dhruv.godvani
03bf0fc923
- discover.py: find app IDs (Play search, iTunes Search API, RSS charts) -> apps.json - scraper.py: fetch metadata + ratings per app/country, threaded, shardable, resumable - netutil.py: shared retry/backoff + adaptive circuit breaker (rate-limit safe) - merge_csv.py: combine sharded outputs from multiple machines - ARCHITECTURE.html: full architecture & flow documentation - english_words.txt: deep search-term wordlist Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
350 lines
14 KiB
Python
350 lines
14 KiB
Python
"""
|
|
discover.py — auto-fill apps.json with LOTS of real apps (target: 100k+).
|
|
|
|
There is no "all apps" endpoint, so we discover apps from several sources and
|
|
MERGE them (de-duplicated) into apps.json, keeping your "settings" block and any
|
|
apps already listed.
|
|
|
|
SOURCES
|
|
Google Play
|
|
- search() across many terms. NOTE: Play search returns only ~30 apps per
|
|
query (a hard server cap), so Play volume comes from using MANY terms.
|
|
App Store
|
|
- iTunes Search API: ~180 apps per term <-- the real workhorse for volume
|
|
- RSS top charts (top free / grossing / paid) per genre, for popular apps
|
|
|
|
REACHING 100k
|
|
Top charts alone top out in the low thousands (only popular apps exist there).
|
|
Volume comes from searching MANY terms. Build a big term list with:
|
|
--deep add all 2-letter combos (aa..zz, 676 terms)
|
|
--terms-file words.txt add your own wordlist (one term per line) -> best lever
|
|
Most of the 100k will be App Store apps; Play contributes fewer (30/query cap).
|
|
|
|
SAFETY
|
|
All requests go through netutil's retry/backoff + shared circuit breaker, so a
|
|
rate-limit just slows us down instead of getting the IP blocked.
|
|
|
|
Run: python discover.py --target 100000 --deep --countries us,gb
|
|
python discover.py --target 100000 --terms-file english_words.txt
|
|
python discover.py --skip-play --target 100000 --deep # App Store only
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import string
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from google_play_scraper import search as gp_search
|
|
|
|
from netutil import Throttle, retry_call
|
|
|
|
# Curated topic terms (used for BOTH stores' search).
|
|
SEARCH_TERMS = [
|
|
# games
|
|
"games", "action games", "puzzle games", "racing games", "strategy games",
|
|
"rpg", "simulation games", "arcade", "card games", "board games", "casino",
|
|
"word games", "trivia", "io games", "tower defense", "shooter", "adventure",
|
|
# social / communication
|
|
"social", "social media", "messaging", "chat", "dating", "video call",
|
|
"community", "forum",
|
|
# music / audio
|
|
"music", "music player", "radio", "podcast", "audiobook", "karaoke",
|
|
# photo / video
|
|
"photo editor", "video editor", "camera", "collage", "selfie",
|
|
"video player", "screen recorder",
|
|
# shopping
|
|
"shopping", "online shopping", "deals", "coupons", "grocery", "fashion",
|
|
# finance
|
|
"finance", "banking", "investing", "crypto", "budget", "payments",
|
|
"insurance", "taxes", "stocks",
|
|
# education
|
|
"education", "language learning", "kids learning", "math", "science",
|
|
"coding", "exam prep", "flashcards",
|
|
# health / fitness
|
|
"health", "fitness", "workout", "yoga", "meditation", "sleep", "diet",
|
|
"period tracker", "medical", "mental health",
|
|
# travel / maps
|
|
"travel", "flights", "hotels", "maps navigation", "ride sharing",
|
|
"public transport",
|
|
# food
|
|
"food delivery", "recipes", "restaurants",
|
|
# news / weather / sports
|
|
"news", "weather", "sports", "live scores",
|
|
# productivity / office
|
|
"productivity", "notes", "calendar", "to do list", "office", "pdf",
|
|
"scanner", "email", "cloud storage",
|
|
# utilities / tools
|
|
"utilities", "file manager", "cleaner", "antivirus", "vpn", "browser",
|
|
"keyboard", "launcher", "wallpaper", "ringtones", "qr scanner",
|
|
"flashlight", "battery",
|
|
# streaming / entertainment
|
|
"streaming", "movies", "tv shows", "live tv", "anime", "entertainment",
|
|
# books / reading
|
|
"books", "comics", "manga", "ebook reader",
|
|
# business / misc
|
|
"business", "crm", "invoicing", "job search", "real estate", "parenting",
|
|
"pregnancy", "art", "drawing", "translator",
|
|
# AI
|
|
"ai chatbot", "ai image generator", "ai assistant",
|
|
]
|
|
|
|
# Apple App Store genre IDs for the RSS charts.
|
|
APPSTORE_GENRES = {
|
|
6005: "Social Networking", 6014: "Games", 6016: "Entertainment",
|
|
6007: "Productivity", 6008: "Photo & Video", 6011: "Music",
|
|
6012: "Lifestyle", 6015: "Finance", 6017: "Education",
|
|
6013: "Health & Fitness", 6023: "Food & Drink", 6024: "Shopping",
|
|
6000: "Business", 6002: "Utilities", 6003: "Travel", 6004: "Sports",
|
|
6009: "News", 6010: "Navigation", 6018: "Books", 6001: "Weather",
|
|
6020: "Medical", 6006: "Reference", 6021: "Magazines & Newspapers",
|
|
}
|
|
|
|
APPSTORE_FEEDS = [
|
|
"topfreeapplications",
|
|
"topgrossingapplications",
|
|
"toppaidapplications",
|
|
]
|
|
|
|
RSS_CHART = (
|
|
"https://itunes.apple.com/{country}/rss/{feed}/"
|
|
"limit={limit}/genre={genre}/json"
|
|
)
|
|
ITUNES_SEARCH = "https://itunes.apple.com/search"
|
|
|
|
|
|
def build_terms(deep: bool, deeper: bool, terms_file: str | None) -> list:
|
|
"""Assemble the search-term list: curated + alphabet sweep + optional file."""
|
|
terms = list(SEARCH_TERMS)
|
|
terms += list(string.ascii_lowercase) # a..z (26)
|
|
if deep:
|
|
terms += [a + b for a in string.ascii_lowercase
|
|
for b in string.ascii_lowercase] # aa..zz (676)
|
|
if deeper:
|
|
terms += [a + b + c for a in string.ascii_lowercase
|
|
for b in string.ascii_lowercase
|
|
for c in string.ascii_lowercase] # aaa..zzz (17576)
|
|
if terms_file:
|
|
words = Path(terms_file).read_text(encoding="utf-8").split()
|
|
terms += words
|
|
# de-dupe, preserve order, drop blanks
|
|
seen, out = set(), []
|
|
for t in terms:
|
|
t = t.strip().lower()
|
|
if t and t not in seen:
|
|
seen.add(t)
|
|
out.append(t)
|
|
return out
|
|
|
|
|
|
def _reached(target: int | None, n: int) -> bool:
|
|
return bool(target) and n >= target
|
|
|
|
|
|
# --- Google Play ------------------------------------------------------------
|
|
def discover_play(terms, lang, countries, hits, target, delay, throttle) -> list:
|
|
seen: dict = {}
|
|
for country in countries:
|
|
for term in terms:
|
|
if _reached(target, len(seen)):
|
|
break
|
|
try:
|
|
items = retry_call(
|
|
lambda: gp_search(term, n_hits=hits, lang=lang, country=country),
|
|
throttle=throttle,
|
|
)
|
|
except Exception as exc:
|
|
print(f" [Play:{country}] '{term}' FAILED -> {exc}")
|
|
continue
|
|
for item in items:
|
|
app_id = item.get("appId")
|
|
if app_id and app_id not in seen:
|
|
seen[app_id] = {"app_id": app_id}
|
|
print(f" [Play:{country}] '{term}': total unique = {len(seen)}"
|
|
f"{f' / {target}' if target else ''}")
|
|
if delay:
|
|
time.sleep(delay)
|
|
if _reached(target, len(seen)):
|
|
break
|
|
apps = list(seen.values())
|
|
return apps[:target] if target else apps
|
|
|
|
|
|
# --- App Store --------------------------------------------------------------
|
|
def _rss_json(url: str) -> dict:
|
|
resp = requests.get(url, timeout=20)
|
|
resp.raise_for_status()
|
|
return resp.json()
|
|
|
|
|
|
def _search_json(term: str, country: str, limit: int) -> dict:
|
|
resp = requests.get(
|
|
ITUNES_SEARCH,
|
|
params={"term": term, "country": country, "entity": "software", "limit": limit},
|
|
timeout=20,
|
|
)
|
|
resp.raise_for_status()
|
|
return resp.json()
|
|
|
|
|
|
def _add_appstore(seen: dict, app_id, name) -> None:
|
|
try:
|
|
app_id = int(app_id)
|
|
except (TypeError, ValueError):
|
|
return
|
|
if app_id not in seen:
|
|
seen[app_id] = {"app_id": app_id, "name": name}
|
|
|
|
|
|
def discover_appstore(terms, countries, chart_limit, search_limit, target,
|
|
delay, throttle) -> list:
|
|
seen: dict = {}
|
|
for country in countries:
|
|
# 1) Top charts — high-quality popular apps (fast, bounded).
|
|
for feed in APPSTORE_FEEDS:
|
|
for genre, name in APPSTORE_GENRES.items():
|
|
if _reached(target, len(seen)):
|
|
break
|
|
url = RSS_CHART.format(
|
|
country=country, feed=feed, limit=chart_limit, genre=genre
|
|
)
|
|
try:
|
|
entries = (retry_call(_rss_json, url, throttle=throttle)
|
|
.get("feed", {}).get("entry", []))
|
|
if isinstance(entries, dict):
|
|
entries = [entries]
|
|
for e in entries:
|
|
_add_appstore(
|
|
seen,
|
|
e.get("id", {}).get("attributes", {}).get("im:id"),
|
|
e.get("im:name", {}).get("label"),
|
|
)
|
|
print(f" [Store:{country}/{feed}] {name}: total unique = "
|
|
f"{len(seen)}{f' / {target}' if target else ''}")
|
|
if delay:
|
|
time.sleep(delay)
|
|
except Exception as exc:
|
|
print(f" [Store:{country}/{feed}] {name} FAILED -> {exc}")
|
|
if _reached(target, len(seen)):
|
|
break
|
|
|
|
# 2) Search API — the deep, high-volume source (~180 apps per term).
|
|
for term in terms:
|
|
if _reached(target, len(seen)):
|
|
break
|
|
try:
|
|
results = retry_call(
|
|
_search_json, term, country, search_limit, throttle=throttle
|
|
).get("results", [])
|
|
except Exception as exc:
|
|
print(f" [Store:{country}/search] '{term}' FAILED -> {exc}")
|
|
continue
|
|
for r in results:
|
|
_add_appstore(seen, r.get("trackId"), r.get("trackName"))
|
|
print(f" [Store:{country}/search] '{term}': total unique = "
|
|
f"{len(seen)}{f' / {target}' if target else ''}")
|
|
if delay:
|
|
time.sleep(delay)
|
|
if _reached(target, len(seen)):
|
|
break
|
|
|
|
apps = list(seen.values())
|
|
return apps[:target] if target else apps
|
|
|
|
|
|
DEFAULT_SETTINGS = {
|
|
"country": "us",
|
|
"countries": ["us", "gb", "in", "ca", "au"],
|
|
"lang": "en",
|
|
"review_count": 100,
|
|
"delay_seconds": 1.0,
|
|
}
|
|
|
|
|
|
def load_config(path: Path) -> dict:
|
|
"""Read apps.json, or start a fresh structure if it doesn't exist yet."""
|
|
if path.exists():
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
print(f"(config {path} not found - creating a fresh one)")
|
|
return {"settings": dict(DEFAULT_SETTINGS), "google_play": [], "app_store": []}
|
|
|
|
|
|
def merge(existing: list, discovered: list, key: str) -> list:
|
|
seen = set()
|
|
merged = []
|
|
for item in existing + discovered:
|
|
k = item.get(key)
|
|
if k is not None and k not in seen:
|
|
seen.add(k)
|
|
merged.append(item)
|
|
return merged
|
|
|
|
|
|
def resolve_countries(args, settings: dict) -> list:
|
|
if args.countries:
|
|
return [c.strip() for c in args.countries.split(",") if c.strip()]
|
|
return settings.get("countries") or [settings.get("country", "us")]
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Auto-fill apps.json with popular apps")
|
|
parser.add_argument("--config", default="apps.json")
|
|
parser.add_argument("--countries", default=None,
|
|
help="comma-separated, e.g. us,gb,in (overrides settings)")
|
|
parser.add_argument("--target", type=int, default=0,
|
|
help="stop each store after N unique apps (0 = use all terms)")
|
|
parser.add_argument("--hits", type=int, default=30,
|
|
help="Play results per term (~30 is the server cap)")
|
|
parser.add_argument("--chart-limit", type=int, default=200,
|
|
help="App Store apps per genre chart (max 200)")
|
|
parser.add_argument("--search-limit", type=int, default=200,
|
|
help="iTunes search results per term (max 200)")
|
|
parser.add_argument("--deep", action="store_true",
|
|
help="add 2-letter terms (aa..zz) for much wider coverage")
|
|
parser.add_argument("--deeper", action="store_true",
|
|
help="add 3-letter terms (aaa..zzz) — 17k terms, very slow")
|
|
parser.add_argument("--terms-file", default=None,
|
|
help="extra search terms, one per line (best way to reach 100k)")
|
|
parser.add_argument("--delay", type=float, default=0.5,
|
|
help="polite delay (s) between discovery requests")
|
|
parser.add_argument("--skip-play", action="store_true", help="skip Google Play")
|
|
parser.add_argument("--skip-appstore", action="store_true", help="skip App Store")
|
|
args = parser.parse_args()
|
|
|
|
config_path = Path(args.config)
|
|
cfg = load_config(config_path)
|
|
s = cfg.get("settings", {})
|
|
lang = s.get("lang", "en")
|
|
countries = resolve_countries(args, s)
|
|
target = args.target or None
|
|
terms = build_terms(args.deep, args.deeper, args.terms_file)
|
|
throttle = Throttle()
|
|
|
|
print(f"Countries: {', '.join(countries)} | search terms: {len(terms)} | "
|
|
f"target/store: {target or 'all terms'}")
|
|
|
|
if not args.skip_play:
|
|
print("\nDiscovering Google Play apps...")
|
|
play = discover_play(terms, lang, countries, args.hits, target,
|
|
args.delay, throttle)
|
|
cfg["google_play"] = merge(cfg.get("google_play", []), play, "app_id")
|
|
|
|
if not args.skip_appstore:
|
|
print("\nDiscovering App Store apps...")
|
|
store = discover_appstore(terms, countries, args.chart_limit,
|
|
args.search_limit, target, args.delay, throttle)
|
|
cfg["app_store"] = merge(cfg.get("app_store", []), store, "app_id")
|
|
|
|
config_path.write_text(
|
|
json.dumps(cfg, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
print(
|
|
f"\nDONE apps.json now has {len(cfg.get('google_play', []))} Play apps "
|
|
f"and {len(cfg.get('app_store', []))} App Store apps."
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|