""" discover.py — auto-fill apps.json with LOTS of real apps (target: 100k+). There is no "all apps" endpoint, so we discover apps from several sources and MERGE them (de-duplicated) into apps.json, keeping your "settings" block and any apps already listed. SOURCES Google Play - search() across many terms. NOTE: Play search returns only ~30 apps per query (a hard server cap), so Play volume comes from using MANY terms. App Store - iTunes Search API: ~180 apps per term <-- the real workhorse for volume - RSS top charts (top free / grossing / paid) per genre, for popular apps REACHING 100k Top charts alone top out in the low thousands (only popular apps exist there). Volume comes from searching MANY terms. Build a big term list with: --deep add all 2-letter combos (aa..zz, 676 terms) --terms-file words.txt add your own wordlist (one term per line) -> best lever Most of the 100k will be App Store apps; Play contributes fewer (30/query cap). SAFETY All requests go through netutil's retry/backoff + shared circuit breaker, so a rate-limit just slows us down instead of getting the IP blocked. Run: python discover.py --target 100000 --deep --countries us,gb python discover.py --target 100000 --terms-file english_words.txt python discover.py --skip-play --target 100000 --deep # App Store only """ import argparse import json import string import time from pathlib import Path import requests from google_play_scraper import search as gp_search from netutil import Throttle, retry_call # Curated topic terms (used for BOTH stores' search). SEARCH_TERMS = [ # games "games", "action games", "puzzle games", "racing games", "strategy games", "rpg", "simulation games", "arcade", "card games", "board games", "casino", "word games", "trivia", "io games", "tower defense", "shooter", "adventure", # social / communication "social", "social media", "messaging", "chat", "dating", "video call", "community", "forum", # music / audio "music", "music player", "radio", "podcast", "audiobook", "karaoke", # photo / video "photo editor", "video editor", "camera", "collage", "selfie", "video player", "screen recorder", # shopping "shopping", "online shopping", "deals", "coupons", "grocery", "fashion", # finance "finance", "banking", "investing", "crypto", "budget", "payments", "insurance", "taxes", "stocks", # education "education", "language learning", "kids learning", "math", "science", "coding", "exam prep", "flashcards", # health / fitness "health", "fitness", "workout", "yoga", "meditation", "sleep", "diet", "period tracker", "medical", "mental health", # travel / maps "travel", "flights", "hotels", "maps navigation", "ride sharing", "public transport", # food "food delivery", "recipes", "restaurants", # news / weather / sports "news", "weather", "sports", "live scores", # productivity / office "productivity", "notes", "calendar", "to do list", "office", "pdf", "scanner", "email", "cloud storage", # utilities / tools "utilities", "file manager", "cleaner", "antivirus", "vpn", "browser", "keyboard", "launcher", "wallpaper", "ringtones", "qr scanner", "flashlight", "battery", # streaming / entertainment "streaming", "movies", "tv shows", "live tv", "anime", "entertainment", # books / reading "books", "comics", "manga", "ebook reader", # business / misc "business", "crm", "invoicing", "job search", "real estate", "parenting", "pregnancy", "art", "drawing", "translator", # AI "ai chatbot", "ai image generator", "ai assistant", ] # Apple App Store genre IDs for the RSS charts. APPSTORE_GENRES = { 6005: "Social Networking", 6014: "Games", 6016: "Entertainment", 6007: "Productivity", 6008: "Photo & Video", 6011: "Music", 6012: "Lifestyle", 6015: "Finance", 6017: "Education", 6013: "Health & Fitness", 6023: "Food & Drink", 6024: "Shopping", 6000: "Business", 6002: "Utilities", 6003: "Travel", 6004: "Sports", 6009: "News", 6010: "Navigation", 6018: "Books", 6001: "Weather", 6020: "Medical", 6006: "Reference", 6021: "Magazines & Newspapers", } APPSTORE_FEEDS = [ "topfreeapplications", "topgrossingapplications", "toppaidapplications", ] RSS_CHART = ( "https://itunes.apple.com/{country}/rss/{feed}/" "limit={limit}/genre={genre}/json" ) ITUNES_SEARCH = "https://itunes.apple.com/search" def build_terms(deep: bool, deeper: bool, terms_file: str | None) -> list: """Assemble the search-term list: curated + alphabet sweep + optional file.""" terms = list(SEARCH_TERMS) terms += list(string.ascii_lowercase) # a..z (26) if deep: terms += [a + b for a in string.ascii_lowercase for b in string.ascii_lowercase] # aa..zz (676) if deeper: terms += [a + b + c for a in string.ascii_lowercase for b in string.ascii_lowercase for c in string.ascii_lowercase] # aaa..zzz (17576) if terms_file: words = Path(terms_file).read_text(encoding="utf-8").split() terms += words # de-dupe, preserve order, drop blanks seen, out = set(), [] for t in terms: t = t.strip().lower() if t and t not in seen: seen.add(t) out.append(t) return out def _reached(target: int | None, n: int) -> bool: return bool(target) and n >= target # --- Google Play ------------------------------------------------------------ def discover_play(terms, lang, countries, hits, target, delay, throttle) -> list: seen: dict = {} for country in countries: for term in terms: if _reached(target, len(seen)): break try: items = retry_call( lambda: gp_search(term, n_hits=hits, lang=lang, country=country), throttle=throttle, ) except Exception as exc: print(f" [Play:{country}] '{term}' FAILED -> {exc}") continue for item in items: app_id = item.get("appId") if app_id and app_id not in seen: seen[app_id] = {"app_id": app_id} print(f" [Play:{country}] '{term}': total unique = {len(seen)}" f"{f' / {target}' if target else ''}") if delay: time.sleep(delay) if _reached(target, len(seen)): break apps = list(seen.values()) return apps[:target] if target else apps # --- App Store -------------------------------------------------------------- def _rss_json(url: str) -> dict: resp = requests.get(url, timeout=20) resp.raise_for_status() return resp.json() def _search_json(term: str, country: str, limit: int) -> dict: resp = requests.get( ITUNES_SEARCH, params={"term": term, "country": country, "entity": "software", "limit": limit}, timeout=20, ) resp.raise_for_status() return resp.json() def _add_appstore(seen: dict, app_id, name) -> None: try: app_id = int(app_id) except (TypeError, ValueError): return if app_id not in seen: seen[app_id] = {"app_id": app_id, "name": name} def discover_appstore(terms, countries, chart_limit, search_limit, target, delay, throttle) -> list: seen: dict = {} for country in countries: # 1) Top charts — high-quality popular apps (fast, bounded). for feed in APPSTORE_FEEDS: for genre, name in APPSTORE_GENRES.items(): if _reached(target, len(seen)): break url = RSS_CHART.format( country=country, feed=feed, limit=chart_limit, genre=genre ) try: entries = (retry_call(_rss_json, url, throttle=throttle) .get("feed", {}).get("entry", [])) if isinstance(entries, dict): entries = [entries] for e in entries: _add_appstore( seen, e.get("id", {}).get("attributes", {}).get("im:id"), e.get("im:name", {}).get("label"), ) print(f" [Store:{country}/{feed}] {name}: total unique = " f"{len(seen)}{f' / {target}' if target else ''}") if delay: time.sleep(delay) except Exception as exc: print(f" [Store:{country}/{feed}] {name} FAILED -> {exc}") if _reached(target, len(seen)): break # 2) Search API — the deep, high-volume source (~180 apps per term). for term in terms: if _reached(target, len(seen)): break try: results = retry_call( _search_json, term, country, search_limit, throttle=throttle ).get("results", []) except Exception as exc: print(f" [Store:{country}/search] '{term}' FAILED -> {exc}") continue for r in results: _add_appstore(seen, r.get("trackId"), r.get("trackName")) print(f" [Store:{country}/search] '{term}': total unique = " f"{len(seen)}{f' / {target}' if target else ''}") if delay: time.sleep(delay) if _reached(target, len(seen)): break apps = list(seen.values()) return apps[:target] if target else apps DEFAULT_SETTINGS = { "country": "us", "countries": ["us", "gb", "in", "ca", "au"], "lang": "en", "review_count": 100, "delay_seconds": 1.0, } def load_config(path: Path) -> dict: """Read apps.json, or start a fresh structure if it doesn't exist yet.""" if path.exists(): return json.loads(path.read_text(encoding="utf-8")) print(f"(config {path} not found - creating a fresh one)") return {"settings": dict(DEFAULT_SETTINGS), "google_play": [], "app_store": []} def merge(existing: list, discovered: list, key: str) -> list: seen = set() merged = [] for item in existing + discovered: k = item.get(key) if k is not None and k not in seen: seen.add(k) merged.append(item) return merged def resolve_countries(args, settings: dict) -> list: if args.countries: return [c.strip() for c in args.countries.split(",") if c.strip()] return settings.get("countries") or [settings.get("country", "us")] def main() -> None: parser = argparse.ArgumentParser(description="Auto-fill apps.json with popular apps") parser.add_argument("--config", default="apps.json") parser.add_argument("--countries", default=None, help="comma-separated, e.g. us,gb,in (overrides settings)") parser.add_argument("--target", type=int, default=0, help="stop each store after N unique apps (0 = use all terms)") parser.add_argument("--hits", type=int, default=30, help="Play results per term (~30 is the server cap)") parser.add_argument("--chart-limit", type=int, default=200, help="App Store apps per genre chart (max 200)") parser.add_argument("--search-limit", type=int, default=200, help="iTunes search results per term (max 200)") parser.add_argument("--deep", action="store_true", help="add 2-letter terms (aa..zz) for much wider coverage") parser.add_argument("--deeper", action="store_true", help="add 3-letter terms (aaa..zzz) — 17k terms, very slow") parser.add_argument("--terms-file", default=None, help="extra search terms, one per line (best way to reach 100k)") parser.add_argument("--delay", type=float, default=0.5, help="polite delay (s) between discovery requests") parser.add_argument("--skip-play", action="store_true", help="skip Google Play") parser.add_argument("--skip-appstore", action="store_true", help="skip App Store") args = parser.parse_args() config_path = Path(args.config) cfg = load_config(config_path) s = cfg.get("settings", {}) lang = s.get("lang", "en") countries = resolve_countries(args, s) target = args.target or None terms = build_terms(args.deep, args.deeper, args.terms_file) throttle = Throttle() print(f"Countries: {', '.join(countries)} | search terms: {len(terms)} | " f"target/store: {target or 'all terms'}") if not args.skip_play: print("\nDiscovering Google Play apps...") play = discover_play(terms, lang, countries, args.hits, target, args.delay, throttle) cfg["google_play"] = merge(cfg.get("google_play", []), play, "app_id") if not args.skip_appstore: print("\nDiscovering App Store apps...") store = discover_appstore(terms, countries, args.chart_limit, args.search_limit, target, args.delay, throttle) cfg["app_store"] = merge(cfg.get("app_store", []), store, "app_id") config_path.write_text( json.dumps(cfg, indent=2, ensure_ascii=False), encoding="utf-8" ) print( f"\nDONE apps.json now has {len(cfg.get('google_play', []))} Play apps " f"and {len(cfg.get('app_store', []))} App Store apps." ) if __name__ == "__main__": main()