"""
discover.py — auto-fill apps.json with LOTS of real apps (target: 100k+).

There is no "all apps" endpoint, so we discover apps from several sources and
MERGE them (de-duplicated) into apps.json, keeping your "settings" block and any
apps already listed.

SOURCES
  Google Play
    - search() across many terms.  NOTE: Play search returns only ~30 apps per
      query (a hard server cap), so Play volume comes from using MANY terms.
  App Store
    - iTunes Search API: ~180 apps per term  <-- the real workhorse for volume
    - RSS top charts (top free / grossing / paid) per genre, for popular apps

REACHING 100k
  Top charts alone top out in the low thousands (only popular apps exist there).
  Volume comes from searching MANY terms. Build a big term list with:
    --deep                 add all 2-letter combos (aa..zz, 676 terms)
    --terms-file words.txt add your own wordlist (one term per line) -> best lever
  Most of the 100k will be App Store apps; Play contributes fewer (30/query cap).

SAFETY
  All requests go through netutil's retry/backoff + shared circuit breaker, so a
  rate-limit just slows us down instead of getting the IP blocked.

Run:  python discover.py --target 100000 --deep --countries us,gb
      python discover.py --target 100000 --terms-file english_words.txt
      python discover.py --skip-play --target 100000 --deep      # App Store only
"""

import argparse
import json
import string
import time
from pathlib import Path

import requests
from google_play_scraper import search as gp_search

from netutil import Throttle, retry_call

# Curated topic terms (used for BOTH stores' search).
SEARCH_TERMS = [
    # games
    "games", "action games", "puzzle games", "racing games", "strategy games",
    "rpg", "simulation games", "arcade", "card games", "board games", "casino",
    "word games", "trivia", "io games", "tower defense", "shooter", "adventure",
    # social / communication
    "social", "social media", "messaging", "chat", "dating", "video call",
    "community", "forum",
    # music / audio
    "music", "music player", "radio", "podcast", "audiobook", "karaoke",
    # photo / video
    "photo editor", "video editor", "camera", "collage", "selfie",
    "video player", "screen recorder",
    # shopping
    "shopping", "online shopping", "deals", "coupons", "grocery", "fashion",
    # finance
    "finance", "banking", "investing", "crypto", "budget", "payments",
    "insurance", "taxes", "stocks",
    # education
    "education", "language learning", "kids learning", "math", "science",
    "coding", "exam prep", "flashcards",
    # health / fitness
    "health", "fitness", "workout", "yoga", "meditation", "sleep", "diet",
    "period tracker", "medical", "mental health",
    # travel / maps
    "travel", "flights", "hotels", "maps navigation", "ride sharing",
    "public transport",
    # food
    "food delivery", "recipes", "restaurants",
    # news / weather / sports
    "news", "weather", "sports", "live scores",
    # productivity / office
    "productivity", "notes", "calendar", "to do list", "office", "pdf",
    "scanner", "email", "cloud storage",
    # utilities / tools
    "utilities", "file manager", "cleaner", "antivirus", "vpn", "browser",
    "keyboard", "launcher", "wallpaper", "ringtones", "qr scanner",
    "flashlight", "battery",
    # streaming / entertainment
    "streaming", "movies", "tv shows", "live tv", "anime", "entertainment",
    # books / reading
    "books", "comics", "manga", "ebook reader",
    # business / misc
    "business", "crm", "invoicing", "job search", "real estate", "parenting",
    "pregnancy", "art", "drawing", "translator",
    # AI
    "ai chatbot", "ai image generator", "ai assistant",
]

# Apple App Store genre IDs for the RSS charts.
APPSTORE_GENRES = {
    6005: "Social Networking", 6014: "Games", 6016: "Entertainment",
    6007: "Productivity", 6008: "Photo & Video", 6011: "Music",
    6012: "Lifestyle", 6015: "Finance", 6017: "Education",
    6013: "Health & Fitness", 6023: "Food & Drink", 6024: "Shopping",
    6000: "Business", 6002: "Utilities", 6003: "Travel", 6004: "Sports",
    6009: "News", 6010: "Navigation", 6018: "Books", 6001: "Weather",
    6020: "Medical", 6006: "Reference", 6021: "Magazines & Newspapers",
}

APPSTORE_FEEDS = [
    "topfreeapplications",
    "topgrossingapplications",
    "toppaidapplications",
]

RSS_CHART = (
    "https://itunes.apple.com/{country}/rss/{feed}/"
    "limit={limit}/genre={genre}/json"
)
ITUNES_SEARCH = "https://itunes.apple.com/search"


def build_terms(deep: bool, deeper: bool, terms_file: str | None) -> list:
    """Assemble the search-term list: curated + alphabet sweep + optional file."""
    terms = list(SEARCH_TERMS)
    terms += list(string.ascii_lowercase)                       # a..z  (26)
    if deep:
        terms += [a + b for a in string.ascii_lowercase
                  for b in string.ascii_lowercase]              # aa..zz (676)
    if deeper:
        terms += [a + b + c for a in string.ascii_lowercase
                  for b in string.ascii_lowercase
                  for c in string.ascii_lowercase]              # aaa..zzz (17576)
    if terms_file:
        words = Path(terms_file).read_text(encoding="utf-8").split()
        terms += words
    # de-dupe, preserve order, drop blanks
    seen, out = set(), []
    for t in terms:
        t = t.strip().lower()
        if t and t not in seen:
            seen.add(t)
            out.append(t)
    return out


def _reached(target: int | None, n: int) -> bool:
    return bool(target) and n >= target


# --- Google Play ------------------------------------------------------------
def discover_play(terms, lang, countries, hits, target, delay, throttle) -> list:
    seen: dict = {}
    for country in countries:
        for term in terms:
            if _reached(target, len(seen)):
                break
            try:
                items = retry_call(
                    lambda: gp_search(term, n_hits=hits, lang=lang, country=country),
                    throttle=throttle,
                )
            except Exception as exc:
                print(f"  [Play:{country}] '{term}' FAILED -> {exc}")
                continue
            for item in items:
                app_id = item.get("appId")
                if app_id and app_id not in seen:
                    seen[app_id] = {"app_id": app_id}
            print(f"  [Play:{country}] '{term}': total unique = {len(seen)}"
                  f"{f' / {target}' if target else ''}")
            if delay:
                time.sleep(delay)
        if _reached(target, len(seen)):
            break
    apps = list(seen.values())
    return apps[:target] if target else apps


# --- App Store --------------------------------------------------------------
def _rss_json(url: str) -> dict:
    resp = requests.get(url, timeout=20)
    resp.raise_for_status()
    return resp.json()


def _search_json(term: str, country: str, limit: int) -> dict:
    resp = requests.get(
        ITUNES_SEARCH,
        params={"term": term, "country": country, "entity": "software", "limit": limit},
        timeout=20,
    )
    resp.raise_for_status()
    return resp.json()


def _add_appstore(seen: dict, app_id, name) -> None:
    try:
        app_id = int(app_id)
    except (TypeError, ValueError):
        return
    if app_id not in seen:
        seen[app_id] = {"app_id": app_id, "name": name}


def discover_appstore(terms, countries, chart_limit, search_limit, target,
                      delay, throttle) -> list:
    seen: dict = {}
    for country in countries:
        # 1) Top charts — high-quality popular apps (fast, bounded).
        for feed in APPSTORE_FEEDS:
            for genre, name in APPSTORE_GENRES.items():
                if _reached(target, len(seen)):
                    break
                url = RSS_CHART.format(
                    country=country, feed=feed, limit=chart_limit, genre=genre
                )
                try:
                    entries = (retry_call(_rss_json, url, throttle=throttle)
                               .get("feed", {}).get("entry", []))
                    if isinstance(entries, dict):
                        entries = [entries]
                    for e in entries:
                        _add_appstore(
                            seen,
                            e.get("id", {}).get("attributes", {}).get("im:id"),
                            e.get("im:name", {}).get("label"),
                        )
                    print(f"  [Store:{country}/{feed}] {name}: total unique = "
                          f"{len(seen)}{f' / {target}' if target else ''}")
                    if delay:
                        time.sleep(delay)
                except Exception as exc:
                    print(f"  [Store:{country}/{feed}] {name} FAILED -> {exc}")
            if _reached(target, len(seen)):
                break

        # 2) Search API — the deep, high-volume source (~180 apps per term).
        for term in terms:
            if _reached(target, len(seen)):
                break
            try:
                results = retry_call(
                    _search_json, term, country, search_limit, throttle=throttle
                ).get("results", [])
            except Exception as exc:
                print(f"  [Store:{country}/search] '{term}' FAILED -> {exc}")
                continue
            for r in results:
                _add_appstore(seen, r.get("trackId"), r.get("trackName"))
            print(f"  [Store:{country}/search] '{term}': total unique = "
                  f"{len(seen)}{f' / {target}' if target else ''}")
            if delay:
                time.sleep(delay)
        if _reached(target, len(seen)):
            break

    apps = list(seen.values())
    return apps[:target] if target else apps


DEFAULT_SETTINGS = {
    "country": "us",
    "countries": ["us", "gb", "in", "ca", "au"],
    "lang": "en",
    "review_count": 100,
    "delay_seconds": 1.0,
}


def load_config(path: Path) -> dict:
    """Read apps.json, or start a fresh structure if it doesn't exist yet."""
    if path.exists():
        return json.loads(path.read_text(encoding="utf-8"))
    print(f"(config {path} not found - creating a fresh one)")
    return {"settings": dict(DEFAULT_SETTINGS), "google_play": [], "app_store": []}


def merge(existing: list, discovered: list, key: str) -> list:
    seen = set()
    merged = []
    for item in existing + discovered:
        k = item.get(key)
        if k is not None and k not in seen:
            seen.add(k)
            merged.append(item)
    return merged


def resolve_countries(args, settings: dict) -> list:
    if args.countries:
        return [c.strip() for c in args.countries.split(",") if c.strip()]
    return settings.get("countries") or [settings.get("country", "us")]


def main() -> None:
    parser = argparse.ArgumentParser(description="Auto-fill apps.json with popular apps")
    parser.add_argument("--config", default="apps.json")
    parser.add_argument("--countries", default=None,
                        help="comma-separated, e.g. us,gb,in (overrides settings)")
    parser.add_argument("--target", type=int, default=0,
                        help="stop each store after N unique apps (0 = use all terms)")
    parser.add_argument("--hits", type=int, default=30,
                        help="Play results per term (~30 is the server cap)")
    parser.add_argument("--chart-limit", type=int, default=200,
                        help="App Store apps per genre chart (max 200)")
    parser.add_argument("--search-limit", type=int, default=200,
                        help="iTunes search results per term (max 200)")
    parser.add_argument("--deep", action="store_true",
                        help="add 2-letter terms (aa..zz) for much wider coverage")
    parser.add_argument("--deeper", action="store_true",
                        help="add 3-letter terms (aaa..zzz) — 17k terms, very slow")
    parser.add_argument("--terms-file", default=None,
                        help="extra search terms, one per line (best way to reach 100k)")
    parser.add_argument("--delay", type=float, default=0.5,
                        help="polite delay (s) between discovery requests")
    parser.add_argument("--skip-play", action="store_true", help="skip Google Play")
    parser.add_argument("--skip-appstore", action="store_true", help="skip App Store")
    args = parser.parse_args()

    config_path = Path(args.config)
    cfg = load_config(config_path)
    s = cfg.get("settings", {})
    lang = s.get("lang", "en")
    countries = resolve_countries(args, s)
    target = args.target or None
    terms = build_terms(args.deep, args.deeper, args.terms_file)
    throttle = Throttle()

    print(f"Countries: {', '.join(countries)}  |  search terms: {len(terms)}  |  "
          f"target/store: {target or 'all terms'}")

    if not args.skip_play:
        print("\nDiscovering Google Play apps...")
        play = discover_play(terms, lang, countries, args.hits, target,
                             args.delay, throttle)
        cfg["google_play"] = merge(cfg.get("google_play", []), play, "app_id")

    if not args.skip_appstore:
        print("\nDiscovering App Store apps...")
        store = discover_appstore(terms, countries, args.chart_limit,
                                 args.search_limit, target, args.delay, throttle)
        cfg["app_store"] = merge(cfg.get("app_store", []), store, "app_id")

    config_path.write_text(
        json.dumps(cfg, indent=2, ensure_ascii=False), encoding="utf-8"
    )
    print(
        f"\nDONE  apps.json now has {len(cfg.get('google_play', []))} Play apps "
        f"and {len(cfg.get('app_store', []))} App Store apps."
    )


if __name__ == "__main__":
    main()
