Initial commit: App Store + Google Play scraper
- discover.py: find app IDs (Play search, iTunes Search API, RSS charts) -> apps.json - scraper.py: fetch metadata + ratings per app/country, threaded, shardable, resumable - netutil.py: shared retry/backoff + adaptive circuit breaker (rate-limit safe) - merge_csv.py: combine sharded outputs from multiple machines - ARCHITECTURE.html: full architecture & flow documentation - english_words.txt: deep search-term wordlist Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+349
@@ -0,0 +1,349 @@
|
||||
"""
|
||||
discover.py — auto-fill apps.json with LOTS of real apps (target: 100k+).
|
||||
|
||||
There is no "all apps" endpoint, so we discover apps from several sources and
|
||||
MERGE them (de-duplicated) into apps.json, keeping your "settings" block and any
|
||||
apps already listed.
|
||||
|
||||
SOURCES
|
||||
Google Play
|
||||
- search() across many terms. NOTE: Play search returns only ~30 apps per
|
||||
query (a hard server cap), so Play volume comes from using MANY terms.
|
||||
App Store
|
||||
- iTunes Search API: ~180 apps per term <-- the real workhorse for volume
|
||||
- RSS top charts (top free / grossing / paid) per genre, for popular apps
|
||||
|
||||
REACHING 100k
|
||||
Top charts alone top out in the low thousands (only popular apps exist there).
|
||||
Volume comes from searching MANY terms. Build a big term list with:
|
||||
--deep add all 2-letter combos (aa..zz, 676 terms)
|
||||
--terms-file words.txt add your own wordlist (one term per line) -> best lever
|
||||
Most of the 100k will be App Store apps; Play contributes fewer (30/query cap).
|
||||
|
||||
SAFETY
|
||||
All requests go through netutil's retry/backoff + shared circuit breaker, so a
|
||||
rate-limit just slows us down instead of getting the IP blocked.
|
||||
|
||||
Run: python discover.py --target 100000 --deep --countries us,gb
|
||||
python discover.py --target 100000 --terms-file english_words.txt
|
||||
python discover.py --skip-play --target 100000 --deep # App Store only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import string
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from google_play_scraper import search as gp_search
|
||||
|
||||
from netutil import Throttle, retry_call
|
||||
|
||||
# Curated topic terms (used for BOTH stores' search).
|
||||
SEARCH_TERMS = [
|
||||
# games
|
||||
"games", "action games", "puzzle games", "racing games", "strategy games",
|
||||
"rpg", "simulation games", "arcade", "card games", "board games", "casino",
|
||||
"word games", "trivia", "io games", "tower defense", "shooter", "adventure",
|
||||
# social / communication
|
||||
"social", "social media", "messaging", "chat", "dating", "video call",
|
||||
"community", "forum",
|
||||
# music / audio
|
||||
"music", "music player", "radio", "podcast", "audiobook", "karaoke",
|
||||
# photo / video
|
||||
"photo editor", "video editor", "camera", "collage", "selfie",
|
||||
"video player", "screen recorder",
|
||||
# shopping
|
||||
"shopping", "online shopping", "deals", "coupons", "grocery", "fashion",
|
||||
# finance
|
||||
"finance", "banking", "investing", "crypto", "budget", "payments",
|
||||
"insurance", "taxes", "stocks",
|
||||
# education
|
||||
"education", "language learning", "kids learning", "math", "science",
|
||||
"coding", "exam prep", "flashcards",
|
||||
# health / fitness
|
||||
"health", "fitness", "workout", "yoga", "meditation", "sleep", "diet",
|
||||
"period tracker", "medical", "mental health",
|
||||
# travel / maps
|
||||
"travel", "flights", "hotels", "maps navigation", "ride sharing",
|
||||
"public transport",
|
||||
# food
|
||||
"food delivery", "recipes", "restaurants",
|
||||
# news / weather / sports
|
||||
"news", "weather", "sports", "live scores",
|
||||
# productivity / office
|
||||
"productivity", "notes", "calendar", "to do list", "office", "pdf",
|
||||
"scanner", "email", "cloud storage",
|
||||
# utilities / tools
|
||||
"utilities", "file manager", "cleaner", "antivirus", "vpn", "browser",
|
||||
"keyboard", "launcher", "wallpaper", "ringtones", "qr scanner",
|
||||
"flashlight", "battery",
|
||||
# streaming / entertainment
|
||||
"streaming", "movies", "tv shows", "live tv", "anime", "entertainment",
|
||||
# books / reading
|
||||
"books", "comics", "manga", "ebook reader",
|
||||
# business / misc
|
||||
"business", "crm", "invoicing", "job search", "real estate", "parenting",
|
||||
"pregnancy", "art", "drawing", "translator",
|
||||
# AI
|
||||
"ai chatbot", "ai image generator", "ai assistant",
|
||||
]
|
||||
|
||||
# Apple App Store genre IDs for the RSS charts.
|
||||
APPSTORE_GENRES = {
|
||||
6005: "Social Networking", 6014: "Games", 6016: "Entertainment",
|
||||
6007: "Productivity", 6008: "Photo & Video", 6011: "Music",
|
||||
6012: "Lifestyle", 6015: "Finance", 6017: "Education",
|
||||
6013: "Health & Fitness", 6023: "Food & Drink", 6024: "Shopping",
|
||||
6000: "Business", 6002: "Utilities", 6003: "Travel", 6004: "Sports",
|
||||
6009: "News", 6010: "Navigation", 6018: "Books", 6001: "Weather",
|
||||
6020: "Medical", 6006: "Reference", 6021: "Magazines & Newspapers",
|
||||
}
|
||||
|
||||
APPSTORE_FEEDS = [
|
||||
"topfreeapplications",
|
||||
"topgrossingapplications",
|
||||
"toppaidapplications",
|
||||
]
|
||||
|
||||
RSS_CHART = (
|
||||
"https://itunes.apple.com/{country}/rss/{feed}/"
|
||||
"limit={limit}/genre={genre}/json"
|
||||
)
|
||||
ITUNES_SEARCH = "https://itunes.apple.com/search"
|
||||
|
||||
|
||||
def build_terms(deep: bool, deeper: bool, terms_file: str | None) -> list:
|
||||
"""Assemble the search-term list: curated + alphabet sweep + optional file."""
|
||||
terms = list(SEARCH_TERMS)
|
||||
terms += list(string.ascii_lowercase) # a..z (26)
|
||||
if deep:
|
||||
terms += [a + b for a in string.ascii_lowercase
|
||||
for b in string.ascii_lowercase] # aa..zz (676)
|
||||
if deeper:
|
||||
terms += [a + b + c for a in string.ascii_lowercase
|
||||
for b in string.ascii_lowercase
|
||||
for c in string.ascii_lowercase] # aaa..zzz (17576)
|
||||
if terms_file:
|
||||
words = Path(terms_file).read_text(encoding="utf-8").split()
|
||||
terms += words
|
||||
# de-dupe, preserve order, drop blanks
|
||||
seen, out = set(), []
|
||||
for t in terms:
|
||||
t = t.strip().lower()
|
||||
if t and t not in seen:
|
||||
seen.add(t)
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def _reached(target: int | None, n: int) -> bool:
|
||||
return bool(target) and n >= target
|
||||
|
||||
|
||||
# --- Google Play ------------------------------------------------------------
|
||||
def discover_play(terms, lang, countries, hits, target, delay, throttle) -> list:
|
||||
seen: dict = {}
|
||||
for country in countries:
|
||||
for term in terms:
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
try:
|
||||
items = retry_call(
|
||||
lambda: gp_search(term, n_hits=hits, lang=lang, country=country),
|
||||
throttle=throttle,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f" [Play:{country}] '{term}' FAILED -> {exc}")
|
||||
continue
|
||||
for item in items:
|
||||
app_id = item.get("appId")
|
||||
if app_id and app_id not in seen:
|
||||
seen[app_id] = {"app_id": app_id}
|
||||
print(f" [Play:{country}] '{term}': total unique = {len(seen)}"
|
||||
f"{f' / {target}' if target else ''}")
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
apps = list(seen.values())
|
||||
return apps[:target] if target else apps
|
||||
|
||||
|
||||
# --- App Store --------------------------------------------------------------
|
||||
def _rss_json(url: str) -> dict:
|
||||
resp = requests.get(url, timeout=20)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _search_json(term: str, country: str, limit: int) -> dict:
|
||||
resp = requests.get(
|
||||
ITUNES_SEARCH,
|
||||
params={"term": term, "country": country, "entity": "software", "limit": limit},
|
||||
timeout=20,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _add_appstore(seen: dict, app_id, name) -> None:
|
||||
try:
|
||||
app_id = int(app_id)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if app_id not in seen:
|
||||
seen[app_id] = {"app_id": app_id, "name": name}
|
||||
|
||||
|
||||
def discover_appstore(terms, countries, chart_limit, search_limit, target,
|
||||
delay, throttle) -> list:
|
||||
seen: dict = {}
|
||||
for country in countries:
|
||||
# 1) Top charts — high-quality popular apps (fast, bounded).
|
||||
for feed in APPSTORE_FEEDS:
|
||||
for genre, name in APPSTORE_GENRES.items():
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
url = RSS_CHART.format(
|
||||
country=country, feed=feed, limit=chart_limit, genre=genre
|
||||
)
|
||||
try:
|
||||
entries = (retry_call(_rss_json, url, throttle=throttle)
|
||||
.get("feed", {}).get("entry", []))
|
||||
if isinstance(entries, dict):
|
||||
entries = [entries]
|
||||
for e in entries:
|
||||
_add_appstore(
|
||||
seen,
|
||||
e.get("id", {}).get("attributes", {}).get("im:id"),
|
||||
e.get("im:name", {}).get("label"),
|
||||
)
|
||||
print(f" [Store:{country}/{feed}] {name}: total unique = "
|
||||
f"{len(seen)}{f' / {target}' if target else ''}")
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
except Exception as exc:
|
||||
print(f" [Store:{country}/{feed}] {name} FAILED -> {exc}")
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
|
||||
# 2) Search API — the deep, high-volume source (~180 apps per term).
|
||||
for term in terms:
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
try:
|
||||
results = retry_call(
|
||||
_search_json, term, country, search_limit, throttle=throttle
|
||||
).get("results", [])
|
||||
except Exception as exc:
|
||||
print(f" [Store:{country}/search] '{term}' FAILED -> {exc}")
|
||||
continue
|
||||
for r in results:
|
||||
_add_appstore(seen, r.get("trackId"), r.get("trackName"))
|
||||
print(f" [Store:{country}/search] '{term}': total unique = "
|
||||
f"{len(seen)}{f' / {target}' if target else ''}")
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
|
||||
apps = list(seen.values())
|
||||
return apps[:target] if target else apps
|
||||
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"country": "us",
|
||||
"countries": ["us", "gb", "in", "ca", "au"],
|
||||
"lang": "en",
|
||||
"review_count": 100,
|
||||
"delay_seconds": 1.0,
|
||||
}
|
||||
|
||||
|
||||
def load_config(path: Path) -> dict:
|
||||
"""Read apps.json, or start a fresh structure if it doesn't exist yet."""
|
||||
if path.exists():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
print(f"(config {path} not found - creating a fresh one)")
|
||||
return {"settings": dict(DEFAULT_SETTINGS), "google_play": [], "app_store": []}
|
||||
|
||||
|
||||
def merge(existing: list, discovered: list, key: str) -> list:
|
||||
seen = set()
|
||||
merged = []
|
||||
for item in existing + discovered:
|
||||
k = item.get(key)
|
||||
if k is not None and k not in seen:
|
||||
seen.add(k)
|
||||
merged.append(item)
|
||||
return merged
|
||||
|
||||
|
||||
def resolve_countries(args, settings: dict) -> list:
|
||||
if args.countries:
|
||||
return [c.strip() for c in args.countries.split(",") if c.strip()]
|
||||
return settings.get("countries") or [settings.get("country", "us")]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Auto-fill apps.json with popular apps")
|
||||
parser.add_argument("--config", default="apps.json")
|
||||
parser.add_argument("--countries", default=None,
|
||||
help="comma-separated, e.g. us,gb,in (overrides settings)")
|
||||
parser.add_argument("--target", type=int, default=0,
|
||||
help="stop each store after N unique apps (0 = use all terms)")
|
||||
parser.add_argument("--hits", type=int, default=30,
|
||||
help="Play results per term (~30 is the server cap)")
|
||||
parser.add_argument("--chart-limit", type=int, default=200,
|
||||
help="App Store apps per genre chart (max 200)")
|
||||
parser.add_argument("--search-limit", type=int, default=200,
|
||||
help="iTunes search results per term (max 200)")
|
||||
parser.add_argument("--deep", action="store_true",
|
||||
help="add 2-letter terms (aa..zz) for much wider coverage")
|
||||
parser.add_argument("--deeper", action="store_true",
|
||||
help="add 3-letter terms (aaa..zzz) — 17k terms, very slow")
|
||||
parser.add_argument("--terms-file", default=None,
|
||||
help="extra search terms, one per line (best way to reach 100k)")
|
||||
parser.add_argument("--delay", type=float, default=0.5,
|
||||
help="polite delay (s) between discovery requests")
|
||||
parser.add_argument("--skip-play", action="store_true", help="skip Google Play")
|
||||
parser.add_argument("--skip-appstore", action="store_true", help="skip App Store")
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = Path(args.config)
|
||||
cfg = load_config(config_path)
|
||||
s = cfg.get("settings", {})
|
||||
lang = s.get("lang", "en")
|
||||
countries = resolve_countries(args, s)
|
||||
target = args.target or None
|
||||
terms = build_terms(args.deep, args.deeper, args.terms_file)
|
||||
throttle = Throttle()
|
||||
|
||||
print(f"Countries: {', '.join(countries)} | search terms: {len(terms)} | "
|
||||
f"target/store: {target or 'all terms'}")
|
||||
|
||||
if not args.skip_play:
|
||||
print("\nDiscovering Google Play apps...")
|
||||
play = discover_play(terms, lang, countries, args.hits, target,
|
||||
args.delay, throttle)
|
||||
cfg["google_play"] = merge(cfg.get("google_play", []), play, "app_id")
|
||||
|
||||
if not args.skip_appstore:
|
||||
print("\nDiscovering App Store apps...")
|
||||
store = discover_appstore(terms, countries, args.chart_limit,
|
||||
args.search_limit, target, args.delay, throttle)
|
||||
cfg["app_store"] = merge(cfg.get("app_store", []), store, "app_id")
|
||||
|
||||
config_path.write_text(
|
||||
json.dumps(cfg, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
print(
|
||||
f"\nDONE apps.json now has {len(cfg.get('google_play', []))} Play apps "
|
||||
f"and {len(cfg.get('app_store', []))} App Store apps."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user