Compare commits
2 Commits
dcac573bcd
...
e958ede9ea
| Author | SHA1 | Date | |
|---|---|---|---|
 dhruv.godvani
|
e958ede9ea | ||
|
dhruv.godvani
|
03bf0fc923 |
+15
-169
@@ -1,176 +1,22 @@
|
||||
# ---> Python
|
||||
# Byte-compiled / optimized / DLL files
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
env/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
# Generated output (CSVs can get very large)
|
||||
output/
|
||||
*.log
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# Ruff stuff:
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
# Backups
|
||||
backups/
|
||||
apps_backup*.json
|
||||
|
||||
# Local tool / editor config
|
||||
.claude/
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
.DS_Store
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>App Store Scraper — Architecture & Flow</title>
|
||||
<style>
|
||||
:root{
|
||||
--bg:#0d1117; --panel:#161b22; --panel2:#1c2330; --border:#30363d;
|
||||
--text:#e6edf3; --muted:#9aa7b4; --accent:#58a6ff; --green:#3fb950;
|
||||
--amber:#d29922; --red:#f85149; --purple:#bc8cff; --cyan:#39c5cf;
|
||||
--code:#0b1020;
|
||||
}
|
||||
*{box-sizing:border-box}
|
||||
html{scroll-behavior:smooth}
|
||||
body{
|
||||
margin:0; background:var(--bg); color:var(--text);
|
||||
font-family:"Segoe UI",system-ui,Roboto,Helvetica,Arial,sans-serif;
|
||||
line-height:1.6; font-size:16px;
|
||||
}
|
||||
.wrap{max-width:1080px; margin:0 auto; padding:0 24px 120px}
|
||||
header.hero{
|
||||
background:linear-gradient(135deg,#11203b 0%,#0d1117 60%);
|
||||
border-bottom:1px solid var(--border); padding:54px 24px 40px; text-align:center;
|
||||
}
|
||||
header.hero h1{margin:0 0 8px; font-size:2.3rem; letter-spacing:.3px}
|
||||
header.hero p{margin:4px 0; color:var(--muted)}
|
||||
.pill{display:inline-block; background:var(--panel2); border:1px solid var(--border);
|
||||
color:var(--cyan); border-radius:999px; padding:4px 14px; font-size:.8rem; margin-top:14px}
|
||||
h2{font-size:1.55rem; margin:54px 0 14px; padding-bottom:8px; border-bottom:2px solid var(--border)}
|
||||
h2 .num{color:var(--accent); font-variant-numeric:tabular-nums; margin-right:10px}
|
||||
h3{font-size:1.15rem; margin:30px 0 10px; color:var(--cyan)}
|
||||
p{margin:10px 0}
|
||||
a{color:var(--accent); text-decoration:none}
|
||||
a:hover{text-decoration:underline}
|
||||
code{background:var(--code); border:1px solid var(--border); padding:1px 6px;
|
||||
border-radius:5px; font-family:"Cascadia Code",Consolas,monospace; font-size:.86em; color:#ffd9a0}
|
||||
pre{background:var(--code); border:1px solid var(--border); border-radius:10px;
|
||||
padding:16px 18px; overflow:auto; font-family:"Cascadia Code",Consolas,monospace;
|
||||
font-size:.85rem; line-height:1.5}
|
||||
pre code{background:none; border:none; padding:0; color:#c9d6e3}
|
||||
.muted{color:var(--muted)}
|
||||
/* TOC */
|
||||
nav.toc{background:var(--panel); border:1px solid var(--border); border-radius:12px;
|
||||
padding:18px 22px; margin:34px 0}
|
||||
nav.toc h4{margin:0 0 10px; color:var(--muted); font-weight:600; text-transform:uppercase; font-size:.78rem; letter-spacing:1px}
|
||||
nav.toc ol{margin:0; padding-left:20px; columns:2; column-gap:40px}
|
||||
nav.toc li{margin:5px 0}
|
||||
/* cards / panels */
|
||||
.panel{background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:18px 22px; margin:18px 0}
|
||||
.grid{display:grid; gap:14px}
|
||||
.grid.c2{grid-template-columns:1fr 1fr}
|
||||
.grid.c3{grid-template-columns:repeat(3,1fr)}
|
||||
@media(max-width:760px){.grid.c2,.grid.c3{grid-template-columns:1fr} nav.toc ol{columns:1}}
|
||||
/* tables */
|
||||
table{width:100%; border-collapse:collapse; margin:16px 0; font-size:.92rem}
|
||||
th,td{border:1px solid var(--border); padding:9px 12px; text-align:left; vertical-align:top}
|
||||
th{background:var(--panel2); color:var(--text); font-weight:600}
|
||||
tr:nth-child(even) td{background:rgba(255,255,255,.02)}
|
||||
/* flow diagram boxes */
|
||||
.flow{display:flex; align-items:center; flex-wrap:wrap; gap:10px; justify-content:center; margin:24px 0}
|
||||
.node{background:var(--panel2); border:1px solid var(--border); border-radius:10px;
|
||||
padding:12px 16px; text-align:center; min-width:130px}
|
||||
.node .t{font-weight:700} .node .s{font-size:.78rem; color:var(--muted)}
|
||||
.node.file{border-color:var(--green)} .node.file .t{color:var(--green)}
|
||||
.node.proc{border-color:var(--accent)} .node.proc .t{color:var(--accent)}
|
||||
.node.out{border-color:var(--purple)} .node.out .t{color:var(--purple)}
|
||||
.arrow{color:var(--muted); font-size:1.5rem; font-weight:bold}
|
||||
.vthread{display:flex; flex-direction:column; align-items:center; gap:6px}
|
||||
/* badges */
|
||||
.badge{display:inline-block; border-radius:6px; padding:2px 8px; font-size:.75rem; font-weight:600; border:1px solid}
|
||||
.b-green{color:var(--green); border-color:var(--green); background:rgba(63,185,80,.08)}
|
||||
.b-amber{color:var(--amber); border-color:var(--amber); background:rgba(210,153,34,.08)}
|
||||
.b-red{color:var(--red); border-color:var(--red); background:rgba(248,81,73,.08)}
|
||||
.b-blue{color:var(--accent); border-color:var(--accent); background:rgba(88,166,255,.08)}
|
||||
/* callouts */
|
||||
.call{border-left:4px solid var(--accent); background:var(--panel); border-radius:0 10px 10px 0; padding:12px 18px; margin:16px 0}
|
||||
.call.warn{border-left-color:var(--amber)}
|
||||
.call.danger{border-left-color:var(--red)}
|
||||
.call.ok{border-left-color:var(--green)}
|
||||
.call .h{font-weight:700; margin-bottom:4px}
|
||||
/* step list */
|
||||
.steps{counter-reset:step; list-style:none; padding:0}
|
||||
.steps li{position:relative; padding:14px 16px 14px 58px; margin:12px 0; background:var(--panel); border:1px solid var(--border); border-radius:10px}
|
||||
.steps li::before{counter-increment:step; content:counter(step);
|
||||
position:absolute; left:16px; top:14px; width:28px; height:28px; border-radius:50%;
|
||||
background:var(--accent); color:#06121f; font-weight:800; display:flex; align-items:center; justify-content:center}
|
||||
/* kpi */
|
||||
.kpi{display:flex; gap:14px; flex-wrap:wrap; margin:18px 0}
|
||||
.kpi .k{flex:1; min-width:150px; background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:16px; text-align:center}
|
||||
.kpi .k .v{font-size:1.7rem; font-weight:800; color:var(--cyan)}
|
||||
.kpi .k .l{font-size:.8rem; color:var(--muted)}
|
||||
footer{color:var(--muted); text-align:center; padding:30px; border-top:1px solid var(--border); margin-top:60px; font-size:.85rem}
|
||||
.legend{display:flex; gap:18px; flex-wrap:wrap; font-size:.82rem; color:var(--muted); justify-content:center; margin-top:10px}
|
||||
.legend span{display:flex; align-items:center; gap:6px}
|
||||
.dot{width:12px; height:12px; border-radius:3px; display:inline-block}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header class="hero">
|
||||
<h1>📱 App Store Scraper</h1>
|
||||
<p>Architecture & Data-Flow Reference</p>
|
||||
<p class="muted">Google Play + Apple App Store · discovery → scraping → combined CSV · scaled across 2 PCs</p>
|
||||
<span class="pill">Self-contained document — open in any browser</span>
|
||||
</header>
|
||||
|
||||
<div class="wrap">
|
||||
|
||||
<nav class="toc">
|
||||
<h4>Contents</h4>
|
||||
<ol>
|
||||
<li><a href="#overview">Big picture</a></li>
|
||||
<li><a href="#pipeline">The pipeline</a></li>
|
||||
<li><a href="#files">Files in the project</a></li>
|
||||
<li><a href="#discover">Phase 1 — Find apps (discover.py)</a></li>
|
||||
<li><a href="#appsjson">apps.json (the app list)</a></li>
|
||||
<li><a href="#scrape">Phase 2 — Get data (scraper.py)</a></li>
|
||||
<li><a href="#safety">Safety layer (netutil.py)</a></li>
|
||||
<li><a href="#concurrency">Concurrency (threads)</a></li>
|
||||
<li><a href="#twopc">Two-PC architecture</a></li>
|
||||
<li><a href="#resume">Resume & crash safety</a></li>
|
||||
<li><a href="#config">Configuration & flags</a></li>
|
||||
<li><a href="#cheatsheet">Command cheat sheet</a></li>
|
||||
<li><a href="#limits">Limits & ground truth</a></li>
|
||||
</ol>
|
||||
</nav>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="overview"><span class="num">0</span>Big picture</h2>
|
||||
<p>
|
||||
There is <strong>no "all apps" endpoint</strong> in either store. So the system works in
|
||||
<strong>two phases</strong>: first it <em>discovers</em> a big list of app IDs, then it
|
||||
<em>scrapes</em> the details for every app on that list into one CSV. Everything is free —
|
||||
no API keys, no paid services.
|
||||
</p>
|
||||
|
||||
<div class="kpi">
|
||||
<div class="k"><div class="v">2</div><div class="l">stores covered<br>(Play + App Store)</div></div>
|
||||
<div class="k"><div class="v">~30</div><div class="l">apps / Google Play query<br>(hard server cap)</div></div>
|
||||
<div class="k"><div class="v">~180</div><div class="l">apps / iTunes search term<br>(the volume engine)</div></div>
|
||||
<div class="k"><div class="v">100k+</div><div class="l">target catalog size<br>(App Store driven)</div></div>
|
||||
</div>
|
||||
|
||||
<div class="call ok">
|
||||
<div class="h">Two verbs, two phases</div>
|
||||
<strong>FIND apps</strong> → <code>discover.py</code> writes <code>apps.json</code>.
|
||||
<strong>GET data</strong> → <code>scraper.py</code> reads <code>apps.json</code> and writes <code>output/app_data.csv</code>.
|
||||
</div>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="pipeline"><span class="num">1</span>The pipeline</h2>
|
||||
<p>End-to-end, data flows left to right. The same flow runs on each PC; only the final merge is shared.</p>
|
||||
|
||||
<div class="flow">
|
||||
<div class="node proc"><div class="t">discover.py</div><div class="s">FIND apps</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node file"><div class="t">apps.json</div><div class="s">list of app IDs</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node proc"><div class="t">scraper.py</div><div class="s">GET data</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node out"><div class="t">app_data.csv</div><div class="s">one row / app / country</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node proc"><div class="t">merge_csv.py</div><div class="s">combine 2 PCs</div></div>
|
||||
</div>
|
||||
<div class="legend">
|
||||
<span><span class="dot" style="background:var(--accent)"></span>process (script)</span>
|
||||
<span><span class="dot" style="background:var(--green)"></span>data file</span>
|
||||
<span><span class="dot" style="background:var(--purple)"></span>output</span>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<tr><th>Step</th><th>Script</th><th>Reads</th><th>Writes</th><th>Purpose</th></tr>
|
||||
<tr><td>Find</td><td><code>discover.py</code></td><td>search APIs + charts</td><td><code>apps.json</code></td><td>build a big list of app IDs</td></tr>
|
||||
<tr><td>Get</td><td><code>scraper.py</code></td><td><code>apps.json</code></td><td><code>output/app_data.csv</code></td><td>fetch metadata + ratings for each app</td></tr>
|
||||
<tr><td>Merge</td><td><code>merge_csv.py</code></td><td>shard CSVs</td><td><code>output/app_data.csv</code></td><td>combine the two PCs' results, de-duped</td></tr>
|
||||
</table>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="files"><span class="num">2</span>Files in the project</h2>
|
||||
<table>
|
||||
<tr><th>File</th><th>Role</th></tr>
|
||||
<tr><td><code>discover.py</code></td><td><span class="badge b-blue">Phase 1</span> Finds app IDs and fills <code>apps.json</code></td></tr>
|
||||
<tr><td><code>scraper.py</code></td><td><span class="badge b-blue">Phase 2</span> Fetches per-app data into the CSV (threaded, shardable, resumable)</td></tr>
|
||||
<tr><td><code>netutil.py</code></td><td><span class="badge b-green">Shared safety</span> Error classification + adaptive throttle / circuit breaker</td></tr>
|
||||
<tr><td><code>merge_csv.py</code></td><td><span class="badge b-green">Helper</span> Merges the two PCs' shard CSVs into one</td></tr>
|
||||
<tr><td><code>apps.json</code></td><td><span class="badge b-amber">Data</span> Settings + the list of app IDs (the contract between the two phases)</td></tr>
|
||||
<tr><td><code>english_words.txt</code></td><td><span class="badge b-amber">Data</span> ~9,868 common words used as deep search terms (path to 100k)</td></tr>
|
||||
<tr><td><code>requirements.txt</code></td><td>Dependencies: <code>google-play-scraper</code>, <code>requests</code></td></tr>
|
||||
<tr><td><code>output/app_data.csv</code></td><td>The final dataset</td></tr>
|
||||
</table>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="discover"><span class="num">3</span>Phase 1 — Find apps <span class="muted" style="font-size:1rem">(discover.py)</span></h2>
|
||||
<p>
|
||||
Because no full catalog exists, discovery <strong>queries many sources with many terms</strong>
|
||||
and keeps only the unique app IDs. Three sources feed one de-duplicated set per store.
|
||||
</p>
|
||||
|
||||
<h3>The three discovery sources</h3>
|
||||
<table>
|
||||
<tr><th>Source</th><th>Store</th><th>How</th><th>Yield / call</th><th>Role</th></tr>
|
||||
<tr><td>Keyword search</td><td>Google Play</td><td><code>google_play_scraper.search(term)</code></td><td><span class="badge b-amber">~30</span> (hard cap)</td><td>breadth via many terms</td></tr>
|
||||
<tr><td>iTunes Search API</td><td>App Store</td><td><code>itunes.apple.com/search?term=</code></td><td><span class="badge b-green">~180</span></td><td><strong>main volume engine</strong></td></tr>
|
||||
<tr><td>RSS top charts</td><td>App Store</td><td><code>.../rss/{feed}/genre={id}</code> (free/grossing/paid × 23 genres)</td><td>up to 200</td><td>popular apps, fast</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>How the search terms are built</h3>
|
||||
<p>The function <code>build_terms()</code> assembles the query list from up to four tiers:</p>
|
||||
<div class="flow">
|
||||
<div class="node"><div class="t">~155 curated</div><div class="s">topic keywords</div></div>
|
||||
<span class="arrow">+</span>
|
||||
<div class="node"><div class="t">a–z</div><div class="s">26 letters</div></div>
|
||||
<span class="arrow">+</span>
|
||||
<div class="node"><div class="t">aa–zz</div><div class="s"><code>--deep</code> (676)</div></div>
|
||||
<span class="arrow">+</span>
|
||||
<div class="node"><div class="t">wordlist</div><div class="s"><code>--terms-file</code> (~9,868)</div></div>
|
||||
<span class="arrow">=</span>
|
||||
<div class="node out"><div class="t">~9,943 terms</div><div class="s">de-duped, ordered</div></div>
|
||||
</div>
|
||||
<p class="muted">
|
||||
More terms = more apps. The wordlist (<code>--terms-file english_words.txt</code>) is the single
|
||||
biggest lever toward 100k. <code>--deeper</code> (aaa–zzz, 17,576 terms) exists but is very slow.
|
||||
</p>
|
||||
|
||||
<h3>Discovery flow (per store)</h3>
|
||||
<ol class="steps">
|
||||
<li>Build the term list and resolve the country list (from <code>--countries</code> or settings).</li>
|
||||
<li>For each country, loop every term; call the source through the safety layer (retry + backoff).</li>
|
||||
<li>Collect each result's <strong>app ID</strong> into a <code>seen</code> dictionary — duplicates are ignored automatically.</li>
|
||||
<li>Print live progress: <code>total unique = N / target</code>.</li>
|
||||
<li>Stop as soon as <code>--target</code> is reached (or terms are exhausted).</li>
|
||||
<li><strong>Merge</strong> into the existing <code>apps.json</code> (keeps your settings + anything already listed), then write the file.</li>
|
||||
</ol>
|
||||
|
||||
<div class="call">
|
||||
<div class="h">Why Google Play can't reach 100k</div>
|
||||
Play search returns only <strong>~30 apps per query no matter what</strong> — that's Google's cap, not the
|
||||
code's. So Play contributes ~15–30k; the App Store's ~180/term search is what carries you to 100k.
|
||||
</div>
|
||||
|
||||
<div class="call ok">
|
||||
<div class="h">De-duplication is automatic and idempotent</div>
|
||||
Discovery keys on app ID, and <code>merge()</code> drops duplicates against what's already in
|
||||
<code>apps.json</code>. Re-running only <em>adds</em> new apps — it never loses or doubles existing ones.
|
||||
If <code>apps.json</code> is missing, a fresh one is created automatically.
|
||||
</div>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="appsjson"><span class="num">4</span>apps.json — the app list</h2>
|
||||
<p>This file is the <strong>contract</strong> between the two phases: discovery writes it, scraping reads it.</p>
|
||||
<pre><code>{
|
||||
"settings": {
|
||||
"country": "us", // default single country
|
||||
"countries": ["us","gb","in","ca","au"], // scrape/discover across these
|
||||
"lang": "en", // language (Google Play)
|
||||
"review_count": 100,
|
||||
"delay_seconds": 1.0 // polite pause; rate ≈ workers / delay
|
||||
},
|
||||
"google_play": [
|
||||
{ "app_id": "com.whatsapp" } // the id= value from the Play URL (string)
|
||||
],
|
||||
"app_store": [
|
||||
{ "app_id": 310633997, "name": "WhatsApp" } // numeric id from the App Store URL
|
||||
]
|
||||
}</code></pre>
|
||||
<table>
|
||||
<tr><th>Store</th><th>ID type</th><th>Where it comes from</th></tr>
|
||||
<tr><td>Google Play</td><td>string</td><td><code>play.google.com/store/apps/details?id=<strong>com.whatsapp</strong></code></td></tr>
|
||||
<tr><td>App Store</td><td>number</td><td><code>apps.apple.com/us/app/.../id<strong>310633997</strong></code></td></tr>
|
||||
</table>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="scrape"><span class="num">5</span>Phase 2 — Get data <span class="muted" style="font-size:1rem">(scraper.py)</span></h2>
|
||||
<p>
|
||||
Scraping turns the ID list into real data. For <strong>every app × every country</strong> it makes one
|
||||
"detail" call and writes one CSV row. Two stores, two API calls, one unified row schema.
|
||||
</p>
|
||||
|
||||
<h3>The two detail calls</h3>
|
||||
<div class="grid c2">
|
||||
<div class="panel">
|
||||
<h3 style="margin-top:0">Google Play</h3>
|
||||
<p><code>google_play_scraper.app(app_id, lang, country)</code></p>
|
||||
<p class="muted">Scrapes the public listing page → title, developer, genre, price, score, ratings count, review count, last-updated, version, URL.</p>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<h3 style="margin-top:0">Apple App Store</h3>
|
||||
<p><code>itunes.apple.com/lookup?id=&country=</code></p>
|
||||
<p class="muted">Official lookup API → trackName, seller, genre, price, average rating, rating count, release date, version, URL.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h3>Output columns — <code>output/app_data.csv</code></h3>
|
||||
<table>
|
||||
<tr><th>Column</th><th>Meaning</th></tr>
|
||||
<tr><td><code>store</code></td><td><code>google_play</code> or <code>app_store</code></td></tr>
|
||||
<tr><td><code>country</code></td><td>which region this row was fetched for</td></tr>
|
||||
<tr><td><code>app_id</code>, <code>title</code>, <code>developer</code>, <code>category</code></td><td>identity</td></tr>
|
||||
<tr><td><code>price</code>, <code>currency</code>, <code>free</code></td><td>pricing</td></tr>
|
||||
<tr><td><code>avg_rating</code></td><td>final average star rating</td></tr>
|
||||
<tr><td><code>total_ratings</code></td><td>total number of ratings</td></tr>
|
||||
<tr><td><code>text_review_count</code></td><td>number of written reviews (Play only)</td></tr>
|
||||
<tr><td><code>last_updated</code></td><td>date the app was last updated</td></tr>
|
||||
<tr><td><code>version</code></td><td>latest version</td></tr>
|
||||
<tr><td><code>url</code></td><td>store listing link</td></tr>
|
||||
</table>
|
||||
<p class="muted">CSV is UTF-8 with BOM → opens cleanly in Excel. The same columns apply to both stores.</p>
|
||||
|
||||
<h3>How one job is processed</h3>
|
||||
<div class="flow">
|
||||
<div class="node">job<br><span class="s">(store, id, country)</span></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node proc">throttle.wait()<br><span class="s">honor cooldown</span></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node proc">fetch<br><span class="s">Play / App Store</span></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node out">row<br><span class="s">flushed to CSV</span></div>
|
||||
</div>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="safety"><span class="num">6</span>Safety layer <span class="muted" style="font-size:1rem">(netutil.py)</span></h2>
|
||||
<div class="call danger">
|
||||
<div class="h">The #1 rule: slow is fine — getting the IP blocked is not.</div>
|
||||
Every request (in both scripts) passes through a shared throttle. The moment a store pushes back,
|
||||
<strong>all workers slow down together</strong> instead of hammering until blocked.
|
||||
</div>
|
||||
|
||||
<h3>Error classification</h3>
|
||||
<table>
|
||||
<tr><th>Signal</th><th>Meaning</th><th>Reaction</th></tr>
|
||||
<tr><td><span class="badge b-red">429 / 403</span> "too many requests", "quota"</td><td>rate-limited</td><td><strong>Trip circuit breaker</strong>, wait it out, retry generously</td></tr>
|
||||
<tr><td><span class="badge b-amber">408 / 5xx</span>, timeout, connection</td><td>transient hiccup</td><td>short exponential backoff, a few retries</td></tr>
|
||||
<tr><td><span class="badge b-green">404</span> / not-found</td><td>healthy answer</td><td>skip immediately — <em>no</em> wasted retries</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>The circuit breaker (a shared state machine)</h3>
|
||||
<div class="flow">
|
||||
<div class="node" style="border-color:var(--green)"><div class="t" style="color:var(--green)">RUNNING</div><div class="s">requests flow freely</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node" style="border-color:var(--red)"><div class="t" style="color:var(--red)">TRIPPED</div><div class="s">all workers pause</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node" style="border-color:var(--amber)"><div class="t" style="color:var(--amber)">COOLDOWN</div><div class="s">30→60→120…→300s</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node" style="border-color:var(--green)"><div class="t" style="color:var(--green)">RECOVER</div><div class="s">speed back up on success</div></div>
|
||||
</div>
|
||||
<ul>
|
||||
<li><strong>Trips on</strong> a rate-limit signal <em>or</em> 5 consecutive unexplained failures (catches "stealth" blocks that don't return a clean 429).</li>
|
||||
<li><strong>Cooldown doubles</strong> each time it re-trips (30s → 60 → 120 … capped at 5 min) and lingers as extra spacing.</li>
|
||||
<li><strong>Recovers automatically</strong>: each success shrinks the cooldown and extra delay back toward normal.</li>
|
||||
</ul>
|
||||
|
||||
<div class="call warn">
|
||||
<div class="h">Reading the logs</div>
|
||||
A line like <code>THROTTLING: pausing all workers ~60s</code> is the protection <em>working</em>, not an error.
|
||||
If it fires constantly, that IP is at its limit → lower <code>--workers</code> on that machine.
|
||||
</div>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="concurrency"><span class="num">7</span>Concurrency — why threads</h2>
|
||||
<p>
|
||||
These are <strong>network-wait</strong> tasks (most time is spent waiting for the server, not computing),
|
||||
which is the ideal case for a <strong>thread pool</strong>. With <code>--workers N</code>, up to N requests are
|
||||
in flight at once.
|
||||
</p>
|
||||
<div class="flow">
|
||||
<div class="node file"><div class="t">job queue</div><div class="s">all apps × countries</div></div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="vthread">
|
||||
<div class="node proc" style="min-width:200px">worker 1</div>
|
||||
<div class="node proc" style="min-width:200px">worker 2</div>
|
||||
<div class="node proc" style="min-width:200px">… worker N</div>
|
||||
</div>
|
||||
<span class="arrow">→</span>
|
||||
<div class="node out"><div class="t">CSV sink</div><div class="s">thread-safe, flush per row</div></div>
|
||||
</div>
|
||||
<p>
|
||||
Effective request rate ≈ <code>workers ÷ delay_seconds</code>. The throttle sits above all workers,
|
||||
so one rate-limit signal pauses the whole pool.
|
||||
</p>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="twopc"><span class="num">8</span>Two-PC architecture <span class="muted" style="font-size:1rem">(2 CPUs, 2 IPs)</span></h2>
|
||||
<p>
|
||||
Two machines with <strong>separate IPs</strong> double throughput <em>and</em> the safe request budget
|
||||
(each store rate-limits per IP). You don't split the file by hand — both PCs read the <em>same</em>
|
||||
<code>apps.json</code> and each takes a different <strong>shard</strong>.
|
||||
</p>
|
||||
|
||||
<h3>How sharding splits the work</h3>
|
||||
<p>
|
||||
All jobs are flattened to a list, then each PC takes <code>jobs[shard::shards]</code> — i.e. every Nth job.
|
||||
This interleaves Play/App-Store/country evenly, so neither PC gets a lopsided slice.
|
||||
</p>
|
||||
<div class="flow">
|
||||
<div class="node file" style="min-width:200px"><div class="t">apps.json</div><div class="s">jobs: 0,1,2,3,4,5,6,7…</div></div>
|
||||
</div>
|
||||
<div class="grid c2">
|
||||
<div class="panel">
|
||||
<h3 style="margin-top:0">PC #1 — IP A</h3>
|
||||
<p><code>--shard 0 --shards 2</code></p>
|
||||
<p>does jobs <strong>0, 2, 4, 6 …</strong></p>
|
||||
<p class="muted">writes <code>app_data_shard0of2.csv</code></p>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<h3 style="margin-top:0">PC #2 — IP B</h3>
|
||||
<p><code>--shard 1 --shards 2</code></p>
|
||||
<p>does jobs <strong>1, 3, 5, 7 …</strong></p>
|
||||
<p class="muted">writes <code>app_data_shard1of2.csv</code></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flow">
|
||||
<div class="node out">shard0 CSV</div>
|
||||
<span class="arrow">↘</span>
|
||||
<div class="node proc"><div class="t">merge_csv.py</div><div class="s">de-duped by store+country+id</div></div>
|
||||
<span class="arrow">↙</span>
|
||||
<div class="node out">shard1 CSV</div>
|
||||
</div>
|
||||
<div class="flow">
|
||||
<span class="arrow">↓</span>
|
||||
</div>
|
||||
<div class="flow">
|
||||
<div class="node out"><div class="t">app_data.csv</div><div class="s">complete combined dataset</div></div>
|
||||
</div>
|
||||
|
||||
<h3>The 3 steps end-to-end</h3>
|
||||
<ol class="steps">
|
||||
<li><strong>Discover once</strong> (either PC): run <code>discover.py</code>, then copy the resulting <code>apps.json</code> to both machines.</li>
|
||||
<li><strong>Scrape in parallel</strong>: PC #1 runs shard 0, PC #2 runs shard 1 (each with its own IP and its own thread pool).</li>
|
||||
<li><strong>Merge</strong>: copy both shard CSVs into one <code>output/</code> folder and run <code>merge_csv.py</code>.</li>
|
||||
</ol>
|
||||
|
||||
<div class="kpi">
|
||||
<div class="k"><div class="v">÷2</div><div class="l">work per PC<br>(sharding)</div></div>
|
||||
<div class="k"><div class="v">×~10</div><div class="l">per PC<br>(thread pool)</div></div>
|
||||
<div class="k"><div class="v">×2 IPs</div><div class="l">safe rate budget<br>(per-IP limits)</div></div>
|
||||
<div class="k"><div class="v">28h → ~1h</div><div class="l">rough wall-clock<br>for 100k records</div></div>
|
||||
</div>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="resume"><span class="num">9</span>Resume & crash safety</h2>
|
||||
<p>A 100k run can't depend on a single perfect execution. Two mechanisms make interruptions harmless:</p>
|
||||
<div class="grid c2">
|
||||
<div class="panel">
|
||||
<h3 style="margin-top:0">💾 Flush-as-you-go</h3>
|
||||
<p>Each row is written and flushed to the CSV <strong>the instant it's fetched</strong>. A crash, block,
|
||||
or <kbd>Ctrl-C</kbd> never loses completed work.</p>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<h3 style="margin-top:0">⏯️ Resume on restart</h3>
|
||||
<p>On startup the scraper reads the existing CSV and <strong>skips apps already done</strong>
|
||||
(keyed by store + country + id). Just re-run the same command to continue.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="call ok">
|
||||
<div class="h">Practical meaning</div>
|
||||
If a PC is interrupted at app 40,000 of 50,000, re-running the exact same command picks up at 40,001.
|
||||
Use <code>--no-resume</code> only if you deliberately want to re-scrape everything.
|
||||
</div>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="config"><span class="num">10</span>Configuration & flags</h2>
|
||||
|
||||
<h3>discover.py</h3>
|
||||
<table>
|
||||
<tr><th>Flag</th><th>Default</th><th>Purpose</th></tr>
|
||||
<tr><td><code>--target N</code></td><td>0 (all terms)</td><td>stop each store after N unique apps</td></tr>
|
||||
<tr><td><code>--terms-file PATH</code></td><td>—</td><td>extra search terms (best lever to 100k)</td></tr>
|
||||
<tr><td><code>--deep</code></td><td>off</td><td>add 2-letter terms aa–zz (676)</td></tr>
|
||||
<tr><td><code>--deeper</code></td><td>off</td><td>add 3-letter terms aaa–zzz (17,576; slow)</td></tr>
|
||||
<tr><td><code>--countries a,b,c</code></td><td>settings</td><td>regions to discover across</td></tr>
|
||||
<tr><td><code>--skip-play</code> / <code>--skip-appstore</code></td><td>off</td><td>run only one store</td></tr>
|
||||
<tr><td><code>--search-limit</code> / <code>--chart-limit</code></td><td>200 / 200</td><td>results per iTunes term / per chart</td></tr>
|
||||
<tr><td><code>--hits</code></td><td>30</td><td>Play results per term (server caps ~30)</td></tr>
|
||||
<tr><td><code>--delay</code></td><td>0.5</td><td>polite pause between discovery requests</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>scraper.py</h3>
|
||||
<table>
|
||||
<tr><th>Flag</th><th>Default</th><th>Purpose</th></tr>
|
||||
<tr><td><code>--workers N</code></td><td>10</td><td>concurrent requests (thread pool)</td></tr>
|
||||
<tr><td><code>--shard i</code></td><td>0</td><td>this machine's slice index</td></tr>
|
||||
<tr><td><code>--shards N</code></td><td>1</td><td>total machines splitting the work</td></tr>
|
||||
<tr><td><code>--countries a,b,c</code></td><td>settings</td><td>regions to scrape (one row each)</td></tr>
|
||||
<tr><td><code>--no-resume</code></td><td>off</td><td>ignore existing CSV, scrape all again</td></tr>
|
||||
<tr><td><code>--out DIR</code> / <code>--config PATH</code></td><td>output / apps.json</td><td>output folder / input file</td></tr>
|
||||
</table>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="cheatsheet"><span class="num">11</span>Command cheat sheet</h2>
|
||||
|
||||
<h3>① Discover ~100k apps (App Store driven)</h3>
|
||||
<pre><code># fastest route to 1 lakh — App Store only, using the wordlist
|
||||
python discover.py --skip-play --target 100000 --terms-file english_words.txt --countries us
|
||||
|
||||
# include Google Play too (adds ~15–30k; can't reach 100k on its own)
|
||||
python discover.py --target 100000 --terms-file english_words.txt --countries us,gb</code></pre>
|
||||
|
||||
<h3>② Scrape across 2 PCs</h3>
|
||||
<pre><code># --- PC #1 (IP A) ---
|
||||
python scraper.py --shard 0 --shards 2 --workers 10
|
||||
|
||||
# --- PC #2 (IP B) ---
|
||||
python scraper.py --shard 1 --shards 2 --workers 10
|
||||
|
||||
# ...interrupted? just run the same command again — it resumes.</code></pre>
|
||||
|
||||
<h3>③ Merge the two PCs' output</h3>
|
||||
<pre><code># copy both app_data_shard*of2.csv into one output/ folder, then:
|
||||
python merge_csv.py</code></pre>
|
||||
|
||||
<h3>Single-PC run (no sharding)</h3>
|
||||
<pre><code>python scraper.py --workers 10</code></pre>
|
||||
|
||||
<!-- ============================================================= -->
|
||||
<h2 id="limits"><span class="num">12</span>Limits & ground truth</h2>
|
||||
<table>
|
||||
<tr><th>Reality</th><th>Detail</th></tr>
|
||||
<tr><td>No full catalog</td><td>Neither store lists "all apps"; you only get what discovery finds.</td></tr>
|
||||
<tr><td>Google Play search cap</td><td><strong>~30 results per query</strong> — Play tops out ~15–30k regardless of terms.</td></tr>
|
||||
<tr><td>App Store is the volume</td><td>iTunes search ~180/term → this is what reaches 100k.</td></tr>
|
||||
<tr><td>Per-IP rate limits</td><td>Both stores throttle per IP; 2 PCs/IPs ≈ 2× safe budget. Proxies extend further.</td></tr>
|
||||
<tr><td>Per-country data</td><td>Ratings & pricing differ by country → one row per app per country.</td></tr>
|
||||
<tr><td>Terms of Service</td><td>Both stores restrict automated scraping. Fine for internal research at modest scale; get sign-off before commercial redistribution.</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="call warn">
|
||||
<div class="h">Tuning rule of thumb</div>
|
||||
Start at <code>--workers 10</code>. If you see constant <code>THROTTLING</code> lines, <strong>lower</strong> workers
|
||||
(e.g. 6) — fewer cooldowns beats pushing too hard. Back up <code>apps.json</code> after a big discovery run;
|
||||
it's the expensive artifact.
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
App Store Scraper — Architecture & Flow · Google Play + Apple App Store · self-contained reference (no internet needed to view)
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,2 +1,73 @@
|
||||
# App_Scraper
|
||||
# App Store Scraper (Google Play + Apple App Store)
|
||||
|
||||
For a **specific list of apps** (defined in `apps.json`) this produces **one combined CSV**
|
||||
with core metadata, the **last-updated date**, and the **final rating** (average + total
|
||||
number of ratings). Python, free libraries only.
|
||||
|
||||
## What it uses
|
||||
|
||||
| Store | Source |
|
||||
|-------|--------|
|
||||
| Google Play | `google-play-scraper` (scrapes public pages) |
|
||||
| Apple App Store | iTunes Lookup API (**official**, free) |
|
||||
|
||||
No paid services, no API keys.
|
||||
|
||||
## Setup
|
||||
|
||||
```powershell
|
||||
python -m pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## Configure — `apps.json`
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"settings": {
|
||||
"country": "us", // store country code
|
||||
"lang": "en", // language (Play only)
|
||||
"delay_seconds": 1.0 // pause between requests (avoid blocks)
|
||||
},
|
||||
"google_play": [
|
||||
{ "app_id": "com.whatsapp" } // the "id=" value in the Play URL
|
||||
],
|
||||
"app_store": [
|
||||
{ "app_id": 310633997, "name": "WhatsApp" } // the numeric "id" in the App Store URL ("name" is just a label)
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Finding IDs**
|
||||
- Play Store: `play.google.com/store/apps/details?id=`**`com.whatsapp`** → use `com.whatsapp`
|
||||
- App Store: `apps.apple.com/us/app/whatsapp-messenger/id`**`310633997`** → use `310633997`
|
||||
|
||||
## Run
|
||||
|
||||
```powershell
|
||||
python scraper.py
|
||||
```
|
||||
|
||||
## Output — `output/app_data.csv` (one row per app, both stores together)
|
||||
|
||||
| Column | Meaning |
|
||||
|--------|---------|
|
||||
| `store` | `google_play` or `app_store` |
|
||||
| `app_id`, `title`, `developer`, `category` | identity |
|
||||
| `price`, `currency`, `free` | pricing |
|
||||
| `avg_rating` | final average star rating |
|
||||
| `total_ratings` | total number of ratings |
|
||||
| `text_review_count` | number of text reviews (Play only) |
|
||||
| `last_updated` | date the app was last updated |
|
||||
| `version` | latest version |
|
||||
| `url` | store listing URL |
|
||||
|
||||
UTF-8 with BOM — opens cleanly in Excel.
|
||||
|
||||
## Limitations (important)
|
||||
|
||||
- **No "all apps" list exists.** Neither store exposes a full catalog; you scrape only
|
||||
the apps you list here. Use `discover.py` to auto-fill `apps.json` from top charts.
|
||||
- **Rate limits / blocking** — keep `delay_seconds > 0`; heavy use may need proxies.
|
||||
- **Terms of Service** — both stores restrict automated scraping. Fine for internal research
|
||||
at small scale; get legal sign-off before commercial redistribution.
|
||||
- **Per-country data** — results differ by `country`.
|
||||
|
||||
+349
@@ -0,0 +1,349 @@
|
||||
"""
|
||||
discover.py — auto-fill apps.json with LOTS of real apps (target: 100k+).
|
||||
|
||||
There is no "all apps" endpoint, so we discover apps from several sources and
|
||||
MERGE them (de-duplicated) into apps.json, keeping your "settings" block and any
|
||||
apps already listed.
|
||||
|
||||
SOURCES
|
||||
Google Play
|
||||
- search() across many terms. NOTE: Play search returns only ~30 apps per
|
||||
query (a hard server cap), so Play volume comes from using MANY terms.
|
||||
App Store
|
||||
- iTunes Search API: ~180 apps per term <-- the real workhorse for volume
|
||||
- RSS top charts (top free / grossing / paid) per genre, for popular apps
|
||||
|
||||
REACHING 100k
|
||||
Top charts alone top out in the low thousands (only popular apps exist there).
|
||||
Volume comes from searching MANY terms. Build a big term list with:
|
||||
--deep add all 2-letter combos (aa..zz, 676 terms)
|
||||
--terms-file words.txt add your own wordlist (one term per line) -> best lever
|
||||
Most of the 100k will be App Store apps; Play contributes fewer (30/query cap).
|
||||
|
||||
SAFETY
|
||||
All requests go through netutil's retry/backoff + shared circuit breaker, so a
|
||||
rate-limit just slows us down instead of getting the IP blocked.
|
||||
|
||||
Run: python discover.py --target 100000 --deep --countries us,gb
|
||||
python discover.py --target 100000 --terms-file english_words.txt
|
||||
python discover.py --skip-play --target 100000 --deep # App Store only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import string
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from google_play_scraper import search as gp_search
|
||||
|
||||
from netutil import Throttle, retry_call
|
||||
|
||||
# Curated topic terms (used for BOTH stores' search).
|
||||
SEARCH_TERMS = [
|
||||
# games
|
||||
"games", "action games", "puzzle games", "racing games", "strategy games",
|
||||
"rpg", "simulation games", "arcade", "card games", "board games", "casino",
|
||||
"word games", "trivia", "io games", "tower defense", "shooter", "adventure",
|
||||
# social / communication
|
||||
"social", "social media", "messaging", "chat", "dating", "video call",
|
||||
"community", "forum",
|
||||
# music / audio
|
||||
"music", "music player", "radio", "podcast", "audiobook", "karaoke",
|
||||
# photo / video
|
||||
"photo editor", "video editor", "camera", "collage", "selfie",
|
||||
"video player", "screen recorder",
|
||||
# shopping
|
||||
"shopping", "online shopping", "deals", "coupons", "grocery", "fashion",
|
||||
# finance
|
||||
"finance", "banking", "investing", "crypto", "budget", "payments",
|
||||
"insurance", "taxes", "stocks",
|
||||
# education
|
||||
"education", "language learning", "kids learning", "math", "science",
|
||||
"coding", "exam prep", "flashcards",
|
||||
# health / fitness
|
||||
"health", "fitness", "workout", "yoga", "meditation", "sleep", "diet",
|
||||
"period tracker", "medical", "mental health",
|
||||
# travel / maps
|
||||
"travel", "flights", "hotels", "maps navigation", "ride sharing",
|
||||
"public transport",
|
||||
# food
|
||||
"food delivery", "recipes", "restaurants",
|
||||
# news / weather / sports
|
||||
"news", "weather", "sports", "live scores",
|
||||
# productivity / office
|
||||
"productivity", "notes", "calendar", "to do list", "office", "pdf",
|
||||
"scanner", "email", "cloud storage",
|
||||
# utilities / tools
|
||||
"utilities", "file manager", "cleaner", "antivirus", "vpn", "browser",
|
||||
"keyboard", "launcher", "wallpaper", "ringtones", "qr scanner",
|
||||
"flashlight", "battery",
|
||||
# streaming / entertainment
|
||||
"streaming", "movies", "tv shows", "live tv", "anime", "entertainment",
|
||||
# books / reading
|
||||
"books", "comics", "manga", "ebook reader",
|
||||
# business / misc
|
||||
"business", "crm", "invoicing", "job search", "real estate", "parenting",
|
||||
"pregnancy", "art", "drawing", "translator",
|
||||
# AI
|
||||
"ai chatbot", "ai image generator", "ai assistant",
|
||||
]
|
||||
|
||||
# Apple App Store genre IDs for the RSS charts.
|
||||
APPSTORE_GENRES = {
|
||||
6005: "Social Networking", 6014: "Games", 6016: "Entertainment",
|
||||
6007: "Productivity", 6008: "Photo & Video", 6011: "Music",
|
||||
6012: "Lifestyle", 6015: "Finance", 6017: "Education",
|
||||
6013: "Health & Fitness", 6023: "Food & Drink", 6024: "Shopping",
|
||||
6000: "Business", 6002: "Utilities", 6003: "Travel", 6004: "Sports",
|
||||
6009: "News", 6010: "Navigation", 6018: "Books", 6001: "Weather",
|
||||
6020: "Medical", 6006: "Reference", 6021: "Magazines & Newspapers",
|
||||
}
|
||||
|
||||
APPSTORE_FEEDS = [
|
||||
"topfreeapplications",
|
||||
"topgrossingapplications",
|
||||
"toppaidapplications",
|
||||
]
|
||||
|
||||
RSS_CHART = (
|
||||
"https://itunes.apple.com/{country}/rss/{feed}/"
|
||||
"limit={limit}/genre={genre}/json"
|
||||
)
|
||||
ITUNES_SEARCH = "https://itunes.apple.com/search"
|
||||
|
||||
|
||||
def build_terms(deep: bool, deeper: bool, terms_file: str | None) -> list:
|
||||
"""Assemble the search-term list: curated + alphabet sweep + optional file."""
|
||||
terms = list(SEARCH_TERMS)
|
||||
terms += list(string.ascii_lowercase) # a..z (26)
|
||||
if deep:
|
||||
terms += [a + b for a in string.ascii_lowercase
|
||||
for b in string.ascii_lowercase] # aa..zz (676)
|
||||
if deeper:
|
||||
terms += [a + b + c for a in string.ascii_lowercase
|
||||
for b in string.ascii_lowercase
|
||||
for c in string.ascii_lowercase] # aaa..zzz (17576)
|
||||
if terms_file:
|
||||
words = Path(terms_file).read_text(encoding="utf-8").split()
|
||||
terms += words
|
||||
# de-dupe, preserve order, drop blanks
|
||||
seen, out = set(), []
|
||||
for t in terms:
|
||||
t = t.strip().lower()
|
||||
if t and t not in seen:
|
||||
seen.add(t)
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def _reached(target: int | None, n: int) -> bool:
|
||||
return bool(target) and n >= target
|
||||
|
||||
|
||||
# --- Google Play ------------------------------------------------------------
|
||||
def discover_play(terms, lang, countries, hits, target, delay, throttle) -> list:
|
||||
seen: dict = {}
|
||||
for country in countries:
|
||||
for term in terms:
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
try:
|
||||
items = retry_call(
|
||||
lambda: gp_search(term, n_hits=hits, lang=lang, country=country),
|
||||
throttle=throttle,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f" [Play:{country}] '{term}' FAILED -> {exc}")
|
||||
continue
|
||||
for item in items:
|
||||
app_id = item.get("appId")
|
||||
if app_id and app_id not in seen:
|
||||
seen[app_id] = {"app_id": app_id}
|
||||
print(f" [Play:{country}] '{term}': total unique = {len(seen)}"
|
||||
f"{f' / {target}' if target else ''}")
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
apps = list(seen.values())
|
||||
return apps[:target] if target else apps
|
||||
|
||||
|
||||
# --- App Store --------------------------------------------------------------
|
||||
def _rss_json(url: str) -> dict:
|
||||
resp = requests.get(url, timeout=20)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _search_json(term: str, country: str, limit: int) -> dict:
|
||||
resp = requests.get(
|
||||
ITUNES_SEARCH,
|
||||
params={"term": term, "country": country, "entity": "software", "limit": limit},
|
||||
timeout=20,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _add_appstore(seen: dict, app_id, name) -> None:
|
||||
try:
|
||||
app_id = int(app_id)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if app_id not in seen:
|
||||
seen[app_id] = {"app_id": app_id, "name": name}
|
||||
|
||||
|
||||
def discover_appstore(terms, countries, chart_limit, search_limit, target,
|
||||
delay, throttle) -> list:
|
||||
seen: dict = {}
|
||||
for country in countries:
|
||||
# 1) Top charts — high-quality popular apps (fast, bounded).
|
||||
for feed in APPSTORE_FEEDS:
|
||||
for genre, name in APPSTORE_GENRES.items():
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
url = RSS_CHART.format(
|
||||
country=country, feed=feed, limit=chart_limit, genre=genre
|
||||
)
|
||||
try:
|
||||
entries = (retry_call(_rss_json, url, throttle=throttle)
|
||||
.get("feed", {}).get("entry", []))
|
||||
if isinstance(entries, dict):
|
||||
entries = [entries]
|
||||
for e in entries:
|
||||
_add_appstore(
|
||||
seen,
|
||||
e.get("id", {}).get("attributes", {}).get("im:id"),
|
||||
e.get("im:name", {}).get("label"),
|
||||
)
|
||||
print(f" [Store:{country}/{feed}] {name}: total unique = "
|
||||
f"{len(seen)}{f' / {target}' if target else ''}")
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
except Exception as exc:
|
||||
print(f" [Store:{country}/{feed}] {name} FAILED -> {exc}")
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
|
||||
# 2) Search API — the deep, high-volume source (~180 apps per term).
|
||||
for term in terms:
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
try:
|
||||
results = retry_call(
|
||||
_search_json, term, country, search_limit, throttle=throttle
|
||||
).get("results", [])
|
||||
except Exception as exc:
|
||||
print(f" [Store:{country}/search] '{term}' FAILED -> {exc}")
|
||||
continue
|
||||
for r in results:
|
||||
_add_appstore(seen, r.get("trackId"), r.get("trackName"))
|
||||
print(f" [Store:{country}/search] '{term}': total unique = "
|
||||
f"{len(seen)}{f' / {target}' if target else ''}")
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
if _reached(target, len(seen)):
|
||||
break
|
||||
|
||||
apps = list(seen.values())
|
||||
return apps[:target] if target else apps
|
||||
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"country": "us",
|
||||
"countries": ["us", "gb", "in", "ca", "au"],
|
||||
"lang": "en",
|
||||
"review_count": 100,
|
||||
"delay_seconds": 1.0,
|
||||
}
|
||||
|
||||
|
||||
def load_config(path: Path) -> dict:
|
||||
"""Read apps.json, or start a fresh structure if it doesn't exist yet."""
|
||||
if path.exists():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
print(f"(config {path} not found - creating a fresh one)")
|
||||
return {"settings": dict(DEFAULT_SETTINGS), "google_play": [], "app_store": []}
|
||||
|
||||
|
||||
def merge(existing: list, discovered: list, key: str) -> list:
|
||||
seen = set()
|
||||
merged = []
|
||||
for item in existing + discovered:
|
||||
k = item.get(key)
|
||||
if k is not None and k not in seen:
|
||||
seen.add(k)
|
||||
merged.append(item)
|
||||
return merged
|
||||
|
||||
|
||||
def resolve_countries(args, settings: dict) -> list:
|
||||
if args.countries:
|
||||
return [c.strip() for c in args.countries.split(",") if c.strip()]
|
||||
return settings.get("countries") or [settings.get("country", "us")]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Auto-fill apps.json with popular apps")
|
||||
parser.add_argument("--config", default="apps.json")
|
||||
parser.add_argument("--countries", default=None,
|
||||
help="comma-separated, e.g. us,gb,in (overrides settings)")
|
||||
parser.add_argument("--target", type=int, default=0,
|
||||
help="stop each store after N unique apps (0 = use all terms)")
|
||||
parser.add_argument("--hits", type=int, default=30,
|
||||
help="Play results per term (~30 is the server cap)")
|
||||
parser.add_argument("--chart-limit", type=int, default=200,
|
||||
help="App Store apps per genre chart (max 200)")
|
||||
parser.add_argument("--search-limit", type=int, default=200,
|
||||
help="iTunes search results per term (max 200)")
|
||||
parser.add_argument("--deep", action="store_true",
|
||||
help="add 2-letter terms (aa..zz) for much wider coverage")
|
||||
parser.add_argument("--deeper", action="store_true",
|
||||
help="add 3-letter terms (aaa..zzz) — 17k terms, very slow")
|
||||
parser.add_argument("--terms-file", default=None,
|
||||
help="extra search terms, one per line (best way to reach 100k)")
|
||||
parser.add_argument("--delay", type=float, default=0.5,
|
||||
help="polite delay (s) between discovery requests")
|
||||
parser.add_argument("--skip-play", action="store_true", help="skip Google Play")
|
||||
parser.add_argument("--skip-appstore", action="store_true", help="skip App Store")
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = Path(args.config)
|
||||
cfg = load_config(config_path)
|
||||
s = cfg.get("settings", {})
|
||||
lang = s.get("lang", "en")
|
||||
countries = resolve_countries(args, s)
|
||||
target = args.target or None
|
||||
terms = build_terms(args.deep, args.deeper, args.terms_file)
|
||||
throttle = Throttle()
|
||||
|
||||
print(f"Countries: {', '.join(countries)} | search terms: {len(terms)} | "
|
||||
f"target/store: {target or 'all terms'}")
|
||||
|
||||
if not args.skip_play:
|
||||
print("\nDiscovering Google Play apps...")
|
||||
play = discover_play(terms, lang, countries, args.hits, target,
|
||||
args.delay, throttle)
|
||||
cfg["google_play"] = merge(cfg.get("google_play", []), play, "app_id")
|
||||
|
||||
if not args.skip_appstore:
|
||||
print("\nDiscovering App Store apps...")
|
||||
store = discover_appstore(terms, countries, args.chart_limit,
|
||||
args.search_limit, target, args.delay, throttle)
|
||||
cfg["app_store"] = merge(cfg.get("app_store", []), store, "app_id")
|
||||
|
||||
config_path.write_text(
|
||||
json.dumps(cfg, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
print(
|
||||
f"\nDONE apps.json now has {len(cfg.get('google_play', []))} Play apps "
|
||||
f"and {len(cfg.get('app_store', []))} App Store apps."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+9868
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
merge_csv.py — combine sharded scraper outputs into one app_data.csv.
|
||||
|
||||
After PC #1 (shard 0) and PC #2 (shard 1) finish, copy both shard CSVs into the
|
||||
same output folder and run this. It concatenates them and drops any duplicate
|
||||
rows (same store + country + app_id).
|
||||
|
||||
Run: python merge_csv.py
|
||||
python merge_csv.py --out output --pattern "app_data_shard*of*.csv"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Merge sharded scraper CSVs")
|
||||
parser.add_argument("--out", default="output", help="folder holding the shard CSVs")
|
||||
parser.add_argument("--pattern", default="app_data_shard*of*.csv",
|
||||
help="glob for the shard files")
|
||||
parser.add_argument("--dest", default="app_data.csv", help="merged filename")
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = Path(args.out)
|
||||
files = sorted(out_dir.glob(args.pattern))
|
||||
if not files:
|
||||
print(f"No shard files matching '{args.pattern}' in {out_dir}")
|
||||
return
|
||||
|
||||
header = None
|
||||
seen = set()
|
||||
rows = []
|
||||
for f in files:
|
||||
with f.open(encoding="utf-8-sig", newline="") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
header = reader.fieldnames
|
||||
for row in reader:
|
||||
key = (row.get("store"), row.get("country"), row.get("app_id"))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
rows.append(row)
|
||||
print(f" read {f.name}")
|
||||
|
||||
dest = out_dir / args.dest
|
||||
with dest.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=header)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
print(f"\nMerged {len(files)} files -> {dest} ({len(rows)} unique rows)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
"""
|
||||
netutil.py — shared network safety: error classification + an adaptive,
|
||||
thread-safe throttle / circuit breaker used by scraper.py and discover.py.
|
||||
|
||||
Goal: NEVER hammer a store into blocking our IP. Being slow is fine; getting
|
||||
blocked is not. So the moment we see a rate-limit signal (or a run of
|
||||
unexplained errors), every worker pauses for a cooldown that grows on repeat
|
||||
and relaxes again once requests succeed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
|
||||
log = logging.getLogger("net")
|
||||
|
||||
# HTTP statuses that mean "you're going too fast / not allowed" -> back off hard.
|
||||
RATE_LIMIT_CODES = {403, 429}
|
||||
# Temporary server hiccups -> short retry.
|
||||
TRANSIENT_CODES = {408, 500, 502, 503, 504}
|
||||
|
||||
|
||||
def http_status(exc: Exception):
|
||||
"""Best-effort extract an HTTP status code from any exception."""
|
||||
resp = getattr(exc, "response", None)
|
||||
code = getattr(resp, "status_code", None)
|
||||
if isinstance(code, int):
|
||||
return code
|
||||
m = re.search(r"\b(\d{3})\b", str(exc)) # libs often embed the code in the message
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def is_not_found(exc: Exception) -> bool:
|
||||
if http_status(exc) == 404:
|
||||
return True
|
||||
return "notfound" in type(exc).__name__.lower()
|
||||
|
||||
|
||||
def is_rate_limited(exc: Exception) -> bool:
|
||||
if http_status(exc) in RATE_LIMIT_CODES:
|
||||
return True
|
||||
msg = str(exc).lower()
|
||||
return (
|
||||
"429" in msg
|
||||
or "too many requests" in msg
|
||||
or ("rate" in msg and "limit" in msg)
|
||||
or "quota" in msg
|
||||
or "throttl" in msg
|
||||
)
|
||||
|
||||
|
||||
def is_transient(exc: Exception) -> bool:
|
||||
if http_status(exc) in TRANSIENT_CODES:
|
||||
return True
|
||||
name = type(exc).__name__.lower()
|
||||
return any(w in name for w in ("timeout", "connection", "ssl", "chunked"))
|
||||
|
||||
|
||||
class Throttle:
|
||||
"""
|
||||
Adaptive, shared-across-threads throttle + circuit breaker.
|
||||
|
||||
- wait() : call BEFORE each request; blocks while a cooldown is active.
|
||||
- record_success() : call after a good response; relaxes the throttle.
|
||||
- record_fail(rate) : call after a failure; trips the cooldown if it's a
|
||||
rate-limit OR enough consecutive failures pile up.
|
||||
"""
|
||||
|
||||
def __init__(self, cooldown_start: float = 30.0, cooldown_max: float = 300.0,
|
||||
fail_threshold: int = 5):
|
||||
self.lock = threading.Lock()
|
||||
self.cooldown_start = cooldown_start
|
||||
self.cooldown_max = cooldown_max
|
||||
self.fail_threshold = fail_threshold
|
||||
self.current_cooldown = cooldown_start
|
||||
self.pause_until = 0.0 # monotonic time until which everyone waits
|
||||
self.extra_delay = 0.0 # added spacing that lingers after a trip
|
||||
self.consecutive_fail = 0
|
||||
|
||||
def wait(self) -> None:
|
||||
while True:
|
||||
with self.lock:
|
||||
remaining = self.pause_until - time.monotonic()
|
||||
extra = self.extra_delay
|
||||
if remaining > 0:
|
||||
time.sleep(min(remaining, 5.0))
|
||||
continue
|
||||
break
|
||||
if extra:
|
||||
time.sleep(extra)
|
||||
|
||||
def record_success(self) -> None:
|
||||
with self.lock:
|
||||
self.consecutive_fail = 0
|
||||
if self.extra_delay > 0:
|
||||
self.extra_delay = max(0.0, self.extra_delay - 0.25)
|
||||
if self.current_cooldown > self.cooldown_start:
|
||||
self.current_cooldown = max(
|
||||
self.cooldown_start, self.current_cooldown * 0.9
|
||||
)
|
||||
|
||||
def record_fail(self, rate_limited: bool) -> float:
|
||||
"""Return the cooldown (seconds) if the breaker tripped, else 0."""
|
||||
with self.lock:
|
||||
self.consecutive_fail += 1
|
||||
tripped = rate_limited or self.consecutive_fail >= self.fail_threshold
|
||||
if not tripped:
|
||||
return 0.0
|
||||
cd = self.current_cooldown
|
||||
self.pause_until = max(self.pause_until, time.monotonic() + cd)
|
||||
self.current_cooldown = min(self.current_cooldown * 2, self.cooldown_max)
|
||||
self.extra_delay = min(self.extra_delay + 1.0, 15.0)
|
||||
self.consecutive_fail = 0
|
||||
return cd
|
||||
|
||||
|
||||
def retry_call(fn, *args, retries: int = 6, base: float = 2.0,
|
||||
max_wait: float = 120.0, throttle: "Throttle | None" = None):
|
||||
"""
|
||||
Sequential call-with-retry. Waits out rate limits (slow but safe).
|
||||
Re-raises not-found immediately (don't retry a healthy 404) and re-raises
|
||||
after the retry budget is exhausted.
|
||||
"""
|
||||
attempt = 0
|
||||
while True:
|
||||
if throttle:
|
||||
throttle.wait()
|
||||
try:
|
||||
result = fn(*args)
|
||||
if throttle:
|
||||
throttle.record_success()
|
||||
return result
|
||||
except Exception as exc:
|
||||
if is_not_found(exc):
|
||||
if throttle:
|
||||
throttle.record_success()
|
||||
raise
|
||||
rate = is_rate_limited(exc)
|
||||
if throttle:
|
||||
throttle.record_fail(rate)
|
||||
attempt += 1
|
||||
if attempt > retries:
|
||||
raise
|
||||
wait = min(base * (2 ** (attempt - 1)), max_wait)
|
||||
if rate:
|
||||
wait = max(wait, 30.0) # rate limits: wait it out, never rush back
|
||||
log.warning("retry %d/%d in %.0fs (%s)", attempt, retries, wait, exc)
|
||||
time.sleep(wait)
|
||||
@@ -0,0 +1,2 @@
|
||||
google-play-scraper>=1.2.7
|
||||
requests>=2.31.0
|
||||
+325
@@ -0,0 +1,325 @@
|
||||
"""
|
||||
App store scraper — Google Play + Apple App Store.
|
||||
|
||||
For a list of apps (apps.json) it collects, into ONE combined CSV:
|
||||
- core metadata (title, developer, category, price, version, LAST UPDATED date)
|
||||
- the final rating (average) + total number of ratings
|
||||
|
||||
Input : apps.json (Play app IDs + App Store numeric IDs)
|
||||
Output: output/app_data.csv (one row per app x country, both stores together)
|
||||
|
||||
SAFETY (never get IP-blocked):
|
||||
- A shared circuit breaker (netutil.Throttle) pauses EVERY worker the moment a
|
||||
rate-limit (HTTP 429/403) or a run of errors appears, then relaxes on success.
|
||||
- 404/not-found is skipped (no retry); rate limits are waited out; timeouts/5xx
|
||||
get a short backoff.
|
||||
- Every row is flushed to disk immediately, so a crash/block loses nothing.
|
||||
|
||||
RESUME:
|
||||
- On restart it reads the existing CSV and SKIPS apps already scraped, so you
|
||||
just re-run after an interruption.
|
||||
|
||||
SPEED / SCALE:
|
||||
--workers N concurrent requests (network-wait tasks -> ~Nx throughput)
|
||||
--shard i with --shards N, this machine only does jobs i, i+N, i+2N, ...
|
||||
--shards N run shard 0 on PC #1 and shard 1 on PC #2 (each its own IP),
|
||||
then combine with merge_csv.py.
|
||||
|
||||
Run: python scraper.py
|
||||
python scraper.py --workers 12
|
||||
python scraper.py --shard 0 --shards 2 --workers 10 # PC #1
|
||||
python scraper.py --shard 1 --shards 2 --workers 10 # PC #2
|
||||
python merge_csv.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from google_play_scraper import app as gp_app
|
||||
|
||||
from netutil import Throttle, is_not_found, is_rate_limited
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
log = logging.getLogger("scraper")
|
||||
|
||||
ITUNES_LOOKUP = "https://itunes.apple.com/lookup"
|
||||
|
||||
# Per-app retry budgets. Rate limits are waited out generously; transient
|
||||
# network errors get a few quick retries before we give up on that one app.
|
||||
MAX_RATE_LIMIT_WAITS = 12
|
||||
MAX_TRANSIENT_RETRIES = 4
|
||||
|
||||
CSV_FIELDS = [
|
||||
"store", "country", "app_id", "title", "developer", "category",
|
||||
"price", "currency", "free",
|
||||
"avg_rating", "total_ratings", "text_review_count",
|
||||
"last_updated", "version", "url",
|
||||
]
|
||||
|
||||
|
||||
def epoch_to_date(ts):
|
||||
try:
|
||||
return datetime.fromtimestamp(int(ts), tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
except (TypeError, ValueError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
# --- Google Play ------------------------------------------------------------
|
||||
def play_row(app_id: str, lang: str, country: str) -> dict:
|
||||
d = gp_app(app_id, lang=lang, country=country)
|
||||
return {
|
||||
"store": "google_play",
|
||||
"country": country,
|
||||
"app_id": app_id,
|
||||
"title": d.get("title"),
|
||||
"developer": d.get("developer"),
|
||||
"category": d.get("genre"),
|
||||
"price": d.get("price"),
|
||||
"currency": d.get("currency"),
|
||||
"free": d.get("free"),
|
||||
"avg_rating": d.get("score"),
|
||||
"total_ratings": d.get("ratings"),
|
||||
"text_review_count": d.get("reviews"),
|
||||
"last_updated": d.get("lastUpdatedOn") or epoch_to_date(d.get("updated")),
|
||||
"version": d.get("version"),
|
||||
"url": d.get("url"),
|
||||
}
|
||||
|
||||
|
||||
# --- Apple App Store --------------------------------------------------------
|
||||
def appstore_metadata(app_id: int, country: str) -> dict | None:
|
||||
resp = requests.get(
|
||||
ITUNES_LOOKUP, params={"id": app_id, "country": country}, timeout=20
|
||||
)
|
||||
resp.raise_for_status() # turns 429/403/5xx into exceptions we classify
|
||||
results = resp.json().get("results", [])
|
||||
return results[0] if results else None
|
||||
|
||||
|
||||
def appstore_row(app_id: int, country: str) -> dict | None:
|
||||
meta = appstore_metadata(app_id, country)
|
||||
if not meta:
|
||||
return None
|
||||
price = meta.get("price")
|
||||
return {
|
||||
"store": "app_store",
|
||||
"country": country,
|
||||
"app_id": meta.get("trackId"),
|
||||
"title": meta.get("trackName"),
|
||||
"developer": meta.get("sellerName"),
|
||||
"category": meta.get("primaryGenreName"),
|
||||
"price": price,
|
||||
"currency": meta.get("currency"),
|
||||
"free": (price == 0.0) if price is not None else None,
|
||||
"avg_rating": meta.get("averageUserRating"),
|
||||
"total_ratings": meta.get("userRatingCount"),
|
||||
"text_review_count": None,
|
||||
"last_updated": (meta.get("currentVersionReleaseDate") or "")[:10] or None,
|
||||
"version": meta.get("version"),
|
||||
"url": meta.get("trackViewUrl"),
|
||||
}
|
||||
|
||||
|
||||
# --- thread-safe, crash-safe CSV sink --------------------------------------
|
||||
class CsvSink:
|
||||
"""Append rows one at a time, flushing each so progress survives a crash."""
|
||||
|
||||
def __init__(self, path: Path, fields: list):
|
||||
self.fields = fields
|
||||
self.lock = threading.Lock()
|
||||
write_header = not path.exists() or path.stat().st_size == 0
|
||||
self.fh = path.open("a", newline="", encoding="utf-8-sig")
|
||||
self.writer = csv.DictWriter(self.fh, fieldnames=fields, extrasaction="ignore")
|
||||
if write_header:
|
||||
self.writer.writeheader()
|
||||
self.fh.flush()
|
||||
|
||||
def write(self, row: dict) -> None:
|
||||
with self.lock:
|
||||
self.writer.writerow({k: row.get(k) for k in self.fields})
|
||||
self.fh.flush()
|
||||
|
||||
def close(self) -> None:
|
||||
self.fh.close()
|
||||
|
||||
|
||||
# --- orchestration ----------------------------------------------------------
|
||||
def job_key(store: str, app_id, country: str) -> tuple:
|
||||
return (store, country, str(app_id))
|
||||
|
||||
|
||||
def build_jobs(cfg: dict, countries: list) -> list:
|
||||
"""Flat (store, app_id, country) list so sharding balances stores+countries."""
|
||||
jobs = []
|
||||
for country in countries:
|
||||
for entry in cfg.get("google_play", []):
|
||||
jobs.append(("google_play", entry["app_id"], country))
|
||||
for entry in cfg.get("app_store", []):
|
||||
jobs.append(("app_store", entry["app_id"], country))
|
||||
return jobs
|
||||
|
||||
|
||||
def load_done(path: Path) -> set:
|
||||
"""Keys already present in the output CSV (for resume)."""
|
||||
done = set()
|
||||
if path.exists() and path.stat().st_size > 0:
|
||||
with path.open(encoding="utf-8-sig", newline="") as fh:
|
||||
for row in csv.DictReader(fh):
|
||||
done.add((row.get("store"), row.get("country"), str(row.get("app_id"))))
|
||||
return done
|
||||
|
||||
|
||||
def fetch_one(job, lang: str, delay: float, throttle: Throttle):
|
||||
"""
|
||||
Scrape one app. Returns a row dict, or None if the app has no data (404).
|
||||
Raises only after exhausting retries. Rate limits trip the shared breaker
|
||||
so ALL workers slow down together.
|
||||
"""
|
||||
store, app_id, country = job
|
||||
rate_waits = 0
|
||||
transient = 0
|
||||
while True:
|
||||
throttle.wait() # honor any global cooldown before requesting
|
||||
if delay:
|
||||
time.sleep(delay) # steady per-thread pacing
|
||||
try:
|
||||
if store == "google_play":
|
||||
row = play_row(app_id, lang, country)
|
||||
else:
|
||||
row = appstore_row(app_id, country)
|
||||
throttle.record_success()
|
||||
return row
|
||||
except Exception as exc:
|
||||
if is_not_found(exc):
|
||||
throttle.record_success() # a 404 is a healthy answer
|
||||
return None
|
||||
rate = is_rate_limited(exc)
|
||||
cd = throttle.record_fail(rate)
|
||||
if cd:
|
||||
log.warning("THROTTLING: pausing all workers ~%.0fs (%s)",
|
||||
cd, "rate-limit" if rate else "error burst")
|
||||
if rate:
|
||||
rate_waits += 1
|
||||
if rate_waits > MAX_RATE_LIMIT_WAITS:
|
||||
raise RuntimeError(f"persistent rate-limit on {app_id}") from exc
|
||||
continue # loop; throttle.wait() enforces the cooldown
|
||||
transient += 1
|
||||
if transient > MAX_TRANSIENT_RETRIES:
|
||||
raise
|
||||
time.sleep(min(2 ** transient, 20))
|
||||
|
||||
|
||||
def run(config_path: Path, out_dir: Path, countries_override: list | None,
|
||||
workers: int, shard: int, shards: int, resume: bool) -> None:
|
||||
if not config_path.exists():
|
||||
log.error("Config %s not found — run discover.py first to create it.",
|
||||
config_path)
|
||||
return
|
||||
cfg = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
s = cfg.get("settings", {})
|
||||
lang = s.get("lang", "en")
|
||||
delay = float(s.get("delay_seconds", 1.0))
|
||||
countries = countries_override or s.get("countries") or [s.get("country", "us")]
|
||||
|
||||
jobs = build_jobs(cfg, countries)
|
||||
if shards > 1:
|
||||
jobs = jobs[shard::shards]
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
name = "app_data.csv" if shards == 1 else f"app_data_shard{shard}of{shards}.csv"
|
||||
out_path = out_dir / name
|
||||
|
||||
skipped = 0
|
||||
if resume:
|
||||
done = load_done(out_path)
|
||||
before = len(jobs)
|
||||
jobs = [j for j in jobs if job_key(*j) not in done]
|
||||
skipped = before - len(jobs)
|
||||
|
||||
total = len(jobs)
|
||||
throttle = Throttle()
|
||||
sink = CsvSink(out_path, CSV_FIELDS)
|
||||
|
||||
log.info(
|
||||
"shard %d/%d | %d jobs (%d already done, skipped) | %d workers | %s",
|
||||
shard, shards, total, skipped, workers, ", ".join(countries),
|
||||
)
|
||||
if total == 0:
|
||||
log.info("Nothing to do. -> %s", out_path)
|
||||
sink.close()
|
||||
return
|
||||
|
||||
stats = {"ok": 0, "empty": 0, "failed": 0}
|
||||
done_n = 0
|
||||
executor = ThreadPoolExecutor(max_workers=workers)
|
||||
try:
|
||||
futures = {
|
||||
executor.submit(fetch_one, job, lang, delay, throttle): job
|
||||
for job in jobs
|
||||
}
|
||||
for fut in as_completed(futures):
|
||||
job = futures[fut]
|
||||
try:
|
||||
row = fut.result()
|
||||
if row:
|
||||
sink.write(row)
|
||||
stats["ok"] += 1
|
||||
else:
|
||||
stats["empty"] += 1
|
||||
except Exception as exc:
|
||||
stats["failed"] += 1
|
||||
log.error("FAILED %s -> %s", job, exc)
|
||||
done_n += 1
|
||||
if done_n % 50 == 0 or done_n == total:
|
||||
log.info("progress %d/%d (ok=%d empty=%d failed=%d)",
|
||||
done_n, total, stats["ok"], stats["empty"], stats["failed"])
|
||||
except KeyboardInterrupt:
|
||||
log.warning("Interrupted — saving progress and stopping. "
|
||||
"Re-run to resume from where you left off.")
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
finally:
|
||||
sink.close()
|
||||
|
||||
log.info("DONE ok=%d empty=%d failed=%d -> %s",
|
||||
stats["ok"], stats["empty"], stats["failed"], out_path)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Play Store + App Store scraper")
|
||||
parser.add_argument("--config", default="apps.json", help="path to apps.json")
|
||||
parser.add_argument("--out", default="output", help="output directory")
|
||||
parser.add_argument("--countries", default=None,
|
||||
help="comma-separated, e.g. us,gb,in (overrides settings)")
|
||||
parser.add_argument("--workers", type=int, default=10,
|
||||
help="concurrent requests (network-wait, so go high)")
|
||||
parser.add_argument("--shard", type=int, default=0,
|
||||
help="this machine's slice index (0-based)")
|
||||
parser.add_argument("--shards", type=int, default=1,
|
||||
help="total number of machines splitting the work")
|
||||
parser.add_argument("--no-resume", action="store_true",
|
||||
help="ignore existing CSV and scrape everything again")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not 0 <= args.shard < args.shards:
|
||||
parser.error("--shard must be between 0 and --shards-1")
|
||||
|
||||
countries = (
|
||||
[c.strip() for c in args.countries.split(",") if c.strip()]
|
||||
if args.countries else None
|
||||
)
|
||||
run(Path(args.config), Path(args.out), countries,
|
||||
args.workers, args.shard, args.shards, resume=not args.no_resume)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user