dhruv.godvani
03bf0fc923
- discover.py: find app IDs (Play search, iTunes Search API, RSS charts) -> apps.json - scraper.py: fetch metadata + ratings per app/country, threaded, shardable, resumable - netutil.py: shared retry/backoff + adaptive circuit breaker (rate-limit safe) - merge_csv.py: combine sharded outputs from multiple machines - ARCHITECTURE.html: full architecture & flow documentation - english_words.txt: deep search-term wordlist Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
57 lines
1.8 KiB
Python
57 lines
1.8 KiB
Python
"""
|
|
merge_csv.py — combine sharded scraper outputs into one app_data.csv.
|
|
|
|
After PC #1 (shard 0) and PC #2 (shard 1) finish, copy both shard CSVs into the
|
|
same output folder and run this. It concatenates them and drops any duplicate
|
|
rows (same store + country + app_id).
|
|
|
|
Run: python merge_csv.py
|
|
python merge_csv.py --out output --pattern "app_data_shard*of*.csv"
|
|
"""
|
|
|
|
import argparse
|
|
import csv
|
|
from pathlib import Path
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Merge sharded scraper CSVs")
|
|
parser.add_argument("--out", default="output", help="folder holding the shard CSVs")
|
|
parser.add_argument("--pattern", default="app_data_shard*of*.csv",
|
|
help="glob for the shard files")
|
|
parser.add_argument("--dest", default="app_data.csv", help="merged filename")
|
|
args = parser.parse_args()
|
|
|
|
out_dir = Path(args.out)
|
|
files = sorted(out_dir.glob(args.pattern))
|
|
if not files:
|
|
print(f"No shard files matching '{args.pattern}' in {out_dir}")
|
|
return
|
|
|
|
header = None
|
|
seen = set()
|
|
rows = []
|
|
for f in files:
|
|
with f.open(encoding="utf-8-sig", newline="") as fh:
|
|
reader = csv.DictReader(fh)
|
|
header = reader.fieldnames
|
|
for row in reader:
|
|
key = (row.get("store"), row.get("country"), row.get("app_id"))
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
rows.append(row)
|
|
print(f" read {f.name}")
|
|
|
|
dest = out_dir / args.dest
|
|
with dest.open("w", encoding="utf-8-sig", newline="") as fh:
|
|
writer = csv.DictWriter(fh, fieldnames=header)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
print(f"\nMerged {len(files)} files -> {dest} ({len(rows)} unique rows)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|