Podcast verification results

Loading…

First pass (original)

\n Verified (keep)\n Review me\n Not found\n

Second pass — recheck of the \"not found\" list (sorted by match score)

👆 Start here: review the highest scores first, then stop when they get too low to bother. Every row has a pod.link to click straight to the show.

\n Recheck verified (0.80+)\n Recheck review (0.35–0.79)\n Still not found (<0.35)\n All recheck results\n

FINAL LIST (merged verified + uncertain + your finds, enriched)

\n Download final list (CSV)\n

Everything

\n All results (CSV)\n\n`;\n\ntype Row = { queried_name?: unknown; status?: unknown; score?: unknown; record?: unknown };\nfunction recOf(r: Row): any {\n try { return JSON.parse((r.record as string) || \"{}\"); } catch { return {}; }\n}\nfunction rowToOut(r: Row) {\n const rec = recOf(r);\n const score = r.score === null || r.score === undefined ? \"\" : Math.round((r.score as number) * 100);\n return {\n queried_name: r.queried_name ?? \"\",\n status: r.status ?? \"\",\n score,\n matched_name: rec.name || \"\",\n itunes_id: rec.id ?? \"\",\n publisher: rec.pub || \"\",\n podlink: rec.link || \"\",\n };\n}\n\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n const download = url.searchParams.get(\"download\");\n const wantSummary = url.searchParams.has(\"summary\");\n\n const vr = await sqlite.execute(\"SELECT queried_name, status, score, record FROM verify_results\");\n const mainRows = vr.rows as Row[];\n const rr = await sqlite.execute(\"SELECT queried_name, status, score, record FROM recheck_results\");\n const recheckRows = rr.rows as Row[];\n\n if (wantSummary) {\n const c: any = { verified: 0, uncertain: 0, not_found: 0, duplicate: 0, total: mainRows.length, rch_verified: 0, rch_uncertain: 0, rch_notfound: 0, rch_total: recheckRows.length };\n for (const r of mainRows) { if ((r.status as string) in c) c[r.status as string]++; }\n for (const r of recheckRows) {\n if (r.status === \"verified\") c.rch_verified++;\n else if (r.status === \"uncertain\") c.rch_uncertain++;\n else if (r.status === \"not_found\") c.rch_notfound++;\n }\n return new Response(JSON.stringify(c), { headers: { \"Content-Type\": \"application/json\" } });\n }\n\n if (!download) return new Response(PAGE, { headers: { \"Content-Type\": \"text/html; charset=utf-8\" } });\n\n const fields = [\"queried_name\", \"status\", \"score\", \"matched_name\", \"itunes_id\", \"publisher\", \"podlink\"];\n let csvRows: any[] = [];\n\n if (download === \"recheck_verified\" || download === \"recheck_uncertain\" ||\n download === \"recheck_notfound\" || download === \"recheck_all\") {\n const st = download.replace(\"recheck_\", \"\");\n let rows = recheckRows;\n if (st === \"verified\") rows = recheckRows.filter((r) => r.status === \"verified\");\n else if (st === \"uncertain\") rows = recheckRows.filter((r) => r.status === \"uncertain\");\n else if (st === \"notfound\") rows = recheckRows.filter((r) => r.status === \"not_found\");\n // sort by score descending so you review most-likely-real first\n rows = [...rows].sort((a, b) => {\n const sa = (a.score as number) || 0, sb = (b.score as number) || 0;\n return sb - sa;\n });\n csvRows = rows.map(rowToOut);\n const fname = st === \"notfound\" ? \"recheck_notfound\" : `recheck_${st}`;\n return new Response(toCsv(csvRows, fields), {\n headers: { \"Content-Type\": \"text/csv\", \"Content-Disposition\": `attachment; filename=\"${fname}.csv\"` },\n });\n }\n\n // FINAL merged list\n if (download === \"final\") {\n const fr = await sqlite.execute(\n \"SELECT id, name, source, pub, genres, primary_genre, eps, release_date, feed, art, link FROM final_list ORDER BY name\",\n );\n const ff = [\"id\", \"name\", \"source\", \"pub\", \"genres\", \"primary_genre\", \"eps\", \"release_date\", \"feed\", \"art\", \"link\"];\n const out = (fr.rows as any[]).map((r) => ({\n id: r.id ?? \"\", name: r.name ?? \"\", source: r.source ?? \"\", pub: r.pub ?? \"\",\n genres: r.genres ?? \"\", primary_genre: r.primary_genre ?? \"\", eps: r.eps ?? \"\",\n release_date: r.release_date ?? \"\", feed: r.feed ?? \"\", art: r.art ?? \"\", link: r.link ?? \"\",\n }));\n return new Response(toCsv(out, ff), {\n headers: { \"Content-Type\": \"text/csv\", \"Content-Disposition\": 'attachment; filename=\"final_list.csv\"' },\n });\n }\n\n // original buckets\n let rows = mainRows;\n if (download !== \"all\") rows = mainRows.filter((r) => r.status === download);\n csvRows = rows.map(rowToOut);\n const fname = (download === \"all\" ? \"all\" : download) || \"all\";\n return new Response(toCsv(csvRows, fields), {\n headers: { \"Content-Type\": \"text/csv\", \"Content-Disposition\": `attachment; filename=\"podcast_${fname}.csv\"` },\n });\n}\n",4,["D",1788743069114],"branchVersion","label",[18],"pulls","branches",1,"forks","envVars",{"_197":26,"_199":26},"connections",{"_197":26,"_198":26},"count","unfilledCount","emptyCount",{"_36":181,"_13":18,"_67":68,"_46":-5,"_38":39,"_186":103,"_211":121,"_40":260,"_42":184,"_69":82,"_182":183,"_71":24,"_187":-5,"_216":261,"_17":262},{"_36":252,"_13":253,"_67":254,"_46":-5,"_38":39,"_186":255,"_211":121,"_40":256,"_42":26,"_69":82,"_182":257,"_71":24,"_187":-5,"_216":258,"_17":259},{"_36":244,"_13":245,"_67":246,"_46":-5,"_38":39,"_186":247,"_211":121,"_40":248,"_42":232,"_69":82,"_182":249,"_71":24,"_187":-5,"_216":250,"_17":251},{"_36":236,"_13":237,"_67":119,"_46":-5,"_38":39,"_186":238,"_211":121,"_40":239,"_42":240,"_69":70,"_182":241,"_71":24,"_187":-5,"_216":242,"_17":243},{"_36":227,"_13":228,"_67":229,"_46":-5,"_38":39,"_186":230,"_211":121,"_40":231,"_42":232,"_69":70,"_182":233,"_71":24,"_187":-5,"_216":234,"_17":235},{"_36":219,"_13":220,"_67":221,"_46":-5,"_38":39,"_186":222,"_211":121,"_40":223,"_42":184,"_69":70,"_182":224,"_71":24,"_187":-5,"_216":225,"_17":226},{"_36":207,"_13":208,"_67":209,"_46":-5,"_38":39,"_186":210,"_211":121,"_40":212,"_42":213,"_69":214,"_182":215,"_71":24,"_187":-5,"_216":217,"_17":218},"6573510b-e5cd-45bf-82e8-ab98982fe58e","worker.ts","af83bf74-a87a-11f1-b8c8-1607ee4eb77e",39,"latest",["D",1788743060779],3,"interval","// Scheduled worker: now enriches rows in the merged final_list table over\n// time (Apple metadata lookup), one small rate-limit-safe chunk per tick.\nimport { mergeBatch } from \"https://esm.town/v/rarefrog5/longlist/merge.ts\";\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nexport default async function () {\n try {\n await mergeBatch();\n } catch (e) {\n console.error(\"worker merge/enrich failed:\", e instanceof Error ? e : e);\n }\n const rows = await sqlite.execute(\"SELECT count(*) AS n FROM final_list WHERE enriched=1\");\n console.log(\"final_list enriched so far:\", rows.rows[0]?.n ?? 0);\n}\n","children",[],[208],"07fb3e66-7484-4b95-9306-9849cf5713e6","recheck.ts","2b40ae70-a951-11f1-b735-1607ee4eb77e",32,["D",1788630375713],"// ============================================================================\n// SECOND-PASS RE-CHECKER for names previously marked \"not_found\".\n// ----------------------------------------------------------------------------\n// The first pass was too strict: it dropped thousands of names that are real\n// podcasts under slightly-off titles. This pass re-checks every \"not_found\"\n// name with multiple aggressive search strategies and a much lower match bar,\n// so real shows get many chances to surface. Results are stored with a match\n// score and pod.link so you can review the most-likely-real first.\n//\n// Statuses produced:\n// verified -> confident real match (score >= 0.8)\n// uncertain -> plausible real match (score >= 0.35) — REVIEW, sorted by score\n// not_found -> no plausible match across all strategies (still visible)\n//\n// RESUMABLE + rate-limit-safe, same as the first pass.\n// ============================================================================\n\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\n// Free-tier executions cap at ~1 minute. The recheck does up to 4 Apple\n// searches per name, so a moderate batch is needed to finish in time.\nconst BATCH = 20;\nconst THROTTLE_MS = 600;\nconst COUNTRY = \"US\";\n\nconst CONFIDENT = 0.8;\nconst PLAUSIBLE = 0.35;\n\nconst FILLER = new Set([\n \"podcast\", \"the\", \"show\", \"radio\", \"with\", \"a\", \"an\", \"official\", \"audio\",\n \"pod\", \"network\", \"series\", \"miniseries\", \"dispatch\", \"file\", \"inquest\",\n \"archive\", \"wire\", \"review\", \"report\", \"watch\", \"voice\", \"front\",\n \"expose\", \"today\", \"studies\", \"theory\", \"reader\", \"seminar\", \"workshop\",\n]);\n\nfunction norm(s: string): string {\n return (s || \"\").toLowerCase().replace(/[^a-z0-9]/g, \"\");\n}\nfunction core(s: string): string {\n return (s || \"\")\n .toLowerCase()\n .replace(/[^a-z0-9\\s]/g, \" \")\n .split(/\\s+/)\n .filter((w) => w && !FILLER.has(w))\n .join(\"\");\n}\nfunction tokens(s: string): string[] {\n return (s || \"\").toLowerCase().replace(/[^a-z0-9\\s]/g, \" \").split(/\\s+/)\n .filter(Boolean);\n}\nfunction titleSim(a: string, b: string): number {\n const na = norm(a), nb = norm(b);\n if (!na || !nb) return 0;\n if (na === nb) return 1;\n const ca = core(a), cb = core(b);\n if (ca && cb && ca === cb) return 1;\n let contain = 0;\n if (ca && cb && (ca.includes(cb) || cb.includes(ca))) {\n const ratio = Math.min(ca.length, cb.length) / Math.max(ca.length, cb.length);\n contain = ratio >= 0.7 ? 0.9 : ratio * 0.5;\n }\n const ta = new Set(tokens(a).filter((w) => !FILLER.has(w)));\n const tb = new Set(tokens(b).filter((w) => !FILLER.has(w)));\n let inter = 0;\n for (const t of ta) if (tb.has(t)) inter++;\n const union = new Set([...ta, ...tb]).size || 1;\n const jac = inter / union;\n return Math.max(contain, jac);\n}\n\nasync function itunesSearch(term: string, limit = 10): Promise {\n const url = \"https://itunes.apple.com/search?\" +\n new URLSearchParams({\n term,\n media: \"podcast\",\n entity: \"podcast\",\n limit: String(limit),\n country: COUNTRY,\n }).toString();\n try {\n const r = await fetch(url, { headers: { \"User-Agent\": \"FF-Directory-Verify/1.0\" } });\n if (!r.ok) return [];\n return (await r.json()).results || [];\n } catch (_e) {\n return [];\n }\n}\n\n// Build several query variants so a real show gets many chances to surface.\nfunction queryVariants(q: string): string[] {\n const out = new Set();\n const trimmed = q.replace(/\\b(podcast|audio|pod|show|radio|series|the|network|today|with|a|an)\\b/ig, \"\")\n .replace(/\\s+/g, \" \").trim();\n if (trimmed && trimmed.length >= 2) out.add(trimmed);\n const toks = tokens(q).filter((w) => !FILLER.has(w));\n if (toks.length >= 2) out.add(toks.slice(0, 2).join(\" \"));\n if (toks.length === 1) out.add(toks[0]);\n const c = core(q);\n if (c && c.length >= 4) out.add(c);\n return Array.from(out);\n}\n\nasync function verifyOne(query: string) {\n const variants = queryVariants(query);\n let best: any = null, bestScore = 0;\n\n for (const v of variants) {\n const results = await itunesSearch(v);\n for (const res of results) {\n const cand = res.collectionName || res.trackName || \"\";\n const s = titleSim(query, cand);\n if (s > bestScore) { bestScore = s; best = res; }\n }\n }\n\n if (!best) return { status: \"not_found\", record: null, score: 0 };\n\n const rec = {\n id: best.collectionId || best.trackId || \"\",\n name: best.collectionName || best.trackName || \"\",\n pub: best.artistName || \"\",\n link: (best.collectionId || best.trackId) ? `https://pod.link/${best.collectionId || best.trackId}` : (best.collectionViewUrl || \"\"),\n score: Math.round(bestScore * 100) / 100,\n };\n\n if (bestScore >= CONFIDENT) return { status: \"verified\", record: rec, score: bestScore };\n if (bestScore >= PLAUSIBLE) return { status: \"uncertain\", record: rec, score: bestScore };\n return { status: \"not_found\", record: null, score: bestScore };\n}\n\nexport async function recheckBatch(batchSize = BATCH): Promise {\n await sqlite.execute(`CREATE TABLE IF NOT EXISTS recheck_results (\n queried_name TEXT PRIMARY KEY,\n status TEXT NOT NULL,\n record TEXT,\n score REAL\n )`);\n\n const nf = await sqlite.execute(\"SELECT queried_name FROM verify_results WHERE status='not_found'\");\n const nfNames = nf.rows.map((r) => r.queried_name as string);\n\n const doneRows = await sqlite.execute(\"SELECT queried_name FROM recheck_results\");\n const done = new Set(doneRows.rows.map((r) => r.queried_name as string));\n const pending = nfNames.filter((n) => !done.has(n));\n\n console.log(`Not-found names to recheck: ${nfNames.length}`);\n console.log(`Already rechecked: ${done.size}`);\n console.log(`Pending: ${pending.length}`);\n\n if (pending.length === 0) { console.log(\"\\n=== RECHECK ALL DONE ===\"); return; }\n\n const todo = pending.slice(0, batchSize);\n const counts = { verified: 0, uncertain: 0, not_found: 0 };\n\n for (let i = 0; i < todo.length; i++) {\n const query = todo[i];\n const { status, record, score } = await verifyOne(query);\n counts[status as keyof typeof counts]++;\n await sqlite.execute({\n sql: \"INSERT OR REPLACE INTO recheck_results (queried_name, status, record, score) VALUES (?, ?, ?, ?)\",\n args: [query, status, record ? JSON.stringify(record) : null, score ?? null],\n });\n await new Promise((r) => setTimeout(r, THROTTLE_MS));\n }\n\n console.log(`\\nThis run rechecked ${todo.length}.`);\n console.log(` verified: ${counts.verified}`);\n console.log(` uncertain: ${counts.uncertain} (review, sorted by score)`);\n console.log(` still not_found: ${counts.not_found}`);\n console.log(`Progress: ${done.size + todo.length}/${nfNames.length} rechecked.`);\n}\n\n// NOTE: recheckBatch is exported for the worker to call. No top-level auto-run\n// here — the scheduled worker (worker.ts) drives one batch per tick, and direct\n// runs can call recheckBatch() themselves.\n",[],[220],"2273f84e-6d79-4ad1-a98c-4d5084716e21","merge.ts","c612ef1a-aa57-11f1-b195-1607ee4eb77e",38,["D",1788742963599],2,"// ============================================================================\n// MERGE + DEDUPE + ENRICH (resumable)\n// ----------------------------------------------------------------------------\n// Step 1 (merge): combine every show the user considers real into one deduped\n// table `final_list`. Sources:\n// - first pass verified + uncertain (verify_results)\n// - second pass verified + uncertain (recheck_results)\n// - user's found shows (blob \"found_names\")\n// Dedupe by Apple id, falling back to normalized name. Durable in SQLite.\n// Step 2 (enrich): fill each row's metadata via Apple lookup by id (or search\n// by name). Runs in small chunks so it fits the free-tier 1-min limit; the\n// worker drives more chunks over time. Rows already enriched are skipped.\n// ============================================================================\nimport { blob } from \"https://esm.town/v/std/blob/main.ts\";\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nconst COUNTRY = \"US\";\nconst THROTTLE_MS = 400;\nconst ENRICH_CHUNK = 20; // Apple lookups per run (stays under 1-min limit)\n\nfunction norm(s: string): string {\n return (s || \"\").toLowerCase().replace(/[^a-z0-9]/g, \"\");\n}\n\nasync function setup(): Promise {\n await sqlite.execute(`CREATE TABLE IF NOT EXISTS final_list (\n dedupe_key TEXT PRIMARY KEY,\n id TEXT,\n name TEXT,\n source TEXT,\n enriched INTEGER DEFAULT 0,\n pub TEXT, genres TEXT, primary_genre TEXT, eps TEXT,\n release_date TEXT, feed TEXT, art TEXT, link TEXT\n )`);\n}\n\nasync function importIntoFinal(): Promise {\n await setup();\n // populate a key->(id,name,source) set\n const rows: { key: string; id: string; name: string; source: string }[] = [];\n const seen = new Set();\n\n for (const [tbl, from] of [[\"verify_results\", \"first_pass\"], [\"recheck_results\", \"recheck_pass\"]] as const) {\n const r = await sqlite.execute(`SELECT queried_name, status, record FROM ${tbl}`);\n for (const row of r.rows) {\n const st = row.status as string;\n if (st !== \"verified\" && st !== \"uncertain\") continue;\n let rec: any = {};\n try { rec = JSON.parse((row.record as string) || \"{}\"); } catch { /* */ }\n const id = String(rec.id ?? \"\");\n const name = (row.queried_name as string) || rec.name || \"\";\n const key = id ? `id:${id}` : `nm:${norm(name)}`;\n if (!seen.has(key)) { seen.add(key); rows.push({ key, id, name, source: from }); }\n }\n }\n try {\n const res = await blob.get(\"found_names\");\n if (res) {\n const found = JSON.parse(await res.text());\n for (const f of found) {\n const name = f.name || \"\";\n const id = String(f.id || \"\");\n const key = id ? `id:${id}` : `nm:${norm(name)}`;\n if (!seen.has(key)) { seen.add(key); rows.push({ key, id, name, source: \"user_found\" }); }\n }\n }\n } catch (_e) { /* no found_names */ }\n\n for (const r of rows) {\n await sqlite.execute({\n sql: \"INSERT OR IGNORE INTO final_list (dedupe_key, id, name, source, enriched) VALUES (?,?,?,?,0)\",\n args: [r.key, r.id, r.name, r.source],\n });\n }\n return rows.length;\n}\n\nasync function itunesLookupId(id: string): Promise {\n try {\n const r = await fetch(`https://itunes.apple.com/lookup?id=${id}&country=${COUNTRY}`, {\n headers: { \"User-Agent\": \"FF-Directory-Verify/1.0\" },\n });\n if (!r.ok) return null;\n const j = await r.json();\n return (j.results && j.results[0]) || null;\n } catch { return null; }\n}\nasync function searchOne(term: string): Promise {\n try {\n const r = await fetch(\"https://itunes.apple.com/search?\" + new URLSearchParams({\n term, media: \"podcast\", entity: \"podcast\", limit: \"5\", country: COUNTRY,\n }).toString(), { headers: { \"User-Agent\": \"FF-Directory-Verify/1.0\" } });\n if (!r.ok) return null;\n const j = await r.json();\n return (j.results && j.results[0]) || null;\n } catch { return null; }\n}\n\nasync function enrichChunk(): Promise {\n const todo = await sqlite.execute(\n \"SELECT dedupe_key, id, name FROM final_list WHERE enriched=0 LIMIT ?\",\n [ENRICH_CHUNK],\n );\n let done = 0;\n for (const row of todo.rows) {\n const key = row.dedupe_key as string;\n const id = row.id as string;\n const name = row.name as string;\n let res: any = null;\n if (id) res = await itunesLookupId(id);\n if (!res && name) res = await searchOne(name);\n await new Promise((r) => setTimeout(r, THROTTLE_MS));\n if (res) {\n const rid = res.collectionId || res.trackId || id;\n await sqlite.execute({\n sql: `UPDATE final_list SET enriched=1, id=?, pub=?, genres=?, primary_genre=?, eps=?,\n release_date=?, feed=?, art=?, link=? WHERE dedupe_key=?`,\n args: [\n String(rid), res.artistName || \"\", (res.genres || []).join(\" | \"),\n res.primaryGenreName || \"\", res.trackCount ?? \"\",\n (res.releaseDate || \"\").slice(0, 10), res.feedUrl || \"\",\n res.artworkUrl600 || res.artworkUrl100 || \"\",\n rid ? `https://pod.link/${rid}` : (res.collectionViewUrl || \"\"),\n key,\n ],\n });\n } else {\n await sqlite.execute({\n sql: \"UPDATE final_list SET enriched=1, link=? WHERE dedupe_key=?\",\n args: [id ? `https://pod.link/${id}` : null, key],\n });\n }\n done++;\n }\n return done;\n}\n\nexport async function mergeBatch(): Promise {\n await setup();\n const imported = await importIntoFinal();\n const total = await sqlite.execute(\"SELECT count(*) AS n FROM final_list\");\n const pending = await sqlite.execute(\"SELECT count(*) AS n FROM final_list WHERE enriched=0\");\n console.log(`Merged/imported this run: ${imported}`);\n console.log(`Total unique in final_list: ${total.rows[0]?.n}`);\n console.log(`Yet to enrich: ${pending.rows[0]?.n}`);\n\n const chunkDone = await enrichChunk();\n const pendingAfter = await sqlite.execute(\"SELECT count(*) AS n FROM final_list WHERE enriched=0\");\n console.log(`Enriched this run: ${chunkDone}`);\n console.log(`Remaining to enrich: ${pendingAfter.rows[0]?.n}`);\n}\n\nexport default async function () {\n await mergeBatch();\n}\n\n// Run when executed directly.\nawait mergeBatch();\n",[],[228],"738f1bd2-b055-4874-8b7f-ffbea650bc9f","main.ts",23,["D",1788538023077],13,"// ============================================================================\n// FORWARD FREQUENCY — PODCAST VERIFICATION SCRIPT (resumable, for Val Town)\n// ----------------------------------------------------------------------------\n// Checks a list of podcast NAMES (stored in this val's blob storage as\n// names_1, via the ingest.ts web page) against the REAL Apple catalog, and\n// keeps only real shows. Runs NOTHING to the \"fake\" bucket without a second,\n// more thorough lookup — because Apple's top-N fuzzy search misses real shows.\n//\n// Each result is one of four statuses:\n// verified -> confidently matched a real Apple show (kept)\n// uncertain -> a plausible near-match existed but wasn't an exact title;\n// kept for HUMAN review so no real show is dropped\n// not_found -> searched generously, no real show found (dropped)\n// (duplicate) -> matched but already in your existing directory\n//\n// RESUMABLE: a single execution has a time limit, so each run:\n// loads names, skips ones already in the DB, verifies BATCH (~150), saves\n// each result, prints totals + how far along. Run repeatedly until ALL DONE.\n// ============================================================================\n\nimport { blob } from \"https://esm.town/v/std/blob/main.ts\";\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\n// How many names to try per run. A run must finish under the execution time\n// limit. Apple rate-limits rapidly-fired requests (we got 403s on a burst),\n// so the batch size and delay below are tuned to stay well clear of that:\n// BATCH requests, each preceded by a THROTTLE_MS delay, and genuinely short.\nconst BATCH = 50;\nconst THROTTLE_MS = 600; // real politeness delay between Apple calls\nconst COUNTRY = \"US\";\n\n// Title-similarity thresholds:\n// >= CONFIDENT : treat as a verified real match\n// >= PLAUSIBLE : a real show might exist under a slightly different title\n// -> mark \"uncertain\" (human review) instead of \"not_found\"\nconst CONFIDENT = 0.8;\n// Real shows are frequently listed under a slightly different title on Apple\n// (e.g. \"Abolition Science\" vs \"Abolition Science Radio\"), and Apple's fuzzy\n// search often surfaces them imperfectly. So anything with at least a partial\n// word overlap is flagged for human review rather than silently dropped. The\n// PLAUSIBLE bar is deliberately low so real shows aren't thrown away.\nconst PLAUSIBLE = 0.35;\n\n// ---- optional dedupe against existing shows (leave empty to skip) ----------\nconst EXISTING: { ids: number[]; names: string[] } = { ids: [], names: [] };\nconst existingIds = new Set(EXISTING.ids);\nconst existingNames = new Set(EXISTING.names);\n\n// ---- text + matching helpers ----------------------------------------------\nfunction norm(s: string): string {\n return (s || \"\").toLowerCase().replace(/[^a-z0-9]/g, \"\");\n}\nconst FILLER = new Set([\n \"podcast\", \"the\", \"show\", \"radio\", \"with\", \"a\", \"an\", \"official\", \"audio\",\n \"pod\", \"network\", \"series\", \"miniseries\", \"dispatch\", \"file\", \"inquest\",\n \"archive\", \"wire\", \"review\", \"report\", \"watch\", \"voice\", \"front\",\n \"expose\", \"today\", \"studies\", \"theory\", \"reader\", \"seminar\", \"workshop\",\n]);\nfunction core(s: string): string {\n return (s || \"\")\n .toLowerCase()\n .replace(/[^a-z0-9\\s]/g, \" \")\n .split(/\\s+/)\n .filter((w) => w && !FILLER.has(w))\n .join(\"\");\n}\nfunction tokens(s: string): string[] {\n return (s || \"\").toLowerCase().replace(/[^a-z0-9\\s]/g, \" \").split(/\\s+/)\n .filter(Boolean);\n}\n// Similarity score 0..1. Handles filler words, minor punctuation/typos.\nfunction titleSim(a: string, b: string): number {\n const na = norm(a), nb = norm(b);\n if (!na || !nb) return 0;\n if (na === nb) return 1;\n const ca = core(a), cb = core(b);\n if (ca && cb && ca === cb) return 1;\n let contain = 0;\n if (ca && cb && (ca.includes(cb) || cb.includes(ca))) {\n const ratio = Math.min(ca.length, cb.length) / Math.max(ca.length, cb.length);\n contain = ratio >= 0.7 ? 0.9 : ratio * 0.5;\n }\n const ta = new Set(tokens(a).filter((w) => !FILLER.has(w)));\n const tb = new Set(tokens(b).filter((w) => !FILLER.has(w)));\n let inter = 0;\n for (const t of ta) if (tb.has(t)) inter++;\n const union = new Set([...ta, ...tb]).size || 1;\n const jac = inter / union;\n return Math.max(contain, jac);\n}\n\n// ---- Apple search (generous: longer result list) --------------------------\nasync function itunesSearch(term: string, limit = 10): Promise {\n const url = \"https://itunes.apple.com/search?\" +\n new URLSearchParams({\n term,\n media: \"podcast\",\n entity: \"podcast\",\n limit: String(limit),\n country: COUNTRY,\n }).toString();\n try {\n const r = await fetch(url, { headers: { \"User-Agent\": \"FF-Directory-Verify/1.0\" } });\n if (!r.ok) return [];\n return (await r.json()).results || [];\n } catch (_e) {\n return [];\n }\n}\n\n// ---- classification (mirrors original directory's logic) ------------------\nconst FOCUS_KEYWORDS: [string, string[]][] = [\n [\"democracy\", [\"democracy\", \"democratic\", \"voting rights\", \"election integrity\", \"authoritarian\", \"fascism\", \"civic\", \"constitution\"]],\n [\"racial_justice\", [\"racial justice\", \"race\", \"black\", \"blm\", \"civil rights\", \"anti-racis\", \"racism\"]],\n [\"climate\", [\"climate\", \"environment\", \"warming\", \"emissions\", \"sustainab\", \"green energy\", \"fossil\"]],\n [\"environmental_justice\", [\"environmental justice\", \"pollution\", \"frontline communit\", \"toxic\"]],\n [\"labor\", [\"labor\", \"union\", \"worker\", \"strike\", \"wages\", \"gig work\", \"organizing\"]],\n [\"immigration\", [\"immigra\", \"migrant\", \"border\", \"asylum\", \"refugee\", \"ice \", \"deportation\"]],\n [\"housing\", [\"housing\", \"rent\", \"tenant\", \"eviction\", \"homeless\", \"affordable housing\"]],\n [\"reproductive\", [\"reproductive\", \"abortion\", \"roe\", \"pro-choice\", \"reproductive rights\"]],\n [\"lgbtq_rights\", [\"lgbtq\", \"queer\", \"trans \", \"transgender\", \"gay \", \"pride\", \"gender identity\"]],\n [\"disability_rights\", [\"disability\", \"disabled\", \"ableism\", \"accessib\", \"neurodiver\"]],\n [\"criminal_justice\", [\"criminal justice\", \"prison\", \"incarcerat\", \"police\", \"carceral\", \"abolition\", \"mass incarceration\"]],\n [\"health\", [\"health care\", \"healthcare\", \"public health\", \"medicaid\", \"medicare\", \"mental health\"]],\n [\"indigenous\", [\"indigenous\", \"native american\", \"tribal\", \"first nations\", \"sovereignty\"]],\n [\"socialism\", [\"socialis\", \"marxis\", \"communis\", \"anti-capitalis\", \"leftist theory\", \"class struggle\"]],\n [\"feminism\", [\"feminis\", \"women's rights\", \"gender equality\", \"patriarchy\"]],\n [\"education\", [\"education\", \"public school\", \"teacher\", \"student debt\", \"curriculum\"]],\n];\nconst FN_KEYWORDS: [string, string[]][] = [\n [\"local_journalism\", [\"local news\", \"city council\", \"statehouse\", \"municipal\", \"county\", \"regional\", \"community journalism\", \"public radio\", \"local report\"]],\n [\"electoral\", [\"election\", \"campaign\", \"candidate\", \"ballot\", \"voter\", \"primary\", \"electoral\", \"politics\", \"political analysis\", \"congress\", \"senate\"]],\n [\"democratic_institutions\", [\"democracy\", \"institution\", \"court\", \"supreme court\", \"constitution\", \"governance\", \"policy analysis\", \"rule of law\"]],\n [\"issue_advocacy\", [\"advocacy\", \"activis\", \"rights\", \"justice\", \"reform\", \"campaign for\", \"fight for\", \"movement for\"]],\n [\"community_movement\", [\"community\", \"movement\", \"grassroots\", \"organizing\", \"mutual aid\", \"solidarity\", \"collective\"]],\n];\nfunction classify(name: string, desc: string, eps: number, genres: string[]) {\n const text = (name + \" \" + desc + \" \" + (genres || []).join(\" \")).toLowerCase();\n let focus = \"\";\n for (const [f, kws] of FOCUS_KEYWORDS) {\n if (kws.some((k) => text.includes(k))) { focus = f; break; }\n }\n let fn = \"other\";\n for (const [f, kws] of FN_KEYWORDS) {\n if (kws.some((k) => text.includes(k))) { fn = f; break; }\n }\n const corePol = [\"politics\", \"election\", \"campaign\", \"congress\", \"senate\", \"policy\", \"government\", \"democrat\", \"republican\", \"left\", \"progressive\", \"conservative\"];\n const tier = corePol.some((k) => text.includes(k)) ? \"core_political\" : \"civic_adjacent\";\n const reach = eps >= 200 ? \"established\" : eps >= 50 ? \"growing\" : \"emerging\";\n const invest = reach === \"emerging\" ? \"high\" : reach === \"growing\" ? \"medium\" : \"low\";\n return { tier, fn, focus, reach, invest };\n}\nfunction toRecord(res: any) {\n const id = res.collectionId || res.trackId;\n const eps = res.trackCount || 0;\n const genres: string[] = res.genres || [];\n const name = res.collectionName || res.trackName || \"\";\n const cls = classify(name, \"\", eps, genres);\n return {\n id, name, pub: res.artistName || \"\", desc: \"\",\n tier: cls.tier, fn: cls.fn, focus: cls.focus, eps,\n yrs: 0, active: true, reach: cls.reach, invest: cls.invest,\n art: res.artworkUrl600 || res.artworkUrl100 || \"\",\n link: id ? `https://pod.link/${id}` : (res.collectionViewUrl || \"\"),\n rec: [], source: \"verified_apple\",\n feed: res.feedUrl || \"\", genres,\n score: 0,\n };\n}\n\n// ---- name loading ----------------------------------------------------------\nasync function parseNamesTxt(txt: string): Promise {\n try {\n const j = JSON.parse(txt);\n if (Array.isArray(j)) return j.map(String);\n } catch (_e) { /* not JSON */ }\n return txt.split(/\\r?\\n/).map((s) => s.trim()).filter(Boolean);\n}\nasync function loadAllNames(): Promise {\n const parts: string[] = [];\n for (let k = 1; k <= 50; k++) {\n try {\n const res = await blob.get(`names_${k}`);\n if (!res) break;\n parts.push(...(await parseNamesTxt(await res.text())));\n } catch (_e) { break; }\n }\n return parts;\n}\n\n// ---- verify ONE name, generously. Never drops on a single flaky search. ---\n// One search per name (Apple rate-limits bursts, so we keep volume down).\n// We search the filler-trimmed query (e.g. \"5-4 Podcast\" -> \"5-4\") which best\n// matches a real Apple title.\nasync function itunesSearchTrimmed(query: string): Promise {\n const trimmed = query\n .replace(/\\b(podcast|audio|pod|show|radio|series|the|network|today)\\b/ig, \"\")\n .replace(/\\s+/g, \" \").trim();\n return itunesSearch(trimmed && trimmed.length >= 2 ? trimmed : query);\n}\nasync function verifyOne(query: string) {\n const all = await itunesSearchTrimmed(query);\n\n let best: any = null, bestScore = 0;\n for (const res of all) {\n const cand = res.collectionName || res.trackName || \"\";\n const s = titleSim(query, cand);\n if (s > bestScore) { bestScore = s; best = res; }\n }\n\n if (!best) return { status: \"not_found\", record: null, score: 0 };\n\n if (bestScore >= CONFIDENT) {\n const rec = toRecord(best);\n rec.score = Math.round(bestScore * 100) / 100;\n // dedupe\n if (existingIds.has(rec.id) || existingNames.has(norm(rec.name))) {\n return { status: \"duplicate\", record: rec, score: bestScore };\n }\n return { status: \"verified\", record: rec, score: bestScore };\n }\n\n if (bestScore >= PLAUSIBLE) {\n // A real show may exist under a near-but-not-exact title. Store it as\n // \"uncertain\" WITH the best candidate, so a human can confirm. Not dropped.\n const rec = toRecord(best);\n rec.score = Math.round(bestScore * 100) / 100;\n return { status: \"uncertain\", record: rec, score: bestScore };\n }\n\n return { status: \"not_found\", record: null, score: bestScore };\n}\n\n// ---- process ONE batch, safely paced, and persist results -----------------\nexport async function processOneBatch(batchSize = BATCH): Promise {\n await sqlite.execute(`CREATE TABLE IF NOT EXISTS verify_results (\n queried_name TEXT PRIMARY KEY,\n status TEXT NOT NULL,\n record TEXT,\n score REAL\n )`);\n\n const names = await loadAllNames();\n const uniqueNames = Array.from(new Set(\n names.map((n) => (n || \"\").trim()).filter((n) => n && n.toLowerCase() !== \"podcast\"),\n ));\n\n const doneRows = await sqlite.execute(\"SELECT queried_name FROM verify_results\");\n const done = new Set(doneRows.rows.map((r) => r.queried_name as string));\n const pending = uniqueNames.filter((n) => !done.has(n));\n\n console.log(`Total unique names: ${uniqueNames.length}`);\n console.log(`Already done: ${done.size}`);\n console.log(`Pending: ${pending.length}`);\n\n if (pending.length === 0) { console.log(\"\\n=== ALL DONE ===\"); return; }\n\n const todo = pending.slice(0, Math.min(batchSize, BATCH));\n const counts = { verified: 0, uncertain: 0, not_found: 0, duplicate: 0 };\n\n for (let i = 0; i < todo.length; i++) {\n const query = todo[i];\n const { status, record, score } = await verifyOne(query);\n counts[status as keyof typeof counts]++;\n await sqlite.execute({\n sql: \"INSERT OR REPLACE INTO verify_results (queried_name, status, record, score) VALUES (?, ?, ?, ?)\",\n args: [query, status, record ? JSON.stringify(record) : null, score ?? null],\n });\n // Real politeness delay so Apple doesn't rate-limit us.\n await new Promise((r) => setTimeout(r, THROTTLE_MS));\n if ((i + 1) % 50 === 0) {\n console.log(`... ${i + 1}/${todo.length} this run | done overall ~${done.size + i + 1}/${uniqueNames.length}`);\n }\n }\n\n console.log(`\\nThis run checked ${todo.length} names.`);\n console.log(` verified: ${counts.verified}`);\n console.log(` uncertain: ${counts.uncertain} (near-match, review before dropping)`);\n console.log(` not_found: ${counts.not_found}`);\n console.log(` duplicate: ${counts.duplicate}`);\n console.log(`Progress: ${done.size + todo.length}/${uniqueNames.length} complete.`);\n}\n\n// Allow running directly (script) OR importing (worker). Only auto-run when\n// executed directly, not when imported as a module.\nconst isMain = import.meta.main;\nif (isMain) {\n await processOneBatch();\n}\n",[],[237],"5d48259b-fdbe-4126-b816-a4424cfeeaee","ingest_found.ts","2294515a-aa55-11f1-97ba-1607ee4eb77e",35,["D",1788742513684],"// Ingest page for the shows YOU found yourself (from the rejects list).\n// Paste them here — one per line, ideally as: Name, https://pod.link/ID\n// (name and pod.link on the same line). Stored under blob key \"found_names\"\n// as a list of {name, id} objects so they can be merged + enriched later.\nimport { blob } from \"https://esm.town/v/std/blob/main.ts\";\n\nconst PAGE = `\n\nAdd shows you found\n\n

Paste the shows YOU found (that the tool marked \"not found\")

One per line, ideally with the pod.link so we match the exact show, e.g.:

 Abolition Science Radio, https://pod.link/1436794285\n  Another Show, https://pod.link/1234567890\n  Just a name works too

\n \n

\n\n`;\n\n// Parse each line into {name, id}. Handles \"Name, https://pod.link/123\" and\n// \"Name | https://pod.link/123\" and \"Name https://pod.link/123\" and bare names.\nfunction parseLine(line: string): { name: string; id: string } {\n let text = line.trim();\n let id = \"\";\n // pull out a pod.link / itunes / apple URL if present\n const urlMatch = text.match(/https?:\\/\\/[^\\s,|]+/i);\n if (urlMatch) {\n const u = urlMatch[0];\n // extract trailing numeric id\n const idm = u.match(/(\\d{6,15})/);\n if (idm) id = idm[1];\n text = text.replace(u, \"\").replace(/[,\\|]/g, \"\").trim();\n }\n return { name: text || line.trim(), id };\n}\n\nfunction parseNames(txt: string): { name: string; id: string }[] {\n try {\n const j = JSON.parse(txt);\n if (Array.isArray(j)) {\n return j.map((s) => {\n if (typeof s === \"string\") return parseLine(s);\n if (s && typeof s === \"object\") {\n return { name: String(s.name ?? \"\"), id: String(s.id ?? \"\") };\n }\n return { name: \"\", id: \"\" };\n }).filter((x) => x.name);\n }\n } catch (_e) { /* not JSON */ }\n return txt.split(/\\r?\\n/).map(parseLine).filter((x) => x.name);\n}\n\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n if (req.method === \"POST\" && url.searchParams.get(\"action\") === \"save\") {\n try {\n const txt = await req.text();\n const entries = parseNames(txt);\n await blob.set(\"found_names\", JSON.stringify(entries));\n const withId = entries.filter((e) => e.id).length;\n return new Response(\n `Saved ${entries.length} found shows (${withId} with a pod.link ID). Now merge them into the verified list.`,\n { headers: { \"Content-Type\": \"text/plain\" } },\n );\n } catch (e) {\n const msg = e instanceof Error ? `${e.name}: ${e.message}` : String(e);\n return new Response(\"ERROR: \" + msg, { status: 500, headers: { \"Content-Type\": \"text/plain\" } });\n }\n }\n return new Response(PAGE, { headers: { \"Content-Type\": \"text/html; charset=utf-8\" } });\n}\n",[],[245],"60680860-1a8f-487e-850c-c426216c229e","ingest.ts","e041c0d2-a86d-11f1-b23f-1607ee4eb77e",8,["D",1788532529232],"// Ingest page for the longlist verifier.\n// Paste your whole list of podcast names here, click Save — it writes them\n// into the val's blob storage under names_1 so main.ts can read them.\nimport { blob } from \"https://esm.town/v/std/blob/main.ts\";\n\nconst PAGE = `\n\nUpload podcast names\n\n

Paste your podcast names here

One per line, or as a JSON array — either works. Then click Save names.

\n \n

\n\n`;\n\nfunction parseNames(txt: string): string[] {\n try {\n const j = JSON.parse(txt);\n if (Array.isArray(j)) return j.map((s) => String(s).trim()).filter(Boolean);\n } catch (_e) {\n /* not JSON */\n }\n return txt.split(/\\r?\\n/).map((s) => s.trim()).filter(Boolean);\n}\n\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n if (req.method === \"POST\" && url.searchParams.get(\"action\") === \"save\") {\n try {\n const txt = await req.text();\n const names = parseNames(txt);\n await blob.set(\"names_1\", JSON.stringify(names));\n return new Response(\n `Saved ${names.length} names. Now run main.ts (the verifier) to check them.`,\n { headers: { \"Content-Type\": \"text/plain\" } },\n );\n } catch (e) {\n const msg = e instanceof Error ? `${e.name}: ${e.message}` : String(e);\n return new Response(\"ERROR: \" + msg, { status: 500, headers: { \"Content-Type\": \"text/plain\" } });\n }\n }\n return new Response(PAGE, { headers: { \"Content-Type\": \"text/html; charset=utf-8\" } });\n}\n",[],[253],["D",1788743069114],[],[18],["D",1788530401973],["D",1789488693443],{"_11":12,"_159":74},{"_11":12,"_155":74,"_73":74,"_30":72,"_156":157,"_158":24},{"_42":103,"_13":16,"_36":39,"_147":109,"_148":-5,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":14,"_154":12},"ctx",{"_57":-5,"_178":301,"_302":303,"_304":305,"_306":24,"_307":24,"_308":24,"_309":121,"_310":-5,"_311":-7,"_312":-7,"_313":26,"_314":24,"_315":24,"_316":-5,"_317":24,"_318":24,"_319":24,"_320":-5},"ENV",{"_272":273,"_274":275,"_276":16,"_277":278,"_279":280,"_281":282,"_283":284,"_285":286,"_287":288,"_289":290,"_291":292,"_293":294,"_295":296,"_297":298,"_299":300},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

History · export.ts