[{"_1":2,"_181":-5,"_182":-5},"loaderData",{"_3":4,"_93":94},"root",{"_5":6,"_36":37,"_66":67},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","7721b06a805a6662fde743a1ec849364175efb5b","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_68":69},"__internal_clerk_state",{"_70":71,"_72":47,"_73":74,"_75":74,"_76":20,"_77":74,"_78":74,"_79":74,"_80":74,"_81":82,"_91":20,"_92":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_83":20,"_19":20,"_84":74,"_85":86,"_87":74,"_88":47,"_89":20,"_90":74},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_95":96,"_97":98,"_99":100,"_101":102,"_155":156,"_178":179,"_180":-5},"handle","ashryanio","name","index-now","path","lib/indexnow.ts","project",{"_103":104,"_97":98,"_105":106,"_107":-5,"_108":20,"_109":110,"_111":74,"_112":-5,"_12":113,"_114":-5,"_115":20,"_116":-5,"_117":-5,"_118":33,"_14":119,"_120":121,"_122":123,"_124":125,"_126":125,"_127":128,"_129":128,"_130":128,"_131":125,"_132":125,"_23":133,"_136":137,"_141":142,"_154":20},"id","e5fae6ec-04d1-11f1-9f4e-42dde27851f2","createdAt",["D",1770543497825],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","97567c3b-8285-4589-880d-547c23383b76","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","112e17ac-118d-4072-aca6-a4ddc6b3970f","tursoName","e5fae6ec04d111f19f4e42dde27851f2","tursoHostname","e5fae6ec04d111f19f4e42dde27851f2-val-town.aws-us-east-2.turso.io","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_95":96,"_134":135},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yaG5kR0hRRTM0S1BSeXlIR2hCdjFqb1dTS3AifQ","org",{"_95":96,"_138":135,"_12":113,"_139":140},"avatar","tier","pro","branch",{"_143":144,"_97":45,"_103":145,"_146":104,"_147":-5,"_148":-5,"_149":-5,"_150":-5,"_151":-5,"_152":98,"_153":96},"version",360,"e5fb619e-04d1-11f1-9f4e-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_103":157,"_97":158,"_159":160,"_161":162,"_163":145,"_164":165,"_166":167,"_105":168,"_143":169,"_170":171,"_172":173,"_174":20,"_175":-5,"_99":176},"8c323baa-07ac-11f1-a5c7-42dde27851f2","indexnow.ts","fileId","bc3f543e-04d4-11f1-bd3b-42dde27851f2","parentId","801a928e-04d4-11f1-9771-42dde27851f2","branchId","branchVersion",358,"latest",true,["D",1770857309283],72,"fileType","script","content","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { XMLParser } from \"https://esm.sh/fast-xml-parser@4.5.0\";\nimport { listFiles, readFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n\nconst URL_STATE_TABLE = \"indexnow_url_state_v1\";\nconst RUN_LOG_TABLE = \"indexnow_run_log_v1\";\nconst RUN_URL_STATUS_TABLE = \"indexnow_run_url_status_v1\";\nconst URL_SNAPSHOT_TABLE = \"indexnow_url_snapshot_v1\";\nconst DEFAULT_ENDPOINT = \"https://api.indexnow.org/indexnow\";\nconst DEFAULT_RUN_URL_LIMIT = 5_000;\nconst RUN_URL_LIMIT_ENV_VAR = \"INDEXNOW_PER_SITE_RUN_URL_LIMIT\";\n// IndexNow protocol hard limit: 10,000 URLs per request payload.\nconst INDEXNOW_MAX_URLS_PER_POST = 10_000;\nconst SITES_CONFIG_DIR = \"config/sites/\";\nconst SNAPSHOT_RETENTION_DAYS = 14;\nconst SNAPSHOT_MIN_PER_URL = 3;\nconst SNAPSHOT_CAPTURE_CONCURRENCY = 4;\nconst BACKFILL_DEFAULT_LIMIT_PER_SITE = 100;\nconst BACKFILL_MAX_LIMIT_PER_SITE = 1_000;\nconst BACKFILL_ERROR_SAMPLE_LIMIT = 20;\nconst BACKFILL_URL_SAMPLE_LIMIT = 20;\n\nconst xmlParser = new XMLParser({\n ignoreAttributes: false,\n trimValues: true,\n});\n\nexport interface IndexNowSiteConfig {\n host: string;\n sitemapUrl: string;\n keyLocation: string;\n alertToEnvVar?: string;\n includePrefixes?: string[];\n excludePrefixes?: string[];\n}\n\ninterface IndexNowSiteFileConfig {\n host?: string;\n sitemapUrl?: string;\n keyLocation?: string;\n keyLocationEnvVar?: string;\n alertToEnvVar?: string;\n includePrefixes?: string[];\n excludePrefixes?: string[];\n}\n\nexport interface IndexNowRuntimeConfig {\n endpoint: string;\n runUrlLimit: number;\n sites: IndexNowSiteConfig[];\n}\n\nexport interface IndexNowRunOptions {\n dryRun?: boolean;\n force?: boolean;\n siteHost?: string;\n reason?: string;\n}\n\ninterface SitemapEntry {\n url: string;\n lastmod?: string;\n}\n\ninterface FilterStats {\n invalidUrl: number;\n wrongHost: number;\n outOfKeyScope: number;\n includePrefixMiss: number;\n excludePrefixHit: number;\n noindexHit: number;\n semanticUnchangedHit: number;\n}\n\ninterface IgnoredUrlsByReason {\n invalid_url: string[];\n wrong_host: string[];\n out_of_key_scope: string[];\n include_prefix_miss: string[];\n exclude_prefix_hit: string[];\n}\n\ninterface PersistedUrlRow {\n url: string;\n last_submitted_lastmod: string | null;\n last_submitted_at: string | null;\n}\n\ninterface SubmitChunkResult {\n index: number;\n urlCount: number;\n status: number;\n accepted: boolean;\n pending: boolean;\n attempts: number;\n responseSnippet: string;\n}\n\nexport type RunUrlOutcome =\n | \"submitted\"\n | \"pending\"\n | \"failed\"\n | \"deferred\"\n | \"dry_run\"\n | \"no_change\"\n | \"ignored\"\n | \"suppressed\";\n\nexport interface RunUrlStatusEntry {\n url: string;\n slug: string;\n changed: boolean;\n queued: boolean;\n attempted: boolean;\n submissionStatus: number | null;\n outcome: RunUrlOutcome;\n note: string | null;\n}\n\ninterface RunUrlStatusRow extends RunUrlStatusEntry {}\n\nexport interface RunUrlStatusesByOutcome {\n submitted: RunUrlStatusEntry[];\n pending: RunUrlStatusEntry[];\n deferred: RunUrlStatusEntry[];\n failed: RunUrlStatusEntry[];\n dry_run: RunUrlStatusEntry[];\n no_change: RunUrlStatusEntry[];\n ignored: RunUrlStatusEntry[];\n suppressed: RunUrlStatusEntry[];\n}\n\ntype UrlSnapshotSource = \"run_queue_capture\" | \"backfill\";\n\ninterface UrlSnapshotFetchResult {\n responseStatus: number | null;\n contentType: string | null;\n etag: string | null;\n responseLastModified: string | null;\n htmlRaw: string | null;\n htmlSha256: string | null;\n fetchError: string | null;\n}\n\ninterface UrlSnapshotInsertRow {\n runId: number | null;\n siteHost: string;\n url: string;\n slug: string;\n capturedAt: string;\n snapshotSource: UrlSnapshotSource;\n dryRun: boolean;\n responseStatus: number | null;\n contentType: string | null;\n etag: string | null;\n responseLastModified: string | null;\n htmlRaw: string | null;\n htmlSha256: string | null;\n sitemapLastmod: string | null;\n previousSubmittedLastmod: string | null;\n fetchError: string | null;\n}\n\nexport interface SiteRunSummary {\n runId: number;\n host: string;\n sitemapUrl: string;\n startedAt: string;\n finishedAt: string;\n discoveredCount: number;\n filteredCount: number;\n changedCount: number;\n queuedCount: number;\n deferredCount: number;\n submittedCount: number;\n pendingCount: number;\n filteredUrls: string[];\n changedUrls: string[];\n queuedUrls: string[];\n dryRun: boolean;\n force: boolean;\n endpoint: string;\n filterStats: FilterStats;\n chunkResults: SubmitChunkResult[];\n errors: string[];\n}\n\nexport interface IndexNowRunSummary {\n startedAt: string;\n finishedAt: string;\n reason: string;\n dryRun: boolean;\n force: boolean;\n endpoint: string;\n runUrlLimit: number;\n siteRuns: SiteRunSummary[];\n hasErrors: boolean;\n}\n\nexport interface IndexNowBackfillOptions {\n dryRun?: boolean;\n siteHost?: string;\n limitPerSite?: number;\n}\n\nexport interface SiteBackfillSummary {\n host: string;\n dryRun: boolean;\n candidatesInBatch: number;\n insertedCount: number;\n fetchErrorCount: number;\n remainingAfterBatch: number;\n sampledUrls: string[];\n errors: string[];\n}\n\nexport interface IndexNowBackfillSummary {\n startedAt: string;\n finishedAt: string;\n dryRun: boolean;\n limitPerSite: number;\n siteHost?: string;\n siteBackfills: SiteBackfillSummary[];\n hasErrors: boolean;\n}\n\ninterface SitemapDocument {\n childSitemaps: string[];\n entries: SitemapEntry[];\n}\n\ninterface SubmitResult {\n status: number;\n body: string;\n attempts: number;\n}\n\ninterface BackfillCandidateRow {\n url: string;\n slug: string;\n lastmod: string | null;\n lastSubmittedLastmod: string | null;\n}\n\nfunction parsePositiveIntEnv(name: string, fallback: number): number {\n const value = Deno.env.get(name);\n if (!value) {\n return fallback;\n }\n\n const parsed = Number.parseInt(value, 10);\n if (!Number.isFinite(parsed) || parsed <= 0) {\n throw new Error(`${name} must be a positive integer, received \"${value}\"`);\n }\n return parsed;\n}\n\nfunction normalizeBackfillLimit(value: number | undefined): number {\n if (typeof value === \"undefined\") {\n return BACKFILL_DEFAULT_LIMIT_PER_SITE;\n }\n if (!Number.isFinite(value) || value <= 0) {\n throw new Error(`limitPerSite must be a positive integer, received \"${value}\"`);\n }\n const integerValue = Math.floor(value);\n return Math.min(integerValue, BACKFILL_MAX_LIMIT_PER_SITE);\n}\n\nfunction normalizeProjectPath(path: string): string {\n return path.startsWith(\"/\") ? path.slice(1) : path;\n}\n\nasync function loadSitesFromProjectFiles(): Promise {\n const allFiles = await listFiles(import.meta.url);\n const siteFiles = allFiles\n .map((entry) => {\n if (typeof entry === \"string\") {\n return normalizeProjectPath(entry);\n }\n const path = (entry as { path?: unknown }).path;\n return typeof path === \"string\" ? normalizeProjectPath(path) : \"\";\n })\n .filter((path) => path.startsWith(SITES_CONFIG_DIR) && path.endsWith(\".json\"))\n .sort((a, b) => a.localeCompare(b));\n\n if (siteFiles.length === 0) {\n throw new Error(\n `No site config files found in ${SITES_CONFIG_DIR}. Add one or more *.json files there.`,\n );\n }\n\n const sites: IndexNowSiteFileConfig[] = [];\n for (const siteFile of siteFiles) {\n const rawContent = await readFile(`/${siteFile}`, import.meta.url);\n let parsed: unknown;\n try {\n parsed = JSON.parse(rawContent);\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n throw new Error(`Invalid JSON in ${siteFile}: ${message}`);\n }\n if (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n throw new Error(`Site file ${siteFile} must contain a single JSON object`);\n }\n sites.push(parsed as IndexNowSiteFileConfig);\n }\n\n return sites.map(normalizeSiteConfig);\n}\n\nfunction readRequiredEnvVar(envVarName: string, context: string): string {\n const name = envVarName.trim();\n if (!name) {\n throw new Error(`${context} requires a non-empty env var name`);\n }\n const value = Deno.env.get(name)?.trim();\n if (!value) {\n throw new Error(`${context} references env var \"${name}\", but it is missing or empty`);\n }\n return value;\n}\n\nfunction withTxtSuffixForKeyPath(url: URL): URL {\n const pathname = url.pathname;\n if (!pathname || pathname.endsWith(\"/\")) {\n return url;\n }\n const lastSlash = pathname.lastIndexOf(\"/\");\n const fileName = pathname.slice(lastSlash + 1);\n if (fileName.toLowerCase().endsWith(\".txt\")) {\n return url;\n }\n if (isValidIndexNowKey(fileName)) {\n url.pathname = `${pathname}.txt`;\n }\n return url;\n}\n\nfunction parseAndNormalizeKeyLocationUrl(rawUrl: string, context: string): string {\n let parsed: URL;\n try {\n parsed = new URL(rawUrl);\n } catch {\n throw new Error(\n `${context} must be a valid URL, a key token, or a key filename`,\n );\n }\n return withTxtSuffixForKeyPath(parsed).toString();\n}\n\nfunction normalizeKeyLocationValue(rawValue: string, host: string, context: string): string {\n const value = rawValue.trim();\n if (!value) {\n throw new Error(`${context} must not be empty`);\n }\n\n if (value.startsWith(\"https://\") || value.startsWith(\"http://\")) {\n return parseAndNormalizeKeyLocationUrl(value, context);\n }\n\n if (isValidIndexNowKey(value)) {\n return `https://${host}/${value}.txt`;\n }\n\n if (value.toLowerCase().endsWith(\".txt\")) {\n const keyPart = value.slice(0, -4);\n if (isValidIndexNowKey(keyPart) && !value.includes(\"/\")) {\n return `https://${host}/${value}`;\n }\n }\n\n const normalizedPath = value.startsWith(\"/\") ? value : `/${value}`;\n return parseAndNormalizeKeyLocationUrl(`https://${host}${normalizedPath}`, context);\n}\n\nfunction resolveKeyLocationFromConfig(site: IndexNowSiteFileConfig, host: string): string {\n const keyLocation = site.keyLocation?.trim();\n const keyLocationEnvVar = site.keyLocationEnvVar?.trim();\n\n if (keyLocation && keyLocationEnvVar) {\n throw new Error(`Site ${host} must set only one of keyLocation or keyLocationEnvVar`);\n }\n if (keyLocationEnvVar) {\n const envValue = readRequiredEnvVar(keyLocationEnvVar, `Site ${host} keyLocationEnvVar`);\n return normalizeKeyLocationValue(\n envValue,\n host,\n `Site ${host} keyLocationEnvVar (${keyLocationEnvVar})`,\n );\n }\n if (keyLocation) {\n return normalizeKeyLocationValue(keyLocation, host, `Site ${host} keyLocation`);\n }\n throw new Error(`Site ${host} requires keyLocation or keyLocationEnvVar`);\n}\n\nfunction normalizeSiteConfig(site: IndexNowSiteFileConfig): IndexNowSiteConfig {\n const rawSite = site as Record;\n if (typeof rawSite[\"key\"] !== \"undefined\" || typeof rawSite[\"keyEnvVar\"] !== \"undefined\") {\n throw new Error(\n \"Site config fields 'key' and 'keyEnvVar' are no longer supported. Use keyLocation or keyLocationEnvVar only.\",\n );\n }\n\n const host = site.host?.trim();\n if (!host) {\n throw new Error(\"Site config requires host\");\n }\n\n const sitemapUrl = site.sitemapUrl?.trim();\n if (!sitemapUrl) {\n throw new Error(`Site ${host} requires sitemapUrl`);\n }\n\n const keyLocation = resolveKeyLocationFromConfig(site, host);\n const alertToEnvVar = site.alertToEnvVar?.trim();\n\n const normalizedSite: IndexNowSiteConfig = {\n host,\n sitemapUrl,\n keyLocation,\n alertToEnvVar: alertToEnvVar && alertToEnvVar.length > 0 ? alertToEnvVar : undefined,\n includePrefixes: site.includePrefixes?.map((prefix) => prefix.trim()),\n excludePrefixes: site.excludePrefixes?.map((prefix) => prefix.trim()),\n };\n\n return validateSiteConfig(normalizedSite);\n}\n\nfunction validateSiteConfig(site: IndexNowSiteConfig): IndexNowSiteConfig {\n const sitemapUrl = new URL(site.sitemapUrl);\n const keyLocationUrl = new URL(site.keyLocation);\n if (sitemapUrl.protocol !== \"https:\" && sitemapUrl.protocol !== \"http:\") {\n throw new Error(`Unsupported sitemap protocol for ${site.host}`);\n }\n if (keyLocationUrl.protocol !== \"https:\" && keyLocationUrl.protocol !== \"http:\") {\n throw new Error(`Unsupported keyLocation protocol for ${site.host}`);\n }\n if (keyLocationUrl.host.toLowerCase() !== site.host.toLowerCase()) {\n throw new Error(\n `keyLocation host (${keyLocationUrl.host}) must match site host (${site.host})`,\n );\n }\n\n return site;\n}\n\nexport async function loadRuntimeConfig(): Promise {\n const endpoint = (Deno.env.get(\"INDEXNOW_ENDPOINT\") ?? DEFAULT_ENDPOINT).trim();\n const runUrlLimit = parsePositiveIntEnv(RUN_URL_LIMIT_ENV_VAR, DEFAULT_RUN_URL_LIMIT);\n const sites = await loadSitesFromProjectFiles();\n\n const uniqueHosts = new Set(sites.map((site) => site.host.toLowerCase()));\n if (uniqueHosts.size !== sites.length) {\n throw new Error(`${SITES_CONFIG_DIR} contains duplicate host values`);\n }\n\n return {\n endpoint,\n runUrlLimit,\n sites,\n };\n}\n\nfunction asArray(value: T | T[] | undefined): T[] {\n if (typeof value === \"undefined\") {\n return [];\n }\n return Array.isArray(value) ? value : [value];\n}\n\nfunction firstString(value: unknown): string | undefined {\n if (typeof value === \"string\") {\n return value.trim();\n }\n if (Array.isArray(value)) {\n return firstString(value[0]);\n }\n if (value && typeof value === \"object\") {\n const maybeRecord = value as Record;\n if (typeof maybeRecord[\"#text\"] === \"string\") {\n return maybeRecord[\"#text\"].trim();\n }\n }\n return undefined;\n}\n\nfunction parseSitemapDocument(xml: string): SitemapDocument {\n const parsed = xmlParser.parse(xml) as Record;\n\n const sitemapIndex = parsed[\"sitemapindex\"] as\n | Record\n | undefined;\n if (sitemapIndex) {\n const childSitemaps = asArray(\n sitemapIndex[\"sitemap\"] as Record | Record[] | undefined,\n )\n .map((item) => firstString(item[\"loc\"]))\n .filter((value): value is string => Boolean(value));\n return { childSitemaps, entries: [] };\n }\n\n const urlSet = parsed[\"urlset\"] as Record | undefined;\n if (urlSet) {\n const entries = asArray(\n urlSet[\"url\"] as Record | Record[] | undefined,\n )\n .map((item) => ({\n url: firstString(item[\"loc\"]) ?? \"\",\n lastmod: firstString(item[\"lastmod\"]),\n }))\n .filter((entry) => entry.url.length > 0);\n return { childSitemaps: [], entries };\n }\n\n throw new Error(\"Unsupported sitemap format\");\n}\n\nasync function fetchText(url: string): Promise {\n const response = await fetch(url, {\n headers: {\n \"user-agent\": \"indexnow-valtown/1.0\",\n },\n });\n if (!response.ok) {\n throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);\n }\n return await response.text();\n}\n\nfunction toNullableHeaderValue(value: string | null): string | null {\n if (!value) {\n return null;\n }\n const trimmed = value.trim();\n return trimmed.length > 0 ? trimmed : null;\n}\n\nfunction toErrorMessage(error: unknown): string {\n return error instanceof Error ? error.message : String(error);\n}\n\nfunction toHex(bytes: Uint8Array): string {\n return Array.from(bytes).map((value) => value.toString(16).padStart(2, \"0\")).join(\"\");\n}\n\nasync function sha256Hex(value: string): Promise {\n const encoded = new TextEncoder().encode(value);\n const digest = await crypto.subtle.digest(\"SHA-256\", encoded);\n return toHex(new Uint8Array(digest));\n}\n\nasync function captureUrlSnapshot(url: string): Promise {\n try {\n const response = await fetch(url, {\n headers: {\n \"user-agent\": \"indexnow-valtown/1.0\",\n },\n });\n const responseStatus = response.status;\n const contentType = toNullableHeaderValue(response.headers.get(\"content-type\"));\n const etag = toNullableHeaderValue(response.headers.get(\"etag\"));\n const responseLastModified = toNullableHeaderValue(response.headers.get(\"last-modified\"));\n\n if (!response.ok) {\n return {\n responseStatus,\n contentType,\n etag,\n responseLastModified,\n htmlRaw: null,\n htmlSha256: null,\n fetchError: `HTTP ${response.status} ${response.statusText}`.trim(),\n };\n }\n\n const htmlRaw = await response.text();\n return {\n responseStatus,\n contentType,\n etag,\n responseLastModified,\n htmlRaw,\n htmlSha256: await sha256Hex(htmlRaw),\n fetchError: null,\n };\n } catch (error) {\n return {\n responseStatus: null,\n contentType: null,\n etag: null,\n responseLastModified: null,\n htmlRaw: null,\n htmlSha256: null,\n fetchError: truncateForLog(`Snapshot fetch failed: ${toErrorMessage(error)}`),\n };\n }\n}\n\nfunction isNewerLastmod(candidate?: string, current?: string): boolean {\n if (!candidate) {\n return false;\n }\n if (!current) {\n return true;\n }\n\n const candidateMs = Date.parse(candidate);\n const currentMs = Date.parse(current);\n if (Number.isFinite(candidateMs) && Number.isFinite(currentMs)) {\n return candidateMs > currentMs;\n }\n return candidate > current;\n}\n\nasync function crawlSitemapRecursive(\n sitemapUrl: string,\n visited: Set,\n): Promise {\n if (visited.has(sitemapUrl)) {\n return [];\n }\n visited.add(sitemapUrl);\n\n const xml = await fetchText(sitemapUrl);\n const doc = parseSitemapDocument(xml);\n\n if (doc.childSitemaps.length === 0) {\n return doc.entries;\n }\n\n const children = await Promise.all(\n doc.childSitemaps.map((childUrl) => crawlSitemapRecursive(childUrl, visited)),\n );\n\n return children.flat();\n}\n\nfunction normalizeUrl(urlText: string): string {\n const parsed = new URL(urlText.trim());\n parsed.hash = \"\";\n return parsed.toString();\n}\n\nfunction isRootKeyLocation(keyLocation: URL): boolean {\n const path = keyLocation.pathname;\n const withoutLeadingSlash = path.startsWith(\"/\") ? path.slice(1) : path;\n return withoutLeadingSlash.length > 0 && !withoutLeadingSlash.includes(\"/\");\n}\n\nfunction isWithinKeyScope(pageUrl: URL, keyLocation: URL): boolean {\n if (isRootKeyLocation(keyLocation)) {\n return true;\n }\n\n const directoryPath = keyLocation.pathname.slice(\n 0,\n keyLocation.pathname.lastIndexOf(\"/\") + 1,\n );\n\n return pageUrl.pathname.startsWith(directoryPath);\n}\n\nfunction matchesPrefix(url: URL, prefix: string): boolean {\n if (prefix.startsWith(\"http://\") || prefix.startsWith(\"https://\")) {\n return url.toString().startsWith(prefix);\n }\n return url.pathname.startsWith(prefix);\n}\n\nfunction emptyIgnoredUrlsByReason(): IgnoredUrlsByReason {\n return {\n invalid_url: [],\n wrong_host: [],\n out_of_key_scope: [],\n include_prefix_miss: [],\n exclude_prefix_hit: [],\n };\n}\n\nfunction appendUniqueValue(target: string[], value: string): void {\n if (!target.includes(value)) {\n target.push(value);\n }\n}\n\nfunction filterAndDeduplicateEntries(\n site: IndexNowSiteConfig,\n entries: SitemapEntry[],\n): { entries: SitemapEntry[]; stats: FilterStats; ignoredByReason: IgnoredUrlsByReason } {\n const stats: FilterStats = {\n invalidUrl: 0,\n wrongHost: 0,\n outOfKeyScope: 0,\n includePrefixMiss: 0,\n excludePrefixHit: 0,\n noindexHit: 0,\n semanticUnchangedHit: 0,\n };\n const ignoredByReason = emptyIgnoredUrlsByReason();\n\n const keyLocation = new URL(site.keyLocation);\n const deduped = new Map();\n\n for (const entry of entries) {\n const rawUrl = typeof entry.url === \"string\" ? entry.url.trim() : \"\";\n let normalized: string;\n try {\n normalized = normalizeUrl(entry.url);\n } catch {\n stats.invalidUrl += 1;\n appendUniqueValue(ignoredByReason.invalid_url, rawUrl.length > 0 ? rawUrl : \"(empty)\");\n continue;\n }\n\n const pageUrl = new URL(normalized);\n if (pageUrl.host.toLowerCase() !== site.host.toLowerCase()) {\n stats.wrongHost += 1;\n appendUniqueValue(ignoredByReason.wrong_host, normalized);\n continue;\n }\n if (!isWithinKeyScope(pageUrl, keyLocation)) {\n stats.outOfKeyScope += 1;\n appendUniqueValue(ignoredByReason.out_of_key_scope, normalized);\n continue;\n }\n\n if (\n site.includePrefixes &&\n site.includePrefixes.length > 0 &&\n !site.includePrefixes.some((prefix) => matchesPrefix(pageUrl, prefix))\n ) {\n stats.includePrefixMiss += 1;\n appendUniqueValue(ignoredByReason.include_prefix_miss, normalized);\n continue;\n }\n\n if (\n site.excludePrefixes &&\n site.excludePrefixes.length > 0 &&\n site.excludePrefixes.some((prefix) => matchesPrefix(pageUrl, prefix))\n ) {\n stats.excludePrefixHit += 1;\n appendUniqueValue(ignoredByReason.exclude_prefix_hit, normalized);\n continue;\n }\n\n const existing = deduped.get(normalized);\n if (!existing || isNewerLastmod(entry.lastmod, existing.lastmod)) {\n deduped.set(normalized, {\n url: normalized,\n lastmod: entry.lastmod,\n });\n }\n }\n\n return {\n entries: Array.from(deduped.values()).sort((a, b) => a.url.localeCompare(b.url)),\n stats,\n ignoredByReason,\n };\n}\n\nfunction isValidIndexNowKey(value: string): boolean {\n return /^[A-Za-z0-9-]{8,128}$/.test(value);\n}\n\nfunction hasNoindexDirective(value: string): boolean {\n const normalized = value.toLowerCase();\n return /\\bnoindex\\b/.test(normalized) || /\\bnone\\b/.test(normalized);\n}\n\nfunction extractHtmlAttribute(tag: string, attribute: string): string | null {\n const pattern = new RegExp(`\\\\b${attribute}\\\\s*=\\\\s*(\\\"([^\\\"]*)\\\"|'([^']*)'|([^\\\\s>]+))`, \"i\");\n const match = tag.match(pattern);\n if (!match) {\n return null;\n }\n const value = match[2] ?? match[3] ?? match[4] ?? \"\";\n const trimmed = value.trim();\n return trimmed.length > 0 ? trimmed : null;\n}\n\nfunction hasNoindexMetaTag(html: string): boolean {\n const headMatch = html.match(/]*>([\\s\\S]*?)<\\/head>/i);\n const scope = headMatch ? headMatch[1] : html.slice(0, 200_000);\n const metaTagMatches = scope.match(/]*>/gi) ?? [];\n for (const tag of metaTagMatches) {\n const name = extractHtmlAttribute(tag, \"name\")?.toLowerCase();\n if (!name || ![\"robots\", \"googlebot\", \"bingbot\", \"slurp\"].includes(name)) {\n continue;\n }\n const content = extractHtmlAttribute(tag, \"content\");\n if (content && hasNoindexDirective(content)) {\n return true;\n }\n }\n return false;\n}\n\nasync function urlHasNoindexDirective(url: string): Promise {\n let response: Response;\n try {\n response = await fetch(url, {\n headers: {\n \"user-agent\": \"indexnow-valtown/1.0\",\n },\n });\n } catch {\n // Fail open: do not suppress submission when the noindex probe itself fails.\n return false;\n }\n\n const xRobotsTag = response.headers.get(\"x-robots-tag\");\n if (xRobotsTag && hasNoindexDirective(xRobotsTag)) {\n return true;\n }\n\n const contentType = response.headers.get(\"content-type\")?.toLowerCase() ?? \"\";\n if (!contentType.includes(\"text/html\") && !contentType.includes(\"application/xhtml+xml\")) {\n return false;\n }\n\n let html: string;\n try {\n html = await response.text();\n } catch {\n return false;\n }\n return hasNoindexMetaTag(html);\n}\n\nasync function excludeNoindexEntries(\n entries: SitemapEntry[],\n): Promise<{ entries: SitemapEntry[]; excludedCount: number; excludedUrls: Set }> {\n if (entries.length === 0) {\n return { entries: [], excludedCount: 0, excludedUrls: new Set() };\n }\n\n const noindexByIndex = new Array(entries.length).fill(false);\n let cursor = 0;\n const workerCount = Math.min(SNAPSHOT_CAPTURE_CONCURRENCY, entries.length);\n const workers = Array.from({ length: workerCount }, async () => {\n while (true) {\n const index = cursor;\n cursor += 1;\n if (index >= entries.length) {\n return;\n }\n noindexByIndex[index] = await urlHasNoindexDirective(entries[index].url);\n }\n });\n await Promise.all(workers);\n\n const indexable: SitemapEntry[] = [];\n let excludedCount = 0;\n const excludedUrls = new Set();\n for (let index = 0; index < entries.length; index += 1) {\n if (noindexByIndex[index]) {\n excludedCount += 1;\n excludedUrls.add(entries[index].url);\n continue;\n }\n indexable.push(entries[index]);\n }\n\n return {\n entries: indexable,\n excludedCount,\n excludedUrls,\n };\n}\n\nfunction normalizeHtmlForSemanticComparison(html: string): string {\n // Neutralize common cache-busting query params while preserving URL structure.\n const withNormalizedQueryValues = html.replace(\n /([?&])(v|ver|version|cache|cb|_)=([A-Za-z0-9._-]+)/gi,\n \"$1$2=__CACHE_BUST__\",\n );\n // Ghost core upgrades can bump generator metadata without meaningful page changes.\n const withNormalizedGeneratorVersion = withNormalizedQueryValues.replace(\n /(]*name=[\"']generator[\"'][^>]*content=[\"']\\s*Ghost\\s+)\\d+(?:\\.\\d+)*([\"'][^>]*>)/gi,\n \"$1__GHOST_VERSION__$2\",\n );\n return withNormalizedGeneratorVersion\n .replace(/\\s+/g, \" \")\n .trim();\n}\n\nasync function loadLatestPreviousSnapshotHtmlByUrl(\n siteHost: string,\n urls: string[],\n currentRunId: number,\n): Promise> {\n const htmlByUrl = new Map();\n for (const url of urls) {\n const result = await sqlite.execute({\n sql: `SELECT html_raw\n FROM ${URL_SNAPSHOT_TABLE}\n WHERE site_host = ?\n AND url = ?\n AND html_raw IS NOT NULL\n AND dry_run = 0\n AND (run_id IS NULL OR run_id <> ?)\n ORDER BY captured_at DESC, id DESC\n LIMIT 1`,\n args: [siteHost, url, currentRunId],\n });\n\n const row = result.rows[0] as { html_raw?: unknown } | undefined;\n if (typeof row?.html_raw === \"string\" && row.html_raw.length > 0) {\n htmlByUrl.set(url, row.html_raw);\n }\n }\n return htmlByUrl;\n}\n\nasync function filterQueuedEntriesBySemanticDiff(\n siteHost: string,\n runId: number,\n queuedEntries: SitemapEntry[],\n stateByUrl: Map,\n snapshotsByUrl: Map,\n): Promise<{ entries: SitemapEntry[]; excludedUrls: Set }> {\n if (queuedEntries.length === 0) {\n return {\n entries: [],\n excludedUrls: new Set(),\n };\n }\n\n const previousHtmlByUrl = await loadLatestPreviousSnapshotHtmlByUrl(\n siteHost,\n queuedEntries.map((entry) => entry.url),\n runId,\n );\n\n const semanticHashesByHtml = new Map();\n const getSemanticHash = async (html: string): Promise => {\n const normalized = normalizeHtmlForSemanticComparison(html);\n const cached = semanticHashesByHtml.get(normalized);\n if (cached) {\n return cached;\n }\n const hash = await sha256Hex(normalized);\n semanticHashesByHtml.set(normalized, hash);\n return hash;\n };\n\n const meaningfulEntries: SitemapEntry[] = [];\n const excludedUrls = new Set();\n\n for (const entry of queuedEntries) {\n const state = stateByUrl.get(entry.url);\n if (!state?.last_submitted_at) {\n // New/never-submitted URLs should always remain eligible.\n meaningfulEntries.push(entry);\n continue;\n }\n\n const currentSnapshot = snapshotsByUrl.get(entry.url);\n const previousHtml = previousHtmlByUrl.get(entry.url);\n if (!currentSnapshot || currentSnapshot.fetchError || !currentSnapshot.htmlRaw || !previousHtml) {\n meaningfulEntries.push(entry);\n continue;\n }\n\n if (currentSnapshot.htmlRaw === previousHtml) {\n excludedUrls.add(entry.url);\n continue;\n }\n\n const [currentHash, previousHash] = await Promise.all([\n getSemanticHash(currentSnapshot.htmlRaw),\n getSemanticHash(previousHtml),\n ]);\n\n if (currentHash === previousHash) {\n excludedUrls.add(entry.url);\n continue;\n }\n\n meaningfulEntries.push(entry);\n }\n\n return {\n entries: meaningfulEntries,\n excludedUrls,\n };\n}\n\nasync function resolveKey(site: IndexNowSiteConfig): Promise {\n const fetchedKey = (await fetchText(site.keyLocation)).trim();\n if (!isValidIndexNowKey(fetchedKey)) {\n throw new Error(\n `Key fetched from ${site.keyLocation} has invalid format for ${site.host}`,\n );\n }\n return fetchedKey;\n}\n\nasync function ensureSchema(): Promise {\n await sqlite.execute({\n sql: `CREATE TABLE IF NOT EXISTS ${URL_STATE_TABLE} (\n site_host TEXT NOT NULL,\n url TEXT NOT NULL,\n lastmod TEXT,\n first_seen_at TEXT NOT NULL,\n last_seen_at TEXT NOT NULL,\n last_attempted_at TEXT,\n last_submitted_at TEXT,\n last_submitted_lastmod TEXT,\n attempt_count INTEGER NOT NULL DEFAULT 0,\n submit_count INTEGER NOT NULL DEFAULT 0,\n last_submission_status INTEGER,\n last_submission_note TEXT,\n PRIMARY KEY (site_host, url)\n )`,\n args: [],\n });\n\n await sqlite.execute({\n sql: `CREATE TABLE IF NOT EXISTS ${RUN_LOG_TABLE} (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n site_host TEXT NOT NULL,\n reason TEXT NOT NULL,\n started_at TEXT NOT NULL,\n finished_at TEXT NOT NULL,\n discovered_count INTEGER NOT NULL,\n eligible_count INTEGER NOT NULL,\n changed_count INTEGER NOT NULL,\n queued_count INTEGER NOT NULL,\n submitted_count INTEGER NOT NULL,\n dry_run INTEGER NOT NULL,\n force_mode INTEGER NOT NULL,\n endpoint TEXT NOT NULL,\n has_errors INTEGER NOT NULL,\n error_summary TEXT\n )`,\n args: [],\n });\n\n await sqlite.execute({\n sql: `CREATE TABLE IF NOT EXISTS ${RUN_URL_STATUS_TABLE} (\n run_id INTEGER NOT NULL,\n site_host TEXT NOT NULL,\n url TEXT NOT NULL,\n slug TEXT NOT NULL,\n changed INTEGER NOT NULL,\n queued INTEGER NOT NULL,\n attempted INTEGER NOT NULL,\n submission_status INTEGER,\n outcome TEXT NOT NULL,\n note TEXT,\n PRIMARY KEY (run_id, url)\n )`,\n args: [],\n });\n\n await sqlite.execute({\n sql: `CREATE INDEX IF NOT EXISTS idx_${RUN_URL_STATUS_TABLE}_lookup\n ON ${RUN_URL_STATUS_TABLE} (site_host, run_id, outcome, slug)`,\n args: [],\n });\n\n await sqlite.execute({\n sql: `CREATE TABLE IF NOT EXISTS ${URL_SNAPSHOT_TABLE} (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n run_id INTEGER,\n site_host TEXT NOT NULL,\n url TEXT NOT NULL,\n slug TEXT NOT NULL,\n captured_at TEXT NOT NULL,\n snapshot_source TEXT NOT NULL,\n dry_run INTEGER NOT NULL,\n response_status INTEGER,\n content_type TEXT,\n etag TEXT,\n response_last_modified TEXT,\n html_raw TEXT,\n html_sha256 TEXT,\n sitemap_lastmod TEXT,\n previous_submitted_lastmod TEXT,\n fetch_error TEXT\n )`,\n args: [],\n });\n\n await sqlite.execute({\n sql: `CREATE UNIQUE INDEX IF NOT EXISTS idx_${URL_SNAPSHOT_TABLE}_run_host_url\n ON ${URL_SNAPSHOT_TABLE} (run_id, site_host, url)\n WHERE run_id IS NOT NULL`,\n args: [],\n });\n\n await sqlite.execute({\n sql: `CREATE INDEX IF NOT EXISTS idx_${URL_SNAPSHOT_TABLE}_site_url_captured\n ON ${URL_SNAPSHOT_TABLE} (site_host, url, captured_at DESC)`,\n args: [],\n });\n\n await sqlite.execute({\n sql: `CREATE INDEX IF NOT EXISTS idx_${URL_SNAPSHOT_TABLE}_run_id\n ON ${URL_SNAPSHOT_TABLE} (run_id)`,\n args: [],\n });\n}\n\nasync function loadStateByUrl(siteHost: string): Promise> {\n const resultSet = await sqlite.execute({\n sql: `SELECT url, last_submitted_lastmod, last_submitted_at\n FROM ${URL_STATE_TABLE}\n WHERE site_host = ?`,\n args: [siteHost],\n });\n\n const map = new Map();\n for (const rawRow of resultSet.rows) {\n const row = rawRow as unknown as Partial;\n if (typeof row.url === \"string\") {\n map.set(row.url, {\n url: row.url,\n last_submitted_lastmod: row.last_submitted_lastmod ?? null,\n last_submitted_at: row.last_submitted_at ?? null,\n });\n }\n }\n return map;\n}\n\nasync function upsertSeenUrls(\n siteHost: string,\n entries: SitemapEntry[],\n seenAt: string,\n): Promise {\n for (const entry of entries) {\n await sqlite.execute({\n sql: `INSERT INTO ${URL_STATE_TABLE}\n (site_host, url, lastmod, first_seen_at, last_seen_at)\n VALUES (?, ?, ?, ?, ?)\n ON CONFLICT(site_host, url) DO UPDATE SET\n lastmod = excluded.lastmod,\n last_seen_at = excluded.last_seen_at`,\n args: [siteHost, entry.url, entry.lastmod ?? null, seenAt, seenAt],\n });\n }\n}\n\nfunction shouldSubmit(\n entry: SitemapEntry,\n state: PersistedUrlRow | undefined,\n force: boolean,\n): boolean {\n if (force) {\n return true;\n }\n if (!state) {\n return true;\n }\n if (!state.last_submitted_at) {\n return true;\n }\n if (!entry.lastmod) {\n return false;\n }\n return entry.lastmod !== state.last_submitted_lastmod;\n}\n\nfunction toChunks(values: T[], chunkSize: number): T[][] {\n const chunks: T[][] = [];\n for (let i = 0; i < values.length; i += chunkSize) {\n chunks.push(values.slice(i, i + chunkSize));\n }\n return chunks;\n}\n\nfunction truncateForLog(value: string, maxLength = 800): string {\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, maxLength)}…`;\n}\n\nfunction sleep(ms: number): Promise {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nasync function submitChunk(\n endpoint: string,\n host: string,\n key: string,\n keyLocation: string,\n urlList: string[],\n): Promise {\n const payload = {\n host,\n key,\n keyLocation,\n urlList,\n };\n\n let attempts = 0;\n let backoffMs = 1_000;\n\n while (true) {\n attempts += 1;\n\n try {\n const response = await fetch(endpoint, {\n method: \"POST\",\n headers: {\n \"content-type\": \"application/json; charset=utf-8\",\n },\n body: JSON.stringify(payload),\n });\n\n const body = truncateForLog(await response.text());\n if ((response.status === 429 || response.status >= 500) && attempts < 3) {\n await sleep(backoffMs);\n backoffMs *= 2;\n continue;\n }\n\n return {\n status: response.status,\n body,\n attempts,\n };\n } catch (error) {\n if (attempts >= 3) {\n throw error;\n }\n await sleep(backoffMs);\n backoffMs *= 2;\n }\n }\n}\n\nasync function markAttemptForUrls(\n siteHost: string,\n urls: SitemapEntry[],\n attemptedAt: string,\n status: number,\n responseSnippet: string,\n accepted: boolean,\n): Promise {\n for (const item of urls) {\n if (accepted) {\n await sqlite.execute({\n sql: `UPDATE ${URL_STATE_TABLE}\n SET last_attempted_at = ?,\n last_submitted_at = ?,\n last_submitted_lastmod = ?,\n attempt_count = attempt_count + 1,\n submit_count = submit_count + 1,\n last_submission_status = ?,\n last_submission_note = ?\n WHERE site_host = ? AND url = ?`,\n args: [\n attemptedAt,\n attemptedAt,\n item.lastmod ?? null,\n status,\n responseSnippet,\n siteHost,\n item.url,\n ],\n });\n } else {\n await sqlite.execute({\n sql: `UPDATE ${URL_STATE_TABLE}\n SET last_attempted_at = ?,\n attempt_count = attempt_count + 1,\n last_submission_status = ?,\n last_submission_note = ?\n WHERE site_host = ? AND url = ?`,\n args: [attemptedAt, status, responseSnippet, siteHost, item.url],\n });\n }\n }\n}\n\nfunction toSlug(urlText: string): string {\n try {\n const parsed = new URL(urlText);\n return `${parsed.pathname}${parsed.search}` || \"/\";\n } catch {\n return urlText;\n }\n}\n\nasync function createRunLogDraft(\n siteHost: string,\n reason: string,\n startedAt: string,\n dryRun: boolean,\n forceMode: boolean,\n endpoint: string,\n): Promise {\n const result = await sqlite.execute({\n sql: `INSERT INTO ${RUN_LOG_TABLE}\n (site_host, reason, started_at, finished_at, discovered_count, eligible_count,\n changed_count, queued_count, submitted_count, dry_run, force_mode, endpoint,\n has_errors, error_summary)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n siteHost,\n reason,\n startedAt,\n startedAt,\n 0,\n 0,\n 0,\n 0,\n 0,\n dryRun ? 1 : 0,\n forceMode ? 1 : 0,\n endpoint,\n 0,\n null,\n ],\n });\n\n const runIdRaw = result.lastInsertRowid;\n const runId = typeof runIdRaw === \"bigint\" ? Number(runIdRaw) : Number(runIdRaw ?? 0);\n if (!Number.isFinite(runId) || runId <= 0) {\n throw new Error(\"Failed to determine run log ID\");\n }\n return runId;\n}\n\nasync function finalizeRunLog(runId: number, summary: SiteRunSummary): Promise {\n const errorSummary = summary.errors.length > 0\n ? truncateForLog(summary.errors.join(\" | \"), 1_600)\n : null;\n await sqlite.execute({\n sql: `UPDATE ${RUN_LOG_TABLE}\n SET started_at = ?,\n finished_at = ?,\n discovered_count = ?,\n eligible_count = ?,\n changed_count = ?,\n queued_count = ?,\n submitted_count = ?,\n dry_run = ?,\n force_mode = ?,\n endpoint = ?,\n has_errors = ?,\n error_summary = ?\n WHERE id = ?`,\n args: [\n summary.startedAt,\n summary.finishedAt,\n summary.discoveredCount,\n summary.filteredCount,\n summary.changedCount,\n summary.queuedCount,\n summary.submittedCount,\n summary.dryRun ? 1 : 0,\n summary.force ? 1 : 0,\n summary.endpoint,\n summary.errors.length > 0 ? 1 : 0,\n errorSummary,\n runId,\n ],\n });\n}\n\nasync function writeUrlSnapshot(row: UrlSnapshotInsertRow): Promise {\n await sqlite.execute({\n sql: `INSERT INTO ${URL_SNAPSHOT_TABLE}\n (run_id, site_host, url, slug, captured_at, snapshot_source, dry_run, response_status,\n content_type, etag, response_last_modified, html_raw, html_sha256, sitemap_lastmod,\n previous_submitted_lastmod, fetch_error)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n row.runId,\n row.siteHost,\n row.url,\n row.slug,\n row.capturedAt,\n row.snapshotSource,\n row.dryRun ? 1 : 0,\n row.responseStatus,\n row.contentType,\n row.etag,\n row.responseLastModified,\n row.htmlRaw,\n row.htmlSha256,\n row.sitemapLastmod,\n row.previousSubmittedLastmod,\n row.fetchError,\n ],\n });\n}\n\nasync function captureQueuedSnapshots(\n runId: number,\n siteHost: string,\n queuedEntries: SitemapEntry[],\n stateByUrl: Map,\n dryRun: boolean,\n): Promise> {\n const snapshotsByUrl = new Map();\n if (queuedEntries.length === 0) {\n return snapshotsByUrl;\n }\n\n let cursor = 0;\n const workerCount = Math.min(SNAPSHOT_CAPTURE_CONCURRENCY, queuedEntries.length);\n const workers = Array.from({ length: workerCount }, async () => {\n while (true) {\n const entryIndex = cursor;\n cursor += 1;\n if (entryIndex >= queuedEntries.length) {\n return;\n }\n\n const entry = queuedEntries[entryIndex];\n const state = stateByUrl.get(entry.url);\n const capturedAt = new Date().toISOString();\n const snapshot = await captureUrlSnapshot(entry.url);\n snapshotsByUrl.set(entry.url, snapshot);\n\n await writeUrlSnapshot({\n runId,\n siteHost,\n url: entry.url,\n slug: toSlug(entry.url),\n capturedAt,\n snapshotSource: \"run_queue_capture\",\n dryRun,\n responseStatus: snapshot.responseStatus,\n contentType: snapshot.contentType,\n etag: snapshot.etag,\n responseLastModified: snapshot.responseLastModified,\n htmlRaw: snapshot.htmlRaw,\n htmlSha256: snapshot.htmlSha256,\n sitemapLastmod: entry.lastmod ?? null,\n previousSubmittedLastmod: state?.last_submitted_lastmod ?? null,\n fetchError: snapshot.fetchError,\n });\n }\n });\n\n await Promise.all(workers);\n return snapshotsByUrl;\n}\n\nasync function pruneUrlSnapshots(siteHost: string): Promise {\n const cutoffMs = Date.now() - SNAPSHOT_RETENTION_DAYS * 24 * 60 * 60 * 1000;\n const cutoffIso = new Date(cutoffMs).toISOString();\n\n await sqlite.execute({\n sql: `DELETE FROM ${URL_SNAPSHOT_TABLE}\n WHERE id IN (\n SELECT id FROM (\n SELECT\n id,\n ROW_NUMBER() OVER (\n PARTITION BY site_host, url\n ORDER BY captured_at DESC, id DESC\n ) AS rn,\n captured_at\n FROM ${URL_SNAPSHOT_TABLE}\n WHERE site_host = ?\n ) ranked\n WHERE rn > ?\n AND captured_at < ?\n )`,\n args: [siteHost, SNAPSHOT_MIN_PER_URL, cutoffIso],\n });\n}\n\nfunction toCount(value: unknown): number {\n if (typeof value === \"number\") {\n return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;\n }\n if (typeof value === \"bigint\") {\n return value > 0 ? Number(value) : 0;\n }\n if (typeof value === \"string\") {\n const parsed = Number.parseInt(value, 10);\n return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;\n }\n return 0;\n}\n\nfunction toRunUrlOutcome(value: unknown): RunUrlOutcome | null {\n const normalized = String(value ?? \"\").toLowerCase();\n switch (normalized) {\n case \"submitted\":\n case \"pending\":\n case \"failed\":\n case \"deferred\":\n case \"dry_run\":\n case \"no_change\":\n case \"ignored\":\n case \"suppressed\":\n return normalized;\n default:\n return null;\n }\n}\n\nfunction toNullableInteger(value: unknown): number | null {\n const parsed = Number(value);\n if (!Number.isFinite(parsed) || parsed <= 0) {\n return null;\n }\n return Math.floor(parsed);\n}\n\nfunction emptyRunUrlStatusesByOutcome(): RunUrlStatusesByOutcome {\n return {\n submitted: [],\n pending: [],\n deferred: [],\n failed: [],\n dry_run: [],\n no_change: [],\n ignored: [],\n suppressed: [],\n };\n}\n\nexport async function loadRunUrlStatusesByOutcome(\n siteHost: string,\n runId: number,\n): Promise {\n if (siteHost.trim().length === 0) {\n throw new Error(\"siteHost is required\");\n }\n if (!Number.isFinite(runId) || runId <= 0) {\n throw new Error(`runId must be a positive number, received \"${runId}\"`);\n }\n\n await ensureSchema();\n const grouped = emptyRunUrlStatusesByOutcome();\n const result = await sqlite.execute({\n sql: `SELECT\n url,\n slug,\n changed,\n queued,\n attempted,\n submission_status,\n outcome,\n note\n FROM ${RUN_URL_STATUS_TABLE}\n WHERE site_host = ?\n AND run_id = ?\n ORDER BY slug ASC, url ASC`,\n args: [siteHost, runId],\n });\n\n for (const rawRow of result.rows) {\n const row = rawRow as {\n url?: unknown;\n slug?: unknown;\n changed?: unknown;\n queued?: unknown;\n attempted?: unknown;\n submission_status?: unknown;\n outcome?: unknown;\n note?: unknown;\n };\n if (typeof row.url !== \"string\" || row.url.length === 0) {\n continue;\n }\n\n const outcome = toRunUrlOutcome(row.outcome);\n if (!outcome) {\n continue;\n }\n\n const item: RunUrlStatusEntry = {\n url: row.url,\n slug: typeof row.slug === \"string\" && row.slug.length > 0 ? row.slug : toSlug(row.url),\n changed: toCount(row.changed) > 0,\n queued: toCount(row.queued) > 0,\n attempted: toCount(row.attempted) > 0,\n submissionStatus: toNullableInteger(row.submission_status),\n outcome,\n note: typeof row.note === \"string\" && row.note.length > 0 ? row.note : null,\n };\n grouped[outcome].push(item);\n }\n\n return grouped;\n}\n\nasync function countRemainingBackfillCandidates(siteHost: string): Promise {\n const result = await sqlite.execute({\n sql: `SELECT COUNT(*) AS remaining_count\n FROM ${URL_STATE_TABLE} AS url_state\n WHERE url_state.site_host = ?\n AND url_state.last_submitted_at IS NOT NULL\n AND NOT EXISTS (\n SELECT 1\n FROM ${URL_SNAPSHOT_TABLE} AS snapshots\n WHERE snapshots.site_host = url_state.site_host\n AND snapshots.url = url_state.url\n AND snapshots.html_raw IS NOT NULL\n )`,\n args: [siteHost],\n });\n const row = result.rows[0] as { remaining_count?: unknown } | undefined;\n return toCount(row?.remaining_count);\n}\n\nasync function loadBackfillCandidates(\n siteHost: string,\n limitPerSite: number,\n): Promise {\n const result = await sqlite.execute({\n sql: `SELECT\n url,\n lastmod,\n last_submitted_lastmod\n FROM ${URL_STATE_TABLE} AS url_state\n WHERE url_state.site_host = ?\n AND url_state.last_submitted_at IS NOT NULL\n AND NOT EXISTS (\n SELECT 1\n FROM ${URL_SNAPSHOT_TABLE} AS snapshots\n WHERE snapshots.site_host = url_state.site_host\n AND snapshots.url = url_state.url\n AND snapshots.html_raw IS NOT NULL\n )\n ORDER BY COALESCE(url_state.last_submitted_at, url_state.last_seen_at) DESC, url_state.url ASC\n LIMIT ?`,\n args: [siteHost, limitPerSite],\n });\n\n return result.rows.flatMap((rawRow) => {\n const row = rawRow as { url?: unknown; lastmod?: unknown; last_submitted_lastmod?: unknown };\n if (typeof row.url !== \"string\" || row.url.length === 0) {\n return [];\n }\n return [{\n url: row.url,\n slug: toSlug(row.url),\n lastmod: typeof row.lastmod === \"string\" && row.lastmod.length > 0 ? row.lastmod : null,\n lastSubmittedLastmod: typeof row.last_submitted_lastmod === \"string\" && row.last_submitted_lastmod.length > 0\n ? row.last_submitted_lastmod\n : null,\n }];\n });\n}\n\nasync function backfillSnapshotsForSite(\n siteHost: string,\n dryRun: boolean,\n limitPerSite: number,\n): Promise {\n const candidates = await loadBackfillCandidates(siteHost, limitPerSite);\n const sampledUrls = candidates.slice(0, BACKFILL_URL_SAMPLE_LIMIT).map((candidate) => candidate.url);\n\n if (dryRun) {\n return {\n host: siteHost,\n dryRun: true,\n candidatesInBatch: candidates.length,\n insertedCount: 0,\n fetchErrorCount: 0,\n remainingAfterBatch: await countRemainingBackfillCandidates(siteHost),\n sampledUrls,\n errors: [],\n };\n }\n\n let cursor = 0;\n let insertedCount = 0;\n let fetchErrorCount = 0;\n const errors: string[] = [];\n const workerCount = Math.min(SNAPSHOT_CAPTURE_CONCURRENCY, candidates.length);\n const workers = Array.from({ length: workerCount }, async () => {\n while (true) {\n const index = cursor;\n cursor += 1;\n if (index >= candidates.length) {\n return;\n }\n\n const candidate = candidates[index];\n const snapshot = await captureUrlSnapshot(candidate.url);\n if (snapshot.fetchError || snapshot.htmlRaw === null || snapshot.htmlSha256 === null) {\n fetchErrorCount += 1;\n if (errors.length < BACKFILL_ERROR_SAMPLE_LIMIT) {\n const detail = snapshot.fetchError ?? \"Snapshot fetch returned no HTML payload.\";\n errors.push(truncateForLog(`${candidate.url}: ${detail}`, 600));\n }\n continue;\n }\n\n await writeUrlSnapshot({\n runId: null,\n siteHost,\n url: candidate.url,\n slug: candidate.slug,\n capturedAt: new Date().toISOString(),\n snapshotSource: \"backfill\",\n dryRun: false,\n responseStatus: snapshot.responseStatus,\n contentType: snapshot.contentType,\n etag: snapshot.etag,\n responseLastModified: snapshot.responseLastModified,\n htmlRaw: snapshot.htmlRaw,\n htmlSha256: snapshot.htmlSha256,\n sitemapLastmod: candidate.lastmod,\n previousSubmittedLastmod: candidate.lastSubmittedLastmod,\n fetchError: null,\n });\n insertedCount += 1;\n }\n });\n\n await Promise.all(workers);\n await pruneUrlSnapshots(siteHost);\n\n return {\n host: siteHost,\n dryRun: false,\n candidatesInBatch: candidates.length,\n insertedCount,\n fetchErrorCount,\n remainingAfterBatch: await countRemainingBackfillCandidates(siteHost),\n sampledUrls,\n errors,\n };\n}\n\ntype IgnoredReasonCode = keyof IgnoredUrlsByReason | \"noindex\";\ntype SuppressedReasonCode = \"semantic_unchanged\";\ntype NoChangeReasonCode = \"unchanged_lastmod\";\n\nfunction ignoredReasonNote(reason: IgnoredReasonCode): string {\n return `ignored:${reason}`;\n}\n\nfunction suppressedReasonNote(reason: SuppressedReasonCode): string {\n return `suppressed:${reason}`;\n}\n\nfunction noChangeReasonNote(reason: NoChangeReasonCode): string {\n return `no_change:${reason}`;\n}\n\nfunction appendIgnoredStatuses(\n rows: RunUrlStatusRow[],\n urls: Iterable,\n reason: IgnoredReasonCode,\n seenUrls: Set,\n): void {\n for (const rawUrl of urls) {\n const normalizedUrl = rawUrl.trim();\n if (normalizedUrl.length === 0 || seenUrls.has(normalizedUrl)) {\n continue;\n }\n seenUrls.add(normalizedUrl);\n rows.push({\n url: normalizedUrl,\n slug: toSlug(normalizedUrl),\n changed: false,\n queued: false,\n attempted: false,\n submissionStatus: null,\n outcome: \"ignored\",\n note: ignoredReasonNote(reason),\n });\n }\n}\n\nfunction appendSuppressedStatuses(\n rows: RunUrlStatusRow[],\n urls: Iterable,\n reason: SuppressedReasonCode,\n seenUrls: Set,\n): void {\n for (const rawUrl of urls) {\n const normalizedUrl = rawUrl.trim();\n if (normalizedUrl.length === 0 || seenUrls.has(normalizedUrl)) {\n continue;\n }\n seenUrls.add(normalizedUrl);\n rows.push({\n url: normalizedUrl,\n slug: toSlug(normalizedUrl),\n changed: false,\n queued: false,\n attempted: false,\n submissionStatus: null,\n outcome: \"suppressed\",\n note: suppressedReasonNote(reason),\n });\n }\n}\n\nfunction appendNoChangeStatuses(\n rows: RunUrlStatusRow[],\n urls: Iterable,\n reason: NoChangeReasonCode,\n seenUrls: Set,\n): void {\n for (const rawUrl of urls) {\n const normalizedUrl = rawUrl.trim();\n if (normalizedUrl.length === 0 || seenUrls.has(normalizedUrl)) {\n continue;\n }\n seenUrls.add(normalizedUrl);\n rows.push({\n url: normalizedUrl,\n slug: toSlug(normalizedUrl),\n changed: false,\n queued: false,\n attempted: false,\n submissionStatus: null,\n outcome: \"no_change\",\n note: noChangeReasonNote(reason),\n });\n }\n}\n\nasync function writeRunUrlStatuses(\n runId: number,\n siteHost: string,\n statuses: RunUrlStatusRow[],\n): Promise {\n if (statuses.length === 0) {\n return;\n }\n\n for (const status of statuses) {\n await sqlite.execute({\n sql: `INSERT INTO ${RUN_URL_STATUS_TABLE}\n (run_id, site_host, url, slug, changed, queued, attempted, submission_status, outcome, note)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n runId,\n siteHost,\n status.url,\n status.slug,\n status.changed ? 1 : 0,\n status.queued ? 1 : 0,\n status.attempted ? 1 : 0,\n status.submissionStatus,\n status.outcome,\n status.note,\n ],\n });\n }\n}\n\nasync function runForSite(\n site: IndexNowSiteConfig,\n config: IndexNowRuntimeConfig,\n options: Required>,\n): Promise {\n const startedAt = new Date().toISOString();\n const rawEntries = await crawlSitemapRecursive(site.sitemapUrl, new Set());\n const filtered = filterAndDeduplicateEntries(site, rawEntries);\n const stateByUrl = await loadStateByUrl(site.host);\n const candidateChangedEntries = filtered.entries.filter((entry) =>\n shouldSubmit(entry, stateByUrl.get(entry.url), options.force)\n );\n const candidateChangedUrls = new Set(candidateChangedEntries.map((entry) => entry.url));\n const noChangeEntries = filtered.entries.filter((entry) => !candidateChangedUrls.has(entry.url));\n const noindexFiltered = await excludeNoindexEntries(candidateChangedEntries);\n let changedEntries = noindexFiltered.entries;\n const noindexExcludedUrls = noindexFiltered.excludedUrls;\n filtered.stats.noindexHit = noindexFiltered.excludedCount;\n const preSemanticQueuedEntries = changedEntries.slice(0, config.runUrlLimit);\n const nowIso = new Date().toISOString();\n\n const chunkResults: SubmitChunkResult[] = [];\n const errors: string[] = [];\n let submittedCount = 0;\n let pendingCount = 0;\n const key = options.dryRun ? null : await resolveKey(site);\n const runId = await createRunLogDraft(\n site.host,\n options.reason,\n startedAt,\n options.dryRun,\n options.force,\n config.endpoint,\n );\n\n const snapshotsByUrl = await captureQueuedSnapshots(\n runId,\n site.host,\n preSemanticQueuedEntries,\n stateByUrl,\n options.dryRun,\n );\n const semanticFiltered = await filterQueuedEntriesBySemanticDiff(\n site.host,\n runId,\n preSemanticQueuedEntries,\n stateByUrl,\n snapshotsByUrl,\n );\n const semanticExcludedUrls = semanticFiltered.excludedUrls;\n filtered.stats.semanticUnchangedHit = semanticExcludedUrls.size;\n if (semanticExcludedUrls.size > 0) {\n changedEntries = changedEntries.filter((entry) => !semanticExcludedUrls.has(entry.url));\n }\n\n if (!options.dryRun) {\n const excludedFromStateUpdate = new Set([\n ...noindexExcludedUrls,\n ...semanticExcludedUrls,\n ]);\n const stateUpdateEntries = excludedFromStateUpdate.size === 0\n ? filtered.entries\n : filtered.entries.filter((entry) => !excludedFromStateUpdate.has(entry.url));\n await upsertSeenUrls(site.host, stateUpdateEntries, nowIso);\n }\n\n const queuedEntries = semanticFiltered.entries;\n const queuedUrlSet = new Set(queuedEntries.map((entry) => entry.url));\n const deferredCount = changedEntries.length - queuedEntries.length;\n const runUrlStatusByUrl = new Map(\n changedEntries.map((entry) => {\n const queued = queuedUrlSet.has(entry.url);\n return [\n entry.url,\n {\n url: entry.url,\n slug: toSlug(entry.url),\n changed: true,\n queued,\n attempted: false,\n submissionStatus: null,\n outcome: queued ? (options.dryRun ? \"dry_run\" : \"pending\") : \"deferred\",\n note: queued && options.dryRun ? \"Dry run: no submission attempted.\" : null,\n } satisfies RunUrlStatusRow,\n ];\n }),\n );\n const syntheticRunStatuses: RunUrlStatusRow[] = [];\n const syntheticStatusSeenUrls = new Set();\n appendIgnoredStatuses(\n syntheticRunStatuses,\n filtered.ignoredByReason.invalid_url,\n \"invalid_url\",\n syntheticStatusSeenUrls,\n );\n appendNoChangeStatuses(\n syntheticRunStatuses,\n noChangeEntries.map((entry) => entry.url),\n \"unchanged_lastmod\",\n syntheticStatusSeenUrls,\n );\n appendIgnoredStatuses(\n syntheticRunStatuses,\n filtered.ignoredByReason.wrong_host,\n \"wrong_host\",\n syntheticStatusSeenUrls,\n );\n appendIgnoredStatuses(\n syntheticRunStatuses,\n filtered.ignoredByReason.out_of_key_scope,\n \"out_of_key_scope\",\n syntheticStatusSeenUrls,\n );\n appendIgnoredStatuses(\n syntheticRunStatuses,\n filtered.ignoredByReason.include_prefix_miss,\n \"include_prefix_miss\",\n syntheticStatusSeenUrls,\n );\n appendIgnoredStatuses(\n syntheticRunStatuses,\n filtered.ignoredByReason.exclude_prefix_hit,\n \"exclude_prefix_hit\",\n syntheticStatusSeenUrls,\n );\n appendIgnoredStatuses(\n syntheticRunStatuses,\n noindexExcludedUrls,\n \"noindex\",\n syntheticStatusSeenUrls,\n );\n appendSuppressedStatuses(\n syntheticRunStatuses,\n semanticExcludedUrls,\n \"semantic_unchanged\",\n syntheticStatusSeenUrls,\n );\n\n if (!options.dryRun) {\n if (!key) {\n throw new Error(`Failed to resolve key for ${site.host}`);\n }\n const chunks = toChunks(queuedEntries, INDEXNOW_MAX_URLS_PER_POST);\n for (let index = 0; index < chunks.length; index += 1) {\n const chunk = chunks[index];\n try {\n const submitResult = await submitChunk(\n config.endpoint,\n site.host,\n key,\n site.keyLocation,\n chunk.map((item) => item.url),\n );\n const accepted = submitResult.status === 200;\n const pending = submitResult.status === 202;\n const acceptedForState = accepted || pending;\n chunkResults.push({\n index,\n urlCount: chunk.length,\n status: submitResult.status,\n accepted,\n pending,\n attempts: submitResult.attempts,\n responseSnippet: submitResult.body,\n });\n\n await markAttemptForUrls(\n site.host,\n chunk,\n new Date().toISOString(),\n submitResult.status,\n submitResult.body,\n acceptedForState,\n );\n\n for (const item of chunk) {\n const statusRow = runUrlStatusByUrl.get(item.url);\n if (!statusRow) {\n continue;\n }\n statusRow.attempted = true;\n statusRow.submissionStatus = submitResult.status;\n statusRow.note = submitResult.body;\n statusRow.outcome = accepted ? \"submitted\" : pending ? \"pending\" : \"failed\";\n }\n\n if (accepted) {\n submittedCount += chunk.length;\n } else if (pending) {\n pendingCount += chunk.length;\n } else {\n errors.push(\n `Chunk ${index} rejected (${submitResult.status}): ${submitResult.body || \"no body\"}`,\n );\n }\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n errors.push(`Chunk ${index} failed: ${message}`);\n for (const item of chunk) {\n const statusRow = runUrlStatusByUrl.get(item.url);\n if (!statusRow) {\n continue;\n }\n statusRow.attempted = true;\n statusRow.submissionStatus = null;\n statusRow.outcome = \"failed\";\n statusRow.note = message;\n }\n }\n }\n }\n\n const finishedAt = new Date().toISOString();\n const summary: SiteRunSummary = {\n runId,\n host: site.host,\n sitemapUrl: site.sitemapUrl,\n startedAt,\n finishedAt,\n discoveredCount: rawEntries.length,\n filteredCount: filtered.entries.length,\n changedCount: changedEntries.length,\n queuedCount: queuedEntries.length,\n deferredCount,\n submittedCount: options.dryRun ? 0 : submittedCount,\n pendingCount: options.dryRun ? 0 : pendingCount,\n filteredUrls: filtered.entries.map((entry) => entry.url),\n changedUrls: changedEntries.map((entry) => entry.url),\n queuedUrls: queuedEntries.map((entry) => entry.url),\n dryRun: options.dryRun,\n force: options.force,\n endpoint: config.endpoint,\n filterStats: filtered.stats,\n chunkResults,\n errors,\n };\n\n await finalizeRunLog(runId, summary);\n const statusesToPersist = [...runUrlStatusByUrl.values(), ...syntheticRunStatuses]\n .sort((a, b) => a.slug.localeCompare(b.slug));\n await writeRunUrlStatuses(\n runId,\n site.host,\n statusesToPersist,\n );\n await pruneUrlSnapshots(site.host);\n return summary;\n}\n\nexport async function runIndexNow(\n options: IndexNowRunOptions = {},\n): Promise {\n const dryRun = options.dryRun ?? false;\n const force = options.force ?? false;\n const reason = options.reason ?? \"manual\";\n const startedAt = new Date().toISOString();\n\n await ensureSchema();\n const config = await loadRuntimeConfig();\n\n const sites = options.siteHost\n ? config.sites.filter((site) => site.host.toLowerCase() === options.siteHost?.toLowerCase())\n : config.sites;\n if (sites.length === 0) {\n throw new Error(`No site configured for host: ${options.siteHost}`);\n }\n\n const siteRuns: SiteRunSummary[] = [];\n for (const site of sites) {\n const summary = await runForSite(site, config, { dryRun, force, reason });\n siteRuns.push(summary);\n }\n\n const finishedAt = new Date().toISOString();\n return {\n startedAt,\n finishedAt,\n reason,\n dryRun,\n force,\n endpoint: config.endpoint,\n runUrlLimit: config.runUrlLimit,\n siteRuns,\n hasErrors: siteRuns.some((siteRun) => siteRun.errors.length > 0),\n };\n}\n\nexport async function backfillSnapshots(\n options: IndexNowBackfillOptions = {},\n): Promise {\n const dryRun = options.dryRun ?? true;\n const limitPerSite = normalizeBackfillLimit(options.limitPerSite);\n const startedAt = new Date().toISOString();\n\n await ensureSchema();\n const config = await loadRuntimeConfig();\n\n const sites = options.siteHost\n ? config.sites.filter((site) => site.host.toLowerCase() === options.siteHost?.toLowerCase())\n : config.sites;\n if (sites.length === 0) {\n throw new Error(`No site configured for host: ${options.siteHost}`);\n }\n\n const siteBackfills: SiteBackfillSummary[] = [];\n for (const site of sites) {\n siteBackfills.push(await backfillSnapshotsForSite(site.host, dryRun, limitPerSite));\n }\n\n const finishedAt = new Date().toISOString();\n return {\n startedAt,\n finishedAt,\n dryRun,\n limitPerSite,\n siteHost: options.siteHost,\n siteBackfills,\n hasErrors: siteBackfills.some((siteBackfill) => siteBackfill.errors.length > 0),\n };\n}\n","deleted","label",[177,158],"lib","permalink","https://www.val.town/x/ashryanio/index-now/code/lib/indexnow.ts","extension","actionData","errors"]