${escapeHtml(label)}

No snapshot available.

\n `;\n }\n\n return `\n

${escapeHtml(label)}

Captured: ${formatTimestamp(snapshot.capturedAt)}
Status: ${snapshotValue(snapshot.responseStatus)}
Content-Type: ${escapeHtml(snapshotValue(snapshot.contentType))}
ETag: ${escapeHtml(snapshotValue(snapshot.etag))}
Last-Modified: ${escapeHtml(snapshotValue(snapshot.responseLastModified))}
SHA-256: ${escapeHtml(snapshotValue(snapshot.htmlSha256))}
Size: ${escapeHtml(formatBytes(snapshot.htmlSizeBytes))}
Sitemap lastmod: ${escapeHtml(snapshotValue(snapshot.sitemapLastmod))}
Prior submitted lastmod: ${escapeHtml(snapshotValue(snapshot.previousSubmittedLastmod))}
Fetch error: ${escapeHtml(snapshotValue(snapshot.fetchError))}

\n `;\n}\n\nfunction renderRunUrlStatusPanel(\n site: SiteMonitor,\n monitor: IndexNowMonitorResponse,\n state: DashboardLinkState,\n): string {\n if (!site.selectedRun) {\n return \"\";\n }\n\n if (!monitor.tables.runUrlStatus) {\n return `\n

Per-run URL statuses will appear after the next run.

\n `;\n }\n\n if (site.selectedRunUrlStatuses.length === 0) {\n const detail = site.selectedRun.changedCount === 0\n ? \"Selected run had no changed URLs.\"\n : \"No URL statuses recorded for this run.\";\n return `\n

${escapeHtml(detail)}

\n `;\n }\n\n const showing = site.selectedRunUrlStatuses.length;\n const total = site.selectedRunUrlStatusTotal;\n const countLabel = total > showing\n ? `Showing ${showing} of ${total} URLs`\n : `Showing ${showing} URLs`;\n\n return `\n

Run URL Statuses

${escapeHtml(countLabel)}

\n \n \n \n \n \n \n \n \n \n \n \n ${site.selectedRunUrlStatuses.map((item) => `\n \n \n \n \n \n \n \n `).join(\"\")}\n \n

Slug	Monitor status	Reason	IndexNow response	Diff
${escapeHtml(item.slug)}	${renderFinalStatusTag(item)}	${escapeHtml(statusReason(item))}	${escapeHtml(indexNowResponse(item))}	\n \n ${monitor.selectedDiffUrl === item.url ? \"Hide diff\" : \"View diff\"}\n \n

\n `;\n}\n\nfunction renderRunDiffPanel(site: SiteMonitor, monitor: IndexNowMonitorResponse): string {\n if (!site.selectedRun || !monitor.selectedDiffUrl) {\n return \"\";\n }\n\n if (!monitor.tables.urlSnapshots) {\n return `\n

Snapshot table is not available yet. Run /run once to initialize it.

\n `;\n }\n\n if (!site.selectedRunDiff) {\n return `\n

No diff data found for this URL in the selected run.

\n `;\n }\n\n const diffText = site.selectedRunDiff.unifiedDiff\n ? `

${escapeHtml(site.selectedRunDiff.unifiedDiff)}

`\n : `

Diff will show here when there are two snapshots to compare.

`;\n\n return `\n

Raw HTML Diff

${escapeHtml(site.selectedRunDiff.url)}

\n ${renderSnapshotMetadata(\"Current Snapshot\", site.selectedRunDiff.currentSnapshot)}\n ${renderSnapshotMetadata(\"Previous Snapshot\", site.selectedRunDiff.previousSnapshot)}\n

\n ${diffText}\n

\n `;\n}\n\nfunction renderQuestionCards(items: QuestionAnswer[]): string {\n const toneEmoji = (tone: QuestionAnswer[\"tone\"]): string => {\n switch (tone) {\n case \"ok\":\n return \"🟢\";\n case \"warn\":\n return \"🟡\";\n default:\n return \"🔴\";\n }\n };\n\n return items.map((item) => `\n

${escapeHtml(item.label)}

${toneEmoji(item.tone)}

${escapeHtml(item.detail)}

\n `).join(\"\");\n}\n\nfunction buildSiteQuestions(site: SiteMonitor, issuesOnlyFilterActive: boolean): QuestionAnswer[] {\n const latestRealRun = latestNonDryRun(site);\n const backlog = latestRealRun ? latestRealRun.queuedCount : 0;\n const acceptedCount = latestRealRun ? latestRealRun.acceptedCount : 0;\n const submittedCount = latestRealRun ? latestRealRun.submittedCount : 0;\n const rejectedCount = latestRealRun ? latestRealRun.rejectedCount : 0;\n const zeroChangeSubmitted = latestRealRun\n ? latestRealRun.changedCount === 0 && submittedCount > 0\n : false;\n const hasInterventionIssues = site.totals.retryPendingUrls > 0 || Boolean(latestRealRun?.hasErrors);\n\n let processingAnswer: QuestionAnswer;\n if (!latestRealRun) {\n const hasRunHistory = site.totals.totalAttemptCount > 0 || site.totals.totalSubmitCount > 0;\n const noKnownIssues = site.totals.retryPendingUrls === 0 && site.totals.needsSubmissionUrls === 0;\n if (issuesOnlyFilterActive && hasRunHistory && noKnownIssues) {\n processingAnswer = {\n label: \"Run processing healthy\",\n detail: \"No issue runs in selected filter.\",\n tone: \"ok\",\n };\n } else {\n processingAnswer = {\n label: \"Run processing healthy\",\n detail: \"No non-dry run history available.\",\n tone: \"warn\",\n };\n }\n } else if (latestRealRun.hasErrors) {\n processingAnswer = {\n label: \"Run processing healthy\",\n detail: \"Latest non-dry run has errors.\",\n tone: \"error\",\n };\n } else if (submittedCount === 0 && backlog === 0) {\n processingAnswer = {\n label: \"Run processing healthy\",\n detail: \"No submissions were needed in the latest non-dry run.\",\n tone: \"ok\",\n };\n } else if (rejectedCount === 0) {\n const pendingSuffix = latestRealRun.pendingCount > 0\n ? ` (${latestRealRun.pendingCount} async 202)`\n : \"\";\n const backlogSuffix = backlog > 0 ? `; ${backlog} queued by run URL limit.` : \".\";\n processingAnswer = {\n label: \"Run processing healthy\",\n detail: `${acceptedCount}/${submittedCount} submitted URLs accepted${pendingSuffix}${backlogSuffix}`,\n tone: backlog > 0 ? \"warn\" : \"ok\",\n };\n } else {\n const backlogSuffix = backlog > 0 ? `; ${backlog} queued by run URL limit.` : \".\";\n processingAnswer = {\n label: \"Run processing healthy\",\n detail: `${acceptedCount}/${submittedCount} submitted URLs accepted; ${rejectedCount} rejected${backlogSuffix}`,\n tone: \"warn\",\n };\n }\n\n return [\n {\n label: \"Coverage is current\",\n detail: `${site.totals.upToDateUrls} up to date, ${site.totals.needsSubmissionUrls} need submission.`,\n tone: site.totals.needsSubmissionUrls === 0 ? \"ok\" : \"warn\",\n },\n {\n label: \"No intervention issues\",\n detail: `${site.totals.retryPendingUrls} retry-pending URLs; last status ${site.lastSubmissionStatus ?? \"n/a\"}.`,\n tone: hasInterventionIssues ? \"error\" : \"ok\",\n },\n processingAnswer,\n {\n label: \"No run-limit backlog\",\n detail: backlog > 0\n ? `${backlog} URLs deferred by run URL limit.`\n : \"No deferred URLs from latest non-dry run.\",\n tone: backlog > 0 ? \"warn\" : \"ok\",\n },\n {\n label: \"No zero-change submissions\",\n detail: zeroChangeSubmitted\n ? \"Latest non-dry run submitted URLs even though changedCount was 0.\"\n : \"No unnecessary submits detected on latest non-dry run.\",\n tone: zeroChangeSubmitted ? \"error\" : \"ok\",\n },\n ];\n}\n\nfunction renderSiteCard(site: SiteMonitor, linkState: DashboardLinkState, monitor: IndexNowMonitorResponse): string {\n const health = siteHealth(site);\n const healthClass = `pill ${health.tone}`;\n const questions = buildSiteQuestions(site, monitor.issuesOnly);\n\n return `\n

\n \n\n

\n ${renderQuestionCards(questions)}\n

\n\n

Run History

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ${renderRunTableRows(site, linkState)}\n \n

Finished	Entry Point	Scanned	Ignored	No change	Suppressed	Changed	Queued	Submitted	Accepted	Rejected

\n\n ${renderRunUrlStatusPanel(site, monitor, linkState)}\n ${renderRunDiffPanel(site, monitor)}\n

\n `;\n}\n\nfunction renderDashboard(\n monitor: IndexNowMonitorResponse,\n runtimeConfig: IndexNowRuntimeConfig,\n): string {\n const valProjectUrl = parseProject(import.meta.url).links.self.project;\n\n const baseState: DashboardLinkState = {\n selectedHost: monitor.selectedHost,\n runHistoryLimit: monitor.runHistoryLimit,\n selectedRunId: monitor.selectedRunId,\n selectedDiffUrl: monitor.selectedDiffUrl,\n issuesOnly: monitor.issuesOnly,\n updatesOnly: monitor.updatesOnly,\n includeDryRuns: monitor.includeDryRuns,\n };\n\n const filterLinks = [\n {\n label: \"All Sites\",\n href: buildDashboardLink({\n ...baseState,\n selectedHost: null,\n selectedRunId: null,\n selectedDiffUrl: null,\n }),\n active: monitor.selectedHost === null,\n },\n ...runtimeConfig.sites.map((site) => ({\n label: site.host,\n href: buildDashboardLink({\n ...baseState,\n selectedHost: site.host,\n selectedRunId: site.host.toLowerCase() === monitor.selectedHost?.toLowerCase()\n ? monitor.selectedRunId\n : null,\n selectedDiffUrl: site.host.toLowerCase() === monitor.selectedHost?.toLowerCase()\n ? monitor.selectedDiffUrl\n : null,\n }),\n active: site.host.toLowerCase() === monitor.selectedHost?.toLowerCase(),\n })),\n ];\n\n const runScopeLinks = [\n {\n label: \"Last 50 Runs\",\n href: buildDashboardLink({\n ...baseState,\n runHistoryLimit: 50,\n issuesOnly: false,\n updatesOnly: false,\n }),\n active: !monitor.issuesOnly && !monitor.updatesOnly && monitor.runHistoryLimit !== 500,\n },\n {\n label: \"Last 500 Runs\",\n href: buildDashboardLink({\n ...baseState,\n runHistoryLimit: 500,\n issuesOnly: false,\n updatesOnly: false,\n }),\n active: !monitor.issuesOnly && !monitor.updatesOnly && monitor.runHistoryLimit === 500,\n },\n {\n label: \"Runs with Issues\",\n href: buildDashboardLink({\n ...baseState,\n runHistoryLimit: 500,\n issuesOnly: true,\n updatesOnly: false,\n }),\n active: monitor.issuesOnly,\n },\n {\n label: \"Runs with Updates\",\n href: buildDashboardLink({\n ...baseState,\n runHistoryLimit: 500,\n issuesOnly: false,\n updatesOnly: true,\n }),\n active: monitor.updatesOnly,\n },\n ];\n\n return `\n\n \n \n \n IndexNow Monitor Dashboard\n \n \n \n

IndexNow Monitor

\n Open Val\n

\n\n

\n ${filterLinks.map((item) =>\n `${escapeHtml(item.label)}`\n ).join(\"\")}\n

\n ${runScopeLinks.map((item) =>\n `${item.label}`\n ).join(\"\")}\n

\n\n

\n ${monitor.sites.map((site) => renderSiteCard(site, baseState, monitor)).join(\"\")}\n

\n \n \n \n`;\n}\n\nasync function tableExists(tableName: string): Promise {\n const result = await sqlite.execute({\n sql: \"SELECT COUNT(*) AS table_count FROM sqlite_master WHERE type = 'table' AND name = ?\",\n args: [tableName],\n });\n const row = result.rows[0] as TableRow | undefined;\n return toNumber(row?.table_count) > 0;\n}\n\nfunction emptySiteTotals(): SiteMonitorTotals {\n return {\n everSubmitted200Urls: 0,\n upToDateUrls: 0,\n needsSubmissionUrls: 0,\n retryPendingUrls: 0,\n totalAttemptCount: 0,\n totalSubmitCount: 0,\n };\n}\n\nfunction isRootKeyLocation(keyLocation: URL): boolean {\n const path = keyLocation.pathname;\n const withoutLeadingSlash = path.startsWith(\"/\") ? path.slice(1) : path;\n return withoutLeadingSlash.length > 0 && !withoutLeadingSlash.includes(\"/\");\n}\n\nfunction isWithinKeyScope(pageUrl: URL, keyLocation: URL): boolean {\n if (isRootKeyLocation(keyLocation)) {\n return true;\n }\n const directoryPath = keyLocation.pathname.slice(\n 0,\n keyLocation.pathname.lastIndexOf(\"/\") + 1,\n );\n return pageUrl.pathname.startsWith(directoryPath);\n}\n\nfunction matchesPrefix(url: URL, prefix: string): boolean {\n if (prefix.startsWith(\"http://\") || prefix.startsWith(\"https://\")) {\n return url.toString().startsWith(prefix);\n }\n return url.pathname.startsWith(prefix);\n}\n\nfunction isUrlInTrackedScope(site: IndexNowSiteConfig, urlText: string): boolean {\n let pageUrl: URL;\n try {\n pageUrl = new URL(urlText);\n } catch {\n return false;\n }\n\n if (pageUrl.host.toLowerCase() !== site.host.toLowerCase()) {\n return false;\n }\n\n const keyLocation = new URL(site.keyLocation);\n if (!isWithinKeyScope(pageUrl, keyLocation)) {\n return false;\n }\n\n if (\n site.includePrefixes &&\n site.includePrefixes.length > 0 &&\n !site.includePrefixes.some((prefix) => matchesPrefix(pageUrl, prefix))\n ) {\n return false;\n }\n\n if (\n site.excludePrefixes &&\n site.excludePrefixes.length > 0 &&\n site.excludePrefixes.some((prefix) => matchesPrefix(pageUrl, prefix))\n ) {\n return false;\n }\n\n return true;\n}\n\nasync function loadSiteTotals(site: IndexNowSiteConfig): Promise<{\n totals: SiteMonitorTotals;\n lastAttemptedAt: string | null;\n lastSubmittedAt: string | null;\n lastSubmissionStatus: number | null;\n}> {\n const result = await sqlite.execute({\n sql: `SELECT\n url,\n lastmod,\n last_submitted_lastmod,\n last_submitted_at,\n last_submission_status,\n attempt_count,\n submit_count,\n last_attempted_at,\n last_seen_at\n FROM ${URL_STATE_TABLE}\n WHERE site_host = ?`,\n args: [site.host],\n });\n\n let everSubmitted200Urls = 0;\n let upToDateUrls = 0;\n let needsSubmissionUrls = 0;\n let retryPendingUrls = 0;\n let totalAttemptCount = 0;\n let totalSubmitCount = 0;\n let lastAttemptedAt: string | null = null;\n let lastSubmittedAt: string | null = null;\n let lastSubmissionStatus: number | null = null;\n let latestStatusAt: string | null = null;\n\n for (const rawRow of result.rows) {\n const row = rawRow as TableRow;\n const url = toNullableString(row.url);\n if (!url || !isUrlInTrackedScope(site, url)) {\n continue;\n }\n const lastmod = toNullableString(row.lastmod);\n const lastSubmittedLastmod = toNullableString(row.last_submitted_lastmod);\n const submittedAt = toNullableString(row.last_submitted_at);\n const attemptedAt = toNullableString(row.last_attempted_at);\n const lastSeenAt = toNullableString(row.last_seen_at);\n const status = toNumber(row.last_submission_status);\n const statusAt = attemptedAt ?? lastSeenAt;\n // Treat a fresh 202 as async-accepted for coverage/health calculations.\n const hasRecentAsyncAcceptance = status === 202 &&\n attemptedAt !== null &&\n (!lastSeenAt || attemptedAt >= lastSeenAt);\n const shouldUseAsyncAcceptance = hasRecentAsyncAcceptance &&\n (!submittedAt || attemptedAt > submittedAt);\n const effectiveSubmittedAt = shouldUseAsyncAcceptance ? attemptedAt : submittedAt;\n const effectiveSubmittedLastmod = shouldUseAsyncAcceptance\n ? (lastmod ?? lastSubmittedLastmod)\n : lastSubmittedLastmod;\n\n if (effectiveSubmittedAt !== null) {\n everSubmitted200Urls += 1;\n }\n\n if (effectiveSubmittedAt !== null && (lastmod === null || (effectiveSubmittedLastmod ?? \"\") === lastmod)) {\n upToDateUrls += 1;\n }\n\n const needsSubmission = effectiveSubmittedAt === null ||\n (lastmod !== null && (effectiveSubmittedLastmod ?? \"\") !== lastmod);\n if (needsSubmission) {\n needsSubmissionUrls += 1;\n if (status > 0 && status !== 200 && status !== 202) {\n retryPendingUrls += 1;\n }\n }\n\n totalAttemptCount += toNumber(row.attempt_count);\n totalSubmitCount += toNumber(row.submit_count);\n\n if (attemptedAt && (!lastAttemptedAt || attemptedAt > lastAttemptedAt)) {\n lastAttemptedAt = attemptedAt;\n }\n if (effectiveSubmittedAt && (!lastSubmittedAt || effectiveSubmittedAt > lastSubmittedAt)) {\n lastSubmittedAt = effectiveSubmittedAt;\n }\n if (status > 0 && statusAt && (!latestStatusAt || statusAt > latestStatusAt)) {\n latestStatusAt = statusAt;\n lastSubmissionStatus = status;\n }\n }\n\n return {\n totals: {\n everSubmitted200Urls,\n upToDateUrls,\n needsSubmissionUrls,\n retryPendingUrls,\n totalAttemptCount,\n totalSubmitCount,\n },\n lastAttemptedAt,\n lastSubmittedAt,\n lastSubmissionStatus,\n };\n}\n\nasync function loadRecentRuns(\n host: string,\n runHistoryLimit: number,\n issuesOnly: boolean,\n updatesOnly: boolean,\n includeDryRuns: boolean,\n hasRunUrlStatusTable: boolean,\n): Promise {\n const dryRunClause = includeDryRuns ? \"\" : \"AND run_log.dry_run = 0\";\n const issuesClause = issuesOnly\n ? hasRunUrlStatusTable\n ? `AND (\n run_log.has_errors = 1\n OR EXISTS (\n SELECT 1\n FROM ${RUN_URL_STATUS_TABLE} AS run_status\n WHERE run_status.site_host = run_log.site_host\n AND run_status.run_id = run_log.id\n AND run_status.outcome IN ('failed', 'deferred')\n )\n )`\n : \"AND (run_log.has_errors = 1 OR (run_log.dry_run = 0 AND run_log.queued_count > run_log.submitted_count))\"\n : \"\";\n const updatesClause = updatesOnly\n ? \"AND (run_log.changed_count > 0 OR run_log.queued_count > 0 OR run_log.submitted_count > 0)\"\n : \"\";\n\n const result = await sqlite.execute({\n sql: `SELECT\n run_log.id,\n run_log.reason,\n run_log.started_at,\n run_log.finished_at,\n run_log.discovered_count,\n run_log.eligible_count AS filtered_count,\n run_log.changed_count,\n run_log.queued_count,\n run_log.submitted_count,\n run_log.dry_run,\n run_log.force_mode,\n run_log.has_errors,\n run_log.error_summary\n FROM ${RUN_LOG_TABLE} AS run_log\n WHERE run_log.site_host = ?\n ${dryRunClause}\n ${issuesClause}\n ${updatesClause}\n ORDER BY run_log.id DESC\n LIMIT ?`,\n args: [host, runHistoryLimit],\n });\n\n const runs = result.rows.map((rawRow) => {\n const row = rawRow as TableRow;\n return {\n id: toNumber(row.id),\n reason: String(row.reason ?? \"\"),\n startedAt: String(row.started_at ?? \"\"),\n finishedAt: String(row.finished_at ?? \"\"),\n discoveredCount: toNumber(row.discovered_count),\n filteredCount: toNumber(row.filtered_count),\n changedCount: toNumber(row.changed_count),\n ignoredCount: 0,\n noChangeCount: 0,\n suppressedCount: 0,\n queuedCount: toNumber(row.queued_count),\n submittedCount: toNumber(row.submitted_count),\n acceptedCount: toNumber(row.submitted_count),\n rejectedCount: 0,\n pendingCount: 0,\n dryRun: toBooleanFromInt(row.dry_run),\n forceMode: toBooleanFromInt(row.force_mode),\n hasErrors: toBooleanFromInt(row.has_errors),\n errorSummary: toNullableString(row.error_summary),\n };\n });\n\n if (!hasRunUrlStatusTable || runs.length === 0) {\n for (const run of runs) {\n run.noChangeCount = Math.max(0, run.discoveredCount - run.changedCount);\n }\n return runs;\n }\n\n const runIds = runs.map((run) => run.id);\n const placeholders = runIds.map(() => \"?\").join(\", \");\n const outcomeCountsResult = await sqlite.execute({\n sql: `SELECT\n run_id,\n SUM(CASE WHEN outcome = 'deferred' THEN 1 ELSE 0 END) AS queued_count,\n SUM(CASE WHEN outcome IN ('submitted', 'pending', 'failed') THEN 1 ELSE 0 END) AS submitted_count,\n SUM(CASE WHEN outcome IN ('submitted', 'pending') THEN 1 ELSE 0 END) AS accepted_count,\n SUM(CASE WHEN outcome = 'failed' THEN 1 ELSE 0 END) AS rejected_count,\n SUM(CASE WHEN outcome = 'pending' THEN 1 ELSE 0 END) AS pending_count,\n SUM(CASE WHEN outcome = 'ignored' THEN 1 ELSE 0 END) AS ignored_count,\n SUM(CASE WHEN outcome = 'suppressed' THEN 1 ELSE 0 END) AS suppressed_count\n FROM ${RUN_URL_STATUS_TABLE}\n WHERE site_host = ?\n AND run_id IN (${placeholders})\n GROUP BY run_id`,\n args: [host, ...runIds],\n });\n\n const countsByRunId = new Map<\n number,\n {\n queued: number;\n submitted: number;\n accepted: number;\n rejected: number;\n pending: number;\n ignored: number;\n suppressed: number;\n }\n >();\n for (const rawRow of outcomeCountsResult.rows) {\n const row = rawRow as TableRow;\n countsByRunId.set(toNumber(row.run_id), {\n queued: toNumber(row.queued_count),\n submitted: toNumber(row.submitted_count),\n accepted: toNumber(row.accepted_count),\n rejected: toNumber(row.rejected_count),\n pending: toNumber(row.pending_count),\n ignored: toNumber(row.ignored_count),\n suppressed: toNumber(row.suppressed_count),\n });\n }\n\n for (const run of runs) {\n const counts = countsByRunId.get(run.id);\n run.queuedCount = counts?.queued ?? run.queuedCount;\n run.submittedCount = counts?.submitted ?? run.submittedCount;\n run.acceptedCount = counts?.accepted ?? run.acceptedCount;\n run.rejectedCount = counts?.rejected ?? run.rejectedCount;\n run.pendingCount = counts?.pending ?? 0;\n run.ignoredCount = counts?.ignored ?? 0;\n run.suppressedCount = counts?.suppressed ?? 0;\n run.noChangeCount = Math.max(\n 0,\n run.discoveredCount - run.ignoredCount - run.suppressedCount - run.changedCount,\n );\n }\n\n return runs;\n}\n\nasync function loadRunUrlStatuses(\n host: string,\n runId: number,\n): Promise<{ rows: RunUrlStatusItem[]; totalCount: number }> {\n const countResult = await sqlite.execute({\n sql: `SELECT COUNT(*) AS total_count\n FROM ${RUN_URL_STATUS_TABLE}\n WHERE site_host = ? AND run_id = ?`,\n args: [host, runId],\n });\n const countRow = countResult.rows[0] as TableRow | undefined;\n const totalCount = toNumber(countRow?.total_count);\n\n const rowsResult = await sqlite.execute({\n sql: `SELECT\n url,\n slug,\n changed,\n queued,\n attempted,\n submission_status,\n outcome,\n note\n FROM ${RUN_URL_STATUS_TABLE}\n WHERE site_host = ? AND run_id = ?\n ORDER BY\n CASE outcome\n WHEN 'failed' THEN 0\n WHEN 'deferred' THEN 1\n WHEN 'pending' THEN 2\n WHEN 'dry_run' THEN 3\n WHEN 'submitted' THEN 4\n WHEN 'suppressed' THEN 5\n WHEN 'ignored' THEN 6\n WHEN 'no_change' THEN 7\n ELSE 8\n END,\n slug ASC\n LIMIT ?`,\n args: [host, runId, SELECTED_RUN_URL_STATUS_LIMIT],\n });\n\n const rows = rowsResult.rows.map((rawRow) => {\n const row = rawRow as TableRow;\n const slug = String(row.slug ?? \"\");\n return {\n url: String(row.url ?? \"\"),\n slug: slug.length > 0 ? slug : String(row.url ?? \"\"),\n changed: toBooleanFromInt(row.changed),\n queued: toBooleanFromInt(row.queued),\n attempted: toBooleanFromInt(row.attempted),\n submissionStatus: toNumber(row.submission_status) || null,\n outcome: toRunUrlOutcome(row.outcome),\n note: toNullableString(row.note),\n } satisfies RunUrlStatusItem;\n });\n\n return { rows, totalCount };\n}\n\nfunction toLoadedUrlSnapshot(rawRow: TableRow | undefined): LoadedUrlSnapshot | null {\n if (!rawRow) {\n return null;\n }\n const id = toNumber(rawRow.id);\n if (id <= 0) {\n return null;\n }\n\n const htmlRawValue = rawRow.html_raw;\n const htmlRaw = typeof htmlRawValue === \"string\" ? htmlRawValue : null;\n const htmlSizeBytes = typeof htmlRawValue === \"string\"\n ? new TextEncoder().encode(htmlRawValue).length\n : null;\n const runIdValue = toNumber(rawRow.run_id);\n\n return {\n id,\n runId: runIdValue > 0 ? runIdValue : null,\n siteHost: String(rawRow.site_host ?? \"\"),\n url: String(rawRow.url ?? \"\"),\n slug: String(rawRow.slug ?? \"\"),\n capturedAt: String(rawRow.captured_at ?? \"\"),\n snapshotSource: String(rawRow.snapshot_source ?? \"\"),\n dryRun: toBooleanFromInt(rawRow.dry_run),\n responseStatus: toNumber(rawRow.response_status) || null,\n contentType: toNullableString(rawRow.content_type),\n etag: toNullableString(rawRow.etag),\n responseLastModified: toNullableString(rawRow.response_last_modified),\n htmlRaw,\n htmlSha256: toNullableString(rawRow.html_sha256),\n sitemapLastmod: toNullableString(rawRow.sitemap_lastmod),\n previousSubmittedLastmod: toNullableString(rawRow.previous_submitted_lastmod),\n fetchError: toNullableString(rawRow.fetch_error),\n htmlSizeBytes,\n };\n}\n\nasync function loadCurrentSnapshot(\n host: string,\n runId: number,\n url: string,\n): Promise {\n const result = await sqlite.execute({\n sql: `SELECT\n id,\n run_id,\n site_host,\n url,\n slug,\n captured_at,\n snapshot_source,\n dry_run,\n response_status,\n content_type,\n etag,\n response_last_modified,\n html_raw,\n html_sha256,\n sitemap_lastmod,\n previous_submitted_lastmod,\n fetch_error\n FROM ${URL_SNAPSHOT_TABLE}\n WHERE site_host = ? AND run_id = ? AND url = ?\n ORDER BY id DESC\n LIMIT 1`,\n args: [host, runId, url],\n });\n\n return toLoadedUrlSnapshot(result.rows[0] as TableRow | undefined);\n}\n\nasync function loadLatestSnapshot(\n host: string,\n url: string,\n): Promise {\n const result = await sqlite.execute({\n sql: `SELECT\n id,\n run_id,\n site_host,\n url,\n slug,\n captured_at,\n snapshot_source,\n dry_run,\n response_status,\n content_type,\n etag,\n response_last_modified,\n html_raw,\n html_sha256,\n sitemap_lastmod,\n previous_submitted_lastmod,\n fetch_error\n FROM ${URL_SNAPSHOT_TABLE}\n WHERE site_host = ? AND url = ?\n ORDER BY captured_at DESC, id DESC\n LIMIT 1`,\n args: [host, url],\n });\n\n return toLoadedUrlSnapshot(result.rows[0] as TableRow | undefined);\n}\n\nasync function loadPreviousSnapshot(\n host: string,\n url: string,\n current: LoadedUrlSnapshot,\n): Promise {\n const result = await sqlite.execute({\n sql: `SELECT\n id,\n run_id,\n site_host,\n url,\n slug,\n captured_at,\n snapshot_source,\n dry_run,\n response_status,\n content_type,\n etag,\n response_last_modified,\n html_raw,\n html_sha256,\n sitemap_lastmod,\n previous_submitted_lastmod,\n fetch_error\n FROM ${URL_SNAPSHOT_TABLE}\n WHERE site_host = ?\n AND url = ?\n AND (\n captured_at < ?\n OR (captured_at = ? AND id < ?)\n )\n ORDER BY captured_at DESC, id DESC\n LIMIT 1`,\n args: [host, url, current.capturedAt, current.capturedAt, current.id],\n });\n\n return toLoadedUrlSnapshot(result.rows[0] as TableRow | undefined);\n}\n\nfunction buildUnifiedRawHtmlDiff(\n previous: LoadedUrlSnapshot,\n current: LoadedUrlSnapshot,\n): string | null {\n if (previous.htmlRaw === null || current.htmlRaw === null) {\n return null;\n }\n return createTwoFilesPatch(\n `previous:${previous.slug || previous.url}`,\n `current:${current.slug || current.url}`,\n previous.htmlRaw,\n current.htmlRaw,\n previous.capturedAt,\n current.capturedAt,\n { context: Number.MAX_SAFE_INTEGER },\n );\n}\n\nfunction toSnapshotMetadata(snapshot: LoadedUrlSnapshot): UrlSnapshotMetadata {\n const { htmlRaw: _htmlRaw, ...metadata } = snapshot;\n return metadata;\n}\n\nasync function loadSelectedRunDiff(\n host: string,\n runId: number,\n diffUrl: string,\n): Promise {\n let detailHint: string | null = null;\n let current = await loadCurrentSnapshot(host, runId, diffUrl);\n if (!current) {\n current = await loadLatestSnapshot(host, diffUrl);\n if (current) {\n detailHint = \"No snapshot linked to selected run; showing latest available snapshot for this URL.\";\n }\n }\n\n if (!current) {\n return {\n url: diffUrl,\n currentSnapshot: null,\n previousSnapshot: null,\n unifiedDiff: null,\n detail: \"No snapshot exists for this URL in the selected run.\",\n };\n }\n\n const previous = await loadPreviousSnapshot(host, diffUrl, current);\n if (!previous) {\n return {\n url: diffUrl,\n currentSnapshot: toSnapshotMetadata(current),\n previousSnapshot: null,\n unifiedDiff: null,\n detail: detailHint\n ? `${detailHint} No previous snapshot exists for this URL yet.`\n : \"No previous snapshot exists for this URL yet.\",\n };\n }\n\n const unifiedDiff = buildUnifiedRawHtmlDiff(previous, current);\n if (!unifiedDiff) {\n return {\n url: diffUrl,\n currentSnapshot: toSnapshotMetadata(current),\n previousSnapshot: toSnapshotMetadata(previous),\n unifiedDiff: null,\n detail: detailHint\n ? `${detailHint} One or both snapshots are missing raw HTML content.`\n : \"One or both snapshots are missing raw HTML content.\",\n };\n }\n\n const hashUnchanged = current.htmlSha256 && previous.htmlSha256 && current.htmlSha256 === previous.htmlSha256;\n const detail = hashUnchanged\n ? detailHint\n ? `${detailHint} Raw HTML hash is unchanged; metadata differences may explain re-submission.`\n : \"Raw HTML hash is unchanged; metadata differences may explain re-submission.\"\n : detailHint;\n\n return {\n url: diffUrl,\n currentSnapshot: toSnapshotMetadata(current),\n previousSnapshot: toSnapshotMetadata(previous),\n unifiedDiff,\n detail,\n };\n}\n\nfunction sumTotals(sites: SiteMonitor[]): SiteMonitorTotals {\n return sites.reduce(\n (acc, site) => ({\n everSubmitted200Urls: acc.everSubmitted200Urls + site.totals.everSubmitted200Urls,\n upToDateUrls: acc.upToDateUrls + site.totals.upToDateUrls,\n needsSubmissionUrls: acc.needsSubmissionUrls + site.totals.needsSubmissionUrls,\n retryPendingUrls: acc.retryPendingUrls + site.totals.retryPendingUrls,\n totalAttemptCount: acc.totalAttemptCount + site.totals.totalAttemptCount,\n totalSubmitCount: acc.totalSubmitCount + site.totals.totalSubmitCount,\n }),\n emptySiteTotals(),\n );\n}\n\nfunction latestRunAt(sites: SiteMonitor[]): string | null {\n let latest: string | null = null;\n for (const site of sites) {\n const candidate = site.latestRunAt;\n if (!candidate) {\n continue;\n }\n if (!latest || candidate > latest) {\n latest = candidate;\n }\n }\n return latest;\n}\n\nasync function loadLatestRunAtForSite(host: string): Promise {\n const result = await sqlite.execute({\n sql: `SELECT\n COALESCE(finished_at, started_at) AS latest_run_at\n FROM ${RUN_LOG_TABLE}\n WHERE site_host = ?\n AND COALESCE(dry_run, 0) = 0\n ORDER BY id DESC\n LIMIT 1`,\n args: [host],\n });\n\n if (result.rows.length === 0) {\n return null;\n }\n\n const row = result.rows[0] as TableRow;\n return toNullableString(row.latest_run_at);\n}\n\nasync function buildSiteMonitor(\n site: IndexNowSiteConfig,\n runHistoryLimit: number,\n selectedRunId: number | null,\n selectedDiffUrl: string | null,\n issuesOnly: boolean,\n updatesOnly: boolean,\n includeDryRuns: boolean,\n hasUrlStateTable: boolean,\n hasRunLogTable: boolean,\n hasRunUrlStatusTable: boolean,\n hasUrlSnapshotTable: boolean,\n): Promise {\n const base: SiteMonitor = {\n host: site.host,\n sitemapUrl: site.sitemapUrl,\n keyLocation: site.keyLocation,\n latestRunAt: null,\n totals: emptySiteTotals(),\n lastAttemptedAt: null,\n lastSubmittedAt: null,\n lastSubmissionStatus: null,\n recentRuns: [],\n latestRun: null,\n selectedRun: null,\n selectedRunUrlStatuses: [],\n selectedRunUrlStatusTotal: 0,\n selectedRunDiff: null,\n };\n\n if (hasUrlStateTable) {\n const siteTotals = await loadSiteTotals(site);\n base.totals = siteTotals.totals;\n base.lastAttemptedAt = siteTotals.lastAttemptedAt;\n base.lastSubmittedAt = siteTotals.lastSubmittedAt;\n base.lastSubmissionStatus = siteTotals.lastSubmissionStatus;\n }\n\n if (hasRunLogTable) {\n base.latestRunAt = await loadLatestRunAtForSite(site.host);\n base.recentRuns = await loadRecentRuns(\n site.host,\n runHistoryLimit,\n issuesOnly,\n updatesOnly,\n includeDryRuns,\n hasRunUrlStatusTable,\n );\n base.latestRun = base.recentRuns.length > 0 ? base.recentRuns[0] : null;\n\n if (selectedRunId !== null) {\n base.selectedRun = base.recentRuns.find((run) => run.id === selectedRunId) ?? null;\n if (base.selectedRun && hasRunUrlStatusTable) {\n const details = await loadRunUrlStatuses(\n site.host,\n selectedRunId,\n );\n base.selectedRunUrlStatuses = details.rows;\n base.selectedRunUrlStatusTotal = details.totalCount;\n }\n\n if (base.selectedRun && selectedDiffUrl && hasUrlSnapshotTable) {\n base.selectedRunDiff = await loadSelectedRunDiff(\n site.host,\n selectedRunId,\n selectedDiffUrl,\n );\n }\n }\n }\n\n return base;\n}\n\nasync function buildMonitor(\n runtimeConfig: IndexNowRuntimeConfig,\n selectedHost: string | null,\n runHistoryLimit: number,\n selectedRunId: number | null,\n selectedDiffUrl: string | null,\n issuesOnly: boolean,\n updatesOnly: boolean,\n includeDryRuns: boolean,\n): Promise {\n const sites = selectedHost\n ? runtimeConfig.sites.filter((site) => site.host.toLowerCase() === selectedHost.toLowerCase())\n : runtimeConfig.sites;\n\n if (sites.length === 0) {\n throw new Error(`No configured site matches host: ${selectedHost}`);\n }\n\n const hasUrlStateTable = await tableExists(URL_STATE_TABLE);\n const hasRunLogTable = await tableExists(RUN_LOG_TABLE);\n const hasRunUrlStatusTable = await tableExists(RUN_URL_STATUS_TABLE);\n const hasUrlSnapshotTable = await tableExists(URL_SNAPSHOT_TABLE);\n const effectiveSelectedRunId = selectedRunId;\n\n const siteMonitors: SiteMonitor[] = [];\n for (const site of sites) {\n siteMonitors.push(\n await buildSiteMonitor(\n site,\n runHistoryLimit,\n effectiveSelectedRunId,\n selectedDiffUrl,\n issuesOnly,\n updatesOnly,\n includeDryRuns,\n hasUrlStateTable,\n hasRunLogTable,\n hasRunUrlStatusTable,\n hasUrlSnapshotTable,\n ),\n );\n }\n\n return {\n generatedAt: new Date().toISOString(),\n endpoint: runtimeConfig.endpoint,\n runUrlLimit: runtimeConfig.runUrlLimit,\n runHistoryLimit,\n issuesOnly,\n updatesOnly,\n includeDryRuns,\n selectedHost,\n selectedRunId: effectiveSelectedRunId,\n selectedDiffUrl,\n siteHosts: runtimeConfig.sites.map((site) => site.host),\n tables: {\n urlState: hasUrlStateTable,\n runLog: hasRunLogTable,\n runUrlStatus: hasRunUrlStatusTable,\n urlSnapshots: hasUrlSnapshotTable,\n },\n totals: sumTotals(siteMonitors),\n latestRunAt: latestRunAt(siteMonitors),\n sites: siteMonitors,\n };\n}\n\nexport async function loadMonitorSnapshotFromUrl(requestUrl: URL): Promise {\n const query = parseMonitorQuery(requestUrl.searchParams);\n const runtimeConfig = await loadRuntimeConfig();\n return buildMonitor(\n runtimeConfig,\n query.selectedHost,\n query.runHistoryLimit,\n query.selectedRunId,\n query.selectedDiffUrl,\n query.issuesOnly,\n query.updatesOnly,\n query.includeDryRuns,\n );\n}\n\nfunction monitorJsonResponse(monitor: IndexNowMonitorResponse): Response {\n return new Response(JSON.stringify(monitor, null, 2), {\n status: 200,\n headers: {\n \"content-type\": \"application/json; charset=utf-8\",\n \"cache-control\": \"no-store\",\n },\n });\n}\n\nexport default async function (req: Request) {\n const requestUrl = new URL(req.url);\n const monitor = await loadMonitorSnapshotFromUrl(requestUrl);\n\n return monitorJsonResponse(monitor);\n}\n","deleted","label",[100],"permalink","https://www.val.town/x/ashryanio/index-now/code/indexnow.monitor.ts","extension","actionData","errors"]