wrapping an\n * with:\n * - h3.card-block__title → name\n * - img.card-block__img-animal → photo (src) + alt\n * - span.card-block__label (×N) → \"Female / Spayed\", breed, age\n * - .card-block__location → adoption location\n */\nexport function parseCats(html: string, baseUrl: string): Cat[] {\n const $ = load(html);\n const cats: Cat[] = [];\n\n $(\"article.card-block__item\").each((_, el) => {\n const card = $(el);\n\n const name = clean(card.find(\".card-block__title\").first().text());\n if (!name) return; // skip non-cat / malformed cards\n\n // The location lives in its own block; grab it first, then exclude it\n // from the generic label list so we don't double-count it.\n const location = clean(card.find(\".card-block__location .card-block__label\").first().text());\n\n // Collect the descriptive labels (sex/status, breed, age) that are NOT\n // inside the location block.\n const labels: string[] = [];\n card.find(\".card-block__main-info .card-block__label\").each((_, l) => {\n const text = clean($(l).text());\n if (text) labels.push(text);\n });\n\n // labels[0] = \"Female / Spayed\", labels[1] = breed, labels[2] = age\n const sexStatus = labels[0] ?? null;\n let sex: string | null = null;\n let spayNeuterStatus: string | null = null;\n if (sexStatus) {\n const [s, status] = sexStatus.split(\"/\").map((x) => x.trim());\n sex = s || null;\n spayNeuterStatus = status || null;\n }\n\n const breed = labels[1] ?? null;\n const age = labels[2] ?? null;\n\n const img = card.find(\"img.card-block__img-animal\").first();\n const rawImg = img.attr(\"src\") ?? null;\n const rawHref = card.find(\"a.card-block__link\").first().attr(\"href\") ?? null;\n\n cats.push({\n name,\n sex,\n spayNeuterStatus,\n breed,\n age,\n location,\n imageUrl: absoluteUrl(rawImg, baseUrl),\n detailUrl: absoluteUrl(rawHref, baseUrl),\n });\n });\n\n return cats;\n}\n\n/** Resolve a possibly-relative URL against the page it came from. */\nfunction absoluteUrl(href: string | null, baseUrl: string): string | null {\n if (!href) return null;\n try {\n return new URL(href, baseUrl).toString();\n } catch {\n return href;\n }\n}\n","createdAt",["D",1780037313706],"branchVersion","branchId","10ca37c0-5b2a-11f1-929f-ee650bb23af1","id","10c99d2e-5b2a-11f1-929f-ee650bb23af1","cat-adoption-scraper",["D",1780037165359],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Scrapes a cat adoption site and returns cats as JSON","imageUrl","15983d2d-b0ea-4ce4-ad7e-b94db72798ae","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","4d411bc9-053e-4a76-9ce5-e64fb3988323","tursoName","10c99d2e5b2a11f1929fee650bb23af1","tursoHostname","10c99d2e5b2a11f1929fee650bb23af1-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789524079207],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_104":103,"_111":107},"org",{"_104":103,"_105":-5,"_106":107,"_19":61,"_108":109,"_110":31},"branch",{"_13":93,"_38":94,"_49":48,"_95":50,"_96":-5,"_97":-5,"_98":-5,"_99":-5,"_100":-5,"_101":51,"_102":103},"isOwned",31,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","matthewtole","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ycUdrd3hwalg1NnBZTEJ3eGhHVWpudjZmSGEifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_49":50,"_38":51,"_44":207,"_53":-5,"_54":31,"_55":56,"_57":56,"_58":59,"_60":-5,"_19":61,"_62":-5,"_63":31,"_64":-5,"_65":-5,"_66":67,"_68":69,"_70":71,"_72":73,"_74":208,"_76":-5,"_77":78,"_79":78,"_80":81,"_82":81,"_83":81,"_84":78,"_85":78,"_86":209,"_88":210,"_90":211,"_92":31},"tree",[186,187,188,189,190],"files",[186,187,188,189,190],"counts",{"_176":67,"_177":152,"_178":67,"_179":180,"_181":182},"contents",[135,136,137,138,139],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles",true,"grants","orgs","customDomains",{},{"_49":169,"_38":170,"_36":171,"_143":-5,"_47":48,"_46":172,"_145":31,"_44":173,"_13":67,"_39":26,"_41":174,"_30":31,"_148":-5,"_33":175},{"_49":162,"_38":163,"_36":164,"_143":-5,"_47":48,"_46":165,"_145":31,"_44":166,"_13":152,"_39":40,"_41":167,"_30":31,"_148":-5,"_33":168},{"_49":154,"_38":155,"_36":156,"_143":-5,"_47":48,"_46":157,"_145":31,"_44":158,"_13":152,"_39":159,"_41":160,"_30":31,"_148":-5,"_33":161},{"_49":150,"_38":34,"_36":37,"_143":-5,"_47":48,"_46":14,"_145":31,"_44":151,"_13":152,"_39":40,"_41":43,"_30":31,"_148":-5,"_33":153},{"_49":140,"_38":141,"_36":142,"_143":-5,"_47":48,"_46":144,"_145":31,"_44":146,"_13":67,"_39":40,"_41":147,"_30":31,"_148":-5,"_33":149},"2b062f0e-5b2a-11f1-9c8c-ee650bb23af1","test.ts","2b06301c-5b2a-11f1-9c8c-ee650bb23af1","parentId",4,"latest",["D",1780037209370],"import { parseCats } from \"./scraper.ts\";\n\n// Sample HTML matching the placeholder selectors in scraper.ts.\nconst sampleHtml = `\n\n

Whiskers

\n 2 years\n Tabby\n Female\n Medium\n

A cuddly lap cat.

Mittens

\n 5 months\n Domestic Shorthair\n Male\n

\n\n`;\n\nconst cats = parseCats(sampleHtml, \"https://example.com/adopt/cats\");\nconsole.log(JSON.stringify(cats, null, 2));\nconsole.log(`Parsed ${cats.length} cats`);\n","label",[141],"693771ca-5b2a-11f1-a50e-ee650bb23af1",["D",1780037313706],1,[34],"1f36a898-5b2a-11f1-a0c2-ee650bb23af1","main.ts","10cd18e6-5b2a-11f1-929f-ee650bb23af1",3,["D",1780037189544],"http","import { LISTING_URL, scrapeCats } from \"./scraper.ts\";\n\n// In-memory cache so repeated requests don't re-scrape the source on every hit.\nconst CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes\nlet cache: { at: number; data: unknown } | null = null;\n\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n\n // Allow overriding the target URL via ?url=… and bypassing cache via ?fresh=1\n const target = url.searchParams.get(\"url\") ?? LISTING_URL;\n const fresh = url.searchParams.get(\"fresh\") === \"1\";\n const usingCustomUrl = url.searchParams.has(\"url\");\n\n // Only the default URL is cached; custom URLs always scrape fresh.\n if (!fresh && !usingCustomUrl && cache && Date.now() - cache.at < CACHE_TTL_MS) {\n return Response.json(cache.data, {\n headers: { \"X-Cache\": \"HIT\" },\n });\n }\n\n try {\n const cats = await scrapeCats(target);\n const payload = {\n source: target,\n scrapedAt: new Date().toISOString(),\n count: cats.length,\n cats,\n };\n\n if (!usingCustomUrl) cache = { at: Date.now(), data: payload };\n\n return Response.json(payload, { headers: { \"X-Cache\": \"MISS\" } });\n } catch (err) {\n return Response.json(\n { error: (err as Error).message, source: target },\n { status: 502 },\n );\n }\n}\n",[155],"5a50c486-5b2a-11f1-81a6-ee650bb23af1","inspect.ts","484ccd5c-5b2a-11f1-a50e-ee650bb23af1",7,["D",1780037288695],"import { load } from \"npm:cheerio\";\n\nconst USER_AGENT =\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \" +\n \"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36\";\n\nconst res = await fetch(\"https://www.paws.org/adopt/cats/\", {\n headers: { \"User-Agent\": USER_AGENT },\n});\nconst html = await res.text();\nconst $ = load(html);\n\n// \"Bahama\" is the first cat. Find the element containing that text and walk up\n// to find the repeating card container.\nconst nameNode = $(\"*\").filter((_, el) => {\n const own = $(el).clone().children().remove().end().text().trim();\n return own === \"Bahama\";\n}).first();\n\nconsole.log(\"Name node tag:\", nameNode.prop(\"tagName\"), \"class:\", nameNode.attr(\"class\"));\n\nlet node = nameNode;\nfor (let i = 0; i < 5; i++) {\n const parent = node.parent();\n console.log(\n `parent[${i}] tag=${parent.prop(\"tagName\")} class=\"${parent.attr(\"class\")}\" href=\"${parent.attr(\"href\") ?? \"\"}\"`,\n );\n node = parent;\n}\n\n// Print the card HTML (3 levels up is usually the card).\nconst card = nameNode.parent().parent().parent();\nconsole.log(\"=== CARD HTML ===\");\nconsole.log($.html(card).slice(0, 1500));\n",[163],"35b6da3e-5b2a-11f1-894a-ee650bb23af1","README.md","35b6db10-5b2a-11f1-894a-ee650bb23af1",5,["D",1780037227303],"# 🐱 Cat Adoption Scraper\n\nScrapes a cat adoption / rescue website and returns all the cats as a JSON list.\n\n## Endpoint\n\n`GET /` returns:\n\n```json\n{\n \"source\": \"https://example.com/adopt/cats\",\n \"scrapedAt\": \"2026-05-29T06:46:00.000Z\",\n \"count\": 2,\n \"cats\": [\n {\n \"name\": \"Whiskers\",\n \"age\": \"2 years\",\n \"breed\": \"Tabby\",\n \"sex\": \"Female\",\n \"size\": \"Medium\",\n \"description\": \"A cuddly lap cat.\",\n \"imageUrl\": \"https://example.com/img/whiskers.jpg\",\n \"detailUrl\": \"https://example.com/cats/whiskers\"\n }\n ]\n}\n```\n\n### Query params\n\n| Param | Effect |\n| -------- | ------------------------------------------------------------- |\n| `?fresh=1` | Bypass the 5-minute cache and re-scrape immediately. |\n| `?url=…` | Scrape a different listing URL (always uncached). |\n\n## ⚠️ Customize it for your site\n\nThis is scaffolded against a placeholder site. To point it at a real one, edit\n**`scraper.ts`**:\n\n1. Set `LISTING_URL` to the adoption listing page you want to scrape.\n2. Update the CSS selectors in `parseCats` to match that site's HTML. Open the\n page in your browser, right-click a cat card → **Inspect**, and copy the\n class names for the card, name, age, breed, etc.\n\nRun `test.ts` to validate your parser against sample HTML without hitting the\nnetwork.\n\n## How it works\n\n```mermaid\nflowchart LR\n A[GET /] --> B{cached?}\n B -- yes --> C[return cached JSON]\n B -- no --> D[fetch listing HTML]\n D --> E[cheerio parse pet cards]\n E --> F[map to Cat objects]\n F --> G[cache + return JSON]\n```\n\n## Files\n\n- `main.ts` — HTTP endpoint, caching, query params\n- `scraper.ts` — fetch + cheerio parsing (**the part you customize**)\n- `test.ts` — offline parser test against sample HTML\n\n## Note on JS-heavy / bot-protected sites\n\nBig aggregators (Petfinder, Adopt-a-Pet) render listings client-side and/or\nblock bots, so a plain `fetch` may return nothing useful. For those, use their\nofficial API or a headless-browser service (Browserbase, Kernel, Steel). Plain\n`cheerio` scraping works great for most individual shelter/rescue sites, which\nserve server-rendered HTML.\n",[170],"pulls","branches","forks","envVars",{"_183":67,"_185":67},"connections",{"_183":67,"_184":67},"count","unfilledCount","emptyCount",{"_49":169,"_38":170,"_36":171,"_143":-5,"_47":48,"_46":172,"_145":31,"_44":204,"_13":67,"_39":26,"_41":174,"_30":31,"_148":-5,"_192":205,"_33":206},{"_49":162,"_38":163,"_36":164,"_143":-5,"_47":48,"_46":165,"_145":31,"_44":201,"_13":152,"_39":40,"_41":167,"_30":31,"_148":-5,"_192":202,"_33":203},{"_49":154,"_38":155,"_36":156,"_143":-5,"_47":48,"_46":157,"_145":31,"_44":198,"_13":152,"_39":159,"_41":160,"_30":31,"_148":-5,"_192":199,"_33":200},{"_49":150,"_38":34,"_36":37,"_143":-5,"_47":48,"_46":14,"_145":31,"_44":195,"_13":152,"_39":40,"_41":43,"_30":31,"_148":-5,"_192":196,"_33":197},{"_49":140,"_38":141,"_36":142,"_143":-5,"_47":48,"_46":144,"_145":31,"_44":191,"_13":67,"_39":40,"_41":147,"_30":31,"_148":-5,"_192":193,"_33":194},["D",1780037209370],"children",[],[141],["D",1780037313706],[],[34],["D",1780037189544],[],[155],["D",1780037288695],[],[163],["D",1780037227303],[],[170],["D",1780037165359],["D",1789524079207],{"_104":103,"_111":107},{"_104":103,"_105":-5,"_106":107,"_19":61,"_108":109,"_110":31},{"_13":93,"_38":94,"_49":48,"_95":50,"_96":-5,"_97":-5,"_98":-5,"_99":-5,"_100":-5,"_101":51,"_102":103},"ctx",{"_86":-5,"_132":245,"_246":247,"_248":249,"_250":31,"_251":31,"_252":31,"_253":130,"_254":-5,"_255":-7,"_256":-7,"_257":67,"_258":31,"_259":31,"_260":-5,"_261":31,"_262":31,"_263":31,"_264":-5},"ENV",{"_216":217,"_218":219,"_220":94,"_221":222,"_223":224,"_225":226,"_227":228,"_229":230,"_231":232,"_233":234,"_235":236,"_237":238,"_239":240,"_241":242,"_243":244},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","fdb67791413f4386b4dbf18f354029a617f6160f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v8