/** @jsxImportSource https://esm.sh/react */ import cheerio from "https://esm.sh/cheerio@1.0.0-rc.12"; import dagre from "https://esm.sh/cytoscape-dagre@2.5.0"; import cytoscape from "https://esm.sh/cytoscape@3.23.0"; import React, { useEffect, useRef, useState } from "https://esm.sh/react"; import { createRoot } from "https://esm.sh/react-dom/client"; cytoscape.use(dagre); const TITLE_SELECTOR = "#firstHeading > span"; function labelPredicate(label) { return label === "Father" || label === "Mother" || /^Parent/.test(label) || label === "Children" || /^Child/.test(label); } async function* crawlWikipedia( url: string, generation: number = 0, visited: Set = new Set(), ): AsyncGenerator<[string, string, string[], number], void, unknown> { const isDescendant = generation < 0; if (Math.abs(generation) > 10) return; // Cap at 10 generations in either direction if (visited.has(url)) { return; } visited.add(url); const response = await fetch(url); const html = await response.text(); const $ = cheerio.load(html); const title = $(TITLE_SELECTOR).text().trim(); const familyLinks: string[] = []; $("table.infobox.vcard > tbody > tr").each((_, row) => { const $row = $(row); const label = $row.find("th.infobox-label").text().trim(); if (labelPredicate(label)) { const links = $row.find("td a:not(.extiw)").map((_, el) => $(el).attr("href")).get(); familyLinks.push(...links); } }); yield [url, title, familyLinks, generation]; for (const link of familyLinks) { const nextUrl = new URL(link, url).toString(); if (isDescendant) { yield* crawlWikipedia(nextUrl, generation - 1, visited); } else { yield* crawlWikipedia(nextUrl, generation + 1, visited); } } // Crawl descendants for the initial person if (generation === 0) { yield* crawlWikipedia(url, -1, new Set()); } } function App() { const [url, setUrl] = useState(""); const [nodes, setNodes] = useState([]); const [edges, setEdges] = useState([]); const cyRef = useRef(null); const [isLoading, setIsLoading] = useState(false); const [isCytoscapeDisabled, setIsCytoscapeDisabled] = useState(false); const [treeDepth, setTreeDepth] = useState({ ancestors: 0, descendants: 0 }); const [maxGenerations, setMaxGenerations] = useState(20); const handleSubmit = async (e, submittedUrl = null) => { if (e) e.preventDefault(); setIsLoading(true); const urlToSubmit = submittedUrl || url; const encodedUrl = encodeURIComponent(urlToSubmit); setNodes([]); setIsCytoscapeDisabled(true); setEdges([]); setTreeDepth({ ancestors: 0, descendants: 0 }); const eventSource = new EventSource(`/stream?url=${encodedUrl}`); const existingNodes = new Set(); eventSource.onmessage = (event) => { const [nodeUrl, name, parentLinks, generation] = JSON.parse(event.data); if (!existingNodes.has(nodeUrl)) { setNodes(prevNodes => [...prevNodes, { data: { id: nodeUrl, label: name } }]); existingNodes.add(nodeUrl); } if (treeDepth.ancestors + treeDepth.descendants >= maxGenerations) { eventSource.close(); } setTreeDepth(prevDepth => ({ ancestors: generation >= 0 ? Math.max(prevDepth.ancestors, generation) : prevDepth.ancestors, descendants: generation < 0 ? Math.max(prevDepth.descendants, Math.abs(generation)) : prevDepth.descendants, })); const isDescendant = generation < 0; parentLinks.forEach((link) => { const parentUrl = new URL(link, nodeUrl).toString(); if (!existingNodes.has(parentUrl)) { const parentName = decodeURIComponent(link.split("/").pop()?.replace(/_/g, " ") || ""); setNodes(prevNodes => [...prevNodes, { data: { id: parentUrl, label: parentName } }]); existingNodes.add(parentUrl); } setEdges( prevEdges => [...prevEdges, { data: { source: parentUrl, target: nodeUrl, id: `${parentUrl}-${nodeUrl}` } }], ); }); }; eventSource.onerror = () => { eventSource.close(); setIsLoading(false); setIsCytoscapeDisabled(false); }; }; useEffect(() => { if (cyRef.current) { cyRef.current.destroy(); } cyRef.current = cytoscape({ container: document.getElementById("cy"), elements: { nodes, edges }, userZoomingEnabled: !isCytoscapeDisabled, userPanningEnabled: !isCytoscapeDisabled, style: cytoscapeStyle, layout: { name: "dagre", rankDir: "TB", align: "UL", ranker: "network-simplex", nodeSep: 50, rankSep: 100, padding: 30, animate: false, fit: true, }, }); cyRef.current.on("tap", "node", function(evt) { if (isCytoscapeDisabled) return; const node = evt.target; window.open(node.id(), "_blank"); }); cyRef.current.on("layoutstop", function() { cyRef.current.center(); cyRef.current.fit(); }); }, [nodes, edges, isCytoscapeDisabled]); useEffect(() => { if (cyRef.current) { cyRef.current.resize(); } }, [treeDepth]); return (

Wikipedia Family Tree Crawler

Try these examples:

{[ "https://en.wikipedia.org/wiki/Martin_Luther_King_Jr.", "https://en.wikipedia.org/wiki/Barack_Obama", "https://en.wikipedia.org/wiki/John_F._Kennedy", ].map((exampleUrl) => ( ))}

{isLoading && (

)}

Ancestor Generations: {treeDepth.ancestors}
Descendant Generations: {treeDepth.descendants}
Total Generations: {treeDepth.ancestors + treeDepth.descendants}

View source

); } function client() { createRoot(document.getElementById("root")).render(); } if (typeof document !== "undefined") { client(); } async function server(req: Request): Promise { const url = new URL(req.url); if (url.pathname === "/stream") { const wikiUrl = url.searchParams.get("url"); if (!wikiUrl) { return new Response("URL parameter is required", { status: 400 }); } const encoder = new TextEncoder(); const stream = new ReadableStream({ async start(controller) { for await (const data of crawlWikipedia(wikiUrl)) { controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)); } controller.close(); }, }); return new Response(stream, { headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", "Connection": "keep-alive", }, }); } return new Response( ` Wikipedia Family Tree Crawler

`, { headers: { "Content-Type": "text/html" }, }, ); } const css = ` body, html { margin: 0; padding: 0; height: 100%; font-family: Arial, sans-serif; } #root { height: 100%; } h1 { margin-top: 0; } form { margin-bottom: 10px; } input { padding: 5px; font-size: 16px; } button { padding: 5px 10px; font-size: 16px; background-color: #4CAF50; color: white; border: none; cursor: pointer; } button:disabled { background-color: #cccccc; } button:hover { background-color: #45a049; } .spinner { border: 4px solid #f3f3f3; border-top: 4px solid #3498db; border-radius: 50%; width: 40px; height: 40px; animation: spin 1s linear infinite; } @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } } `; const cytoscapeStyle = [ { selector: "node", style: { "background-color": "#666", "label": "data(label)", "color": "#fff", "text-valign": "center", "text-halign": "center", "text-wrap": "wrap", "text-max-width": "100px", "font-size": "10px", "width": "label", "height": "label", "padding": "10px", }, }, { selector: "edge", style: { "width": 2, "line-color": "#ccc", "target-arrow-color": "#ccc", "curve-style": "bezier", }, }, ]; export default server;

infoboxCrawler