Fork
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import { PuppeteerDeno } from "https://deno.land/x/puppeteer@16.2.0/src/deno/Puppeteer.ts";
import process from "node:process";
export async function extractTwitterContent(req) {
try {
console.log("Starting to extract Twitter content...");
const url = new URL(req.url);
const twurl = decodeURIComponent(url.searchParams.get("twurl") || "");
if (twurl.length == 0) { return new Response("Error: No twitter url provided", { status: 400 }); }
console.log(`Received twitter URL: ${twurl}`);
// convert twurl to threadurl via regex
const postURLRegex = /https:\/\/twitter\.com\/(.*)\/status\/(\d+)/;
const matches = twurl.match(postURLRegex);
console.log("Matching URL against Twitter post URL regex...");
let threadurl = "";
if (matches) {
const authorHandle = matches[1];
const postID = matches[2];
threadurl = `https://twitter.com/${authorHandle}/thread/${postID}`;
console.log(`Thread URL constructed: ${threadurl}`);
} else {
threadurl = twurl;
console.log("No match found, using original URL.");
}
const puppeteer = new PuppeteerDeno({ productName: "chrome" });
console.log("Connecting to browser...");
const browser = await puppeteer.connect({
browserWSEndpoint: `wss://chrome.browserless.io?token=${process.env.browserlessKey}`,
defaultViewport: null,
args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"],
});
const page = await browser.newPage();
console.log(`Navigating to thread URL: ${threadurl}`);
await page.goto(threadurl, { waitUntil: "domcontentloaded" }).catch(e => {
console.error("Navigation to thread URL failed:", e);
browser.close();
return new Response("Error: Navigation to Twitter thread URL timed out", { status: 504 });
});
await page.screenshot({ path: "threadurl-screenshot.png" });
// Check if login required
const loginSelector = "div[data-testid=\"loginButton\"]";
console.log("Checking if login is required...");
const login = await page.$(loginSelector);
let status = "";
if (login) {
👆 This is a val. Vals are TypeScript snippets of code, written in the browser and run on our servers. Create scheduled functions, email yourself, and persist small pieces of data — all from the browser.
Comments
stevekrouse avatar

This is awesome! What is it for?

stevekrouse avatar

I tried it and it timed out. I think you may need to upgrade to Val Town Pro to run vals longer than a minute for this