1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import { PuppeteerDeno } from "https://deno.land/x/puppeteer@16.2.0/src/deno/Puppeteer.ts";
import process from "node:process";
export async function extractTwitterContent(req) {
try {
console.log("Starting to extract Twitter content...");
const url = new URL(req.url);
const twurl = decodeURIComponent(url.searchParams.get("twurl") || "");
if (twurl.length == 0) { return new Response("Error: No twitter url provided", { status: 400 }); }
console.log(`Received twitter URL: ${twurl}`);
// convert twurl to threadurl via regex
const postURLRegex = /https:\/\/twitter\.com\/(.*)\/status\/(\d+)/;
const matches = twurl.match(postURLRegex);
console.log("Matching URL against Twitter post URL regex...");
let threadurl = "";
if (matches) {
const authorHandle = matches[1];
const postID = matches[2];
threadurl = `https://twitter.com/${authorHandle}/thread/${postID}`;
console.log(`Thread URL constructed: ${threadurl}`);
} else {
threadurl = twurl;
console.log("No match found, using original URL.");
}
const puppeteer = new PuppeteerDeno({ productName: "chrome" });
console.log("Connecting to browser...");
const browser = await puppeteer.connect({
browserWSEndpoint: `wss://chrome.browserless.io?token=${process.env.browserlessKey}`,
defaultViewport: null,
args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"],
});
const page = await browser.newPage();
console.log(`Navigating to thread URL: ${threadurl}`);
await page.goto(threadurl, { waitUntil: "domcontentloaded" }).catch(e => {
console.error("Navigation to thread URL failed:", e);
browser.close();
return new Response("Error: Navigation to Twitter thread URL timed out", { status: 504 });
});
await page.screenshot({ path: "threadurl-screenshot.png" });
// Check if login required
const loginSelector = "div[data-testid=\"loginButton\"]";
console.log("Checking if login is required...");
const login = await page.$(loginSelector);
let status = "";
if (login) {
status = "login_required";
console.log("Login is required.");
} else {
status = "ok";
console.log("No login required.");
}
if (status == "login_required") {
console.log("Attempting to log in...");
await page.goto("https://twitter.com/i/flow/login", { waitUntil: "domcontentloaded" }).catch(e => {
console.error("Navigation to login page failed:", e);
browser.close();
return new Response("Error: Navigation to Twitter login page timed out", { status: 504 });
});
// if login required, click the login button which has data-testid="loginButton"
const loginButton = "div[data-testid=\"loginButton\"]";
await page.click(loginButton).catch(e => {
console.error("Clicking login button failed:", e);
browser.close();
return new Response("Error: Clicking Twitter login button failed", { status: 500 });
});
console.log("Clicked on login button.");
// wait for the login form to appear, input with autocomplete="username" and name="text"
await page.waitForSelector("input[autocomplete=\"username\"][name=\"text\"]").catch(e => {
console.error("Waiting for username input failed:", e);
browser.close();
return new Response("Error: Waiting for Twitter username input failed", { status: 500 });
});
console.log("Typing in username...");
await page.type("input[autocomplete=\"username\"][name=\"text\"]", process.env.twitterUsername).catch(e => {
console.error("Typing in username failed:", e);
browser.close();
return new Response("Error: Typing in Twitter username failed", { status: 500 });
});
// look for the div with role="button" that is somewhere below the input
const loginButtonSelector = "div[role=\"button\"]";
await page.waitForSelector(loginButtonSelector).catch(e => {
console.error("Waiting for next button after username failed:", e);
browser.close();
return new Response("Error: Waiting for next button after username failed", { status: 500 });
});
console.log("Clicking on next button after username...");
await page.click(loginButtonSelector).catch(e => {
console.error("Clicking next button after username failed:", e);
browser.close();
return new Response("Error: Clicking next button after username failed", { status: 500 });
});
👆 This is a val. Vals are TypeScript snippets of code, written in the browser and run on our servers. Create scheduled functions, email yourself, and persist small pieces of data — all from the browser.