Back to packages list
Vals using jsdom
Description from the NPM package:
A JavaScript implementation of many web standards
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
// Might look complicated, don't be scared! Just follow the "//", I'll walk you through it.
import { Buffer } from "node:buffer";
import jsdom from "npm:jsdom";
const JSDOM = jsdom.JSDOM;
export default async function getMetadata(url) {
// Do we have a URL?
if (url == null || url.trim() == "") {
return { "status": 400, "error_msg": "Valid URL is required as input. URL was empty." };
}
let resp = await fetch(url);
let text = await resp.text();
// We parse the page as HTML and then pick the data we want.
// Right now we have the page as a looong string, let's convert it into a HTML document
let frag = new JSDOM(text).window.document;
// Get the data we want and send
let imgUrl = getImageUrl(frag, url);
let imgData = await getImageDataFromUrl(imgUrl);
let title = getTitle(frag);
let description = getDescription(frag);
return { "status": 200, "url": url, title, description, imgUrl, imgData };
}
function getImageUrl(frag, url) {
let imgUrl = "";
let selectors = [
"meta[property=\"og:image:secure_url\"]",
"meta[property=\"og:image:url\"]",
"meta[property=\"og:image\"]",
"meta[name=\"twitter:image:src\"]",
"meta[property=\"twitter:image:src\"]",
"meta[name=\"twitter:image\"]",
"meta[property=\"twitter:image\"]",
"meta[itemprop=\"image\"]",
];
// Get image from the HTML fragment
let element;
for (let i = 0; i < selectors.length; i++) {
element = frag.querySelector(selectors[i]);
if (!imgUrl && element && element.content) {
imgUrl = element.content;
}
}
// Still not present? Try to get the image of the author and use it instead
element = frag.querySelector("img[alt*=\"author\" i]");
if (!imgUrl && element && element.src) {
imgUrl = element.getAttribute("src");
}
// Still not present? Well let's take ANY visible image from the page.
// You leave me no choice, my friend.
element = frag.querySelector("img[src]:not([aria-hidden=\"true\"])");
if (!imgUrl && element && element.src) {
imgUrl = element.getAttribute("src");
}
if (imgUrl !== "") {
// Some img src URLs are relative. In that case, convert to absolute.
// https://stackoverflow.com/a/44547904/12415069
imgUrl = new URL(imgUrl, url).href;
}
return imgUrl;
}
async function getImageDataFromUrl(url) {
const response = await fetch(url);
const base64data = Buffer.from(await response.arrayBuffer()).toString("base64");
return "data:image/png;base64," + base64data;
}
function getTitle(frag) {
let element;
let title = "";
let selectors = [
"meta[property=\"og:title\"]",
"meta[name=\"twitter:title\"]",
"meta[property=\"twitter:title\"]",
];
// Get image from the HTML fragment
for (let i = 0; i < selectors.length; i++) {
element = frag.querySelector(selectors[i]);
if (element && element.content) {
title = element.content;
return title;
}
}
// Not present? Take the text that shows up in the browser tab
element = frag.querySelector("title");
if (element && (element.innerText || element.text || element.textContent)) {
title = element.innerText || element.text || element.textContent;
return title;
}
return title;
}
Runs every 1 hrs
Fork1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
import { blob } from "https://esm.town/v/std/blob?v=11";
import { email } from "https://esm.town/v/std/email?v=11";
import { fetchText } from "https://esm.town/v/stevekrouse/fetchText?v=6";
import { JSDOM } from "npm:jsdom";
const username = "ccorcos";
export default async function(interval: Interval) {
const url = "https://news.ycombinator.com/threads?id=" + username;
const html = await fetchText(url);
console.log("html", html.slice(0, 100));
const { document } = (new JSDOM(html)).window;
const comments = Array.from(document.querySelectorAll(".athing.comtr")) as any[];
const currentIds = new Set(comments.map(x => x.id));
const newIds = new Set(comments.map(x => x.id));
const key = "hn2:" + username;
const prevIds = (await blob.getJSON(key)) || [] as string[];
for (const id of prevIds) newIds.delete(id);
if (newIds.size === 0) {
console.log("No new comments.");
return;
}
for (const id of Array.from(newIds)) prevIds.push(id)
console.log(newIds.size + " new comments.");
await blob.setJSON(key, prevIds);
const newComments = comments.filter(comment => newIds.has(comment.id))
.map(comment => comment.textContent);
await email({
subject: "New HN comments",
text: url + "\n\n" + newComments.join("\n---\n"),
});
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import { fetch } from "https://esm.town/v/std/fetch";
import { parseBlock } from "https://esm.town/v/usefulthink/parseBlock";
export async function getAvailableCourses() {
const JSDOM = await import("npm:jsdom");
const qs = await import("node:querystring");
const requestData = {
"args[kurskategorie_id]": "18",
"args[kurstyp_id]": "84",
"args[standort_id]": "",
"args[datum]": "",
"phery[method]": "POST",
"phery[submit_id]": "kurse-filter",
"phery[remote]": "get_kurse",
};
const res = await fetch(
"https://www.baederland-shop.de/schwimmschule?_=" + Date.now(),
{
headers: {
accept: "application/json, text/javascript, */*; q=0.01",
"content-type": "application/x-www-form-urlencoded",
"x-phery": "1",
"x-requested-with": "XMLHttpRequest",
},
body: qs.encode(requestData),
method: "POST",
},
).then((res) => res.json());
const dom = new JSDOM(res["#buchung"][0].a[0]);
const doc = dom.window.document;
const blocks = doc.querySelectorAll("[id^=kursblock]");
const availableCourses = [];
for (let el of blocks) {
const data = await parseBlock(el);
availableCourses.push(data);
}
return availableCourses;
}