@rlesser
Product Engineer at https://nomic.ai | MBA Candidate at Columbia Business School | Building ways to clear the fog
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import { api } from "https://esm.town/v/pomdtr/api";
type Module = {
slug: string;
id: string;
websiteUrl: string;
category: string | null;
};
export async function getReverseDependencies(valUrlOrSlug: string): Promise<Module[]> {
// clean the url or slug
const initialSlug = valUrlOrSlug.includes("/v/")
? valUrlOrSlug.split("/v/")[1]
: valUrlOrSlug;
// we leave the trailing quote off, in case its imported with a "?v=X" version string
// this does leave open the possibility of capturing vals that have this val as a
// prefix like (bob/getData and bob/getData). Search also returns vals where the
// search appears in the readme. We'll filter all these out later.
const searchString = ` from "https://esm.town/v/${initialSlug}`;
const { data } = await api(`/v1/search/vals?query=${encodeURI(searchString)}&offset=0&limit=100`);
return data
.map(d => {
const slug = d.author.username.replace("@", "") + "/" + d.name;
return {
slug,
id: "https://esm.town/v/" + slug,
websiteUrl: "https://www.val.town/v/" + slug,
category: "valtown",
};
});
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import { fetchJSON } from "https://esm.town/v/stevekrouse/fetchJSON?v=41";
export let googleSheetBatchUpdate = async (
serviceAccount: string,
spreadsheetId: string,
batchUpdateData:
import("npm:googleapis").sheets_v4.Params$Resource$Spreadsheets$Batchupdate[
"requestBody"
]["requests"],
) => {
const { getToken } = await import(
"https://deno.land/x/google_jwt_sa@v0.2.3/mod.ts"
);
const token = await getToken(serviceAccount, {
scope: ["https://www.googleapis.com/auth/spreadsheets"],
});
const requestBody = {
requests: batchUpdateData,
};
const result = await fetchJSON(
`https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}:batchUpdate`,
{
method: "POST",
headers: {
Authorization: `Bearer ${token.access_token}`,
},
body: JSON.stringify(requestBody),
},
);
return result;
};
getNextSSR - fetch data from Next.js SSR-based sites
Many modern websites use Next.js and Server-side Rendering (SSR) to serve their website and the data to populate it. Instead of standard API calls, this manifests in the browser as calls routed through dynamic endpoints.
This val handles the url construction and response parsing, giving you access to the endpoint and data.
Input
- websiteUrl - The website's url, like
google.comorval.townlet getNextSSR: (websiteUrl: string) => Promise<(endpoint: string) => Promise<Record<string, any>>>
Returns
- fetching function - A function that takes in an endpoint and returns the endpoint's response data:
-
- Input
-
- endpoint - The endpoint string, like
results.jsonortrending.json?page=2
- endpoint - The endpoint string, like
-
- Returns
-
- data - The endpoint response data, without NextJS artifacts.
Example
const fetch = await @rlesser.getNextSSR("example.com");
const data = fetch("results.json");
return data;
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
import { fetchJSON } from "https://esm.town/v/stevekrouse/fetchJSON?v=41";
import { fetchText } from "https://esm.town/v/stevekrouse/fetchText?v=5";
export let getNextSSR = async (websiteUrl: string) => {
const html = await fetchText(websiteUrl);
const staticId = html.split("/_buildManifest")[0].split("/").slice(-1)[0];
return async (endpoint: string) => {
const responseData = await fetchJSON(
`https://${websiteUrl}/_next/data/${staticId}/${endpoint}`,
);
// one possible data path is response.pageProps.ssrResponses.[some_path]
const ssrResponses = responseData?.pageProps?.ssrResponses;
if (ssrResponses == undefined) {
throw Error(
"Unexpected response structure - leave a comment with the expected structure!",
);
}
const realData = Object.values(ssrResponses)[0];
return realData as Record<string, any>;
};
};
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
import { fetchJSON } from "https://esm.town/v/stevekrouse/fetchJSON?v=41";
type AuthorData = {
avatarUrl: string;
fullname: string;
name: string;
type: string;
isHf: boolean;
};
interface Repo {
author: string;
id: string;
isLikedByUser: boolean;
lastModified: string;
likes: number;
private: boolean;
}
type ModelRepo = Repo & {
authorData: AuthorData;
downloads: number;
gated: boolean;
pipeline_tag: string;
repoType: "model";
};
type DatasetRepo = Repo & {
downloads: number;
gated: boolean;
previewable: boolean;
repoType: "dataset";
};
type SpaceRepo = Repo & {
authorData: AuthorData;
colorFrom: string;
colorTo: string;
emoji: string;
pinned: boolean;
runtime: object;
title: string;
repoType: "space";
};
type TrendingRepo = {
likes: number;
typeRank: number;
allRank?: number;
repoType: "model" | "dataset" | "space";
repoData: ModelRepo | DatasetRepo | SpaceRepo;
};
export const HuggingFace_getTrending = async (params: {
type: "all" | "model" | "dataset" | "space";
}): Promise<TrendingRepo[]> => {
const url = "https://huggingface.co/api/trending";
const query = new URLSearchParams({ type: params.type, limit: String(100) });
const json = await fetchJSON(
url + "?" + query.toString(),
);
const data: TrendingRepo[] = json.recentlyTrending;
["model", "dataset", "space"].forEach((type) =>
data.filter((repo) => repo.repoType == type).forEach((repo, i) => {
repo.typeRank = i + 1;
})
);
if (params.type == "all") {
data.forEach((repo, i) => repo.allRank = i + 1);
}
return data;
};
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
import { fetch } from "https://esm.town/v/std/fetch";
type RepoData = {
date: string;
rank: number;
full_name: string;
description: string;
language: string;
stars: number;
forks: number;
stars_recent: number;
};
export const Github_ScrapeTrendingRepos = async (
timeframe: string,
): Promise<RepoData[]> => {
const { cheerioJsonMapper: cjm } = await import("npm:cheerio-json-mapper");
const res = await fetch(`https://github.com/trending?since=${timeframe}`);
const html = await res.text();
const template = [
{
$: "article",
full_name: " > h2 > a | attr:href",
description: " > p",
language: " span[itemprop]",
stars: " > .f6 > a:first",
forks: " > .f6 > a:last",
stars_recent: " > .f6 > span:last",
},
];
const data = await cjm(html, template);
const date = new Date();
return data.map((d, i) => ({
date: date.toISOString(),
rank: i + 1,
full_name: d.full_name.slice(1),
description: d.description,
language: d.language,
stars: +d.stars.replace(",", ""),
forks: +d.forks.replace(",", ""),
stars_recent: +d.stars_recent.split(" ")[0].replace(",", ""),
}));
};
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
export const Github_SearchTopRepos = async (
start: string | Date,
days: number = 1,
) => {
const { Octokit } = await import("npm:@octokit/rest");
const octokit = new Octokit();
if (typeof start == "string")
start = new Date(start);
const end = new Date(start);
end.setDate(start.getDate() + days);
return await octokit.rest.search.repos({
q: `created:${start.toISOString()}..${end.toISOString()}`,
sort: "stars",
order: "desc",
per_page: 100,
});
};
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
import { fetchText } from "https://esm.town/v/stevekrouse/fetchText?v=5";
export default async function(req: Request): Promise<Response> {
// get the bracketId, gender, and year query parameters
const searchParams = new URL(req.url).searchParams;
const bracketId = searchParams.get("bracketId");
const gender = searchParams.get("gender") == "mens" ? "mens" : "womens";
const year = searchParams.get("year");
if (!bracketId || !gender || !year) {
return new Response("Missing query parameters", { status: 400 });
}
const groups = await espntcGetUsersGroups(bracketId, gender, year);
return new Response(JSON.stringify(groups), {
headers: { "Content-Type": "application/json", "cache-control": "public, max-age=3600" },
});
}
const espntcGetUsersGroups = async (bracketId: string, gender: "mens" | "womens", year: string) => {
const url = `https://fantasy.espn.com/tournament-challenge-bracket${
gender == "womens" ? "-women" : ""
}/${year}/en/game?entryID=${bracketId}`;
const text = await fetchText(url);
const selectorDivs = text
.split("group?groupID=")
.slice(1)
.map((d) => +d.split("\"")[0]);
return [...new Set(selectorDivs)];
};
1
2
3
4
5
6
import { espntcFindGroups } from "https://esm.town/v/rlesser/espntcFindGroups";
export let espntcFindUserGroups = async (search: string, gender?: "m" | "f") => {
const res = await espntcFindGroups({ search, gender });
return res.filter((d) => d.size > 0 && d.type == "user");
};
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import { fetchJSON } from "https://esm.town/v/stevekrouse/fetchJSON?v=41";
export let espntcFindGroups = async (options?: {
search?: string,
access?: "all" | "private" | "public",
type?: "all" | "celebrity" | "fans",
locked?: "all" | "unlocked" | "locked",
dropWorst?: boolean,
start?: number,
num?: number,
gender?: "m" | "f",
}) => {
const params = {
search: options?.search || "",
type_access: ["all", "private", "public"].indexOf(options?.access ?? "all"),
type_group: ["all", "celebrity", "fans"].indexOf(options?.type ?? "all"),
type_lock: ["all", "unlocked", "locked"].indexOf(options?.locked ?? "all"),
type_dropWorst: options?.dropWorst ? 1 : 0,
start: options?.start || 0,
num: options?.num || 30,
gender: options?.gender || "m",
};
const baseURL =
"https://fantasy.espncdn.com/tournament-challenge-bracket" +
(params.gender == "f" ? "-women" : "") +
"/2023/en/api/findGroups";
console.log(`${baseURL}?${new URLSearchParams(params)}`);
const res = await fetchJSON(
`${baseURL}?${new URLSearchParams(params)}`
);
interface Group {
id: number;
name: string;
motto: string;
size: number;
private: boolean;
locked: boolean;
type: "user" | "celebrity" | "fans";
raw: any;
}
console.log(res);
const groups: Group[] = res.g.map((g): Group => ({
id: g.id,
name: g.n,
motto: g.m,
size: g.s,
private: g.p,
locked: g.l,
type: g.t == "u" ? "user" : g.t == "c" ? "celebrity" : "fans",
raw: g,
}));
return groups;
};
1
2
3
4
5
6
7
8
9
10
11
12
13
14
import { fetchJSON } from "https://esm.town/v/stevekrouse/fetchJSON?v=41";
export let CohereCommon = (key: string, endpoint: string, body: Object) => {
return fetchJSON("https://api.cohere.ai/" + endpoint, {
method: "POST",
headers: {
accept: "application/json",
"Cohere-Version": "2022-12-06",
"content-type": "application/json",
authorization: "Bearer " + key,
},
body: JSON.stringify(body),
});
};