The Big Story
This val, along with @tmcw.big_story, which requests from the New York Times API, and @tmcw.big_stories_ranks, which contains the data, generates a visualization of top stories on the NYTimes homepage.
This is here just to ask the question – what happens to cover stories over time? Do they slowly drop down the page, or just get replaced by a fully new lede? So far it doesn't have quite enough data to answer that question.
But also, it might be neat because it'll show which kinds of stories make the front page - is it climate, war, politics, or something else?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
export let big_story_visualization = async (req, res) => {
const { default: htm } = await import("npm:htm");
const { default: vhtml } = await import("npm:vhtml");
const html = htm.bind(vhtml);
const raw = (html) =>
vhtml(null, {
dangerouslySetInnerHTML: { __html: html },
});
res.send(html`
<html>
<body style="font-family: sans-serif;padding:30px;">
<h1>Big stories on the New York Times</h1>
<p>Below is a chart of headlines from the New York Times homepage, ordered
by their rank over time. Most stories start near the top and then get lower-ranked
when other news breaks. Some stories pop back up the ranks.
</p>
<p>Check out <a href='https://www.val.town/v/tmcw.big_story_visualization'>the code that makes this work</a> on <a href='https://val.town/'>val.town</a>.
</p>
<p>Hover over a line to see the story title, click on a line to view the story.</p>
<div style='padding-top:40px' id='chart'></div>
<script type="module">
${
raw(`
import * as Plot from "https://cdn.jsdelivr.net/npm/@observablehq/plot@0.6.9/+esm";
import {debounce} from "https://cdn.jsdelivr.net/npm/lodash-es@4.17.21/+esm";
const ranks = await (await fetch('https://api.val.town/v1/run/tmcw.big_stories_ranks')).json();
let allDates = new Set();
for (let rank of Object.values(ranks)) {
for (let [date] of rank.ranks) {
allDates.add(date);
}
}
allDates = [...allDates.values()].sort((a, b) => a - b);
const normalized = Object.entries(ranks).flatMap(([url, data]) => {
return allDates.map(date => {
return {
...data,
ranks: undefined,
url: url,
rank: data.ranks.find(rank => rank[0] === date)?.[1],
date: new Date(date),
};
});
});
function render() {
const width = window.innerWidth;
const output = Plot.plot({
grid: true,
width,
height: 800,
color: {
legend: true,
},
y: {
reverse: true,
},
x: {
type: 'time'
},
marginLeft: 20,
marginRight: 10,
marks: [
Plot.lineY(normalized, {
x: "date",
y: "rank",
stroke: 'section',
strokeLinecap: 'butt',
strokeLinejoin: 'miter-clip',
z: 'url',
href: 'url',
strokeWidth: 10,
}),
Plot.tip(normalized, Plot.pointer({
x: "date",
y: "rank",
href: 'url',
frameAnchor: "top-left",
title: (d) => d.title + ' ' + d.section
}))
]});
chart.innerHTML = '';
output.addEventListener('click', (e) => {
if (e.target instanceof SVGPathElement && e.target.ariaDescription) {
window.open(e.target.ariaDescription);
}
});
chart.append(output);
}
render();
window.addEventListener('resize', debounce(() => {
render();
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
import { fetch } from "https://esm.town/v/std/fetch";
import { big_stories_ranks } from "https://esm.town/v/tmcw/big_stories_ranks";
import process from "node:process";
export let big_story = (async () => {
const nytimes =
await (await fetch(
`https://api.nytimes.com/svc/topstories/v2/home.json?api-key=${process.env.nytimes_api_key}`,
)).json();
const now = Date.now();
const newDataPoints = nytimes.results.slice(0, 20).forEach(
({ title, url, created_date, section, uri }, rank) => {
if (big_stories_ranks[url]) {
big_stories_ranks[url].ranks.push([now, rank]);
}
else {
big_stories_ranks[url] = {
title,
url,
section,
ranks: [[now, rank]],
};
}
},
);
for (let url of Object.keys(big_stories_ranks)) {
// Prune stories over 3 days ago
if (
big_stories_ranks[url].ranks.every((rank) => {
return rank[0] < Date.now() - (1000 * 60 * 60 * 24 * 3);
})
) {
delete big_stories_ranks[url];
}
}
});