mirror of
https://github.com/prdlk/x-bookmarks-rss.git
synced 2026-08-02 17:31:42 +00:00
init(src): add X bookmark RSS feed implementation using Hono and D1
This commit is contained in:
Prad Nukala
+204
@@ -0,0 +1,204 @@
|
|||||||
|
import { Hono } from "hono";
|
||||||
|
|
||||||
|
// Env bindings (D1 + secrets/vars). Secrets are set via `wrangler secret put`.
|
||||||
|
// X_REFRESH_TOKEN is your personal OAuth2 refresh token — seeded once, then rotated into D1.
|
||||||
|
type Env = {
|
||||||
|
DB: D1Database;
|
||||||
|
X_CLIENT_ID: string;
|
||||||
|
X_CLIENT_SECRET: string;
|
||||||
|
X_REFRESH_TOKEN?: string; // optional seed fallback; primary path is the refresh_token row in D1 meta
|
||||||
|
SYNC_TTL_SECONDS?: string;
|
||||||
|
MAX_ITEMS?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const TOKEN_URL = "https://api.x.com/2/oauth2/token";
|
||||||
|
|
||||||
|
const app = new Hono<{ Bindings: Env }>();
|
||||||
|
|
||||||
|
// ---------- meta table (key/value) ----------
|
||||||
|
|
||||||
|
const getMeta = async (db: D1Database, key: string): Promise<string | null> => {
|
||||||
|
const row = await db.prepare("SELECT value FROM meta WHERE key = ?").bind(key).first<{ value: string }>();
|
||||||
|
return row?.value ?? null;
|
||||||
|
};
|
||||||
|
const setMeta = (db: D1Database, key: string, value: string) =>
|
||||||
|
db.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").bind(key, value).run();
|
||||||
|
|
||||||
|
// ---------- small helpers ----------
|
||||||
|
|
||||||
|
const xmlEscape = (s: string): string =>
|
||||||
|
s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
||||||
|
// CDATA can't contain the literal "]]>"; split it so the payload stays intact.
|
||||||
|
const cdata = (s: string): string => `<![CDATA[${s.replace(/]]>/g, "]]]]><![CDATA[>")}]]>`;
|
||||||
|
const rfc822 = (iso: string | null): string => {
|
||||||
|
const d = new Date(iso ?? "");
|
||||||
|
return isNaN(d.getTime()) ? new Date().toUTCString() : d.toUTCString();
|
||||||
|
};
|
||||||
|
|
||||||
|
// ---------- X API ----------
|
||||||
|
|
||||||
|
type Tweet = {
|
||||||
|
id: string;
|
||||||
|
text: string;
|
||||||
|
created_at?: string;
|
||||||
|
author_id?: string;
|
||||||
|
attachments?: { media_keys?: string[] };
|
||||||
|
};
|
||||||
|
type XUser = { id: string; username: string; name: string };
|
||||||
|
type Media = { media_key: string; type: string; url?: string; preview_image_url?: string };
|
||||||
|
|
||||||
|
// Exchange the rotating refresh_token for a fresh access_token. X rotates the
|
||||||
|
// refresh_token on every use, so the caller MUST persist the new one immediately.
|
||||||
|
async function refreshAccessToken(env: Env, refreshToken: string) {
|
||||||
|
const basic = btoa(`${env.X_CLIENT_ID}:${env.X_CLIENT_SECRET}`);
|
||||||
|
const res = await fetch(TOKEN_URL, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/x-www-form-urlencoded", Authorization: `Basic ${basic}` },
|
||||||
|
body: new URLSearchParams({ grant_type: "refresh_token", refresh_token: refreshToken, client_id: env.X_CLIENT_ID }),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`refresh failed: ${res.status} ${await res.text()}`);
|
||||||
|
return (await res.json()) as { access_token: string; refresh_token: string };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pull bookmarks (paginated) up to maxItems. Uses the all-bookmarks endpoint —
|
||||||
|
// the folder endpoint has a broken pagination bug, so we deliberately avoid it.
|
||||||
|
async function fetchBookmarks(userId: string, accessToken: string, maxItems: number) {
|
||||||
|
const tweets: Tweet[] = [];
|
||||||
|
const users = new Map<string, XUser>();
|
||||||
|
const media = new Map<string, Media>();
|
||||||
|
let token: string | undefined;
|
||||||
|
|
||||||
|
do {
|
||||||
|
const url = new URL(`https://api.x.com/2/users/${userId}/bookmarks`);
|
||||||
|
url.searchParams.set("max_results", "100");
|
||||||
|
url.searchParams.set("tweet.fields", "created_at,text,author_id");
|
||||||
|
url.searchParams.set("expansions", "author_id,attachments.media_keys");
|
||||||
|
url.searchParams.set("user.fields", "username,name");
|
||||||
|
url.searchParams.set("media.fields", "url,preview_image_url,type");
|
||||||
|
if (token) url.searchParams.set("pagination_token", token);
|
||||||
|
|
||||||
|
const res = await fetch(url, { headers: { Authorization: `Bearer ${accessToken}` } });
|
||||||
|
if (!res.ok) throw new Error(`bookmarks fetch failed: ${res.status} ${await res.text()}`);
|
||||||
|
|
||||||
|
const json = (await res.json()) as {
|
||||||
|
data?: Tweet[];
|
||||||
|
includes?: { users?: XUser[]; media?: Media[] };
|
||||||
|
meta?: { next_token?: string };
|
||||||
|
};
|
||||||
|
for (const t of json.data ?? []) tweets.push(t);
|
||||||
|
for (const u of json.includes?.users ?? []) users.set(u.id, u);
|
||||||
|
for (const m of json.includes?.media ?? []) media.set(m.media_key, m);
|
||||||
|
token = json.meta?.next_token;
|
||||||
|
} while (token && tweets.length < maxItems);
|
||||||
|
|
||||||
|
return { tweets: tweets.slice(0, maxItems), users, media };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve the authenticated user's id (cached in D1 after the first call).
|
||||||
|
async function resolveUserId(env: Env, accessToken: string): Promise<string> {
|
||||||
|
const cached = await getMeta(env.DB, "user_id");
|
||||||
|
if (cached) return cached;
|
||||||
|
const res = await fetch("https://api.x.com/2/users/me", { headers: { Authorization: `Bearer ${accessToken}` } });
|
||||||
|
if (!res.ok) throw new Error(`/users/me failed: ${res.status} ${await res.text()}`);
|
||||||
|
const me = (await res.json()) as { data: { id: string } };
|
||||||
|
await setMeta(env.DB, "user_id", me.data.id);
|
||||||
|
return me.data.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refresh + fetch + upsert into D1, throttled by SYNC_TTL_SECONDS. Best-effort:
|
||||||
|
// any failure is swallowed so the feed still serves whatever is already stored.
|
||||||
|
// The refresh token is seeded from the X_REFRESH_TOKEN secret on first run, then
|
||||||
|
// the rotated token lives in D1 (secrets can't be written back to from a Worker).
|
||||||
|
async function syncIfStale(env: Env): Promise<void> {
|
||||||
|
const ttl = Number(env.SYNC_TTL_SECONDS ?? "900");
|
||||||
|
const lastSync = Number((await getMeta(env.DB, "last_sync")) ?? "0");
|
||||||
|
if (Date.now() - lastSync < ttl * 1000) return;
|
||||||
|
|
||||||
|
const refreshToken = (await getMeta(env.DB, "refresh_token")) ?? env.X_REFRESH_TOKEN;
|
||||||
|
if (!refreshToken) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const t = await refreshAccessToken(env, refreshToken);
|
||||||
|
await setMeta(env.DB, "refresh_token", t.refresh_token); // persist rotated token immediately
|
||||||
|
|
||||||
|
const userId = await resolveUserId(env, t.access_token);
|
||||||
|
const { tweets, users, media } = await fetchBookmarks(userId, t.access_token, Number(env.MAX_ITEMS ?? "100"));
|
||||||
|
const now = Date.now();
|
||||||
|
const stmt = env.DB.prepare(
|
||||||
|
`INSERT INTO bookmarks (id, text, author_id, username, name, created_at, media_urls, url, fetched_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(id) DO UPDATE SET text=excluded.text, username=excluded.username, name=excluded.name, media_urls=excluded.media_urls, url=excluded.url`,
|
||||||
|
);
|
||||||
|
const rows = tweets.map((t) => {
|
||||||
|
const u = t.author_id ? users.get(t.author_id) : undefined;
|
||||||
|
const username = u?.username ?? "i"; // /i/status/<id> still resolves on X
|
||||||
|
const mediaUrls = (t.attachments?.media_keys ?? [])
|
||||||
|
.map((k) => media.get(k))
|
||||||
|
.map((m) => m?.url ?? m?.preview_image_url)
|
||||||
|
.filter((x): x is string => !!x);
|
||||||
|
return stmt.bind(
|
||||||
|
t.id,
|
||||||
|
t.text,
|
||||||
|
t.author_id ?? null,
|
||||||
|
u?.username ?? null,
|
||||||
|
u?.name ?? null,
|
||||||
|
t.created_at ?? null,
|
||||||
|
JSON.stringify(mediaUrls),
|
||||||
|
`https://x.com/${username}/status/${t.id}`,
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
if (rows.length) await env.DB.batch(rows);
|
||||||
|
await setMeta(env.DB, "last_sync", String(now));
|
||||||
|
} catch (e) {
|
||||||
|
console.error("sync failed:", (e as Error).message); // serve stale DB content
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- RSS rendering ----------
|
||||||
|
|
||||||
|
type Row = { id: string; text: string; username: string | null; created_at: string | null; media_urls: string | null; url: string | null };
|
||||||
|
|
||||||
|
function renderFeed(rows: Row[]): string {
|
||||||
|
const items = rows
|
||||||
|
.map((r) => {
|
||||||
|
const link = r.url ?? `https://x.com/i/status/${r.id}`;
|
||||||
|
const title = xmlEscape(r.text.slice(0, 80).replace(/\s+/g, " ").trim());
|
||||||
|
const mediaUrls: string[] = r.media_urls ? JSON.parse(r.media_urls) : [];
|
||||||
|
const body = [r.text, ...mediaUrls].join("\n");
|
||||||
|
return ` <item>
|
||||||
|
<title>${title}</title>
|
||||||
|
<link>${xmlEscape(link)}</link>
|
||||||
|
<guid isPermaLink="false">${xmlEscape(r.id)}</guid>
|
||||||
|
<pubDate>${rfc822(r.created_at)}</pubDate>
|
||||||
|
<description>${cdata(body)}</description>
|
||||||
|
</item>`;
|
||||||
|
})
|
||||||
|
.join("\n");
|
||||||
|
|
||||||
|
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<rss version="2.0">
|
||||||
|
<channel>
|
||||||
|
<title>X Bookmarks</title>
|
||||||
|
<link>https://x.com/i/bookmarks</link>
|
||||||
|
<description>Your X (Twitter) bookmarks as an RSS feed.</description>
|
||||||
|
<lastBuildDate>${new Date().toUTCString()}</lastBuildDate>
|
||||||
|
${items}
|
||||||
|
</channel>
|
||||||
|
</rss>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- routes ----------
|
||||||
|
|
||||||
|
// The live feed. Personal & unguarded: syncs from X when stale, renders all stored rows.
|
||||||
|
app.get("/feed.xml", async (c) => {
|
||||||
|
await syncIfStale(c.env);
|
||||||
|
const { results } = await c.env.DB.prepare(
|
||||||
|
"SELECT id, text, username, created_at, media_urls, url FROM bookmarks ORDER BY created_at DESC",
|
||||||
|
).all<Row>();
|
||||||
|
return c.body(renderFeed(results ?? []), 200, { "Content-Type": "application/rss+xml; charset=utf-8" });
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get("/", (c) => c.text("X Bookmarks → RSS. Read /feed.xml"));
|
||||||
|
|
||||||
|
export default app;
|
||||||
Reference in New Issue
Block a user