amico papers digest — intelligent daily arXiv picks to Slack (#412)

packages/amico-run/src/papers_digest.ts

@@ -0,0 +1,231 @@
+// papers_digest.ts — the intelligent daily digest engine (#412):
+// parse (arXiv RSS subset, zero-dep) → profile (the corpus IS the lab's
+// taste: tag/system frequencies, recency-decayed) → score (explainable:
+// matched terms, title-weighted, word-boundary safe) → rank (drop zeros,
+// skip corpus + posted identities) → format (Slack mrkdwn).
+// Deterministic: same inputs → same digest. No model in the loop — the
+// intelligence is the corpus, and every pick says why it matched.
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import type { CorpusReport } from "./papers.js";
+
+// ── the RSS subset ───────────────────────────────────────────────────────────
+
+export interface RssItem {
+ arxiv: string;
+ title: string;
+ abstract: string;
+}
+
+function unescapeXml(s: string): string {
+ return s
+ .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+ .replace(/&lt;/g, "<")
+ .replace(/&gt;/g, ">")
+ .replace(/&quot;/g, '"')
+ .replace(/&#39;|&apos;/g, "'")
+ .replace(/&amp;/g, "&");
+}
+
+function stripTags(s: string): string {
+ return unescapeXml(s).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
+}
+
+/** Parse arXiv's RSS: <item><title/><link>…abs/<id></link><description/></item>.
+ * Malformed input degrades to [] — a bad feed never crashes the digest. */
+export function parseArxivRss(xml: string): RssItem[] {
+ const items: RssItem[] = [];
+ const entries = xml.match(/<item>([\s\S]*?)<\/item>/g) ?? [];
+ for (const e of entries) {
+ const title = e.match(/<title>([\s\S]*?)<\/title>/)?.[1];
+ const link = e.match(/<link>([\s\S]*?)<\/link>/)?.[1];
+ const desc = e.match(/<description>([\s\S]*?)<\/description>/)?.[1];
+ if (!title || !link) continue;
+ const arxiv = link.trim().match(/abs\/([0-9]{4}\.[0-9]{4,5}|[a-z-]+\/[0-9]{7})(v\d+)?/)?.[1];
+ if (!arxiv) continue;
+ items.push({
+ arxiv,
+ title: stripTags(title),
+ abstract: desc ? stripTags(desc).slice(0, 2000) : "",
+ });
+ }
+ return items;
+}
+
+// ── the corpus-derived profile ───────────────────────────────────────────────
+
+const HALF_LIFE_DAYS = 180; // a term from a note read 6 months ago weighs half
+
+export interface LabProfile {
+ terms: Map<string, number>;
+ weight(term: string): number;
+}
+
+/** The lab's demonstrated taste: every tag + system across the corpus,
+ * frequency-weighted and recency-decayed (date_read half-life). Fresh
+ * reading defines the current profile; old interests fade, never vanish. */
+export function buildProfile(corpus: CorpusReport, now = new Date()): LabProfile {
+ const terms = new Map<string, number>();
+ const bump = (term: string, dateRead: string | undefined) => {
+ const t = term.toLowerCase().trim();
+ if (!t) return;
+ let w = 1;
+ if (dateRead) {
+ const days = Math.max(0, (now.getTime() - Date.parse(dateRead)) / 86_400_000);
+ if (!Number.isNaN(days)) w *= Math.pow(0.5, days / HALF_LIFE_DAYS);
+ }
+ terms.set(t, (terms.get(t) ?? 0) + w);
+ };
+ for (const p of corpus.papers) {
+ const dateRead = p.frontmatter.date_read as string | undefined;
+ for (const s of p.systems) bump(s, dateRead);
+ for (const t of p.tags) if (t.toLowerCase() !== "paper") bump(t, dateRead);
+ }
+ return { terms, weight: (t) => terms.get(t.toLowerCase().trim()) ?? 0 };
+}
+
+// ── scoring ──────────────────────────────────────────────────────────────────
+
+export interface ScoredPick {
+ item: RssItem;
+ score: number;
+ terms: string[];
+}
+
+const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/** Word-boundary matcher for one term against a text (already lowercased). */
+function countMatches(textLower: string, term: string): number {
+ const m = textLower.match(new RegExp(`(^|[^a-z0-9-])${escapeRe(term)}([^a-z0-9-]|$)`, "g"));
+ return m ? m.length : 0;
+}
+
+/** Explainable score: Σ term-weight × occurrences (title ×3, abstract ×1).
+ * Zero-score papers are uninteresting to this lab — dropped, not padded. */
+export function scorePaper(item: RssItem, profile: LabProfile): ScoredPick {
+ const title = item.title.toLowerCase();
+ const abstract = item.abstract.toLowerCase();
+ let score = 0;
+ const terms: string[] = [];
+ for (const [term, weight] of profile.terms) {
+ const inTitle = countMatches(title, term);
+ const inAbs = countMatches(abstract, term);
+ if (inTitle + inAbs === 0) continue;
+ // multi-word / compound terms (e.g. "trapped-ion", "neutral-atoms") are
+ // more specific — scale with their length
+ const specificity = 1 + Math.min(1, term.length / 16);
+ score += weight * specificity * (inTitle * 3 + inAbs);
+ terms.push(term);
+ }
+ return { item, score: Math.round(score * 100) / 100, terms };
+}
+
+// ── ranking ──────────────────────────────────────────────────────────────────
+
+export interface RankOpts {
+ posted: string[]; // previously-posted ids (the state file)
+ top: number;
+}
+
+export interface RankResult {
+ picks: ScoredPick[];
+ skipped: { corpus: string[]; posted: string[] };
+ dropped: string[]; // zero-score: irrelevant to this lab
+}
+
+/** Rank: score every item, drop zeros, skip corpus/posted identities, take top N. */
+export function rankDigest(items: RssItem[], profile: LabProfile, corpus: CorpusReport, opts: RankOpts): RankResult {
+ const known = new Set<string>();
+ for (const p of corpus.papers) {
+ if (p.arxiv) known.add(p.arxiv);
+ }
+ const posted = new Set(opts.posted);
+ const scored = items.map((i) => scorePaper(i, profile));
+ const picks: ScoredPick[] = [];
+ const skipped = { corpus: [] as string[], posted: [] as string[] };
+ const dropped: string[] = [];
+ for (const s of scored) {
+ const id = s.item.arxiv;
+ if (known.has(id)) skipped.corpus.push(id);
+ else if (posted.has(id)) skipped.posted.push(id);
+ else if (s.score <= 0) dropped.push(id);
+ else picks.push(s);
+ }
+ picks.sort((a, b) => b.score - a.score || a.item.arxiv.localeCompare(b.item.arxiv));
+ return { picks: picks.slice(0, opts.top), skipped, dropped };
+}
+
+// ── formatting ───────────────────────────────────────────────────────────────
+
+export interface DigestForFormat {
+ feedName: string;
+ total: number;
+ picks: ScoredPick[];
+ skipped: { corpus: string[]; posted: string[] };
+}
+
+/** Slack mrkdwn: header, numbered picks with links and why-lines. */
+export function formatDigest(d: DigestForFormat): string {
+ const today = new Date().toISOString().slice(0, 10);
+ const lines: string[] = [`*arXiv ${d.feedName} picks for ${today}* (${d.picks.length} of ${d.total} new, ranked against the lab corpus)`];
+ d.picks.forEach((p, i) => {
+ lines.push(
+ `${i + 1}. <http://arxiv.org/abs/${p.item.arxiv}|${p.item.title}>`,
+ ` _why:_ ${p.terms.slice(0, 5).map((t) => `\`${t}\``).join(" · ")} (score ${p.score})`,
+ );
+ });
+ if (d.picks.length === 0) lines.push("_nothing matched the lab profile today_");
+ if (d.skipped.corpus.length) lines.push(`_${d.skipped.corpus.length} already in the lab corpus — skipped_`);
+ return lines.join("\n");
+}
+
+// ── the posted-state (tiny, content-addressed idempotence) ───────────────────
+
+export function stateFile(): string {
+ return join(homedir(), ".amico", "amicode", "papers-digest-state.json");
+}
+
+export function readPostedIds(file = stateFile()): string[] {
+ try {
+ const j = JSON.parse(readFileSync(file, "utf8")) as { posted?: string[] };
+ return Array.isArray(j.posted) ? j.posted.slice(-2000) : [];
+ } catch {
+ return [];
+ }
+}
+
+/** Append ids; cap the log at 2000 entries (the digest only needs recent). */
+export function writePostedIds(ids: string[], file = stateFile()): void {
+ const prev = readPostedIds(file);
+ const merged = [...new Set([...prev, ...ids])].slice(-2000);
+ mkdirSync(join(file, ".."), { recursive: true });
+ writeFileSync(file, JSON.stringify({ posted: merged, updated: new Date().toISOString() }) + "\n");
+}
+
+/** Stable digest fingerprint (for tests: same inputs → same output). */
+export function digestFingerprint(text: string): string {
+ return createHash("sha256").update(text).digest("hex").slice(0, 16);
+}
+
+export function feedUrl(name: string): string {
+ return `https://export.arxiv.org/rss/${name}`;
+}
+
+/** Feed retrieval via curl (the S31-compliant network seam — network I/O
+ * rides subprocess curl; same zero-dep doctrine as the slack post). */
+export function fetchFeed(url: string): string {
+ const out = execFileSync(
+ "curl",
+ ["-sS", "--max-time", "30", "-H", "user-agent: amicode-papers-digest/0.1 (harmoniqs)", url],
+ { encoding: "utf8", maxBuffer: 4 << 20 },
+ );
+ return out;
+}
+
+export function libraryRootLegacy(): string {
+ return join(homedir(), ".amico", "library");
+}
+void existsSync;

packages/amico-run/src/papers_digest_verb.ts

@@ -0,0 +1,110 @@
+// papers_digest_verb.ts — `amico papers digest [--feed q] [--top N] [--dry-run | --post <channel>]`
+// (#412): fetch → rank against the lab corpus → dedup → print (dry-run default)
+// or post to Slack as the Amico bot. The posted-state file makes reruns
+// idempotent. Runs on the server (the Slack token is server-only by posture).
+import {
+ parseArxivRss,
+ buildProfile,
+ rankDigest,
+ formatDigest,
+ readPostedIds,
+ writePostedIds,
+ fetchFeed,
+ feedUrl,
+ digestFingerprint,
+} from "./papers_digest.js";
+import { foldCorpus } from "./papers.js";
+import { studioPathsOrLegacy } from "@amicode/schema";
+import { readFileSync, existsSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import type { VerbResult } from "./verbs.js";
+
+function flagValue(argv: string[], name: string): string | undefined {
+ const i = argv.indexOf(name);
+ return i >= 0 && i + 1 < argv.length ? argv[i + 1] : undefined;
+}
+
+function slackPost(channelName: string, text: string): { ok: boolean; error?: string; ts?: string } {
+ const tokenFile = join(homedir(), ".amico", "slack", "token");
+ const channelsFile = join(homedir(), ".amico", "slack", "channels.json");
+ if (!existsSync(tokenFile) || !existsSync(channelsFile))
+ return { ok: false, error: "slack credentials not found (~/.amico/slack/{token,channels.json}) — run on the server" };
+ const token = readFileSync(tokenFile, "utf8").trim();
+ const channels = JSON.parse(readFileSync(channelsFile, "utf8")) as Record<string, string>;
+ const cid = channels[channelName];
+ if (!cid) return { ok: false, error: `unknown channel '${channelName}' (known: ${Object.keys(channels).join(", ")})` };
+ // synchronous CLI context: curl is the zero-dep POST path
+ const args = [
+ "-sS", "-X", "POST", "https://slack.com/api/chat.postMessage",
+ "-H", `Authorization: Bearer ${token}`,
+ "-H", "Content-type: application/json; charset=utf-8",
+ "--data-binary", JSON.stringify({ channel: cid, text, unfurl_links: false }),
+ ];
+ try {
+ const out = execFileSync("curl", args, { encoding: "utf8", maxBuffer: 1 << 20 });
+ const j = JSON.parse(out) as { ok: boolean; error?: string; ts?: string };
+ return j;
+ } catch (e) {
+ return { ok: false, error: String(e) };
+ }
+}
+
+export async function papersDigestVerb(argv: string[]): Promise<VerbResult> {
+ const feed = flagValue(argv, "--feed") ?? "quant-ph";
+ const top = Number(flagValue(argv, "--top") ?? 5);
+ const post = flagValue(argv, "--post");
+ const dryRun = argv.includes("--dry-run") || post === undefined;
+
+ // hermetic escapes (tests) → studio ladder (production)
+ const vaults = process.env.AMICO_PAPERS_VAULTS ?? studioPathsOrLegacy().vaultsRoot;
+ const library = process.env.AMICO_PAPERS_LIBRARY ?? join(homedir(), ".amico", "library");
+
+ let xml: string;
+ try {
+ xml = fetchFeed(feedUrl(feed));
+ } catch (e) {
+ return { json: { ok: false, error: `feed fetch failed: ${e}` }, code: 1 };
+ }
+ const items = parseArxivRss(xml);
+ if (items.length === 0) {
+ return { json: { ok: false, error: `feed parsed to zero items (${feed}) — refusing to post an empty digest` }, code: 1 };
+ }
+
+ const corpus = foldCorpus([vaults], library);
+ const profile = buildProfile(corpus);
+ const posted = readPostedIds();
+ const r = rankDigest(items, profile, corpus, { posted, top });
+
+ if (r.picks.length === 0) {
+ writePostedIds(r.skipped.posted); // no repost risk either way; state stays fresh
+ return {
+ json: { ok: true, posted: false, reason: "no items matched the lab profile today", total: items.length },
+ code: 0,
+ };
+ }
+
+ const text = formatDigest({ feedName: feed, total: items.length, picks: r.picks, skipped: r.skipped });
+
+ if (dryRun) {
+ return {
+ json: {
+ ok: true,
+ dry_run: true,
+ text,
+ fingerprint: digestFingerprint(text),
+ counts: { total: items.length, picks: r.picks.length, skipped_corpus: r.skipped.corpus.length, dropped: r.dropped.length },
+ },
+ code: 0,
+ };
+ }
+
+ const res = slackPost(post!, text);
+ if (!res.ok) return { json: { ok: false, error: `slack post failed: ${res.error}` }, code: 1 };
+ writePostedIds(r.picks.map((p) => p.item.arxiv));
+ return {
+ json: { ok: true, posted: true, channel: post, ts: res.ts, fingerprint: digestFingerprint(text), counts: { total: items.length, picks: r.picks.length } },
+ code: 0,
+ };
+}

packages/amico-run/src/papers_verb.ts

@@ -8,15 +8,17 @@
 // Read-only (the fold never writes). $AMICO_PAPERS_VAULTS / $AMICO_PAPERS_LIBRARY
 // are the hermetic test escapes; production roots come from the studio ladder.
 import { papersList } from "./papers_list.js";
+import { papersDigestVerb } from "./papers_digest_verb.js";
 import type { VerbResult } from "./verbs.js";
 
-export function papersVerb(argv: string[]): VerbResult {
+// The Verb.run signature accepts Promise (digest fetches the feed); the
+// router awaits. List stays sync.
+export async function papersVerb(argv: string[]): Promise<VerbResult> {
  const [sub, ...rest] = argv;
- if (sub !== "list") {
- return {
- json: { ok: false, error: `papers: unknown subcommand '${sub ?? ""}' — usage: amico papers list [--status|--tag|--platform|--q] [--json]` },
- code: 64,
- };
- }
- return papersList(rest);
+ if (sub === "list") return papersList(rest);
+ if (sub === "digest") return papersDigestVerb(rest);
+ return {
+ json: { ok: false, error: `papers: unknown subcommand '${sub ?? ""}' — usage: amico papers list […] | amico papers digest [--feed <f>] [--top <n>] [--dry-run|--post <channel>]` },
+ code: 64,
+ };
 }