From a56f72ef4510b43a9de7f49401db81c2c158b98c Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 23:41:55 -0400 Subject: [PATCH 1/3] feat(twister): add isNoReplySender to the signals entry point Connectors extracting mail signals no longer need a separate package dependency for sender-identity detection. --- .changeset/no-reply-sender.md | 9 +++++++++ connectors/apple/package.json | 1 - connectors/google/package.json | 1 - connectors/google/src/mail/gmail-api.ts | 2 +- connectors/outlook/package.json | 1 - connectors/outlook/src/mail/graph-mail-api.ts | 2 +- pnpm-lock.yaml | 9 --------- twister/src/signals.ts | 20 +++++++++++++++++++ 8 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 .changeset/no-reply-sender.md diff --git a/.changeset/no-reply-sender.md b/.changeset/no-reply-sender.md new file mode 100644 index 00000000..50906027 --- /dev/null +++ b/.changeset/no-reply-sender.md @@ -0,0 +1,9 @@ +--- +"@plotday/twister": minor +--- + +Added: `isNoReplySender` to the signals entry point. + +Identifies an address whose local part marks it as an automated or no-reply +sender. Connectors extracting mail signals can use it without depending on a +separate package. diff --git a/connectors/apple/package.json b/connectors/apple/package.json index 7bd8d991..6de53508 100644 --- a/connectors/apple/package.json +++ b/connectors/apple/package.json @@ -29,7 +29,6 @@ "test": "vitest run" }, "dependencies": { - "@plotday/email-classifier": "workspace:^", "@plotday/twister": "workspace:^" }, "devDependencies": { diff --git a/connectors/google/package.json b/connectors/google/package.json index 4ceea6d3..fc8016a2 100644 --- a/connectors/google/package.json +++ b/connectors/google/package.json @@ -30,7 +30,6 @@ "test:watch": "vitest" }, "dependencies": { - "@plotday/email-classifier": "workspace:^", "@plotday/google-contacts": "workspace:^", "@plotday/twister": "workspace:^" }, diff --git a/connectors/google/src/mail/gmail-api.ts b/connectors/google/src/mail/gmail-api.ts index e177cd69..0a865fe5 100644 --- a/connectors/google/src/mail/gmail-api.ts +++ b/connectors/google/src/mail/gmail-api.ts @@ -9,7 +9,7 @@ import type { } from "@plotday/twister/plot"; import { markdownToPlainText } from "@plotday/twister/utils/markdown"; import { markdownToHtml } from "@plotday/twister/utils/markdown-html"; -import { isNoReplySender } from "@plotday/email-classifier"; +import { isNoReplySender } from "@plotday/twister/signals"; export type GmailLabel = { diff --git a/connectors/outlook/package.json b/connectors/outlook/package.json index d5d539b6..de914d0f 100644 --- a/connectors/outlook/package.json +++ b/connectors/outlook/package.json @@ -30,7 +30,6 @@ "test:watch": "vitest" }, "dependencies": { - "@plotday/email-classifier": "workspace:^", "@plotday/twister": "workspace:^" }, "devDependencies": { diff --git a/connectors/outlook/src/mail/graph-mail-api.ts b/connectors/outlook/src/mail/graph-mail-api.ts index bf44aaac..54da0fea 100644 --- a/connectors/outlook/src/mail/graph-mail-api.ts +++ b/connectors/outlook/src/mail/graph-mail-api.ts @@ -6,7 +6,7 @@ import type { NewContact, NewLinkWithNotes, } from "@plotday/twister/plot"; -import { isNoReplySender } from "@plotday/email-classifier"; +import { isNoReplySender } from "@plotday/twister/signals"; import { stripQuotedReply } from "./email-parsing"; export type GraphRecipient = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e9b472f..786f5441 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,9 +39,6 @@ importers: connectors/apple: dependencies: - '@plotday/email-classifier': - specifier: workspace:^ - version: link:../../libs/email-classifier '@plotday/twister': specifier: workspace:^ version: link:../../twister @@ -110,9 +107,6 @@ importers: connectors/google: dependencies: - '@plotday/email-classifier': - specifier: workspace:^ - version: link:../../libs/email-classifier '@plotday/google-contacts': specifier: workspace:^ version: link:../../libs/google-contacts @@ -232,9 +226,6 @@ importers: connectors/outlook: dependencies: - '@plotday/email-classifier': - specifier: workspace:^ - version: link:../../libs/email-classifier '@plotday/twister': specifier: workspace:^ version: link:../../twister diff --git a/twister/src/signals.ts b/twister/src/signals.ts index ea4c130c..b5ba2ba7 100644 --- a/twister/src/signals.ts +++ b/twister/src/signals.ts @@ -82,3 +82,23 @@ export type LinkSignals = { */ noteKey?: string | null; }; + +const NOREPLY_LOCALPART = + /^(no-?reply|do-?not-?reply|donotreply|notifications?|notify|mailer-daemon|bounce|postmaster|automated|auto|alerts?|updates?)\b/; + +function localPart(address: string | null): string { + if (!address) return ""; + const at = address.indexOf("@"); + return (at === -1 ? address : address.slice(0, at)).toLowerCase(); +} + +/** + * True when an address's local part marks it as an automated / no-reply / + * notification sender (no-reply@, notify@, notifications@, alerts@, …). This is + * the identity-trust signal used to enable name-conflict detection for shared + * sender addresses; it deliberately ignores list/precedence headers (those are + * per-message automation signals, not shared-identity signals). + */ +export function isNoReplySender(address: string | null): boolean { + return NOREPLY_LOCALPART.test(localPart(address)); +} From d62ccba0d0b9d8297ee76c7a04d5ce114f2e2167 Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Mon, 3 Aug 2026 00:15:17 -0400 Subject: [PATCH 2/3] chore: remove the email classifier package Its classification logic now lives with the platform that runs it. Connectors emit raw signals rather than classifying, so nothing in this repository consumes it; the sender-identity helper they did use is now part of the SDK. --- connectors/AGENTS.md | 2 +- libs/email-classifier/CHANGELOG.md | 249 ------------- libs/email-classifier/package.json | 33 -- .../src/classify-email.test.ts | 221 ------------ libs/email-classifier/src/classify-email.ts | 171 --------- libs/email-classifier/src/extract-cta.test.ts | 327 ------------------ libs/email-classifier/src/extract-cta.ts | 262 -------------- .../src/extract-link-candidates.test.ts | 27 -- .../src/extract-link-candidates.ts | 29 -- libs/email-classifier/src/index.ts | 3 - libs/email-classifier/tsconfig.json | 8 - libs/email-classifier/vitest.config.ts | 8 - pnpm-lock.yaml | 13 - 13 files changed, 1 insertion(+), 1352 deletions(-) delete mode 100644 libs/email-classifier/CHANGELOG.md delete mode 100644 libs/email-classifier/package.json delete mode 100644 libs/email-classifier/src/classify-email.test.ts delete mode 100644 libs/email-classifier/src/classify-email.ts delete mode 100644 libs/email-classifier/src/extract-cta.test.ts delete mode 100644 libs/email-classifier/src/extract-cta.ts delete mode 100644 libs/email-classifier/src/extract-link-candidates.test.ts delete mode 100644 libs/email-classifier/src/extract-link-candidates.ts delete mode 100644 libs/email-classifier/src/index.ts delete mode 100644 libs/email-classifier/tsconfig.json delete mode 100644 libs/email-classifier/vitest.config.ts diff --git a/connectors/AGENTS.md b/connectors/AGENTS.md index 5af31519..15f24c6a 100644 --- a/connectors/AGENTS.md +++ b/connectors/AGENTS.md @@ -712,7 +712,7 @@ Add to `pnpm-workspace.yaml` if not already covered by a glob. Every directory under `connectors/` is a deployable connector. Shared code that isn't a connection in its own right lives in `../libs/` — currently `@plotday/google-contacts` (contact enrichment -under a shared Google auth) and `@plotday/email-classifier`. +under a shared Google auth). **Composite connectors** (`google/`, `outlook/`) offer several products under one OAuth grant. Each product's sync lives in its own subdirectory of the connector's `src/` — `google/src/{mail,calendar,tasks}`, diff --git a/libs/email-classifier/CHANGELOG.md b/libs/email-classifier/CHANGELOG.md deleted file mode 100644 index e90d9623..00000000 --- a/libs/email-classifier/CHANGELOG.md +++ /dev/null @@ -1,249 +0,0 @@ -# @plotday/email-classifier - -## 0.2.13 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.91.0 - -## 0.2.12 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.90.0 - -## 0.2.11 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.89.0 - -## 0.2.10 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.88.0 - -## 0.2.9 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.87.0 - -## 0.2.8 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.86.0 - -## 0.2.7 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.85.0 - -## 0.2.6 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.84.0 - -## 0.2.5 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.83.0 - -## 0.2.4 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.82.0 - -## 0.2.3 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.81.0 - -## 0.2.2 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.80.0 - -## 0.2.1 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.79.0 - -## 0.2.0 - -### Added - -- isNoReplySender(address) helper that reports whether an address's local part marks it as an automated/no-reply/notification sender. ([#293](https://github.com/plotday/plot/pull/293) [`8b8b972`](https://github.com/plotday/plot/commit/8b8b972690cced88b3226c39874f4030c981bed1)) - -### Changed - -- Updated dependencies: -- @plotday/twister@0.78.0 - -## 0.1.21 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.77.0 - -## 0.1.20 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.76.0 - -## 0.1.19 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.75.0 - -## 0.1.18 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.74.0 - -## 0.1.17 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.73.0 - -## 0.1.16 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.72.0 - -## 0.1.15 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.71.0 - -## 0.1.14 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.70.0 - -## 0.1.13 - -### Fixed - -- a directly-addressed reply is now classified as a message instead of a notification, even when the sending system stamps automated headers (support desks, ticketing systems). Previously short automated replies were swept into the muted FYI focus, burying real two-way conversations. ([#252](https://github.com/plotday/plot/pull/252) [`b575f76`](https://github.com/plotday/plot/commit/b575f76337a3bd6b51b0ddc28f6286b57d339d54)) - -## 0.1.12 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.69.0 - -## 0.1.11 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.68.0 - -## 0.1.10 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.67.0 - -## 0.1.9 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.66.0 - -## 0.1.8 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.65.0 - -## 0.1.7 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.64.0 - -## 0.1.6 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.63.0 - -## 0.1.5 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.62.0 - -## 0.1.4 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.61.0 - -## 0.1.3 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.60.0 - -## 0.1.2 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.59.0 - -## 0.1.1 - -### Changed - -- Updated dependencies: -- @plotday/twister@0.58.0 diff --git a/libs/email-classifier/package.json b/libs/email-classifier/package.json deleted file mode 100644 index 4920ba9c..00000000 --- a/libs/email-classifier/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@plotday/email-classifier", - "private": true, - "version": "0.2.13", - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "exports": { - ".": { - "@plotday/connector": "./src/index.ts", - "types": "./dist/index.d.ts", - "default": "./dist/index.js" - } - }, - "files": [ - "dist", - "README.md" - ], - "scripts": { - "build": "tsc", - "clean": "rm -rf dist", - "lint": "tsc --noEmit", - "test": "vitest run", - "test:watch": "vitest" - }, - "dependencies": { - "@plotday/twister": "workspace:^" - }, - "devDependencies": { - "typescript": "^5.9.3", - "vitest": "^2.1.8" - } -} diff --git a/libs/email-classifier/src/classify-email.test.ts b/libs/email-classifier/src/classify-email.test.ts deleted file mode 100644 index e8ab54c6..00000000 --- a/libs/email-classifier/src/classify-email.test.ts +++ /dev/null @@ -1,221 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { classifyEmail, isNoReplySender, type EmailSignals } from "./classify-email"; - -function signals(overrides: Partial = {}): EmailSignals { - return { - listId: null, - listUnsubscribe: null, - precedence: null, - autoSubmitted: null, - returnPath: null, - importance: null, - fromAddress: "jane@example.com", - recipientCount: 1, - isReply: false, - subject: "Hello", - bodyLength: 300, - gmailCategories: [], - bodyText: null, - fromName: null, - links: [], - authResults: null, - ...overrides, - }; -} - -describe("classifyEmail — automation", () => { - it("flags no-reply senders automated", () => { - expect(classifyEmail(signals({ fromAddress: "no-reply@acme.com" })).automation).toBe("automated"); - }); - it("flags Precedence: bulk automated", () => { - expect(classifyEmail(signals({ precedence: "bulk" })).automation).toBe("automated"); - }); - it("flags Auto-Submitted automated", () => { - expect(classifyEmail(signals({ autoSubmitted: "auto-generated" })).automation).toBe("automated"); - }); - it("flags mailing-list mail automated", () => { - expect(classifyEmail(signals({ listId: "" })).automation).toBe("automated"); - }); - it("treats a plain person email as human", () => { - expect(classifyEmail(signals()).automation).toBe("human"); - }); - it("flags a sender whose display NAME signals automation, even with a benign address", () => { - // Marketing/phishing blasts often omit List-* headers and use an ordinary - // local-part, so the address heuristics miss them — but the display name - // ("… NOTIFICATION SYSTEM") is a strong automated signal. - expect( - classifyEmail( - signals({ fromName: "BLUEREWARDS NOTIFICATION SYSTEM", fromAddress: "massage@teravistawellness.com" }) - ).automation - ).toBe("automated"); - }); - it("flags a 'do not reply' display name with an ordinary address", () => { - expect( - classifyEmail(signals({ fromName: "Acme (do not reply)", fromAddress: "hello@acme.com" })).automation - ).toBe("automated"); - }); - it("does not flag an ordinary personal display name as automated", () => { - expect(classifyEmail(signals({ fromName: "Jane Smith", fromAddress: "jane@example.com" })).automation).toBe("human"); - }); -}); - -describe("classifyEmail — reach", () => { - it("flags List-Id as list", () => { - expect(classifyEmail(signals({ listId: "" })).reach).toBe("list"); - }); - it("flags List-Unsubscribe as list", () => { - expect(classifyEmail(signals({ listUnsubscribe: "" })).reach).toBe("list"); - }); - it("flags high recipient count as list", () => { - expect(classifyEmail(signals({ recipientCount: 12 })).reach).toBe("list"); - }); - it("treats a 1:1 email as direct", () => { - expect(classifyEmail(signals()).reach).toBe("direct"); - }); -}); - -describe("classifyEmail — format", () => { - it("invoice from subject keywords", () => { - expect(classifyEmail(signals({ subject: "Your invoice is due", fromAddress: "billing@acme.com" })).format).toBe("invoice"); - }); - it("receipt from subject keywords", () => { - expect(classifyEmail(signals({ subject: "Your order confirmation #1234", fromAddress: "orders@shop.com" })).format).toBe("receipt"); - }); - it("promotion from Gmail category", () => { - expect(classifyEmail(signals({ gmailCategories: ["CATEGORY_PROMOTIONS"], listUnsubscribe: "" })).format).toBe("promotion"); - }); - it("reading from a long list email", () => { - expect( - classifyEmail(signals({ listId: "", listUnsubscribe: "", bodyLength: 4000, subject: "Weekly digest" })).format - ).toBe("reading"); - }); - it("notification from a short automated update", () => { - expect( - classifyEmail(signals({ fromAddress: "notifications@github.com", bodyLength: 200, gmailCategories: ["CATEGORY_UPDATES"] })).format - ).toBe("notification"); - }); - it("message for a normal human email", () => { - expect(classifyEmail(signals({ bodyLength: 800 })).format).toBe("message"); - }); - it("classifies a short automated email as a notification", () => { - expect(classifyEmail(signals({ fromAddress: "no-reply@x.com", bodyLength: 0, subject: null })).format).toBe("notification"); - }); - it("keeps a short automated DIRECT reply a message, not a notification", () => { - // A support desk / ticketing system stamps automated headers, but a reply - // addressed directly to the user is a two-way conversation. Without the - // direct-reply guard this short automated body falls through to the - // notification branch and gets swept into the muted FYI focus. - expect( - classifyEmail( - signals({ isReply: true, autoSubmitted: "auto-generated", bodyLength: 200, subject: "Re: RESP Withdrawals" }) - ).format - ).toBe("message"); - }); - it("still classifies a short automated DIRECT non-reply as a notification", () => { - expect( - classifyEmail( - signals({ isReply: false, fromAddress: "no-reply@x.com", bodyLength: 200 }) - ).format - ).toBe("notification"); - }); - it("does not treat a reply on a LIST email as a message", () => { - // The guard is scoped to direct reach — bulk/list replies stay as they were. - expect( - classifyEmail( - signals({ isReply: true, listId: "", listUnsubscribe: "", bodyLength: 200 }) - ).format - ).toBe("notification"); - }); - it("leaves format null when no heuristic is confident", () => { - // automated (auto_reply) but long body, direct, no categories, neutral subject → - // none of the format branches fire → null. - expect( - classifyEmail(signals({ precedence: "auto_reply", bodyLength: 900, subject: "Re: status" })).format - ).toBeNull(); - }); -}); - -describe("classifyEmail — calendar invitation responses", () => { - // Google/Outlook send automated "Accepted:/Declined:/Tentative:" emails when - // an invitee responds to a meeting invite. An acceptance is a passive - // confirmation — route it to a notification so it lands in the muted FYI focus - // ("skip active"). A decline or tentative may need follow-up (reschedule, find - // a new time), so it must stay active. - const RESPONSE_SUBJECT_TAIL = "Beth <> Kris Collab @ Wed Jun 10, 2026 10:30am (EDT)"; - - it("classifies an acceptance as a notification regardless of body length", () => { - expect( - classifyEmail( - signals({ - subject: `Accepted: ${RESPONSE_SUBJECT_TAIL}`, - fromAddress: "beth@example.com", - autoSubmitted: "auto-generated", - bodyLength: 1200, - }) - ).format - ).toBe("notification"); - }); - - it("keeps a decline active even when short and automated", () => { - // Without the calendar-response guard, a short automated email would fall - // through to the generic short-automated → notification branch and get - // swept into FYI. Declines must stay active. - expect( - classifyEmail( - signals({ - subject: `Declined: ${RESPONSE_SUBJECT_TAIL}`, - fromAddress: "beth@example.com", - autoSubmitted: "auto-generated", - bodyLength: 200, - }) - ).format - ).toBeNull(); - }); - - it("keeps a tentative response active", () => { - expect( - classifyEmail( - signals({ - subject: `Tentative: ${RESPONSE_SUBJECT_TAIL}`, - fromAddress: "beth@example.com", - autoSubmitted: "auto-generated", - bodyLength: 200, - }) - ).format - ).toBeNull(); - }); - - it("does not treat a human 'Accepted:' email as a calendar notification", () => { - // A real person writing "Accepted: ..." is not a calendar response; only - // automated invitation replies skip active. - expect( - classifyEmail( - signals({ - subject: "Accepted: your proposal", - fromAddress: "jane@example.com", - bodyLength: 800, - }) - ).format - ).toBe("message"); - }); -}); - -describe("isNoReplySender", () => { - it("flags notify@ / no-reply@ / notifications@ localparts", () => { - expect(isNoReplySender("notify@payments.interac.ca")).toBe(true); - expect(isNoReplySender("no-reply@github.com")).toBe(true); - expect(isNoReplySender("noreply@github.com")).toBe(true); - expect(isNoReplySender("notifications@github.com")).toBe(true); - expect(isNoReplySender("alerts@bank.com")).toBe(true); - }); - - it("does not flag ordinary personal addresses", () => { - expect(isNoReplySender("susan.braun@gmail.com")).toBe(false); - expect(isNoReplySender("shane@company.com")).toBe(false); - }); - - it("is null-safe", () => { - expect(isNoReplySender(null)).toBe(false); - expect(isNoReplySender("")).toBe(false); - }); -}); diff --git a/libs/email-classifier/src/classify-email.ts b/libs/email-classifier/src/classify-email.ts deleted file mode 100644 index cd62e293..00000000 --- a/libs/email-classifier/src/classify-email.ts +++ /dev/null @@ -1,171 +0,0 @@ -import type { Automation, Format, Reach, ThreadFacets } from "@plotday/twister/facets"; - -/** - * Normalized email signals an email connector assembles from raw RFC 5322 - * headers + Gmail labels. Every field is optional-by-nullability so connectors - * can populate only what they have. - */ -export type EmailSignals = { - /** List-Id header value, or null. */ - listId: string | null; - /** List-Unsubscribe header value, or null. */ - listUnsubscribe: string | null; - /** Precedence header (e.g. "bulk", "list", "auto_reply"), or null. */ - precedence: string | null; - /** Auto-Submitted header (e.g. "auto-generated"), or null. */ - autoSubmitted: string | null; - /** - * Return-Path header; "<>" / "" indicates a bounce/auto sender. (Connectors - * whose header getter coerces an empty value to null will pass null here, so - * the empty-string bounce signal only fires when the value is literally "<>".) - */ - returnPath: string | null; - /** Importance / X-Priority header, or null. Carried by connectors; reserved for future heuristics. */ - importance: string | null; - /** Sender email address, lowercased, or null. */ - fromAddress: string | null; - /** Count of To + Cc recipients. */ - recipientCount: number; - /** Whether In-Reply-To / References was present. Carried by connectors; reserved for future heuristics. */ - isReply: boolean; - /** Subject line, or null. */ - subject: string | null; - /** Length (chars) of the message body text. */ - bodyLength: number; - /** Gmail system category labels (e.g. ["CATEGORY_PROMOTIONS"]). */ - gmailCategories: string[]; - /** Plain-text body for code-keyword scanning. Null if unavailable. */ - bodyText: string | null; - /** Sender display name (e.g. "Acme Security"), for service-name derivation. Null if unavailable. */ - fromName: string | null; - /** Anchor candidates from the HTML body: visible text → href. Empty if none. */ - links: { text: string; href: string }[]; - /** Raw Authentication-Results header value, for DMARC parsing. Null if unavailable. */ - authResults: string | null; -}; - -// Recipient count at/above which a directly-addressed email is treated as a list. -const LIST_RECIPIENT_THRESHOLD = 8; -// Body length at/above which a list email reads as long-form "reading". -const READING_MIN_BODY = 1200; -// Body length below which an automated email reads as a "notification". -const NOTIFICATION_MAX_BODY = 700; - -const NOREPLY_LOCALPART = - /^(no-?reply|do-?not-?reply|donotreply|notifications?|notify|mailer-daemon|bounce|postmaster|automated|auto|alerts?|updates?)\b/; - -// Automation signals in the sender DISPLAY NAME (anywhere in the name, not just -// the start). Bulk / marketing / phishing senders routinely omit List-* headers -// and use an ordinary local-part, so the address heuristics miss them — but the -// display name ("… NOTIFICATION SYSTEM", "Acme (do not reply)") gives them away. -// Kept narrower than NOREPLY_LOCALPART: names are noisier than local-parts, so -// generic tokens (auto, alerts, updates, bounce) are excluded to avoid flagging -// real people ("Auto Desk", "Sales Updates"). -const NOREPLY_NAME = - /\b(no[-\s]?reply|do[-\s]?not[-\s]?reply|donotreply|notifications?|notify|mailer(?:[-\s]?daemon)?|postmaster|automated|auto[-\s]?(?:reply|responder))\b/i; - -// Calendar invitation-response notification emails (Google/Outlook). The -// subject is prefixed with the responder's verdict, e.g. "Accepted: ". -const CAL_RESPONSE_RE = /^\s*(accepted|declined|tentative(?:ly accepted)?):\s/i; - -function calendarResponseVerdict( - subject: string, - automation: Automation -): "accepted" | "declined" | "tentative" | null { - // Only automated invitation replies count — a human writing "Accepted: ..." - // is ordinary correspondence, not a calendar response. - if (automation !== "automated") return null; - const m = CAL_RESPONSE_RE.exec(subject); - if (!m) return null; - const verb = m[1].toLowerCase(); - if (verb === "accepted") return "accepted"; - if (verb === "declined") return "declined"; - return "tentative"; -} - -const INVOICE_RE = /\b(invoice|amount due|payment due|past due|statement|bill)\b/i; -const RECEIPT_RE = - /\b(receipt|order (confirmation|#|number)|your order|payment (received|confirmation)|thanks for your (order|purchase)|purchase confirmation)\b/i; -const PROMO_RE = /\b(sale|% off|\d+% ?off|deal|special offer|discount|coupon|save \$|limited time)\b/i; - -function localPart(address: string | null): string { - if (!address) return ""; - const at = address.indexOf("@"); - return (at === -1 ? address : address.slice(0, at)).toLowerCase(); -} - -/** - * True when an address's local part marks it as an automated / no-reply / - * notification sender (no-reply@, notify@, notifications@, alerts@, …). This is - * the identity-trust signal used to enable name-conflict detection for shared - * sender addresses; it deliberately ignores list/precedence headers (those are - * per-message automation signals, not shared-identity signals). - */ -export function isNoReplySender(address: string | null): boolean { - return NOREPLY_LOCALPART.test(localPart(address)); -} - -function computeAutomation(s: EmailSignals): Automation { - const prec = (s.precedence ?? "").toLowerCase(); - if (prec === "bulk" || prec === "list" || prec === "junk" || prec === "auto_reply") return "automated"; - const auto = (s.autoSubmitted ?? "").toLowerCase(); - if (auto && auto !== "no") return "automated"; - if (s.returnPath !== null && (s.returnPath === "" || s.returnPath === "<>")) return "automated"; - // Mailing-list headers (List-Id / List-Unsubscribe) indicate bulk/automated - // mail — newsletters, announcements, notifications. (A human posting to a - // discussion list is the rare exception we accept under best-effort.) - if (s.listId || s.listUnsubscribe) return "automated"; - if (NOREPLY_LOCALPART.test(localPart(s.fromAddress))) return "automated"; - if (s.fromName && NOREPLY_NAME.test(s.fromName)) return "automated"; - return "human"; -} - -function computeReach(s: EmailSignals): Reach { - if (s.listId || s.listUnsubscribe) return "list"; - if (s.recipientCount >= LIST_RECIPIENT_THRESHOLD) return "list"; - const prec = (s.precedence ?? "").toLowerCase(); - if (prec === "bulk" || prec === "list") return "list"; - return "direct"; -} - -function computeFormat(s: EmailSignals, automation: Automation, reach: Reach): Format | null { - const subject = s.subject ?? ""; - // Calendar invitation responses: an acceptance is a passive confirmation, so - // mark it a notification (routes to the muted FYI focus — "skip active"). A - // decline or tentative may need follow-up, so return null to keep it active - // AND to stop it falling through to the generic short-automated → notification - // branch below, which would otherwise sweep it into FYI. - const verdict = calendarResponseVerdict(subject, automation); - if (verdict === "accepted") return "notification"; - if (verdict !== null) return null; - if (INVOICE_RE.test(subject)) return "invoice"; - if (RECEIPT_RE.test(subject)) return "receipt"; - if (s.gmailCategories.includes("CATEGORY_PROMOTIONS")) return "promotion"; - if (reach === "list" && PROMO_RE.test(subject)) return "promotion"; - if (reach === "list" && s.bodyLength >= READING_MIN_BODY) return "reading"; - // A directly-addressed reply is part of a two-way conversation, so treat it - // as correspondence even when the sending system stamps automated headers - // (support desks, ticketing systems like Zendesk/Front). Without this, a - // short automated reply falls through to the notification branch below and - // gets swept into the muted FYI focus, burying real back-and-forth — e.g. a - // support agent replying "we need more info" on a request the user opened. - // Scoped to `direct` reach so list/bulk mail is unaffected. - if (s.isReply && reach === "direct") return "message"; - if (automation === "automated" && s.bodyLength < NOTIFICATION_MAX_BODY) return "notification"; - if (s.gmailCategories.includes("CATEGORY_UPDATES") || s.gmailCategories.includes("CATEGORY_SOCIAL")) { - return "notification"; - } - if (automation === "human") return "message"; - return null; -} - -/** Classify an email's intrinsic facets from normalized signals. */ -export function classifyEmail(s: EmailSignals): ThreadFacets { - const automation = computeAutomation(s); - const reach = computeReach(s); - return { - format: computeFormat(s, automation, reach), - automation, - reach, - }; -} diff --git a/libs/email-classifier/src/extract-cta.test.ts b/libs/email-classifier/src/extract-cta.test.ts deleted file mode 100644 index 838051da..00000000 --- a/libs/email-classifier/src/extract-cta.test.ts +++ /dev/null @@ -1,327 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { extractCta } from "./extract-cta"; -import type { EmailSignals } from "./classify-email"; - -function signals(over: Partial): EmailSignals { - return { - listId: null, listUnsubscribe: null, precedence: null, autoSubmitted: null, - returnPath: null, importance: null, fromAddress: null, recipientCount: 1, - isReply: false, subject: null, bodyLength: 0, gmailCategories: [], - bodyText: null, fromName: null, links: [], authResults: null, - ...over, - }; -} - -describe("extractCta — OTP", () => { - it("extracts a keyword-anchored numeric code", () => { - const cta = extractCta(signals({ - fromAddress: "no-reply@acme.com", fromName: "Acme", - subject: "Your verification code", - bodyText: "Your verification code is 482913. It expires in 10 minutes.", - })); - expect(cta).toEqual({ kind: "otp", service: "Acme", code: "482913", url: null }); - }); - it("extracts a grouped/alphanumeric code (Google style)", () => { - const cta = extractCta(signals({ - fromAddress: "no-reply@google.com", - subject: "G-557812 is your Google verification code", - bodyText: "G-557812 is your Google verification code.", - })); - expect(cta?.kind).toBe("otp"); - expect(cta?.code).toBe("G-557812"); - expect(cta?.service).toBe("Google"); - }); - it("does NOT treat an order total / price as a code", () => { - expect(extractCta(signals({ - fromAddress: "orders@shop.com", - subject: "Order confirmation", - bodyText: "Your order confirmation code total is $129456.", - }))).toBeNull(); - }); - it("does NOT treat a bare 4-digit year as a code", () => { - expect(extractCta(signals({ - bodyText: "Your verification code expires in 2026.", - }))).toBeNull(); - }); - it("extracts a code shown on the line AFTER the keyword line", () => { - // Many providers display the code prominently on its own line/box, below - // the "enter the following code" sentence — the keyword and the code are - // NOT on the same line. - const cta = extractCta(signals({ - fromAddress: "no-reply@example.com", fromName: "Example", - subject: "Verifying it's you", - bodyText: "To access Example, enter the following code:\n\nFP9I0Z\n\nThis code expires soon.", - })); - expect(cta?.kind).toBe("otp"); - expect(cta?.code).toBe("FP9I0Z"); - }); - it("extracts an interleaved alphanumeric code (e.g. FP9I0Z) on a keyword line", () => { - const cta = extractCta(signals({ - fromAddress: "no-reply@example.com", fromName: "Example", - subject: "Your access code", - bodyText: "Your access code is FP9I0Z.", - })); - expect(cta?.kind).toBe("otp"); - expect(cta?.code).toBe("FP9I0Z"); - }); - it("extracts a code from an HTML body where the code sits in its own block element", () => { - // Connectors commonly pass the raw HTML note body. Block boundaries, not - // newlines, separate the keyword sentence from the displayed code. - const cta = extractCta(signals({ - fromAddress: "no-reply@example.com", fromName: "Example", - subject: "Verifying it's you", - bodyText: - "

To access Example, enter the following code:

" + - '
FP9I0Z
' + - "

This code expires soon.

", - })); - expect(cta?.kind).toBe("otp"); - expect(cta?.code).toBe("FP9I0Z"); - }); - it("does NOT treat a number in an HTML tag attribute as a code", () => { - // Numbers living inside tag attributes (widths, tracking ids) must never - // surface — tag stripping removes them before scanning. - expect(extractCta(signals({ - fromAddress: "no-reply@shop.com", fromName: "Shop", - subject: "Your verification code", - bodyText: - "

Your verification code:

" + - '
Welcome aboard!
', - }))).toBeNull(); - }); - it("does NOT treat a bare word on the line after a keyword as a code", () => { - // The line after the keyword must contain digits to be a code — a plain - // word (sign-off, service name) must not be picked up. - expect(extractCta(signals({ - fromAddress: "no-reply@acme.com", fromName: "Acme", - subject: "Your verification code", - bodyText: "Enter the verification code below:\n\nRegards\n\nThe Acme team", - }))).toBeNull(); - }); -}); - -describe("extractCta — transactional OTP on mailing-list mail", () => { - // Real transactional senders (identity/security mail) increasingly stamp - // List-Unsubscribe on their OTP messages, which classifies them reach=list. - // A genuine one-time code must NOT be suppressed just because of that header - // — only promotional mail (format=promotion) is the false-positive class. - it("extracts an OTP from a list-classified (List-Unsubscribe) transactional mail with code in subject", () => { - const cta = extractCta(signals({ - fromAddress: "no-reply@asana.com", fromName: "Asana", - listUnsubscribe: "", - subject: "Asana confirmation code: 412855", - bodyText: - "Confirm your email address\n\n" + - "Thank you for signing up for Asana! Enter the code below in your open web browser window.\n\n" + - "412855", - bodyLength: 1400, - })); - expect(cta?.kind).toBe("otp"); - expect(cta?.code).toBe("412855"); - expect(cta?.service).toBe("Asana"); - }); - it("extracts an OTP from list-classified mail with the code only in the body", () => { - const cta = extractCta(signals({ - fromAddress: "no-reply@service.com", fromName: "Service", - listUnsubscribe: "", - subject: "Your verification code", - bodyText: "Your verification code is 771234. It expires in 10 minutes.", - })); - expect(cta?.kind).toBe("otp"); - expect(cta?.code).toBe("771234"); - }); - it("STILL suppresses a promotional (format=promotion) discount code on list mail", () => { - // The promo false-positive class stays suppressed — promotion, not merely - // list membership, is what disqualifies a code. - expect(extractCta(signals({ - fromAddress: "sale@promo.com", fromName: "Promo", - listUnsubscribe: "", - subject: "50% OFF everything this weekend!", - bodyText: "Use code 8558 at checkout for an extra discount.", - }))).toBeNull(); - }); -}); - -describe("extractCta — confirm link", () => { - const dmarcPass = "spf=pass; dkim=pass; dmarc=pass header.from=acme.com"; - it("extracts a confirm link when DMARC passes and anchor text is positive", () => { - const cta = extractCta(signals({ - fromAddress: "hello@acme.com", fromName: "Acme", - subject: "Confirm your email", - authResults: dmarcPass, - links: [{ text: "Confirm email", href: "https://acme.com/confirm?t=xyz" }], - })); - expect(cta).toEqual({ kind: "confirm", service: "Acme", code: null, url: "https://acme.com/confirm?t=xyz" }); - }); - it("SKIPS the link when DMARC does not pass", () => { - expect(extractCta(signals({ - fromAddress: "hello@acme.com", subject: "Confirm your email", - authResults: "spf=fail; dkim=none; dmarc=fail header.from=acme.com", - links: [{ text: "Confirm email", href: "https://evil.example/confirm" }], - }))).toBeNull(); - }); - it("SKIPS negative-context links (wasn't you / reset / unsubscribe)", () => { - expect(extractCta(signals({ - fromAddress: "hello@acme.com", subject: "Security alert", - authResults: dmarcPass, - links: [ - { text: "This wasn't me", href: "https://acme.com/secure" }, - { text: "Reset your password", href: "https://acme.com/reset" }, - { text: "Unsubscribe", href: "https://acme.com/u" }, - ], - }))).toBeNull(); - }); - it("SKIPS when two distinct confirm-verb links conflict", () => { - expect(extractCta(signals({ - fromAddress: "hello@acme.com", subject: "Confirm", - authResults: dmarcPass, - links: [ - { text: "Confirm email", href: "https://acme.com/a" }, - { text: "Verify account", href: "https://acme.com/b" }, - ], - }))).toBeNull(); - }); - it("prefers OTP when both a code and a confirm link are present", () => { - const cta = extractCta(signals({ - fromAddress: "hello@acme.com", fromName: "Acme", - subject: "Confirm your email", - authResults: dmarcPass, - bodyText: "Your code is 224466 or click below.", - links: [{ text: "Confirm email", href: "https://acme.com/confirm" }], - })); - expect(cta?.kind).toBe("otp"); - expect(cta?.code).toBe("224466"); - }); - - it("SKIPS a confirm link whose host is NOT the DMARC-verified sender domain", () => { - expect(extractCta(signals({ - fromAddress: "hello@acme.com", subject: "Confirm your email", - authResults: dmarcPass, // dmarc=pass header.from=acme.com - links: [{ text: "Confirm email", href: "https://evil.example/confirm" }], - }))).toBeNull(); - }); - - it("SKIPS when dmarc=pass is for a different header.from than the sender", () => { - expect(extractCta(signals({ - fromAddress: "hello@acme.com", subject: "Confirm your email", - authResults: "spf=pass; dkim=pass; dmarc=pass header.from=evil.com", - links: [{ text: "Confirm email", href: "https://evil.com/confirm" }], - }))).toBeNull(); - }); - - it("SKIPS a non-http(s) scheme link even with valid confirm text", () => { - expect(extractCta(signals({ - fromAddress: "hello@acme.com", subject: "Confirm your email", - authResults: dmarcPass, - links: [{ text: "Confirm email", href: "javascript:alert(1)" }], - }))).toBeNull(); - }); - - it("SKIPS a userinfo-spoofed link (https://sender.com@evil.com/...)", () => { - expect(extractCta(signals({ - fromAddress: "hello@acme.com", subject: "Confirm", - authResults: dmarcPass, // header.from=acme.com - links: [{ text: "Confirm email", href: "https://acme.com@evil.example/confirm" }], - }))).toBeNull(); - }); - - it("ALLOWS a confirm link on a subdomain of the verified sender domain", () => { - const cta = extractCta(signals({ - fromAddress: "hello@acme.com", fromName: "Acme", subject: "Confirm your email", - authResults: dmarcPass, - links: [{ text: "Confirm email", href: "https://login.acme.com/confirm?t=1" }], - })); - expect(cta).toEqual({ kind: "confirm", service: "Acme", code: null, url: "https://login.acme.com/confirm?t=1" }); - }); -}); - -describe("extractCta — promotional false positives", () => { - // Real samples mined from prod (martha.braun@gmail.com): bulk marketing mail - // that the old detector mis-read as an OTP because a 4-8 digit number sat near - // the bare word "code". A genuine one-time code is transactional and direct — - // never a bulk mailing-list blast — so reach=list / promotion must suppress it. - it("does NOT treat a bulk-list promo with a 'code' number as an OTP", () => { - expect(extractCta(signals({ - fromAddress: "sale@l904gw.fi86.fdske.com", fromName: "Ashley Rose Reeves", - listUnsubscribe: "", - subject: "Hydrojugs are 50% OFF!! 💥", - bodyText: "Summer blowout! Use code 8558 at checkout for an extra discount.", - }))).toBeNull(); - }); - it("does NOT treat a Gmail CATEGORY_PROMOTIONS mail (reach=direct) as an OTP", () => { - expect(extractCta(signals({ - fromAddress: "deals@modlily.com", fromName: "Modlily", - gmailCategories: ["CATEGORY_PROMOTIONS"], - subject: "Fresh Dress Arrivals Just For You 🎁", - bodyText: "Shop now with code 4070 for free shipping.", - }))).toBeNull(); - }); - it("does NOT treat a promo-context 'promo code' line as an OTP", () => { - expect(extractCta(signals({ - fromAddress: "hello@store.com", fromName: "Store", - subject: "20% off everything", - bodyText: "Enter promo code 5678 for 20% off your order.", - }))).toBeNull(); - }); - it("does NOT treat an all-same-digit placeholder as an OTP", () => { - expect(extractCta(signals({ - fromAddress: "no-reply@penningtons.com", fromName: "Penningtons", - subject: "Ends Tonight: 40% Off", - bodyText: "Your code: 000000. Shop the sale now!", - }))).toBeNull(); - }); - // Real samples (martha.braun@gmail.com): direct, automated transactional mail - // — NOT list/promotion, so the reach gate doesn't catch them — where the old - // detector spliced a digit fragment out of a tracking-link token sitting on a - // line that happened to carry a code keyword. A genuine OTP is a small, - // STANDALONE code, never a slice of a longer URL/UUID/identifier. - it("does NOT splice a digit fragment out of a tracking-link token (Reclaim)", () => { - // thread CcRuoq3tNqLxdMEYHVYLh — "Weekly Report" report email. The fragment - // 39378156 lived inside …f76e-39378156-bc8f… on a line with "Sign in". - expect(extractCta(signals({ - fromAddress: "no-reply@reclaim.ai", fromName: "Reclaim.ai", - subject: "🎉 Weekly Report at Reclaim: Jun 20 - 26", - bodyText: - "Sign in to view your stats: https://app.reclaim.ai/i/CL0/stats/0100019f0401f76e-39378156-bc8f-4558-a478-526a2151ab73-0", - }))).toBeNull(); - }); - it("does NOT treat an order-number id or a URL fragment as an OTP (Walmart)", () => { - // thread CcRquZiM1Uy9ri8GsPk7r — "Thank you for shopping with us!". The - // 15-digit order number must not be chopped to an 8-digit code, and the - // fragment 750993 inside a clickTracker URL must not surface either. - expect(extractCta(signals({ - fromAddress: "no-reply@walmart.ca", fromName: "Walmart Canada", - subject: "Thank you for shopping with us!", - bodyText: - "Thank you for shopping with us! Order number: 600000097650390.\n" + - "Access your account here: https://w-mt.ca/g/rptrcks/clickTracker?redirectTo=msnpt+750993/dFeijEb7W5", - }))).toBeNull(); - }); - it("does NOT extract a confirm CTA from a bulk-list mailing", () => { - const dmarcPass = "spf=pass; dkim=pass; dmarc=pass header.from=shop.com"; - expect(extractCta(signals({ - fromAddress: "news@shop.com", fromName: "Shop", - listUnsubscribe: "", - subject: "Confirm you still want our deals", - authResults: dmarcPass, - links: [{ text: "Confirm preferences", href: "https://shop.com/confirm" }], - }))).toBeNull(); - }); - it("STILL extracts a genuine OTP from a direct transactional mail", () => { - const cta = extractCta(signals({ - fromAddress: "no-reply@acme.com", fromName: "Acme", - subject: "Your verification code", - bodyText: "Your verification code is 482913. It expires in 10 minutes.", - })); - expect(cta).toEqual({ kind: "otp", service: "Acme", code: "482913", url: null }); - }); -}); - -describe("extractCta — none", () => { - it("returns null for ordinary mail", () => { - expect(extractCta(signals({ - fromAddress: "jane@friend.com", fromName: "Jane", - subject: "Lunch tomorrow?", bodyText: "Want to grab lunch at noon?", - }))).toBeNull(); - }); -}); diff --git a/libs/email-classifier/src/extract-cta.ts b/libs/email-classifier/src/extract-cta.ts deleted file mode 100644 index 2cdf52d7..00000000 --- a/libs/email-classifier/src/extract-cta.ts +++ /dev/null @@ -1,262 +0,0 @@ -import type { Cta } from "@plotday/twister/facets"; -import { classifyEmail, type EmailSignals } from "./classify-email"; - -// ---- domains --------------------------------------------------------------- -// Minimal multi-label public suffixes for registrable-domain comparison. -// Not a full PSL — conservative: unknown suffixes fall back to last 2 labels. -const MULTI_SUFFIX = new Set([ - "co.uk", "org.uk", "gov.uk", "ac.uk", "co.jp", "com.au", "net.au", "org.au", - "co.nz", "com.br", "co.in", "co.za", -]); - -function registrableDomain(host: string | null): string | null { - if (!host) return null; - let h = host.toLowerCase().trim().replace(/^www\./, "").replace(/:\d+$/, ""); - const parts = h.split(".").filter(Boolean); - if (parts.length < 2) return null; - const last2 = parts.slice(-2).join("."); - if (parts.length >= 3 && MULTI_SUFFIX.has(last2)) return parts.slice(-3).join("."); - return last2; -} - -function domainOfAddress(address: string | null): string | null { - if (!address) return null; - const at = address.indexOf("@"); - return at === -1 ? null : address.slice(at + 1); -} - -// ---- service name ---------------------------------------------------------- -const SERVICE_NOISE = - /\b(no-?reply|do-?not-?reply|notifications?|notify|team|support|security|account|alerts?|mail(er)?|info|hello|accounts?)\b/gi; - -function titleCase(s: string): string { - return s.replace(/\b\w/g, (c) => c.toUpperCase()); -} - -function serviceName(s: EmailSignals): string { - const name = (s.fromName ?? "") - .replace(SERVICE_NOISE, " ") - .replace(/\s+/g, " ") - .trim() - .slice(0, 60); - if (name) return name; - const reg = registrableDomain(domainOfAddress(s.fromAddress)); - if (reg) return titleCase(reg.split(".")[0]); - return "this service"; -} - -// ---- OTP code -------------------------------------------------------------- -const CODE_KEYWORD = - /(one[\s-]?time|verification|security|confirmation|access|login|sign[\s-]?in|auth(entication)?|2fa|two[\s-]?factor|otp|passcode|pass\s?code|pin|code)/i; -// A code, anchored to a WHOLE standalone token: alphanumeric (e.g. "G-557812", -// "ABZ419") OR a 4–8 digit numeric code. Numeric-only requires ≥4 digits to -// avoid matching short incidental numbers, and ≤8 so a long identifier (a -// 15-digit order number) can never satisfy it even as a slice. -const CODE_CORE = /^([A-Z]{1,4}-\d{3,8}|[A-Z]{1,4}\d{3,8}|\d{4,8})$/; -// A code with letters and digits INTERLEAVED (e.g. "FP9I0Z"), which CODE_CORE's -// letters-then-digits shape misses. Constrained to 6–8 uppercase alphanumerics -// carrying at least one letter AND one digit — tight enough that ordinary prose -// words (lowercase, no digits) and long tracking identifiers (>8 chars) can't -// satisfy it. Kept separate from CODE_CORE so the looser shape only applies -// where a code is strongly signalled (see extractOtp). -const CODE_ALNUM = /^(?=[A-Z0-9]*[A-Z])(?=[A-Z0-9]*\d)[A-Z0-9]{6,8}$/; -// Wrapper punctuation stripped from a token's ends before the whole-token test. -// Deliberately excludes '-' '/' '+' '=' '%' '.' (and alphanumerics): those are -// identifier/URL/decimal characters, so a token glued to them is NOT a -// standalone code and must fail CODE_CORE (e.g. "…f76e-39378156-bc8f", -// "…+750993/…", "1234.56"). -const WRAP = /^[.,;:!?()[\]{}<>"'*`|]+|[.,;:!?()[\]{}<>"'*`|]+$/g; -// A label that turns the following number into an identifier, not a code -// ("Order number: 600…", "Account number 12345678", "Reference number …"). -const ID_LABEL = /\bnumber\s*[:#]?\s*$/i; - -function looksLikeYear(t: string): boolean { - return /^\d{4}$/.test(t) && Number(t) >= 1900 && Number(t) <= 2100; -} - -// All-same-digit numeric tokens (0000, 9999, 000000, …) are promo placeholders, -// never real one-time codes. Cheap, high-precision FP filter for marketing mail -// that survives the reach/promotion gate (e.g. a direct "Your code: 000000"). -function looksLikePlaceholder(t: string): boolean { - return /^(\d)\1{3,}$/.test(t); -} - -// Price / order-identifier context that turns a nearby number into a total or -// an order/invoice number rather than a one-time code. -const PRICE_ID_LINE = /\$\s?\d|#\s?\d|\border\b|\binvoice\b|\btotal\b/i; - -// Block-level tags whose boundaries are visual line breaks. Turned into -// newlines so a code displayed in its own block element becomes its own line. -const BLOCK_BREAK = - /<\s*(br|\/?(p|div|td|tr|table|h[1-6]|li|ul|ol|section|header|footer|blockquote))\b[^>]*>/gi; -const HTML_ENTITY: Record = { - " ": " ", "&": "&", "<": "<", ">": ">", """: '"', "'": "'", "'": "'", -}; - -// Normalize a body that may be HTML (some connectors pass the raw HTML note -// body rather than plain text) into line-structured text for code scanning: -// block boundaries → newlines, remaining tags stripped (so attribute numbers -// like width="480123" or tracking ids never survive), common entities decoded. -// Already-plain text passes through essentially unchanged. -function htmlToLines(body: string): string { - if (!/<[a-z!/]/i.test(body)) return body; // no tags — plain text, leave as-is - return body - .replace(/<\s*(script|style)\b[^>]*>[\s\S]*?<\s*\/\s*\1\s*>/gi, " ") - .replace(BLOCK_BREAK, "\n") - .replace(/<[^>]+>/g, "") // strip any remaining tags (attributes go with them) - .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n))) - .replace(/&[a-z]+;|&#\d+;/gi, (m) => HTML_ENTITY[m.toLowerCase()] ?? " "); -} - -// True when a whole line is a single standalone token (no whitespace), the shape -// providers use to display a code prominently on its own line/box. -function isBareTokenLine(line: string): boolean { - return line.length > 0 && !/\s/.test(line); -} - -// Scan one line for a standalone code token. `allowAlnum` additionally accepts -// the looser interleaved-alphanumeric shape (CODE_ALNUM) — enabled only where a -// code is strongly signalled (a bare code line beneath a keyword line). -function scanLineForCode(line: string, allowAlnum: boolean): string | null { - for (const m of line.matchAll(/\S+/g)) { - const core = m[0].replace(WRAP, ""); - if (!CODE_CORE.test(core) && !(allowAlnum && CODE_ALNUM.test(core))) continue; - if (looksLikeYear(core)) continue; - if (looksLikePlaceholder(core)) continue; - // Reject numbers introduced by an identifier label ("order/account number"). - if (ID_LABEL.test(line.slice(0, m.index ?? 0))) continue; - return core; - } - return null; -} - -// A real one-time code is a small STANDALONE token, never a fragment spliced -// out of a longer number, URL, or UUID-like tracking token. We scan each keyword -// line word-by-word and only accept a whitespace-delimited token that matches a -// code shape in full — rejecting the dominant residual FP where a 4–8 digit run -// sat inside a tracking link (…-39378156-…, …+750993/…) on a line that happened -// to carry a code keyword. -// -// Codes are also commonly displayed on the line BELOW the "enter this code" -// sentence rather than inline, so a bare code line immediately following a -// keyword line is accepted too (and there the interleaved-alphanumeric shape is -// allowed, since the surrounding keyword context makes a false positive unlikely). -function extractOtp(s: EmailSignals): string | null { - const hay = `${s.subject ?? ""}\n${htmlToLines(s.bodyText ?? "")}`; - if (!hay.trim()) return null; - const lines = hay - .split(/\n+/) - .map((l) => l.trim()) - .filter((l) => l.length > 0); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - const isKeyword = CODE_KEYWORD.test(line) && !PRICE_ID_LINE.test(line); - // (1) Inline: a code token sitting on the keyword line itself. - if (isKeyword) { - const found = scanLineForCode(line, /* allowAlnum */ true); - if (found) return found; - } - // (2) Below: a bare code line directly under a keyword line. - if (i > 0 && isBareTokenLine(line)) { - const prev = lines[i - 1]; - const prevIsKeyword = CODE_KEYWORD.test(prev) && !PRICE_ID_LINE.test(prev); - if (prevIsKeyword) { - const found = scanLineForCode(line, /* allowAlnum */ true); - if (found) return found; - } - } - } - return null; -} - -// ---- DMARC + confirm link -------------------------------------------------- -// The DMARC-verified registrable domain from the RECEIVING provider's trusted -// Authentication-Results. Returns null unless there is a `dmarc=pass` WITH a -// `header.from` domain. -// -// SECURITY CONTRACT: the connector MUST pass only its provider MTA's -// Authentication-Results header value (selected by authserv-id), NEVER a -// sender-inserted one — otherwise `dmarc=pass` can be forged. See the Gmail / -// Outlook connector tasks for trusted-header selection. -function dmarcVerifiedDomain(authResults: string | null): string | null { - if (!authResults) return null; - // NOTE: header.from is normally a bare domain (RFC 7489 §6.6.1), but some - // MTAs emit the full mailbox form (user@domain) — strip the localpart. - const m = authResults.match( - /dmarc\s*=\s*pass\b[^;]*?header\.from\s*=\s*"?([a-z0-9.@-]+)"?/i - ); - if (!m) return null; - const raw = m[1].includes("@") ? m[1].split("@")[1] : m[1]; - return registrableDomain(raw); -} - -function httpHost(href: string): string | null { - let u: URL; - try { - u = new URL(href); - } catch { - return null; - } - if (u.protocol !== "https:" && u.protocol !== "http:") return null; - return u.hostname; -} - -const CONFIRM_VERB = - /\b(confirm|verify|activate|complete (your )?(sign[\s-]?up|registration))\b/i; -const NEGATIVE_LINK = - /\b(wasn'?t (you|me)|was not (you|me)|did ?n'?t (request|sign)|not (you|me)|reset|change (your )?password|unsubscribe|report|cancel|decline|manage|view (in|on) (browser|web)|privacy|terms|help|update preferences)\b/i; - -function extractConfirmUrl(s: EmailSignals): string | null { - const verified = dmarcVerifiedDomain(s.authResults); - if (!verified) return null; - // The DMARC-verified domain must match the sender's own domain (reject a - // dmarc=pass issued for some other header.from). - const sender = registrableDomain(domainOfAddress(s.fromAddress)); - if (!sender || sender !== verified) return null; - // The link must use http(s) AND sit on the verified registrable domain - // (subdomains allowed). This is what makes a DMARC pass meaningful for links. - const matches = s.links.filter((l) => { - if (!CONFIRM_VERB.test(l.text) || NEGATIVE_LINK.test(l.text)) return false; - const host = httpHost(l.href); - return host !== null && registrableDomain(host) === verified; - }); - if (matches.length === 0) return null; - const distinct = Array.from(new Set(matches.map((m) => m.href))); - if (distinct.length !== 1) return null; - return distinct[0]; -} - -// ---- public API ------------------------------------------------------------ -/** - * Extract a time-sensitive CTA from an email's signals. OTP wins over confirm - * when both are present. Confirm links require: a DMARC pass aligned to the - * sender's domain, an http(s) scheme, and the link host on the verified - * registrable domain. Returns null unless a high-confidence detection is made - * (bias to false-negative). - * - * Promotional mail is suppressed up front: the dominant false-positive class is - * a 4-8 digit discount code, price, or SKU sitting near the word "code" in a - * marketing email, which would otherwise fire an immediate, gate-bypassing OTP - * push for every promo the user receives. `format === "promotion"` (a - * promotional subject or a Gmail CATEGORY_PROMOTIONS label) captures that class. - * - * A genuine one-time code is NOT suppressed merely for being reach=list: many - * legitimate identity/security senders now stamp List-Unsubscribe on their OTP - * mail, which classifies it list even though the code is real. The precise - * extractor (standalone token, no price/order/placeholder/year context) plus the - * promotion gate above are what reject discount-code blasts. Confirm LINKS, - * however, stay gated to direct mail — a "confirm"/"verify" link in bulk mail is - * almost always a manage-preferences / re-engagement CTA, not account verification. - */ -export function extractCta(s: EmailSignals): Cta | null { - const { reach, format } = classifyEmail(s); - if (format === "promotion") return null; - - const service = serviceName(s); - const code = extractOtp(s); - if (code) return { kind: "otp", service, code, url: extractConfirmUrl(s) }; - if (reach === "list") return null; - const url = extractConfirmUrl(s); - if (url) return { kind: "confirm", service, code: null, url }; - return null; -} diff --git a/libs/email-classifier/src/extract-link-candidates.test.ts b/libs/email-classifier/src/extract-link-candidates.test.ts deleted file mode 100644 index 72459b7a..00000000 --- a/libs/email-classifier/src/extract-link-candidates.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { extractLinkCandidates } from "./extract-link-candidates"; - -describe("extractLinkCandidates", () => { - it("pairs anchor text with href", () => { - const html = `

Hi

Confirm email`; - expect(extractLinkCandidates(html)).toEqual([ - { text: "Confirm email", href: "https://acme.com/confirm?t=abc" }, - ]); - }); - it("collapses inner tags and whitespace in anchor text", () => { - const html = `Verify\n account`; - expect(extractLinkCandidates(html)).toEqual([ - { text: "Verify account", href: "https://x.io/v" }, - ]); - }); - it("ignores anchors without an href and non-http schemes", () => { - const html = `nopemailgo`; - expect(extractLinkCandidates(html)).toEqual([ - { text: "go", href: "https://ok.io/c" }, - ]); - }); - it("returns [] for empty/no-anchor html", () => { - expect(extractLinkCandidates("

no links

")).toEqual([]); - expect(extractLinkCandidates("")).toEqual([]); - }); -}); diff --git a/libs/email-classifier/src/extract-link-candidates.ts b/libs/email-classifier/src/extract-link-candidates.ts deleted file mode 100644 index 0164eb11..00000000 --- a/libs/email-classifier/src/extract-link-candidates.ts +++ /dev/null @@ -1,29 +0,0 @@ -export type LinkCandidate = { text: string; href: string }; - -const ANCHOR_RE = /]*?\bhref\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>([\s\S]*?)<\/a>/gi; -const TAG_RE = /<[^>]+>/g; - -function decodeEntities(s: string): string { - return s - .replace(/&/gi, "&") - .replace(/</gi, "<") - .replace(/>/gi, ">") - .replace(/"/gi, '"') - .replace(/'|'/gi, "'") - .replace(/ /gi, " "); -} - -/** Extract visible-text → href pairs from email HTML. http(s) only. */ -export function extractLinkCandidates(html: string): LinkCandidate[] { - if (!html) return []; - const out: LinkCandidate[] = []; - for (const m of html.matchAll(ANCHOR_RE)) { - const href = decodeEntities((m[2] ?? m[3] ?? m[4] ?? "").trim()); - if (!/^https?:\/\//i.test(href)) continue; - const text = decodeEntities(m[5].replace(TAG_RE, " ")) - .replace(/\s+/g, " ") - .trim(); - out.push({ text, href }); - } - return out; -} diff --git a/libs/email-classifier/src/index.ts b/libs/email-classifier/src/index.ts deleted file mode 100644 index a1d84c90..00000000 --- a/libs/email-classifier/src/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { classifyEmail, isNoReplySender, type EmailSignals } from "./classify-email"; -export { extractCta } from "./extract-cta"; -export { extractLinkCandidates, type LinkCandidate } from "./extract-link-candidates"; diff --git a/libs/email-classifier/tsconfig.json b/libs/email-classifier/tsconfig.json deleted file mode 100644 index b98a1162..00000000 --- a/libs/email-classifier/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "@plotday/twister/tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist" - }, - "include": ["src/**/*.ts"] -} diff --git a/libs/email-classifier/vitest.config.ts b/libs/email-classifier/vitest.config.ts deleted file mode 100644 index 044bd637..00000000 --- a/libs/email-classifier/vitest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - resolve: { - conditions: ["@plotday/connector", "default"], - }, - test: {}, -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 786f5441..3bfacb6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -292,19 +292,6 @@ importers: specifier: ^2.1.8 version: 2.1.9(@types/node@25.0.3) - libs/email-classifier: - dependencies: - '@plotday/twister': - specifier: workspace:^ - version: link:../../twister - devDependencies: - typescript: - specifier: ^5.9.3 - version: 5.9.3 - vitest: - specifier: ^2.1.8 - version: 2.1.9(@types/node@25.0.3) - libs/google-contacts: dependencies: '@plotday/twister': From 38bd2b731950e50fe9b7ff468104519b10b785d1 Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Mon, 3 Aug 2026 00:35:32 -0400 Subject: [PATCH 3/3] docs(twister): point NewContact.automated doc at isNoReplySender The doc comment referenced "the email classifier", which no longer exists as a concept in this package now that isNoReplySender lives directly in the signals entry point. Update the reference so connector authors reading the SDK types land on the right helper. --- twister/src/plot.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twister/src/plot.ts b/twister/src/plot.ts index 1f9ceb9d..53d5d63d 100644 --- a/twister/src/plot.ts +++ b/twister/src/plot.ts @@ -1228,8 +1228,8 @@ type NewContactBase = { * notify@payments.interac.ca, which puts a different person's name on every * message. When two different names are seen for such an address, the * runtime suppresses its name and it displays as the email address instead. - * Connectors set this from the email classifier (isNoReplySender). Omitted ⇒ - * treated as false (normal identity trust). + * Connectors set this from `isNoReplySender` in the signals entry point. + * Omitted ⇒ treated as false (normal identity trust). */ automated?: boolean; /**