From 7a46f0c7e27895bc8c99dbd94e7faf2b7b0f0d90 Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 18:28:01 -0400 Subject: [PATCH 1/8] feat(twister): add NewLink.signals for connector-emitted raw signals Connectors can emit the raw header and metadata signals they extract instead of a finished facets verdict, so classification can be improved without redeploying connectors. facets continues to work unchanged; signals wins when both are present. --- .changeset/mail-signals.md | 13 ++++++++ twister/package.json | 5 ++++ twister/src/plot.ts | 10 +++++++ twister/src/signals.ts | 61 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 .changeset/mail-signals.md create mode 100644 twister/src/signals.ts diff --git a/.changeset/mail-signals.md b/.changeset/mail-signals.md new file mode 100644 index 00000000..915e1e45 --- /dev/null +++ b/.changeset/mail-signals.md @@ -0,0 +1,13 @@ +--- +"@plotday/twister": minor +--- + +Added: `NewLink.signals` and the `@plotday/twister/signals` entry point. + +Connectors can now emit the raw signals they extract from a source item — +email headers, provider categories, recipient counts — instead of a finished +`facets` verdict. The platform derives classification from those signals, so +classification can improve without redeploying every connector, and can be +combined with recipient-relative context a connector cannot observe. + +`facets` continues to work unchanged. When a link carries both, `signals` wins. diff --git a/twister/package.json b/twister/package.json index 67441382..41b4e0f8 100644 --- a/twister/package.json +++ b/twister/package.json @@ -55,6 +55,11 @@ "types": "./dist/facets.d.ts", "default": "./dist/facets.js" }, + "./signals": { + "@plotday/connector": "./src/signals.ts", + "types": "./dist/signals.d.ts", + "default": "./dist/signals.js" + }, "./options": { "@plotday/connector": "./src/options.ts", "types": "./dist/options.d.ts", diff --git a/twister/src/plot.ts b/twister/src/plot.ts index 4c4d4b92..f2a611ae 100644 --- a/twister/src/plot.ts +++ b/twister/src/plot.ts @@ -1,5 +1,6 @@ import type { Cta, ThreadFacets } from "./facets"; import type { NewSchedule, NewScheduleOccurrence, Schedule } from "./schedule"; +import type { LinkSignals } from "./signals"; import { type Tag } from "./tag"; import { type Callback } from "./tools/callbacks"; import { type JSONValue } from "./utils/types"; @@ -1326,6 +1327,15 @@ export type NewLink = Partial< * no heuristic is confident. See `@plotday/twister/facets`. */ facets?: ThreadFacets; + /** + * Raw signals this item was derived from (email headers, provider + * categories). The platform derives classification from these. Prefer this + * over `facets`: emitting signals lets classification improve without a + * connector redeploy. See `@plotday/twister/signals`. + * + * When both are set, `signals` wins. + */ + signals?: LinkSignals; /** * The person who created this item in the external system. * diff --git a/twister/src/signals.ts b/twister/src/signals.ts new file mode 100644 index 00000000..27ef63d3 --- /dev/null +++ b/twister/src/signals.ts @@ -0,0 +1,61 @@ +/** + * Raw, normalized signals a connector extracts from a source item. Connectors + * emit these; the platform derives classification from them. + * + * This is the inverse of `./facets`: facets are a *verdict* (a connector's + * conclusion), signals are *evidence* (what the connector observed). Emitting + * signals lets the platform re-derive classification without a connector + * redeploy, and lets it combine connector observations with recipient-relative + * facts a connector cannot see (does this user know this sender?). + * + * Every field is nullable — populate only what the source actually provides. + */ + +/** Header and metadata signals from an email message. */ +export type MailSignals = { + /** List-Id header value, verbatim, or null. */ + listId: string | null; + /** List-Unsubscribe header value, verbatim, or null. */ + listUnsubscribe: string | null; + /** Precedence header ("bulk", "list", "junk", "auto_reply"), or null. */ + precedence: string | null; + /** Auto-Submitted header ("auto-generated", "no", …), or null. */ + autoSubmitted: string | null; + /** Return-Path header. "<>" or "" indicates a bounce/automated sender. */ + returnPath: string | null; + /** Importance or X-Priority header, or the provider's equivalent. */ + importance: string | null; + /** Sender address, lowercased, or null. */ + fromAddress: string | null; + /** Sender display name, or null. */ + fromName: string | null; + /** Number of To recipients. `1` means the user was the sole direct recipient. */ + toCount: number | null; + /** Number of Cc recipients. */ + ccCount: number | null; + /** True when In-Reply-To or References was present. */ + isReply: boolean | null; + /** Subject line, or null. */ + subject: string | null; + /** Length in characters of the extracted plain-text body. */ + bodyLength: number | null; + /** + * The Authentication-Results header carrying the receiving MTA's own verdict. + * The CONNECTOR selects which header to trust (only it knows its provider's + * authserv-id); the platform parses the value. Null when absent or untrusted. + */ + authResults: string | null; + /** + * Provider content categories, verbatim. Gmail's CATEGORY_PROMOTIONS / + * CATEGORY_UPDATES / CATEGORY_SOCIAL / CATEGORY_FORUMS / CATEGORY_PERSONAL, + * or another provider's equivalent bucket. + */ + providerCategories: string[]; + /** Provider user-state flags, verbatim: IMPORTANT, STARRED, FLAGGED, … */ + providerFlags: string[]; +}; + +/** Signals a connector attaches to a `NewLink`. */ +export type LinkSignals = { + mail?: MailSignals; +}; From c4d475734b075a5ae63db8021f4c169700f6af18 Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 19:26:12 -0400 Subject: [PATCH 2/8] feat(google): emit mail signals instead of classifying locally The Gmail product now reports the headers and provider categories it observed and lets the platform derive classification, so classification can improve without redeploying the connector. To and Cc counts are emitted separately rather than summed. --- .../google/src/mail/gmail-facets.test.ts | 187 +++++++++--------- connectors/google/src/mail/gmail-facets.ts | 34 ++-- connectors/google/src/mail/sync.test.ts | 6 +- connectors/google/src/mail/sync.ts | 17 +- 4 files changed, 121 insertions(+), 123 deletions(-) diff --git a/connectors/google/src/mail/gmail-facets.test.ts b/connectors/google/src/mail/gmail-facets.test.ts index 17c87965..c694e813 100644 --- a/connectors/google/src/mail/gmail-facets.test.ts +++ b/connectors/google/src/mail/gmail-facets.test.ts @@ -1,45 +1,8 @@ import { describe, expect, it } from "vitest"; -import { gmailFacets } from "./gmail-facets"; -import type { GmailMessage, GmailMessagePart } from "./gmail-api"; - -/** Encode a UTF-8 string as base64url (matches Gmail's wire format). */ -function b64url(s: string): string { - return Buffer.from(s, "utf8").toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); -} - -function msg(opts: { - headers: Array<[string, string]>; - labelIds?: string[]; - htmlBody?: string; -}): GmailMessage { - const topHeaders = opts.headers.map(([name, value]) => ({ name, value })); - - let payload: GmailMessagePart; - if (opts.htmlBody !== undefined) { - // multipart/alternative with an html part so getMessageHtml (via findPartContent) can find it - payload = { - mimeType: "multipart/alternative", - headers: topHeaders, - parts: [ - { - mimeType: "text/plain", - headers: [], - body: { size: 0, data: b64url("") }, - }, - { - mimeType: "text/html", - headers: [], - body: { size: opts.htmlBody.length, data: b64url(opts.htmlBody) }, - }, - ], - }; - } else { - payload = { - mimeType: "text/plain", - headers: topHeaders, - }; - } +import { gmailSignals } from "./gmail-facets"; +import type { GmailMessage } from "./gmail-api"; +function msg(opts: { headers: Array<[string, string]>; labelIds?: string[] }): GmailMessage { return { id: "m1", threadId: "t1", @@ -48,75 +11,113 @@ function msg(opts: { historyId: "1", internalDate: "1700000000000", sizeEstimate: 0, - payload, + payload: { + mimeType: "text/plain", + headers: opts.headers.map(([name, value]) => ({ name, value })), + }, }; } -describe("gmailFacets", () => { - it("classifies a newsletter as reading/automated/list", () => { - const { facets } = gmailFacets( - msg({ - headers: [ - ["From", "news@substack.com"], - ["To", "me@x.com"], - ["Subject", "The Weekly Digest"], - ["List-Id", ""], - ["List-Unsubscribe", ""], - ], - }), - "a".repeat(4000) - ); - expect(facets).toEqual({ format: "reading", automation: "automated", reach: "list" }); +describe("gmailSignals", () => { + it("extracts List-Id and List-Unsubscribe from a newsletter", () => { + // Previously asserted format: "reading", automation: "automated", reach: + // "list" via classifyEmail. The automated/list verdict came from these two + // headers; the reading/notification split came from body length, which is + // classifier logic now covered by derive-facets.test.ts, not this file. + const message = msg({ + headers: [ + ["From", "news@substack.com"], + ["To", "me@x.com"], + ["Subject", "The Weekly Digest"], + ["List-Id", ""], + ["List-Unsubscribe", ""], + ], + }); + const s = gmailSignals(message, 4000); + expect(s.listId).toBe(""); + expect(s.listUnsubscribe).toBe(""); }); - it("classifies a personal 1:1 email as message/human/direct", () => { - const { facets } = gmailFacets( - msg({ headers: [["From", "jane@friends.com"], ["To", "me@x.com"], ["Subject", "Lunch?"]] }), - "a".repeat(500) - ); - expect(facets).toEqual({ format: "message", automation: "human", reach: "direct" }); + it("extracts no automation signals for a personal 1:1 email", () => { + // Previously asserted format: "message", automation: "human", reach: + // "direct" via classifyEmail. The human verdict came from the absence of + // list/precedence/auto-submitted signals (now classifier logic, covered + // by derive-facets.test.ts); the direct verdict came from a single To + // recipient (toCount), covered here plus by the toCount/ccCount case below. + const message = msg({ + headers: [ + ["From", "jane@friends.com"], + ["To", "me@x.com"], + ["Subject", "Lunch?"], + ], + }); + const s = gmailSignals(message, 500); + expect(s.listId).toBeNull(); + expect(s.precedence).toBeNull(); + expect(s.autoSubmitted).toBeNull(); + expect(s.fromAddress).toBe("jane@friends.com"); + expect(s.toCount).toBe(1); }); - it("classifies a GitHub notification", () => { - const { facets } = gmailFacets( - msg({ - headers: [["From", "notifications@github.com"], ["To", "me@x.com"], ["Subject", "[repo] PR merged"]], - labelIds: ["CATEGORY_UPDATES"], - }), - "short" - ); - expect(facets.format).toBe("notification"); - expect(facets.automation).toBe("automated"); + it("extracts CATEGORY_UPDATES as a provider category", () => { + // Previously asserted format: "notification", automation: "automated" for + // a GitHub notification. The automated verdict is classifier logic over + // the sender/labels (covered by derive-facets.test.ts); the label itself + // is the signal this connector is responsible for extracting. + const message = msg({ + headers: [ + ["From", "notifications@github.com"], + ["To", "me@x.com"], + ["Subject", "[repo] PR merged"], + ], + labelIds: ["CATEGORY_UPDATES"], + }); + const s = gmailSignals(message, 5); + expect(s.providerCategories).toEqual(["CATEGORY_UPDATES"]); + }); + + it("selects the Authentication-Results header added by Google's receiving MTA", () => { + // Previously exercised CTA extraction from an HTML body link alongside + // trusted-DMARC selection; CTA extraction moved server-side (the platform + // now derives it from signals), so only the authserv-id selection — + // connector-only knowledge — remains here. + const authResults = + "mx.google.com; spf=pass smtp.mailfrom=acme.com; dkim=pass header.d=acme.com; dmarc=pass header.from=acme.com"; + const message = msg({ + headers: [ + ["From", "Acme "], + ["To", "user@example.com"], + ["Subject", "Confirm your email"], + ["Authentication-Results", authResults], + ], + }); + const s = gmailSignals(message, 100); + expect(s.authResults).toBe(authResults); }); - it("extracts a confirm cta from an HTML body link with trusted DMARC", () => { - // This test exercises getMessageHtml, which calls findPartContent (already decoded). - // With the double-decode bug (decodeBase64Url(findPartContent(...))), the HTML - // would be re-decoded as base64url and produce garbage, so extractLinkCandidates - // would find no links and cta would be null. Fix: getMessageHtml returns - // findPartContent directly without re-decoding. + it("ignores an Authentication-Results header from an untrusted authserv-id", () => { const message = msg({ headers: [ ["From", "Acme "], ["To", "user@example.com"], ["Subject", "Confirm your email"], - [ - "Authentication-Results", - "mx.google.com; spf=pass smtp.mailfrom=acme.com; dkim=pass header.d=acme.com; dmarc=pass header.from=acme.com", - ], + ["Authentication-Results", "spf1.example.net; spf=pass smtp.mailfrom=acme.com"], ], - htmlBody: `

Welcome to Acme

Confirm your email`, }); - const { facets, cta } = gmailFacets(message, "Welcome to Acme Confirm your email"); - expect(cta).toEqual({ - kind: "confirm", - service: "Acme", - code: null, - url: "https://acme.com/confirm?t=abc123", + const s = gmailSignals(message, 100); + expect(s.authResults).toBeNull(); + }); + + it("emits To and Cc counts separately", () => { + const message = msg({ + headers: [ + ["From", "a@example.com"], + ["To", "a@example.com, b@example.com"], + ["Cc", "c@example.com"], + ], }); - // gmailFacets returns raw classifyEmail output; the caller merges cta.kind into - // facets.format (see gmail.ts: `cta ? { ...facets, format: cta.kind } : facets`). - // Here we just confirm cta is present and facets is non-null. - expect(facets).not.toBeNull(); + const s = gmailSignals(message, 100); + expect(s.toCount).toBe(2); + expect(s.ccCount).toBe(1); }); }); diff --git a/connectors/google/src/mail/gmail-facets.ts b/connectors/google/src/mail/gmail-facets.ts index 2bd0025d..c17dd124 100644 --- a/connectors/google/src/mail/gmail-facets.ts +++ b/connectors/google/src/mail/gmail-facets.ts @@ -1,6 +1,5 @@ -import { classifyEmail, extractCta, extractLinkCandidates, type EmailSignals } from "@plotday/email-classifier"; -import type { Cta, ThreadFacets } from "@plotday/twister/facets"; -import { getHeader, getHeaders, getMessageHtml, parseEmailAddress, parseEmailAddresses, type GmailMessage } from "./gmail-api"; +import type { MailSignals } from "@plotday/twister/signals"; +import { getHeader, getHeaders, parseEmailAddress, parseEmailAddresses, type GmailMessage } from "./gmail-api"; const GMAIL_CATEGORY_LABELS = new Set([ "CATEGORY_PROMOTIONS", @@ -21,15 +20,21 @@ function trustedAuthResults(message: GmailMessage): string | null { return null; } -export type GmailClassification = { facets: ThreadFacets; cta: Cta | null }; +const GMAIL_FLAG_LABELS = new Set(["IMPORTANT", "STARRED"]); /** - * Compute facets and extract CTA for a Gmail message. `bodyText` is the extracted body used - * for the length heuristic (pass the same string the note will carry). + * Extract the normalized mail signals for a Gmail message. `bodyLength` is the + * character length of the extracted plain-text body — pass the length of the + * same string the note will carry, so the platform's format thresholds match + * what the connector saw. + * + * This connector no longer classifies: it reports what it observed and the + * platform decides. */ -export function gmailFacets(message: GmailMessage, bodyText: string): GmailClassification { +export function gmailSignals(message: GmailMessage, bodyLength: number): MailSignals { const from = parseEmailAddress(getHeader(message, "From") ?? ""); - const signals: EmailSignals = { + const labels = message.labelIds ?? []; + return { listId: getHeader(message, "List-Id"), listUnsubscribe: getHeader(message, "List-Unsubscribe"), precedence: getHeader(message, "Precedence"), @@ -38,16 +43,13 @@ export function gmailFacets(message: GmailMessage, bodyText: string): GmailClass importance: getHeader(message, "Importance") ?? getHeader(message, "X-Priority"), fromAddress: from?.email.toLowerCase() ?? null, fromName: from?.name ?? null, - recipientCount: - parseEmailAddresses(getHeader(message, "To")).length + - parseEmailAddresses(getHeader(message, "Cc")).length, + toCount: parseEmailAddresses(getHeader(message, "To")).length, + ccCount: parseEmailAddresses(getHeader(message, "Cc")).length, isReply: getHeader(message, "In-Reply-To") !== null || getHeader(message, "References") !== null, subject: getHeader(message, "Subject"), - bodyText, - bodyLength: bodyText.length, - links: extractLinkCandidates(getMessageHtml(message)), + bodyLength, authResults: trustedAuthResults(message), - gmailCategories: (message.labelIds ?? []).filter((l) => GMAIL_CATEGORY_LABELS.has(l)), + providerCategories: labels.filter((l) => GMAIL_CATEGORY_LABELS.has(l)), + providerFlags: labels.filter((l) => GMAIL_FLAG_LABELS.has(l)), }; - return { facets: classifyEmail(signals), cta: extractCta(signals) }; } diff --git a/connectors/google/src/mail/sync.test.ts b/connectors/google/src/mail/sync.test.ts index de1a38f3..16a2df64 100644 --- a/connectors/google/src/mail/sync.test.ts +++ b/connectors/google/src/mail/sync.test.ts @@ -1122,11 +1122,11 @@ describe("processEmailThreadsFn — attendee responses fold onto the event", () // human reply instead. expect(links[0].preview).toBe("No problem"); - // Facets must be computed from the surviving human reply, not the + // Signals must be computed from the surviving human reply, not the // folded notification: the RSVP message carries an Auto-Submitted - // header (see rsvpThread), so picking it would classify this thread as + // header (see rsvpThread), so picking it would report this thread as // automated even though a real person wrote the surviving message. - expect(links[0].facets?.automation).toBe("human"); + expect(links[0].signals?.mail?.autoSubmitted).toBeNull(); }); it("keeps the email thread when the event thread cannot be resolved", async () => { diff --git a/connectors/google/src/mail/sync.ts b/connectors/google/src/mail/sync.ts index cee65cd7..394fc655 100644 --- a/connectors/google/src/mail/sync.ts +++ b/connectors/google/src/mail/sync.ts @@ -24,7 +24,6 @@ import { type NoteWriteBackResult, resolveOutboundReplyRecipients, } from "@plotday/twister"; -import type { Cta } from "@plotday/twister/facets"; import { ActionType } from "@plotday/twister/plot"; import type { Actor, @@ -65,7 +64,7 @@ import { syncGmailMailboxIncremental, transformGmailThread, } from "./gmail-api"; -import { gmailFacets } from "./gmail-facets"; +import { gmailSignals } from "./gmail-facets"; import { type ClassifiedSendError, classifySendError, @@ -1790,9 +1789,9 @@ async function saveTransformedThread( } } - // Compute classifier facets from the parent message's headers + body. - // When the fold above dropped one or more notes, restrict the candidate - // to messages whose note survived — otherwise a folded RSVP notification + // Compute mail signals from the parent message's headers + body. When + // the fold above dropped one or more notes, restrict the candidate to + // messages whose note survived — otherwise a folded RSVP notification // (headers + snippet of an automated message) can still be picked here // and get a real human reply misclassified as automated. Skipped // entirely (same `.find()` as before) when nothing was folded, which is @@ -1812,16 +1811,12 @@ async function saveTransformedThread( ); if (facetParent) { // Use the parent message's full note body (not the short preview snippet) - // so the classifier's reading-vs-notification length split can fire. + // so the platform's reading-vs-notification length split can fire. const facetNote = plotThread.notes?.find( (n) => "key" in n && (n as { key: string }).key === facetParent.id ); const facetBody = facetNote?.content ?? plotThread.preview ?? ""; - const { facets, cta } = gmailFacets(facetParent, facetBody); - plotThread.facets = cta ? { ...facets, format: cta.kind } : facets; - if (cta && facetNote) { - (facetNote as { cta?: Cta | null }).cta = cta; - } + plotThread.signals = { mail: gmailSignals(facetParent, facetBody.length) }; } // Star ↔ todo sync: detect star changes and sync to Plot todo status. From d033bf958ae9b7b2dea727e68503f5b184d250e8 Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 20:04:24 -0400 Subject: [PATCH 3/8] fix(google): drop bodyLength from mail signals; compute it from real text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MailSignals.bodyLength let each mail connector report its own plain-text body length for classification. In practice every connector passed the extracted body's raw string length, which is markup length whenever the body is HTML — so a short, heavily-templated email could look long enough to read on tag bloat alone. The platform already holds the note content and already has an HTML-to-text fallback path, so it now derives body length itself and connectors no longer report it. gmailSignals() drops its bodyLength parameter accordingly. Also removes getMessageHtml(), an export left with no callers after an earlier change moved link/CTA extraction server-side. --- connectors/google/src/mail/gmail-api.ts | 5 ---- .../google/src/mail/gmail-facets.test.ts | 24 ++++++++++--------- connectors/google/src/mail/gmail-facets.ts | 13 +++++----- connectors/google/src/mail/sync.ts | 14 ++++------- twister/src/signals.ts | 8 +++++-- 5 files changed, 29 insertions(+), 35 deletions(-) diff --git a/connectors/google/src/mail/gmail-api.ts b/connectors/google/src/mail/gmail-api.ts index 71e8464e..e177cd69 100644 --- a/connectors/google/src/mail/gmail-api.ts +++ b/connectors/google/src/mail/gmail-api.ts @@ -624,11 +624,6 @@ export function getHeaders(message: GmailMessage, name: string): string[] { .map((h) => h.value); } -/** Decoded HTML body for a message (empty string if none). For link extraction. */ -export function getMessageHtml(message: GmailMessage): string { - return findPartContent(message.payload, "text/html") ?? ""; -} - /** * True when a mailing list (Google Groups, etc.) rewrote the `From` *address* * for DMARC alignment, so the From display name no longer belongs to the From diff --git a/connectors/google/src/mail/gmail-facets.test.ts b/connectors/google/src/mail/gmail-facets.test.ts index c694e813..bdd2beb9 100644 --- a/connectors/google/src/mail/gmail-facets.test.ts +++ b/connectors/google/src/mail/gmail-facets.test.ts @@ -23,7 +23,8 @@ describe("gmailSignals", () => { // Previously asserted format: "reading", automation: "automated", reach: // "list" via classifyEmail. The automated/list verdict came from these two // headers; the reading/notification split came from body length, which is - // classifier logic now covered by derive-facets.test.ts, not this file. + // now classifier logic covered by the platform's own facet-derivation + // tests, not this file. const message = msg({ headers: [ ["From", "news@substack.com"], @@ -33,7 +34,7 @@ describe("gmailSignals", () => { ["List-Unsubscribe", ""], ], }); - const s = gmailSignals(message, 4000); + const s = gmailSignals(message); expect(s.listId).toBe(""); expect(s.listUnsubscribe).toBe(""); }); @@ -42,8 +43,9 @@ describe("gmailSignals", () => { // Previously asserted format: "message", automation: "human", reach: // "direct" via classifyEmail. The human verdict came from the absence of // list/precedence/auto-submitted signals (now classifier logic, covered - // by derive-facets.test.ts); the direct verdict came from a single To - // recipient (toCount), covered here plus by the toCount/ccCount case below. + // elsewhere on the platform side); the direct verdict came from a single + // To recipient (toCount), covered here plus by the toCount/ccCount case + // below. const message = msg({ headers: [ ["From", "jane@friends.com"], @@ -51,7 +53,7 @@ describe("gmailSignals", () => { ["Subject", "Lunch?"], ], }); - const s = gmailSignals(message, 500); + const s = gmailSignals(message); expect(s.listId).toBeNull(); expect(s.precedence).toBeNull(); expect(s.autoSubmitted).toBeNull(); @@ -62,8 +64,8 @@ describe("gmailSignals", () => { it("extracts CATEGORY_UPDATES as a provider category", () => { // Previously asserted format: "notification", automation: "automated" for // a GitHub notification. The automated verdict is classifier logic over - // the sender/labels (covered by derive-facets.test.ts); the label itself - // is the signal this connector is responsible for extracting. + // the sender/labels (covered elsewhere on the platform side); the label + // itself is the signal this connector is responsible for extracting. const message = msg({ headers: [ ["From", "notifications@github.com"], @@ -72,7 +74,7 @@ describe("gmailSignals", () => { ], labelIds: ["CATEGORY_UPDATES"], }); - const s = gmailSignals(message, 5); + const s = gmailSignals(message); expect(s.providerCategories).toEqual(["CATEGORY_UPDATES"]); }); @@ -91,7 +93,7 @@ describe("gmailSignals", () => { ["Authentication-Results", authResults], ], }); - const s = gmailSignals(message, 100); + const s = gmailSignals(message); expect(s.authResults).toBe(authResults); }); @@ -104,7 +106,7 @@ describe("gmailSignals", () => { ["Authentication-Results", "spf1.example.net; spf=pass smtp.mailfrom=acme.com"], ], }); - const s = gmailSignals(message, 100); + const s = gmailSignals(message); expect(s.authResults).toBeNull(); }); @@ -116,7 +118,7 @@ describe("gmailSignals", () => { ["Cc", "c@example.com"], ], }); - const s = gmailSignals(message, 100); + const s = gmailSignals(message); expect(s.toCount).toBe(2); expect(s.ccCount).toBe(1); }); diff --git a/connectors/google/src/mail/gmail-facets.ts b/connectors/google/src/mail/gmail-facets.ts index c17dd124..20044183 100644 --- a/connectors/google/src/mail/gmail-facets.ts +++ b/connectors/google/src/mail/gmail-facets.ts @@ -23,15 +23,15 @@ function trustedAuthResults(message: GmailMessage): string | null { const GMAIL_FLAG_LABELS = new Set(["IMPORTANT", "STARRED"]); /** - * Extract the normalized mail signals for a Gmail message. `bodyLength` is the - * character length of the extracted plain-text body — pass the length of the - * same string the note will carry, so the platform's format thresholds match - * what the connector saw. + * Extract the normalized mail signals for a Gmail message. * * This connector no longer classifies: it reports what it observed and the - * platform decides. + * platform decides. Body length is not part of this contract — the platform + * already holds the note content and derives plain-text length from it + * itself (see `derive-facets.ts`'s `toEmailSignals`), rather than trusting a + * number each connector would otherwise have to compute independently. */ -export function gmailSignals(message: GmailMessage, bodyLength: number): MailSignals { +export function gmailSignals(message: GmailMessage): MailSignals { const from = parseEmailAddress(getHeader(message, "From") ?? ""); const labels = message.labelIds ?? []; return { @@ -47,7 +47,6 @@ export function gmailSignals(message: GmailMessage, bodyLength: number): MailSig ccCount: parseEmailAddresses(getHeader(message, "Cc")).length, isReply: getHeader(message, "In-Reply-To") !== null || getHeader(message, "References") !== null, subject: getHeader(message, "Subject"), - bodyLength, authResults: trustedAuthResults(message), providerCategories: labels.filter((l) => GMAIL_CATEGORY_LABELS.has(l)), providerFlags: labels.filter((l) => GMAIL_FLAG_LABELS.has(l)), diff --git a/connectors/google/src/mail/sync.ts b/connectors/google/src/mail/sync.ts index 394fc655..93027603 100644 --- a/connectors/google/src/mail/sync.ts +++ b/connectors/google/src/mail/sync.ts @@ -1789,9 +1789,9 @@ async function saveTransformedThread( } } - // Compute mail signals from the parent message's headers + body. When - // the fold above dropped one or more notes, restrict the candidate to - // messages whose note survived — otherwise a folded RSVP notification + // Compute mail signals from the parent message's headers. When the fold + // above dropped one or more notes, restrict the candidate to messages + // whose note survived — otherwise a folded RSVP notification // (headers + snippet of an automated message) can still be picked here // and get a real human reply misclassified as automated. Skipped // entirely (same `.find()` as before) when nothing was folded, which is @@ -1810,13 +1810,7 @@ async function saveTransformedThread( (survivingNoteKeys === null || survivingNoteKeys.has(m.id)) ); if (facetParent) { - // Use the parent message's full note body (not the short preview snippet) - // so the platform's reading-vs-notification length split can fire. - const facetNote = plotThread.notes?.find( - (n) => "key" in n && (n as { key: string }).key === facetParent.id - ); - const facetBody = facetNote?.content ?? plotThread.preview ?? ""; - plotThread.signals = { mail: gmailSignals(facetParent, facetBody.length) }; + plotThread.signals = { mail: gmailSignals(facetParent) }; } // Star ↔ todo sync: detect star changes and sync to Plot todo status. diff --git a/twister/src/signals.ts b/twister/src/signals.ts index 27ef63d3..e08ead76 100644 --- a/twister/src/signals.ts +++ b/twister/src/signals.ts @@ -9,6 +9,12 @@ * facts a connector cannot see (does this user know this sender?). * * Every field is nullable — populate only what the source actually provides. + * + * Deliberately absent: a body-length field. The platform already holds the + * note content and already has an HTML-to-plain-text fallback path + * (`stripHtmlToText` in the API's thread helpers), so it derives body length + * itself instead of trusting a number connectors would each have to compute + * (and, historically, got wrong — see `derive-facets.ts`'s `toEmailSignals`). */ /** Header and metadata signals from an email message. */ @@ -37,8 +43,6 @@ export type MailSignals = { isReply: boolean | null; /** Subject line, or null. */ subject: string | null; - /** Length in characters of the extracted plain-text body. */ - bodyLength: number | null; /** * The Authentication-Results header carrying the receiving MTA's own verdict. * The CONNECTOR selects which header to trust (only it knows its provider's From 6a6fe89a3a9aaf1cfcc15de27afed2f33fe2e575 Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 20:20:46 -0400 Subject: [PATCH 4/8] docs: remove private-repo path references from public comments A handful of comments across connectors named files that only exist in Plot's private core repository (not readable from this public repo), or described private implementation details more specifically than a reader here can act on. Reworded each to describe the platform behavior a connector author needs to know, without the internal reference. --- connectors/apple/src/mail/transform.ts | 4 ++-- connectors/google-drive/src/google-drive.ts | 4 ++-- connectors/google/src/mail/gmail-facets.ts | 4 ++-- connectors/google/src/mail/gmail-send-errors.ts | 4 ++-- connectors/slack/src/slack.test.ts | 7 +++---- connectors/slack/src/slack.ts | 6 +++--- twister/src/signals.ts | 8 ++++---- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/connectors/apple/src/mail/transform.ts b/connectors/apple/src/mail/transform.ts index 15d339ba..ed4b1cff 100644 --- a/connectors/apple/src/mail/transform.ts +++ b/connectors/apple/src/mail/transform.ts @@ -434,8 +434,8 @@ export function transformMessages( // precedence over `sentOnly`), and `title` has NO default at all: the // runtime substitutes the literal placeholder "Untitled", // PERMANENTLY, since every later pass for a still-Sent-only thread would - // also omit the key (`thread-helpers.ts`'s - // `cleanTitle(activity.title?.trim() || "Untitled")`). A degraded + // also omit the key (the platform's title-cleaning fallback substitutes + // "Untitled" whenever the title is empty). A degraded // "Re: …" subject from the Sent copy is strictly better than "Untitled", // and gets overwritten with the real subject the moment an inbound // message enters the window. See `sync.test.ts`'s "never 'Untitled'" diff --git a/connectors/google-drive/src/google-drive.ts b/connectors/google-drive/src/google-drive.ts index ed67a9fd..b2d3e0f9 100644 --- a/connectors/google-drive/src/google-drive.ts +++ b/connectors/google-drive/src/google-drive.ts @@ -74,8 +74,8 @@ function isVirtualChannel(id: string): boolean { * `(provider, accountId)` mappings in `contact_external_account`, so a * source-only contact created by a comment will be transparently merged with * the email-having contact created by the file owner (or any later sync that - * sees the same user with an email) — see `addContacts` in - * `workers/api/src/twist/tools/plot/contacts.ts`. + * sees the same user with an email) — handled automatically by the platform's + * contact-merging logic. * * The pattern (used by Linear, Asana, GitHub, Google Chat, Jira, etc.): * - Always emit `source` when you have a stable provider-side user ID. diff --git a/connectors/google/src/mail/gmail-facets.ts b/connectors/google/src/mail/gmail-facets.ts index 20044183..4a156af3 100644 --- a/connectors/google/src/mail/gmail-facets.ts +++ b/connectors/google/src/mail/gmail-facets.ts @@ -28,8 +28,8 @@ const GMAIL_FLAG_LABELS = new Set(["IMPORTANT", "STARRED"]); * This connector no longer classifies: it reports what it observed and the * platform decides. Body length is not part of this contract — the platform * already holds the note content and derives plain-text length from it - * itself (see `derive-facets.ts`'s `toEmailSignals`), rather than trusting a - * number each connector would otherwise have to compute independently. + * itself, rather than trusting a number each connector would otherwise have + * to compute independently. */ export function gmailSignals(message: GmailMessage): MailSignals { const from = parseEmailAddress(getHeader(message, "From") ?? ""); diff --git a/connectors/google/src/mail/gmail-send-errors.ts b/connectors/google/src/mail/gmail-send-errors.ts index 10c3d8b1..fd91d20a 100644 --- a/connectors/google/src/mail/gmail-send-errors.ts +++ b/connectors/google/src/mail/gmail-send-errors.ts @@ -22,8 +22,8 @@ export interface ClassifiedSendError { message: string | null; } -// Google reason markers (mirrors the server-side classifier vocabulary in -// workers/api/src/utils/transient-error.ts, which the connector can't import). +// Google reason markers (mirrors the platform's own transient-error +// classifier vocabulary, which the connector can't import). const RATE_LIMIT_MARKERS = [ "rateLimitExceeded", "userRateLimitExceeded", diff --git a/connectors/slack/src/slack.test.ts b/connectors/slack/src/slack.test.ts index 65fc59fa..91f2d3a0 100644 --- a/connectors/slack/src/slack.test.ts +++ b/connectors/slack/src/slack.test.ts @@ -3287,10 +3287,9 @@ describe("channel message routing", () => { it("never resolves a Slack token while routing a channel message", async () => { // integrations.get(channelId) treats a deliberately-disabled channel's // falsy `enabled` as "never configured" and writes a migration fallback - // that re-enables it (workers/api/src/twist/tools/integrations.ts - // ~694-728). Calling it here, on every inbound channel message, would - // silently re-enable channels the user disabled. Identity must come - // only from the `slack_user_id` cache (see `cachedMentionContext`). + // that re-enables it. Calling it here, on every inbound channel message, + // would silently re-enable channels the user disabled. Identity must + // come only from the `slack_user_id` cache (see `cachedMentionContext`). const { slack, integrationsGet } = setup(); await (slack as any).handleChannelMessage({ diff --git a/connectors/slack/src/slack.ts b/connectors/slack/src/slack.ts index 3a7593fa..1dd66f3b 100644 --- a/connectors/slack/src/slack.ts +++ b/connectors/slack/src/slack.ts @@ -178,9 +178,9 @@ const REACTION_REFRESH_MAX_ATTEMPTS = 20; * connection that covers ALL of the user's DM/MPIM conversations, instead of * one callback per conversation (which would require enumerating and * tracking hundreds of `channel` rows with no corresponding Settings UI to - * manage them). `GetSlackCallbacks` (workers/api/src/twist/tools/network.ts) - * broadcasts every incoming Slack event to every callback registered for the - * team whose granted scopes cover the event type — this sentinel is just + * manage them). The platform broadcasts every incoming Slack event to every + * callback registered for the team whose granted scopes cover the event + * type — this sentinel is just * another registered callback's extraArg, distinguished from a real * channelId by never matching a Slack conversation id format. */ diff --git a/twister/src/signals.ts b/twister/src/signals.ts index e08ead76..2f4e5ca1 100644 --- a/twister/src/signals.ts +++ b/twister/src/signals.ts @@ -11,10 +11,10 @@ * Every field is nullable — populate only what the source actually provides. * * Deliberately absent: a body-length field. The platform already holds the - * note content and already has an HTML-to-plain-text fallback path - * (`stripHtmlToText` in the API's thread helpers), so it derives body length - * itself instead of trusting a number connectors would each have to compute - * (and, historically, got wrong — see `derive-facets.ts`'s `toEmailSignals`). + * note content, so it derives body length itself instead of trusting a + * number connectors would each have to compute independently — and, + * historically, got wrong: connectors used to report the raw content + * string's length, which is markup length whenever the note is HTML. */ /** Header and metadata signals from an email message. */ From d915262d6b7fbba43fd25462b3f6035dc9ce8efe Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 20:32:09 -0400 Subject: [PATCH 5/8] feat(outlook): emit mail signals instead of classifying locally The Outlook mail product now reports observed headers, recipient counts and the Focused Inbox bucket, and lets the platform derive classification. The Focused/Other bucket is emitted in the provider's own vocabulary rather than being mapped locally. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013nPrTKxi7VmhzefnX5MDK1 --- .../outlook/src/mail/outlook-facets.test.ts | 125 ++++++++++-------- connectors/outlook/src/mail/outlook-facets.ts | 43 ++---- connectors/outlook/src/mail/sync.ts | 23 +--- 3 files changed, 84 insertions(+), 107 deletions(-) diff --git a/connectors/outlook/src/mail/outlook-facets.test.ts b/connectors/outlook/src/mail/outlook-facets.test.ts index df7369e3..88083e0c 100644 --- a/connectors/outlook/src/mail/outlook-facets.test.ts +++ b/connectors/outlook/src/mail/outlook-facets.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { outlookFacets } from "./outlook-facets"; +import { outlookSignals } from "./outlook-facets"; import type { GraphMessage } from "./graph-mail-api"; const m = (over: Partial): GraphMessage => ({ @@ -11,53 +11,58 @@ const m = (over: Partial): GraphMessage => ({ ...over, }); -describe("outlookFacets", () => { - it("newsletter with List-Id → automated/list", () => { - const { facets } = outlookFacets( - [{ name: "List-Id", value: "" }], - m({}), - "x".repeat(2000) - ); - expect(facets.automation).toBe("automated"); - expect(facets.reach).toBe("list"); - expect(facets.format).toBe("reading"); +describe("outlookSignals", () => { + it("captures a List-Id header, marking the message as list mail", () => { + // Previously asserted format: "reading", automation: "automated", reach: + // "list" via outlookFacets/classifyEmail. The automated/list verdict came + // from this header; the reading/notification split came from body + // length, which is now classifier logic covered by the platform's own + // facet-derivation tests, not this file. + const s = outlookSignals([{ name: "List-Id", value: "" }], m({})); + expect(s.listId).toBe(""); }); - it("plain human reply → human/direct/message", () => { - const { facets } = outlookFacets( - [{ name: "In-Reply-To", value: "" }], - m({ subject: "Re: Hi" }), - "short" - ); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); - expect(facets.format).toBe("message"); + it("marks a message as a reply when In-Reply-To is present", () => { + // Previously asserted format: "message", automation: "human", reach: + // "direct". The human/direct verdict is classifier logic over the + // absence of list/precedence headers and the recipient count (covered + // elsewhere on the platform side); the reply signal itself is what this + // connector is responsible for extracting. + const s = outlookSignals([{ name: "In-Reply-To", value: "" }], m({ subject: "Re: Hi" })); + expect(s.isReply).toBe(true); }); - it("short Other-inbox automated mail → notification (no headers available)", () => { - const { facets } = outlookFacets( + it("emits the provider's own inference classification verbatim", () => { + // Previously asserted automation: "automated", format: "notification" for + // a short Focused-Inbox "Other" message. The Focused/Other bucket used to + // be mapped onto a Gmail category locally; that mapping is now the + // platform's decision, so the connector just reports the provider's own + // bucket name verbatim. + const s = outlookSignals( null, m({ inferenceClassification: "other", from: { emailAddress: { address: "noreply@svc.com" } }, - }), - "tiny" + }) ); - expect(facets.automation).toBe("automated"); - expect(facets.format).toBe("notification"); + expect(s.providerCategories).toEqual(["other"]); }); - it("null headers degrade gracefully", () => { - const { facets } = outlookFacets(null, m({}), "hello there"); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); + it("null headers degrade gracefully — header-driven signals stay null", () => { + const s = outlookSignals(null, m({})); + expect(s.listId).toBeNull(); + expect(s.listUnsubscribe).toBeNull(); + expect(s.precedence).toBeNull(); + expect(s.isReply).toBe(false); }); - it("extracts a confirm cta from an HTML body link with trusted EOP auth-results", () => { - // Outlook has no decode bug (body.content is already a plain string), so this - // is coverage-only: verifies the full outlookFacets path (HTML link extraction + - // DMARC trust) works end-to-end and that the tightened authserv-id suffix match - // still accepts a real EOP sub-domain like bl0pr01.prod.protection.outlook.com. + it("captures the trusted EOP Authentication-Results header verbatim", () => { + // Previously exercised CTA extraction from an HTML body link alongside + // trusted-EOP selection; CTA extraction moved server-side (the platform + // now derives it from signals), so only the authserv-id selection — + // connector-only knowledge — remains here. Also coverage that the + // tightened authserv-id suffix match still accepts a real EOP sub-domain + // like bl0pr01.prod.protection.outlook.com. const headers = [ { name: "Authentication-Results", @@ -68,21 +73,9 @@ describe("outlookFacets", () => { const message = m({ from: { emailAddress: { address: "hello@contoso.com", name: "Contoso" } }, subject: "Confirm your account", - body: { - contentType: "html", - content: `

Welcome

Confirm your account`, - }, - }); - const { facets, cta } = outlookFacets(headers, message, "Welcome Confirm your account"); - expect(cta).toEqual({ - kind: "confirm", - service: "Contoso", - code: null, - url: "https://contoso.com/verify?token=xyz", }); - // outlookFacets returns raw classifyEmail output; the caller merges cta.kind into - // facets.format (see outlook-mail.ts: `cta ? { ...facets, format: cta.kind } : facets`). - expect(facets).not.toBeNull(); + const s = outlookSignals(headers, message); + expect(s.authResults).toBe(headers[0].value); }); it("rejects a spoofed authserv-id that merely contains protection.outlook.com", () => { @@ -90,19 +83,35 @@ describe("outlookFacets", () => { const headers = [ { name: "Authentication-Results", - value: - "evil-protection.outlook.com.attacker.com; dmarc=pass header.from=victim.com", + value: "evil-protection.outlook.com.attacker.com; dmarc=pass header.from=victim.com", }, ]; const message = m({ from: { emailAddress: { address: "no-reply@victim.com", name: "Victim" } }, - body: { - contentType: "html", - content: `Confirm`, - }, }); - const { cta } = outlookFacets(headers, message, "Confirm"); - // Without trusted auth-results the link host can't be validated → no confirm cta - expect(cta?.kind).not.toBe("confirm"); + const s = outlookSignals(headers, message); + expect(s.authResults).toBeNull(); + }); + + it("emits To and Cc counts separately", () => { + const s = outlookSignals( + null, + m({ + toRecipients: [{ emailAddress: { address: "a@x.com" } }, { emailAddress: { address: "b@x.com" } }], + ccRecipients: [{ emailAddress: { address: "c@x.com" } }], + }) + ); + expect(s.toCount).toBe(2); + expect(s.ccCount).toBe(1); + }); + + it("reports a flagged message as a FLAGGED provider flag", () => { + const s = outlookSignals(null, m({ flag: { flagStatus: "flagged" } })); + expect(s.providerFlags).toEqual(["FLAGGED"]); + }); + + it("reports no provider flags for an unflagged message", () => { + const s = outlookSignals(null, m({})); + expect(s.providerFlags).toEqual([]); }); }); diff --git a/connectors/outlook/src/mail/outlook-facets.ts b/connectors/outlook/src/mail/outlook-facets.ts index f9937e04..1aee3715 100644 --- a/connectors/outlook/src/mail/outlook-facets.ts +++ b/connectors/outlook/src/mail/outlook-facets.ts @@ -1,5 +1,4 @@ -import { classifyEmail, extractCta, extractLinkCandidates, type EmailSignals } from "@plotday/email-classifier"; -import type { Cta, ThreadFacets } from "@plotday/twister/facets"; +import type { MailSignals } from "@plotday/twister/signals"; import type { GraphHeader, GraphMessage } from "./graph-mail-api"; function header(headers: GraphHeader[] | null, name: string): string | null { @@ -17,47 +16,33 @@ function trustedAuthResults(headers: GraphHeader[] | null): string | null { return null; } -export type OutlookClassification = { facets: ThreadFacets; cta: Cta | null }; - /** - * Compute facets and extract CTA for an Outlook conversation's parent message. `headers` is - * the parent's internetMessageHeaders (separate single-message fetch; null - * when that fetch failed — header-driven signals just stay null). - * `inferenceClassification === "other"` (Focused Inbox's bulk bucket) maps to - * the classifier's CATEGORY_UPDATES slot so short automated "Other" mail - * classifies as notification, mirroring Gmail's category labels. + * Extract normalized mail signals for an Outlook conversation's parent message. + * `headers` is the parent's internetMessageHeaders (a separate single-message + * fetch; null when that fetch failed — header-driven signals then stay null). + * + * Focused Inbox's bucket is emitted verbatim ("focused" / "other"); mapping it + * onto a content category is the platform's decision, not the connector's. */ -export function outlookFacets( - headers: GraphHeader[] | null, - message: GraphMessage, - bodyText: string -): OutlookClassification { - const html = message.body?.contentType === "html" ? (message.body.content ?? "") : ""; - const signals: EmailSignals = { +export function outlookSignals(headers: GraphHeader[] | null, message: GraphMessage): MailSignals { + return { listId: header(headers, "List-Id"), listUnsubscribe: header(headers, "List-Unsubscribe"), precedence: header(headers, "Precedence"), autoSubmitted: header(headers, "Auto-Submitted"), returnPath: header(headers, "Return-Path"), - importance: - message.importance ?? - header(headers, "Importance") ?? - header(headers, "X-Priority"), + importance: message.importance ?? header(headers, "Importance") ?? header(headers, "X-Priority"), fromAddress: message.from?.emailAddress?.address?.toLowerCase() ?? null, fromName: message.from?.emailAddress?.name ?? null, - recipientCount: - (message.toRecipients?.length ?? 0) + (message.ccRecipients?.length ?? 0), + toCount: message.toRecipients?.length ?? 0, + ccCount: message.ccRecipients?.length ?? 0, isReply: header(headers, "In-Reply-To") !== null || header(headers, "References") !== null || /^re:/i.test(message.subject ?? ""), subject: message.subject ?? null, - bodyText, - bodyLength: bodyText.length, - links: extractLinkCandidates(html), authResults: trustedAuthResults(headers), - gmailCategories: - message.inferenceClassification === "other" ? ["CATEGORY_UPDATES"] : [], + providerCategories: message.inferenceClassification ? [message.inferenceClassification] : [], + providerFlags: message.flag?.flagStatus === "flagged" ? ["FLAGGED"] : [], }; - return { facets: classifyEmail(signals), cta: extractCta(signals) }; } diff --git a/connectors/outlook/src/mail/sync.ts b/connectors/outlook/src/mail/sync.ts index f8fcf933..ef689b2a 100644 --- a/connectors/outlook/src/mail/sync.ts +++ b/connectors/outlook/src/mail/sync.ts @@ -35,7 +35,6 @@ import type { Thread, } from "@plotday/twister/plot"; import type { WebhookRequest } from "@plotday/twister/tools/network"; -import type { Cta } from "@plotday/twister/facets"; import { markdownToHtml } from "@plotday/twister/utils/markdown-html"; import { enrichLinkContactsFromOutlook } from "./enrich"; @@ -55,7 +54,7 @@ import { type GraphMessage, type WellKnownFolders, } from "./graph-mail-api"; -import { outlookFacets } from "./outlook-facets"; +import { outlookSignals } from "./outlook-facets"; // --------------------------------------------------------------------------- // Constants @@ -1499,28 +1498,12 @@ export async function processConversationsFn( } } - // Compute classifier facets from the parent message's headers + body. + // Compute mail signals from the parent message's headers. const facetParent = sortConversation(item.messages).find( (m) => !m.isDraft ); if (facetParent) { - const parentKey = facetParent.internetMessageId ?? facetParent.id; - const facetNote = plotThread.notes?.find( - (n) => "key" in n && (n as { key: string }).key === parentKey - ); - const facetBody = - (facetNote as { content?: string } | undefined)?.content ?? - plotThread.preview ?? - ""; - const { facets, cta } = outlookFacets( - item.parentHeaders, - facetParent, - facetBody - ); - plotThread.facets = cta ? { ...facets, format: cta.kind } : facets; - if (cta && facetNote) { - (facetNote as { cta?: Cta | null }).cta = cta; - } + plotThread.signals = { mail: outlookSignals(item.parentHeaders, facetParent) }; } const isFlagged = isConversationFlagged(item.messages); From 207b68a36739ba7a7b283b104a99cb74f0215de4 Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 20:54:46 -0400 Subject: [PATCH 6/8] feat(apple): emit mail signals instead of classifying locally The Apple Mail product now reports observed headers and recipient counts and lets the platform derive classification. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013nPrTKxi7VmhzefnX5MDK1 --- .../apple/src/mail/apple-facets.test.ts | 154 +++++++++--------- connectors/apple/src/mail/apple-facets.ts | 23 +-- connectors/apple/src/mail/transform.test.ts | 55 +++---- connectors/apple/src/mail/transform.ts | 24 +-- 4 files changed, 112 insertions(+), 144 deletions(-) diff --git a/connectors/apple/src/mail/apple-facets.test.ts b/connectors/apple/src/mail/apple-facets.test.ts index 25e71e51..c2ad5883 100644 --- a/connectors/apple/src/mail/apple-facets.test.ts +++ b/connectors/apple/src/mail/apple-facets.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { appleMailFacets } from "./apple-facets"; +import { appleMailSignals } from "./apple-facets"; import type { MailMessage } from "./transform"; function msg(over: Partial): MailMessage { @@ -13,74 +13,72 @@ function msg(over: Partial): MailMessage { }; } -describe("appleMailFacets", () => { - it("newsletter with List-Id → automated/list", () => { - const { facets } = appleMailFacets( - msg({ listId: "" }), - "x".repeat(2000) - ); - expect(facets.automation).toBe("automated"); - expect(facets.reach).toBe("list"); - expect(facets.format).toBe("reading"); +describe("appleMailSignals", () => { + it("extracts a List-Id header from a newsletter", () => { + // Previously asserted automation: "automated", reach: "list", format: + // "reading" via classifyEmail. The automated/list verdict came from this + // header; the reading/notification split came from body length, which is + // now classifier logic covered platform-side, not this file. + const s = appleMailSignals(msg({ listId: "" })); + expect(s.listId).toBe(""); }); - it("plain human reply → human/direct/message", () => { - const { facets } = appleMailFacets( - msg({ inReplyTo: "", subject: "Re: Hi" }), - "short" - ); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); - expect(facets.format).toBe("message"); + it("marks a message as a reply when In-Reply-To is present", () => { + // Previously asserted automation: "human", reach: "direct", format: + // "message" for a plain human reply. The human/direct verdict is + // classifier logic over the absence of list/precedence headers and the + // recipient count (covered platform-side); the reply signal itself is + // what this connector is responsible for extracting. + const s = appleMailSignals(msg({ inReplyTo: "", subject: "Re: Hi" })); + expect(s.isReply).toBe(true); }); - it("short automated mail from a no-reply sender → notification", () => { - const { facets } = appleMailFacets( - msg({ from: [{ address: "no-reply@svc.com" }] }), - "tiny" - ); - expect(facets.automation).toBe("automated"); - expect(facets.format).toBe("notification"); + it("extracts a no-reply sender's address verbatim (lowercased)", () => { + // Previously asserted automation: "automated", format: "notification" + // for a short automated mail from a no-reply sender. That verdict came + // from classifier logic over the sender-address pattern and body + // length (now platform-side); the address extraction itself is what + // this connector is responsible for. + const s = appleMailSignals(msg({ from: [{ address: "No-Reply@SVC.com" }] })); + expect(s.fromAddress).toBe("no-reply@svc.com"); }); - it("message with no facet-signal headers degrades gracefully", () => { - const { facets } = appleMailFacets(msg({}), "hello there"); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); + it("extracts no automation signals for a message with no facet-signal headers", () => { + // Previously asserted automation: "human", reach: "direct" for a message + // with none of the automation-indicating headers set. Same classifier + // logic as above, covered platform-side. + const s = appleMailSignals(msg({})); + expect(s.listId).toBeNull(); + expect(s.precedence).toBeNull(); + expect(s.autoSubmitted).toBeNull(); + expect(s.isReply).toBe(false); }); - it("extracts a confirm cta from an HTML body link with a trusted iCloud auth-results", () => { + it("captures a trusted iCloud Authentication-Results header verbatim", () => { + // Previously exercised CTA extraction from an HTML body link alongside + // trusted-DMARC selection; CTA extraction moved server-side (the + // platform now derives it from signals), so only the authserv-id + // selection — connector-only knowledge — remains here. + const authResults = "icloud.com; spf=pass smtp.mailfrom=contoso.com; dkim=pass header.d=contoso.com; dmarc=pass header.from=contoso.com"; const message = msg({ from: [{ address: "hello@contoso.com", name: "Contoso" }], - subject: "Confirm your account", - bodyHtml: `

Welcome

Confirm your account`, - authenticationResults: [ - "icloud.com; spf=pass smtp.mailfrom=contoso.com; dkim=pass header.d=contoso.com; dmarc=pass header.from=contoso.com", - ], - }); - const { facets, cta } = appleMailFacets( - message, - "Welcome Confirm your account" - ); - expect(cta).toEqual({ - kind: "confirm", - service: "Contoso", - code: null, - url: "https://contoso.com/verify?token=xyz", + authenticationResults: [authResults], }); - expect(facets).not.toBeNull(); + const s = appleMailSignals(message); + expect(s.authResults).toBe(authResults); }); it("accepts a trusted iCloud auth-results reported by a specific mail-exchanger subdomain", () => { + // Coverage that the suffix match (authservId.endsWith(".icloud.com")), + // not just an exact "icloud.com" match, still accepts a real + // mail-exchanger sub-host like mx05.mail.icloud.com. + const authResults = "mx05.mail.icloud.com; dmarc=pass header.from=contoso.com"; const message = msg({ from: [{ address: "hello@contoso.com", name: "Contoso" }], - bodyHtml: `Confirm your account`, - authenticationResults: [ - "mx05.mail.icloud.com; dmarc=pass header.from=contoso.com", - ], + authenticationResults: [authResults], }); - const { cta } = appleMailFacets(message, "Confirm your account"); - expect(cta?.kind).toBe("confirm"); + const s = appleMailSignals(message); + expect(s.authResults).toBe(authResults); }); it("finds the DMARC verdict when iCloud splits SPF/DKIM/DMARC/BIMI across separate Authentication-Results headers", () => { @@ -93,49 +91,51 @@ describe("appleMailFacets", () => { // ends with .icloud.com" pick would return it and the DMARC regex would // never match — this test pins that the correct (dmarc=-bearing) header // is found regardless of header order. + const dmarcResult = "dmarc.icloud.com; dmarc=pass header.from=contoso.com"; const message = msg({ from: [{ address: "hello@contoso.com", name: "Contoso" }], - bodyHtml: `Confirm your account`, authenticationResults: [ "bimi.icloud.com; bimi=pass header.d=contoso.com header.selector=default policy.authority=pass", - "dmarc.icloud.com; dmarc=pass header.from=contoso.com", + dmarcResult, "dkim-verifier.icloud.com; dkim=pass header.d=contoso.com header.i=@contoso.com", "spf.icloud.com; spf=pass smtp.mailfrom=contoso.com", ], }); - const { cta } = appleMailFacets(message, "Confirm your account"); - expect(cta).toEqual({ - kind: "confirm", - service: "Contoso", - code: null, - url: "https://contoso.com/verify", - }); + const s = appleMailSignals(message); + expect(s.authResults).toBe(dmarcResult); }); it("rejects a spoofed authserv-id that merely contains icloud.com", () => { // evil-icloud.com.attacker.com should NOT match after the suffix tightening. const message = msg({ from: [{ address: "no-reply@victim.com", name: "Victim" }], - bodyHtml: `Confirm`, - authenticationResults: [ - "evil-icloud.com.attacker.com; dmarc=pass header.from=victim.com", - ], + authenticationResults: ["evil-icloud.com.attacker.com; dmarc=pass header.from=victim.com"], }); - const { cta } = appleMailFacets(message, "Confirm"); - // Without trusted auth-results the link host can't be validated → no confirm cta. - expect(cta?.kind).not.toBe("confirm"); + const s = appleMailSignals(message); + expect(s.authResults).toBeNull(); }); it("selects Importance over X-Priority, falling back to X-Priority when Importance is absent", () => { - const withImportance = appleMailFacets( - msg({ importance: "high", xPriority: "1" }), - "hi" + const withImportance = appleMailSignals(msg({ importance: "high", xPriority: "1" })); + expect(withImportance.importance).toBe("high"); + const withXPriorityOnly = appleMailSignals(msg({ xPriority: "1" })); + expect(withXPriorityOnly.importance).toBe("1"); + }); + + it("emits To and Cc counts separately", () => { + const s = appleMailSignals( + msg({ + to: [{ address: "me@icloud.com" }, { address: "friend@x.com" }], + cc: [{ address: "cc@x.com" }], + }) ); - const withXPriorityOnly = appleMailFacets(msg({ xPriority: "1" }), "hi"); - // Both just need to not throw and to have run the classifier — importance - // itself isn't asserted on `facets` directly (it's carried as raw signal), - // so this test pins that the fallback wiring doesn't crash either way. - expect(withImportance.facets).not.toBeNull(); - expect(withXPriorityOnly.facets).not.toBeNull(); + expect(s.toCount).toBe(2); + expect(s.ccCount).toBe(1); + }); + + it("always emits empty provider categories and flags — IMAP has no equivalent", () => { + const s = appleMailSignals(msg({})); + expect(s.providerCategories).toEqual([]); + expect(s.providerFlags).toEqual([]); }); }); diff --git a/connectors/apple/src/mail/apple-facets.ts b/connectors/apple/src/mail/apple-facets.ts index eb5a8cce..24a7abaa 100644 --- a/connectors/apple/src/mail/apple-facets.ts +++ b/connectors/apple/src/mail/apple-facets.ts @@ -1,5 +1,4 @@ -import { classifyEmail, extractCta, extractLinkCandidates, type EmailSignals } from "@plotday/email-classifier"; -import type { Cta, ThreadFacets } from "@plotday/twister/facets"; +import type { MailSignals } from "@plotday/twister/signals"; import type { MailMessage } from "./transform"; /** @@ -35,16 +34,12 @@ function trustedAuthResults(results: string[] | undefined): string | null { return null; } -export type AppleMailClassification = { facets: ThreadFacets; cta: Cta | null }; - /** - * Compute facets and extract CTA for an Apple Mail (IMAP) message. `bodyText` - * is the extracted body used for the length heuristic (pass the same string - * the note will carry). + * Extract normalized mail signals for an Apple Mail (IMAP) message. */ -export function appleMailFacets(message: MailMessage, bodyText: string): AppleMailClassification { +export function appleMailSignals(message: MailMessage): MailSignals { const from = message.from && message.from[0] ? message.from[0] : null; - const signals: EmailSignals = { + return { listId: message.listId ?? null, listUnsubscribe: message.listUnsubscribe ?? null, precedence: message.precedence ?? null, @@ -53,14 +48,12 @@ export function appleMailFacets(message: MailMessage, bodyText: string): AppleMa importance: message.importance ?? message.xPriority ?? null, fromAddress: from?.address.toLowerCase() ?? null, fromName: from?.name ?? null, - recipientCount: (message.to?.length ?? 0) + (message.cc?.length ?? 0), + toCount: message.to?.length ?? 0, + ccCount: message.cc?.length ?? 0, isReply: message.inReplyTo != null || (message.references?.length ?? 0) > 0, subject: message.subject ?? null, - bodyText, - bodyLength: bodyText.length, - links: extractLinkCandidates(message.bodyHtml ?? ""), authResults: trustedAuthResults(message.authenticationResults), - gmailCategories: [], + providerCategories: [], + providerFlags: [], }; - return { facets: classifyEmail(signals), cta: extractCta(signals) }; } diff --git a/connectors/apple/src/mail/transform.test.ts b/connectors/apple/src/mail/transform.test.ts index 54823567..a24534de 100644 --- a/connectors/apple/src/mail/transform.test.ts +++ b/connectors/apple/src/mail/transform.test.ts @@ -936,22 +936,22 @@ describe("transformMessages — Sent-only roots", () => { }); }); -describe("transformMessages — facets", () => { - it("sets link.facets from the thread's originating (earliest) message", () => { - const originator = msg({ listId: "", bodyText: "x".repeat(2000) }); +describe("transformMessages — signals", () => { + it("sets link.signals.mail from the thread's originating (earliest) message", () => { + // Previously asserted format: "reading", automation: "automated", reach: + // "list" via classifyEmail run over the originator's facets. The + // automated/list verdict came from the List-Id header below; the + // reading/notification split came from body length, which is now + // classifier logic covered platform-side, not this file. + const originator = msg({ listId: "" }); const link = transform([originator])[0]; - expect(link.facets).toEqual({ - format: "reading", - automation: "automated", - reach: "list", - }); + expect(link.signals?.mail?.listId).toBe(""); }); - it("classifies from the ORIGINATOR even when a later reply in the thread looks different", () => { + it("computes signals from the ORIGINATOR even when a later reply in the thread looks different", () => { const originator = msg({ messageId: "", listId: "", - bodyText: "x".repeat(2000), date: new Date("2026-07-15T09:00:00Z"), }); const reply = msg({ @@ -964,33 +964,16 @@ describe("transformMessages — facets", () => { date: new Date("2026-07-15T10:00:00Z"), }); const link = transform([originator, reply])[0]; - // Still classified off the newsletter-shaped originator, not the short + // Still computed off the newsletter-shaped originator, not the short // human reply — matches Gmail/Outlook's "parent message" convention. - expect(link.facets?.reach).toBe("list"); + expect(link.signals?.mail?.listId).toBe(""); }); - it("attaches an extracted CTA to the originating note and overrides facets.format with cta.kind", () => { - const originator = msg({ - from: [{ address: "security@example.com", name: "Example Security" }], - subject: "Your verification code", - bodyText: "Your one-time code is 482913. It expires in 10 minutes.", - }); - const link = transform([originator])[0]; - expect(link.facets?.format).toBe("otp"); - const note = link.notes?.[0]; - expect(note?.cta).toEqual({ - kind: "otp", - // serviceName() strips SERVICE_NOISE words like "Security" from the - // From display name — see @plotday/email-classifier/extract-cta.ts. - service: "Example", - code: "482913", - url: null, - }); - }); - - it("leaves cta unset on the note when no CTA is detected", () => { - const originator = msg({ bodyText: "just saying hi" }); - const link = transform([originator])[0]; - expect(link.notes?.[0].cta).toBeFalsy(); - }); + // The CTA-extraction cases previously here ("attaches an extracted CTA to + // the originating note …", "leaves cta unset on the note when no CTA is + // detected") are dropped, not converted: CTA extraction moved server-side + // (the platform now derives it from signals), so there is no connector + // behavior left for them to cover — matching Gmail's and Outlook's facet + // test suites, which dropped their equivalent CTA cases outright rather + // than replacing them with a signal assertion. }); diff --git a/connectors/apple/src/mail/transform.ts b/connectors/apple/src/mail/transform.ts index ed4b1cff..e37b8966 100644 --- a/connectors/apple/src/mail/transform.ts +++ b/connectors/apple/src/mail/transform.ts @@ -1,9 +1,8 @@ import type { ImapAddress, ImapMessage } from "@plotday/twister/tools/imap"; import { ActionType, type Action, type NewContact, type NewLinkWithNotes } from "@plotday/twister"; -import type { Cta } from "@plotday/twister/facets"; import { parse } from "../product-channel"; -import { appleMailFacets } from "./apple-facets"; +import { appleMailSignals } from "./apple-facets"; import { buildAttachmentRef } from "./attachments"; import { isCalendarAttachment, type CalendarBundle } from "./calendar-bundle"; import { looksLikeHtml } from "./html"; @@ -361,15 +360,14 @@ export function transformMessages( const originator = msgs[0]; const originatorFrom = originator.from && originator.from[0] ? originator.from[0] : null; - // Classifier facets are computed from the ORIGINATING message only (same + // Mail signals are computed from the ORIGINATING message only (same // "parent message" convention as the Gmail/Outlook connectors) — the // thread's overall nature (newsletter, notification, direct message) is - // set by how it started, not by whatever reply happens to be in this - // batch. `originatorBody` is recomputed below inside notes.map for the - // same message; cheap and pure, so duplicating it here (rather than - // threading the value through) keeps this block self-contained. - const originatorBody = bodyOf(originator); - const { facets, cta } = appleMailFacets(originator, originatorBody?.content ?? ""); + // derived from how it started, not from whatever reply happens to be in + // this batch. + const signals = { + mail: appleMailSignals(originator), + }; // Union of participants for thread access. const participants = new Map(); @@ -396,9 +394,6 @@ export function transformMessages( : { author: from ? toContact(from) : null }), ...(actions ? { actions } : {}), accessContacts: messageContacts(m, ownEmail), - // A time-sensitive CTA (OTP/confirm) is only ever extracted from the - // originating message — see appleMailFacets above. - ...(m === originator && cta ? { cta: cta as Cta } : {}), }; }); @@ -501,10 +496,7 @@ export function transformMessages( // owner-sent threads); explicit null when the sender is unknown, so a // From-less message is never mis-credited to the connector. author: originatorFrom ? toContact(originatorFrom) : null, - // A detected CTA overrides format with its own kind ("otp"/"confirm"), - // same convention as the Gmail/Outlook connectors — the classifier's - // generic format guess is superseded by the more specific signal. - facets: cta ? { ...facets, format: cta.kind } : facets, + signals, ...readState, ...(calendarBundle ? { sources: [`icaluid:${calendarBundle.uid}`] } : {}), ...(calendarBundle?.eventKnown || sentOnlyKnown From 36e823a92a60457960a7cacc73659cb240d2f80f Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 21:38:19 -0400 Subject: [PATCH 7/8] docs(connectors): document the signals contract for messaging connectors Replaces the facets-derivation guidance with the signals contract: report what was observed and let the platform classify, so classification logic can improve without a connector redeploy. Points the checklist/pitfalls sweep at the same section (no stale references found elsewhere). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013nPrTKxi7VmhzefnX5MDK1 --- connectors/AGENTS.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/connectors/AGENTS.md b/connectors/AGENTS.md index 71e7e71d..7087162d 100644 --- a/connectors/AGENTS.md +++ b/connectors/AGENTS.md @@ -312,9 +312,24 @@ link.meta = { ...link.meta, syncProvider: "myprovider" }; `onCreateLink` is the one exception: its return type is `CreateLinkResult`, where `channelId` is optional — the platform auto-fills it from `draft.channelId` (the channel the user composed into) if you omit it. -## Classifier facets (optional) +## Classifier signals (optional) -Messaging-style connectors may set `link.facets` (`format` / `automation` / `reach` from `@plotday/twister/facets`) as internal classifier signal. Set a dimension only when a heuristic is confident; leave it `null`/omitted otherwise. See `google/src/mail/gmail-facets.ts` and `slack/src/slack-facets.ts`. +Messaging-style connectors report the raw signals they observe and let the +platform classify. Set `link.signals` (`@plotday/twister/signals`) — for email, +`signals.mail` carries the header bundle, recipient counts, and the provider's +own category vocabulary verbatim. Do not translate a provider's vocabulary or +derive a verdict locally; the platform does both, so the logic can improve +without redeploying every connector. + +The one judgement a connector still makes is selecting which +`Authentication-Results` header to trust — only the connector knows its +provider's `authserv-id`. Emit that header's value and let the platform parse it. + +`link.facets` (a finished `{format, automation, reach}` verdict) is still +supported for connectors that have not migrated, and for non-email sources whose +signals do not fit the mail shape. When a link carries both, `signals` wins. + +See `google/src/mail/gmail-facets.ts` and `apple/src/mail/apple-facets.ts`. ## Initial vs incremental sync (REQUIRED) From 27a2b389d1cd031ac562250ff83eb7534bdadf2c Mon Sep 17 00:00:00 2001 From: Kris Braun Date: Sun, 2 Aug 2026 22:11:09 -0400 Subject: [PATCH 8/8] feat(twister): name the note that mail signals were read from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A conversation link carries one note per message, but `LinkSignals` only described one of them — the message a connector picked as the conversation's classification parent. The platform had no way to tell which, so body-derived classification (message length, links in the body) read the link's first note instead. On an incremental sync that is often a later reply, so a thread's classification could change every time someone replied to it. `LinkSignals.noteKey` closes that gap: set it to the `key` of the note built from the message the signals came from, using the same expression the note itself was keyed with. Omit it for single-note links; the platform still falls back to the first note. When the key matches no note in the link, the platform classifies from `link.preview`. The Gmail, Outlook and iCloud Mail connectors now set it — Gmail's case is the clearest: when an attendee-response message is folded onto its calendar event, the conversation's first message no longer has a note at all. --- connectors/AGENTS.md | 9 ++++++ connectors/apple/src/mail/transform.test.ts | 6 ++++ connectors/apple/src/mail/transform.ts | 4 +++ connectors/google/src/mail/sync.test.ts | 7 +++++ connectors/google/src/mail/sync.ts | 9 +++++- connectors/outlook/src/mail/bundle.test.ts | 35 +++++++++++++++++++++ connectors/outlook/src/mail/sync.ts | 10 +++++- twister/src/signals.ts | 19 +++++++++++ 8 files changed, 97 insertions(+), 2 deletions(-) diff --git a/connectors/AGENTS.md b/connectors/AGENTS.md index 7087162d..5af31519 100644 --- a/connectors/AGENTS.md +++ b/connectors/AGENTS.md @@ -325,6 +325,15 @@ The one judgement a connector still makes is selecting which `Authentication-Results` header to trust — only the connector knows its provider's `authserv-id`. Emit that header's value and let the platform parse it. +When a link carries several notes (one per message in a conversation), also set +`signals.noteKey` to the `key` of the note the signals were read from — the +message you picked as the classification parent. Body-derived classification +(how long the message is, what it links to) reads that note's content, so +without the pointer the platform falls back to the link's first note, which on +an incremental sync is often a later reply — and the thread's classification +churns every time someone replies. Use the same expression the note was keyed +with. + `link.facets` (a finished `{format, automation, reach}` verdict) is still supported for connectors that have not migrated, and for non-email sources whose signals do not fit the mail shape. When a link carries both, `signals` wins. diff --git a/connectors/apple/src/mail/transform.test.ts b/connectors/apple/src/mail/transform.test.ts index a24534de..660776f3 100644 --- a/connectors/apple/src/mail/transform.test.ts +++ b/connectors/apple/src/mail/transform.test.ts @@ -967,6 +967,12 @@ describe("transformMessages — signals", () => { // Still computed off the newsletter-shaped originator, not the short // human reply — matches Gmail/Outlook's "parent message" convention. expect(link.signals?.mail?.listId).toBe(""); + // `noteKey` names the originator's own note, so the platform reads that + // message's body for classification rather than the reply's. + expect(link.signals?.noteKey).toBe("root@example.com"); + expect((link.notes ?? []).map((n) => (n as { key?: string }).key)).toContain( + link.signals?.noteKey + ); }); // The CTA-extraction cases previously here ("attaches an extracted CTA to diff --git a/connectors/apple/src/mail/transform.ts b/connectors/apple/src/mail/transform.ts index e37b8966..24840867 100644 --- a/connectors/apple/src/mail/transform.ts +++ b/connectors/apple/src/mail/transform.ts @@ -367,6 +367,10 @@ export function transformMessages( // this batch. const signals = { mail: appleMailSignals(originator), + // Points the platform at the originator's own note (same `noteKeyOf` + // the notes below are keyed with) so body-derived classification reads + // that message, not whichever note happens to be first in this batch. + noteKey: noteKeyOf(originator), }; // Union of participants for thread access. diff --git a/connectors/google/src/mail/sync.test.ts b/connectors/google/src/mail/sync.test.ts index 16a2df64..7166e6cd 100644 --- a/connectors/google/src/mail/sync.test.ts +++ b/connectors/google/src/mail/sync.test.ts @@ -1127,6 +1127,13 @@ describe("processEmailThreadsFn — attendee responses fold onto the event", () // header (see rsvpThread), so picking it would report this thread as // automated even though a real person wrote the surviving message. expect(links[0].signals?.mail?.autoSubmitted).toBeNull(); + + // …and `noteKey` must name that same surviving message, so the platform + // reads ITS body when classifying. The folded RSVP notification is the + // thread's first message, so a link that named no note (or named the + // first one) would classify from a message this link no longer carries. + expect(links[0].signals?.noteKey).toBe("rsvp-mixed-msg-2"); + expect(keys).toContain(links[0].signals?.noteKey); }); it("keeps the email thread when the event thread cannot be resolved", async () => { diff --git a/connectors/google/src/mail/sync.ts b/connectors/google/src/mail/sync.ts index 93027603..e40aed26 100644 --- a/connectors/google/src/mail/sync.ts +++ b/connectors/google/src/mail/sync.ts @@ -1810,7 +1810,14 @@ async function saveTransformedThread( (survivingNoteKeys === null || survivingNoteKeys.has(m.id)) ); if (facetParent) { - plotThread.signals = { mail: gmailSignals(facetParent) }; + // `noteKey` points the platform at THIS message's note (notes are keyed + // on the Gmail message id — see transformGmailThread), so body-derived + // classification reads the same message the headers came from rather + // than whichever note happens to be first in this batch. + plotThread.signals = { + mail: gmailSignals(facetParent), + noteKey: facetParent.id, + }; } // Star ↔ todo sync: detect star changes and sync to Plot todo status. diff --git a/connectors/outlook/src/mail/bundle.test.ts b/connectors/outlook/src/mail/bundle.test.ts index d6afee33..2473d0c6 100644 --- a/connectors/outlook/src/mail/bundle.test.ts +++ b/connectors/outlook/src/mail/bundle.test.ts @@ -137,6 +137,41 @@ describe("processConversationsFn — calendar-thread bundling", () => { expect(saved[0].sources).toContain("icaluid:uid-3"); }); + it("names the parent message's note on the signals so the platform classifies from it", async () => { + // A draft sits ahead of the real message in the conversation. The parent + // (first non-draft) is what the signals are read from, and `noteKey` must + // point at THAT message's note — the same key the note itself carries — + // so body-derived classification never reads a different message. + const { host, saved } = makeHost(); + const item = { + messages: [ + baseMessage({ + id: "msg-draft", + internetMessageId: "", + conversationId: "conv-5", + isDraft: true, + receivedDateTime: "2026-06-01T09:00:00Z", + }), + baseMessage({ + id: "msg-5", + internetMessageId: "", + conversationId: "conv-5", + receivedDateTime: "2026-06-01T10:00:00Z", + }), + ], + attachmentsByMessageId: emptyAttachments, + parentHeaders: null as GraphHeader[] | null, + }; + + await processConversationsFn(host, [item], false, "inbox-folder"); + + expect(saved).toHaveLength(1); + expect(saved[0].signals?.noteKey).toBe(""); + expect( + (saved[0].notes ?? []).map((n) => (n as { key?: string }).key) + ).toContain(saved[0].signals?.noteKey); + }); + it("does not add icaluid sources for a plain conversation with no calendar signal", async () => { const { host, saved } = makeHost(); const item = { diff --git a/connectors/outlook/src/mail/sync.ts b/connectors/outlook/src/mail/sync.ts index ef689b2a..6879f03f 100644 --- a/connectors/outlook/src/mail/sync.ts +++ b/connectors/outlook/src/mail/sync.ts @@ -1503,7 +1503,15 @@ export async function processConversationsFn( (m) => !m.isDraft ); if (facetParent) { - plotThread.signals = { mail: outlookSignals(item.parentHeaders, facetParent) }; + // `noteKey` points the platform at THIS message's note — same + // expression the note itself is keyed with (see + // graph-mail-api.ts) — so body-derived classification reads the + // message the headers came from, not whichever note happens to be + // first in this batch. + plotThread.signals = { + mail: outlookSignals(item.parentHeaders, facetParent), + noteKey: facetParent.internetMessageId ?? facetParent.id, + }; } const isFlagged = isConversationFlagged(item.messages); diff --git a/twister/src/signals.ts b/twister/src/signals.ts index 2f4e5ca1..ea4c130c 100644 --- a/twister/src/signals.ts +++ b/twister/src/signals.ts @@ -62,4 +62,23 @@ export type MailSignals = { /** Signals a connector attaches to a `NewLink`. */ export type LinkSignals = { mail?: MailSignals; + /** + * The `key` of the note these signals were read from — the message the + * connector picked as the conversation's classification parent (normally + * the originating message). + * + * A conversation link carries one note per message, and body-derived + * classification (how long the message really is, what it links to) must + * read the SAME message the headers came from. Without this pointer the + * platform can only use the link's first note, which on an incremental + * sync is often a later reply — so a thread's classification would churn + * every time someone replies to it. + * + * Set it to the `key` of the note built from that message, using the same + * expression the note itself was keyed with. Omit it when the link has a + * single note, or when its notes are unkeyed; the platform then falls back + * to the first note. When the key matches no note in the link, the + * platform classifies from `link.preview` instead. + */ + noteKey?: string | null; };