diff --git a/.changeset/mail-signals.md b/.changeset/mail-signals.md new file mode 100644 index 00000000..915e1e45 --- /dev/null +++ b/.changeset/mail-signals.md @@ -0,0 +1,13 @@ +--- +"@plotday/twister": minor +--- + +Added: `NewLink.signals` and the `@plotday/twister/signals` entry point. + +Connectors can now emit the raw signals they extract from a source item — +email headers, provider categories, recipient counts — instead of a finished +`facets` verdict. The platform derives classification from those signals, so +classification can improve without redeploying every connector, and can be +combined with recipient-relative context a connector cannot observe. + +`facets` continues to work unchanged. When a link carries both, `signals` wins. diff --git a/connectors/AGENTS.md b/connectors/AGENTS.md index 71e7e71d..5af31519 100644 --- a/connectors/AGENTS.md +++ b/connectors/AGENTS.md @@ -312,9 +312,33 @@ link.meta = { ...link.meta, syncProvider: "myprovider" }; `onCreateLink` is the one exception: its return type is `CreateLinkResult`, where `channelId` is optional — the platform auto-fills it from `draft.channelId` (the channel the user composed into) if you omit it. -## Classifier facets (optional) - -Messaging-style connectors may set `link.facets` (`format` / `automation` / `reach` from `@plotday/twister/facets`) as internal classifier signal. Set a dimension only when a heuristic is confident; leave it `null`/omitted otherwise. See `google/src/mail/gmail-facets.ts` and `slack/src/slack-facets.ts`. +## Classifier signals (optional) + +Messaging-style connectors report the raw signals they observe and let the +platform classify. Set `link.signals` (`@plotday/twister/signals`) — for email, +`signals.mail` carries the header bundle, recipient counts, and the provider's +own category vocabulary verbatim. Do not translate a provider's vocabulary or +derive a verdict locally; the platform does both, so the logic can improve +without redeploying every connector. + +The one judgement a connector still makes is selecting which +`Authentication-Results` header to trust — only the connector knows its +provider's `authserv-id`. Emit that header's value and let the platform parse it. + +When a link carries several notes (one per message in a conversation), also set +`signals.noteKey` to the `key` of the note the signals were read from — the +message you picked as the classification parent. Body-derived classification +(how long the message is, what it links to) reads that note's content, so +without the pointer the platform falls back to the link's first note, which on +an incremental sync is often a later reply — and the thread's classification +churns every time someone replies. Use the same expression the note was keyed +with. + +`link.facets` (a finished `{format, automation, reach}` verdict) is still +supported for connectors that have not migrated, and for non-email sources whose +signals do not fit the mail shape. When a link carries both, `signals` wins. + +See `google/src/mail/gmail-facets.ts` and `apple/src/mail/apple-facets.ts`. ## Initial vs incremental sync (REQUIRED) diff --git a/connectors/apple/src/mail/apple-facets.test.ts b/connectors/apple/src/mail/apple-facets.test.ts index 25e71e51..c2ad5883 100644 --- a/connectors/apple/src/mail/apple-facets.test.ts +++ b/connectors/apple/src/mail/apple-facets.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { appleMailFacets } from "./apple-facets"; +import { appleMailSignals } from "./apple-facets"; import type { MailMessage } from "./transform"; function msg(over: Partial): MailMessage { @@ -13,74 +13,72 @@ function msg(over: Partial): MailMessage { }; } -describe("appleMailFacets", () => { - it("newsletter with List-Id → automated/list", () => { - const { facets } = appleMailFacets( - msg({ listId: "" }), - "x".repeat(2000) - ); - expect(facets.automation).toBe("automated"); - expect(facets.reach).toBe("list"); - expect(facets.format).toBe("reading"); +describe("appleMailSignals", () => { + it("extracts a List-Id header from a newsletter", () => { + // Previously asserted automation: "automated", reach: "list", format: + // "reading" via classifyEmail. The automated/list verdict came from this + // header; the reading/notification split came from body length, which is + // now classifier logic covered platform-side, not this file. + const s = appleMailSignals(msg({ listId: "" })); + expect(s.listId).toBe(""); }); - it("plain human reply → human/direct/message", () => { - const { facets } = appleMailFacets( - msg({ inReplyTo: "", subject: "Re: Hi" }), - "short" - ); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); - expect(facets.format).toBe("message"); + it("marks a message as a reply when In-Reply-To is present", () => { + // Previously asserted automation: "human", reach: "direct", format: + // "message" for a plain human reply. The human/direct verdict is + // classifier logic over the absence of list/precedence headers and the + // recipient count (covered platform-side); the reply signal itself is + // what this connector is responsible for extracting. + const s = appleMailSignals(msg({ inReplyTo: "", subject: "Re: Hi" })); + expect(s.isReply).toBe(true); }); - it("short automated mail from a no-reply sender → notification", () => { - const { facets } = appleMailFacets( - msg({ from: [{ address: "no-reply@svc.com" }] }), - "tiny" - ); - expect(facets.automation).toBe("automated"); - expect(facets.format).toBe("notification"); + it("extracts a no-reply sender's address verbatim (lowercased)", () => { + // Previously asserted automation: "automated", format: "notification" + // for a short automated mail from a no-reply sender. That verdict came + // from classifier logic over the sender-address pattern and body + // length (now platform-side); the address extraction itself is what + // this connector is responsible for. + const s = appleMailSignals(msg({ from: [{ address: "No-Reply@SVC.com" }] })); + expect(s.fromAddress).toBe("no-reply@svc.com"); }); - it("message with no facet-signal headers degrades gracefully", () => { - const { facets } = appleMailFacets(msg({}), "hello there"); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); + it("extracts no automation signals for a message with no facet-signal headers", () => { + // Previously asserted automation: "human", reach: "direct" for a message + // with none of the automation-indicating headers set. Same classifier + // logic as above, covered platform-side. + const s = appleMailSignals(msg({})); + expect(s.listId).toBeNull(); + expect(s.precedence).toBeNull(); + expect(s.autoSubmitted).toBeNull(); + expect(s.isReply).toBe(false); }); - it("extracts a confirm cta from an HTML body link with a trusted iCloud auth-results", () => { + it("captures a trusted iCloud Authentication-Results header verbatim", () => { + // Previously exercised CTA extraction from an HTML body link alongside + // trusted-DMARC selection; CTA extraction moved server-side (the + // platform now derives it from signals), so only the authserv-id + // selection — connector-only knowledge — remains here. + const authResults = "icloud.com; spf=pass smtp.mailfrom=contoso.com; dkim=pass header.d=contoso.com; dmarc=pass header.from=contoso.com"; const message = msg({ from: [{ address: "hello@contoso.com", name: "Contoso" }], - subject: "Confirm your account", - bodyHtml: `

Welcome

Confirm your account`, - authenticationResults: [ - "icloud.com; spf=pass smtp.mailfrom=contoso.com; dkim=pass header.d=contoso.com; dmarc=pass header.from=contoso.com", - ], - }); - const { facets, cta } = appleMailFacets( - message, - "Welcome Confirm your account" - ); - expect(cta).toEqual({ - kind: "confirm", - service: "Contoso", - code: null, - url: "https://contoso.com/verify?token=xyz", + authenticationResults: [authResults], }); - expect(facets).not.toBeNull(); + const s = appleMailSignals(message); + expect(s.authResults).toBe(authResults); }); it("accepts a trusted iCloud auth-results reported by a specific mail-exchanger subdomain", () => { + // Coverage that the suffix match (authservId.endsWith(".icloud.com")), + // not just an exact "icloud.com" match, still accepts a real + // mail-exchanger sub-host like mx05.mail.icloud.com. + const authResults = "mx05.mail.icloud.com; dmarc=pass header.from=contoso.com"; const message = msg({ from: [{ address: "hello@contoso.com", name: "Contoso" }], - bodyHtml: `Confirm your account`, - authenticationResults: [ - "mx05.mail.icloud.com; dmarc=pass header.from=contoso.com", - ], + authenticationResults: [authResults], }); - const { cta } = appleMailFacets(message, "Confirm your account"); - expect(cta?.kind).toBe("confirm"); + const s = appleMailSignals(message); + expect(s.authResults).toBe(authResults); }); it("finds the DMARC verdict when iCloud splits SPF/DKIM/DMARC/BIMI across separate Authentication-Results headers", () => { @@ -93,49 +91,51 @@ describe("appleMailFacets", () => { // ends with .icloud.com" pick would return it and the DMARC regex would // never match — this test pins that the correct (dmarc=-bearing) header // is found regardless of header order. + const dmarcResult = "dmarc.icloud.com; dmarc=pass header.from=contoso.com"; const message = msg({ from: [{ address: "hello@contoso.com", name: "Contoso" }], - bodyHtml: `Confirm your account`, authenticationResults: [ "bimi.icloud.com; bimi=pass header.d=contoso.com header.selector=default policy.authority=pass", - "dmarc.icloud.com; dmarc=pass header.from=contoso.com", + dmarcResult, "dkim-verifier.icloud.com; dkim=pass header.d=contoso.com header.i=@contoso.com", "spf.icloud.com; spf=pass smtp.mailfrom=contoso.com", ], }); - const { cta } = appleMailFacets(message, "Confirm your account"); - expect(cta).toEqual({ - kind: "confirm", - service: "Contoso", - code: null, - url: "https://contoso.com/verify", - }); + const s = appleMailSignals(message); + expect(s.authResults).toBe(dmarcResult); }); it("rejects a spoofed authserv-id that merely contains icloud.com", () => { // evil-icloud.com.attacker.com should NOT match after the suffix tightening. const message = msg({ from: [{ address: "no-reply@victim.com", name: "Victim" }], - bodyHtml: `Confirm`, - authenticationResults: [ - "evil-icloud.com.attacker.com; dmarc=pass header.from=victim.com", - ], + authenticationResults: ["evil-icloud.com.attacker.com; dmarc=pass header.from=victim.com"], }); - const { cta } = appleMailFacets(message, "Confirm"); - // Without trusted auth-results the link host can't be validated → no confirm cta. - expect(cta?.kind).not.toBe("confirm"); + const s = appleMailSignals(message); + expect(s.authResults).toBeNull(); }); it("selects Importance over X-Priority, falling back to X-Priority when Importance is absent", () => { - const withImportance = appleMailFacets( - msg({ importance: "high", xPriority: "1" }), - "hi" + const withImportance = appleMailSignals(msg({ importance: "high", xPriority: "1" })); + expect(withImportance.importance).toBe("high"); + const withXPriorityOnly = appleMailSignals(msg({ xPriority: "1" })); + expect(withXPriorityOnly.importance).toBe("1"); + }); + + it("emits To and Cc counts separately", () => { + const s = appleMailSignals( + msg({ + to: [{ address: "me@icloud.com" }, { address: "friend@x.com" }], + cc: [{ address: "cc@x.com" }], + }) ); - const withXPriorityOnly = appleMailFacets(msg({ xPriority: "1" }), "hi"); - // Both just need to not throw and to have run the classifier — importance - // itself isn't asserted on `facets` directly (it's carried as raw signal), - // so this test pins that the fallback wiring doesn't crash either way. - expect(withImportance.facets).not.toBeNull(); - expect(withXPriorityOnly.facets).not.toBeNull(); + expect(s.toCount).toBe(2); + expect(s.ccCount).toBe(1); + }); + + it("always emits empty provider categories and flags — IMAP has no equivalent", () => { + const s = appleMailSignals(msg({})); + expect(s.providerCategories).toEqual([]); + expect(s.providerFlags).toEqual([]); }); }); diff --git a/connectors/apple/src/mail/apple-facets.ts b/connectors/apple/src/mail/apple-facets.ts index eb5a8cce..24a7abaa 100644 --- a/connectors/apple/src/mail/apple-facets.ts +++ b/connectors/apple/src/mail/apple-facets.ts @@ -1,5 +1,4 @@ -import { classifyEmail, extractCta, extractLinkCandidates, type EmailSignals } from "@plotday/email-classifier"; -import type { Cta, ThreadFacets } from "@plotday/twister/facets"; +import type { MailSignals } from "@plotday/twister/signals"; import type { MailMessage } from "./transform"; /** @@ -35,16 +34,12 @@ function trustedAuthResults(results: string[] | undefined): string | null { return null; } -export type AppleMailClassification = { facets: ThreadFacets; cta: Cta | null }; - /** - * Compute facets and extract CTA for an Apple Mail (IMAP) message. `bodyText` - * is the extracted body used for the length heuristic (pass the same string - * the note will carry). + * Extract normalized mail signals for an Apple Mail (IMAP) message. */ -export function appleMailFacets(message: MailMessage, bodyText: string): AppleMailClassification { +export function appleMailSignals(message: MailMessage): MailSignals { const from = message.from && message.from[0] ? message.from[0] : null; - const signals: EmailSignals = { + return { listId: message.listId ?? null, listUnsubscribe: message.listUnsubscribe ?? null, precedence: message.precedence ?? null, @@ -53,14 +48,12 @@ export function appleMailFacets(message: MailMessage, bodyText: string): AppleMa importance: message.importance ?? message.xPriority ?? null, fromAddress: from?.address.toLowerCase() ?? null, fromName: from?.name ?? null, - recipientCount: (message.to?.length ?? 0) + (message.cc?.length ?? 0), + toCount: message.to?.length ?? 0, + ccCount: message.cc?.length ?? 0, isReply: message.inReplyTo != null || (message.references?.length ?? 0) > 0, subject: message.subject ?? null, - bodyText, - bodyLength: bodyText.length, - links: extractLinkCandidates(message.bodyHtml ?? ""), authResults: trustedAuthResults(message.authenticationResults), - gmailCategories: [], + providerCategories: [], + providerFlags: [], }; - return { facets: classifyEmail(signals), cta: extractCta(signals) }; } diff --git a/connectors/apple/src/mail/transform.test.ts b/connectors/apple/src/mail/transform.test.ts index 54823567..660776f3 100644 --- a/connectors/apple/src/mail/transform.test.ts +++ b/connectors/apple/src/mail/transform.test.ts @@ -936,22 +936,22 @@ describe("transformMessages — Sent-only roots", () => { }); }); -describe("transformMessages — facets", () => { - it("sets link.facets from the thread's originating (earliest) message", () => { - const originator = msg({ listId: "", bodyText: "x".repeat(2000) }); +describe("transformMessages — signals", () => { + it("sets link.signals.mail from the thread's originating (earliest) message", () => { + // Previously asserted format: "reading", automation: "automated", reach: + // "list" via classifyEmail run over the originator's facets. The + // automated/list verdict came from the List-Id header below; the + // reading/notification split came from body length, which is now + // classifier logic covered platform-side, not this file. + const originator = msg({ listId: "" }); const link = transform([originator])[0]; - expect(link.facets).toEqual({ - format: "reading", - automation: "automated", - reach: "list", - }); + expect(link.signals?.mail?.listId).toBe(""); }); - it("classifies from the ORIGINATOR even when a later reply in the thread looks different", () => { + it("computes signals from the ORIGINATOR even when a later reply in the thread looks different", () => { const originator = msg({ messageId: "", listId: "", - bodyText: "x".repeat(2000), date: new Date("2026-07-15T09:00:00Z"), }); const reply = msg({ @@ -964,33 +964,22 @@ describe("transformMessages — facets", () => { date: new Date("2026-07-15T10:00:00Z"), }); const link = transform([originator, reply])[0]; - // Still classified off the newsletter-shaped originator, not the short + // Still computed off the newsletter-shaped originator, not the short // human reply — matches Gmail/Outlook's "parent message" convention. - expect(link.facets?.reach).toBe("list"); - }); - - it("attaches an extracted CTA to the originating note and overrides facets.format with cta.kind", () => { - const originator = msg({ - from: [{ address: "security@example.com", name: "Example Security" }], - subject: "Your verification code", - bodyText: "Your one-time code is 482913. It expires in 10 minutes.", - }); - const link = transform([originator])[0]; - expect(link.facets?.format).toBe("otp"); - const note = link.notes?.[0]; - expect(note?.cta).toEqual({ - kind: "otp", - // serviceName() strips SERVICE_NOISE words like "Security" from the - // From display name — see @plotday/email-classifier/extract-cta.ts. - service: "Example", - code: "482913", - url: null, - }); + expect(link.signals?.mail?.listId).toBe(""); + // `noteKey` names the originator's own note, so the platform reads that + // message's body for classification rather than the reply's. + expect(link.signals?.noteKey).toBe("root@example.com"); + expect((link.notes ?? []).map((n) => (n as { key?: string }).key)).toContain( + link.signals?.noteKey + ); }); - it("leaves cta unset on the note when no CTA is detected", () => { - const originator = msg({ bodyText: "just saying hi" }); - const link = transform([originator])[0]; - expect(link.notes?.[0].cta).toBeFalsy(); - }); + // The CTA-extraction cases previously here ("attaches an extracted CTA to + // the originating note …", "leaves cta unset on the note when no CTA is + // detected") are dropped, not converted: CTA extraction moved server-side + // (the platform now derives it from signals), so there is no connector + // behavior left for them to cover — matching Gmail's and Outlook's facet + // test suites, which dropped their equivalent CTA cases outright rather + // than replacing them with a signal assertion. }); diff --git a/connectors/apple/src/mail/transform.ts b/connectors/apple/src/mail/transform.ts index 15d339ba..24840867 100644 --- a/connectors/apple/src/mail/transform.ts +++ b/connectors/apple/src/mail/transform.ts @@ -1,9 +1,8 @@ import type { ImapAddress, ImapMessage } from "@plotday/twister/tools/imap"; import { ActionType, type Action, type NewContact, type NewLinkWithNotes } from "@plotday/twister"; -import type { Cta } from "@plotday/twister/facets"; import { parse } from "../product-channel"; -import { appleMailFacets } from "./apple-facets"; +import { appleMailSignals } from "./apple-facets"; import { buildAttachmentRef } from "./attachments"; import { isCalendarAttachment, type CalendarBundle } from "./calendar-bundle"; import { looksLikeHtml } from "./html"; @@ -361,15 +360,18 @@ export function transformMessages( const originator = msgs[0]; const originatorFrom = originator.from && originator.from[0] ? originator.from[0] : null; - // Classifier facets are computed from the ORIGINATING message only (same + // Mail signals are computed from the ORIGINATING message only (same // "parent message" convention as the Gmail/Outlook connectors) — the // thread's overall nature (newsletter, notification, direct message) is - // set by how it started, not by whatever reply happens to be in this - // batch. `originatorBody` is recomputed below inside notes.map for the - // same message; cheap and pure, so duplicating it here (rather than - // threading the value through) keeps this block self-contained. - const originatorBody = bodyOf(originator); - const { facets, cta } = appleMailFacets(originator, originatorBody?.content ?? ""); + // derived from how it started, not from whatever reply happens to be in + // this batch. + const signals = { + mail: appleMailSignals(originator), + // Points the platform at the originator's own note (same `noteKeyOf` + // the notes below are keyed with) so body-derived classification reads + // that message, not whichever note happens to be first in this batch. + noteKey: noteKeyOf(originator), + }; // Union of participants for thread access. const participants = new Map(); @@ -396,9 +398,6 @@ export function transformMessages( : { author: from ? toContact(from) : null }), ...(actions ? { actions } : {}), accessContacts: messageContacts(m, ownEmail), - // A time-sensitive CTA (OTP/confirm) is only ever extracted from the - // originating message — see appleMailFacets above. - ...(m === originator && cta ? { cta: cta as Cta } : {}), }; }); @@ -434,8 +433,8 @@ export function transformMessages( // precedence over `sentOnly`), and `title` has NO default at all: the // runtime substitutes the literal placeholder "Untitled", // PERMANENTLY, since every later pass for a still-Sent-only thread would - // also omit the key (`thread-helpers.ts`'s - // `cleanTitle(activity.title?.trim() || "Untitled")`). A degraded + // also omit the key (the platform's title-cleaning fallback substitutes + // "Untitled" whenever the title is empty). A degraded // "Re: …" subject from the Sent copy is strictly better than "Untitled", // and gets overwritten with the real subject the moment an inbound // message enters the window. See `sync.test.ts`'s "never 'Untitled'" @@ -501,10 +500,7 @@ export function transformMessages( // owner-sent threads); explicit null when the sender is unknown, so a // From-less message is never mis-credited to the connector. author: originatorFrom ? toContact(originatorFrom) : null, - // A detected CTA overrides format with its own kind ("otp"/"confirm"), - // same convention as the Gmail/Outlook connectors — the classifier's - // generic format guess is superseded by the more specific signal. - facets: cta ? { ...facets, format: cta.kind } : facets, + signals, ...readState, ...(calendarBundle ? { sources: [`icaluid:${calendarBundle.uid}`] } : {}), ...(calendarBundle?.eventKnown || sentOnlyKnown diff --git a/connectors/google-drive/src/google-drive.ts b/connectors/google-drive/src/google-drive.ts index ed67a9fd..b2d3e0f9 100644 --- a/connectors/google-drive/src/google-drive.ts +++ b/connectors/google-drive/src/google-drive.ts @@ -74,8 +74,8 @@ function isVirtualChannel(id: string): boolean { * `(provider, accountId)` mappings in `contact_external_account`, so a * source-only contact created by a comment will be transparently merged with * the email-having contact created by the file owner (or any later sync that - * sees the same user with an email) — see `addContacts` in - * `workers/api/src/twist/tools/plot/contacts.ts`. + * sees the same user with an email) — handled automatically by the platform's + * contact-merging logic. * * The pattern (used by Linear, Asana, GitHub, Google Chat, Jira, etc.): * - Always emit `source` when you have a stable provider-side user ID. diff --git a/connectors/google/src/mail/gmail-api.ts b/connectors/google/src/mail/gmail-api.ts index 71e8464e..e177cd69 100644 --- a/connectors/google/src/mail/gmail-api.ts +++ b/connectors/google/src/mail/gmail-api.ts @@ -624,11 +624,6 @@ export function getHeaders(message: GmailMessage, name: string): string[] { .map((h) => h.value); } -/** Decoded HTML body for a message (empty string if none). For link extraction. */ -export function getMessageHtml(message: GmailMessage): string { - return findPartContent(message.payload, "text/html") ?? ""; -} - /** * True when a mailing list (Google Groups, etc.) rewrote the `From` *address* * for DMARC alignment, so the From display name no longer belongs to the From diff --git a/connectors/google/src/mail/gmail-facets.test.ts b/connectors/google/src/mail/gmail-facets.test.ts index 17c87965..bdd2beb9 100644 --- a/connectors/google/src/mail/gmail-facets.test.ts +++ b/connectors/google/src/mail/gmail-facets.test.ts @@ -1,45 +1,8 @@ import { describe, expect, it } from "vitest"; -import { gmailFacets } from "./gmail-facets"; -import type { GmailMessage, GmailMessagePart } from "./gmail-api"; - -/** Encode a UTF-8 string as base64url (matches Gmail's wire format). */ -function b64url(s: string): string { - return Buffer.from(s, "utf8").toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); -} - -function msg(opts: { - headers: Array<[string, string]>; - labelIds?: string[]; - htmlBody?: string; -}): GmailMessage { - const topHeaders = opts.headers.map(([name, value]) => ({ name, value })); - - let payload: GmailMessagePart; - if (opts.htmlBody !== undefined) { - // multipart/alternative with an html part so getMessageHtml (via findPartContent) can find it - payload = { - mimeType: "multipart/alternative", - headers: topHeaders, - parts: [ - { - mimeType: "text/plain", - headers: [], - body: { size: 0, data: b64url("") }, - }, - { - mimeType: "text/html", - headers: [], - body: { size: opts.htmlBody.length, data: b64url(opts.htmlBody) }, - }, - ], - }; - } else { - payload = { - mimeType: "text/plain", - headers: topHeaders, - }; - } +import { gmailSignals } from "./gmail-facets"; +import type { GmailMessage } from "./gmail-api"; +function msg(opts: { headers: Array<[string, string]>; labelIds?: string[] }): GmailMessage { return { id: "m1", threadId: "t1", @@ -48,75 +11,115 @@ function msg(opts: { historyId: "1", internalDate: "1700000000000", sizeEstimate: 0, - payload, + payload: { + mimeType: "text/plain", + headers: opts.headers.map(([name, value]) => ({ name, value })), + }, }; } -describe("gmailFacets", () => { - it("classifies a newsletter as reading/automated/list", () => { - const { facets } = gmailFacets( - msg({ - headers: [ - ["From", "news@substack.com"], - ["To", "me@x.com"], - ["Subject", "The Weekly Digest"], - ["List-Id", ""], - ["List-Unsubscribe", ""], - ], - }), - "a".repeat(4000) - ); - expect(facets).toEqual({ format: "reading", automation: "automated", reach: "list" }); +describe("gmailSignals", () => { + it("extracts List-Id and List-Unsubscribe from a newsletter", () => { + // Previously asserted format: "reading", automation: "automated", reach: + // "list" via classifyEmail. The automated/list verdict came from these two + // headers; the reading/notification split came from body length, which is + // now classifier logic covered by the platform's own facet-derivation + // tests, not this file. + const message = msg({ + headers: [ + ["From", "news@substack.com"], + ["To", "me@x.com"], + ["Subject", "The Weekly Digest"], + ["List-Id", ""], + ["List-Unsubscribe", ""], + ], + }); + const s = gmailSignals(message); + expect(s.listId).toBe(""); + expect(s.listUnsubscribe).toBe(""); }); - it("classifies a personal 1:1 email as message/human/direct", () => { - const { facets } = gmailFacets( - msg({ headers: [["From", "jane@friends.com"], ["To", "me@x.com"], ["Subject", "Lunch?"]] }), - "a".repeat(500) - ); - expect(facets).toEqual({ format: "message", automation: "human", reach: "direct" }); + it("extracts no automation signals for a personal 1:1 email", () => { + // Previously asserted format: "message", automation: "human", reach: + // "direct" via classifyEmail. The human verdict came from the absence of + // list/precedence/auto-submitted signals (now classifier logic, covered + // elsewhere on the platform side); the direct verdict came from a single + // To recipient (toCount), covered here plus by the toCount/ccCount case + // below. + const message = msg({ + headers: [ + ["From", "jane@friends.com"], + ["To", "me@x.com"], + ["Subject", "Lunch?"], + ], + }); + const s = gmailSignals(message); + expect(s.listId).toBeNull(); + expect(s.precedence).toBeNull(); + expect(s.autoSubmitted).toBeNull(); + expect(s.fromAddress).toBe("jane@friends.com"); + expect(s.toCount).toBe(1); }); - it("classifies a GitHub notification", () => { - const { facets } = gmailFacets( - msg({ - headers: [["From", "notifications@github.com"], ["To", "me@x.com"], ["Subject", "[repo] PR merged"]], - labelIds: ["CATEGORY_UPDATES"], - }), - "short" - ); - expect(facets.format).toBe("notification"); - expect(facets.automation).toBe("automated"); + it("extracts CATEGORY_UPDATES as a provider category", () => { + // Previously asserted format: "notification", automation: "automated" for + // a GitHub notification. The automated verdict is classifier logic over + // the sender/labels (covered elsewhere on the platform side); the label + // itself is the signal this connector is responsible for extracting. + const message = msg({ + headers: [ + ["From", "notifications@github.com"], + ["To", "me@x.com"], + ["Subject", "[repo] PR merged"], + ], + labelIds: ["CATEGORY_UPDATES"], + }); + const s = gmailSignals(message); + expect(s.providerCategories).toEqual(["CATEGORY_UPDATES"]); + }); + + it("selects the Authentication-Results header added by Google's receiving MTA", () => { + // Previously exercised CTA extraction from an HTML body link alongside + // trusted-DMARC selection; CTA extraction moved server-side (the platform + // now derives it from signals), so only the authserv-id selection — + // connector-only knowledge — remains here. + const authResults = + "mx.google.com; spf=pass smtp.mailfrom=acme.com; dkim=pass header.d=acme.com; dmarc=pass header.from=acme.com"; + const message = msg({ + headers: [ + ["From", "Acme "], + ["To", "user@example.com"], + ["Subject", "Confirm your email"], + ["Authentication-Results", authResults], + ], + }); + const s = gmailSignals(message); + expect(s.authResults).toBe(authResults); }); - it("extracts a confirm cta from an HTML body link with trusted DMARC", () => { - // This test exercises getMessageHtml, which calls findPartContent (already decoded). - // With the double-decode bug (decodeBase64Url(findPartContent(...))), the HTML - // would be re-decoded as base64url and produce garbage, so extractLinkCandidates - // would find no links and cta would be null. Fix: getMessageHtml returns - // findPartContent directly without re-decoding. + it("ignores an Authentication-Results header from an untrusted authserv-id", () => { const message = msg({ headers: [ ["From", "Acme "], ["To", "user@example.com"], ["Subject", "Confirm your email"], - [ - "Authentication-Results", - "mx.google.com; spf=pass smtp.mailfrom=acme.com; dkim=pass header.d=acme.com; dmarc=pass header.from=acme.com", - ], + ["Authentication-Results", "spf1.example.net; spf=pass smtp.mailfrom=acme.com"], ], - htmlBody: `

Welcome to Acme

Confirm your email`, }); - const { facets, cta } = gmailFacets(message, "Welcome to Acme Confirm your email"); - expect(cta).toEqual({ - kind: "confirm", - service: "Acme", - code: null, - url: "https://acme.com/confirm?t=abc123", + const s = gmailSignals(message); + expect(s.authResults).toBeNull(); + }); + + it("emits To and Cc counts separately", () => { + const message = msg({ + headers: [ + ["From", "a@example.com"], + ["To", "a@example.com, b@example.com"], + ["Cc", "c@example.com"], + ], }); - // gmailFacets returns raw classifyEmail output; the caller merges cta.kind into - // facets.format (see gmail.ts: `cta ? { ...facets, format: cta.kind } : facets`). - // Here we just confirm cta is present and facets is non-null. - expect(facets).not.toBeNull(); + const s = gmailSignals(message); + expect(s.toCount).toBe(2); + expect(s.ccCount).toBe(1); }); }); diff --git a/connectors/google/src/mail/gmail-facets.ts b/connectors/google/src/mail/gmail-facets.ts index 2bd0025d..4a156af3 100644 --- a/connectors/google/src/mail/gmail-facets.ts +++ b/connectors/google/src/mail/gmail-facets.ts @@ -1,6 +1,5 @@ -import { classifyEmail, extractCta, extractLinkCandidates, type EmailSignals } from "@plotday/email-classifier"; -import type { Cta, ThreadFacets } from "@plotday/twister/facets"; -import { getHeader, getHeaders, getMessageHtml, parseEmailAddress, parseEmailAddresses, type GmailMessage } from "./gmail-api"; +import type { MailSignals } from "@plotday/twister/signals"; +import { getHeader, getHeaders, parseEmailAddress, parseEmailAddresses, type GmailMessage } from "./gmail-api"; const GMAIL_CATEGORY_LABELS = new Set([ "CATEGORY_PROMOTIONS", @@ -21,15 +20,21 @@ function trustedAuthResults(message: GmailMessage): string | null { return null; } -export type GmailClassification = { facets: ThreadFacets; cta: Cta | null }; +const GMAIL_FLAG_LABELS = new Set(["IMPORTANT", "STARRED"]); /** - * Compute facets and extract CTA for a Gmail message. `bodyText` is the extracted body used - * for the length heuristic (pass the same string the note will carry). + * Extract the normalized mail signals for a Gmail message. + * + * This connector no longer classifies: it reports what it observed and the + * platform decides. Body length is not part of this contract — the platform + * already holds the note content and derives plain-text length from it + * itself, rather than trusting a number each connector would otherwise have + * to compute independently. */ -export function gmailFacets(message: GmailMessage, bodyText: string): GmailClassification { +export function gmailSignals(message: GmailMessage): MailSignals { const from = parseEmailAddress(getHeader(message, "From") ?? ""); - const signals: EmailSignals = { + const labels = message.labelIds ?? []; + return { listId: getHeader(message, "List-Id"), listUnsubscribe: getHeader(message, "List-Unsubscribe"), precedence: getHeader(message, "Precedence"), @@ -38,16 +43,12 @@ export function gmailFacets(message: GmailMessage, bodyText: string): GmailClass importance: getHeader(message, "Importance") ?? getHeader(message, "X-Priority"), fromAddress: from?.email.toLowerCase() ?? null, fromName: from?.name ?? null, - recipientCount: - parseEmailAddresses(getHeader(message, "To")).length + - parseEmailAddresses(getHeader(message, "Cc")).length, + toCount: parseEmailAddresses(getHeader(message, "To")).length, + ccCount: parseEmailAddresses(getHeader(message, "Cc")).length, isReply: getHeader(message, "In-Reply-To") !== null || getHeader(message, "References") !== null, subject: getHeader(message, "Subject"), - bodyText, - bodyLength: bodyText.length, - links: extractLinkCandidates(getMessageHtml(message)), authResults: trustedAuthResults(message), - gmailCategories: (message.labelIds ?? []).filter((l) => GMAIL_CATEGORY_LABELS.has(l)), + providerCategories: labels.filter((l) => GMAIL_CATEGORY_LABELS.has(l)), + providerFlags: labels.filter((l) => GMAIL_FLAG_LABELS.has(l)), }; - return { facets: classifyEmail(signals), cta: extractCta(signals) }; } diff --git a/connectors/google/src/mail/gmail-send-errors.ts b/connectors/google/src/mail/gmail-send-errors.ts index 10c3d8b1..fd91d20a 100644 --- a/connectors/google/src/mail/gmail-send-errors.ts +++ b/connectors/google/src/mail/gmail-send-errors.ts @@ -22,8 +22,8 @@ export interface ClassifiedSendError { message: string | null; } -// Google reason markers (mirrors the server-side classifier vocabulary in -// workers/api/src/utils/transient-error.ts, which the connector can't import). +// Google reason markers (mirrors the platform's own transient-error +// classifier vocabulary, which the connector can't import). const RATE_LIMIT_MARKERS = [ "rateLimitExceeded", "userRateLimitExceeded", diff --git a/connectors/google/src/mail/sync.test.ts b/connectors/google/src/mail/sync.test.ts index de1a38f3..7166e6cd 100644 --- a/connectors/google/src/mail/sync.test.ts +++ b/connectors/google/src/mail/sync.test.ts @@ -1122,11 +1122,18 @@ describe("processEmailThreadsFn — attendee responses fold onto the event", () // human reply instead. expect(links[0].preview).toBe("No problem"); - // Facets must be computed from the surviving human reply, not the + // Signals must be computed from the surviving human reply, not the // folded notification: the RSVP message carries an Auto-Submitted - // header (see rsvpThread), so picking it would classify this thread as + // header (see rsvpThread), so picking it would report this thread as // automated even though a real person wrote the surviving message. - expect(links[0].facets?.automation).toBe("human"); + expect(links[0].signals?.mail?.autoSubmitted).toBeNull(); + + // …and `noteKey` must name that same surviving message, so the platform + // reads ITS body when classifying. The folded RSVP notification is the + // thread's first message, so a link that named no note (or named the + // first one) would classify from a message this link no longer carries. + expect(links[0].signals?.noteKey).toBe("rsvp-mixed-msg-2"); + expect(keys).toContain(links[0].signals?.noteKey); }); it("keeps the email thread when the event thread cannot be resolved", async () => { diff --git a/connectors/google/src/mail/sync.ts b/connectors/google/src/mail/sync.ts index cee65cd7..e40aed26 100644 --- a/connectors/google/src/mail/sync.ts +++ b/connectors/google/src/mail/sync.ts @@ -24,7 +24,6 @@ import { type NoteWriteBackResult, resolveOutboundReplyRecipients, } from "@plotday/twister"; -import type { Cta } from "@plotday/twister/facets"; import { ActionType } from "@plotday/twister/plot"; import type { Actor, @@ -65,7 +64,7 @@ import { syncGmailMailboxIncremental, transformGmailThread, } from "./gmail-api"; -import { gmailFacets } from "./gmail-facets"; +import { gmailSignals } from "./gmail-facets"; import { type ClassifiedSendError, classifySendError, @@ -1790,9 +1789,9 @@ async function saveTransformedThread( } } - // Compute classifier facets from the parent message's headers + body. - // When the fold above dropped one or more notes, restrict the candidate - // to messages whose note survived — otherwise a folded RSVP notification + // Compute mail signals from the parent message's headers. When the fold + // above dropped one or more notes, restrict the candidate to messages + // whose note survived — otherwise a folded RSVP notification // (headers + snippet of an automated message) can still be picked here // and get a real human reply misclassified as automated. Skipped // entirely (same `.find()` as before) when nothing was folded, which is @@ -1811,17 +1810,14 @@ async function saveTransformedThread( (survivingNoteKeys === null || survivingNoteKeys.has(m.id)) ); if (facetParent) { - // Use the parent message's full note body (not the short preview snippet) - // so the classifier's reading-vs-notification length split can fire. - const facetNote = plotThread.notes?.find( - (n) => "key" in n && (n as { key: string }).key === facetParent.id - ); - const facetBody = facetNote?.content ?? plotThread.preview ?? ""; - const { facets, cta } = gmailFacets(facetParent, facetBody); - plotThread.facets = cta ? { ...facets, format: cta.kind } : facets; - if (cta && facetNote) { - (facetNote as { cta?: Cta | null }).cta = cta; - } + // `noteKey` points the platform at THIS message's note (notes are keyed + // on the Gmail message id — see transformGmailThread), so body-derived + // classification reads the same message the headers came from rather + // than whichever note happens to be first in this batch. + plotThread.signals = { + mail: gmailSignals(facetParent), + noteKey: facetParent.id, + }; } // Star ↔ todo sync: detect star changes and sync to Plot todo status. diff --git a/connectors/outlook/src/mail/bundle.test.ts b/connectors/outlook/src/mail/bundle.test.ts index d6afee33..2473d0c6 100644 --- a/connectors/outlook/src/mail/bundle.test.ts +++ b/connectors/outlook/src/mail/bundle.test.ts @@ -137,6 +137,41 @@ describe("processConversationsFn — calendar-thread bundling", () => { expect(saved[0].sources).toContain("icaluid:uid-3"); }); + it("names the parent message's note on the signals so the platform classifies from it", async () => { + // A draft sits ahead of the real message in the conversation. The parent + // (first non-draft) is what the signals are read from, and `noteKey` must + // point at THAT message's note — the same key the note itself carries — + // so body-derived classification never reads a different message. + const { host, saved } = makeHost(); + const item = { + messages: [ + baseMessage({ + id: "msg-draft", + internetMessageId: "", + conversationId: "conv-5", + isDraft: true, + receivedDateTime: "2026-06-01T09:00:00Z", + }), + baseMessage({ + id: "msg-5", + internetMessageId: "", + conversationId: "conv-5", + receivedDateTime: "2026-06-01T10:00:00Z", + }), + ], + attachmentsByMessageId: emptyAttachments, + parentHeaders: null as GraphHeader[] | null, + }; + + await processConversationsFn(host, [item], false, "inbox-folder"); + + expect(saved).toHaveLength(1); + expect(saved[0].signals?.noteKey).toBe(""); + expect( + (saved[0].notes ?? []).map((n) => (n as { key?: string }).key) + ).toContain(saved[0].signals?.noteKey); + }); + it("does not add icaluid sources for a plain conversation with no calendar signal", async () => { const { host, saved } = makeHost(); const item = { diff --git a/connectors/outlook/src/mail/outlook-facets.test.ts b/connectors/outlook/src/mail/outlook-facets.test.ts index df7369e3..88083e0c 100644 --- a/connectors/outlook/src/mail/outlook-facets.test.ts +++ b/connectors/outlook/src/mail/outlook-facets.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { outlookFacets } from "./outlook-facets"; +import { outlookSignals } from "./outlook-facets"; import type { GraphMessage } from "./graph-mail-api"; const m = (over: Partial): GraphMessage => ({ @@ -11,53 +11,58 @@ const m = (over: Partial): GraphMessage => ({ ...over, }); -describe("outlookFacets", () => { - it("newsletter with List-Id → automated/list", () => { - const { facets } = outlookFacets( - [{ name: "List-Id", value: "" }], - m({}), - "x".repeat(2000) - ); - expect(facets.automation).toBe("automated"); - expect(facets.reach).toBe("list"); - expect(facets.format).toBe("reading"); +describe("outlookSignals", () => { + it("captures a List-Id header, marking the message as list mail", () => { + // Previously asserted format: "reading", automation: "automated", reach: + // "list" via outlookFacets/classifyEmail. The automated/list verdict came + // from this header; the reading/notification split came from body + // length, which is now classifier logic covered by the platform's own + // facet-derivation tests, not this file. + const s = outlookSignals([{ name: "List-Id", value: "" }], m({})); + expect(s.listId).toBe(""); }); - it("plain human reply → human/direct/message", () => { - const { facets } = outlookFacets( - [{ name: "In-Reply-To", value: "" }], - m({ subject: "Re: Hi" }), - "short" - ); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); - expect(facets.format).toBe("message"); + it("marks a message as a reply when In-Reply-To is present", () => { + // Previously asserted format: "message", automation: "human", reach: + // "direct". The human/direct verdict is classifier logic over the + // absence of list/precedence headers and the recipient count (covered + // elsewhere on the platform side); the reply signal itself is what this + // connector is responsible for extracting. + const s = outlookSignals([{ name: "In-Reply-To", value: "" }], m({ subject: "Re: Hi" })); + expect(s.isReply).toBe(true); }); - it("short Other-inbox automated mail → notification (no headers available)", () => { - const { facets } = outlookFacets( + it("emits the provider's own inference classification verbatim", () => { + // Previously asserted automation: "automated", format: "notification" for + // a short Focused-Inbox "Other" message. The Focused/Other bucket used to + // be mapped onto a Gmail category locally; that mapping is now the + // platform's decision, so the connector just reports the provider's own + // bucket name verbatim. + const s = outlookSignals( null, m({ inferenceClassification: "other", from: { emailAddress: { address: "noreply@svc.com" } }, - }), - "tiny" + }) ); - expect(facets.automation).toBe("automated"); - expect(facets.format).toBe("notification"); + expect(s.providerCategories).toEqual(["other"]); }); - it("null headers degrade gracefully", () => { - const { facets } = outlookFacets(null, m({}), "hello there"); - expect(facets.automation).toBe("human"); - expect(facets.reach).toBe("direct"); + it("null headers degrade gracefully — header-driven signals stay null", () => { + const s = outlookSignals(null, m({})); + expect(s.listId).toBeNull(); + expect(s.listUnsubscribe).toBeNull(); + expect(s.precedence).toBeNull(); + expect(s.isReply).toBe(false); }); - it("extracts a confirm cta from an HTML body link with trusted EOP auth-results", () => { - // Outlook has no decode bug (body.content is already a plain string), so this - // is coverage-only: verifies the full outlookFacets path (HTML link extraction + - // DMARC trust) works end-to-end and that the tightened authserv-id suffix match - // still accepts a real EOP sub-domain like bl0pr01.prod.protection.outlook.com. + it("captures the trusted EOP Authentication-Results header verbatim", () => { + // Previously exercised CTA extraction from an HTML body link alongside + // trusted-EOP selection; CTA extraction moved server-side (the platform + // now derives it from signals), so only the authserv-id selection — + // connector-only knowledge — remains here. Also coverage that the + // tightened authserv-id suffix match still accepts a real EOP sub-domain + // like bl0pr01.prod.protection.outlook.com. const headers = [ { name: "Authentication-Results", @@ -68,21 +73,9 @@ describe("outlookFacets", () => { const message = m({ from: { emailAddress: { address: "hello@contoso.com", name: "Contoso" } }, subject: "Confirm your account", - body: { - contentType: "html", - content: `

Welcome

Confirm your account`, - }, - }); - const { facets, cta } = outlookFacets(headers, message, "Welcome Confirm your account"); - expect(cta).toEqual({ - kind: "confirm", - service: "Contoso", - code: null, - url: "https://contoso.com/verify?token=xyz", }); - // outlookFacets returns raw classifyEmail output; the caller merges cta.kind into - // facets.format (see outlook-mail.ts: `cta ? { ...facets, format: cta.kind } : facets`). - expect(facets).not.toBeNull(); + const s = outlookSignals(headers, message); + expect(s.authResults).toBe(headers[0].value); }); it("rejects a spoofed authserv-id that merely contains protection.outlook.com", () => { @@ -90,19 +83,35 @@ describe("outlookFacets", () => { const headers = [ { name: "Authentication-Results", - value: - "evil-protection.outlook.com.attacker.com; dmarc=pass header.from=victim.com", + value: "evil-protection.outlook.com.attacker.com; dmarc=pass header.from=victim.com", }, ]; const message = m({ from: { emailAddress: { address: "no-reply@victim.com", name: "Victim" } }, - body: { - contentType: "html", - content: `Confirm`, - }, }); - const { cta } = outlookFacets(headers, message, "Confirm"); - // Without trusted auth-results the link host can't be validated → no confirm cta - expect(cta?.kind).not.toBe("confirm"); + const s = outlookSignals(headers, message); + expect(s.authResults).toBeNull(); + }); + + it("emits To and Cc counts separately", () => { + const s = outlookSignals( + null, + m({ + toRecipients: [{ emailAddress: { address: "a@x.com" } }, { emailAddress: { address: "b@x.com" } }], + ccRecipients: [{ emailAddress: { address: "c@x.com" } }], + }) + ); + expect(s.toCount).toBe(2); + expect(s.ccCount).toBe(1); + }); + + it("reports a flagged message as a FLAGGED provider flag", () => { + const s = outlookSignals(null, m({ flag: { flagStatus: "flagged" } })); + expect(s.providerFlags).toEqual(["FLAGGED"]); + }); + + it("reports no provider flags for an unflagged message", () => { + const s = outlookSignals(null, m({})); + expect(s.providerFlags).toEqual([]); }); }); diff --git a/connectors/outlook/src/mail/outlook-facets.ts b/connectors/outlook/src/mail/outlook-facets.ts index f9937e04..1aee3715 100644 --- a/connectors/outlook/src/mail/outlook-facets.ts +++ b/connectors/outlook/src/mail/outlook-facets.ts @@ -1,5 +1,4 @@ -import { classifyEmail, extractCta, extractLinkCandidates, type EmailSignals } from "@plotday/email-classifier"; -import type { Cta, ThreadFacets } from "@plotday/twister/facets"; +import type { MailSignals } from "@plotday/twister/signals"; import type { GraphHeader, GraphMessage } from "./graph-mail-api"; function header(headers: GraphHeader[] | null, name: string): string | null { @@ -17,47 +16,33 @@ function trustedAuthResults(headers: GraphHeader[] | null): string | null { return null; } -export type OutlookClassification = { facets: ThreadFacets; cta: Cta | null }; - /** - * Compute facets and extract CTA for an Outlook conversation's parent message. `headers` is - * the parent's internetMessageHeaders (separate single-message fetch; null - * when that fetch failed — header-driven signals just stay null). - * `inferenceClassification === "other"` (Focused Inbox's bulk bucket) maps to - * the classifier's CATEGORY_UPDATES slot so short automated "Other" mail - * classifies as notification, mirroring Gmail's category labels. + * Extract normalized mail signals for an Outlook conversation's parent message. + * `headers` is the parent's internetMessageHeaders (a separate single-message + * fetch; null when that fetch failed — header-driven signals then stay null). + * + * Focused Inbox's bucket is emitted verbatim ("focused" / "other"); mapping it + * onto a content category is the platform's decision, not the connector's. */ -export function outlookFacets( - headers: GraphHeader[] | null, - message: GraphMessage, - bodyText: string -): OutlookClassification { - const html = message.body?.contentType === "html" ? (message.body.content ?? "") : ""; - const signals: EmailSignals = { +export function outlookSignals(headers: GraphHeader[] | null, message: GraphMessage): MailSignals { + return { listId: header(headers, "List-Id"), listUnsubscribe: header(headers, "List-Unsubscribe"), precedence: header(headers, "Precedence"), autoSubmitted: header(headers, "Auto-Submitted"), returnPath: header(headers, "Return-Path"), - importance: - message.importance ?? - header(headers, "Importance") ?? - header(headers, "X-Priority"), + importance: message.importance ?? header(headers, "Importance") ?? header(headers, "X-Priority"), fromAddress: message.from?.emailAddress?.address?.toLowerCase() ?? null, fromName: message.from?.emailAddress?.name ?? null, - recipientCount: - (message.toRecipients?.length ?? 0) + (message.ccRecipients?.length ?? 0), + toCount: message.toRecipients?.length ?? 0, + ccCount: message.ccRecipients?.length ?? 0, isReply: header(headers, "In-Reply-To") !== null || header(headers, "References") !== null || /^re:/i.test(message.subject ?? ""), subject: message.subject ?? null, - bodyText, - bodyLength: bodyText.length, - links: extractLinkCandidates(html), authResults: trustedAuthResults(headers), - gmailCategories: - message.inferenceClassification === "other" ? ["CATEGORY_UPDATES"] : [], + providerCategories: message.inferenceClassification ? [message.inferenceClassification] : [], + providerFlags: message.flag?.flagStatus === "flagged" ? ["FLAGGED"] : [], }; - return { facets: classifyEmail(signals), cta: extractCta(signals) }; } diff --git a/connectors/outlook/src/mail/sync.ts b/connectors/outlook/src/mail/sync.ts index f8fcf933..6879f03f 100644 --- a/connectors/outlook/src/mail/sync.ts +++ b/connectors/outlook/src/mail/sync.ts @@ -35,7 +35,6 @@ import type { Thread, } from "@plotday/twister/plot"; import type { WebhookRequest } from "@plotday/twister/tools/network"; -import type { Cta } from "@plotday/twister/facets"; import { markdownToHtml } from "@plotday/twister/utils/markdown-html"; import { enrichLinkContactsFromOutlook } from "./enrich"; @@ -55,7 +54,7 @@ import { type GraphMessage, type WellKnownFolders, } from "./graph-mail-api"; -import { outlookFacets } from "./outlook-facets"; +import { outlookSignals } from "./outlook-facets"; // --------------------------------------------------------------------------- // Constants @@ -1499,28 +1498,20 @@ export async function processConversationsFn( } } - // Compute classifier facets from the parent message's headers + body. + // Compute mail signals from the parent message's headers. const facetParent = sortConversation(item.messages).find( (m) => !m.isDraft ); if (facetParent) { - const parentKey = facetParent.internetMessageId ?? facetParent.id; - const facetNote = plotThread.notes?.find( - (n) => "key" in n && (n as { key: string }).key === parentKey - ); - const facetBody = - (facetNote as { content?: string } | undefined)?.content ?? - plotThread.preview ?? - ""; - const { facets, cta } = outlookFacets( - item.parentHeaders, - facetParent, - facetBody - ); - plotThread.facets = cta ? { ...facets, format: cta.kind } : facets; - if (cta && facetNote) { - (facetNote as { cta?: Cta | null }).cta = cta; - } + // `noteKey` points the platform at THIS message's note — same + // expression the note itself is keyed with (see + // graph-mail-api.ts) — so body-derived classification reads the + // message the headers came from, not whichever note happens to be + // first in this batch. + plotThread.signals = { + mail: outlookSignals(item.parentHeaders, facetParent), + noteKey: facetParent.internetMessageId ?? facetParent.id, + }; } const isFlagged = isConversationFlagged(item.messages); diff --git a/connectors/slack/src/slack.test.ts b/connectors/slack/src/slack.test.ts index 65fc59fa..91f2d3a0 100644 --- a/connectors/slack/src/slack.test.ts +++ b/connectors/slack/src/slack.test.ts @@ -3287,10 +3287,9 @@ describe("channel message routing", () => { it("never resolves a Slack token while routing a channel message", async () => { // integrations.get(channelId) treats a deliberately-disabled channel's // falsy `enabled` as "never configured" and writes a migration fallback - // that re-enables it (workers/api/src/twist/tools/integrations.ts - // ~694-728). Calling it here, on every inbound channel message, would - // silently re-enable channels the user disabled. Identity must come - // only from the `slack_user_id` cache (see `cachedMentionContext`). + // that re-enables it. Calling it here, on every inbound channel message, + // would silently re-enable channels the user disabled. Identity must + // come only from the `slack_user_id` cache (see `cachedMentionContext`). const { slack, integrationsGet } = setup(); await (slack as any).handleChannelMessage({ diff --git a/connectors/slack/src/slack.ts b/connectors/slack/src/slack.ts index 3a7593fa..1dd66f3b 100644 --- a/connectors/slack/src/slack.ts +++ b/connectors/slack/src/slack.ts @@ -178,9 +178,9 @@ const REACTION_REFRESH_MAX_ATTEMPTS = 20; * connection that covers ALL of the user's DM/MPIM conversations, instead of * one callback per conversation (which would require enumerating and * tracking hundreds of `channel` rows with no corresponding Settings UI to - * manage them). `GetSlackCallbacks` (workers/api/src/twist/tools/network.ts) - * broadcasts every incoming Slack event to every callback registered for the - * team whose granted scopes cover the event type — this sentinel is just + * manage them). The platform broadcasts every incoming Slack event to every + * callback registered for the team whose granted scopes cover the event + * type — this sentinel is just * another registered callback's extraArg, distinguished from a real * channelId by never matching a Slack conversation id format. */ diff --git a/twister/package.json b/twister/package.json index 67441382..41b4e0f8 100644 --- a/twister/package.json +++ b/twister/package.json @@ -55,6 +55,11 @@ "types": "./dist/facets.d.ts", "default": "./dist/facets.js" }, + "./signals": { + "@plotday/connector": "./src/signals.ts", + "types": "./dist/signals.d.ts", + "default": "./dist/signals.js" + }, "./options": { "@plotday/connector": "./src/options.ts", "types": "./dist/options.d.ts", diff --git a/twister/src/plot.ts b/twister/src/plot.ts index 4c4d4b92..f2a611ae 100644 --- a/twister/src/plot.ts +++ b/twister/src/plot.ts @@ -1,5 +1,6 @@ import type { Cta, ThreadFacets } from "./facets"; import type { NewSchedule, NewScheduleOccurrence, Schedule } from "./schedule"; +import type { LinkSignals } from "./signals"; import { type Tag } from "./tag"; import { type Callback } from "./tools/callbacks"; import { type JSONValue } from "./utils/types"; @@ -1326,6 +1327,15 @@ export type NewLink = Partial< * no heuristic is confident. See `@plotday/twister/facets`. */ facets?: ThreadFacets; + /** + * Raw signals this item was derived from (email headers, provider + * categories). The platform derives classification from these. Prefer this + * over `facets`: emitting signals lets classification improve without a + * connector redeploy. See `@plotday/twister/signals`. + * + * When both are set, `signals` wins. + */ + signals?: LinkSignals; /** * The person who created this item in the external system. * diff --git a/twister/src/signals.ts b/twister/src/signals.ts new file mode 100644 index 00000000..ea4c130c --- /dev/null +++ b/twister/src/signals.ts @@ -0,0 +1,84 @@ +/** + * Raw, normalized signals a connector extracts from a source item. Connectors + * emit these; the platform derives classification from them. + * + * This is the inverse of `./facets`: facets are a *verdict* (a connector's + * conclusion), signals are *evidence* (what the connector observed). Emitting + * signals lets the platform re-derive classification without a connector + * redeploy, and lets it combine connector observations with recipient-relative + * facts a connector cannot see (does this user know this sender?). + * + * Every field is nullable — populate only what the source actually provides. + * + * Deliberately absent: a body-length field. The platform already holds the + * note content, so it derives body length itself instead of trusting a + * number connectors would each have to compute independently — and, + * historically, got wrong: connectors used to report the raw content + * string's length, which is markup length whenever the note is HTML. + */ + +/** Header and metadata signals from an email message. */ +export type MailSignals = { + /** List-Id header value, verbatim, or null. */ + listId: string | null; + /** List-Unsubscribe header value, verbatim, or null. */ + listUnsubscribe: string | null; + /** Precedence header ("bulk", "list", "junk", "auto_reply"), or null. */ + precedence: string | null; + /** Auto-Submitted header ("auto-generated", "no", …), or null. */ + autoSubmitted: string | null; + /** Return-Path header. "<>" or "" indicates a bounce/automated sender. */ + returnPath: string | null; + /** Importance or X-Priority header, or the provider's equivalent. */ + importance: string | null; + /** Sender address, lowercased, or null. */ + fromAddress: string | null; + /** Sender display name, or null. */ + fromName: string | null; + /** Number of To recipients. `1` means the user was the sole direct recipient. */ + toCount: number | null; + /** Number of Cc recipients. */ + ccCount: number | null; + /** True when In-Reply-To or References was present. */ + isReply: boolean | null; + /** Subject line, or null. */ + subject: string | null; + /** + * The Authentication-Results header carrying the receiving MTA's own verdict. + * The CONNECTOR selects which header to trust (only it knows its provider's + * authserv-id); the platform parses the value. Null when absent or untrusted. + */ + authResults: string | null; + /** + * Provider content categories, verbatim. Gmail's CATEGORY_PROMOTIONS / + * CATEGORY_UPDATES / CATEGORY_SOCIAL / CATEGORY_FORUMS / CATEGORY_PERSONAL, + * or another provider's equivalent bucket. + */ + providerCategories: string[]; + /** Provider user-state flags, verbatim: IMPORTANT, STARRED, FLAGGED, … */ + providerFlags: string[]; +}; + +/** Signals a connector attaches to a `NewLink`. */ +export type LinkSignals = { + mail?: MailSignals; + /** + * The `key` of the note these signals were read from — the message the + * connector picked as the conversation's classification parent (normally + * the originating message). + * + * A conversation link carries one note per message, and body-derived + * classification (how long the message really is, what it links to) must + * read the SAME message the headers came from. Without this pointer the + * platform can only use the link's first note, which on an incremental + * sync is often a later reply — so a thread's classification would churn + * every time someone replies to it. + * + * Set it to the `key` of the note built from that message, using the same + * expression the note itself was keyed with. Omit it when the link has a + * single note, or when its notes are unkeyed; the platform then falls back + * to the first note. When the key matches no note in the link, the + * platform classifies from `link.preview` instead. + */ + noteKey?: string | null; +};