diff --git a/supabase/migrations/20260827120000_create_social_snapshots.sql b/supabase/migrations/20260827120000_create_social_snapshots.sql new file mode 100644 index 0000000..859e6b2 --- /dev/null +++ b/supabase/migrations/20260827120000_create_social_snapshots.sql @@ -0,0 +1,73 @@ +-- Create social_snapshots: one follower-count point per social per day +-- (recoupable/app#2018 keystone; contract recoupable/docs#316; originally +-- app#2026). +-- +-- socials.followerCount is overwritten by every scrape (api upsertSocials on +-- profile_url), so the previous value is gone the moment a new scrape lands. +-- A weekly report that wants "followers this week vs last week" has to keep +-- its own file in a sandbox, and no other surface (artist page, chat) can +-- show a trend at all. This table keeps every value the platform paid for. +-- +-- Shape: append-only, written by the api socials-upsert wrapper on every +-- scrape that reports a follower count, across all seven platform handlers. +-- captured_on is the dedupe key: one row per social per UTC day, and the +-- latest scrape that day wins (the api upserts on (social_id, captured_on)). +-- A date column with a plain unique constraint instead of an expression index +-- so the upsert's ON CONFLICT target is a column list, not an expression. +-- +-- Read by GET /api/artists/{id}/socials?history= as the `history` array +-- on each profile. `socials` keeps the latest value for existing readers. + +CREATE TABLE IF NOT EXISTS public.social_snapshots ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + social_id UUID NOT NULL REFERENCES public.socials(id) ON DELETE CASCADE, + -- The scrape's completion time; the day partition is derived once here so + -- readers never re-derive it in a different timezone. + captured_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), + -- Always derived from captured_at by the trigger below (a generated + -- column can't: timezone() is STABLE, not IMMUTABLE), so a caller that + -- supplies a historical captured_at can never land on the wrong day. + captured_on DATE NOT NULL, + follower_count BIGINT NOT NULL CHECK (follower_count >= 0), + following_count BIGINT CHECK (following_count IS NULL OR following_count >= 0), + -- Lifetime post count where the platform reports one (Instagram + -- postsCount, TikTok authorMeta.video, YouTube channelTotalVideos, X + -- statusesCount); NULL on LinkedIn, Threads, Facebook. + post_count BIGINT CHECK (post_count IS NULL OR post_count >= 0), + UNIQUE (social_id, captured_on) +); + +-- captured_on is never written by callers; it is the UTC day of captured_at. +CREATE OR REPLACE FUNCTION public.social_snapshots_set_captured_on() +RETURNS TRIGGER AS $fn$ +BEGIN + NEW.captured_on := (NEW.captured_at AT TIME ZONE 'utc')::date; + RETURN NEW; +END; +$fn$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS set_captured_on ON public.social_snapshots; +CREATE TRIGGER set_captured_on + BEFORE INSERT OR UPDATE OF captured_at ON public.social_snapshots + FOR EACH ROW EXECUTE FUNCTION public.social_snapshots_set_captured_on(); + +-- The history read: one social's points, newest first, bounded by days. +CREATE INDEX IF NOT EXISTS social_snapshots_social_captured_idx + ON public.social_snapshots (social_id, captured_at DESC); + +-- RLS on with zero policies: the api reads and writes via the service role, +-- which bypasses it; anon/authenticated get nothing, matching socials' +-- neighbours (playcount_snapshots, music_generations). +ALTER TABLE public.social_snapshots ENABLE ROW LEVEL SECURITY; + +-- Backfill: history starts today rather than at the next scrape. One row per +-- social that already has a follower count, stamped with the socials row's +-- updated_at (the last scrape that wrote it). Idempotent via the unique key. +INSERT INTO public.social_snapshots (social_id, captured_at, follower_count, following_count) +SELECT s.id, + s.updated_at, + s."followerCount", + CASE WHEN s."followingCount" >= 0 THEN s."followingCount" END +FROM public.socials s +WHERE s."followerCount" IS NOT NULL AND s."followerCount" >= 0 +ON CONFLICT (social_id, captured_on) DO NOTHING; diff --git a/supabase/migrations/20260827120100_posts_engagement_columns.sql b/supabase/migrations/20260827120100_posts_engagement_columns.sql new file mode 100644 index 0000000..110e7a8 --- /dev/null +++ b/supabase/migrations/20260827120100_posts_engagement_columns.sql @@ -0,0 +1,30 @@ +-- posts: engagement counts per post, and updated_at becomes the publish date +-- it already was in practice (recoupable/app#2018 keystone; contract +-- recoupable/docs#316). +-- +-- Every platform scraper returns engagement with each post (TikTok playCount/ +-- diggCount/commentCount/shareCount, YouTube viewCount/likes/commentsCount, +-- X viewCount/likeCount/replyCount/retweetCount, Instagram likesCount/ +-- commentsCount, LinkedIn likes/comments/shares) and the api threw all of it +-- away after the run was read, so "which post outperformed this week" needed +-- a fresh scrape every time. Four named columns rather than a metrics JSONB +-- so the table can be ordered and filtered on them directly. +-- +-- Nullable: Threads and Facebook report nothing per post, rows written +-- before this migration have nothing to backfill from, and NULL keeps +-- "not reported" distinct from 0. + +ALTER TABLE public.posts + ADD COLUMN IF NOT EXISTS views BIGINT CHECK (views IS NULL OR views >= 0), + ADD COLUMN IF NOT EXISTS likes BIGINT CHECK (likes IS NULL OR likes >= 0), + ADD COLUMN IF NOT EXISTS comments BIGINT CHECK (comments IS NULL OR comments >= 0), + ADD COLUMN IF NOT EXISTS reposts BIGINT CHECK (reposts IS NULL OR reposts >= 0); + +-- updated_at on posts has always been written by the handlers as the post's +-- platform publish timestamp (it is what GET /api/artists/{id}/posts orders +-- by), never as a row-modification time. The BEFORE UPDATE trigger from the +-- create migration (20250130161836) would overwrite that value with now() on +-- the first re-scrape that refreshes engagement, turning every re-scraped +-- post into "published just now". Drop it; the api sets updated_at +-- explicitly on every upsert. +DROP TRIGGER IF EXISTS set_updated_at ON public.posts; diff --git a/supabase/migrations/20260827120200_apify_scraper_runs_lineage.sql b/supabase/migrations/20260827120200_apify_scraper_runs_lineage.sql new file mode 100644 index 0000000..7993069 --- /dev/null +++ b/supabase/migrations/20260827120200_apify_scraper_runs_lineage.sql @@ -0,0 +1,37 @@ +-- apify_scraper_runs: run lineage for webhook-spawned runs (recoupable/app#2018 +-- keystone, budget PR follows). +-- +-- An artist profile scrape spawns a comments run, which spawns one commenter +-- profile run. Until now only the artist-batch route registered its runs +-- here; spawned runs were invisible, which is how an unbounded fan-of-fan +-- crawl ran at ~300 runs/hour for a day (app#2018 evidence) with nothing in +-- our own data to count or trace it. Every run the api starts now registers +-- with where it came from: +-- origin 'artist' — the profile belongs to a roster artist; the +-- handler may schedule follow-ups. +-- 'fan' — a commenter profile batch; terminal by +-- construction, never followed up. +-- parent_run_id the run whose webhook started this one (NULL for the run a +-- scrape endpoint started). Walk it to the root for the +-- account and the originating scrape. +-- +-- account_id becomes nullable: a spawned run inherits its parent's account +-- when the parent is registered, but a chain whose root predates this +-- migration has no account to inherit, and refusing to register it would +-- hide exactly the runs the budget PR needs to count. + +ALTER TABLE public.apify_scraper_runs + ALTER COLUMN account_id DROP NOT NULL, + ADD COLUMN IF NOT EXISTS origin TEXT + CHECK (origin IS NULL OR origin IN ('artist', 'fan')), + ADD COLUMN IF NOT EXISTS parent_run_id TEXT; + +-- Lineage walk (child -> parent) and "how many runs did this scrape spawn". +CREATE INDEX IF NOT EXISTS apify_scraper_runs_parent_run_id_idx + ON public.apify_scraper_runs (parent_run_id) + WHERE parent_run_id IS NOT NULL; + +-- Per-account hourly budget count (budget PR). +CREATE INDEX IF NOT EXISTS apify_scraper_runs_account_created_idx + ON public.apify_scraper_runs (account_id, created_at DESC) + WHERE account_id IS NOT NULL;