diff --git a/.gitignore b/.gitignore index fa02aa2..2b4ddbc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ frontend/.openclaude/ # Build artifacts frontend/next-env.d.ts CLAUDE 2.md + +# Trace logs +*.log diff --git a/backend/.env.example b/backend/.env.example index 8b97706..64e9ae7 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -2,3 +2,4 @@ OPENAI_API_KEY="YOUR_KEY" QDRANT_URL=http://qdrant:6333 QDRANT_CLOUD_API_KEY="YOUR_KEY" DATABASE_CONNECTION_STRING=postgresql://npmatch:npmatch@postgres:5432/npmatch +ANALYTICS_TOKEN="YOUR_TOKEN" diff --git a/backend/app/analytics.py b/backend/app/analytics.py new file mode 100644 index 0000000..ff786c5 --- /dev/null +++ b/backend/app/analytics.py @@ -0,0 +1,152 @@ +import hashlib +import logging +import os +from datetime import UTC, datetime, timedelta + +from fastapi import Request + +from app.db import get_pool + +logger = logging.getLogger(__name__) + +SEARCH_EVENT_INSERT = """ +INSERT INTO search_events (query, framework, priorities, result_count, ip_hash, user_agent) +VALUES ($1, $2, $3, $4, $5, $6) +""" + + +def hash_ip(ip: str) -> str: + """SHA-256 hash of the client IP, salted so it isn't reversible.""" + salt = os.environ.get("ANALYTICS_SALT", "npmatch") + return hashlib.sha256(f"{salt}:{ip}".encode()).hexdigest() + + +def client_ip(request: Request) -> str: + """Client IP, honoring X-Forwarded-For (set by the Next.js proxy).""" + forwarded = request.headers.get("x-forwarded-for") + if forwarded: + return forwarded.split(",")[0].strip() + if request.client: + return request.client.host + return "unknown" + + +async def record_page_view(*, ip_hash: str, user_agent: str | None, referrer: str | None) -> None: + pool = await get_pool() + await pool.execute( + "INSERT INTO page_views (ip_hash, user_agent, referrer) VALUES ($1, $2, $3)", + ip_hash, + user_agent, + referrer, + ) + + +async def record_search( + *, + query: str, + framework: str | None, + priorities: list[str] | None, + result_count: int, + ip_hash: str, + user_agent: str | None, +) -> None: + pool = await get_pool() + await pool.execute( + SEARCH_EVENT_INSERT, + query, + framework, + ", ".join(priorities) if priorities else None, + result_count, + ip_hash, + user_agent, + ) + + +async def get_analytics_summary() -> dict: + pool = await get_pool() + cutoff = datetime.now(UTC) - timedelta(hours=24) + + row = await pool.fetchrow( + """ + SELECT + (SELECT count(*) FROM page_views) AS total_visits, + (SELECT count(DISTINCT ip_hash) FROM page_views) AS unique_visitors, + (SELECT count(*) FROM search_events) AS total_searches, + (SELECT count(*) FROM page_views WHERE visited_at >= $1) AS visits_last_24h, + (SELECT count(*) FROM search_events WHERE searched_at >= $1) AS searches_last_24h + """, + cutoff, + ) + + top_queries = await pool.fetch( + """ + SELECT query, count(*) AS count + FROM search_events + GROUP BY query + ORDER BY count DESC, query ASC + LIMIT 10 + """ + ) + top_frameworks = await pool.fetch( + """ + SELECT COALESCE(NULLIF(framework, ''), 'any') AS framework, count(*) AS count + FROM search_events + GROUP BY COALESCE(NULLIF(framework, ''), 'any') + ORDER BY count DESC, framework ASC + LIMIT 10 + """ + ) + top_referrers = await pool.fetch( + """ + SELECT COALESCE(NULLIF(referrer, ''), '(direct)') AS referrer, count(*) AS count + FROM page_views + GROUP BY COALESCE(NULLIF(referrer, ''), '(direct)') + ORDER BY count DESC, referrer ASC + LIMIT 10 + """ + ) + + return { + "total_visits": row["total_visits"], + "unique_visitors": row["unique_visitors"], + "total_searches": row["total_searches"], + "visits_last_24h": row["visits_last_24h"], + "searches_last_24h": row["searches_last_24h"], + "top_queries": [{"label": r["query"], "count": r["count"]} for r in top_queries], + "top_frameworks": [{"label": r["framework"], "count": r["count"]} for r in top_frameworks], + "top_referrers": [{"label": r["referrer"], "count": r["count"]} for r in top_referrers], + } + + +async def list_page_views(limit: int = 50) -> list[dict]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT visited_at, ip_hash, user_agent, referrer + FROM page_views + ORDER BY visited_at DESC + LIMIT $1 + """, + limit, + ) + return [dict(r) for r in rows] + + +async def list_searches(limit: int = 50) -> list[dict]: + pool = await get_pool() + rows = await pool.fetch( + """ + SELECT searched_at, query, framework, priorities, result_count + FROM search_events + ORDER BY searched_at DESC + LIMIT $1 + """, + limit, + ) + out = [] + for row in rows: + item = dict(row) + priorities = item["priorities"] + item["priorities"] = [p.strip() for p in priorities.split(",")] if priorities else None + out.append(item) + return out diff --git a/backend/app/db.py b/backend/app/db.py new file mode 100644 index 0000000..f42481e --- /dev/null +++ b/backend/app/db.py @@ -0,0 +1,19 @@ +import os + +import asyncpg + +_pg_pool: asyncpg.Pool | None = None + + +async def get_pool() -> asyncpg.Pool: + """Lazy initialization of the shared asyncpg pool.""" + global _pg_pool + + if _pg_pool is None: + _pg_pool = await asyncpg.create_pool( + dsn=os.environ["DATABASE_CONNECTION_STRING"], + min_size=1, + max_size=5, + ) + + return _pg_pool diff --git a/backend/app/main.py b/backend/app/main.py index 7d36e6d..78bb717 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,6 +10,15 @@ from slowapi.util import get_remote_address import app.env +from app.analytics import ( + client_ip, + get_analytics_summary, + hash_ip, + list_page_views, + list_searches, + record_page_view, + record_search, +) from app.llm import stream_response from app.models import SearchRequest from app.search import package_search @@ -50,6 +59,66 @@ async def health(): return {"status": "ok"} +def _require_analytics_token(request: Request) -> None: + expected = os.environ.get("ANALYTICS_TOKEN") + if not expected: + raise HTTPException(status_code=503, detail="Analytics token not configured") + if request.headers.get("x-analytics-token") != expected: + raise HTTPException(status_code=401, detail="Unauthorized") + + +@app.post("/api/track/pageview") +@limiter.limit("120/minute") +async def track_pageview(request: Request): + referrer = None + try: + body = await request.json() + referrer = body.get("referrer") + except Exception: + pass + + try: + await record_page_view( + ip_hash=hash_ip(client_ip(request)), + user_agent=request.headers.get("user-agent"), + referrer=referrer or request.headers.get("referer"), + ) + except Exception as e: + logger.warning(f"Failed to record page view: {e}") + + return {"ok": True} + + +@app.get("/api/analytics/summary") +async def analytics_summary(request: Request): + _require_analytics_token(request) + try: + return await get_analytics_summary() + except Exception: + logger.exception("Analytics summary failed") + raise HTTPException(status_code=502, detail="Analytics query failed") from None + + +@app.get("/api/analytics/visits") +async def analytics_visits(request: Request, limit: int = 50): + _require_analytics_token(request) + try: + return await list_page_views(limit=min(limit, 200)) + except Exception: + logger.exception("Analytics visits failed") + raise HTTPException(status_code=502, detail="Analytics query failed") from None + + +@app.get("/api/analytics/searches") +async def analytics_searches(request: Request, limit: int = 50): + _require_analytics_token(request) + try: + return await list_searches(limit=min(limit, 200)) + except Exception: + logger.exception("Analytics searches failed") + raise HTTPException(status_code=502, detail="Analytics query failed") from None + + @app.post("/api/search") @limiter.limit("2/minute") async def search(request: Request, body: SearchRequest): @@ -60,9 +129,21 @@ async def search(request: Request, body: SearchRequest): try: packages = await package_search(body.query) + except Exception: + logger.exception("Package search failed") + raise HTTPException(status_code=502, detail="Failed in hybrid search") from None + + try: + await record_search( + query=body.query, + framework=body.framework, + priorities=body.priorities, + result_count=len(packages), + ip_hash=hash_ip(client_ip(request)), + user_agent=request.headers.get("user-agent"), + ) except Exception as e: - logger.error(f"Package search failed: {e}") - raise HTTPException(status_code=502, detail="Failed in hybrid search") from e + logger.warning(f"Failed to record search event: {e}") if not packages: logger.info("No packages found for query, returning empty response") diff --git a/backend/app/models.py b/backend/app/models.py index 4ef64fe..8813a64 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,3 +1,5 @@ +from datetime import datetime + from pydantic import BaseModel, Field @@ -18,3 +20,34 @@ class Package(BaseModel): class SearchResponse(BaseModel): packages: list[Package] + + +class PageView(BaseModel): + visited_at: datetime + ip_hash: str + user_agent: str | None = None + referrer: str | None = None + + +class SearchEvent(BaseModel): + searched_at: datetime + query: str + framework: str | None = None + priorities: list[str] | None = None + result_count: int + + +class TopItem(BaseModel): + label: str + count: int + + +class AnalyticsSummary(BaseModel): + total_visits: int + unique_visitors: int + total_searches: int + visits_last_24h: int + searches_last_24h: int + top_queries: list[TopItem] + top_frameworks: list[TopItem] + top_referrers: list[TopItem] diff --git a/backend/app/search.py b/backend/app/search.py index 8cde67f..f096b4c 100644 --- a/backend/app/search.py +++ b/backend/app/search.py @@ -2,10 +2,11 @@ import logging import os -import asyncpg from openai import AsyncOpenAI from qdrant_client import AsyncQdrantClient +from app.db import get_pool + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -40,19 +41,6 @@ def get_qdrant_client() -> AsyncQdrantClient: COLLECTION_NAME = "npmatch" CANDIDATE_LIMIT = 20 -_pg_pool: asyncpg.Pool | None = None - - -async def _get_pool() -> asyncpg.Pool: - global _pg_pool - if _pg_pool is None: - _pg_pool = await asyncpg.create_pool( - dsn=os.environ["DATABASE_CONNECTION_STRING"], - min_size=1, - max_size=5, - ) - return _pg_pool - def _rrf(rankings: list[list[str]], k: int = 60) -> list[str]: scores: dict[str, float] = {} @@ -89,7 +77,7 @@ async def _vector_search(embedding: list[float]) -> list[str]: async def _fts_search(query: str) -> list[str]: - pool = await _get_pool() + pool = await get_pool() rows = await pool.fetch( """ SELECT name, @@ -115,7 +103,7 @@ def _rrf(rankings: list[list[str]], k: int = 60) -> list[str]: async def _fetch_metadata(names: list[str]) -> list[dict]: - pool = await _get_pool() + pool = await get_pool() rows = await pool.fetch( "SELECT name, description, keywords, version FROM packages WHERE name = ANY($1)", names, diff --git a/backend/app/test/test_analytics.py b/backend/app/test/test_analytics.py new file mode 100644 index 0000000..3f1046f --- /dev/null +++ b/backend/app/test/test_analytics.py @@ -0,0 +1,300 @@ +import hashlib +import logging +from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.analytics import ( + client_ip, + get_analytics_summary, + hash_ip, + list_page_views, + list_searches, + record_page_view, + record_search, +) +from app.main import app + + +class TestHashIp: + def test_deterministic(self): + assert hash_ip("203.0.113.5") == hash_ip("203.0.113.5") + + def test_different_ips_differ(self): + assert hash_ip("203.0.113.5") != hash_ip("203.0.113.6") + + def test_uses_default_salt(self): + expected = hashlib.sha256(b"npmatch:203.0.113.5").hexdigest() + assert hash_ip("203.0.113.5") == expected + + def test_salt_from_env(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_SALT", "custom-salt") + expected = hashlib.sha256(b"custom-salt:203.0.113.5").hexdigest() + assert hash_ip("203.0.113.5") == expected + + +class TestClientIp: + def test_uses_first_xff_value(self): + request = MagicMock() + request.headers = {"x-forwarded-for": "203.0.113.5, 10.0.0.1"} + assert client_ip(request) == "203.0.113.5" + + def test_falls_back_to_client_host(self): + request = MagicMock() + request.headers = {} + request.client.host = "127.0.0.1" + assert client_ip(request) == "127.0.0.1" + + def test_unknown_when_no_client(self): + request = MagicMock() + request.headers = {} + request.client = None + assert client_ip(request) == "unknown" + + +class TestRecordPageView: + @pytest.mark.asyncio + async def test_inserts_row(self): + pool = MagicMock() + pool.execute = AsyncMock() + + with patch("app.analytics.get_pool", AsyncMock(return_value=pool)): + await record_page_view(ip_hash="h", user_agent="ua", referrer="https://example.com") + + pool.execute.assert_awaited_once() + args = pool.execute.await_args.args + assert "page_views" in args[0] + assert args[1:] == ("h", "ua", "https://example.com") + + +class TestRecordSearch: + @pytest.mark.asyncio + async def test_joins_priorities(self): + pool = MagicMock() + pool.execute = AsyncMock() + + with patch("app.analytics.get_pool", AsyncMock(return_value=pool)): + await record_search( + query="react", + framework="react", + priorities=["bundle size", "typescript"], + result_count=3, + ip_hash="h", + user_agent="ua", + ) + + pool.execute.assert_awaited_once() + args = pool.execute.await_args.args + assert "search_events" in args[0] + assert args[1:] == ("react", "react", "bundle size, typescript", 3, "h", "ua") + + @pytest.mark.asyncio + async def test_none_priorities(self): + pool = MagicMock() + pool.execute = AsyncMock() + + with patch("app.analytics.get_pool", AsyncMock(return_value=pool)): + await record_search( + query="react", + framework=None, + priorities=None, + result_count=0, + ip_hash="h", + user_agent=None, + ) + + args = pool.execute.await_args.args + assert args[1:] == ("react", None, None, 0, "h", None) + + +class TestGetAnalyticsSummary: + @pytest.mark.asyncio + async def test_returns_structured_summary(self): + pool = MagicMock() + pool.fetchrow = AsyncMock( + return_value={ + "total_visits": 10, + "unique_visitors": 5, + "total_searches": 3, + "visits_last_24h": 2, + "searches_last_24h": 1, + } + ) + pool.fetch = AsyncMock( + side_effect=[ + [{"query": "react", "count": 4}], + [{"framework": "react", "count": 3}], + [{"referrer": "(direct)", "count": 2}], + ] + ) + + with patch("app.analytics.get_pool", AsyncMock(return_value=pool)): + summary = await get_analytics_summary() + + assert summary["total_visits"] == 10 + assert summary["unique_visitors"] == 5 + assert summary["searches_last_24h"] == 1 + assert summary["top_queries"] == [{"label": "react", "count": 4}] + assert summary["top_frameworks"] == [{"label": "react", "count": 3}] + assert summary["top_referrers"] == [{"label": "(direct)", "count": 2}] + + +class TestListPageViews: + @pytest.mark.asyncio + async def test_returns_dicts(self): + pool = MagicMock() + pool.fetch = AsyncMock( + return_value=[ + { + "visited_at": datetime(2026, 8, 11, tzinfo=UTC), + "ip_hash": "abc", + "user_agent": "ua", + "referrer": None, + } + ] + ) + + with patch("app.analytics.get_pool", AsyncMock(return_value=pool)): + result = await list_page_views(limit=5) + + assert result[0]["ip_hash"] == "abc" + assert result[0]["referrer"] is None + + +class TestListSearches: + @pytest.mark.asyncio + async def test_parses_priorities(self): + pool = MagicMock() + pool.fetch = AsyncMock( + return_value=[ + { + "searched_at": datetime(2026, 8, 11, tzinfo=UTC), + "query": "react", + "framework": "react", + "priorities": "bundle size, typescript", + "result_count": 3, + } + ] + ) + + with patch("app.analytics.get_pool", AsyncMock(return_value=pool)): + result = await list_searches(limit=5) + + assert result[0]["priorities"] == ["bundle size", "typescript"] + + @pytest.mark.asyncio + async def test_none_priorities(self): + pool = MagicMock() + pool.fetch = AsyncMock( + return_value=[ + { + "searched_at": datetime(2026, 8, 11, tzinfo=UTC), + "query": "react", + "framework": None, + "priorities": None, + "result_count": 0, + } + ] + ) + + with patch("app.analytics.get_pool", AsyncMock(return_value=pool)): + result = await list_searches(limit=5) + + assert result[0]["priorities"] is None + + +class TestAnalyticsEndpoints: + def test_summary_requires_token(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + with TestClient(app) as client: + res = client.get("/api/analytics/summary") + assert res.status_code == 401 + + def test_summary_rejects_wrong_token(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + with TestClient(app) as client: + res = client.get("/api/analytics/summary", headers={"x-analytics-token": "wrong"}) + assert res.status_code == 401 + + def test_summary_with_token(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + mock_summary = { + "total_visits": 10, + "unique_visitors": 5, + "total_searches": 3, + "visits_last_24h": 2, + "searches_last_24h": 1, + "top_queries": [], + "top_frameworks": [], + "top_referrers": [], + } + with ( + patch("app.main.get_analytics_summary", AsyncMock(return_value=mock_summary)), + TestClient(app) as client, + ): + res = client.get("/api/analytics/summary", headers={"x-analytics-token": "secret"}) + assert res.status_code == 200 + assert res.json()["total_visits"] == 10 + + def test_visits_requires_token(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + with TestClient(app) as client: + res = client.get("/api/analytics/visits") + assert res.status_code == 401 + + def test_searches_requires_token(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + with TestClient(app) as client: + res = client.get("/api/analytics/searches") + assert res.status_code == 401 + + def test_summary_502_on_db_error_logs_traceback(self, monkeypatch, caplog): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + with ( + patch( + "app.main.get_analytics_summary", + AsyncMock(side_effect=RuntimeError("db down")), + ), + TestClient(app) as client, + caplog.at_level(logging.ERROR), + ): + res = client.get( + "/api/analytics/summary", headers={"x-analytics-token": "secret"} + ) + assert res.status_code == 502 + assert res.json() == {"detail": "Analytics query failed"} + assert any( + "Analytics summary failed" in r.message and r.exc_info for r in caplog.records + ) + + def test_visits_502_on_db_error(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + with ( + patch( + "app.main.list_page_views", + AsyncMock(side_effect=RuntimeError("db down")), + ), + TestClient(app) as client, + ): + res = client.get( + "/api/analytics/visits", headers={"x-analytics-token": "secret"} + ) + assert res.status_code == 502 + assert res.json() == {"detail": "Analytics query failed"} + + def test_searches_502_on_db_error(self, monkeypatch): + monkeypatch.setenv("ANALYTICS_TOKEN", "secret") + with ( + patch( + "app.main.list_searches", + AsyncMock(side_effect=RuntimeError("db down")), + ), + TestClient(app) as client, + ): + res = client.get( + "/api/analytics/searches", headers={"x-analytics-token": "secret"} + ) + assert res.status_code == 502 + assert res.json() == {"detail": "Analytics query failed"} diff --git a/backend/app/test/test_config.py b/backend/app/test/test_config.py new file mode 100644 index 0000000..f093455 --- /dev/null +++ b/backend/app/test/test_config.py @@ -0,0 +1,26 @@ +import os +from urllib.parse import urlparse + +import pytest + +import app.env # noqa: F401 (loads backend/.env via python-dotenv) + + +def test_db_connection_string_uses_supabase_session_pooler(): + """Config guard: cloud DB must use the Supabase session pooler (5432). + + Port 6543 is the transaction pooler, which does not track asyncpg's named + prepared statements across connection reassignment and intermittently raises + DuplicatePreparedStatementError -> 502. See doc/analytics-dashboard-blocker.md. + """ + db = os.environ.get("DATABASE_CONNECTION_STRING", "") + if not db: + pytest.skip("DATABASE_CONNECTION_STRING not set (CI or offline dev)") + if "pooler.supabase.com" not in db: + pytest.skip("not running against the Supabase pooler (local docker)") + + port = urlparse(db).port + assert port == 5432, ( + f"Supabase pooler port is {port}; expected 5432 (session pooler). " + "Port 6543 (transaction pooler) breaks asyncpg named prepared statements." + ) diff --git a/backend/app/test/test_search.py b/backend/app/test/test_search.py index 04ad1f6..017fe88 100644 --- a/backend/app/test/test_search.py +++ b/backend/app/test/test_search.py @@ -18,7 +18,7 @@ async def test_vector_search_skips_missing_metadata(): with ( patch("app.search.get_qdrant_client", return_value=qdrant), - patch("app.search._get_pool", AsyncMock(return_value=mock_pool)), + patch("app.search.get_pool", AsyncMock(return_value=mock_pool)), patch("app.search._fts_search", AsyncMock(return_value=[])), patch("app.search._embed_query", AsyncMock(return_value=[0.1])), ): @@ -57,7 +57,7 @@ async def test_hybrid_fusion_vector_and_fts(): with ( patch("app.search.get_qdrant_client", return_value=qdrant), - patch("app.search._get_pool", AsyncMock(return_value=mock_pool)), + patch("app.search.get_pool", AsyncMock(return_value=mock_pool)), patch("app.search._fts_search", AsyncMock(return_value=fts_results)), patch("app.search._embed_query", AsyncMock(return_value=[0.1, 0.2])), ): @@ -91,7 +91,7 @@ async def test_rrf_ordering_prefers_shared_results(): with ( patch("app.search.get_qdrant_client", return_value=qdrant), - patch("app.search._get_pool", AsyncMock(return_value=mock_pool)), + patch("app.search.get_pool", AsyncMock(return_value=mock_pool)), patch("app.search._fts_search", AsyncMock(return_value=fts_results)), patch("app.search._embed_query", AsyncMock(return_value=[0.1])), ): diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index d8b0b69..651cacb 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -1,11 +1,11 @@ # CLAUDE.md - Frontend -Next.js 15 app with HeroUI v3, Tailwind CSS v4, and Jest. +Next.js 16 app with HeroUI v3, Tailwind CSS v4, and Jest. ## Commands ```bash -npm run dev # Next.js dev server (port 3000) +npm run dev # Next.js dev server (port 3000, Turbopack) npm run build # Production build npm run lint # ESLint on app/ directory npm run lint:fix # ESLint auto-fix @@ -48,3 +48,11 @@ The SSE parser in `useSearch` handles `event: packages`, `data:`, `event: error` This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean. + + +## Turbopack troubleshooting + +If `next dev` hangs silently after `▲ Next.js 16.3.0 (Turbopack)` and never binds port 3000, the most likely cause is a stale `node_modules`. Run `rm -rf node_modules && npm install` to fix. + +Binary-search isolation confirmed this is NOT caused by: `"type": "module"`, `next.config.ts` settings, dependency set, or app code — all work fine with fresh node_modules. + diff --git a/frontend/app/analytics/page.tsx b/frontend/app/analytics/page.tsx new file mode 100644 index 0000000..7382175 --- /dev/null +++ b/frontend/app/analytics/page.tsx @@ -0,0 +1,323 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { Link, Spinner } from "@heroui/react"; +import { analyticsErrorMessage, fetchAnalytics } from "@/lib/analytics"; +import type { AnalyticsResult, AnalyticsSummary, PageView, SearchEvent, TopItem } from "@/types"; + +const TOKEN_KEY = "npmatch.analytics.token"; + +type Status = "gate" | "loading" | "error" | "done"; + +export default function AnalyticsPage() { + const [status, setStatus] = useState("loading"); + const [token, setToken] = useState(""); + const [savedToken, setSavedToken] = useState(null); + const [result, setResult] = useState(null); + const [error, setError] = useState(""); + + useEffect(() => { + const stored = window.sessionStorage.getItem(TOKEN_KEY); + if (stored) { + setSavedToken(stored); + } else { + setStatus("gate"); + } + }, []); + + useEffect(() => { + if (!savedToken) return; + let cancelled = false; + setStatus("loading"); + fetchAnalytics(savedToken).then((r) => { + if (cancelled) return; + if (r.summary !== null) { + // Only persist a token that was just validated + window.sessionStorage.setItem(TOKEN_KEY, savedToken); + setResult(r); + setStatus("done"); + } else { + if (r.errors.summary === 401) { + window.sessionStorage.removeItem(TOKEN_KEY); + } + setError(analyticsErrorMessage(r.errors.summary ?? null)); + setStatus("error"); + } + }); + return () => { + cancelled = true; + }; + }, [savedToken]); + + const handleUnlock = (e: React.FormEvent) => { + e.preventDefault(); + const trimmed = token.trim(); + if (!trimmed) return; + setSavedToken(trimmed); + }; + + const handleLock = useCallback(() => { + window.sessionStorage.removeItem(TOKEN_KEY); + setSavedToken(null); + setResult(null); + setToken(""); + setStatus("gate"); + }, []); + + return ( + <> +
+
+
+
+ + npmatch + + / analytics +
+
+
+ +
+ {status === "gate" && ( +
+
+

Analytics

+

Enter your analytics token to unlock the dashboard.

+
+ setToken(e.target.value)} + placeholder="Analytics token" + aria-label="Analytics token" + className="bg-white/5 border border-white/10 rounded-lg px-3 py-2 text-sm font-mono text-white/90 placeholder:text-white/25 focus:outline-none focus:border-npm-red transition-colors" + /> + +
+ )} + + {status === "loading" && ( +
+ + Loading analytics… +
+ )} + + {status === "error" && ( +
+

{error}

+ +
+ )} + + {status === "done" && result && result.summary && ( + + )} +
+
+ + ); +} + +function Dashboard({ + summary, + visits, + searches, + errors, + onLock, +}: { + summary: AnalyticsSummary; + visits: PageView[] | null; + searches: SearchEvent[] | null; + errors: AnalyticsResult["errors"]; + onLock: () => void; +}) { + return ( +
+
+
+

Analytics

+

Who visits, and what they search.

+
+ +
+ +
+ + + + + +
+ +
+ + + +
+ + + +
+ ); +} + +function Stat({ label, value }: { label: string; value: number }) { + return ( +
+ {label} + {value.toLocaleString()} +
+ ); +} + +function TopList({ title, items }: { title: string; items: TopItem[] }) { + const max = Math.max(1, ...items.map((i) => i.count)); + return ( +
+

{title}

+ {items.length === 0 ? ( +

No data yet

+ ) : ( + items.map((item) => ( +
+
+ {item.label} + {item.count} +
+
+
+
+
+ )) + )} +
+ ); +} + +function RecentVisits({ visits, error }: { visits: PageView[] | null; error: string }) { + return ( +
+

Recent visitors

+ {visits === null ? ( +
+ Failed to load recent visitors — {error} +
+ ) : ( +
+ + + + + + + + + + + {visits.length === 0 && ( + + + + )} + {visits.map((v, i) => ( + + + + + + + ))} + +
TimeVisitorReferrerUser agent
+ No visits yet +
{formatTime(v.visited_at)}{v.ip_hash.slice(0, 8)}…{referrerDomain(v.referrer)}{v.user_agent}
+
+ )} +
+ ); +} + +function RecentSearches({ searches, error }: { searches: SearchEvent[] | null; error: string }) { + return ( +
+

Recent searches

+ {searches === null ? ( +
+ Failed to load recent searches — {error} +
+ ) : ( +
+ + + + + + + + + + + + {searches.length === 0 && ( + + + + )} + {searches.map((s, i) => ( + + + + + + + + ))} + +
TimeQueryFrameworkPrioritiesResults
+ No searches yet +
{formatTime(s.searched_at)}{s.query}{s.framework ?? "any"}{s.priorities?.join(", ") ?? "—"}{s.result_count}
+
+ )} +
+ ); +} + +function formatTime(iso: string): string { + return new Date(iso).toLocaleString(); +} + +function referrerDomain(ref: string | null): string { + if (!ref) return "—"; + try { + return new URL(ref).hostname; + } catch { + return ref; + } +} diff --git a/frontend/app/api/analytics/route.ts b/frontend/app/api/analytics/route.ts new file mode 100644 index 0000000..14a6016 --- /dev/null +++ b/frontend/app/api/analytics/route.ts @@ -0,0 +1,40 @@ +import { NextRequest } from "next/server"; + +const API_URL = process.env.API_URL ?? "http://localhost:8000"; +const ANALYTICS_TOKEN = process.env.ANALYTICS_TOKEN; + +const ENDPOINTS: Record string> = { + summary: () => "/api/analytics/summary", + visits: (limit) => `/api/analytics/visits?limit=${limit}`, + searches: (limit) => `/api/analytics/searches?limit=${limit}`, +}; + +function json(body: unknown, status: number, cacheControl = "no-store") { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json", "Cache-Control": cacheControl }, + }); +} + +export async function GET(req: NextRequest) { + if (!ANALYTICS_TOKEN || req.headers.get("x-analytics-token") !== ANALYTICS_TOKEN) { + return json({ error: "Unauthorized" }, 401); + } + + const kind = req.nextUrl.searchParams.get("kind") ?? ""; + const buildPath = ENDPOINTS[kind]; + if (!buildPath) { + return json({ error: "Unknown analytics kind" }, 400); + } + + const limit = Number(req.nextUrl.searchParams.get("limit") ?? 25); + const upstream = await fetch(`${API_URL}${buildPath(limit)}`, { + headers: { "x-analytics-token": ANALYTICS_TOKEN }, + }); + + if (!upstream.ok) { + return json({ error: "Backend analytics request failed" }, upstream.status); + } + + return json(await upstream.json(), 200); +} diff --git a/frontend/app/api/search/route.ts b/frontend/app/api/search/route.ts index 4314699..c9939ed 100644 --- a/frontend/app/api/search/route.ts +++ b/frontend/app/api/search/route.ts @@ -5,9 +5,15 @@ const API_URL = process.env.API_URL ?? "http://localhost:8000"; export async function POST(req: NextRequest) { const body = await req.json(); + const headers: Record = { "Content-Type": "application/json" }; + const xff = req.headers.get("x-forwarded-for") ?? req.headers.get("x-real-ip"); + if (xff) headers["x-forwarded-for"] = xff; + const userAgent = req.headers.get("user-agent"); + if (userAgent) headers["user-agent"] = userAgent; + const upstream = await fetch(`${API_URL}/api/search`, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers, body: JSON.stringify(body), }); diff --git a/frontend/app/api/track/pageview/route.ts b/frontend/app/api/track/pageview/route.ts new file mode 100644 index 0000000..265037f --- /dev/null +++ b/frontend/app/api/track/pageview/route.ts @@ -0,0 +1,23 @@ +import { NextRequest } from "next/server"; + +const API_URL = process.env.API_URL ?? "http://localhost:8000"; + +export async function POST(req: NextRequest) { + const body = await req.json(); + + const headers: Record = { "Content-Type": "application/json" }; + const xff = req.headers.get("x-forwarded-for") ?? req.headers.get("x-real-ip"); + if (xff) headers["x-forwarded-for"] = xff; + const userAgent = req.headers.get("user-agent"); + if (userAgent) headers["user-agent"] = userAgent; + + const upstream = await fetch(`${API_URL}/api/track/pageview`, { + method: "POST", + headers, + body: JSON.stringify(body), + }); + + return new Response(JSON.stringify(await upstream.json()), { + headers: { "Content-Type": "application/json" }, + }); +} diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index 8aa719f..2b57a26 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -1,4 +1,5 @@ import type { Metadata } from "next"; +import { PageTracker } from "@/components/PageTracker"; import "./globals.css"; export const metadata: Metadata = { @@ -24,6 +25,7 @@ export default function RootLayout({ {children} + ); diff --git a/frontend/components/PageTracker.tsx b/frontend/components/PageTracker.tsx new file mode 100644 index 0000000..38bd1c1 --- /dev/null +++ b/frontend/components/PageTracker.tsx @@ -0,0 +1,18 @@ +"use client"; + +import { useEffect } from "react"; + +export function PageTracker() { + useEffect(() => { + fetch("/api/track/pageview", { + method: "POST", + keepalive: true, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ referrer: document.referrer }), + }).catch(() => { + // Fire-and-forget: analytics must never break the page. + }); + }, []); + + return null; +} diff --git a/frontend/components/test/AnalyticsPage.test.tsx b/frontend/components/test/AnalyticsPage.test.tsx new file mode 100644 index 0000000..d2f2f72 --- /dev/null +++ b/frontend/components/test/AnalyticsPage.test.tsx @@ -0,0 +1,189 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import "@testing-library/jest-dom"; +import AnalyticsPage from "../../app/analytics/page"; +import { fetchAnalytics } from "@/lib/analytics"; +import type { AnalyticsData, AnalyticsResult } from "@/types"; + +jest.mock("@/lib/analytics", () => ({ + fetchAnalytics: jest.fn(), + analyticsErrorMessage: jest.fn( + (status: number | null) => + status === 401 ? "Wrong token." : "Analytics backend unreachable." + ), +})); + +const mockFetchAnalytics = fetchAnalytics as jest.Mock; + +const TOKEN_KEY = "npmatch.analytics.token"; + +const mockData: AnalyticsData = { + summary: { + total_visits: 42, + unique_visitors: 7, + total_searches: 13, + visits_last_24h: 3, + searches_last_24h: 1, + top_queries: [{ label: "react", count: 4 }], + top_frameworks: [{ label: "react", count: 3 }], + top_referrers: [{ label: "(direct)", count: 2 }], + }, + visits: [ + { + visited_at: "2026-08-11T10:00:00Z", + ip_hash: "a1b2c3d4e5f6", + user_agent: "Mozilla/5.0", + referrer: "https://example.com/ref", + }, + ], + searches: [ + { + searched_at: "2026-08-11T10:00:00Z", + query: "react", + framework: "react", + priorities: ["bundle size"], + result_count: 5, + }, + ], +}; + +function okResult(): AnalyticsResult { + return { + summary: mockData.summary, + visits: mockData.visits, + searches: mockData.searches, + errors: {}, + }; +} + +describe("AnalyticsPage", () => { + beforeEach(() => { + window.sessionStorage.clear(); + jest.clearAllMocks(); + }); + + it("shows the gate form when no token is stored", async () => { + render(); + expect(await screen.findByLabelText("Analytics token")).toBeInTheDocument(); + expect(screen.getByText(/Enter your analytics token/i)).toBeInTheDocument(); + expect(mockFetchAnalytics).not.toHaveBeenCalled(); + }); + + it("unlocks with a token and renders the dashboard", async () => { + const user = userEvent.setup(); + mockFetchAnalytics.mockResolvedValue(okResult()); + + render(); + const input = await screen.findByLabelText("Analytics token"); + await user.type(input, "secret"); + await user.click(screen.getByRole("button", { name: /Unlock/i })); + + expect(mockFetchAnalytics).toHaveBeenCalledWith("secret"); + + expect(await screen.findByText("42")).toBeInTheDocument(); + expect(screen.getByText("7")).toBeInTheDocument(); + expect(screen.getByText("Top queries")).toBeInTheDocument(); + expect(screen.getByText("Recent visitors")).toBeInTheDocument(); + expect(screen.getByText("a1b2c3d4…")).toBeInTheDocument(); + expect(screen.getByText("Recent searches")).toBeInTheDocument(); + expect(screen.getAllByText("react").length).toBeGreaterThan(0); + }); + + it("persists a validated token to sessionStorage", async () => { + const user = userEvent.setup(); + mockFetchAnalytics.mockResolvedValue(okResult()); + + render(); + const input = await screen.findByLabelText("Analytics token"); + await user.type(input, "secret"); + await user.click(screen.getByRole("button", { name: /Unlock/i })); + + await screen.findByText("42"); + expect(window.sessionStorage.getItem(TOKEN_KEY)).toBe("secret"); + }); + + it("renders the dashboard directly when a token is already stored", async () => { + window.sessionStorage.setItem(TOKEN_KEY, "stored-token"); + mockFetchAnalytics.mockResolvedValue(okResult()); + + render(); + + expect(await screen.findByText("42")).toBeInTheDocument(); + expect(mockFetchAnalytics).toHaveBeenCalledWith("stored-token"); + }); + + it("shows a distinct wrong-token error and does not persist a bad token", async () => { + const user = userEvent.setup(); + mockFetchAnalytics.mockResolvedValue({ + summary: null, + visits: null, + searches: null, + errors: { summary: 401, visits: 401, searches: 401 }, + }); + + render(); + const input = await screen.findByLabelText("Analytics token"); + await user.type(input, "wrong"); + await user.click(screen.getByRole("button", { name: /Unlock/i })); + + expect(await screen.findByText("Wrong token.")).toBeInTheDocument(); + expect(window.sessionStorage.getItem(TOKEN_KEY)).toBeNull(); + }); + + it("shows a backend-unreachable error for a 5xx summary failure", async () => { + const user = userEvent.setup(); + mockFetchAnalytics.mockResolvedValue({ + summary: null, + visits: null, + searches: null, + errors: { summary: 502, visits: 502, searches: 502 }, + }); + + render(); + const input = await screen.findByLabelText("Analytics token"); + await user.type(input, "secret"); + await user.click(screen.getByRole("button", { name: /Unlock/i })); + + expect( + await screen.findByText("Analytics backend unreachable.") + ).toBeInTheDocument(); + }); + + it("clears a stored token that comes back 401", async () => { + window.sessionStorage.setItem(TOKEN_KEY, "stale-token"); + mockFetchAnalytics.mockResolvedValue({ + summary: null, + visits: null, + searches: null, + errors: { summary: 401, visits: 401, searches: 401 }, + }); + + render(); + + expect(await screen.findByText("Wrong token.")).toBeInTheDocument(); + expect(window.sessionStorage.getItem(TOKEN_KEY)).toBeNull(); + }); + + it("degrades gracefully when only the searches endpoint fails", async () => { + const user = userEvent.setup(); + mockFetchAnalytics.mockResolvedValue({ + summary: mockData.summary, + visits: mockData.visits, + searches: null, + errors: { searches: 502 }, + }); + + render(); + const input = await screen.findByLabelText("Analytics token"); + await user.type(input, "secret"); + await user.click(screen.getByRole("button", { name: /Unlock/i })); + + expect(await screen.findByText("42")).toBeInTheDocument(); + expect(screen.getByText("Recent visitors")).toBeInTheDocument(); + expect(screen.getByText("a1b2c3d4…")).toBeInTheDocument(); + expect( + screen.getByText(/Failed to load recent searches/) + ).toBeInTheDocument(); + expect(screen.queryByText("No searches yet")).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/components/test/PageTracker.test.tsx b/frontend/components/test/PageTracker.test.tsx new file mode 100644 index 0000000..8a8aabb --- /dev/null +++ b/frontend/components/test/PageTracker.test.tsx @@ -0,0 +1,29 @@ +import { render } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import { PageTracker } from "../PageTracker"; + +describe("PageTracker", () => { + beforeEach(() => { + global.fetch = jest.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch; + }); + + it("renders nothing", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("posts a pageview to the tracking route on mount", async () => { + const fetchMock = global.fetch as jest.Mock; + + render(); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(fetchMock).toHaveBeenCalledTimes(1); + + const [url, opts] = fetchMock.mock.calls[0]; + expect(url).toBe("/api/track/pageview"); + expect(opts.method).toBe("POST"); + expect(opts.keepalive).toBe(true); + expect(JSON.parse(opts.body)).toEqual({ referrer: "" }); + }); +}); diff --git a/frontend/jest.config.js b/frontend/jest.config.mjs similarity index 100% rename from frontend/jest.config.js rename to frontend/jest.config.mjs diff --git a/frontend/lib/analytics.ts b/frontend/lib/analytics.ts new file mode 100644 index 0000000..d070a7d --- /dev/null +++ b/frontend/lib/analytics.ts @@ -0,0 +1,56 @@ +import type { AnalyticsData, AnalyticsResult } from "@/types"; + +type SectionResult = { ok: true; value: T } | { ok: false; status: number | null }; + +async function getJson(url: string, token: string): Promise> { + try { + const res = await fetch(url, { + headers: { "x-analytics-token": token }, + cache: "no-store", + }); + if (!res.ok) { + return { ok: false, status: res.status }; + } + return { ok: true, value: (await res.json()) as T }; + } catch { + // Network failure (fetch rejected) — distinct from an HTTP error status + return { ok: false, status: null }; + } +} + +/** User-facing message for an analytics section failure, keyed by failure class. */ +export function analyticsErrorMessage(status: number | null): string { + if (status === 401) return "Wrong token."; + if (status === null || status >= 500) return "Analytics backend unreachable."; + return `Analytics request failed (${status}).`; +} + +export async function fetchAnalytics(token: string): Promise { + const result: AnalyticsResult = { summary: null, visits: null, searches: null, errors: {} }; + + const [summary, visits, searches] = await Promise.all([ + getJson("/api/analytics?kind=summary", token), + getJson("/api/analytics?kind=visits&limit=25", token), + getJson("/api/analytics?kind=searches&limit=25", token), + ]); + + if (summary.ok) { + result.summary = summary.value; + } else { + result.errors.summary = summary.status; + } + + if (visits.ok) { + result.visits = visits.value; + } else { + result.errors.visits = visits.status; + } + + if (searches.ok) { + result.searches = searches.value; + } else { + result.errors.searches = searches.status; + } + + return result; +} diff --git a/frontend/types/index.ts b/frontend/types/index.ts index 1a7b0ff..fac8bf3 100644 --- a/frontend/types/index.ts +++ b/frontend/types/index.ts @@ -38,4 +38,51 @@ export const PRIORITY_OPTIONS = [ "well documented", ] as const; -export type Priority = (typeof PRIORITY_OPTIONS)[number]; \ No newline at end of file +export type Priority = (typeof PRIORITY_OPTIONS)[number]; + +export interface TopItem { + label: string; + count: number; +} + +export interface AnalyticsSummary { + total_visits: number; + unique_visitors: number; + total_searches: number; + visits_last_24h: number; + searches_last_24h: number; + top_queries: TopItem[]; + top_frameworks: TopItem[]; + top_referrers: TopItem[]; +} + +export interface PageView { + visited_at: string; + ip_hash: string; + user_agent: string | null; + referrer: string | null; +} + +export interface SearchEvent { + searched_at: string; + query: string; + framework: string | null; + priorities: string[] | null; + result_count: number; +} + +export interface AnalyticsData { + summary: AnalyticsSummary; + visits: PageView[]; + searches: SearchEvent[]; +} + +export type AnalyticsSection = "summary" | "visits" | "searches"; + +export interface AnalyticsResult { + summary: AnalyticsSummary | null; + visits: PageView[] | null; + searches: SearchEvent[] | null; + /** Per-section HTTP status on failure; null means a network error. Absent = success. */ + errors: Partial>; +} \ No newline at end of file diff --git a/ingestion/src/upsert.ts b/ingestion/src/upsert.ts index 6383fb0..cd9c956 100644 --- a/ingestion/src/upsert.ts +++ b/ingestion/src/upsert.ts @@ -61,6 +61,39 @@ async function ensurePgTable(): Promise { ON packages USING GIN(search_vector) `); + await pool.query(` + CREATE TABLE IF NOT EXISTS page_views ( + id BIGSERIAL PRIMARY KEY, + visited_at TIMESTAMPTZ NOT NULL DEFAULT now(), + ip_hash TEXT, + user_agent TEXT, + referrer TEXT + ) + `); + + await pool.query(` + CREATE INDEX IF NOT EXISTS idx_page_views_visited_at + ON page_views (visited_at DESC) + `); + + await pool.query(` + CREATE TABLE IF NOT EXISTS search_events ( + id BIGSERIAL PRIMARY KEY, + searched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + query TEXT NOT NULL, + framework TEXT, + priorities TEXT, + result_count INTEGER, + ip_hash TEXT, + user_agent TEXT + ) + `); + + await pool.query(` + CREATE INDEX IF NOT EXISTS idx_search_events_searched_at + ON search_events (searched_at DESC) + `); + console.log("Postgres table ready"); }