feat(operations): Phase 3 operations, widgets and CLI auth bridge

.env.example

@@ -0,0 +1,35 @@
+# TCRM Toolkit Configuration
+# Copy this file to .env and fill in your values
+
+# Application Settings
+APP_NAME=tcrm-toolkit
+APP_VERSION=0.1.0
+DEBUG=false
+LOG_LEVEL=INFO
+
+# Salesforce API Settings
+SF_API_VERSION=v60.0
+SF_DEFAULT_DOMAIN=login.salesforce.com
+
+# Encryption Settings
+# Generate a secure key: python -c "import base64, os; print(base64.urlsafe_b64encode(os.urandom(32)).decode())"
+ENCRYPTION_KEY=your-base64-encoded-32-byte-key-here
+
+# JWT Settings (for Connected App JWT Bearer flow)
+JWT_SECRET_KEY=your-jwt-secret-key-min-32-chars
+JWT_ALGORITHM=HS256
+ACCESS_TOKEN_EXPIRE_MINUTES=30
+REFRESH_TOKEN_EXPIRE_DAYS=30
+
+# Connected App Credentials (for JWT Bearer flow)
+# SF_CONNECTED_APP_CLIENT_ID=your-consumer-key
+# SF_CONNECTED_APP_CLIENT_SECRET=your-consumer-secret
+# SF_CONNECTED_APP_USERNAME=your-salesforce-username
+
+# Web OAuth Settings (for PKCE flow)
+# SF_WEB_OAUTH_CLIENT_ID=your-consumer-key
+# SF_WEB_OAUTH_CLIENT_SECRET=your-consumer-secret
+# SF_WEB_OAUTH_REDIRECT_URI=http://localhost:8080/callback
+
+# Device Flow Settings (for headless/terminal login)
+# SF_DEVICE_FLOW_CLIENT_ID=your-consumer-key

.github/workflows/ci.yml

@@ -0,0 +1,54 @@
+name: CI
+
+on:
+ push:
+ branches: [main, refactor/**, feature/**]
+ pull_request:
+ branches: [main]
+
+jobs:
+ test:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-latest, windows-latest, macos-latest]
+ python-version: ["3.11", "3.12", "3.13"]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v3
+
+ - name: Install dependencies
+ run: uv sync --extra interactive --extra dev
+
+ - name: Run linting
+ run: uv run ruff check .
+ continue-on-error: true
+
+ - name: Run type checking
+ run: uv run mypy tcrm_toolkit
+ continue-on-error: true
+
+ - name: Run tests
+ run: uv run pytest -v --tb=short
+
+ - name: Verify cross-platform
+ run: uv run python scripts/verify-cross-platform.py
+
+ docker:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Build Docker image
+ run: docker build -t crma-toolkit:test .
+
+ - name: Test Docker image
+ run: docker run --rm crma-toolkit:test --help

.gitignore

@@ -0,0 +1,52 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Virtual environments
+.venv/
+venv/
+env/
+.env
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+*.egg
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+
+# Test coverage
+.coverage
+htmlcov/
+.pytest_cache/
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Ruff
+.ruff_cache/
+
+# Local development
+.env.local
+.env.*.local
+
+# Keyring (if using file backend)
+*.keyring
+
+# PDF outputs
+*.pdf

AGENTS.md

@@ -0,0 +1,103 @@
+# AGENTS.md — TCRM Toolkit
+
+Guidance for agents working in this repository. Read before editing.
+
+## Project
+
+Modern async Python CLI for Salesforce Tableau CRM (TCRM). Rebuilds the legacy
+`FOSS_Toolkit.py` (now preserved under `_legacy/`) with a `typer` CLI +
+`textual` interactive TUI. Current work lives on `feature/interactive-tui-complete`
+(the old `main` only holds the dead legacy toolkit).
+
+---
+
+## Mandatory GitHub & Git Workflow
+
+- **Repository**: `pg-dev-git/foss_analytics_toolkit`
+- **Never work directly on `main` (or `master`).**
+- **Branch Naming**: Always create a feature branch from `main`:
+ - `feature/` for new features (e.g., `feature/order-form`, `feature/product-image-upload`)
+ - `fix/` for bug fixes (e.g., `fix/payment-code-generation`)
+ - `chore/` for maintenance/infrastructure (e.g., `chore/docker-compose`)
+ - `docs/` for documentation updates (e.g., `docs/prd-and-phases`)
+- **Standard GitHub Flow**:
+ 1. Create and switch to feature branch: `git checkout -b feature/your-feature-name`
+ 2. Make small, atomic commits with conventional commit messages.
+ 3. Push branch to remote: `git push -u origin feature/your-feature-name`
+ 4. Open a Pull Request against `main` using `create_pull_request` (do not merge to `main` without review / completion of phase).
+
+### Conventional Commits Guidelines
+Use the format `<type>(<scope>): <short description>`:
+- `feat(cart): add quantity increment/decrement steppers`
+- `fix(checkout): resolve missing phone number validation error`
+- `style(ui): update mobile touch target padding in product grid`
+- `refactor(pb): extract pocketbase query helper functions`
+- `docs(readme): add cloudflare tunnel setup instructions`
+
+---
+
+## Environment & Commands
+
+- Use **uv**, not pip. Run everything via `uv run ...`.
+- Install deps: `uv sync --extra interactive --extra dev`
+- Tests: `uv run pytest -v --tb=short` (expect 42 passing)
+- Lint: `uv run ruff check .` · Typecheck: `uv run mypy tcrm_toolkit`
+- Cross-platform check: `uv run python scripts/verify-cross-platform.py`
+- CLI entrypoint: `tcrm` (`tcrm --help`, `tcrm doctor`)
+- Tests need real encryption keys — `conftest.py` provides them; copy
+ `.env.example` → `.env` only if you need live settings.
+
+## Architecture (layered — respect the dependency direction)
+
+`cli/` → `interactive/` → `core/services/` → `core/` (client, models, config, crypto, auth)
+
+- **cli/** — `typer` command groups (`commands/{auth,dashboards,dataflows,datasets,jobs}.py`).
+ Wire user actions directly to `core/services/`. Register with
+ `app.add_typer(sub_app, name="...")` in `cli/main.py`.
+- **interactive/** — `textual` TUI: `screens/` (Login, Main, OrgPicker, SafetyModal, Help),
+ `widgets/` (DataBrowser, DetailPanel, StatusBar, CommandPalette, ...),
+ `operations/` (extract/upload/backup/control), `tasks.py` (`TaskRunner` + `ProcessPoolExecutor`).
+- **core/services/** — the real business logic: `dataset_service`, `dashboard_service`,
+ `dataflow_service`, `auth_service`. These are async and already exercised by tests.
+ `session.py::SessionManager` owns auth + `client_context()`; every network call must go
+ through it, never construct a bare client.
+- **core/** — `client.py` (httpx + tenacity retries), `models/` (pydantic), `config.py`
+ (pydantic-settings, `TCRM_*` env vars), `crypto.py` (dynamic-salt encryption),
+ `auth/{sf_cli_auth,token_store}.py`.
+
+## Conventions
+
+- Python 3.11+, `pathlib.Path` everywhere (no path separators hardcoded).
+- Async throughout: `async/await`, `asyncio.gather`/`Semaphore` for I/O,
+ `ProcessPoolExecutor` only for CPU-bound work (pandas).
+- `structlog` for logging. `datetime.now(timezone.utc)` — **not** `datetime.utcnow()`
+ (deprecated, flagged in CI warnings).
+- **Error Logging**: Centralized structured JSON logging via `core/logger.py` writes concurrently to `stderr` and persistent log file `~/.tcrm/tcrm.log`. TUI unhandled errors and background worker failures are intercepted and logged via `TCRMApp.on_error`.
+- Keep it simple: no special-case glue, no dead stubs. Prefer eliminating a branch over
+ guarding it.
+
+## Testing
+
+- `tests/unit/` (crypto, client retry, auth schemas, interactive) +
+ `tests/integration/test_api_endpoints.py` (uses `AsyncMock` on `SalesforceClient._client`).
+- Add tests for new/changed behaviour. Assert on real outputs/state, not mocked calls.
+- Integration tests mock the HTTP layer via the `mock_client` fixture — exercise real
+ service code, not the client.
+- **Live Session Seeding & E2E Testing**:
+ - To test against a live Salesforce org without interactive browser login, use `scripts/seed_session.py`:
+ ```bash
+ TCRM_ACCESS_TOKEN="..." TCRM_INSTANCE_URL="..." TCRM_USERNAME="..." uv run python scripts/seed_session.py
+ ```
+ - **Token Expiry**: If a live session token expires or encounters authentication errors during testing, the agent must immediately notify the user to request a fresh token.
+
+## Gotchas
+
+- **Phases are NOT all done.** Phases 0–2 are complete. Phase 3 (operations/background
+ tasks) and Phase 4 (polish/DX) are partial — see the phase-completion review.
+ `TASK_LOG.md` checkboxes are stale; trust `docs/plans/phases/README.md`.
+- CLI write commands are currently **stubs** (`"Authentication integration pending"`,
+ `datasets.py` even raises `NotImplementedError`). The service layer is done; the
+ CLI↔keyring↔service auth bridge is not.
+- `widgets/progress_panel.py` and `widgets/task_history.py` are empty `Static` subclasses.
+- CI on `main` does not run (workflow targets `main`/`refactor/**`/`feature/**`; `main`
+ holds only the legacy toolkit).

Dockerfile

@@ -0,0 +1,52 @@
+# =============================================================================
+# CRMA Toolkit - Interactive TUI Docker Image
+# Multi-stage build for minimal production image
+# =============================================================================
+
+# ---- Build Stage ----
+FROM python:3.12-slim AS builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ build-essential \
+ curl \
+ git \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock* ./
+COPY . .
+
+RUN uv pip install --system -e .[interactive,dev]
+
+# ---- Runtime Stage ----
+FROM python:3.12-slim AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ curl \
+ gnupg \
+ procps \
+ npm \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g @salesforce/cli
+
+RUN useradd -m -s /bin/bash tcrm && \
+ mkdir -p /home/tcrm/.config /home/tcrm/.local/share && \
+ chown -R tcrm:tcrm /home/tcrm
+
+COPY --from=builder /app /app
+RUN chown -R tcrm:tcrm /app
+WORKDIR /app
+RUN pip install -e .
+
+ENV HOME="/home/tcrm"
+ENV USER="tcrm"
+
+USER tcrm
+WORKDIR /home/tcrm
+
+ENTRYPOINT ["tcrm"]
+CMD ["--help"]

FOSS_Toolkit.py

@@ -1,17 +1,20 @@
-import json, requests, os, configparser, datetime, time, multiprocessing as mp
-from misc_tasks.terminal_colors import *
-from misc_tasks.sfdc_login import *
-from dataset_tasks.get_datasets import *
-from dataset_tasks.csv_new_dataset import *
-from dataflow_tasks.get_dataflows import *
-from data_manager_tasks.get_dataflowjobs import *
-from misc_tasks.initial_checks import *
+import multiprocessing as mp
+import os
+import time
+
 from dashboards_tasks.get_dashboards import *
 from dashboards_tasks.mass_dashboard_backup import *
-from misc_tasks.get_ea_limits import *
-from misc_tasks.line import *
+from data_manager_tasks.get_dataflowjobs import *
+from dataflow_tasks.get_dataflows import *
 from dataflow_tasks.mass_dataflows_backup import *
+from dataset_tasks.csv_new_dataset import *
+from dataset_tasks.get_datasets import *
 from dataset_tasks.mass_user_xmd_backup import *
+from misc_tasks.get_ea_limits import *
+from misc_tasks.initial_checks import *
+from misc_tasks.line import *
+from misc_tasks.sfdc_login import *
+from misc_tasks.terminal_colors import *
 
 if __name__ == "__main__":