Add Jira multi-account support, connection-aware sync/matching, and related fixes

backend/analytics_server/mhq/api/integrations.py

@@ -7,9 +7,12 @@
 from typing import Dict, List, Tuple
 from flask import Blueprint, jsonify
 from github import GithubException
+from sqlalchemy.exc import IntegrityError
 
-# CLUSTOX: Required is used by the Jenkins mapping request schemas below.
-from voluptuous import Schema, Optional, Required, Coerce, Range, All
+# CLUSTOX: Required is used by the Jenkins mapping request schemas below. Any
+# is used by the JiraConnection routes' generated_by field, which is either a
+# uuid string or None.
+from voluptuous import Schema, Optional, Required, Coerce, Range, All, Any
 
 from mhq.exapi.models.gitlab import GitlabRepo
 
@@ -26,6 +29,13 @@
 from mhq.store.repos.code import CodeRepoService
 from mhq.store.repos.workflows import WorkflowRepoService
 
+# CLUSTOX: JiraConnection routes -- see docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 3.
+from mhq.store.repos.jira_connection import (
+    JiraConnectionInUseError,
+    JiraConnectionNotFoundError,
+    JiraConnectionRepoService,
+)
+
 # END CLUSTOX
 from mhq.utils.github import github_org_data_multi_thread_worker
 
@@ -652,3 +662,118 @@ def get_gitlab_user_projects(org_id: str, page_size: int, page: int):
         }
         for project in projects
     ]
+
+
+# CLUSTOX: JiraConnection routes -- Task 3 of docs/JIRA_MULTI_ACCOUNT_PLAN.md.
+# The web-server's jira-connections/* BFF routes proxy straight through to
+# these, the same way the Jenkins mapping routes above do, rather than writing
+# JiraConnection/OrgProjectConnection from knex directly: create/delete/
+# set-default all carry business rules (encrypt-on-write, delete-blocked-by-
+# reference, default-switch atomicity) that belong in one place, enforced
+# for every caller, not re-implemented in TypeScript.
+def _serialize_jira_connection(connection) -> Dict:
+    # access_token_enc_chunks is deliberately never included -- there is no
+    # caller of this serializer that should ever see even the encrypted form.
+    return {
+        "id": str(connection.id),
+        "site_url": connection.site_url,
+        "email": connection.email,
+        "is_default": connection.is_default,
+        "provider_meta": connection.provider_meta or {},
+        "created_at": (
+            connection.created_at.isoformat() if connection.created_at else None
+        ),
+    }
+
+
+@app.route("/orgs/<org_id>/integrations/jira-connections", methods={"GET"})
+def list_jira_connections(org_id: str):
+    query_validator = get_query_validator()
+    query_validator.org_validator(org_id)
+
+    service = JiraConnectionRepoService()
+    connections = service.list_jira_connections(org_id)
+    return jsonify([_serialize_jira_connection(c) for c in connections])
+
+
+@app.route("/orgs/<org_id>/integrations/jira-connections", methods={"POST"})
+@dataschema(
+    Schema(
+        {
+            Required("site_url"): str,
+            Required("email"): str,
+            Required("access_token"): str,
+            Optional("provider_meta", default={}): dict,
+            Optional("generated_by", default=None): Any(
+                All(str, Coerce(uuid_validator)), None
+            ),
+        }
+    ),
+)
+def create_jira_connection(
+    org_id: str,
+    site_url: str,
+    email: str,
+    access_token: str,
+    provider_meta: dict,
+    generated_by: str = None,
+):
+    query_validator = get_query_validator()
+    query_validator.org_validator(org_id)
+
+    service = JiraConnectionRepoService()
+    try:
+        connection = service.create_jira_connection(
+            org_id, site_url, email, access_token, provider_meta, generated_by
+        )
+    except IntegrityError:
+        # jira_connection_unique_account: this (org_id, site_url, email) is
+        # already connected. Named explicitly rather than surfacing the raw
+        # constraint violation -- see delete/set-default below for the same
+        # rule applied to their own DB-level invariants.
+        return (
+            jsonify(
+                {
+                    "error": (
+                        f"{email} is already connected to {site_url} for this "
+                        "workspace"
+                    )
+                }
+            ),
+            409,
+        )
+    return jsonify(_serialize_jira_connection(connection)), 201
+
+
+@app.route(
+    "/orgs/<org_id>/integrations/jira-connections/<connection_id>",
+    methods={"DELETE"},
+)
+def delete_jira_connection(org_id: str, connection_id: str):
+    query_validator = get_query_validator()
+    query_validator.org_validator(org_id)
+
+    service = JiraConnectionRepoService()
+    try:
+        service.delete_jira_connection(org_id, connection_id)
+    except JiraConnectionNotFoundError as e:
+        return jsonify({"error": str(e)}), 404
+    except JiraConnectionInUseError as e:
+        return jsonify({"error": str(e)}), 409
+    return jsonify({"ok": True})
+
+
+@app.route(
+    "/orgs/<org_id>/integrations/jira-connections/<connection_id>",
+    methods={"PATCH"},
+)
+def set_default_jira_connection(org_id: str, connection_id: str):
+    query_validator = get_query_validator()
+    query_validator.org_validator(org_id)
+
+    service = JiraConnectionRepoService()
+    try:
+        service.set_default_jira_connection(org_id, connection_id)
+    except JiraConnectionNotFoundError as e:
+        return jsonify({"error": str(e)}), 404
+    return jsonify({"ok": True})

backend/analytics_server/mhq/api/request_utils.py

@@ -126,6 +126,7 @@ def coerce_org_project(project: Dict[str, str]) -> RawTeamOrgProject:
         key=project.get("key"),
         name=project.get("name"),
         idempotency_key=project.get("idempotency_key"),
+        connection_id=project.get("connection_id"),
     )
 
 

backend/analytics_server/mhq/api/teams.py

@@ -36,8 +36,10 @@
     coerce_team_repos,
     dataschema,
     queryschema,
+    uuid_validator,
 )
 from mhq.service.query_validator import get_query_validator
+from mhq.store.repos.team_repo_project_mapping import TeamRepoProjectMappingRepoService
 
 app = Blueprint("teams", __name__)
 
@@ -216,6 +218,76 @@ def update_team_projects(team_id: str, projects: List[RawTeamOrgProject]):
     return adapt_org_projects(updated_org_projects)
 
 
+# CLUSTOX: explicit, informational repo<->Jira-project pairing -- see
+# docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on Jira<->repo relationships
+# and TeamRepoProjectMapping's own docstring for why this never drives
+# PR<->ticket matching. Same "GET the current set / PUT the full
+# replacement set" shape as /teams/<team_id>/projects above.
+@app.route("/teams/<team_id>/repo_project_mappings", methods={"GET"})
+def fetch_team_repo_project_mappings(team_id: str):
+    query_validator = get_query_validator()
+    team: Team = query_validator.team_validator(team_id)
+
+    mappings = TeamRepoProjectMappingRepoService().get_mappings_for_team(team.id)
+    return [
+        {
+            "org_repo_id": str(mapping.org_repo_id),
+            "org_project_id": str(mapping.org_project_id),
+        }
+        for mapping in mappings
+    ]
+
+
+@app.route("/teams/<team_id>/repo_project_mappings", methods={"PUT"})
+@dataschema(
+    Schema(
+        {
+            Required("mappings"): [
+                {
+                    Required("org_repo_id"): All(str, Coerce(uuid_validator)),
+                    Required("org_project_id"): All(str, Coerce(uuid_validator)),
+                }
+            ],
+        }
+    ),
+)
+def update_team_repo_project_mappings(team_id: str, mappings: List[Dict[str, str]]):
+    query_validator = get_query_validator()
+    team: Team = query_validator.team_validator(team_id)
+
+    # Informational only, but a pair naming a repo or project this team
+    # hasn't actually selected would be a mapping to nothing meaningful --
+    # rejected rather than silently stored, same "fail loud at the
+    # boundary" convention the rest of this API follows.
+    team_repo_ids = {
+        str(repo.id) for repo in get_repository_service().get_team_repos(team)
+    }
+    team_project_ids = {
+        str(project.id) for project in get_project_service().get_team_projects(team)
+    }
+    for mapping in mappings:
+        if mapping["org_repo_id"] not in team_repo_ids:
+            raise BadRequest(
+                f"Repo {mapping['org_repo_id']} is not tracked by team {team_id}."
+            )
+        if mapping["org_project_id"] not in team_project_ids:
+            raise BadRequest(
+                f"Project {mapping['org_project_id']} is not tracked by team {team_id}."  # noqa E501
+            )
+
+    saved = TeamRepoProjectMappingRepoService().set_mappings_for_team(
+        team.id,
+        [(mapping["org_repo_id"], mapping["org_project_id"]) for mapping in mappings],
+    )
+    return [
+        {
+            "org_repo_id": str(mapping.org_repo_id),
+            "org_project_id": str(mapping.org_project_id),
+        }
+        for mapping in saved
+    ]
+
+
 # CLUSTOX: Jira integration, Phase 4 (§6C/§6E) -- the DORA Metrics page's
 # ticket-cycle-time widget and "N PRs merged with no linked ticket"
 # callout. Deliberately its own read-only endpoint, not folded into the

backend/analytics_server/mhq/service/project/integration.py

@@ -2,6 +2,7 @@
 
 from mhq.store.models import Integration, UserIdentityProvider
 from mhq.store.repos.core import CoreRepoService
+from mhq.store.repos.jira_connection import JiraConnectionRepoService
 
 # CLUSTOX: Jira integration, Phase 2 (issue sync). Mirrors
 # mhq/service/code/integration.py's CODE_INTEGRATION_BUCKET -- one entry
@@ -14,19 +15,36 @@
 
 
 class ProjectIntegrationService:
-    def __init__(self, core_repo_service: CoreRepoService):
+    def __init__(
+        self,
+        core_repo_service: CoreRepoService,
+        jira_connection_repo_service: JiraConnectionRepoService,
+    ):
         self.core_repo_service = core_repo_service
+        self.jira_connection_repo_service = jira_connection_repo_service
 
     def get_org_providers(self, org_id: str) -> List[str]:
         integrations: List[Integration] = (
             self.core_repo_service.get_org_integrations_for_names(
                 org_id, PROJECT_INTEGRATION_BUCKET
             )
         )
-        if not integrations:
-            return []
-        return [integration.name for integration in integrations]
+        providers = {integration.name for integration in integrations}
+
+        # CLUSTOX: an org can go straight to JiraConnection without ever
+        # linking the legacy Integration(name='jira') row -- see
+        # docs/JIRA_MULTI_ACCOUNT_PLAN.md. Checking only Integration here
+        # left such an org with an empty provider list, and
+        # sync_project_issues returns before ever calling the Jira ETL
+        # factory -- a JiraConnection-only org would silently never sync.
+        if self.jira_connection_repo_service.list_jira_connections(org_id):
+            providers.add(UserIdentityProvider.JIRA.value)
+
+        return list(providers)
 
 
 def get_project_integration_service():
-    return ProjectIntegrationService(core_repo_service=CoreRepoService())
+    return ProjectIntegrationService(
+        core_repo_service=CoreRepoService(),
+        jira_connection_repo_service=JiraConnectionRepoService(),
+    )

backend/analytics_server/mhq/service/project/models/org_project.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Optional
 
 
 @dataclass
@@ -8,3 +9,7 @@ class RawTeamOrgProject:
     key: str
     name: str
     idempotency_key: str
+    # CLUSTOX: which JiraConnection this project was picked under, if any --
+    # None means either a non-Jira provider or the legacy single-account
+    # Integration flow. See docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6 part 2.
+    connection_id: Optional[str] = None

backend/analytics_server/mhq/service/project/repository_service.py

@@ -2,7 +2,7 @@
 
 from mhq.service.project.models.org_project import RawTeamOrgProject
 from mhq.store.models.core import Team
-from mhq.store.models.projects import OrgProject
+from mhq.store.models.projects import OrgProject, OrgProjectConnection
 from mhq.store.repos.projects import ProjectRepoService, TeamProjects
 from mhq.utils.string import uuid4_str
 
@@ -59,6 +59,14 @@ def _update_org_projects(
         }
 
         updated_org_projects = []
+        # CLUSTOX: which connection each project (new or existing) came from,
+        # keyed by the project's own id -- collected alongside the loop below
+        # since that's the only place both the project id (freshly minted for
+        # a new row) and the raw request's connection_id are in scope
+        # together. Written only after updated_org_projects is committed
+        # (see the OrgProjectConnection FK), so this stays a plan, not a
+        # write, until after the return below.
+        connection_id_by_project_id: Dict[str, str] = {}
         for raw_project in raw_org_projects:
             existing_project = idempotency_key_to_project_map.get(
                 raw_project.idempotency_key
@@ -75,27 +83,44 @@ def _update_org_projects(
                 existing_project.is_active = True
                 existing_project.key = raw_project.key
                 existing_project.name = raw_project.name
-                updated_org_projects.append(existing_project)
+                project = existing_project
             else:
-                updated_org_projects.append(
-                    OrgProject(
-                        id=uuid4_str(),
-                        org_id=org_id,
-                        key=raw_project.key,
-                        name=raw_project.name,
-                        provider=raw_project.provider,
-                        idempotency_key=raw_project.idempotency_key,
-                        # Explicit rather than relying on the column's
-                        # SQLAlchemy-level default -- that default only
-                        # materializes on this instance once it's actually
-                        # flushed through a real DB session, which makes
-                        # the object momentarily wrong (is_active=None) to
-                        # anything inspecting it beforehand.
-                        is_active=True,
-                    )
+                project = OrgProject(
+                    id=uuid4_str(),
+                    org_id=org_id,
+                    key=raw_project.key,
+                    name=raw_project.name,
+                    provider=raw_project.provider,
+                    idempotency_key=raw_project.idempotency_key,
+                    # Explicit rather than relying on the column's
+                    # SQLAlchemy-level default -- that default only
+                    # materializes on this instance once it's actually
+                    # flushed through a real DB session, which makes
+                    # the object momentarily wrong (is_active=None) to
+                    # anything inspecting it beforehand.
+                    is_active=True,
                 )
-
-        return self._project_repo_service.update_org_projects(updated_org_projects)
+            updated_org_projects.append(project)
+            # A project saved without a connection_id (legacy flow, or a
+            # non-Jira provider) is left alone here -- not de-associated --
+            # see save_org_project_connections's own docstring.
+            if raw_project.connection_id:
+                connection_id_by_project_id[str(project.id)] = raw_project.connection_id
+
+        saved_projects = self._project_repo_service.update_org_projects(
+            updated_org_projects
+        )
+        if connection_id_by_project_id:
+            self._project_repo_service.save_org_project_connections(
+                [
+                    OrgProjectConnection(
+                        org_project_id=project_id,
+                        jira_connection_id=connection_id,
+                    )
+                    for project_id, connection_id in connection_id_by_project_id.items()
+                ]
+            )
+        return saved_projects
 
     def _update_team_projects(
         self,