From 76bf0e6efe5348cc1ddd32f04f394eaff3006e39 Mon Sep 17 00:00:00 2001 From: Chris Dayne Date: Tue, 4 Aug 2026 09:48:32 +1000 Subject: [PATCH] feat: add tests for transitive dependencies extraction and publishing - Implemented unit tests for extracting backend pool members and policy fragment dependencies transitively in extract-service. - Added tests to ensure API exclusions are respected during product publishing in product-publisher. - Enhanced publish-service tests to cover filtering resources, including transitive dependencies and dry-run scenarios. - Introduced tests for resource-publisher to skip entries based on exclusion filters for subscriptions and API tags. - Added tests for workspace-extractor to validate workspace exclusions and transitive extraction of policy fragments. - Created transitive-extractor service to handle extraction of transitive dependencies with proper logging and error handling. --- README.md | 15 +- docs/architecture.md | 1 + docs/commands/publish.md | 18 +- docs/guides/filtering-resources.md | 58 ++- src/cli/publish-command.ts | 28 +- src/lib/resource-path.ts | 17 + src/models/config.ts | 2 + src/services/api-publisher.ts | 167 ++++++++- src/services/extract-service.ts | 97 ++--- src/services/filter-service.ts | 49 ++- src/services/product-publisher.ts | 100 +++++- src/services/publish-service.ts | 251 +++++++++++-- src/services/resource-publisher.ts | 86 +++++ src/services/transitive-extractor.ts | 99 +++++ src/services/transitive-resolver.ts | 189 +++++++++- src/services/workspace-extractor.ts | 140 ++++---- tests/unit/cli/publish-command.test.ts | 20 ++ tests/unit/lib/resource-path.test.ts | 32 ++ tests/unit/services/api-publisher.test.ts | 257 +++++++++++++ tests/unit/services/extract-service.test.ts | 72 ++++ tests/unit/services/product-publisher.test.ts | 88 +++++ tests/unit/services/publish-service.test.ts | 339 ++++++++++++++++++ .../unit/services/resource-publisher.test.ts | 184 ++++++++++ .../unit/services/transitive-resolver.test.ts | 117 +++++- .../unit/services/workspace-extractor.test.ts | 140 ++++++++ 25 files changed, 2332 insertions(+), 234 deletions(-) create mode 100644 src/services/transitive-extractor.ts diff --git a/README.md b/README.md index aec7ad71..363ff38e 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,11 @@ Publish local artifact files to an Azure APIM service. | `--service-name ` | *(required)* | APIM service name | | `--source ` | `./apim-artifacts` | Source artifacts directory | | `--overrides ` | | Path to overrides file | +| `--filter ` | | Filter YAML file (same format as `extract`) | +| `--no-transitive` | | Publish only filter matches, without referenced dependencies | | `--commit-id ` | | Git commit SHA for incremental publish | | `--dry-run` | | Preview changes without applying | -| `--delete-unmatched` | | Delete resources not in artifacts (mutually exclusive with `--commit-id`) | +| `--delete-unmatched` | | Delete resources not in artifacts (mutually exclusive with `--filter` and `--commit-id`) | ```bash apiops publish --help @@ -107,8 +109,19 @@ apiops publish \ --resource-group \ --service-name \ --commit-id + +# Publish a filtered subset; transitive dependencies are included by default +apiops publish \ + --resource-group \ + --service-name \ + --source ./apim-artifacts \ + --filter ./filter.yaml ``` +The publish filter uses the same YAML file and matching rules as `apiops extract --filter`. +Referenced named values, backends (including backend pool members), policy fragments, and version +sets are included automatically. Add `--no-transitive` to publish only the exact filter matches. + ### `apiops init` Scaffold a new APIM artifacts repository with CI/CD pipelines. diff --git a/docs/architecture.md b/docs/architecture.md index 08c20f48..e4e1b211 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -109,6 +109,7 @@ flowchart TB extract_svc --> apim_client extract_svc --> store + publish_svc --> filter_svc publish_svc --> override_svc publish_svc --> git_svc publish_svc --> dry_svc diff --git a/docs/commands/publish.md b/docs/commands/publish.md index d074d5fe..ef58ffbe 100644 --- a/docs/commands/publish.md +++ b/docs/commands/publish.md @@ -44,6 +44,20 @@ apiops publish \ --commit-id abc123def456 ``` +### Publish a filtered subset + +```bash +apiops publish \ + --resource-group my-rg \ + --service-name my-apim \ + --filter ./configuration.extractor.yaml +``` + +The filter uses the same YAML format as `apiops extract --filter`. Referenced named values, +backends (including backend pool members), policy fragments, and version sets are included by +default; use `--no-transitive` to publish only exact matches. Product, gateway, and subscription +links do not pull their API or Product targets into the publish set. + ### Delete resources not in source ```bash @@ -73,11 +87,13 @@ apiops publish \ | `--service-name ` | string | — | Yes | APIM service instance name | | `--source ` | string | `./apim-artifacts` | No | Source directory containing artifacts | | `--overrides ` | string | — | No | Override configuration YAML file | +| `--filter ` | string | — | No | Filter YAML file shared with `extract` | +| `--no-transitive` | boolean | `false` | No | Publish only exact filter matches | | `--commit-id ` | string | env: `COMMIT_ID` | No | Git commit SHA for incremental publish | | `--dry-run` | boolean | `false` | No | Preview changes without applying | | `--delete-unmatched` | boolean | `false` | No | Delete APIM resources not present in source | -> **Note:** `--commit-id` and `--delete-unmatched` are **mutually exclusive**. The CLI will error if both are specified. +> **Note:** `--filter` and `--commit-id` can be combined; `--delete-unmatched` cannot be combined with either option. ### Global flags diff --git a/docs/guides/filtering-resources.md b/docs/guides/filtering-resources.md index f643f53b..51ec2adc 100644 --- a/docs/guides/filtering-resources.md +++ b/docs/guides/filtering-resources.md @@ -64,7 +64,7 @@ documentations: [] workspaces: [] ``` -With transitive resolution enabled (the default), any version set, backend, named value, policy fragment, or tag directly referenced by `my-own-api` or its policies is still pulled in automatically — even though those keys are set to `[]`. Use `--no-transitive` to disable that behavior. +With transitive resolution enabled (the default), any version set, backend, named value, or policy fragment directly referenced by `my-own-api` or its policies is still pulled in automatically — even though those keys are set to `[]`. Use `--no-transitive` to disable that behavior. The three states for every key are: @@ -200,16 +200,14 @@ namedValues: - '!keyvault-*' ``` -Exclusions work anywhere a string list is accepted, including sub-filter fields inside `apiSubFilters` and `workspaceSubFilters`: +Exclusions work anywhere a string list is accepted, including nested API and workspace sub-filters: ```yaml apis: - - 'my-api' -apiSubFilters: - my-api: - operations: - - 'get-*' - - '!get-internal-*' # keep all get-* operations except internal ones + - 'my-api': + operations: + - 'get-*' + - '!get-internal-*' # keep all get-* operations except internal ones ``` --- @@ -260,7 +258,7 @@ Supported workspace sub-filter keys: `apis`, `backends`, `diagnostics`, `groups` --- -## All 16 Filterable Resource Types +## All 17 Filterable Resource Types | Filter Field | APIM Resource | Example Values | |-------------|---------------|----------------| @@ -295,10 +293,9 @@ flowchart TD A[Filtered API] --> B[Backends referenced in policies] A --> C[Named Values referenced in policies] A --> D[Policy Fragments included in policies] - A --> E[Tags attached to the API] - A --> F[Products containing the API] - A --> G[Diagnostics configured on the API] - A --> H[Version Set the API belongs to] + A --> E[Version Set the API belongs to] + B --> F[Backend pool members] + D --> G[Resources referenced by fragment content] ``` ### Example @@ -315,11 +312,23 @@ If `petstore-api` has a policy that references: - Named value `petstore-api-key` → **auto-included** - Policy fragment `rate-limit-fragment` → **auto-included** -And `petstore-api` is assigned to: -- Product `starter` → **auto-included** -- Tag `production` → **auto-included** +The extract output includes these dependencies even though only `apis` was specified in the filter. +Product, gateway, tag, and subscription links do not reverse the dependency direction or +auto-include composite API or Product targets. + +### API Children Are Still Extracted + +Tags and diagnostics were not removed from API extraction. They are API child artifacts, not +transitive dependencies: -The extract output includes all of these, even though only `apiNames` was specified in the filter. +- Selecting an API also extracts its API tag associations and API diagnostics. +- Use a nested API `diagnostics` filter to narrow or exclude API diagnostics. +- Product selection remains independent. Selecting an API does not find and include Products that + contain it; omit `products` to include all Products, list specific Products, or use `products: []` + to include none. + +Top-level `tags` and `diagnostics` filter their resource definitions independently from API child +artifacts. This follows the general rule that every top-level filter key is independent. ### Why Transitive Matters @@ -329,7 +338,7 @@ Without transitive resolution, publishing the extracted artifacts to a new APIM ## Disabling Transitive Dependencies -Use `--no-transitive` to extract **only** the explicitly listed resources: +Use `--no-transitive` to disable referenced dependency expansion: ```bash apiops extract \ @@ -345,6 +354,9 @@ apiops extract \ - You want a minimal extract and will handle missing references manually - Debugging — to see exactly what was explicitly filtered +Selected parent resources still include their normal child artifacts. For example, selecting an API +still extracts its tags and diagnostics unless nested API sub-filters exclude them. + > ⚠️ **Caution:** Extracted artifacts without transitive dependencies may not be publishable standalone. You'll need to ensure all referenced resources exist in the target APIM instance. --- @@ -401,7 +413,15 @@ backends: ### Full Extract Minus Specific Resources -There is no "exclude" syntax. To extract everything except certain resources, list all the resources you _do_ want. For large instances, it's often easier to extract everything and use `.gitignore` or separate branches to manage visibility. +Use `!`-prefixed entries to extract everything except specific resources. For example: + +```yaml +apis: + - '*' + - '!legacy-api' +``` + +For large instances, it can still be easier to extract everything and use `.gitignore` or separate branches to manage visibility. ### Pattern-Based Team Filtering diff --git a/src/cli/publish-command.ts b/src/cli/publish-command.ts index 61bfc165..469da08c 100644 --- a/src/cli/publish-command.ts +++ b/src/cli/publish-command.ts @@ -3,7 +3,7 @@ /** * Publish command CLI registration * Commander subcommand with --resource-group, --service-name, --source, - * --overrides, --dry-run, --delete-unmatched flags. + * --overrides, --filter, --no-transitive, --dry-run, --delete-unmatched flags. * Includes --format json: machine-readable JSON output mode. */ @@ -11,7 +11,7 @@ import { Command } from 'commander'; import { PublishConfig } from '../models/config.js'; import { ApimServiceContext } from '../models/types.js'; import { runPublish, PublishResult } from '../services/publish-service.js'; -import { loadOverrideConfig } from '../lib/config-loader.js'; +import { loadFilterConfig, loadOverrideConfig } from '../lib/config-loader.js'; import { logger, parseLogLevel } from '../lib/logger.js'; import { ApimClient } from '../clients/apim-client.js'; import { ArtifactStore } from '../clients/artifact-store.js'; @@ -25,6 +25,8 @@ interface PublishOptions { serviceName: string; source: string; overrides?: string; + filter?: string; + transitive: boolean; commitId?: string; dryRun: boolean; deleteUnmatched: boolean; @@ -40,6 +42,8 @@ export function createPublishCommand(): Command { .requiredOption('--service-name ', 'APIM service instance name') .option('--source ', 'Source directory with artifacts', './apim-artifacts') .option('--overrides ', 'Override configuration YAML file') + .option('--filter ', 'Filter configuration YAML file') + .option('--no-transitive', 'Disable transitive dependency inclusion') .option( '--commit-id ', 'Git commit SHA for incremental publish (overrides COMMIT_ID env var)' @@ -121,15 +125,24 @@ async function executePublish( } } + let filterConfig; + if (options.filter) { + filterConfig = await loadFilterConfig(options.filter); + if (!filterConfig) { + logger.error(`Filter file not found: ${options.filter}`); + process.exit(2); + } + } + // Resolve commit ID for incremental publish const commitId = options.commitId ?? process.env.COMMIT_ID; if (commitId) { logger.debug(`Using incremental publish with commit ID: ${commitId}`); } - if (hasMutuallyExclusivePublishOptions(options.deleteUnmatched, commitId)) { + if (hasMutuallyExclusivePublishOptions(options.deleteUnmatched, commitId, Boolean(options.filter))) { logger.error( - 'Options --commit-id (or COMMIT_ID) and --delete-unmatched are mutually exclusive.' + 'Option --delete-unmatched cannot be combined with --filter or --commit-id (or COMMIT_ID).' ); process.exit(2); } @@ -138,6 +151,8 @@ async function executePublish( const publishConfig: PublishConfig = { service: context, sourceDir: options.source, + filter: filterConfig, + includeTransitive: options.transitive, overrides: overrideConfig, dryRun: options.dryRun, deleteUnmatched: options.deleteUnmatched, @@ -167,9 +182,10 @@ async function executePublish( */ export function hasMutuallyExclusivePublishOptions( deleteUnmatched: boolean, - commitId?: string + commitId?: string, + hasFilter = false ): boolean { - return deleteUnmatched && Boolean(commitId); + return deleteUnmatched && (Boolean(commitId) || hasFilter); } /** diff --git a/src/lib/resource-path.ts b/src/lib/resource-path.ts index 0dc9835d..b8032224 100644 --- a/src/lib/resource-path.ts +++ b/src/lib/resource-path.ts @@ -143,6 +143,23 @@ export function getNamePart(nameParts: string[], index: number): string { return value; } +/** + * Create a case-insensitive identity key for a resource descriptor. + */ +export function getResourceDescriptorKey(descriptor: ResourceDescriptor): string { + return `${descriptor.type}:${descriptor.workspace ?? ''}:${descriptor.nameParts.join('/')}`.toLowerCase(); +} + +/** + * Compare resource descriptors using APIM's case-insensitive names. + */ +export function sameResourceDescriptor( + left: ResourceDescriptor, + right: ResourceDescriptor +): boolean { + return getResourceDescriptorKey(left) === getResourceDescriptorKey(right); +} + /** * Converts a positional template string to a capturing regex. * Each `{i}` placeholder becomes a `([^/]+)` capture group; all other diff --git a/src/models/config.ts b/src/models/config.ts index d16f9243..644b24b1 100644 --- a/src/models/config.ts +++ b/src/models/config.ts @@ -75,6 +75,8 @@ export interface FilterConfig { export interface PublishConfig { service: ApimServiceContext; sourceDir: string; + filter?: FilterConfig; + includeTransitive?: boolean; overrides?: OverrideConfig; dryRun: boolean; deleteUnmatched: boolean; diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index cbb9c248..0720604d 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -21,8 +21,13 @@ import { import { runParallel } from '../lib/parallel-runner.js'; import { applyOverrides } from './override-merger.js'; import { logger } from '../lib/logger.js'; -import { getNamePart, getPublishTier } from '../lib/resource-path.js'; +import { + getNamePart, + getPublishTier, + getResourceDescriptorKey, +} from '../lib/resource-path.js'; import { isAutoGeneratedId } from '../lib/auto-generated.js'; +import { resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; /** * API child resource types that should be published after the API itself @@ -51,11 +56,20 @@ export async function publishApi( store: IArtifactStore, context: ApimServiceContext, descriptor: ResourceDescriptor, - config: PublishConfig + config: PublishConfig, + allowedDescriptors?: ResourceDescriptor[] ): Promise { try { + const includeSpecification = await shouldImportSpecification( + store, + descriptor, + config, + allowedDescriptors + ); // Step 1: Publish root API (with spec import if available) - const rootResult = await publishRootApi(client, store, context, descriptor, config); + const rootResult = await publishRootApi(client, store, context, descriptor, config, { + includeSpecification, + }); if (rootResult.status !== 'success') { return rootResult; } @@ -70,7 +84,14 @@ export async function publishApi( } // Step 2: Find and publish revisions in numeric order - const publishedRevisionCount = await publishApiRevisions(client, store, context, descriptor, config); + const publishedRevisionCount = await publishApiRevisions( + client, + store, + context, + descriptor, + config, + allowedDescriptors + ); // Step 2b: Align root API only when source marks it as current. // Source of truth is properties.isCurrent in root apiInformation.json. @@ -89,7 +110,15 @@ export async function publishApi( // Step 3: Publish child resources in parallel // When a spec was imported, operations and schemas are auto-created by APIM - await publishApiChildren(client, store, context, descriptor, config, rootResult.specImported); + await publishApiChildren( + client, + store, + context, + descriptor, + config, + rootResult.specImported, + allowedDescriptors + ); return { descriptor, @@ -106,6 +135,43 @@ export async function publishApi( } } +async function shouldImportSpecification( + store: IArtifactStore, + apiDescriptor: ResourceDescriptor, + config: PublishConfig, + allowedDescriptors?: ResourceDescriptor[] +): Promise { + if (!allowedDescriptors) { + return true; + } + + const apiName = getNamePart(apiDescriptor.nameParts, 0); + const managedChildren = (await store.listResources(config.sourceDir)).filter( + (descriptor) => + SPEC_MANAGED_CHILD_TYPES.has(descriptor.type) && + getNamePart(descriptor.nameParts, 0) === apiName && + descriptor.workspace === apiDescriptor.workspace + ); + const effectiveFilter = apiDescriptor.workspace + ? resolveWorkspaceFilter(apiDescriptor.workspace, config.filter) + : config.filter; + if (managedChildren.length === 0) { + const subFilter = Object.entries(effectiveFilter?.apiSubFilters ?? {}).find( + ([name]) => name.toLowerCase() === apiName.toLowerCase() + )?.[1]; + return subFilter?.operations === undefined && subFilter?.schemas === undefined; + } + + if (!config.filter) { + const allowed = new Set(allowedDescriptors.map(getResourceDescriptorKey)); + return managedChildren.every((descriptor) => allowed.has(getResourceDescriptorKey(descriptor))); + } + + return managedChildren.every((descriptor) => + shouldIncludeResource(descriptor, effectiveFilter) + ); +} + /** * Maps spec file format to APIM ContentFormat for inline import. * @param specDialect - The spec dialect of a JSON spec. Swagger 2.0 ('swagger2') @@ -275,16 +341,32 @@ async function publishApiRevisions( store: IArtifactStore, context: ApimServiceContext, apiDescriptor: ResourceDescriptor, - config: PublishConfig + config: PublishConfig, + allowedDescriptors?: ResourceDescriptor[] ): Promise { // List all resources from store const allDescriptors = await store.listResources(config.sourceDir); + const allowed = allowedDescriptors + ? new Set(allowedDescriptors.map(getResourceDescriptorKey)) + : undefined; + const effectiveFilter = apiDescriptor.workspace + ? resolveWorkspaceFilter(apiDescriptor.workspace, config.filter) + : config.filter; + const rootName = getNamePart(apiDescriptor.nameParts, 0).toLowerCase(); // Find revision descriptors for this API const revisionDescriptors = allDescriptors.filter( (d) => d.type === ResourceType.Api && - getNamePart(d.nameParts, 0).startsWith(`${getNamePart(apiDescriptor.nameParts, 0)};rev=`) + d.workspace === apiDescriptor.workspace && + getNamePart(d.nameParts, 0).toLowerCase().startsWith(`${rootName};rev=`) && + ( + !allowed || + allowed.has(getResourceDescriptorKey(d)) || + // Incremental mode only diffs changed files; fall back to the filter + // so an unchanged revision still republishes with its root API. + (config.commitId !== undefined && shouldIncludeResource(d, effectiveFilter)) + ) ); // Sort revisions by revision number @@ -343,19 +425,35 @@ async function publishApiChildren( context: ApimServiceContext, apiDescriptor: ResourceDescriptor, config: PublishConfig, - specImported: boolean = false + specImported: boolean = false, + allowedDescriptors?: ResourceDescriptor[] ): Promise { // List all resources from store const allDescriptors = await store.listResources(config.sourceDir); + const effectiveFilter = apiDescriptor.workspace + ? resolveWorkspaceFilter(apiDescriptor.workspace, config.filter) + : config.filter; // Find child descriptors for this API let childDescriptors = allDescriptors.filter( (d) => API_CHILD_TYPES.includes(d.type) && - getNamePart(d.nameParts, 0) === getNamePart(apiDescriptor.nameParts, 0) && + getNamePart(d.nameParts, 0).toLowerCase() === getNamePart(apiDescriptor.nameParts, 0).toLowerCase() && + d.workspace === apiDescriptor.workspace && !(specImported && SPEC_MANAGED_CHILD_TYPES.has(d.type)) ); + if (allowedDescriptors) { + const allowed = new Set(allowedDescriptors.map(getResourceDescriptorKey)); + childDescriptors = childDescriptors.filter( + (descriptor) => + allowed.has(getResourceDescriptorKey(descriptor)) || + // Incremental mode only diffs changed files; fall back to the filter + // so an unchanged child still republishes with its parent API. + (config.commitId !== undefined && shouldIncludeResource(descriptor, effectiveFilter)) + ); + } + if (specImported) { // Re-include explicitly named schemas (non-auto-generated IDs). // Auto-generated schemas have 24-char hex names and are recreated by spec import. @@ -363,15 +461,27 @@ async function publishApiChildren( const explicitSchemas = allDescriptors.filter( (d) => d.type === ResourceType.ApiSchema && - getNamePart(d.nameParts, 0) === getNamePart(apiDescriptor.nameParts, 0) && + getNamePart(d.nameParts, 0).toLowerCase() === getNamePart(apiDescriptor.nameParts, 0).toLowerCase() && + d.workspace === apiDescriptor.workspace && !isAutoGeneratedId(getNamePart(d.nameParts, 1)) ); - if (explicitSchemas.length > 0) { + const filteredExplicitSchemas = config.filter + ? explicitSchemas.filter((descriptor) => + shouldIncludeResource(descriptor, effectiveFilter) + ) + : allowedDescriptors + ? explicitSchemas.filter((d) => + allowedDescriptors.some( + (allowed) => getResourceDescriptorKey(allowed) === getResourceDescriptorKey(d) + ) + ) + : explicitSchemas; + if (filteredExplicitSchemas.length > 0) { logger.debug( - `Re-publishing ${explicitSchemas.length} explicit schema(s) after spec import for "${getNamePart(apiDescriptor.nameParts, 0)}"` + `Re-publishing ${filteredExplicitSchemas.length} explicit schema(s) after spec import for "${getNamePart(apiDescriptor.nameParts, 0)}"` ); - childDescriptors = [...childDescriptors, ...explicitSchemas]; + childDescriptors = [...childDescriptors, ...filteredExplicitSchemas]; } } @@ -404,7 +514,15 @@ async function publishApiChildren( // regardless of what the importer defaulted. PATCH is idempotent and only updates // the fields present in the persisted JSON. if (specImported) { - await reconcileOperationsAfterSpecImport(client, store, context, apiDescriptor, config, allDescriptors); + await reconcileOperationsAfterSpecImport( + client, + store, + context, + apiDescriptor, + config, + allDescriptors, + allowedDescriptors + ); } } @@ -423,14 +541,29 @@ async function reconcileOperationsAfterSpecImport( context: ApimServiceContext, apiDescriptor: ResourceDescriptor, config: PublishConfig, - allDescriptors: ResourceDescriptor[] + allDescriptors: ResourceDescriptor[], + allowedDescriptors?: ResourceDescriptor[] ): Promise { - const operationDescriptors = allDescriptors.filter( + let operationDescriptors = allDescriptors.filter( (d) => d.type === ResourceType.ApiOperation && - getNamePart(d.nameParts, 0) === getNamePart(apiDescriptor.nameParts, 0) && + getNamePart(d.nameParts, 0).toLowerCase() === getNamePart(apiDescriptor.nameParts, 0).toLowerCase() && + d.workspace === apiDescriptor.workspace && !isAutoGeneratedId(getNamePart(d.nameParts, 1)) ); + if (config.filter) { + const effectiveFilter = apiDescriptor.workspace + ? resolveWorkspaceFilter(apiDescriptor.workspace, config.filter) + : config.filter; + operationDescriptors = operationDescriptors.filter((descriptor) => + shouldIncludeResource(descriptor, effectiveFilter) + ); + } else if (allowedDescriptors) { + const allowed = new Set(allowedDescriptors.map(getResourceDescriptorKey)); + operationDescriptors = operationDescriptors.filter((d) => + allowed.has(getResourceDescriptorKey(d)) + ); + } if (operationDescriptors.length === 0) return; diff --git a/src/services/extract-service.ts b/src/services/extract-service.ts index e48eb20a..543d98c4 100644 --- a/src/services/extract-service.ts +++ b/src/services/extract-service.ts @@ -26,9 +26,7 @@ import { isSingletonType, isChildType } from '../lib/resource-path.js'; import { extractApiResources, ApiExtractionResult } from './api-extractor.js'; import { extractProductResources, ProductExtractionResult } from './product-extractor.js'; import { extractWorkspaces, WorkspaceExtractionResult } from './workspace-extractor.js'; -import { - findTransitiveDependencies, -} from './transitive-resolver.js'; +import { extractTransitiveDependencies } from './transitive-extractor.js'; import { redactAndWarnPolicySecrets } from './secret-redactor.js'; import { shouldIncludeResource } from './filter-service.js'; import { logger } from '../lib/logger.js'; @@ -132,7 +130,15 @@ export async function runExtraction( } // Phase 7: Extract workspace-scoped resources - await extractWorkspaceResources(client, store, service, outputDir, filter, result); + await extractWorkspaceResources( + client, + store, + service, + outputDir, + filter, + config.includeTransitive, + result + ); // Compute exit code if (result.totalErrors > 0 && result.totalExtracted > 0) { @@ -463,14 +469,6 @@ async function extractGatewayAssociations( } } -/** - * Result of extracting a single transitive dependency. - */ -interface TransitiveTaskResult { - dep: ResourceDescriptor; - success: boolean; -} - /** * Resolve transitive dependencies and extract any additional resources. * Collects results per-task and merges after all tasks complete to avoid @@ -499,61 +497,27 @@ async function resolveAndExtractTransitive( } } - // Find transitive dependencies - const transitiveDeps = findTransitiveDependencies( - result.collectedPolicies, - apiJsonMap + const resources = result.typeResults.flatMap((typeResult) => + typeResult.extracted + .filter((extracted) => extracted.status === 'success') + .map((extracted) => ({ + descriptor: extracted.descriptor, + json: extracted.json, + })) ); - - // Filter out already-extracted resources - // Use buildResourceLabel for the key — it handles singleton types (e.g. - // ServicePolicy) whose nameParts are empty, avoiding a getNamePart crash. - const alreadyExtracted = new Set( - result.extractedDescriptors.map( - (d) => `${d.type}:${buildResourceLabel(d).toLowerCase()}` - ) - ); - - const newDeps = transitiveDeps.filter( - (dep) => !alreadyExtracted.has(`${dep.type}:${buildResourceLabel(dep).toLowerCase()}`) + const transitiveResult = await extractTransitiveDependencies( + client, + store, + context, + outputDir, + result.collectedPolicies, + apiJsonMap, + resources, + result.extractedDescriptors ); - - if (newDeps.length === 0) { - logger.debug('No additional transitive dependencies found'); - return; - } - - logger.info(`Found ${newDeps.length} transitive dependencies to extract`); - - // Extract each transitive dependency - const tasks = newDeps.map((dep) => async (): Promise => { - try { - const json = await client.getResource(context, dep); - if (json) { - await store.writeResource(outputDir, dep, json); - logger.info(`Extracted transitive dependency ${buildResourceLabel(dep)}`); - return { dep, success: true }; - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - logger.warn(`Failed to extract transitive dependency ${buildResourceLabel(dep)}: ${errorMessage}`); - } - return { dep, success: false }; - }); - - const taskResults = await runParallel(tasks, DEFAULT_CONCURRENCY); - - // Merge results sequentially after parallel execution completes - for (const taskResult of taskResults) { - if (taskResult.status === 'fulfilled' && taskResult.value) { - if (taskResult.value.success) { - result.totalExtracted++; - result.extractedDescriptors.push(taskResult.value.dep); - } else { - result.totalErrors++; - } - } - } + result.totalExtracted += transitiveResult.extractedDescriptors.length; + result.totalErrors += transitiveResult.errorCount; + result.extractedDescriptors.push(...transitiveResult.extractedDescriptors); } /** @@ -565,10 +529,11 @@ async function extractWorkspaceResources( context: ApimServiceContext, outputDir: string, filter: FilterConfig | undefined, + includeTransitive: boolean, result: ExtractionResult ): Promise { const wsResults = await extractWorkspaces( - client, store, context, outputDir, filter + client, store, context, outputDir, filter, includeTransitive ); result.workspaceResults = wsResults; diff --git a/src/services/filter-service.ts b/src/services/filter-service.ts index e3a82962..98e59807 100644 --- a/src/services/filter-service.ts +++ b/src/services/filter-service.ts @@ -6,7 +6,7 @@ * case-insensitive matching, API root-name matching for revisions. */ -import { FilterConfig, ApiSubFilter } from '../models/config.js'; +import { FilterConfig, ApiSubFilter, WorkspaceSubFilter } from '../models/config.js'; import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; import { ResourceDescriptor } from '../models/types.js'; import { logger } from '../lib/logger.js'; @@ -123,6 +123,40 @@ export function shouldIncludeResource( return true; } +/** + * Resolve the resource filter configured for a workspace. + * Returns undefined when the workspace has no nested filter. + */ +export function resolveWorkspaceFilter( + workspaceName: string, + filter?: FilterConfig +): FilterConfig | undefined { + const lowerName = workspaceName.toLowerCase(); + const matchingKey = Object.keys(filter?.workspaceSubFilters ?? {}).find( + (key) => key.toLowerCase() === lowerName + ); + const subFilter = matchingKey ? filter?.workspaceSubFilters?.[matchingKey] : undefined; + return subFilter ? workspaceSubFilterToFilterConfig(subFilter) : undefined; +} + +function workspaceSubFilterToFilterConfig(sub: WorkspaceSubFilter): FilterConfig { + return { + apis: sub.apis, + apiSubFilters: sub.apiSubFilters, + backends: sub.backends, + diagnostics: sub.diagnostics, + groups: sub.groups, + loggers: sub.loggers, + namedValues: sub.namedValues, + policyFragments: sub.policyFragments, + products: sub.products, + schemas: sub.schemas, + subscriptions: sub.subscriptions, + tags: sub.tags, + versionSets: sub.versionSets, + }; +} + /** * Get the fixed singleton name for a resource type from its ARM path. * E.g., ServicePolicy → "policy" @@ -190,6 +224,19 @@ function getParentNameForFilter(descriptor: ResourceDescriptor): string | undefi : parentName; } +/** + * Whether the filter's field for this resource type is a defined array + * (user explicitly scoped it down), as opposed to simply absent. + */ +export function hasExplicitTypeFilter(type: ResourceType, filter?: FilterConfig): boolean { + if (!filter) { + return false; + } + + const field = FILTER_FIELD_MAP[type]; + return field !== undefined && filter[field] !== undefined; +} + /** * Extract root API name from a potentially revision-qualified name. * E.g., "my-api;rev=2" → "my-api" diff --git a/src/services/product-publisher.ts b/src/services/product-publisher.ts index 2720b0cb..3cbed42f 100644 --- a/src/services/product-publisher.ts +++ b/src/services/product-publisher.ts @@ -12,10 +12,11 @@ import type { PublishConfig } from '../models/config.js'; import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; import { publishResource, type ResourcePublishResult } from './resource-publisher.js'; import { logger } from '../lib/logger.js'; -import { getNamePart } from '../lib/resource-path.js'; +import { getNamePart, sameResourceDescriptor } from '../lib/resource-path.js'; import { parseArmUri } from '../lib/resource-uri.js'; import { isWorkspaceScope, buildLinkPayload } from '../lib/workspace-link.js'; import { isLinkAlreadyExistsError } from '../clients/apim-client.js'; +import { hasExplicitTypeFilter, resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; /** * Publish a Product with all its associations (APIs, Groups, Tags). @@ -26,7 +27,8 @@ export async function publishProduct( store: IArtifactStore, context: ApimServiceContext, descriptor: ResourceDescriptor, - config: PublishConfig + config: PublishConfig, + allowedDescriptors?: ResourceDescriptor[] ): Promise { try { const productName = getNamePart(descriptor.nameParts, 0); @@ -50,7 +52,8 @@ export async function publishProduct( descriptor, config, 'apis', - ResourceType.ProductApi + ResourceType.ProductApi, + allowedDescriptors ); // Step 3: Publish ProductGroup associations @@ -61,12 +64,13 @@ export async function publishProduct( descriptor, config, 'groups', - ResourceType.ProductGroup + ResourceType.ProductGroup, + allowedDescriptors ); // Step 4: Publish ProductTag associations // Tags are stored in the product directory, need to check for tags - await publishProductTags(client, store, context, descriptor, config); + await publishProductTags(client, store, context, descriptor, config, allowedDescriptors); // Step 5: Publish ProductPolicy if exists const policyDescriptor: ResourceDescriptor = { @@ -75,7 +79,7 @@ export async function publishProduct( workspace: descriptor.workspace, }; const policyContent = await store.readContent(config.sourceDir, policyDescriptor, 'policy'); - if (policyContent) { + if (policyContent && isDescriptorAllowed(policyDescriptor, config, allowedDescriptors)) { await publishResource(client, store, context, policyDescriptor, config); logger.debug(`Published policy for product: ${productName}`); } @@ -164,7 +168,8 @@ async function publishProductAssociations( productDescriptor: ResourceDescriptor, config: PublishConfig, associationType: 'apis' | 'groups', - resourceType: ResourceType + resourceType: ResourceType, + allowedDescriptors?: ResourceDescriptor[] ): Promise { const productName = getNamePart(productDescriptor.nameParts, 0); @@ -194,6 +199,17 @@ async function publishProductAssociations( nameParts: [productName, name], workspace: productDescriptor.workspace, }; + const targetDescriptor: ResourceDescriptor = { + type: associationType === 'apis' ? ResourceType.Api : ResourceType.Group, + nameParts: [name], + workspace: entry.scope === 'service' ? undefined : productDescriptor.workspace, + }; + if (!isAssociationAllowed(assocDescriptor, targetDescriptor, config, allowedDescriptors)) { + logger.warn( + `Skipping ${resourceType} association "${assocDescriptor.nameParts.join('/')}" because its target is excluded by the filter or unavailable in the publish set` + ); + continue; + } try { // In workspace scope, PUT with link payload; otherwise empty body. @@ -226,7 +242,8 @@ async function publishProductTags( store: IArtifactStore, context: ApimServiceContext, productDescriptor: ResourceDescriptor, - config: PublishConfig + config: PublishConfig, + allowedDescriptors?: ResourceDescriptor[] ): Promise { const productName = getNamePart(productDescriptor.nameParts, 0); @@ -253,6 +270,17 @@ async function publishProductTags( nameParts: [productName, tagName], workspace: productDescriptor.workspace, }; + const targetDescriptor: ResourceDescriptor = { + type: ResourceType.Tag, + nameParts: [tagName], + workspace: productDescriptor.workspace, + }; + if (!isAssociationAllowed(tagDescriptor, targetDescriptor, config, allowedDescriptors)) { + logger.warn( + `Skipping ProductTag association "${tagDescriptor.nameParts.join('/')}" because its target is excluded by the filter or unavailable in the publish set` + ); + continue; + } try { // The workspace ProductTag link references the product (productId), which @@ -274,3 +302,59 @@ async function publishProductTags( logger.info(`Published ${tagEntries.length} tags for product: ${productName}`); } + +function isDescriptorAllowed( + descriptor: ResourceDescriptor, + config: PublishConfig, + allowedDescriptors?: ResourceDescriptor[] +): boolean { + if (!allowedDescriptors) { + return true; + } + + if (allowedDescriptors.some((allowed) => sameResourceDescriptor(allowed, descriptor))) { + return true; + } + + // Incremental mode only diffs changed files; an unchanged product policy + // can still need republishing when its product does. + return config.commitId !== undefined; +} + +function isAssociationAllowed( + association: ResourceDescriptor, + target: ResourceDescriptor, + config: PublishConfig, + allowedDescriptors?: ResourceDescriptor[] +): boolean { + if (!allowedDescriptors) { + return true; + } + + const effectiveFilter = target.workspace + ? resolveWorkspaceFilter(target.workspace, config.filter) + : config.filter; + if (effectiveFilter && !shouldIncludeResource(target, effectiveFilter)) { + return false; + } + + if (allowedDescriptors.some((allowed) => sameResourceDescriptor(allowed, association))) { + return true; + } + + if (allowedDescriptors.some((allowed) => sameResourceDescriptor(allowed, target))) { + return true; + } + + // Incremental targets contain only changed resources. An unchanged target + // can still exist in APIM and remain explicitly allowed by the filter. + if (config.commitId) { + return true; + } + + if (hasExplicitTypeFilter(target.type, effectiveFilter)) { + return false; + } + + return true; +} diff --git a/src/services/publish-service.ts b/src/services/publish-service.ts index ed9604c9..4139d601 100644 --- a/src/services/publish-service.ts +++ b/src/services/publish-service.ts @@ -8,7 +8,7 @@ import { IApimClient } from '../clients/iapim-client.js'; import { IArtifactStore } from '../clients/iartifact-store.js'; -import { PublishConfig, OverrideConfig } from '../models/config.js'; +import { FilterConfig, PublishConfig, OverrideConfig } from '../models/config.js'; import { ApimServiceContext, ResourceDescriptor } from '../models/types.js'; import { ResourceType } from '../models/resource-types.js'; import { getResourceTier } from '../lib/dependency-graph.js'; @@ -17,7 +17,17 @@ import { logger } from '../lib/logger.js'; import { isAutoGeneratedId } from '../lib/auto-generated.js'; import { EXIT_SUCCESS, EXIT_PARTIAL, EXIT_FATAL } from '../lib/exit-codes.js'; import { buildResourceLabel } from '../lib/resource-uri.js'; -import { getNamePart, isChildType, isTopLevelSingleton } from '../lib/resource-path.js'; +import { + getNamePart, + getResourceDescriptorKey, + isChildType, + isTopLevelSingleton, +} from '../lib/resource-path.js'; +import { + extractRootApiName, + resolveWorkspaceFilter, + shouldIncludeResource, +} from './filter-service.js'; // Import from other agents' files (will be created in parallel) import { publishResource, ResourcePublishResult } from './resource-publisher.js'; @@ -29,6 +39,7 @@ import { computeGitDiff } from './git-diff-service.js'; import { scanForRedactionMarkers } from './secret-redaction-guard.js'; import { hasNamedValueOverride } from './override-merger.js'; import { REDACTION_MARKER } from './secret-redactor.js'; +import { scanArtifactReferences } from './transitive-resolver.js'; /** * The APIM Backend properties.type value that identifies a pool backend. @@ -205,24 +216,146 @@ async function determinePublishTargets( store: IArtifactStore, config: PublishConfig ): Promise { + let targetDescriptors: ResourceDescriptor[]; + let deletedDescriptors: ResourceDescriptor[]; + if (config.commitId) { // Incremental mode: use git diff logger.debug( `Using incremental publish mode with commit ID: ${config.commitId}` ); const diffResult = await computeGitDiff(config.sourceDir, config.commitId); - return { - targetDescriptors: diffResult.changedDescriptors, - deletedDescriptors: diffResult.deletedDescriptors, - }; + targetDescriptors = diffResult.changedDescriptors; + deletedDescriptors = diffResult.deletedDescriptors; } else { // Full mode: publish all artifacts logger.debug('Using full publish mode (all artifacts)'); - return { - targetDescriptors: await store.listResources(config.sourceDir), - deletedDescriptors: [], - }; + targetDescriptors = await store.listResources(config.sourceDir); + deletedDescriptors = []; + } + + targetDescriptors = filterPublishDescriptors(targetDescriptors, config.filter); + deletedDescriptors = filterPublishDescriptors(deletedDescriptors, config.filter); + + if (config.filter && config.includeTransitive !== false) { + const availableDescriptors = await store.listResources(config.sourceDir); + targetDescriptors = await expandTransitivePublishTargets( + store, + config.sourceDir, + targetDescriptors, + availableDescriptors + ); + } + + return { targetDescriptors, deletedDescriptors }; +} + +async function expandTransitivePublishTargets( + store: IArtifactStore, + sourceDir: string, + initialDescriptors: ResourceDescriptor[], + availableDescriptors: ResourceDescriptor[] +): Promise { + const availableByKey = new Map( + availableDescriptors.map((descriptor) => [getResourceDescriptorKey(descriptor), descriptor]) + ); + const expanded = [...initialDescriptors]; + const included = new Set(expanded.map(getResourceDescriptorKey)); + const scanQueue = [...initialDescriptors]; + const scanQueued = new Set(scanQueue.map(getResourceDescriptorKey)); + + // A changed API or product info file can be the only diff entry even when + // its policy/association artifacts contain the references to resolve. + for (const descriptor of initialDescriptors) { + if (descriptor.type !== ResourceType.Api && descriptor.type !== ResourceType.Product) { + continue; + } + const parentName = descriptor.type === ResourceType.Api + ? extractRootApiName(descriptor.nameParts[0] ?? '').toLowerCase() + : (descriptor.nameParts[0] ?? '').toLowerCase(); + for (const candidate of availableDescriptors) { + const isSameResourceType = candidate.type === descriptor.type; + const isOwnedChild = getParentType(candidate.type) === descriptor.type; + if ( + (!isSameResourceType && !isOwnedChild) || + candidate.workspace !== descriptor.workspace || + candidate.nameParts.length < 1 + ) { + continue; + } + const candidateName = candidate.type === ResourceType.Api + ? extractRootApiName(candidate.nameParts[0] ?? '').toLowerCase() + : (candidate.nameParts[0] ?? '').toLowerCase(); + if (candidateName === parentName && !scanQueued.has(getResourceDescriptorKey(candidate))) { + scanQueue.push(candidate); + scanQueued.add(getResourceDescriptorKey(candidate)); + } + } + } + + for (let index = 0; index < scanQueue.length; index++) { + let references: ResourceDescriptor[]; + try { + references = await scanArtifactReferences(store, sourceDir, scanQueue[index]); + } catch (error) { + logger.warn( + `Unable to scan transitive references for ${scanQueue[index].type} "${scanQueue[index].nameParts.join('/')}": ${String(error)}` + ); + continue; + } + for (const reference of references) { + const available = availableByKey.get(getResourceDescriptorKey(reference)); + if (available && !included.has(getResourceDescriptorKey(available))) { + included.add(getResourceDescriptorKey(available)); + expanded.push(available); + scanQueue.push(available); + } + } + } + + if (expanded.length > initialDescriptors.length) { + logger.debug( + `Transitive publish resolution added ${expanded.length - initialDescriptors.length} resources` + ); + } + return expanded; +} + +function filterPublishDescriptors( + descriptors: ResourceDescriptor[], + filter?: FilterConfig +): ResourceDescriptor[] { + if (!filter) { + return descriptors; + } + + return descriptors.filter((descriptor) => { + if (descriptor.type === ResourceType.Workspace) { + return matchesWorkspaceName(descriptor.nameParts[0] ?? '', filter.workspaces); + } + + if (!descriptor.workspace) { + return shouldIncludeResource(descriptor, filter); + } + + if (!matchesWorkspaceName(descriptor.workspace, filter.workspaces)) { + return false; + } + + const workspaceFilter = resolveWorkspaceFilter(descriptor.workspace, filter); + return shouldIncludeResource(descriptor, workspaceFilter ?? undefined); + }); +} + +function matchesWorkspaceName(name: string, allowlist: string[] | undefined): boolean { + if (allowlist === undefined) { + return true; } + + return shouldIncludeResource( + { type: ResourceType.Workspace, nameParts: [name] }, + { workspaces: allowlist } + ); } /** @@ -263,7 +396,7 @@ async function executePuts( if (!parentNamesByType.has(d.type)) { parentNamesByType.set(d.type, new Set()); } - parentNamesByType.get(d.type)!.add(getNamePart(d.nameParts, 0)); + parentNamesByType.get(d.type)!.add(parentScopeKey(d)); } } @@ -279,7 +412,7 @@ async function executePuts( if (workspaces.length > 0) { logger.debug(`Publishing ${workspaces.length} workspace container(s) first (wave 0 of tier 1)`); - await publishAndOutput(client, store, context, config, workspaces, results); + await publishAndOutput(client, store, context, config, workspaces, targetDescriptors, results); } // Within Tier 1 we publish in three ordered waves to satisfy implicit @@ -302,7 +435,7 @@ async function executePuts( if (namedValues.length > 0) { logger.debug(`Publishing ${namedValues.length} named value(s) first (wave 1 of tier 1)`); - await publishAndOutput(client, store, context, config, namedValues, results); + await publishAndOutput(client, store, context, config, namedValues, targetDescriptors, results); } const { poolBackends, regularTier1 } = await splitPoolBackends( @@ -310,7 +443,7 @@ async function executePuts( config.sourceDir, otherTier1 ); - await publishAndOutput(client, store, context, config, regularTier1, results); + await publishAndOutput(client, store, context, config, regularTier1, targetDescriptors, results); if (poolBackends.length > 0) { logger.debug( `Publishing ${poolBackends.length} pool backend(s) after regular backends` @@ -321,6 +454,7 @@ async function executePuts( context, config, poolBackends, + targetDescriptors, results ); } @@ -333,11 +467,11 @@ async function executePuts( tier2Descriptors ); - await publishAndOutput(client, store, context, config, regularTier2, results); + await publishAndOutput(client, store, context, config, regularTier2, targetDescriptors, results); if (mcpApis.length > 0) { logger.debug(`Publishing ${mcpApis.length} MCP API resource(s) after regular tier 2 resources`); - await publishAndOutput(client, store, context, config, mcpApis, results); + await publishAndOutput(client, store, context, config, mcpApis, targetDescriptors, results); } } else { // For tiers 3/4, exclude child resources whose parent is being published @@ -352,27 +486,67 @@ async function executePuts( if (!isChildType(d.type)) return true; // Child resources have the parent name as nameParts[0] - const owningParent = getNamePart(d.nameParts, 0); + const parentType = getParentType(d.type); + if (!parentType) return true; - // Skip if this parent name is being published in tier 2 - for (const parentNames of parentNamesByType.values()) { - if (parentNames.has(owningParent)) return false; - } - return true; + return !parentNamesByType.get(parentType)?.has(parentScopeKey(d)); }); const skipped = countBefore - tierDescriptors.length; if (skipped > 0) { logger.debug(`Skipping ${skipped} child resource(s) in tier ${tier} (handled by parent publisher)`); } + } - await publishAndOutput(client, store, context, config, tierDescriptors, results); + await publishAndOutput(client, store, context, config, tierDescriptors, targetDescriptors, results); } } return results; } +function parentScopeKey(descriptor: ResourceDescriptor): string { + return `${descriptor.workspace ?? ''}:${getNamePart(descriptor.nameParts, 0)}`.toLowerCase(); +} + +function getParentType(type: ResourceType): ResourceType | undefined { + if ( + [ + ResourceType.ApiPolicy, + ResourceType.ApiTag, + ResourceType.ApiDiagnostic, + ResourceType.ApiOperation, + ResourceType.ApiOperationPolicy, + ResourceType.ApiSchema, + ResourceType.ApiRelease, + ResourceType.ApiTagDescription, + ResourceType.ApiWiki, + ResourceType.GraphQLResolver, + ResourceType.GraphQLResolverPolicy, + ].includes(type) + ) { + return ResourceType.Api; + } + + if ( + [ + ResourceType.ProductPolicy, + ResourceType.ProductApi, + ResourceType.ProductGroup, + ResourceType.ProductTag, + ResourceType.ProductWiki, + ].includes(type) + ) { + return ResourceType.Product; + } + + if (type === ResourceType.GatewayApi) { + return ResourceType.Gateway; + } + + return undefined; +} + function splitWorkspaces( descriptors: ResourceDescriptor[] ): { workspaces: ResourceDescriptor[]; nonWorkspaceTier1: ResourceDescriptor[] } { @@ -399,10 +573,11 @@ async function publishAndOutput( context: ApimServiceContext, config: PublishConfig, descriptors: ResourceDescriptor[], + targetDescriptors: ResourceDescriptor[], results: PublishActionResult[] ): Promise { if (descriptors.length === 0) return; - const tierResults = await publishTier(client, store, context, config, descriptors); + const tierResults = await publishTier(client, store, context, config, descriptors, targetDescriptors); results.push(...tierResults); for (const result of tierResults) { outputActionStatus(result); @@ -513,7 +688,8 @@ async function publishTier( store: IArtifactStore, context: ApimServiceContext, config: PublishConfig, - descriptors: ResourceDescriptor[] + descriptors: ResourceDescriptor[], + allTargetDescriptors: ResourceDescriptor[] ): Promise { const tasks = descriptors.map((descriptor) => async () => { try { @@ -525,9 +701,18 @@ async function publishTier( // Use specialized publishers for Api and Product types if (descriptor.type === ResourceType.Api && !isApiRevision) { - publishResult = await publishApi(client, store, context, descriptor, config); + publishResult = await publishApi( + client, + store, + context, + descriptor, + config, + config.filter ? allTargetDescriptors : undefined + ); } else if (descriptor.type === ResourceType.Product) { - publishResult = await publishProduct(client, store, context, descriptor, config); + publishResult = config.filter + ? await publishProduct(client, store, context, descriptor, config, allTargetDescriptors) + : await publishProduct(client, store, context, descriptor, config); } else { publishResult = await publishResource( client, @@ -577,7 +762,7 @@ async function publishTier( function filterApiRevisionsHandledByRootApis( descriptors: ResourceDescriptor[] ): ResourceDescriptor[] { - const rootApiNames = new Set(); + const rootApiKeys = new Set(); for (const descriptor of descriptors) { if (descriptor.type !== ResourceType.Api) { continue; @@ -585,11 +770,11 @@ function filterApiRevisionsHandledByRootApis( const apiName = getNamePart(descriptor.nameParts, 0); if (!isApiRevisionName(apiName)) { - rootApiNames.add(apiName); + rootApiKeys.add(apiScopeKey(descriptor.workspace, apiName)); } } - if (rootApiNames.size === 0) { + if (rootApiKeys.size === 0) { return descriptors; } @@ -603,10 +788,14 @@ function filterApiRevisionsHandledByRootApis( return true; } - return !rootApiNames.has(getApiRootName(apiName)); + return !rootApiKeys.has(apiScopeKey(descriptor.workspace, getApiRootName(apiName))); }); } +function apiScopeKey(workspace: string | undefined, apiName: string): string { + return `${workspace ?? ''}:${apiName}`.toLowerCase(); +} + function isApiRevisionName(apiName: string): boolean { return apiName.includes(';rev='); } diff --git a/src/services/resource-publisher.ts b/src/services/resource-publisher.ts index f41d0473..773f6961 100644 --- a/src/services/resource-publisher.ts +++ b/src/services/resource-publisher.ts @@ -22,6 +22,8 @@ import { REDACTION_MARKER } from './secret-redactor.js'; import { isLinkAlreadyExistsError } from '../clients/apim-client.js'; import type { OverrideConfig } from '../models/config.js'; import { buildResourceLabel } from '../lib/resource-uri.js'; +import { hasExplicitTypeFilter, resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; +import { findSubscriptionTargets } from './transitive-resolver.js'; export interface ResourcePublishResult { descriptor: ResourceDescriptor; @@ -95,6 +97,24 @@ export async function publishResource( ); } + if (descriptor.type === ResourceType.ApiTag) { + const targetDescriptor: ResourceDescriptor = { + type: ResourceType.Tag, + nameParts: [getNamePart(descriptor.nameParts, 1)], + workspace: descriptor.workspace, + }; + if (!(await isTargetAllowedByFilter(store, targetDescriptor, config))) { + logger.warn( + `Skipping ApiTag association "${descriptor.nameParts.join('/')}" because its target is excluded by the filter or was never extracted` + ); + return { + descriptor, + status: 'skipped', + action: 'noop', + }; + } + } + // Handle workspace ApiTag — uses link endpoint with link payload. // In service scope, ApiTag is a regular PUT with the tag JSON; in workspace // scope it becomes a link resource at `tags/{tag}/apiLinks/{api}`. @@ -223,6 +243,17 @@ export async function publishResource( }; } + if (!(await areSubscriptionTargetsAllowed(store, json, descriptor, config))) { + logger.warn( + `Skipping subscription "${subscriptionName}" because its API or Product target is excluded by the filter or was never extracted` + ); + return { + descriptor, + status: 'skipped', + action: 'noop', + }; + } + json = normalizeSubscriptionScope(json, context); } @@ -309,7 +340,23 @@ async function publishAssociation( const assocDescriptor: ResourceDescriptor = { type: descriptor.type, nameParts: [getNamePart(descriptor.nameParts, 0), entry.name], + workspace: descriptor.workspace, + }; + const targetDescriptor: ResourceDescriptor = { + type: descriptor.type === ResourceType.GatewayApi + ? ResourceType.Api + : descriptor.type === ResourceType.ProductApi + ? ResourceType.Api + : ResourceType.Group, + nameParts: [entry.name], + workspace: entry.scope === 'service' ? undefined : descriptor.workspace, }; + if (!(await isTargetAllowedByFilter(store, targetDescriptor, config))) { + logger.warn( + `Skipping ${descriptor.type} association "${assocDescriptor.nameParts.join('/')}" because its target is excluded by the filter or was never extracted` + ); + continue; + } try { // PUT empty body for association (APIM uses PUT to create association) await client.putResource(context, assocDescriptor, {}); @@ -336,6 +383,45 @@ async function publishAssociation( } } +async function areSubscriptionTargetsAllowed( + store: IArtifactStore, + json: Record, + descriptor: ResourceDescriptor, + config: PublishConfig +): Promise { + for (const target of findSubscriptionTargets(json, descriptor.workspace)) { + if (!(await isTargetAllowedByFilter(store, target, config))) { + return false; + } + } + return true; +} + +async function isTargetAllowedByFilter( + store: IArtifactStore, + target: ResourceDescriptor, + config: PublishConfig +): Promise { + if (!config.filter) { + return true; + } + + const effectiveFilter = target.workspace + ? resolveWorkspaceFilter(target.workspace, config.filter) + : config.filter; + if (effectiveFilter && !shouldIncludeResource(target, effectiveFilter)) { + return false; + } + + // Explicitly scoped types must actually exist among the artifacts, not + // just pass the filter — catches dangling references (mirrors publishProduct). + if (hasExplicitTypeFilter(target.type, effectiveFilter)) { + return (await store.readResource(config.sourceDir, target)) !== undefined; + } + + return true; +} + /** * Publish wiki resource (ApiWiki, ProductWiki) */ diff --git a/src/services/transitive-extractor.ts b/src/services/transitive-extractor.ts new file mode 100644 index 00000000..4717d95f --- /dev/null +++ b/src/services/transitive-extractor.ts @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import type { IApimClient } from '../clients/iapim-client.js'; +import type { IArtifactStore } from '../clients/iartifact-store.js'; +import type { ApimServiceContext, ResourceDescriptor } from '../models/types.js'; +import { getResourceDescriptorKey } from '../lib/resource-path.js'; +import { buildResourceLabel } from '../lib/resource-uri.js'; +import { logger } from '../lib/logger.js'; +import { runParallel } from '../lib/parallel-runner.js'; +import { redactSecrets } from './secret-redactor.js'; +import { findTransitiveDependencies } from './transitive-resolver.js'; + +const DEFAULT_CONCURRENCY = 5; + +export interface TransitiveResourceArtifact { + descriptor: ResourceDescriptor; + json: Record; +} + +export interface TransitiveExtractionResult { + extractedDescriptors: ResourceDescriptor[]; + errorCount: number; +} + +export async function extractTransitiveDependencies( + client: IApimClient, + store: IArtifactStore, + context: ApimServiceContext, + outputDir: string, + policies: Map, + apis: Map>, + resources: TransitiveResourceArtifact[], + alreadyExtracted: ResourceDescriptor[], + workspace?: string, + serviceContext?: ApimServiceContext +): Promise { + const attempted = new Set(alreadyExtracted.map(getResourceDescriptorKey)); + const extractedDescriptors: ResourceDescriptor[] = []; + let errorCount = 0; + let foundDependencies = false; + + while (true) { + const newDeps = findTransitiveDependencies( + policies, + apis, + workspace, + resources + ).filter((dep) => !attempted.has(getResourceDescriptorKey(dep))); + + if (newDeps.length === 0) { + if (!foundDependencies) { + logger.debug('No additional transitive dependencies found'); + } + return { extractedDescriptors, errorCount }; + } + + foundDependencies = true; + logger.info(`Found ${newDeps.length} transitive dependencies to extract`); + for (const dep of newDeps) { + attempted.add(getResourceDescriptorKey(dep)); + } + + const tasks = newDeps.map((dep) => async () => { + try { + const dependencyContext = + serviceContext && dep.workspace !== workspace ? serviceContext : context; + const json = await client.getResource(dependencyContext, dep); + if (json) { + const safeJson = redactSecrets(dep, json); + await store.writeResource(outputDir, dep, safeJson); + logger.info(`Extracted transitive dependency ${buildResourceLabel(dep)}`); + return { dep, json: safeJson }; + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.warn( + `Failed to extract transitive dependency ${buildResourceLabel(dep)}: ${message}` + ); + } + return { dep }; + }); + + const taskResults = await runParallel(tasks, DEFAULT_CONCURRENCY); + for (const taskResult of taskResults) { + const value = taskResult.status === 'fulfilled' ? taskResult.value : undefined; + if (!value?.json) { + errorCount++; + continue; + } + + extractedDescriptors.push(value.dep); + resources.push({ + descriptor: value.dep, + json: value.json, + }); + } + } +} diff --git a/src/services/transitive-resolver.ts b/src/services/transitive-resolver.ts index d0b4d4cc..51f961b8 100644 --- a/src/services/transitive-resolver.ts +++ b/src/services/transitive-resolver.ts @@ -8,9 +8,11 @@ */ import { FilterConfig } from '../models/config.js'; -import { ResourceType } from '../models/resource-types.js'; +import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; import { ResourceDescriptor } from '../models/types.js'; +import type { IArtifactStore } from '../clients/iartifact-store.js'; import { logger } from '../lib/logger.js'; +import { getResourceDescriptorKey } from '../lib/resource-path.js'; /** * Reference detection patterns for policy XML content. @@ -27,6 +29,14 @@ export interface TransitiveDependency { name: string; } +const POLICY_RESOURCE_TYPES = new Set([ + ResourceType.ServicePolicy, + ResourceType.ApiPolicy, + ResourceType.ApiOperationPolicy, + ResourceType.ProductPolicy, + ResourceType.GraphQLResolverPolicy, +]); + /** * Scan policy XML content for references to other resources. * @@ -87,10 +97,7 @@ export function scanApiVersionSetReference( return undefined; } - // Extract version set name from ARM resource ID - // Format: /subscriptions/.../apiVersionSets/{name} - const parts = versionSetId.split('/'); - const name = parts[parts.length - 1]; + const name = extractResourceNameFromId(versionSetId, 'apiVersionSets'); if (!name) { return undefined; } @@ -196,33 +203,177 @@ function addToFilter( */ export function findTransitiveDependencies( policies: Map, - apis: Map> + apis: Map>, + workspace?: string, + resources: ReadonlyArray<{ + descriptor: ResourceDescriptor; + json: Record; + }> = [] ): ResourceDescriptor[] { const dependencies: ResourceDescriptor[] = []; - const seen = new Set(); - // Scan all policies for (const [, policyXml] of policies) { for (const dep of scanPolicyReferences(policyXml)) { - const key = `${dep.type}:${dep.name.toLowerCase()}`; - if (!seen.has(key)) { - seen.add(key); - dependencies.push({ type: dep.type, nameParts: [dep.name] }); - } + dependencies.push({ type: dep.type, nameParts: [dep.name], workspace }); } } - // Scan API version set references for (const [, apiJson] of apis) { const dep = scanApiVersionSetReference(apiJson); if (dep) { - const key = `${dep.type}:${dep.name.toLowerCase()}`; - if (!seen.has(key)) { - seen.add(key); - dependencies.push({ type: dep.type, nameParts: [dep.name] }); + dependencies.push({ type: dep.type, nameParts: [dep.name], workspace }); + } + } + + for (const { descriptor, json } of resources) { + const properties = json.properties as Record | undefined; + + if (descriptor.type === ResourceType.Backend) { + const pool = isRecord(properties?.pool) ? properties.pool : undefined; + const services = pool?.services; + if (Array.isArray(services)) { + for (const service of services) { + if (isRecord(service) && typeof service.id === 'string') { + const name = extractResourceNameFromId(service.id, 'backends'); + if (name) { + dependencies.push({ + type: ResourceType.Backend, + nameParts: [name], + workspace: workspaceFromReference(service.id, descriptor.workspace), + }); + } + } + } + } + } + + if (descriptor.type === ResourceType.PolicyFragment) { + for (const value of [properties?.value, properties?.policyContent]) { + if (typeof value !== 'string') { + continue; + } + for (const dep of scanPolicyReferences(value)) { + dependencies.push({ + type: dep.type, + nameParts: [dep.name], + workspace: descriptor.workspace, + }); + } } } } - return dependencies; + return deduplicateDescriptors(dependencies); +} + +/** + * Read intrinsic dependencies from one on-disk artifact. + * + * Association and subscription targets are links to independently selected + * composite resources, not transitive dependencies. + */ +export async function scanArtifactReferences( + store: IArtifactStore, + sourceDir: string, + descriptor: ResourceDescriptor +): Promise { + const references: ResourceDescriptor[] = []; + const policies = new Map(); + const apis = new Map>(); + + if (POLICY_RESOURCE_TYPES.has(descriptor.type)) { + const content = await store.readContent(sourceDir, descriptor, 'policy'); + if (content) { + policies.set(descriptor.nameParts.join('/'), content.content); + } + } + + const infoFile = RESOURCE_TYPE_METADATA[descriptor.type]?.infoFile; + const json = POLICY_RESOURCE_TYPES.has(descriptor.type) || !infoFile?.endsWith('.json') + ? undefined + : await store.readResource(sourceDir, descriptor); + if (json) { + if (descriptor.type === ResourceType.Api) { + apis.set(descriptor.nameParts.join('/'), json); + } + + } + + references.push( + ...findTransitiveDependencies( + policies, + apis, + descriptor.workspace, + json ? [{ descriptor, json }] : [] + ) + ); + + return deduplicateDescriptors(references); +} + +/** + * Find API or Product targets referenced by a subscription payload. + * + * These targets are used to gate link publication; they must not be fed into + * transitive expansion because APIs and Products are composite resources. + */ +export function findSubscriptionTargets( + json: Record, + workspace?: string +): ResourceDescriptor[] { + const references: ResourceDescriptor[] = []; + const properties = json.properties as Record | undefined; + for (const value of [properties?.scope, properties?.apiId]) { + if (typeof value !== 'string') { + continue; + } + for (const [segment, type] of [ + ['apis', ResourceType.Api], + ['products', ResourceType.Product], + ] as const) { + const name = extractResourceNameFromId(value, segment); + if (name) { + references.push({ + type, + nameParts: [name], + workspace: workspaceFromReference(value, workspace), + }); + } + } + } + + return deduplicateDescriptors(references); +} + +function extractResourceNameFromId(value: string, segment: string): string | undefined { + const match = value.match(new RegExp(`(?:^|/)${segment}/([^/]+)(?:/|$)`, 'i')); + return match?.[1] ? decodeArmSegment(match[1]) : undefined; +} + +function workspaceFromReference(value: string, fallback?: string): string | undefined { + const match = value.match(/\/workspaces\/([^/]+)/i); + return match?.[1] ? decodeArmSegment(match[1]) : (value.startsWith('/') ? undefined : fallback); +} + +function decodeArmSegment(value: string): string { + try { + return decodeURIComponent(value); + } catch (error) { + logger.warn(`Unable to decode ARM resource ID segment; using the raw value: ${String(error)}`); + return value; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function deduplicateDescriptors(descriptors: ResourceDescriptor[]): ResourceDescriptor[] { + const seen = new Set(); + return descriptors.filter((descriptor) => { + const key = getResourceDescriptorKey(descriptor); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); } diff --git a/src/services/workspace-extractor.ts b/src/services/workspace-extractor.ts index b3aa3d9e..4118f75d 100644 --- a/src/services/workspace-extractor.ts +++ b/src/services/workspace-extractor.ts @@ -7,15 +7,25 @@ */ import { IApimClient } from '../clients/iapim-client.js'; import { IArtifactStore } from '../clients/iartifact-store.js'; -import { ApimServiceContext } from '../models/types.js'; +import { ApimServiceContext, ResourceDescriptor } from '../models/types.js'; import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; -import { FilterConfig, WorkspaceSubFilter } from '../models/config.js'; +import { FilterConfig } from '../models/config.js'; import { extractResourceType, ExtractedResource } from './resource-extractor.js'; import { extractApiResources, extractWorkspaceApiTags } from './api-extractor.js'; import { extractProductResources, extractWorkspaceProductTags } from './product-extractor.js'; import { logger } from '../lib/logger.js'; import { getNamePart } from '../lib/resource-path.js'; -import { isWildcardPattern, wildcardMatch } from './filter-service.js'; +import { + isWildcardPattern, + resolveWorkspaceFilter, + shouldIncludeResource, +} from './filter-service.js'; +import { + extractTransitiveDependencies, + type TransitiveResourceArtifact, +} from './transitive-extractor.js'; + +export { resolveWorkspaceFilter }; /** * Types that can exist at the workspace level, derived from RESOURCE_TYPE_METADATA. @@ -47,7 +57,8 @@ export async function extractWorkspaces( store: IArtifactStore, context: ApimServiceContext, outputDir: string, - filter?: FilterConfig + filter?: FilterConfig, + includeTransitive: boolean = false ): Promise { const results: WorkspaceExtractionResult[] = []; let workspaceNames: string[]; @@ -60,21 +71,23 @@ export async function extractWorkspaces( return results; } - const hasWildcards = filter.workspaces.some(isWildcardPattern); - if (hasWildcards) { - // Wildcard patterns require discovery so we can match against real names + const requiresDiscovery = filter.workspaces.some( + (entry) => isWildcardPattern(entry) || entry.startsWith('!') + ); + if (requiresDiscovery) { + // Wildcards and exclusions require discovery so shared filter semantics + // can be applied against actual workspace names. const discovered = await discoverWorkspaceNames(client, context); workspaceNames = discovered.filter((name) => - filter.workspaces!.some((pattern) => - isWildcardPattern(pattern) - ? wildcardMatch(pattern, name) - : pattern.toLowerCase() === name.toLowerCase() + shouldIncludeResource( + { type: ResourceType.Workspace, nameParts: [name] }, + { workspaces: filter.workspaces } ) ); // Warn about exact (non-wildcard) entries that didn't match any discovered workspace for (const entry of filter.workspaces) { - if (!isWildcardPattern(entry)) { + if (!entry.startsWith('!') && !isWildcardPattern(entry)) { const matched = discovered.some((d) => d.toLowerCase() === entry.toLowerCase()); if (!matched) { logger.warn(`Workspace filter entry "${entry}" did not match any discovered workspace`); @@ -111,7 +124,8 @@ export async function extractWorkspaces( const wsResult = await extractWorkspace( client, store, context, wsName, outputDir, - resolveWorkspaceFilter(wsName, filter) + resolveWorkspaceFilter(wsName, filter), + includeTransitive ); results.push(wsResult); } @@ -125,7 +139,8 @@ async function extractWorkspace( context: ApimServiceContext, workspaceName: string, outputDir: string, - filter?: FilterConfig + filter?: FilterConfig, + includeTransitive: boolean = false ): Promise { logger.info(`Extracting workspace "${workspaceName}"...`); @@ -143,6 +158,10 @@ async function extractWorkspace( let extractedTagNames: string[] = []; const extractedApiNames = new Set(); let extractedProducts: ExtractedResource[] = []; + const extractedDescriptors: ResourceDescriptor[] = []; + const resources: TransitiveResourceArtifact[] = []; + const policies = new Map(); + const apis = new Map>(); for (const type of WORKSPACE_SUPPORTED_TYPES) { try { @@ -152,6 +171,15 @@ async function extractWorkspace( ); resourceCount += result.extracted.filter((r) => r.status === 'success').length; errorCount += result.errorCount; + for (const extracted of result.extracted) { + if (extracted.status === 'success') { + extractedDescriptors.push(extracted.descriptor); + resources.push({ + descriptor: extracted.descriptor, + json: extracted.json, + }); + } + } // Track extracted tags for later ApiTag/ProductTag extraction if (type === ResourceType.Tag) { @@ -170,6 +198,14 @@ async function extractWorkspace( client, store, wsContext, api.descriptor, api.json, outputDir, filter, workspaceName ); + const apiName = getNamePart(api.descriptor.nameParts, 0); + apis.set(apiName, api.json); + for (let index = 0; index < apiResult.policies.length; index++) { + const policy = apiResult.policies[index]; + if (policy !== undefined) { + policies.set(`api:${apiName}:policy:${index}`, policy); + } + } resourceCount += apiResult.operations.length + apiResult.tags.length + apiResult.schemas.length; @@ -185,10 +221,17 @@ async function extractWorkspace( extractedProducts = result.extracted.filter((r) => r.status === 'success'); for (const product of extractedProducts) { try { - await extractProductResources( + const productResult = await extractProductResources( client, store, wsContext, product.descriptor, outputDir, filter, workspaceName ); + const productName = getNamePart(product.descriptor.nameParts, 0); + for (let index = 0; index < productResult.policies.length; index++) { + const policy = productResult.policies[index]; + if (policy !== undefined) { + policies.set(`product:${productName}:policy:${index}`, policy); + } + } resourceCount++; } catch (error) { logger.warn(`Failed to extract product details for workspace "${workspaceName}": ${(error as Error).message}`); @@ -233,6 +276,24 @@ async function extractWorkspace( } } + if (includeTransitive && filter) { + logger.info(`Resolving transitive dependencies for workspace "${workspaceName}"...`); + const transitiveResult = await extractTransitiveDependencies( + client, + store, + wsContext, + outputDir, + policies, + apis, + resources, + extractedDescriptors, + workspaceName, + context + ); + resourceCount += transitiveResult.extractedDescriptors.length; + errorCount += transitiveResult.errorCount; + } + logger.info(`Workspace "${workspaceName}": extracted ${resourceCount} resources, ${errorCount} errors`); return { workspaceName, resourceCount, errorCount }; @@ -254,52 +315,3 @@ async function discoverWorkspaceNames( } return names; } - -/** - * Resolve the effective FilterConfig for a workspace. - * If the workspace has a sub-filter in workspaceSubFilters, convert it to a FilterConfig. - * Otherwise return undefined (no filter = extract everything in the workspace). - */ -export function resolveWorkspaceFilter( - workspaceName: string, - filter?: FilterConfig -): FilterConfig | undefined { - if (!filter?.workspaceSubFilters) { - return undefined; - } - - // Case-insensitive lookup of workspace sub-filter - const lowerName = workspaceName.toLowerCase(); - const matchingKey = Object.keys(filter.workspaceSubFilters).find( - (k) => k.toLowerCase() === lowerName - ); - - if (!matchingKey) { - return undefined; - } - - const sub = filter.workspaceSubFilters[matchingKey]; - return workspaceSubFilterToFilterConfig(sub); -} - -/** - * Convert a WorkspaceSubFilter to a FilterConfig so the standard - * filter-service matching logic can be applied to workspace-scoped resources. - */ -function workspaceSubFilterToFilterConfig(sub: WorkspaceSubFilter): FilterConfig { - return { - apis: sub.apis, - apiSubFilters: sub.apiSubFilters, - backends: sub.backends, - diagnostics: sub.diagnostics, - groups: sub.groups, - loggers: sub.loggers, - namedValues: sub.namedValues, - policyFragments: sub.policyFragments, - products: sub.products, - schemas: sub.schemas, - subscriptions: sub.subscriptions, - tags: sub.tags, - versionSets: sub.versionSets, - }; -} diff --git a/tests/unit/cli/publish-command.test.ts b/tests/unit/cli/publish-command.test.ts index ff7c1346..abf2ad02 100644 --- a/tests/unit/cli/publish-command.test.ts +++ b/tests/unit/cli/publish-command.test.ts @@ -48,6 +48,18 @@ describe('publish-command', () => { expect(overridesOpt).toBeDefined(); }); + it('should have --filter option', () => { + const cmd = createPublishCommand(); + expect(cmd.options.find((o) => o.long === '--filter')).toBeDefined(); + }); + + it('should have a negated --no-transitive flag', () => { + const cmd = createPublishCommand(); + const transitiveOpt = cmd.options.find((o) => o.long === '--no-transitive'); + expect(transitiveOpt).toBeDefined(); + expect(transitiveOpt?.negate).toBe(true); + }); + it('should have --commit-id option', () => { const cmd = createPublishCommand(); const opts = cmd.options; @@ -137,5 +149,13 @@ describe('publish-command', () => { it('should allow commit-id incremental mode without delete-unmatched', () => { expect(hasMutuallyExclusivePublishOptions(false, 'abc123')).toBe(false); }); + + it('should reject filter with delete-unmatched', () => { + expect(hasMutuallyExclusivePublishOptions(true, undefined, true)).toBe(true); + }); + + it('should allow filter without delete-unmatched', () => { + expect(hasMutuallyExclusivePublishOptions(false, undefined, true)).toBe(false); + }); }); }); diff --git a/tests/unit/lib/resource-path.test.ts b/tests/unit/lib/resource-path.test.ts index 5ba3899b..174e0f21 100644 --- a/tests/unit/lib/resource-path.test.ts +++ b/tests/unit/lib/resource-path.test.ts @@ -9,6 +9,8 @@ import { parseTemplatePath, getNamePart, getNameFromNameParts, + getResourceDescriptorKey, + sameResourceDescriptor, buildArtifactDirectory, buildArtifactFilePath, buildPolicyFilePath, @@ -650,6 +652,36 @@ describe('getNamePart', () => { }); }); +describe('resource descriptor identity', () => { + const descriptor: ResourceDescriptor = { + type: ResourceType.ApiOperation, + nameParts: ['Orders', 'Get-Order'], + workspace: 'Team-A', + }; + + it('creates a case-insensitive key containing type, workspace, and name parts', () => { + expect(getResourceDescriptorKey(descriptor)).toBe( + 'apioperation:team-a:orders/get-order' + ); + }); + + it('matches equivalent descriptors case-insensitively without crossing scopes', () => { + expect( + sameResourceDescriptor(descriptor, { + type: ResourceType.ApiOperation, + nameParts: ['orders', 'get-order'], + workspace: 'team-a', + }) + ).toBe(true); + expect( + sameResourceDescriptor(descriptor, { + type: ResourceType.ApiOperation, + nameParts: ['orders', 'get-order'], + }) + ).toBe(false); + }); +}); + describe('getNameFromNameParts', () => { it('returns the last element for a 1-part array', () => { expect(getNameFromNameParts(['petstore'])).toBe('petstore'); diff --git a/tests/unit/services/api-publisher.test.ts b/tests/unit/services/api-publisher.test.ts index ea9d4df6..a5cdd469 100644 --- a/tests/unit/services/api-publisher.test.ts +++ b/tests/unit/services/api-publisher.test.ts @@ -94,6 +94,224 @@ describe('api-publisher', () => { }); describe('publishApi', () => { + it('should publish only API children included in the filtered target set', async () => { + const client = createMockClient(); + const operation: ResourceDescriptor = { + type: ResourceType.ApiOperation, + nameParts: ['orders-api', 'get-orders'], + }; + const diagnostic: ResourceDescriptor = { + type: ResourceType.ApiDiagnostic, + nameParts: ['orders-api', 'application-insights'], + }; + const store = createMockStore([operation, diagnostic]); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + }; + + await publishApi(client, store, testContext, apiDescriptor, testConfig, [ + apiDescriptor, + operation, + ]); + + const publishedTasks = mockRunParallel.mock.calls.reduce((sum, call) => { + return sum + (call[0] as unknown[]).length; + }, 0); + expect(publishedTasks).toBe(1); + }); + + it('should not import a full specification when filtered children are excluded', async () => { + const client = createMockClient(); + const operation: ResourceDescriptor = { + type: ResourceType.ApiOperation, + nameParts: ['orders-api', 'get-orders'], + }; + const schema: ResourceDescriptor = { + type: ResourceType.ApiSchema, + nameParts: ['orders-api', 'order-schema'], + }; + const store = createMockStore([operation, schema]); + store.readContent.mockResolvedValue({ content: 'openapi: 3.0.0', format: 'yaml' }); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + }; + + await publishApi(client, store, testContext, apiDescriptor, testConfig, [ + apiDescriptor, + operation, + ]); + + expect(client.putResource).toHaveBeenCalledWith( + testContext, + apiDescriptor, + expect.not.objectContaining({ + properties: expect.objectContaining({ format: expect.anything() }), + }) + ); + }); + + it('should retain specification import when filtered APIs have no child artifacts', async () => { + const client = createMockClient(); + const store = createMockStore([]); + store.readContent.mockResolvedValue({ content: 'openapi: 3.0.0', format: 'yaml' }); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + }; + + await publishApi(client, store, testContext, apiDescriptor, testConfig, [apiDescriptor]); + + expect(client.putResource).toHaveBeenCalledWith( + testContext, + apiDescriptor, + expect.objectContaining({ + properties: expect.objectContaining({ format: 'openapi' }), + }) + ); + }); + + it('should restore filter-eligible children after an incremental specification import', async () => { + const client = createMockClient(); + const operation: ResourceDescriptor = { + type: ResourceType.ApiOperation, + nameParts: ['orders-api', 'get-orders'], + }; + const schema: ResourceDescriptor = { + type: ResourceType.ApiSchema, + nameParts: ['orders-api', 'order-schema'], + }; + const operationPolicy: ResourceDescriptor = { + type: ResourceType.ApiOperationPolicy, + nameParts: ['orders-api', 'get-orders'], + }; + const store = createMockStore([operation, schema, operationPolicy]); + store.readContent.mockResolvedValue({ content: 'openapi: 3.0.0', format: 'yaml' }); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + }; + const config: PublishConfig = { + ...testConfig, + commitId: 'abc123', + filter: { apis: ['orders-api'] }, + }; + + await publishApi(client, store, testContext, apiDescriptor, config, [apiDescriptor]); + + const totalTasks = mockRunParallel.mock.calls.reduce((sum, call) => { + return sum + (call[0] as unknown[]).length; + }, 0); + expect(totalTasks).toBe(3); + }); + + it('should republish an unchanged, filter-eligible child when its API changes in incremental mode', async () => { + const client = createMockClient(); + const tag: ResourceDescriptor = { + type: ResourceType.ApiTag, + nameParts: ['orders-api', 'production'], + }; + const store = createMockStore([tag]); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + }; + const config: PublishConfig = { + ...testConfig, + commitId: 'abc123', + filter: { apis: ['orders-api'] }, + }; + + // Only the API itself is in the incremental diff/expansion set — the + // tag association did not change in this commit. + await publishApi(client, store, testContext, apiDescriptor, config, [apiDescriptor]); + + const publishedTasks = mockRunParallel.mock.calls.reduce((sum, call) => { + return sum + (call[0] as unknown[]).length; + }, 0); + expect(publishedTasks).toBe(1); + }); + + it('should not republish a child excluded by an API sub-filter in incremental mode', async () => { + const client = createMockClient(); + const diagnostic: ResourceDescriptor = { + type: ResourceType.ApiDiagnostic, + nameParts: ['orders-api', 'application-insights'], + }; + const store = createMockStore([diagnostic]); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + }; + const config: PublishConfig = { + ...testConfig, + commitId: 'abc123', + filter: { + apis: ['orders-api'], + apiSubFilters: { 'orders-api': { diagnostics: [] } }, + }, + }; + + await publishApi(client, store, testContext, apiDescriptor, config, [apiDescriptor]); + + const publishedTasks = mockRunParallel.mock.calls.reduce((sum, call) => { + return sum + (call[0] as unknown[]).length; + }, 0); + expect(publishedTasks).toBe(0); + }); + + it('should republish an unchanged API revision when the root API changes in incremental mode', async () => { + const client = createMockClient(); + const revision: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api;rev=2'], + }; + const store = createMockStore([revision]); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + }; + const config: PublishConfig = { + ...testConfig, + commitId: 'abc123', + filter: { apis: ['orders-api'] }, + }; + + // Only the root API is in the incremental diff/expansion set — the + // revision itself did not change in this commit. + await publishApi(client, store, testContext, apiDescriptor, config, [apiDescriptor]); + + expect(mockPublishResource).toHaveBeenCalledTimes(1); + expect(mockPublishResource.mock.calls[0][3]).toEqual(revision); + }); + + it('should ignore managed children from other workspaces when deciding specification import', async () => { + const client = createMockClient(); + const otherWorkspaceSchema: ResourceDescriptor = { + type: ResourceType.ApiSchema, + nameParts: ['orders-api', 'other-schema'], + workspace: 'other', + }; + const store = createMockStore([otherWorkspaceSchema]); + store.readContent.mockResolvedValue({ content: 'openapi: 3.0.0', format: 'yaml' }); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + workspace: 'current', + }; + + await publishApi(client, store, testContext, apiDescriptor, testConfig, [apiDescriptor]); + + expect(client.putResource).toHaveBeenCalledWith( + testContext, + apiDescriptor, + expect.objectContaining({ + properties: expect.objectContaining({ format: 'openapi' }), + }) + ); + }); + it('should publish root API first', async () => { const client = createMockClient(); const store = createMockStore([]); @@ -413,6 +631,45 @@ describe('api-publisher', () => { expect(mockPublishResource.mock.calls[0][3].nameParts[0]).toBe('orders-api;rev=2'); }); + it('should publish only allowed revisions from the same workspace', async () => { + const client = createMockClient(); + const allowedRevision: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api;rev=2'], + workspace: 'team-a', + }; + const revisions: ResourceDescriptor[] = [ + allowedRevision, + { + type: ResourceType.Api, + nameParts: ['orders-api;rev=3'], + workspace: 'team-a', + }, + { + type: ResourceType.Api, + nameParts: ['orders-api;rev=4'], + }, + ]; + const store = createMockStore(revisions); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['orders-api'], + workspace: 'team-a', + }; + + await publishApi( + client, + store, + testContext, + apiDescriptor, + { ...testConfig, filter: { workspaces: ['team-a'] } }, + [apiDescriptor, allowedRevision] + ); + + expect(mockPublishResource).toHaveBeenCalledTimes(1); + expect(mockPublishResource.mock.calls[0][3]).toEqual(allowedRevision); + }); + it('should publish API child resources in parallel', async () => { const client = createMockClient(); const children = [ diff --git a/tests/unit/services/extract-service.test.ts b/tests/unit/services/extract-service.test.ts index 3aeccc42..5f08f2d3 100644 --- a/tests/unit/services/extract-service.test.ts +++ b/tests/unit/services/extract-service.test.ts @@ -518,6 +518,78 @@ describe('extract-service', () => { expect(result.exitCode).toBe(0); }); + it('should extract backend pool members and policy fragment dependencies transitively', async () => { + const backendId = + '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/backends/member'; + const client = createMockClient({ + [ResourceType.Backend]: [ + { + name: 'pool', + properties: { + type: 'Pool', + pool: { services: [{ id: backendId }] }, + }, + }, + ], + [ResourceType.PolicyFragment]: [ + { + name: 'shared-fragment', + properties: { + value: '{{shared-secret}}', + }, + }, + ], + }); + client.getResource = vi.fn().mockImplementation(async (_ctx, descriptor) => { + if (descriptor.type === ResourceType.Backend && descriptor.nameParts[0] === 'member') { + return { name: 'member', properties: {} }; + } + if ( + descriptor.type === ResourceType.NamedValue && + descriptor.nameParts[0] === 'shared-secret' + ) { + return { name: 'shared-secret', properties: { secret: true, value: 'secret' } }; + } + return undefined; + }); + const store = createMockStore(); + + const result = await runExtraction(client, store, { + service: testContext, + outputDir: '/output', + includeTransitive: true, + filter: { + apis: [], + backends: ['pool'], + namedValues: [], + policyFragments: ['shared-fragment'], + }, + logLevel: LogLevel.INFO, + }); + + expect(result.exitCode).toBe(0); + expect(store.writeResource).toHaveBeenCalledWith( + '/output', + expect.objectContaining({ + type: ResourceType.Backend, + nameParts: ['member'], + }), + expect.anything() + ); + expect(store.writeResource).toHaveBeenCalledWith( + '/output', + expect.objectContaining({ + type: ResourceType.NamedValue, + nameParts: ['shared-secret'], + }), + expect.objectContaining({ + properties: expect.objectContaining({ + value: REDACTION_MARKER, + }), + }) + ); + }); + it('should handle transitive dependency not found (getResource returns null)', async () => { const client = createMockClient({}); diff --git a/tests/unit/services/product-publisher.test.ts b/tests/unit/services/product-publisher.test.ts index a54ad722..8145738c 100644 --- a/tests/unit/services/product-publisher.test.ts +++ b/tests/unit/services/product-publisher.test.ts @@ -178,6 +178,94 @@ describe('product-publisher', () => { ); }); + it('does not let resolved targets override explicit API exclusions', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation + .mockResolvedValueOnce([{ name: 'legacy-api' }]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]); + const legacyApi: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['legacy-api'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + products: ['my-product'], + apis: ['!legacy-api', '*'], + }, + }; + + await publishProduct( + client, + store, + testContext, + productDescriptor, + config, + [productDescriptor, legacyApi] + ); + + expect(client.putResource).not.toHaveBeenCalled(); + }); + + it('publishes links to unchanged targets allowed during incremental publish', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation + .mockResolvedValueOnce([{ name: 'orders-api' }]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]); + const config: PublishConfig = { + ...testConfig, + commitId: 'abc123', + filter: { + products: ['my-product'], + apis: ['orders-api'], + }, + }; + + await publishProduct( + client, + store, + testContext, + productDescriptor, + config, + [productDescriptor] + ); + + expect(client.putResource).toHaveBeenCalledWith( + testContext, + expect.objectContaining({ + type: ResourceType.ProductApi, + nameParts: ['my-product', 'orders-api'], + }), + {} + ); + }); + + it('republishes an unchanged product policy when the product changes in incremental mode', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation.mockResolvedValue([]); + store.readContent.mockResolvedValue({ content: '', format: 'xml' }); + const config: PublishConfig = { + ...testConfig, + commitId: 'abc123', + filter: { products: ['my-product'] }, + }; + + // Only the product itself is in the incremental diff/expansion set — + // policy.xml did not change in this commit. + await publishProduct(client, store, testContext, productDescriptor, config, [productDescriptor]); + + expect(mockPublishResource).toHaveBeenCalledWith( + client, store, testContext, + expect.objectContaining({ type: ResourceType.ProductPolicy, nameParts: ['my-product'] }), + config + ); + }); + it('groups association: calls putResource with ProductGroup descriptor', async () => { const client = createMockClient(); const store = createMockStore(); diff --git a/tests/unit/services/publish-service.test.ts b/tests/unit/services/publish-service.test.ts index 2f842263..ff615e23 100644 --- a/tests/unit/services/publish-service.test.ts +++ b/tests/unit/services/publish-service.test.ts @@ -137,6 +137,345 @@ describe('publish-service', () => { expect(result.totalErrors).toBe(0); }); + it('should publish only resources matched by a filter', async () => { + const resources = [ + { type: ResourceType.NamedValue, nameParts: ['keep'] }, + { type: ResourceType.NamedValue, nameParts: ['skip'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { namedValues: ['keep'] }, + includeTransitive: false, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(1); + expect(client.putResource).toHaveBeenCalledTimes(1); + expect(client.putResource.mock.calls[0]?.[1].nameParts).toEqual(['keep']); + }); + + it('should include an artifact-backed version set transitively', async () => { + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.VersionSet, nameParts: ['orders-v1'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => { + if (descriptor.type === ResourceType.Api) { + return { + name: 'orders', + properties: { + apiVersionSetId: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/apiVersionSets/orders-v1', + }, + }; + } + return { name: descriptor.nameParts[0] ?? '', properties: {} }; + }); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], versionSets: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(2); + expect(client.putResource.mock.calls.some((call) => + call[1].type === ResourceType.VersionSet && + call[1].nameParts[0] === 'orders-v1' + )).toBe(true); + }); + + it('should include every dependency type supported by extract', async () => { + const resources: ResourceDescriptor[] = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.ApiPolicy, nameParts: ['orders'] }, + { type: ResourceType.NamedValue, nameParts: ['orders-key'] }, + { type: ResourceType.Backend, nameParts: ['orders-backend'] }, + { type: ResourceType.PolicyFragment, nameParts: ['shared-auth'] }, + { type: ResourceType.VersionSet, nameParts: ['orders-v1'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.Api + ? { + name: 'orders', + properties: { + apiVersionSetId: + '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/apiVersionSets/orders-v1', + }, + } + : { name: descriptor.nameParts[0] ?? '', properties: {} } + ); + store.readContent.mockImplementation(async (_sourceDir, descriptor) => + descriptor.type === ResourceType.ApiPolicy + ? { + content: [ + '', + '{{orders-key}}', + '', + '', + '', + ].join(''), + format: 'xml', + } + : undefined + ); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { + apis: ['orders'], + namedValues: [], + backends: [], + policyFragments: [], + versionSets: [], + }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(5); + expect(publishApi).toHaveBeenCalledWith( + client, + store, + testContext, + expect.objectContaining({ type: ResourceType.Api, nameParts: ['orders'] }), + expect.anything(), + expect.arrayContaining([ + expect.objectContaining({ type: ResourceType.ApiPolicy, nameParts: ['orders'] }), + expect.objectContaining({ type: ResourceType.NamedValue, nameParts: ['orders-key'] }), + expect.objectContaining({ type: ResourceType.Backend, nameParts: ['orders-backend'] }), + expect.objectContaining({ type: ResourceType.PolicyFragment, nameParts: ['shared-auth'] }), + expect.objectContaining({ type: ResourceType.VersionSet, nameParts: ['orders-v1'] }), + ]) + ); + }); + + it('should not scan unrelated resources that share the selected parent name', async () => { + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.Product, nameParts: ['orders'] }, + { type: ResourceType.Api, nameParts: ['shipping'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readAssociation.mockImplementation( + async (_sourceDir: string, descriptor: ResourceDescriptor, associationType: string) => + descriptor.type === ResourceType.Product && associationType === 'apis' + ? [{ name: 'shipping' }] + : [] + ); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], products: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(publishApi).toHaveBeenCalledTimes(1); + expect(publishApi).toHaveBeenCalledWith( + client, + store, + testContext, + expect.objectContaining({ nameParts: ['orders'] }), + expect.anything(), + expect.anything() + ); + }); + + it('should not pull composite API targets from product associations', async () => { + const resources = [ + { type: ResourceType.Product, nameParts: ['starter'] }, + { type: ResourceType.Api, nameParts: ['legacy-api'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readAssociation.mockResolvedValue([{ name: 'legacy-api' }]); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { products: ['starter'], apis: [] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(1); + expect(publishProduct).toHaveBeenCalledTimes(1); + expect(publishApi).not.toHaveBeenCalled(); + }); + + it('should include backend pool members transitively', async () => { + const resources = [ + { type: ResourceType.Backend, nameParts: ['pool'] }, + { type: ResourceType.Backend, nameParts: ['member'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockImplementation(async (_sourceDir, descriptor) => + descriptor.nameParts[0] === 'pool' + ? { + properties: { + type: 'Pool', + pool: { + services: [{ + id: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/backends/member', + }], + }, + }, + } + : { properties: {} } + ); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { backends: ['pool'] }, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(2); + expect(client.putResource.mock.calls.some((call) => + call[1].type === ResourceType.Backend && + call[1].nameParts[0] === 'member' + )).toBe(true); + }); + + it('should not include transitive dependencies when disabled', async () => { + const resources = [ + { type: ResourceType.Api, nameParts: ['orders'] }, + { type: ResourceType.VersionSet, nameParts: ['orders-v1'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + store.readResource.mockResolvedValue({ + name: 'orders', + properties: { + apiVersionSetId: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/apiVersionSets/orders-v1', + }, + }); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], versionSets: [] }, + includeTransitive: false, + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(1); + expect(client.putResource.mock.calls.some((call) => call[1].type === ResourceType.VersionSet)).toBe(false); + }); + + it('should intersect incremental changes with a filter', async () => { + const changedDescriptors = [ + { type: ResourceType.NamedValue, nameParts: ['keep'] }, + { type: ResourceType.NamedValue, nameParts: ['skip'] }, + ]; + vi.mocked(computeGitDiff).mockResolvedValue({ + changedDescriptors, + deletedDescriptors: [], + }); + const client = createMockClient(); + const store = createMockStore(changedDescriptors); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { namedValues: ['keep'] }, + includeTransitive: false, + commitId: 'abc123', + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(1); + expect(client.putResource.mock.calls[0]?.[1].nameParts).toEqual(['keep']); + }); + + it('should resolve policy dependencies from unchanged child artifacts in incremental mode', async () => { + const api = { type: ResourceType.Api, nameParts: ['orders'] }; + const apiPolicy = { type: ResourceType.ApiPolicy, nameParts: ['orders'] }; + const backend = { type: ResourceType.Backend, nameParts: ['orders-backend'] }; + vi.mocked(computeGitDiff).mockResolvedValue({ + changedDescriptors: [api], + deletedDescriptors: [], + }); + const client = createMockClient(); + const store = createMockStore([api, apiPolicy, backend]); + store.readContent.mockImplementation(async (_sourceDir: string, descriptor: ResourceDescriptor) => + descriptor.type === ResourceType.ApiPolicy + ? { content: '' } + : undefined + ); + + const result = await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { apis: ['orders'], backends: [] }, + commitId: 'abc123', + dryRun: false, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(result.totalPuts).toBe(2); + expect(client.putResource.mock.calls.some((call) => + call[1].type === ResourceType.Backend && + call[1].nameParts[0] === 'orders-backend' + )).toBe(true); + }); + + it('should report filtered targets in dry-run mode', async () => { + const resources = [ + { type: ResourceType.NamedValue, nameParts: ['keep'] }, + { type: ResourceType.NamedValue, nameParts: ['skip'] }, + ]; + const client = createMockClient(); + const store = createMockStore(resources); + + await runPublish(client, store, { + service: testContext, + sourceDir: '/source', + filter: { namedValues: ['keep'] }, + includeTransitive: false, + dryRun: true, + deleteUnmatched: false, + logLevel: LogLevel.INFO, + }); + + expect(generateDryRunReport).toHaveBeenCalledWith( + store, + client, + testContext, + expect.objectContaining({ filter: { namedValues: ['keep'] } }), + [{ type: ResourceType.NamedValue, nameParts: ['keep'] }], + [] + ); + }); + it('should return exit code 1 when some fail', async () => { const resources = [ { type: ResourceType.NamedValue, nameParts: ['nv1'] }, diff --git a/tests/unit/services/resource-publisher.test.ts b/tests/unit/services/resource-publisher.test.ts index 79f05637..5f67af11 100644 --- a/tests/unit/services/resource-publisher.test.ts +++ b/tests/unit/services/resource-publisher.test.ts @@ -350,6 +350,125 @@ describe('resource-publisher', () => { ); }); + it('should skip GatewayApi entries whose API target is excluded', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation.mockResolvedValue([{ name: 'legacy-api' }]); + + const descriptor: ResourceDescriptor = { + type: ResourceType.GatewayApi, + nameParts: ['my-gateway'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + gateways: ['my-gateway'], + apis: ['!legacy-api', '*'], + }, + }; + + const result = await publishResource(client, store, testContext, descriptor, config); + + expect(result.status).toBe('success'); + expect(client.putResource).not.toHaveBeenCalled(); + }); + + it('should skip ApiTag links whose Tag target is excluded', async () => { + const client = createMockClient(); + const store = createMockStore(); + const descriptor: ResourceDescriptor = { + type: ResourceType.ApiTag, + nameParts: ['orders', 'internal'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + apis: ['orders'], + tags: ['!internal', '*'], + }, + }; + + const result = await publishResource(client, store, testContext, descriptor, config); + + expect(result.status).toBe('skipped'); + expect(store.readResource).not.toHaveBeenCalled(); + expect(client.putResource).not.toHaveBeenCalled(); + }); + + it('should publish a GatewayApi association when its target passes an explicit filter and was extracted', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation.mockResolvedValue([{ name: 'orders-api' }]); + store.readResource.mockResolvedValue({ name: 'orders-api', properties: {} }); + + const descriptor: ResourceDescriptor = { + type: ResourceType.GatewayApi, + nameParts: ['my-gateway'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + gateways: ['my-gateway'], + apis: ['orders-api'], + }, + }; + + const result = await publishResource(client, store, testContext, descriptor, config); + + expect(result.status).toBe('success'); + expect(client.putResource).toHaveBeenCalledWith( + testContext, + expect.objectContaining({ type: ResourceType.GatewayApi, nameParts: ['my-gateway', 'orders-api'] }), + {} + ); + }); + + it('should skip a GatewayApi association whose target passes the filter but was never extracted', async () => { + const client = createMockClient(); + const store = createMockStore(); + store.readAssociation.mockResolvedValue([{ name: 'orders-api' }]); + // readResource left unmocked (resolves undefined) — simulates a dangling + // reference to an API name that matches the filter but has no artifact. + + const descriptor: ResourceDescriptor = { + type: ResourceType.GatewayApi, + nameParts: ['my-gateway'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + gateways: ['my-gateway'], + apis: ['orders-api'], + }, + }; + + const result = await publishResource(client, store, testContext, descriptor, config); + + expect(result.status).toBe('success'); + expect(client.putResource).not.toHaveBeenCalled(); + }); + + it('should skip an ApiTag link whose Tag target passes the filter but was never extracted', async () => { + const client = createMockClient(); + const store = createMockStore(); + const descriptor: ResourceDescriptor = { + type: ResourceType.ApiTag, + nameParts: ['orders', 'production'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + apis: ['orders'], + tags: ['production'], + }, + }; + + const result = await publishResource(client, store, testContext, descriptor, config); + + expect(result.status).toBe('skipped'); + expect(client.putResource).not.toHaveBeenCalled(); + }); + it('should strip properties.value from KeyVault-backed NamedValue PUT payload', async () => { const client = createMockClient(); const store = createMockStore(); @@ -612,6 +731,71 @@ describe('resource-publisher', () => { expect(props.scope).toBe('/products/my-product'); }); + it('should skip a Subscription whose Product target is excluded', async () => { + const client = createMockClient(); + const store = createMockStore(); + const armScopePrefix = + '/subscriptions/sub-1/resourceGroups/rg-1/providers/Microsoft.ApiManagement/service/apim-1'; + store.readResource.mockResolvedValue({ + name: 'sub-1', + properties: { + scope: `${armScopePrefix}/products/legacy-product`, + }, + }); + + const descriptor: ResourceDescriptor = { + type: ResourceType.Subscription, + nameParts: ['sub-1'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + subscriptions: ['sub-1'], + products: ['!legacy-product', '*'], + }, + }; + + const result = await publishResource(client, store, testContext, descriptor, config); + + expect(result.status).toBe('skipped'); + expect(client.putResource).not.toHaveBeenCalled(); + }); + + it('should skip a Subscription whose Product target passes the filter but was never extracted', async () => { + const client = createMockClient(); + const store = createMockStore(); + const armScopePrefix = + '/subscriptions/sub-1/resourceGroups/rg-1/providers/Microsoft.ApiManagement/service/apim-1'; + const subscriptionJson = { + name: 'sub-1', + properties: { + scope: `${armScopePrefix}/products/orders-product`, + }, + }; + // Only the subscription's own artifact exists; the referenced product + // ("orders-product") was never extracted, even though it passes the filter. + store.readResource.mockImplementation(async (_dir: string, desc: ResourceDescriptor) => + desc.type === ResourceType.Subscription ? subscriptionJson : undefined + ); + + const descriptor: ResourceDescriptor = { + type: ResourceType.Subscription, + nameParts: ['sub-1'], + }; + const config: PublishConfig = { + ...testConfig, + filter: { + subscriptions: ['sub-1'], + products: ['orders-product'], + }, + }; + + const result = await publishResource(client, store, testContext, descriptor, config); + + expect(result.status).toBe('skipped'); + expect(client.putResource).not.toHaveBeenCalled(); + }); + it('should leave scope unchanged when it is already a relative APIM path', async () => { const client = createMockClient(); const store = createMockStore(); diff --git a/tests/unit/services/transitive-resolver.test.ts b/tests/unit/services/transitive-resolver.test.ts index 34f16248..f111fa33 100644 --- a/tests/unit/services/transitive-resolver.test.ts +++ b/tests/unit/services/transitive-resolver.test.ts @@ -4,7 +4,7 @@ * Unit tests for Transitive dependency resolver */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { ResourceType } from '../../../src/models/resource-types.js'; import { FilterConfig } from '../../../src/models/config.js'; import { @@ -12,6 +12,8 @@ import { scanApiVersionSetReference, resolveTransitiveDependencies, findTransitiveDependencies, + findSubscriptionTargets, + scanArtifactReferences, } from '../../../src/services/transitive-resolver.js'; describe('transitive-resolver', () => { @@ -105,6 +107,19 @@ describe('transitive-resolver', () => { const apiJson = { name: 'my-api' }; expect(scanApiVersionSetReference(apiJson)).toBeUndefined(); }); + + it('should preserve malformed encoded names instead of throwing', () => { + const apiJson = { + properties: { + apiVersionSetId: '/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.ApiManagement/service/svc1/apiVersionSets/version%', + }, + }; + + expect(scanApiVersionSetReference(apiJson)).toEqual({ + type: ResourceType.VersionSet, + name: 'version%', + }); + }); }); describe('resolveTransitiveDependencies', () => { @@ -194,5 +209,105 @@ describe('transitive-resolver', () => { const deps = findTransitiveDependencies(policies, apis); expect(deps).toHaveLength(0); }); + + describe('scanArtifactReferences', () => { + it('scans policy references without parsing policy XML as JSON', async () => { + const store = { + readResource: vi.fn().mockRejectedValue(new SyntaxError('Unexpected token <')), + readContent: vi.fn().mockResolvedValue({ + content: '{{shared-secret}}', + format: 'xml', + }), + readAssociation: vi.fn(), + }; + + await expect( + scanArtifactReferences(store, '/source', { + type: ResourceType.ApiPolicy, + nameParts: ['orders'], + workspace: 'team-a', + }) + ).resolves.toContainEqual({ + type: ResourceType.NamedValue, + nameParts: ['shared-secret'], + workspace: 'team-a', + }); + expect(store.readResource).not.toHaveBeenCalled(); + }); + + it('should scan backend pools without treating links as transitive dependencies', async () => { + const store = { + readResource: vi.fn() + .mockResolvedValueOnce({ + properties: { + type: 'Pool', + pool: { + services: [{ id: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/backends/backend-1' }], + }, + }, + }), + readContent: vi.fn().mockResolvedValue(undefined), + readAssociation: vi.fn(), + }; + + const backendRefs = await scanArtifactReferences( + store, + '/source', + { type: ResourceType.Backend, nameParts: ['pool'] } + ); + expect(backendRefs).toContainEqual({ + type: ResourceType.Backend, + nameParts: ['backend-1'], + workspace: undefined, + }); + + await expect(scanArtifactReferences( + store, + '/source', + { type: ResourceType.Subscription, nameParts: ['sub'] } + )).resolves.toEqual([]); + + await expect(scanArtifactReferences( + store, + '/source', + { type: ResourceType.Product, nameParts: ['starter'] } + )).resolves.toEqual([]); + expect(store.readAssociation).not.toHaveBeenCalled(); + }); + + it('should identify subscription targets separately from transitive dependencies', () => { + const targets = findSubscriptionTargets({ + properties: { + scope: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/apis/orders', + apiId: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/products/starter', + }, + }); + + expect(targets).toEqual([ + { + type: ResourceType.Api, + nameParts: ['orders'], + workspace: undefined, + }, + { + type: ResourceType.Product, + nameParts: ['starter'], + workspace: undefined, + }, + ]); + }); + + it('should preserve service scope for absolute subscription targets', () => { + expect(findSubscriptionTargets({ + properties: { + scope: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/apis/shared-api', + }, + }, 'team-a')).toContainEqual({ + type: ResourceType.Api, + nameParts: ['shared-api'], + workspace: undefined, + }); + }); + }); }); }); diff --git a/tests/unit/services/workspace-extractor.test.ts b/tests/unit/services/workspace-extractor.test.ts index c1aff29f..9b5e34c9 100644 --- a/tests/unit/services/workspace-extractor.test.ts +++ b/tests/unit/services/workspace-extractor.test.ts @@ -284,6 +284,55 @@ describe('workspace-extractor', () => { ]); }); + it('should apply workspace exclusions with shared filter semantics', async () => { + const client = createMockClient(); + client.listResources = async function* ( + _ctx: ApimServiceContext, + type: ResourceType + ) { + if (type === ResourceType.Workspace) { + yield { name: 'team-a', properties: {} }; + yield { name: 'team-b', properties: {} }; + } + }; + const store = createMockStore(); + + const results = await extractWorkspaces( + client, + store, + testContext, + '/output', + { workspaces: ['!team-b'] } + ); + + expect(results.map((result) => result.workspaceName)).toEqual(['team-a']); + }); + + it('should apply exclusions after wildcard workspace inclusions', async () => { + const client = createMockClient(); + client.listResources = async function* ( + _ctx: ApimServiceContext, + type: ResourceType + ) { + if (type === ResourceType.Workspace) { + yield { name: 'team-a', properties: {} }; + yield { name: 'team-b', properties: {} }; + yield { name: 'shared', properties: {} }; + } + }; + const store = createMockStore(); + + const results = await extractWorkspaces( + client, + store, + testContext, + '/output', + { workspaces: ['team-*', '!team-b'] } + ); + + expect(results.map((result) => result.workspaceName)).toEqual(['team-a']); + }); + it('should apply sub-filter with wildcard workspace name patterns', async () => { const client = createMockClient(); let firstCall = true; @@ -317,6 +366,97 @@ describe('workspace-extractor', () => { // Only nv-1 should be extracted due to sub-filter expect(results[0]?.resourceCount).toBe(1); }); + + it('should extract workspace policy fragment dependencies transitively', async () => { + const client = createMockClient(); + client.listResources = async function* ( + _ctx: ApimServiceContext, + type: ResourceType + ) { + if (type === ResourceType.PolicyFragment) { + yield { + name: 'shared-fragment', + properties: { + value: '{{workspace-secret}}', + }, + }; + } + if (type === ResourceType.Backend) { + yield { + name: 'workspace-pool', + properties: { + type: 'Pool', + pool: { + services: [{ + id: '/subscriptions/s/resourceGroups/r/providers/Microsoft.ApiManagement/service/a/backends/service-member', + }], + }, + }, + }; + } + }; + client.getResource.mockImplementation(async (_ctx, descriptor) => { + if (descriptor.type === ResourceType.Workspace) { + return { name: 'team-a', properties: {} }; + } + if ( + descriptor.type === ResourceType.NamedValue && + descriptor.nameParts[0] === 'workspace-secret' + ) { + return { name: 'workspace-secret', properties: { secret: true, value: 'secret' } }; + } + if ( + descriptor.type === ResourceType.Backend && + descriptor.nameParts[0] === 'service-member' + ) { + return { name: 'service-member', properties: {} }; + } + return undefined; + }); + const store = createMockStore(); + const filter: FilterConfig = { + workspaces: ['team-a'], + workspaceSubFilters: { + 'team-a': { + backends: ['workspace-pool'], + namedValues: [], + policyFragments: ['shared-fragment'], + }, + }, + }; + + const results = await extractWorkspaces( + client, + store, + testContext, + '/output', + filter, + true + ); + + expect(results[0]?.resourceCount).toBe(4); + expect(client.getResource).toHaveBeenCalledWith( + testContext, + expect.objectContaining({ + type: ResourceType.Backend, + nameParts: ['service-member'], + workspace: undefined, + }) + ); + expect(store.writeResource).toHaveBeenCalledWith( + '/output', + expect.objectContaining({ + type: ResourceType.NamedValue, + nameParts: ['workspace-secret'], + workspace: 'team-a', + }), + expect.objectContaining({ + properties: expect.objectContaining({ + value: '*** REDACTED ***', + }), + }) + ); + }); }); describe('resolveWorkspaceFilter', () => {