diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 965b2739410..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -# Prometheus has switched to GitHub action. -# Circle CI is not disabled repository-wise so that previous pull requests -# continue working. -# This file does not generate any CircleCI workflow. - -version: 2.1 - -executors: - golang: - docker: - - image: busybox - -jobs: - noopjob: - executor: golang - - steps: - - run: - command: "true" - -workflows: - version: 2 - prometheus: - jobs: - - noopjob - triggers: - - schedule: - cron: "0 0 30 2 *" - filters: - branches: - only: - - main diff --git a/.dockerignore b/.dockerignore index 5eca8e1b808..09670991552 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,8 @@ data/ .tarballs/ !.build/linux-amd64/ -!.build/linux-armv7/ !.build/linux-arm64/ +!.build/linux-armv7/ !.build/linux-ppc64le/ +!.build/linux-riscv64/ !.build/linux-s390x/ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..432caee6f78 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +web/api/v1/testdata/openapi_golden.yaml linguist-generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 7f7cec9cda9..00000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,10 +0,0 @@ -/web/ui @juliusv -/web/ui/module @juliusv @nexucis -/storage/remote @cstyan @bwplotka @tomwilkie -/storage/remote/otlptranslator @aknuds1 @jesusvazquez -/discovery/kubernetes @brancz -/tsdb @jesusvazquez -/promql @roidelapluie -/cmd/promtool @dgl -/documentation/prometheus-mixin @metalmatze - diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index f4d17b3596b..bb4e2d24c99 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,4 @@ -blank_issues_enabled: false +blank_issues_enabled: true contact_links: - name: Prometheus Community Support url: https://prometheus.io/community/ diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index cf90177b1d1..58641d17d8a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,5 @@ + +#### Which issue(s) does the PR fix: + + +#### Release notes for end users (**ALL** commits must be considered). +*Reviewers should verify clarity and quality.* + + +```release-notes + +``` diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 89b2f4d0b6f..00000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "gomod" - directory: "/" - schedule: - interval: "monthly" - groups: - k8s.io: - patterns: - - "k8s.io/*" - go.opentelemetry.io: - patterns: - - "go.opentelemetry.io/*" - open-pull-requests-limit: 20 - - package-ecosystem: "gomod" - directory: "/documentation/examples/remote_storage" - schedule: - interval: "monthly" - - package-ecosystem: "npm" - directory: "/web/ui" - schedule: - interval: "monthly" - open-pull-requests-limit: 20 - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "monthly" - - package-ecosystem: "github-actions" - directory: "/scripts" - schedule: - interval: "monthly" - - package-ecosystem: "docker" - directory: "/" - schedule: - interval: "monthly" diff --git a/.github/workflows/automerge-dependabot.yml b/.github/workflows/automerge-dependabot.yml new file mode 100644 index 00000000000..8b07f4df959 --- /dev/null +++ b/.github/workflows/automerge-dependabot.yml @@ -0,0 +1,30 @@ +--- +name: Dependabot auto-merge +on: pull_request + +concurrency: + group: ${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + dependabot: + permissions: + contents: write + pull-requests: write + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.user.login == 'dependabot[bot]' && github.repository_owner == 'prometheus' }} + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@21025c705c08248db411dc16f3619e6b5f9ea21a # v2.5.0 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Enable auto-merge for Dependabot PRs + if: ${{steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'version-update:semver-patch'}} + run: gh pr merge --auto --merge "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/buf-lint.yml b/.github/workflows/buf-lint.yml index 3f6cf76e16f..d2c78cd6def 100644 --- a/.github/workflows/buf-lint.yml +++ b/.github/workflows/buf-lint.yml @@ -12,8 +12,10 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: bufbuild/buf-setup-action@54abbed4fe8d8d45173eca4798b0c39a53a7b658 # v1.39.0 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1 diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml index 632d38cb009..fcfdc43c6df 100644 --- a/.github/workflows/buf.yml +++ b/.github/workflows/buf.yml @@ -12,8 +12,10 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'prometheus' steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: bufbuild/buf-setup-action@54abbed4fe8d8d45173eca4798b0c39a53a7b658 # v1.39.0 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1 @@ -23,7 +25,7 @@ jobs: with: input: 'prompb' against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD~1,subdir=prompb' - - uses: bufbuild/buf-push-action@a654ff18effe4641ebea4a4ce242c49800728459 # v1.1.1 + - uses: bufbuild/buf-push-action@a654ff18effe4641ebea4a4ce242c49800728459 # v1.2.0 with: input: 'prompb' buf_token: ${{ secrets.BUF_TOKEN }} diff --git a/.github/workflows/check_release_notes.yml b/.github/workflows/check_release_notes.yml new file mode 100644 index 00000000000..c2bd1b1d640 --- /dev/null +++ b/.github/workflows/check_release_notes.yml @@ -0,0 +1,28 @@ +name: 'Check release notes' +on: + pull_request: + branches: [main, 'release-*'] + types: + - opened + - reopened + - edited + - synchronize +permissions: + contents: read + pull-requests: read + +jobs: + check_release_notes: + name: check + runs-on: ubuntu-latest + # Don't run this workflow on forks. + # Don't run it on dependabot PRs either as humans would take control in case a bump introduces a breaking change. + if: (github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community') && github.event.pull_request.user.login != 'dependabot[bot]' + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - env: + PR_DESCRIPTION: ${{ github.event.pull_request.body }} + run: | + echo "$PR_DESCRIPTION" | ./scripts/check_release_notes.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98d3d9a754f..7d9e613b8d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,11 @@ name: CI on: pull_request: push: + branches: [main, 'release-*'] + tags: ['v*'] + +permissions: + contents: read jobs: test_go: @@ -11,13 +16,16 @@ jobs: container: # Whenever the Go version is updated here, .promu.yml # should also be updated. - image: quay.io/prometheus/golang-builder:1.23-base + image: quay.io/prometheus/golang-builder:1.26-base steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 - - uses: ./.github/promci/actions/setup_environment - - run: make GOOPTS=--tags=stringlabels GO_ONLY=1 SKIP_GOLANGCI_LINT=1 - - run: go test --tags=stringlabels ./tsdb/ -test.tsdb-isolation=false + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: prometheus/promci-setup@5af30ba8c199a91d6c04ebdc3c48e630e355f62d # v0.1.0 + with: + enable_npm: true + - run: make GO_ONLY=1 SKIP_GOLANGCI_LINT=1 + - run: go test ./tsdb/ -test.tsdb-isolation=false - run: make -C documentation/examples/remote_storage - run: make -C documentation/examples @@ -25,16 +33,64 @@ jobs: name: More Go tests runs-on: ubuntu-latest container: - image: quay.io/prometheus/golang-builder:1.23-base + image: quay.io/prometheus/golang-builder:1.26-base steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 - - uses: ./.github/promci/actions/setup_environment + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: prometheus/promci-setup@5af30ba8c199a91d6c04ebdc3c48e630e355f62d # v0.1.0 - run: go test --tags=dedupelabels ./... - - run: GOARCH=386 go test ./cmd/prometheus - - uses: ./.github/promci/actions/check_proto + - run: go test --tags=slicelabels -race ./cmd/prometheus ./model/textparse ./prompb/... + - run: go test --tags=forcedirectio -race ./tsdb/ + - run: make protoc + - run: make proto + - run: git diff --exit-code + + test_go_386: + name: Go tests for 32-bit x86 + runs-on: ubuntu-latest + container: + image: quay.io/prometheus/golang-builder:1.26-base + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - version: "3.15.8" + persist-credentials: false + - uses: prometheus/promci-setup@5af30ba8c199a91d6c04ebdc3c48e630e355f62d # v0.1.0 + # NOTE(bwplotka): We limit concurrency to avoid issues around too many concurrent mmaps + # caused by parallel tests. See context: https://github.com/prometheus/prometheus/pull/18709 + # Alternatively we could adjust each relevant test to have 386 aware parallelization setting. + - run: GOARCH=386 go test -parallel=1 ./... + + list_lts_releases: + name: List active LTS releases + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.lts.outputs.versions }} + steps: + - id: lts + # Emit the active LTS versions as a JSON array consumed by the matrix below. + run: | + versions=$(curl -sSf https://prometheus.io/download.json \ + | jq -c '[.prometheus[] | select(.lts == true) | (.version | ltrimstr("v"))]') + echo "versions=$versions" >> "$GITHUB_OUTPUT" + + test_version_upgrade: + name: Go tests for Prometheus upgrades and downgrades + needs: list_lts_releases + runs-on: ubuntu-latest + strategy: + # Run every LTS release even if one fails. + fail-fast: false + matrix: + lts_version: ${{ fromJSON(needs.list_lts_releases.outputs.versions) }} + container: + image: quay.io/prometheus/golang-builder:1.26-base + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: prometheus/promci-setup@5af30ba8c199a91d6c04ebdc3c48e630e355f62d # v0.1.0 + - run: go test -v --race ./cmd/prometheus/ --test.version-upgrade=true --test.lts-version=${{ matrix.lts_version }} -run TestVersionUpgrade test_go_oldest: name: Go tests with previous Go version @@ -44,9 +100,11 @@ jobs: GOTOOLCHAIN: local container: # The go version in this image should be N-1 wrt test_go. - image: quay.io/prometheus/golang-builder:1.22-base + image: quay.io/prometheus/golang-builder:1.25-base steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - run: make build # Don't run NPM build; don't run race-detector. - run: make test GO_ONLY=1 test-flags="" @@ -57,19 +115,20 @@ jobs: # Whenever the Go version is updated here, .promu.yml # should also be updated. container: - image: quay.io/prometheus/golang-builder:1.23-base + image: quay.io/prometheus/golang-builder:1.26-base steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 - - uses: ./.github/promci/actions/setup_environment + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: prometheus/promci-setup@5af30ba8c199a91d6c04ebdc3c48e630e355f62d # v0.1.0 with: enable_go: false enable_npm: true - run: make assets-tarball - run: make ui-lint - run: make ui-test - - uses: ./.github/promci/actions/save_artifacts + - uses: prometheus/promci-artifacts/save@f9a587dbc0b2c78a0c54f8ad1cde71ea29a4b76f # v0.1.0 with: directory: .tarballs @@ -77,12 +136,14 @@ jobs: name: Go tests on Windows runs-on: windows-latest steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - go-version: 1.23.x + go-version: 1.26.x - run: | - $TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"} + $TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"} go test $TestTargets -vet=off -v shell: powershell @@ -92,9 +153,11 @@ jobs: # Whenever the Go version is updated here, .promu.yml # should also be updated. container: - image: quay.io/prometheus/golang-builder:1.23-base + image: quay.io/prometheus/golang-builder:1.26-base steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - run: go install ./cmd/promtool/. - run: go install github.com/google/go-jsonnet/cmd/jsonnet@latest - run: go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest @@ -104,12 +167,35 @@ jobs: - run: make -C documentation/prometheus-mixin - run: git diff --exit-code + test-compliance: + name: Compliance testing + runs-on: ubuntu-latest + container: + # Whenever the Go version is updated here, .promu.yml + # should also be updated. + image: quay.io/prometheus/golang-builder:1.26-base + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: prometheus/promci-setup@5af30ba8c199a91d6c04ebdc3c48e630e355f62d # v0.1.0 + with: + enable_npm: false + # NOTE: Those tests are based on https://github.com/prometheus/compliance and + # are executed against the ./cmd/prometheus main package. + - run: go test -v --tags=compliance ./compliance/... + build: name: Build Prometheus for common architectures runs-on: ubuntu-latest + # Reuse the web UI built and tested by test_ui instead of rebuilding it in + # every crossbuild container. + needs: [test_ui] if: | !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')) && + !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.')) + && !(github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-')) && !(github.event_name == 'push' && github.event.ref == 'refs/heads/main') @@ -117,19 +203,33 @@ jobs: matrix: thread: [ 0, 1, 2 ] steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 - - uses: ./.github/promci/actions/build + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: prometheus/promci-artifacts/restore@f9a587dbc0b2c78a0c54f8ad1cde71ea29a4b76f # v0.1.0 + - name: Extract pre-built UI assets + run: | + mkdir -p .prebuilt-ui + tar -xzvf .tarballs/prometheus-web-ui-*.tar.gz -C .prebuilt-ui + - uses: prometheus/promci/build@c7d527128ffb38ad3d7934795da35b252c7d51dd # v0.8.3 with: - promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386" + checkout: false + # Feed the restored UI assets into the crossbuild containers so make + # skips the UI build (see PREBUILT_ASSETS_STATIC_DIR in the Makefile). + promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386 --env PREBUILT_ASSETS_STATIC_DIR=.prebuilt-ui/static" parallelism: 3 thread: ${{ matrix.thread }} build_all: name: Build Prometheus for all architectures runs-on: ubuntu-latest + # Reuse the web UI built and tested by test_ui instead of rebuilding it in + # every crossbuild container. + needs: [test_ui] if: | (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')) || + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.')) + || (github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-')) || (github.event_name == 'push' && github.event.ref == 'refs/heads/main') @@ -140,17 +240,38 @@ jobs: # Whenever the Go version is updated here, .promu.yml # should also be updated. steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 - - uses: ./.github/promci/actions/build + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: prometheus/promci-artifacts/restore@f9a587dbc0b2c78a0c54f8ad1cde71ea29a4b76f # v0.1.0 + - name: Extract pre-built UI assets + run: | + mkdir -p .prebuilt-ui + tar -xzvf .tarballs/prometheus-web-ui-*.tar.gz -C .prebuilt-ui + - uses: prometheus/promci/build@c7d527128ffb38ad3d7934795da35b252c7d51dd # v0.8.3 with: + checkout: false parallelism: 12 thread: ${{ matrix.thread }} + # Feed the restored UI assets into the crossbuild containers so make + # skips the UI build (see PREBUILT_ASSETS_STATIC_DIR in the Makefile). + promu_opts: --env PREBUILT_ASSETS_STATIC_DIR=.prebuilt-ui/static build_all_status: + # This status check aggregates the individual matrix jobs of the "Build + # Prometheus for all architectures" step into a final status. Fails if a + # single matrix job fails, succeeds if all matrix jobs succeed. + # See https://github.com/orgs/community/discussions/4324 for why this is + # needed name: Report status of build Prometheus for all architectures runs-on: ubuntu-latest needs: [build_all] - if: github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-') + # The run condition needs to include always(). Otherwise actions + # behave unexpected: + # only "needs" will make the Status Report be skipped if one of the builds fails https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/using-jobs-in-a-workflow#defining-prerequisite-jobs + # And skipped is treated as success https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborat[…]n-repositories-with-code-quality-features/about-status-checks + # Adding always ensures that the status check is run independently of the + # results of Build All + if: always() && github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-') steps: - name: Successful build if: ${{ !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled')) }} @@ -159,105 +280,158 @@ jobs: if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} run: exit 1 check_generated_parser: + # Checks generated parser and UI functions list. Not renaming as it is a required check. name: Check generated parser runs-on: ubuntu-latest + container: + image: quay.io/prometheus/golang-builder:1.26-base steps: - name: Checkout repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - name: Install Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - cache: false - go-version: 1.23.x - - name: Run goyacc and check for diff - run: make install-goyacc check-generated-parser + persist-credentials: false + - uses: prometheus/promci-setup@5af30ba8c199a91d6c04ebdc3c48e630e355f62d # v0.1.0 + with: + enable_npm: true + - run: make install-goyacc check-generated-parser + - run: make check-generated-promql-functions golangci: name: golangci-lint runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Install Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - go-version: 1.23.x + go-version: 1.26.x - name: Install snmp_exporter/generator dependencies run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' + - name: Get golangci-lint version + id: golangci-lint-version + run: echo "version=$(make print-golangci-lint-version)" >> $GITHUB_OUTPUT - name: Lint - uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0 + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 with: args: --verbose - # Make sure to sync this with Makefile.common and scripts/golangci-lint.yml. - version: v1.60.2 + version: ${{ steps.golangci-lint-version.outputs.version }} + - name: Lint with slicelabels + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 + with: + args: --verbose --build-tags=slicelabels + version: ${{ steps.golangci-lint-version.outputs.version }} + - name: Lint with dedupelabels + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 + with: + args: --verbose --build-tags=dedupelabels + version: ${{ steps.golangci-lint-version.outputs.version }} + - name: Lint in compliance + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 + with: + args: --verbose + working-directory: compliance + version: ${{ steps.golangci-lint-version.outputs.version }} + - name: Lint in documentation/examples/remote_storage + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 + with: + args: --verbose + working-directory: documentation/examples/remote_storage + version: ${{ steps.golangci-lint-version.outputs.version }} fuzzing: uses: ./.github/workflows/fuzzing.yml if: github.event_name == 'pull_request' codeql: uses: ./.github/workflows/codeql-analysis.yml + permissions: + contents: read + security-events: write publish_main: name: Publish main branch artifacts runs-on: ubuntu-latest + permissions: + packages: write needs: [test_ui, test_go, test_go_more, test_go_oldest, test_windows, golangci, codeql, build_all] if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 - - uses: ./.github/promci/actions/publish_main + - uses: prometheus/promci/publish_main@c7d527128ffb38ad3d7934795da35b252c7d51dd # v0.8.3 with: docker_hub_login: ${{ secrets.docker_hub_login }} docker_hub_password: ${{ secrets.docker_hub_password }} + ghcr_io_password: ${{ github.token }} quay_io_login: ${{ secrets.quay_io_login }} quay_io_password: ${{ secrets.quay_io_password }} publish_release: name: Publish release artefacts runs-on: ubuntu-latest + permissions: + contents: write + packages: write needs: [test_ui, test_go, test_go_more, test_go_oldest, test_windows, golangci, codeql, build_all] - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.') + if: | + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')) + || + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.')) steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 - - uses: ./.github/promci/actions/publish_release + - uses: prometheus/promci/publish_release@c7d527128ffb38ad3d7934795da35b252c7d51dd # v0.8.3 with: docker_hub_login: ${{ secrets.docker_hub_login }} docker_hub_password: ${{ secrets.docker_hub_password }} + ghcr_io_password: ${{ github.token }} quay_io_login: ${{ secrets.quay_io_login }} quay_io_password: ${{ secrets.quay_io_password }} - github_token: ${{ secrets.PROMBOT_GITHUB_TOKEN }} + github_token: ${{ github.token }} publish_ui_release: name: Publish UI on npm Registry runs-on: ubuntu-latest needs: [test_ui, codeql] + permissions: + contents: read + # Required for npm trusted publishing via OIDC. + id-token: write steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - uses: prometheus/promci@45166329da36d74895901808f1c8c97efafc7f84 # v0.3.0 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Install nodejs - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version-file: "web/ui/.nvmrc" registry-url: "https://registry.npmjs.org" - - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + version: 11 + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/.local/share/pnpm/store + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | - ${{ runner.os }}-node- + ${{ runner.os }}-pnpm- - name: Check libraries version - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.') - run: ./scripts/ui_release.sh --check-package "$(echo ${{ github.ref_name }}|sed s/v2/v0/)" + if: | + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')) + || + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.')) + run: ./scripts/ui_release.sh --check-package "$(./scripts/get_module_version.sh ${GH_REF_NAME})" + env: + GH_REF_NAME: ${{ github.ref_name }} - name: build run: make assets - name: Copy files before publishing libs run: ./scripts/ui_release.sh --copy - name: Publish dry-run libraries - if: "!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))" + if: | + !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')) + && + !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.')) run: ./scripts/ui_release.sh --publish dry-run - name: Publish libraries - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.') + if: | + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')) + || + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.')) run: ./scripts/ui_release.sh --publish - env: - # The setup-node action writes an .npmrc file with this env variable - # as the placeholder for the auth token - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 89aa2ba29b1..51c08d879e9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -6,15 +6,14 @@ on: schedule: - cron: "26 14 * * 1" -permissions: - contents: read - security-events: write +permissions: {} jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: + contents: read security-events: write strategy: @@ -24,15 +23,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 + uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 + uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 + uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 diff --git a/.github/workflows/container_description.yml b/.github/workflows/container_description.yml index 8ddbc34aebb..3bb36ccf43d 100644 --- a/.github/workflows/container_description.yml +++ b/.github/workflows/container_description.yml @@ -18,7 +18,9 @@ jobs: if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. steps: - name: git checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Set docker hub repo name run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV - name: Push README to Dockerhub @@ -40,7 +42,9 @@ jobs: if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. steps: - name: git checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Set quay.io org name run: echo "DOCKER_REPO=$(echo quay.io/${GITHUB_REPOSITORY_OWNER} | tr -d '-')" >> $GITHUB_ENV - name: Set quay.io repo name diff --git a/.github/workflows/funcbench.yml b/.github/workflows/funcbench.yml deleted file mode 100644 index 8959f82142e..00000000000 --- a/.github/workflows/funcbench.yml +++ /dev/null @@ -1,61 +0,0 @@ -on: - repository_dispatch: - types: [funcbench_start] -name: Funcbench Workflow -permissions: - contents: read - -jobs: - run_funcbench: - name: Running funcbench - if: github.event.action == 'funcbench_start' - runs-on: ubuntu-latest - env: - AUTH_FILE: ${{ secrets.TEST_INFRA_PROVIDER_AUTH }} - BRANCH: ${{ github.event.client_payload.BRANCH }} - BENCH_FUNC_REGEX: ${{ github.event.client_payload.BENCH_FUNC_REGEX }} - PACKAGE_PATH: ${{ github.event.client_payload.PACKAGE_PATH }} - GITHUB_TOKEN: ${{ secrets.PROMBOT_GITHUB_TOKEN }} - GITHUB_ORG: prometheus - GITHUB_REPO: prometheus - GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }} - GKE_PROJECT_ID: macro-mile-203600 - PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }} - PROVIDER: gke - ZONE: europe-west3-a - steps: - - name: Update status to pending - run: >- - curl -i -X POST - -H "Authorization: Bearer $GITHUB_TOKEN" - -H "Content-Type: application/json" - --data '{"state":"pending","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' - "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA" - - name: Prepare nodepool - uses: docker://prominfra/funcbench:master - with: - entrypoint: "docker_entrypoint" - args: make deploy - - name: Delete all resources - if: always() - uses: docker://prominfra/funcbench:master - with: - entrypoint: "docker_entrypoint" - args: make clean - - name: Update status to failure - if: failure() - run: >- - curl -i -X POST - -H "Authorization: Bearer $GITHUB_TOKEN" - -H "Content-Type: application/json" - --data '{"state":"failure","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' - "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA" - - name: Update status to success - if: success() - run: >- - curl -i -X POST - -H "Authorization: Bearer $GITHUB_TOKEN" - -H "Content-Type: application/json" - --data '{"state":"success","context":"funcbench-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' - "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA" diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index f3953cb2a48..856e01e35cb 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -1,28 +1,49 @@ -name: CIFuzz +name: fuzzing on: workflow_call: permissions: contents: read jobs: - Fuzzing: + fuzzing: + name: Run Go Fuzz Tests runs-on: ubuntu-latest + strategy: + matrix: + fuzz_test: [[FuzzParseMetricText, FuzzParseOpenMetric], [FuzzParseMetricSelector, FuzzParseExpr], [FuzzXORChunk, FuzzXOR2Chunk], [FuzzParseProtobuf]] steps: - - name: Build Fuzzers - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - oss-fuzz-project-name: "prometheus" - dry-run: false - - name: Run Fuzzers - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + persist-credentials: false + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - oss-fuzz-project-name: "prometheus" - fuzz-seconds: 600 - dry-run: false - - name: Upload Crash - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 - if: failure() && steps.build.outcome == 'success' + go-version: 1.26.x + - name: Run Fuzzing + run: | + for fuzz_test in ${{ join(matrix.fuzz_test, ' ') }}; do + go test -fuzz="${fuzz_test}$" -fuzztime=4m ./util/fuzzing + done + id: fuzz + - name: Upload Crash Artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() with: - name: artifacts - path: ./out/artifacts + name: fuzz-artifacts-${{ join(matrix.fuzz_test, '-') }} + path: util/fuzzing/testdata/fuzz/ + fuzzing_status: + # This status check aggregates the individual matrix jobs of the fuzzing + # step into a final status. Fails if a single matrix job fails, succeeds if + # all matrix jobs succeed. + name: Fuzzing + runs-on: ubuntu-latest + needs: [fuzzing] + if: always() + steps: + - name: Successful fuzzing + if: ${{ !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled')) }} + run: exit 0 + - name: Failing or cancelled fuzzing + if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + run: exit 1 diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml new file mode 100644 index 00000000000..de9ea09c2a2 --- /dev/null +++ b/.github/workflows/go-test.yml @@ -0,0 +1,26 @@ +name: go-test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# Lifted from https://github.com/airbnb/prometheus/blob/main/.circleci/config.yml +jobs: + test: + name: go test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: prometheus/setup-environment + run: make promu + - name: run make + run: make GO_ONLY=1 test + env: + GOGC: 20 + GOOPTS: "-p 2" + GOMAXPROCS: "2" + GO111MODULE: "on" + - name: test tsdb + run: go test ./tsdb/ -test.tsdb-isolation=false diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml new file mode 100644 index 00000000000..621476dec41 --- /dev/null +++ b/.github/workflows/govulncheck.yml @@ -0,0 +1,30 @@ +--- +name: govulncheck +on: + pull_request: + paths: + - VERSION + - .github/workflows/govulncheck.yml + push: + branches: + - main + - master + schedule: + - cron: '33 2 * * *' + +permissions: + contents: read + +jobs: + govulncheck: + runs-on: ubuntu-latest + name: Run govulncheck + steps: + - name: Install snmp_exporter/generator dependencies + id: snmp-deps + run: sudo apt-get update && sudo apt-get -y install libsnmp-dev + if: github.repository == 'prometheus/snmp_exporter' + - id: govulncheck + uses: golang/govulncheck-action@3fa7bd9cee2cfdf3499a8803b226e43de7b7cdb4 # master + env: + GOOS: ${{ contains(github.repository, 'windows_exporter') && 'windows' || '' }} diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index e7e813e3b64..de630953056 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'prometheus' steps: - - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5.0.1 + - uses: dessant/lock-threads@89ae32b08ed1a541efecbab17912962a5e38981c # v6.0.2 with: process-only: 'issues' issue-inactive-days: '180' diff --git a/.github/workflows/prombench.yml b/.github/workflows/prombench.yml index 6ee172662bf..34bea41d546 100644 --- a/.github/workflows/prombench.yml +++ b/.github/workflows/prombench.yml @@ -2,6 +2,8 @@ on: repository_dispatch: types: [prombench_start, prombench_restart, prombench_stop] name: Prombench Workflow +permissions: + contents: read env: AUTH_FILE: ${{ secrets.TEST_INFRA_PROVIDER_AUTH }} CLUSTER_NAME: test-infra @@ -15,11 +17,15 @@ env: PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }} PROVIDER: gke RELEASE: ${{ github.event.client_payload.RELEASE }} + BENCHMARK_VERSION: ${{ github.event.client_payload.BENCHMARK_VERSION }} + BENCHMARK_DIRECTORY: ${{ github.event.client_payload.BENCHMARK_DIRECTORY }} ZONE: europe-west3-a jobs: benchmark_start: name: Benchmark Start if: github.event.action == 'prombench_start' + permissions: + statuses: write runs-on: ubuntu-latest steps: - name: Update status to pending @@ -34,8 +40,8 @@ jobs: uses: docker://prominfra/prombench:master with: args: >- - until make all_nodes_deleted; do echo "waiting for nodepools to be deleted"; sleep 10; done; make deploy; + until make all_nodes_running; do echo "waiting for nodepools to be created"; sleep 10; done; - name: Update status to failure if: failure() run: >- @@ -55,6 +61,8 @@ jobs: benchmark_cancel: name: Benchmark Cancel if: github.event.action == 'prombench_stop' + permissions: + statuses: write runs-on: ubuntu-latest steps: - name: Update status to pending @@ -69,8 +77,8 @@ jobs: uses: docker://prominfra/prombench:master with: args: >- - until make all_nodes_running; do echo "waiting for nodepools to be created"; sleep 10; done; make clean; + until make all_nodes_deleted; do echo "waiting for nodepools to be deleted"; sleep 10; done; - name: Update status to failure if: failure() run: >- @@ -90,6 +98,8 @@ jobs: benchmark_restart: name: Benchmark Restart if: github.event.action == 'prombench_restart' + permissions: + statuses: write runs-on: ubuntu-latest steps: - name: Update status to pending @@ -104,10 +114,10 @@ jobs: uses: docker://prominfra/prombench:master with: args: >- - until make all_nodes_running; do echo "waiting for nodepools to be created"; sleep 10; done; make clean; until make all_nodes_deleted; do echo "waiting for nodepools to be deleted"; sleep 10; done; make deploy; + until make all_nodes_running; do echo "waiting for nodepools to be created"; sleep 10; done; - name: Update status to failure if: failure() run: >- diff --git a/.github/workflows/repo_sync.yml b/.github/workflows/repo_sync.yml index 537e9abd84d..9395f2d036f 100644 --- a/.github/workflows/repo_sync.yml +++ b/.github/workflows/repo_sync.yml @@ -3,6 +3,7 @@ name: Sync repo files on: schedule: - cron: '44 17 * * *' + workflow_dispatch: {} permissions: contents: read @@ -13,7 +14,11 @@ jobs: container: image: quay.io/prometheus/golang-builder steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - run: ./scripts/sync_repo_files.sh env: - GITHUB_TOKEN: ${{ secrets.PROMBOT_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.PROMBOT_REPOSYNC_TOKEN }} + GITHUB_TOKEN_PROMETHEUS: ${{ secrets.PROMBOT_REPOSYNC_TOKEN_PROMETHEUS }} + GITHUB_TOKEN_PROMETHEUS_COMMUNITY: ${{ secrets.PROMBOT_REPOSYNC_TOKEN_PROMETHEUS_COMMUNITY }} diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 82cccb2bc13..bfaad919c92 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -21,12 +21,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # tag=v4.1.6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # tag=v2.4.0 + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # tag=v2.4.3 with: results_file: results.sarif results_format: sarif @@ -37,7 +37,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # tag=v4.3.4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: SARIF file path: results.sarif @@ -45,6 +45,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # tag=v3.26.6 + uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 with: sarif_file: results.sarif diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d71bcbc9d83..74d037f8f17 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -11,7 +11,7 @@ jobs: if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. runs-on: ubuntu-latest steps: - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # opt out of defaults to avoid marking issues as stale and closing them diff --git a/.gitignore b/.gitignore index e85d766b095..cc4cf38e75b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,12 +22,14 @@ benchmark.txt /documentation/examples/remote_storage/example_write_adapter/example_write_adapter npm_licenses.tar.bz2 -/web/ui/static/react +/web/ui/static /vendor /.build +/go.work.sum /**/node_modules +.pnpm-store # Ignore parser debug y.output diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile deleted file mode 100644 index 2370ec5f5c2..00000000000 --- a/.gitpod.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM gitpod/workspace-full - -# Set Node.js version as an environment variable. -ENV CUSTOM_NODE_VERSION=16 - -# Install and use the specified Node.js version via nvm. -RUN bash -c ". .nvm/nvm.sh && nvm install ${CUSTOM_NODE_VERSION} && nvm use ${CUSTOM_NODE_VERSION} && nvm alias default ${CUSTOM_NODE_VERSION}" - -# Ensure nvm uses the default Node.js version in all new shells. -RUN echo "nvm use default &>/dev/null" >> ~/.bashrc.d/51-nvm-fix - -# Remove any existing Go installation in $HOME path. -RUN rm -rf $HOME/go $HOME/go-packages - -# Export go environment variables. -RUN echo "export GOPATH=/workspace/go" >> ~/.bashrc.d/300-go && \ - echo "export GOBIN=\$GOPATH/bin" >> ~/.bashrc.d/300-go && \ - echo "export GOROOT=${HOME}/go" >> ~/.bashrc.d/300-go && \ - echo "export PATH=\$GOROOT/bin:\$GOBIN:\$PATH" >> ~/.bashrc - -# Reload the environment variables to ensure go environment variables are -# available in subsequent commands. -RUN bash -c "source ~/.bashrc && source ~/.bashrc.d/300-go" - -# Fetch the Go version dynamically from the Prometheus go.mod file and Install Go in $HOME path. -RUN export CUSTOM_GO_VERSION=$(curl -sSL "https://raw.githubusercontent.com/prometheus/prometheus/main/go.mod" | awk '/^go/{print $2".0"}') && \ - curl -fsSL "https://dl.google.com/go/go${CUSTOM_GO_VERSION}.linux-amd64.tar.gz" | \ - tar -xz -C $HOME - -# Fetch the goyacc parser version dynamically from the Prometheus Makefile -# and install it globally in $GOBIN path. -RUN GOYACC_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/prometheus/prometheus/main/Makefile" | awk -F'=' '/GOYACC_VERSION \?=/{gsub(/ /, "", $2); print $2}') && \ - go install "golang.org/x/tools/cmd/goyacc@${GOYACC_VERSION}" diff --git a/.gitpod.yml b/.gitpod.yml deleted file mode 100644 index bb38f6d2c4b..00000000000 --- a/.gitpod.yml +++ /dev/null @@ -1,20 +0,0 @@ -image: - file: .gitpod.Dockerfile -tasks: - - init: - make build - command: | - gp sync-done build - ./prometheus --config.file=documentation/examples/prometheus.yml - - command: | - cd web/ui/ - gp sync-await build - unset BROWSER - export DANGEROUSLY_DISABLE_HOST_CHECK=true - npm start - openMode: split-right -ports: - - port: 3000 - onOpen: open-preview - - port: 9090 - onOpen: ignore diff --git a/.golangci.yml b/.golangci.yml index 303cd33d8b0..b85ba104663 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,166 +1,243 @@ -run: - timeout: 15m +formatters: + enable: + - gci + - gofumpt + - goimports + settings: + gci: + sections: + - standard + - default + - prefix(github.com/prometheus/prometheus) + gofumpt: + extra-rules: true + goimports: + local-prefixes: + - github.com/prometheus/prometheus -output: - sort-results: true +issues: + max-issues-per-linter: 0 + max-same-issues: 0 linters: + # Keep this list sorted alphabetically enable: - depguard - errorlint + - exptostd + - fatcontext - gocritic - godot - - gofumpt - - goimports + - govet + - loggercheck - misspell - - nolintlint + - modernize + - nilnesserr + # TODO(bwplotka): Enable once https://github.com/golangci/golangci-lint/issues/3228 is fixed. + # - nolintlint - perfsprint - predeclared - revive + - sloglint + - staticcheck - testifylint - unconvert - unused - usestdlibvars - whitespace - - loggercheck -issues: - max-issues-per-linter: 0 - max-same-issues: 0 - # The default exclusions are too aggressive. For one, they - # essentially disable any linting on doc comments. We disable - # default exclusions here and add exclusions fitting our codebase - # further down. - exclude-use-default: false - exclude-files: - # Skip autogenerated files. - - ^.*\.(pb|y)\.go$ - exclude-dirs: - # Copied it from a different source. - - storage/remote/otlptranslator/prometheusremotewrite - - storage/remote/otlptranslator/prometheus - exclude-rules: - - linters: - - errcheck - # Taken from the default exclusions (that are otherwise disabled above). - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - - linters: - - govet - # We use many Seek methods that do not follow the usual pattern. - text: "stdmethods: method Seek.* should have signature Seek" - - linters: - - revive - # We have stopped at some point to write doc comments on exported symbols. - # TODO(beorn7): Maybe we should enforce this again? There are ~500 offenders right now. - text: exported (.+) should have comment( \(or a comment on this block\))? or be unexported - - linters: - - gocritic - text: "appendAssign" - - path: _test.go - linters: - - errcheck - - path: "tsdb/head_wal.go" - linters: - - errorlint - - linters: - - godot - source: "^// ===" - - linters: - - perfsprint - text: "fmt.Sprintf can be replaced with string concatenation" -linters-settings: - depguard: - rules: - main: - deny: - - pkg: "sync/atomic" - desc: "Use go.uber.org/atomic instead of sync/atomic" - - pkg: "github.com/stretchr/testify/assert" - desc: "Use github.com/stretchr/testify/require instead of github.com/stretchr/testify/assert" - - pkg: "github.com/go-kit/kit/log" - desc: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log" - - pkg: "io/ioutil" - desc: "Use corresponding 'os' or 'io' functions instead." - - pkg: "regexp" - desc: "Use github.com/grafana/regexp instead of regexp" - - pkg: "github.com/pkg/errors" - desc: "Use 'errors' or 'fmt' instead of github.com/pkg/errors" - - pkg: "gzip" - desc: "Use github.com/klauspost/compress instead of gzip" - - pkg: "zlib" - desc: "Use github.com/klauspost/compress instead of zlib" - - pkg: "golang.org/x/exp/slices" - desc: "Use 'slices' instead." - errcheck: - exclude-functions: - # Don't flag lines such as "io.Copy(io.Discard, resp.Body)". - - io.Copy - # The next two are used in HTTP handlers, any error is handled by the server itself. - - io.WriteString - - (net/http.ResponseWriter).Write - # No need to check for errors on server's shutdown. - - (*net/http.Server).Shutdown - # Never check for logger errors. - - (github.com/go-kit/log.Logger).Log - # Never check for rollback errors as Rollback() is called when a previous error was detected. - - (github.com/prometheus/prometheus/storage.Appender).Rollback - goimports: - local-prefixes: github.com/prometheus/prometheus - gofumpt: - extra-rules: true - perfsprint: - # Optimizes `fmt.Errorf`. - errorf: false - revive: - # By default, revive will enable only the linting rules that are named in the configuration file. - # So, it's needed to explicitly enable all required rules here. + exclusions: + paths: + # Skip autogenerated files. + - ^.*\.(l|pb|y)\.go$ rules: - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md - - name: blank-imports - - name: comment-spacings - - name: context-as-argument - arguments: - # Allow functions with test or bench signatures. - - allowTypesBefore: "*testing.T,testing.TB" - - name: context-keys-type - - name: dot-imports - # A lot of false positives: incorrectly identifies channel draining as "empty code block". - # See https://github.com/mgechev/revive/issues/386 - - name: empty-block - disabled: true - - name: error-naming - - name: error-return - - name: error-strings - - name: errorf - - name: exported - - name: increment-decrement - - name: indent-error-flow - - name: package-comments - # TODO(beorn7): Currently, we have a lot of missing package doc comments. Maybe we should have them. - disabled: true - - name: range - - name: receiver-naming - - name: redefines-builtin-id - - name: superfluous-else - - name: time-naming - - name: unexported-return - - name: unreachable-code - - name: unused-parameter - disabled: true - - name: var-declaration - - name: var-naming - testifylint: - disable: - - float-compare - - go-require - enable: - - bool-compare - - compares - - empty - - error-is-as - - error-nil - - expected-actual - - len - - require-error - - suite-dont-use-pkg - - suite-extra-assert-call + - linters: + - errcheck + # Taken from the default exclusions in v1. + text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked + - linters: + - govet + # We use many Seek methods that do not follow the usual pattern. + text: "stdmethods: method Seek.* should have signature Seek" + - linters: + - revive + # We have stopped at some point to write doc comments on exported symbols. + # TODO(beorn7): Maybe we should enforce this again? There are ~500 offenders right now. + text: exported (.+) should have comment( \(or a comment on this block\))? or be unexported + - linters: + - errcheck + path: _test.go + - linters: + - errorlint + path: "tsdb/head_wal.go" + - linters: + - godot + source: "^// ===" + - linters: + - staticcheck + text: '(labels.MetricName|v1\.(Endpoints|EndpointSubset|EndpointPort|EndpointAddress)) is deprecated: .*' + warn-unused: true + settings: + depguard: + rules: + main: + deny: + - pkg: "sync/atomic" + desc: "Use go.uber.org/atomic instead of sync/atomic" + - pkg: "github.com/stretchr/testify/assert" + desc: "Use github.com/stretchr/testify/require instead of github.com/stretchr/testify/assert" + - pkg: "github.com/go-kit/kit/log" + desc: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log" + - pkg: "io/ioutil" + desc: "Use corresponding 'os' or 'io' functions instead." + - pkg: "regexp" + desc: "Use github.com/grafana/regexp instead of regexp" + - pkg: "github.com/pkg/errors" + desc: "Use 'errors' or 'fmt' instead of github.com/pkg/errors" + - pkg: "gzip" + desc: "Use github.com/klauspost/compress instead of gzip" + - pkg: "zlib" + desc: "Use github.com/klauspost/compress instead of zlib" + - pkg: "golang.org/x/exp/slices" + desc: "Use 'slices' instead." + - pkg: "gopkg.in/yaml.v2" + desc: "Use go.yaml.in/yaml/v2 instead of gopkg.in/yaml.v2" + - pkg: "gopkg.in/yaml.v3" + desc: "Use go.yaml.in/yaml/v3 instead of gopkg.in/yaml.v3" + errcheck: + exclude-functions: + # Don't flag lines such as "io.Copy(io.Discard, resp.Body)". + - io.Copy + # The next two are used in HTTP handlers, any error is handled by the server itself. + - io.WriteString + - (net/http.ResponseWriter).Write + # No need to check for errors on server's shutdown. + - (*net/http.Server).Shutdown + # Never check for rollback errors as Rollback() is called when a previous error was detected. + - (github.com/prometheus/prometheus/storage.Appender).Rollback + gocritic: + enable-all: true + disabled-checks: + - appendAssign + - appendCombine + - badLock + - boolExprSimplify + - commentedOutCode + - deferInLoop + - docStub + - equalFold + - evalOrder + - exposedSyncMutex + - hexLiteral + - hugeParam + - importShadow + - nestingReduce + - ptrToRefParam + - preferStringWriter + - rangeValCopy + - redundantSprint + - regexpSimplify + - returnAfterHttpError + - sloppyReassign + - sprintfQuotedString + - stringsCompare + - todoCommentWithoutDetail + - tooManyResultsChecker + - typeUnparen + - truncateCmp + - typeDefFirst + - unlabelStmt + - unnamedResult + - unnecessaryBlock + - unnecessaryDefer + - whyNoLint + govet: + disable: + - shadow + - fieldalignment + enable-all: true + modernize: + disable: + # Suggest replacing omitempty with omitzero for struct fields. + # Disable this check for now since it introduces too many changes in our existing codebase. + # See https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/modernize#hdr-Analyzer_omitzero for more details. + - omitzero + # Disable newexpr check for now since it introduces too many changes in our existing codebase. + # To be re-enabled as a part of https://github.com/prometheus/prometheus/issues/18066. + - newexpr + perfsprint: + # Optimizes even if it requires an int or uint type cast. + int-conversion: true + # Optimizes into `err.Error()` even if it is only equivalent for non-nil errors. + err-error: true + # Optimizes `fmt.Errorf`. + errorf: true + # Optimizes `fmt.Sprintf` with only one argument. + sprintf1: true + # Optimizes into strings concatenation. + strconcat: false + # Disable optimization of concat loop. + concat-loop: false + revive: + # By default, revive will enable only the linting rules that are named in the configuration file. + # So, it's needed to explicitly enable all required rules here. + rules: + # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md + - name: blank-imports + - name: comment-spacings + - name: context-as-argument + arguments: + # Allow functions with test or bench signatures. + - allowTypesBefore: '*testing.T,testing.TB' + - name: context-keys-type + - name: dot-imports + - name: early-return + arguments: + - "preserveScope" + # A lot of false positives: incorrectly identifies channel draining as "empty code block". + # See https://github.com/mgechev/revive/issues/386 + - name: empty-block + disabled: true + - name: error-naming + - name: error-return + - name: error-strings + - name: errorf + - name: exported + - name: increment-decrement + - name: indent-error-flow + arguments: + - "preserveScope" + - name: package-comments + # TODO(beorn7): Currently, we have a lot of missing package doc comments. Maybe we should have them. + disabled: true + - name: range + - name: receiver-naming + - name: redefines-builtin-id + - name: superfluous-else + arguments: + - "preserveScope" + - name: time-naming + - name: unexported-return + - name: unreachable-code + - name: unused-parameter + - name: unused-receiver + - name: var-declaration + - name: var-naming + # TODO(SuperQ): See: https://github.com/prometheus/prometheus/issues/17766 + arguments: + - [] + - [] + - - skip-package-name-checks: true + testifylint: + disable: + - float-compare + - go-require + enable-all: true + +output: + show-stats: false + +run: + timeout: 15m + +version: "2" diff --git a/.promu.yml b/.promu.yml index 699a7fa5859..15b2f0a4977 100644 --- a/.promu.yml +++ b/.promu.yml @@ -1,7 +1,7 @@ go: # Whenever the Go version is updated here, # .github/workflows should also be updated. - version: 1.23 + version: 1.26 repository: path: github.com/prometheus/prometheus build: @@ -14,10 +14,8 @@ build: all: - netgo - builtinassets - - stringlabels windows: - builtinassets - - stringlabels ldflags: | -X github.com/prometheus/common/version.Version={{.Version}} -X github.com/prometheus/common/version.Revision={{.Revision}} @@ -28,14 +26,12 @@ tarball: # Whenever there are new files to include in the tarball, # remember to make sure the new files will be generated after `make build`. files: - - consoles - - console_libraries - documentation/examples/prometheus.yml - LICENSE - NOTICE - - npm_licenses.tar.bz2 crossbuild: platforms: + - aix - darwin - dragonfly - freebsd diff --git a/.yamllint b/.yamllint index 1859cb624bd..b329f464fb3 100644 --- a/.yamllint +++ b/.yamllint @@ -1,7 +1,8 @@ --- extends: default ignore: | - ui/react-app/node_modules + **/node_modules + web/api/v1/testdata/openapi_*_golden.yaml rules: braces: diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..bdc352b8f09 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,148 @@ +# Agents Guide for Prometheus + +This document captures patterns and preferences observed from maintainer reviews +of recently merged pull requests. Use it to align your contributions with what +maintainers expect. + +--- + +## PR Title Format + +Titles must follow `area: short description`, using a prefix that identifies the +subsystem. Examples from merged PRs: + +``` +tsdb/wlog: optimize WAL watcher reads +fix(PromQL): do not skip histogram buckets when trimming +feat(agent): fix ST append; add compliance RW sender test +chore: fix emptyStringTest issues from gocritic +ci: add statuses write permission to prombench workflow +docs: clarify that `lookback_delta` query parameter takes either a duration or number of seconds +``` + +Common area prefixes: `tsdb`, `tsdb/wlog`, `promql`, `discovery/`, `agent`, +`alerting`, `textparse`, `ui`, `build`, `ci`, `docs`, `chore`. + +For performance work, append `[PERF]` to the area segment or use the `perf(area):` +convention. + +--- + +## Commits + +- Each commit must compile and pass tests independently, except when one commit adds a test to expose a bug and then the next commit fixes the bug. +- Keep commits small and focused. Do not bundle unrelated changes in one commit. +- Sign off every commit with `git commit -s` to satisfy the DCO requirement. +- Do not include unrelated local changes in the PR. + +--- + +## Release Notes Block + +Every PR must include a `release-notes` fenced code block in the description. +If there is no user-facing change, write `NONE`: + +```` +```release-notes +NONE +``` +```` + +Otherwise use one of these prefixes, matching the CHANGELOG style: + +``` +[FEATURE] new capability +[ENHANCEMENT] improvement to existing behaviour +[PERF] performance improvement +[BUGFIX] bug fix +[SECURITY] security fix +[CHANGE] breaking or behavioural change +``` + +Example: +```` +```release-notes +[BUGFIX] PromQL: Do not skip histogram buckets in queries where histogram trimming is used. +``` +```` + +--- + +## Tests + +- Bug fixes require a test that reproduces the bug. +- New behaviour or exported API changes require unit or e2e tests. +- Tests should attempt to mirror realistic data and/or behaviour. +- Use only exported APIs in tests where possible — this keeps tests closer to + real library usage and simplifies review. + +--- + +## Performance Work + +Maintainers take performance seriously. For any PERF PR: + +- Performance improvements require a benchmark that demonstrates the improvement. +- Run benchmarks before and after the change using `go test -count=6 -benchmem -bench ` +- Provide benchmark numbers in the PR body using `benchstat` output. +- If a subset of benchmark results show a regression, address this or explain why the case is not important. +- Reuse allocations in hot paths where possible (slices, buffers). +- When reusing buffers passed to interfaces, document that callers must copy + the contents and must not retain references. +- Link to supporting analysis (Google Doc, issue, etc.) for complex changes. + +--- + +## Code Style + +- Follow [Go Code Review Comments](https://go.dev/wiki/CodeReviewComments) + and the formatting/style section of + [Go: Best Practices for Production Environments](https://peter.bourgon.org/go-in-production/#formatting-and-style). +- State your assumptions. +- Interface contracts: when ownership or lifetime semantics (e.g. buffer reuse) are important, + document it at the interface definition, not just in the implementation. +- All exposed objects must have a doc comment. +- All comments must start with a capital letter and end with a full stop. +- Run `make lint` before submitting. The project uses `golangci-lint` including + `gocritic` rules such as `emptyStringTest` — fix linter findings rather than + suppressing them with `//nolint` unless there is a clear false-positive. +- Use `//nolint:linter1[,linter2,...]` sparingly; prefer fixing the code. + +--- + +## Linking Issues + +Use GitHub closing keywords in the PR body so the linked issue closes +automatically on merge: + +``` +Fixes #18243 +``` + +--- + +## Scope Discipline + +- Do not include unrelated changes in a PR; make a separate PR instead. +- If a refactor is necessary to make a change, do those in separate commits. +- If a PR is large, split it into preparatory and follow-up PRs and reference + them with "Part of #NNNN" or "Depends on #NNNN". + +--- + +## Documentation Changes + +- Docs PRs are welcome for clarifying ambiguous parameter descriptions, + fixing Markdown formatting, and keeping the OpenAPI spec consistent with + the implementation. +- When changing documented behaviour, update any relevant text in the docs/ directory. + Check whether the OpenAPI spec also needs updating. + +--- + +## CI / Workflow Changes + +- Workflow files need the correct GitHub token permissions declared explicitly. + Missing permissions (e.g. `statuses: write`) cause silent 403 failures. + +--- diff --git a/CHANGELOG.md b/CHANGELOG.md index 19af3f460fc..98e0e9894f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,671 @@ # Changelog -## unreleased - -## 2.55.1 / 2024-01-04 +## 3.13.1 / 2026-07-10 + +- [BUGFIX] TSDB: Fix the head-chunk cache returning samples from the wrong chunk, or spurious not-found errors, to range queries after head-chunk truncation. #19134 + +## 3.13.0 / 2026-07-01 + +- [SECURITY] UI: Bump `sanitize-html` to fix a cross-site scripting vulnerability (CVE-2026-44990). #18697 +- [CHANGE] UI: Third-party npm dependency licenses are now embedded in the Prometheus binary and served at `/assets/third-party-licenses.txt`, replacing the `npm_licenses.tar.bz2` archive previously shipped in release tarballs and container images. #18997 +- [CHANGE] API: Use SHA-256 instead of SHA-1 to generate rule group pagination tokens. #18927 +- [CHANGE] HTTP clients: Credentials (Authorization header, basic auth, bearer token, OAuth2, configured headers) are no longer forwarded when following a redirect to a different host; affects scraping, remote read/write, alerting, and service discovery. Via prometheus/common v0.69.0 (CVE-2025-4673 CVE-2023-45289). #18949 +- [CHANGE] promtool: Relative file paths in the file passed to `--http.config.file` are now resolved relative to that config file's directory instead of its parent directory. Via prometheus/common v0.69.0. #18949 +- [CHANGE] PromQL: Rename the `min()` and `max()` duration-expression functions (experimental feature flag `experimental-duration-expr`) to `min_of()` and `max_of()` to avoid confusion with the `min` and `max` aggregate operators. #18687 +- [FEATURE] API: Add experimental search endpoints to search metric names, label names, and label values. #18573 +- [FEATURE] Discovery/AWS: Add ability to filter RDS instances. #18859 +- [FEATURE] PromQL: Add `min_of(a, b)` and `max_of(a, b)` scalar experimental functions, returning the smaller or larger of two scalar values. #18687 +- [FEATURE] PromQL: Add support for smoothed/anchored rate with native histograms. #18564 +- [FEATURE] PromQL: Expose per-query `samplesRead` (and `samplesReadPerStep` with `stats=all` and the `promql-per-step-stats` feature flag) in the query stats response, and add the `prometheus_engine_query_samples_read_total` engine counter. `samplesRead` reflects storage I/O distinct from `totalQueryableSamples`, which counts samples loaded into the evaluator (and so over-counts when a sample is reused across multiple range-vector windows). #18081 +- [FEATURE] Scrape: Add `__convert_classic_histograms_to_nhcb__` internal label to allow per-target override of `convert_classic_histograms_to_nhcb` scrape configuration via relabeling. #18840 +- [FEATURE] TSDB: Add `storage.tsdb.chunk_encoding.floats` configuration field to select float chunk encoding (`xor` or `xor2`) at runtime, independently of the `--enable-feature=xor2-encoding` flag. #18769 +- [FEATURE] remote_write: Add Certificate support for ingesting data into an Azure Monitor Workspace. #18217 +- [FEATURE] Scrape: Add `__always_scrape_classic_histograms__` and `__scrape_native_histograms__` internal labels to allow per-target override of the `always_scrape_classic_histograms` and `scrape_native_histograms` scrape configuration via relabeling. #18929 +- [ENHANCEMENT] Release: Container images are now also published to the GitHub Container Registry (ghcr.io). #18791 +- [ENHANCEMENT] PromQL: Prettify `fill_left(x) fill_right(x)` as `fill(x)` when both fill values are equal. #18851 +- [ENHANCEMENT] UI: Improve autocompletion after closing a function bracket. #18894 +- [PERF] Labels: Add case-insensitive prefix matching to speed up evaluation of long case-insensitive regular expressions (up to ~2x faster). #18540 +- [PERF] TSDB: Reduce per-sample overhead in chunk population, speeding up affected queries by ~12-15% in benchmarks. #18699 +- [PERF] TSDB: Eliminate unnecessary heap allocations in the V2 histogram WAL decoder, reducing allocations by up to 50% and memory by up to 10% for deployments using native histograms with created-timestamp storage enabled (`--enable-feature=created-timestamp-zero-ingestion`). #18813 +- [BUGFIX] Discovery/AWS: Fix failure when processing an AWS RDS cluster without instances. #18845 +- [BUGFIX] Fix race condition in initTime that could cause ErrOutOfBounds. #18629 +- [BUGFIX] PromQL: A range query whose `end` was not aligned to `step` caused subqueries inside it to evaluate past the parent's last actual step, inflating `peakSamples` in the query stats and against the `query.max-samples` limit, and wasting storage I/O reading samples that were never used in the result. #18081 +- [BUGFIX] PromQL: A range query containing an at-modifier-unsafe function over a range-vector with an `@` modifier (e.g. `predict_linear(metric[60s] @ T, X)`) silently under-counted `totalQueryableSamples` for steps after step 0. #18081 +- [BUGFIX] PromQL: Fix `fill_left`/`fill_right` producing missing samples in range queries when using `group_left`/`group_right`. #18850 +- [BUGFIX] PromQL: Fix for resets() and changes() in anchored range extenders with histograms. #18906 +- [BUGFIX] PromQL: Fix panic on `1[5m] smoothed` and similar expressions when extended range selectors are enabled. #18764 +- [BUGFIX] PromQL: Fix panic when a `smoothed` instant vector selector produces no samples for a series. #18943 +- [BUGFIX] PromQL: Fix panic when using a parenthesised plain number as an offset (e.g. `foo offset -(5)`). #18768 +- [BUGFIX] promtool: Fix panic when parsing exposition text containing empty braces `{}`. Via prometheus/common v0.69.0. #18949 +- [BUGFIX] Promtool: Fix `check healthy` and `check ready` when `--url` ends with a trailing slash. #18854 +- [BUGFIX] Rules: Close PromQL query after each rule evaluation to ensure resources are released. #18733 +- [BUGFIX] Scaleway SD: Resolve VPC/IPAM-only instances that have no legacy `private_ip` or `public_ip` field, but do have private NICs attached. #18772 +- [BUGFIX] TSDB: Do not leak head series when an integer histogram append is rejected (e.g. out-of-order). #18838 +- [BUGFIX] UI: Escape label values offered by PromQL autocomplete. #18658 +- [BUGFIX] TSDB: Fix chunk snapshot encoding for EncXOR2 chunks, preventing corruption on TSDB restart when EncXOR2-encoded series were present. #18739 +- [BUGFIX] TSDB: Store a millisecond timestamp (not a WAL segment number) in walExpiries when a series is evicted via CompactStaleHead/CompactSelectedSeries, so the series's label record is correctly retained in the next WAL checkpoint and replays cleanly. #18847 +- [BUGFIX] TSDB: Prevent loss of samples at the chunk-range boundary when CompactSelectedSeries (and CompactStaleHead) evict the series — the per-slice compaction loop now runs one more iteration so the boundary timestamp is captured in a block before the in-memory copy is removed. #18849 + +## 3.12.0 / 2026-05-28 + +- [SECURITY] Remote-write: Reject snappy-compressed requests whose declared decoded length exceeds the 32MB. Thanks to @hibrian827 for reporting it. #18642 +- [SECURITY] STACKIT SD: Fix secrets being exposed in plaintext via `/-/config` endpoint. Thanks to @August829 and @Phaxma for reporting. GHSA-39j6-789q-qxvh #18649 +- [CHANGE] TSDB/Agent: Adds Start Timestamp field to all WAL Histogram samples in memory; used `st-storage` flag is enabled. #18221 +- [FEATURE] API: Add `/api/v1/status/self_metrics` endpoint returning the current state of the Prometheus server's own metrics about itself as JSON. #18411 +- [FEATURE] Discovery: Add DigitalOcean Managed Databases service discovery #18287 +- [FEATURE] Prometheus: Add support for the aix/ppc64 compilation target #18321 +- [FEATURE] Discovery: Add Outscale VM service discovery (`outscale_sd_configs`) for discovering scrape targets from the Outscale Cloud API. #18139 +- [FEATURE] PromQL: Emit a warning when `sort`, `sort_by_label` or `sort_by_label_desc` is used within range (matrix) queries, as these functions do not have effect in that context. #18498 +- [FEATURE] PromQL: Add `start()`, `end()`, `range()`, and `step()` experimental functions #17877 +- [FEATURE] PromQL: Update `resets()` function to consider start timestamp resets. Hidden behind `use-start-timestamps` feature flag. #18627 +- [FEATURE] Prometheus: Promote auto-reload-config as stable #18620 +- [FEATURE] TSDB/Agent: Add `CheckpointFromInMemorySeries` option to `agent.DB` that enables checkpoint based on in-memory series. #17948 +- [FEATURE] UI: Add a web interface for deleting time series and cleaning tombstones, accessible from the Status menu. #18390 +- [FEATURE] PromQL: Use start timestamps for `rate()`, `irate(), and `increase()` calculations, behind a feature flag `use-start-timestamps`. Doesn't work together with extended range selectors `anchored` and `smoothed`. #18344 +- [FEATURE] Scrape: Added a feature flag `st-synthesis` which synthesizes unknown STs for scraped cumulative metrics. Useful when Remote Writing 2.0 with delta or Otel-based backends. #18279 +- [FEATURE] promqltest: support `@st` annotation in `load` blocks to specify per-sample start timestamps. #18360 +- [ENHANCEMENT] API: reject concurrent fgprof profiles. #18651 +- [ENHANCEMENT] AWS SD: Add optional `external_id` field to ECS/MSK/RDS/Elasticache. #18579 +- [ENHANCEMENT] AWS SD: Add optional `external_id` field. #17171 +- [ENHANCEMENT] Discovery: Propagate SD target updates faster by introducing dynamic backoff interval instead of static 5s interval for throttling. #18187 +- [ENHANCEMENT] Promtool: Add `--header` flag to `query instant` command, matching existing `query range` behaviour. #18418 +- [ENHANCEMENT]: AWS SD: Allows EC2 service discovery to discover IPv6 addresses to communicate with target endpoints. The private IPv4 address remains the default when both IPv4 and IPv6 addresses are present. #16088 +- [PERF] TSDB: Make head chunk lookup in range queries constant time instead of quadratic time #18302 +- [PERF] TSDB: Skip entire stripes in mmapHeadChunks when no series need mmapping, reducing CPU utilization significantly at production-relevant scales. #18541 +- [PERF] TSDB: Skip clean series during periodic head chunk mmap using cached head chunk count #18272 +- [PERF] PromQL: Address FloatHistogram.KahanAdd performance regression on Go 1.26. #18568 +- [BUGFIX] PromQL: Fix `info()` function incorrectly handling negated `__name__` matchers #17932 +- [BUGFIX] API: Return duration expressions in `/parse_ast`. #18624 +- [BUGFIX] API: correctly document formats accepted for duration query request parameters (step, timeout and lookback delta) in OpenAPI spec #18305 +- [BUGFIX] Scrape: AppenderV2 now tracks staleness even when OOO/duplicate series errors happen similar to AppenderV1 #18567 +- [BUGFIX] Config: Validate remote_write queue_config fields at load time to prevent runtime panic and silent misconfiguration. #18209 +- [BUGFIX] Discovery/Consul: Add `health_filter` for Health API filtering, fixing breakage when using Catalog-only fields like `ServiceTags` in `filter`. #18479 #18499 +- [BUGFIX] OTLP: limit decompressed body size for gzip-encoded OTLP write requests. #18408 +- [BUGFIX] PromQL: Fix `smoothed` rate/increase returning zero instead of no result when all data falls strictly after the query range. #18523 +- [BUGFIX] PromQL: Fix metric name not being dropped when last_over_time or first_over_time is applied to subqueries containing name-dropping functions like abs(). #18409 +- [BUGFIX] PromQL: Fix missing warning when mixing exponential and custom-bucket histograms in stats queries. #18660 +- [BUGFIX] PromQL: Fix parsing of `range()` keyword in duration expressions such as `foo[5m+range()]`. #18623 +- [BUGFIX] PromQL: Fix smoothed vector selector returning no results in binary operations when the `@` modifier is used. #18531 +- [BUGFIX] PromQL: Reject NaN, infinite, and out-of-range duration expressions instead of silently producing an out-of-range time.Duration. #18639 +- [BUGFIX] Scrape: Fix panic when scraping malformed native histograms. #18414 +- [BUGFIX] Scrape: fix panic when scraping a target exposing a summary with no quantiles via the protobuf format. #18382 +- [BUGFIX] Scrape: fix scrape failure log file occasionally not applied after a configuration reload. #18421 +- [BUGFIX] TSDB: Allow retention percentage with new data path. #18628 +- [BUGFIX] TSDB: Preserve decimal precision in percentage-based retention #18374 +- [BUGFIX] TSDB: fix prometheus_tsdb_head_chunks going negative after WAL replay #18401 +- [BUGFIX] TSDB: panic with native histograms during query of overlapping chunks. #18692 +- [BUGFIX] Tracing: fix startup failure for insecure OTLP HTTP tracing #18469 +- [BUGFIX] UI: Escape label values offered by PromQL autocomplete. #18658 +- [BUGFIX] UI: Improve Y-axis tick label precision for graph values over small ranges. #18682 +- [BUGFIX] `prometheus_sd_refresh*` and `prometheus_sd_discovered_targets` metrics for specific scrape jobs are deleted when the scrape job is removed. #17614 +- [BUGFIX] Remote: fixed validation for received RW2 requests when parsing metadata unit symbols. This fixes a case when request would cause (recovered) handler panic. #18641 +- [BUGFIX] TSDB/Agent: fix race in agent appender where concurrent appends for the same label set could produce duplicate in-memory series and duplicate WAL records. #18292 +- [BUGFIX] Config: Update `--enable-feature` flag description and sort feature names. #18487 + +## 3.11.3 / 2026-04-27 + +This release fixes mutiple security issues. + +We would like to thank the following people for the responsible disclosures: +- Shadowbyte (4c1dr3aper) - Charlie Lewis for the Remote-Read snappy decode vulnerability. +- Brett Gervasoni for the AzureAD OAuth `client_secret` vulnerability. +- @iiihaiii and @Ngocnn97 for the Old UI XSS vulnerability. + +- [SECURITY] AzureAD remote write: Fix OAuth `client_secret` being exposed in plaintext via `/-/config` endpoint. GHSA-wg65-39gg-5wfj / CVE-2026-42151 #18590 +- [SECURITY] Remote-read: Reject snappy-compressed requests whose declared decoded length exceeds the decode limit. GHSA-8rm2-7qqf-34qm / CVE-2026-42154 #18584 +- [SECURITY] UI: Fix stored XSS via unescaped `le` label values in old UI heatmap chart tick labels. GHSA-fw8g-cg8f-9j28 #18588 + +## 3.11.2 / 2026-04-13 + +This release has a fix for a Stored XSS vulnerability that can be triggered via crafted metric names and label values in Prometheus web UI tooltips and metrics explorer. Thanks to Duc Anh Nguyen from TinyxLab for reporting it. + +- [SECURITY] UI: Fix stored XSS via unescaped metric names and labels. CVE-2026-40179. #18506 +- [ENHANCEMENT] Consul SD: Introduce `health_filter` field for Health API filtering. #18499 +- [BUGFIX] Consul SD: Fix filter parameter being incorrectly applied to the Health API. #18499 + +## 3.11.1 / 2026-04-07 + +- [BUGFIX] Tracing: Fix startup failure for OTLP HTTP tracing with `insecure: true`. #18469 + +## 3.11.0 / 2026-04-02 + +- [CHANGE] Hetzner SD: The `__meta_hetzner_datacenter` label is deprecated for the role `robot` but kept for backward compatibility, use the `__meta_hetzner_robot_datacenter` label instead. For the role `hcloud`, the label is deprecated and will stop working after the 1 July 2026. #17850 +- [CHANGE] Hetzner SD: The `__meta_hetzner_hcloud_datacenter_location` and `__meta_hetzner_hcloud_datacenter_location_network_zone` labels are deprecated, use the `__meta_hetzner_hcloud_location` and `__meta_hetzner_hcloud_location_network_zone` labels instead. #17850 +- [CHANGE] Promtool: Redirect debug output to stderr to avoid interfering with stdout-based tool output. #18346 +- [FEATURE] AWS SD: Add Elasticache Role. #18099 +- [FEATURE] AWS SD: Add RDS Role. #18206 +- [FEATURE] Azure SD: Add support for Azure Workload Identity authentication method. #17207 +- [FEATURE] Discovery: Introduce `prometheus_sd_last_update_timestamp_seconds` metric to track the last time a service discovery update was sent to consumers. #18194 +- [FEATURE] Kubernetes SD: Add support for node role selectors for pod roles. #18006 +- [FEATURE] Kubernetes SD: Introduce pod-based labels for deployment, cronjob, and job controller names: `__meta_kubernetes_pod_deployment_name`, `__meta_kubernetes_pod_cronjob_name` and `__meta_kubernetes_pod_job_name`, respectively. #17774 +- [FEATURE] PromQL: Add `/` operators for trimming observations from native histograms. #17904 +- [FEATURE] PromQL: Add experimental `histogram_quantiles` variadic function for computing multiple quantiles at once. #17285 +- [FEATURE] TSDB: Add `storage.tsdb.retention.percentage` configuration to configure the maximum percent of disk usable for TSDB storage. #18080 +- [FEATURE] TSDB: Add an experimental `fast-startup` feature flag that writes a `series_state.json` file to the WAL directory to track active series state across restarts. #18303 +- [FEATURE] TSDB: Add an experimental `st-storage` feature flag. When enabled, Prometheus stores ingested start timestamps (ST, previously called Created Timestamp) from scrape or OTLP in the TSDB and Agent WAL, and exposes them via Remote Write 2. #18062 +- [FEATURE] TSDB: Add an experimental `xor2-encoding` feature flag for the new TSDB block float sample chunk encoding that is optimized for scraped data and allows encoding start timestamps. #18062 +- [ENHANCEMENT] HTTP client: Add AWS `external_id` support for sigv4. #17916 +- [ENHANCEMENT] Kubernetes SD: Deduplicate deprecation warning logs from the Kubernetes API to reduce noise. #17829 +- [ENHANCEMENT] TSDB: Remove old temporary checkpoints when creating a Checkpoint. #17598 +- [ENHANCEMENT] UI: Add autocomplete support for experimental `first_over_time` and `ts_of_first_over_time` PromQL functions. #18318 +- [ENHANCEMENT] Vultr SD: Upgrade govultr library from v2 to v3 for continued security patches and maintenance. #18347 +- [PERF] PromQL: Improve performance and reduce heap allocations in joins (VectorBinop)/And/Or/Unless. #17159 +- [PERF] PromQL: Partially address performance regression in native histogram aggregations due to using `KahanAdd`. #18252 +- [PERF] Remote write: Optimize WAL watching used for RW sending to reuse internal buffers. #18250 +- [PERF] TSDB: Optimize LabelValues intersection performance for matchers. #18069 +- [PERF] UI: Skip restacking on hover in stacked series charts. #18230 +- [BUGFIX] AWS SD: Fix EC2 SD ignoring the configured `endpoint` option, a regression from the AWS SDK v2 migration. #18133 +- [BUGFIX] AWS SD: Fix panic in EC2 SD when DescribeAvailabilityZones returns nil ZoneName or ZoneId. #18133 +- [BUGFIX] Agent: Fix memory leak caused by duplicate SeriesRefs being loaded as active series. #17538 +- [BUGFIX] Alerting: Fix alert state incorrectly resetting to pending when the FOR period is increased in the config file. #18244 +- [BUGFIX] Azure SD: Fix system-assigned managed identity not working when `client_id` is empty. #18323 +- [BUGFIX] Consul SD: Fix filter parameter not being applied to health service endpoint, causing Node and Node.Meta filters to be ignored. #17349 +- [BUGFIX] Kubernetes SD: Fix duplicate targets generated by `*DualStack` EndpointSlices policies. #18192 +- [BUGFIX] OTLP: Fix ErrTooOldSample being returned as HTTP 500 instead of 400 in PRW v2 histogram write paths, preventing infinite client retry loops. #18084 +- [BUGFIX] OTLP: Fix exemplars getting mixed between incorrect parts of a histogram. #18056 +- [BUGFIX] PromQL: Do not skip histogram buckets in queries where histogram trimming is used. #18263 +- [BUGFIX] Remote write: Fix `prometheus_remote_storage_sent_batch_duration_seconds` measuring before the request was sent. #18214 +- [BUGFIX] Rules: Fix alert state restoration when rule labels contain Go template expressions. #18375 +- [BUGFIX] Scrape: Fix panic when parsing bare label names without an equal sign in brace-only metric notation. #18229 +- [BUGFIX] TSDB: Fail early when `use-uncached-io` feature flag is set on unsupported environments. #18219 +- [BUGFIX] TSDB: Fall back to CLI flag values when retention is removed from config file. #18200 +- [BUGFIX] TSDB: Fix memory leaks in buffer pools by clearing reference fields before returning buffers to pools. #17895 +- [BUGFIX] TSDB: Fix missing mmap of histogram chunks during WAL replay. #18306 +- [BUGFIX] TSDB: Fix storage.tsdb.retention.time unit mismatch in file causing retention to be 1e6 times longer than configured. #18200 +- [BUGFIX] Tracing: Fix missing traceID in query log when tracing is enabled, previously only spanID was emitted. #18189 +- [BUGFIX] UI: Fix tooltip Y-offset drift when using multiple graph panels. #18228 +- [BUGFIX] UI: Update retention display in runtime info when config is reloaded. #18200 + +## 3.10.0 / 2026-02-24 + +Prometheus now offers a distroless Docker image variant alongside the default +busybox image. The distroless variant provides enhanced security with a minimal +base image, uses UID/GID 65532 (nonroot) instead of nobody, and removes the +VOLUME declaration. Both variants are available with `-busybox` and `-distroless` +tag suffixes (e.g., `prom/prometheus:latest-busybox`, `prom/prometheus:latest-distroless`). +The busybox image remains the default with no suffix for backwards compatibility +(e.g., `prom/prometheus:latest` points to the busybox variant). + +For users migrating existing **named** volumes from the busybox image to the distroless variant, the ownership can be adjusted with: +``` +docker run --rm -v prometheus-data:/prometheus alpine chown -R 65532:65532 /prometheus +``` +Then, the container can be started with the old volume with: +``` +docker run -v prometheus-data:/prometheus prom/prometheus:latest-distroless +``` +User migrating from bind mounts might need to ajust permissions too, depending on their setup. + +- [CHANGE] Alerting: Add `alertmanager` dimension to following metrics: `prometheus_notifications_dropped_total`, `prometheus_notifications_queue_capacity`, `prometheus_notifications_queue_length`. #16355 +- [CHANGE] UI: Hide expanded alert annotations by default, enabling more information density on the `/alerts` page. #17611 +- [FEATURE] AWS SD: Add MSK Role. #17600 +- [FEATURE] PromQL: Add `fill()` / `fill_left()` / `fill_right()` binop modifiers for specifying default values for missing series. #17644 +- [FEATURE] Web: Add OpenAPI 3.2 specification for the HTTP API at `/api/v1/openapi.yaml`. #17825 +- [FEATURE] Dockerfile: Add distroless image variant using UID/GID 65532 and no VOLUME declaration. Busybox image remains default. #17876 +- [FEATURE] Web: Add on-demand wall time profiling under `/debug/pprof/fgprof`. #18027 +- [ENHANCEMENT] PromQL: Add more detail to histogram quantile monotonicity info annotations. #15578 +- [ENHANCEMENT] Alerting: Independent alertmanager sendloops. #16355 +- [ENHANCEMENT] TSDB: Experimental support for early compaction of stale series in the memory with configurable threshold `stale_series_compaction_threshold` in the config file. #16929 +- [ENHANCEMENT] Service Discovery: Service discoveries are now removable from the Prometheus binary through the Go build tag `remove_all_sd` and individual service discoveries can be re-added with the build tags `enable__sd`. Users can build a custom Prometheus with only the necessary SDs for a smaller binary size. #17736 +- [ENHANCEMENT] Promtool: Support promql syntax features `promql-duration-expr` and `promql-extended-range-selectors`. #17926 +- [PERF] PromQL: Avoid unnecessary label extraction in PromQL functions. #17676 +- [PERF] PromQL: Improve performance of regex matchers like `.*-.*-.*`. #17707 +- [PERF] OTLP: Add label caching for OTLP-to-Prometheus conversion to reduce allocations and improve latency. #17860 +- [PERF] API: Compute `/api/v1/targets/relabel_steps` in a single pass instead of re-running relabeling for each prefix. #17969 +- [PERF] tsdb: Optimize LabelValues intersection performance for matchers. #18069 +- [BUGFIX] PromQL: Prevent query strings containing only UTF-8 continuation bytes from crashing Prometheus. #17735 +- [BUGFIX] Web: Fix missing `X-Prometheus-Stopping` header for `/-/ready` endpoint in `NotReady` state. #17795 +- [BUGFIX] PromQL: Fix PromQL `info()` function returning empty results when filtering by a label that exists on both the input metric and `target_info`. #17817 +- [BUGFIX] TSDB: Fix a bug during exemplar buffer grow/shrink that could cause exemplars to be incorrectly discarded. #17863 +- [BUGFIX] UI: Fix broken graph display after page reload, due to broken Y axis min encoding/decoding. #17869 +- [BUGFIX] TSDB: Fix memory leaks in buffer pools by clearing reference fields (Labels, Histogram pointers, metadata strings) before returning buffers to pools. #17879 +- [BUGFIX] PromQL: info function: fix series without identifying labels not being returned. #17898 +- [BUGFIX] OTLP: Filter `__name__` from OTLP attributes to prevent duplicate labels. #17917 +- [BUGFIX] TSDB: Fix division by zero when computing stale series ratio with empty head. #17952 +- [BUGFIX] OTLP: Fix potential silent data loss for sum metrics. #17954 +- [BUGFIX] PromQL: Fix smoothed interpolation across counter resets. #17988 +- [BUGFIX] PromQL: Fix panic with `@` modifier on empty ranges. #18020 +- [BUGFIX] PromQL: Fix `avg_over_time` for a single native histogram. #18058 + +## 3.9.1 / 2026-01-07 + + - [BUGFIX] Agent: fix crash shortly after startup from invalid type of object. #17802 + - [BUGFIX] Scraping: fix relabel keep/drop not working. #17807 + +## 3.9.0 / 2026-01-06 + +- [CHANGE] Native Histograms are no longer experimental! Make the `native-histogram` feature flag a no-op. Use `scrape_native_histograms` config option instead. #17528 +- [CHANGE] API: Add maximum limit of 10,000 sets of statistics to TSDB status endpoint. #17647 +- [FEATURE] API: Add /api/v1/features for clients to understand which features are supported. #17427 +- [FEATURE] Promtool: Add `start_timestamp` field for unit tests. #17636 +- [FEATURE] Promtool: Add `--format seriesjson` option to `tsdb dump` to output just series labels in JSON format. #13409 +- [FEATURE] Add `--storage.tsdb.delay-compact-file.path` flag for better interoperability with Thanos. #17435 +- [FEATURE] UI: Add an option on the query drop-down menu to duplicate that query panel. #17714 +- [ENHANCEMENT]: TSDB: add flag `--storage.tsdb.block-reload-interval` to configure TSDB Block Reload Interval. #16728 +- [ENHANCEMENT] UI: Add graph option to start the chart's Y axis at zero. #17565 +- [ENHANCEMENT] Scraping: Classic protobuf format no longer requires the unit in the metric name. #16834 +- [ENHANCEMENT] PromQL, Rules, SD, Scraping: Add native histograms to complement existing summaries. #17374 +- [ENHANCEMENT] Notifications: Add a histogram `prometheus_notifications_latency_histogram_seconds` to complement the existing summary. #16637 +- [ENHANCEMENT] Remote-write: Add custom scope support for AzureAD authentication. #17483 +- [ENHANCEMENT] SD: add a `config` label with job name for most `prometheus_sd_refresh` metrics. #17138 +- [ENHANCEMENT] TSDB: New histogram `prometheus_tsdb_sample_ooo_delta`, the distribution of out-of-order samples in seconds. Collected for all samples, accepted or not. #17477 +- [ENHANCEMENT] Remote-read: Validate histograms received via remote-read. #17561 +- [PERF] TSDB: Small optimizations to postings index. #17439 +- [PERF] Scraping: Speed up relabelling of series. #17530 +- [PERF] PromQL: Small optimisations in binary operators. #17524, #17519. +- [BUGFIX] UI: PromQL autocomplete now shows the correct type and HELP text for OpenMetrics counters whose samples end in `_total`. #17682 +- [BUGFIX] UI: Fixed codemirror-promql incorrectly showing label completion suggestions after the closing curly brace of a vector selector. #17602 +- [BUGFIX] UI: Query editor no longer suggests a duration unit if one is already present after a number. #17605 +- [BUGFIX] PromQL: Fix some "vector cannot contain metrics with the same labelset" errors when experimental delayed name removal is enabled. #17678 +- [BUGFIX] PromQL: Fix possible corruption of PromQL text if the query had an empty `ignoring()` and non-empty grouping. #17643 +- [BUGFIX] PromQL: Fix resets/changes to return empty results for anchored selectors when all samples are outside the range. #17479 +- [BUGFIX] PromQL: Check more consistently for many-to-one matching in filter binary operators. #17668 +- [BUGFIX] PromQL: Fix collision in unary negation with non-overlapping series. #17708 +- [BUGFIX] PromQL: Fix collision in label_join and label_replace with non-overlapping series. #17703 +- [BUGFIX] PromQL: Fix bug with inconsistent results for queries with OR expression when experimental delayed name removal is enabled. #17161 +- [BUGFIX] PromQL: Ensure that `rate`/`increase`/`delta` of histograms results in a gauge histogram. #17608 +- [BUGFIX] PromQL: Do not panic while iterating over invalid histograms. #17559 +- [BUGFIX] TSDB: Reject chunk files whose encoded chunk length overflows int. #17533 +- [BUGFIX] TSDB: Do not panic during resolution reduction of invalid histograms. #17561 +- [BUGFIX] Remote-write Receive: Avoid duplicate labels when experimental type-and-unit-label feature is enabled. #17546 +- [BUGFIX] OTLP Receiver: Only write metadata to disk when experimental metadata-wal-records feature is enabled. #17472 + +## 3.8.1 / 2025-12-16 + +* [SECURITY] Remote-Write: Reject snappy-compressed requests whose declared decoded length exceeds the decode limit. #17683 +* [BUGFIX] remote: Fix Remote Write receiver, so it does not send wrong response headers for v1 flow and cause Prometheus senders to emit false partial error log and metrics. #17683 + +## 3.8.0 / 2025-11-28 + +* [CHANGE] Remote-write: Update receiving to [2.0-rc.4 spec](https://github.com/prometheus/docs/blob/60c24e450010df38cfcb4f65df874f6f9b26dbcb/docs/specs/prw/remote_write_spec_2_0.md). "created timestamp" (CT) is now called "start timestamp" (ST). #17411 +* [CHANGE] TSDB: Native Histogram Custom Bounds with a NaN threshold are now rejected. #17287 +* [FEATURE] OAuth2: support jwt-bearer grant-type (RFC7523 3.1). #17592 +* [FEATURE] Dockerfile: Add OpenContainers spec labels to Dockerfile. #16483 +* [FEATURE] SD: Add unified AWS service discovery for ec2, lightsail and ecs services. #17406 +* [FEATURE] Native histograms are now a stable, but optional feature, use the `scrape_native_histograms` config setting. #17232 #17315 +* [FEATURE] UI: Support anchored and smoothed keyword in promql editor. #17239 +* [FEATURE] UI: Show detailed relabeling steps for each discovered target. #17337 +* [FEATURE] Alerting: Add urlQueryEscape to template functions. #17403 +* [FEATURE] Promtool: Add Remote-Write 2.0 support to `promtool push metrics` via the `--protobuf_message` flag. #17417 +* [ENHANCEMENT] Clarify the docs about handling negative native histograms. #17249 +* [ENHANCEMENT] Mixin: Add static UID to the remote-write dashboard. #17256 +* [ENHANCEMENT] PromQL: Reconcile mismatched NHCB bounds in `Add` and `Sub`. #17278 +* [ENHANCEMENT] Alerting: Add "unknown" state for alerting rules that haven't been evaluated yet. #17282 +* [ENHANCEMENT] Scrape: Allow simultaneous use of classic histogram → NHCB conversion and zero-timestamp ingestion. #17305 +* [ENHANCEMENT] UI: Add smoothed/anchored in explain. #17334 +* [ENHANCEMENT] OTLP: De-duplicate any `target_info` samples with the same timestamp for the same series. #17400 +* [ENHANCEMENT] Document `use_fips_sts_endpoint` in `sigv4` config sections. #17304 +* [ENHANCEMENT] Document Prometheus Agent. #14519 +* [PERF] PromQL: Speed up parsing of variadic functions. #17316 +* [PERF] UI: Speed up alerts/rules/... pages by not rendering collapsed content. #17485 +* [PERF] UI: Performance improvement when getting label name and values in promql editor. #17194 +* [PERF] UI: Speed up /alerts for many firing alerts via virtual scrolling. #17254 +* [BUGFIX] PromQL: Fix slice indexing bug in info function on churning series. #17199 +* [BUGFIX] API: Reduce lock contention on `/api/v1/targets`. #17306 +* [BUGFIX] PromQL: Consistent handling of gauge vs. counter histograms in aggregations. #17312 +* [BUGFIX] TSDB: Allow NHCB with -Inf as the first custom value. #17320 +* [BUGFIX] UI: Fix duplicate loading of data from the API speed up rendering of some pages. #17357 +* [BUGFIX] Old UI: Fix createExpressionLink to correctly build /graph URLs so links from Alerts/Rules work again. #17365 +* [BUGFIX] PromQL: Avoid panic when parsing malformed `info` call. #17379 +* [BUGFIX] PromQL: Include histograms when enforcing sample_limit. #17390 +* [BUGFIX] Config: Fix panic if TLS CA file is absent. #17418 +* [BUGFIX] PromQL: Fix `histogram_fraction` for classic histograms and NHCB if lower bound is in the first bucket. #17424 + +## 3.7.3 / 2025-10-29 + +* [BUGFIX] UI: Revert changed (and breaking) redirect behavior for `-web.external-url` if `-web.route-prefix` is configured, which was introduced in #17240. #17389 +* [BUGFIX] Fix federation of some native histograms. #17299 #17409 +* [BUGFIX] promtool: `check config` would fail when `--lint=none` flag was set. #17399 #17414 +* [BUGFIX] Remote-write: fix a deadlock in the queue resharding logic that can lead to suboptimal queue behavior. #17412 + +## 3.7.2 / 2025-10-22 + +* [BUGFIX] AWS SD: Fix AWS SDK v2 credentials handling for EC2 and Lightsail discovery. #17355 +* [BUGFIX] AWS SD: Load AWS region from IMDS when not set. #17376 +* [BUGFIX] Relabeling: Fix `labelmap` action validation with the legacy metric name validation scheme. #17372 +* [BUGFIX] PromQL: Fix parsing failure when `anchored` and `smoothed` are used as metric names and label names. #17353 +* [BUGFIX] PromQL: Fix formatting of range vector selectors with `smoothed`/`anchored` modifier. #17354 + +## 3.7.1 / 2025-10-16 + +* [BUGFIX] OTLP: Prefix `key_` to label name when translating an OTel attribute name starting with a single underscore, and keep multiple consecutive underscores in label name when translating an OTel attribute name. This reverts the breaking changes introduced in 3.7.0. #17344 + +## 3.7.0 / 2025-10-15 + +* [CHANGE] Remote-write: the following metrics are deprecated: + - `prometheus_remote_storage_samples_in_total`, use `prometheus_wal_watcher_records_read_total{type="samples"}` and `prometheus_remote_storage_samples_dropped_total` instead, + - `prometheus_remote_storage_histograms_in_total`, use `prometheus_wal_watcher_records_read_total{type=~".*histogram_samples"}` and `prometheus_remote_storage_histograms_dropped_total` instead, + - `prometheus_remote_storage_exemplars_in_total`, use `prometheus_wal_watcher_records_read_total{type="exemplars"}` and `prometheus_remote_storage_exemplars_dropped_total` instead, + - `prometheus_remote_storage_highest_timestamp_in_seconds`, use the more accurate `prometheus_remote_storage_queue_highest_timestamp_seconds` instead in dashboards and alerts to properly account for relabeling and for more accuracy. #17065 +* [FEATURE] PromQL: Add support for experimental anchored and smoothed rate behind feature flag `promql-extended-range-selectors`. #16457 +* [FEATURE] Federation: Add support for native histograms with custom buckets (NHCB). #17215 +* [FEATURE] PromQL: Add `first_over_time(...)` and `ts_of_first_over_time(...)` behind feature flag `experimental-promql-functions`. #16963 #17021 +* [FEATURE] Remote-write: Add support for Azure Workload Identity as an authentication method for the receiver. #16788 +* [FEATURE] Remote-write: Add type and unit labels to outgoing time series in remote-write 2.0 when the `type-and-unit-labels` feature flag is enabled. #17033 +* [FEATURE] OTLP: Write start time of metrics as created time zero samples into TSDB when `created-timestamp-zero-ingestion` feature flag is enabled. #16951 +* [ENHANCEMENT] PromQL: Add warn-level annotations for counter reset conflicts in certain histogram operations. #17051 #17094 +* [ENHANCEMENT] UI: Add scrape interval and scrape timeout to targets page. #17158 +* [ENHANCEMENT] TSDB: Reduce the resolution of native histograms read from chunks or remote read if the schema is exponential. #17213 +* [ENHANCEMENT] Remote write: Add logging for unexpected metadata in sample batches, when metadata entries are found in samples-only batches. #17034 #17082 +* [ENHANCEMENT] Rules: Support concurrent evaluation for rules querying `ALERTS` and `ALERTS_FOR_STATE`. #17064 +* [ENHANCEMENT] TSDB: Add logs to improve visibility into internal operations. #17074 +* [PERF] OTLP: Write directly to TSDB instead of passing through a Remote-Write adapter when receiving OTLP metrics. #16951 +* [PERF] OTLP: Reduce number of logs emitted from OTLP endpoint. No need to log duplicate sample errors. #17201 +* [PERF] PromQL: Move more work to preprocessing step. #16896 +* [PERF] PromQL: Reduce allocations when walking the syntax tree. #16593 +* [PERF] TSDB: Optimize appender creation, slightly speeding up startup. #16922 +* [PERF] TSDB: Improve speed of querying a series with multiple matchers. #13971 +* [BUGFIX] Alerting: Mutating alerts relabeling (using `replace` actions, etc.) within a `alertmanager_config.alert_relabel_configs` block is now scoped correctly and no longer yields altered alerts to subsequent blocks. #17063 +* [BUGFIX] Config: Infer valid escaping scheme when scrape config validation scheme is set. #16923 +* [BUGFIX] TSDB: Correctly handle appending mixed-typed samples to the same series. #17071 #17241 #17290 #17295 #17296 +* [BUGFIX] Remote-write: Prevent sending unsupported native histograms with custom buckets (NHCB) over Remote-write 1.0, log warning. #17146 +* [BUGFIX] TSDB: Fix metadata entries handling on `metadata-wal-records` experimental feature for native histograms with custom buckets (NHCB) in protobuf scraping. #17156 +* [BUGFIX] TSDB: Ignore Native Histograms with invalid schemas during WAL/WBL replay. #17214 +* [BUGFIX] PromQL: Avoid empty metric names in annotations for `histogram_quantile()`. #16794 +* [BUGFIX] PromQL: Correct inaccurate character positions in errors for some aggregate expressions. #16996 #17031 +* [BUGFIX] PromQL: Fix `info()` function on churning series. #17135 +* [BUGFIX] PromQL: Set native histogram to gauge type when subtracting or multiplying/dividing with negative factors. #17004 +* [BUGFIX] TSDB: Reject unsupported native histogram schemas when attempting to append to TSDB. For scrape and remote-write implement reducing the resolution to fit the maximum if the schema is within the -9 to 52. #17189 +* [BUGFIX] Remote-write: Fix HTTP handler to return after writing error response for invalid compression. #17050 +* [BUGFIX] Remote-write: Return HTTP error `400` instead of `5xx` for wrongly formatted Native Histograms. #17210 +* [BUGFIX] Scrape: Prevent staleness markers from generating unnecessary series. #16429 +* [BUGFIX] TSDB: Avoid misleading `Failed to calculate size of \"wal\" dir` error logs during WAL clean-up. #17006 +* [BUGFIX] TSDB: Prevent erroneously dropping series records during WAL checkpoints. #17029 +* [BUGFIX] UI: Fix redirect to path of `-web.external-url` if `-web.route-prefix` is configured. #17240 +* [BUGFIX] Remote-write: Do not panic on invalid symbol table in remote-write 2.0. #17160 + +## 3.6.0 / 2025-09-17 + +* [FEATURE] PromQL: Add `step()`, and `min()` and `max()` on durations, behind feature flag `promql-duration-expr`. #16777 +* [FEATURE] API: Add a `/v1/status/tsdb/blocks` endpoint exposing metadata about loaded blocks. #16695 +* [FEATURE] Templates: Add `toDuration()` and `now()` functions. #16619 +* [ENHANCEMENT] Discovery: Add support for attaching namespace metadata to targets. #16831 +* [ENHANCEMENT] OTLP: Support new `UnderscoreEscapingWithoutSuffixes` strategy via `otlp.translation_strategy`. #16849 +* [ENHANCEMENT] OTLP: Support including scope metadata as metric labels via `otlp.promote_scope_metadata`. #16878 +* [ENHANCEMENT] OTLP: Add `__type__` and `__unit__` labels when feature flag `type-and-unit-labels` is enabled. #16630 +* [ENHANCEMENT] Tracing: Send the traceparent HTTP header during scrapes. #16425 +* [ENHANCEMENT] UI: Add option to disable info and warning query messages under `Query page settings`. #16901 +* [ENHANCEMENT] UI: Improve metadata handling for `_count/_sum/_bucket` suffixes. #16910 +* [ENHANCEMENT] TSDB: Track stale series in the Head block via the `prometheus_tsdb_head_stale_series` metric. #16925 +* [PERF] PromQL: Improve performance due to internal optimizations. #16797 +* [BUGFIX] Config: Fix "unknown global name escaping method" error messages produced during config validation. #16801 +* [BUGFIX] Discovery: Fix race condition during shutdown. #16820 +* [BUGFIX] OTLP: Generate `target_info` samples between the earliest and latest samples per resource. #16737 +* [BUGFIX] PromQL: Fail when `NaN` is passed as parameter to `topk()`, `bottomk()`, `limitk()` and `limit_ratio()`. #16725 +* [BUGFIX] PromQL: Fix extrapolation for native counter histograms. #16828 +* [BUGFIX] PromQL: Reduce numerical errors by disabling some optimizations. #16895 +* [BUGFIX] PromQL: Fix inconsistencies when using native histograms in subqueries. #16879 +* [BUGFIX] PromQL: Fix inconsistent annotations for `rate()` and `increase()` on histograms when feature flag `type-and-unit-labels` is enabled. #16915 +* [BUGFIX] Scraping: Fix memory corruption in `slicelabels` builds. #16946 +* [BUGFIX] TSDB: Fix panic on append when feature flag `created-timestamp-zero-ingestion` is enabled. #16332 +* [BUGFIX] TSDB: Fix panic on append for native histograms with empty buckets. #16893 + +## 3.5.0 / 2025-07-14 + +* [FEATURE] PromQL: Add experimental type and unit metadata labels, behind feature flag `type-and-unit-labels`. #16228 #16632 #16718 #16743 +* [FEATURE] PromQL: Add `ts_of_(min|max|last)_over_time`, behind feature flag `experimental-promql-functions`. #16722 #16733 +* [FEATURE] Scraping: Add global option `always_scrape_classic_histograms` to scrape a classic histogram even if it is also exposed as native. #16452 +* [FEATURE] OTLP: New config options `promote_all_resource_attributes` and `ignore_resource_attributes`. #16426 +* [FEATURE] Discovery: New service discovery for STACKIT Cloud. #16401 +* [ENHANCEMENT] Hetzner SD: Add `label_selector` to filter servers. #16512 +* [ENHANCEMENT] PromQL: support non-constant parameter in aggregations like `quantile` and `topk`. #16404 +* [ENHANCEMENT] UI: Better total target count display when using `keep_dropped_targets` option. #16604 +* [ENHANCEMENT] UI: Add simple filtering on the `/rules` page. #16605 +* [ENHANCEMENT] UI: Display query stats in hover tooltip over table query tab. #16723 +* [ENHANCEMENT] UI: Clear search field on `/targets` page. #16567 +* [ENHANCEMENT] Rules: Check that rules parse without error earlier at startup. #16601 +* [ENHANCEMENT] Promtool: Optional fuzzy float64 comparison in rules unittests. #16395 +* [PERF] PromQL: Reuse `histogramStatsIterator` where possible. #16686 +* [PERF] PromQL: Reuse storage for custom bucket values for native histograms. #16565 +* [PERF] UI: Optimize memoization and search debouncing on `/targets` page. #16589 +* [PERF] UI: Fix full-page re-rendering when opening status nav menu. #16590 +* [PERF] Kubernetes SD: use service cache.Indexer to achieve better performance. #16365 +* [PERF] TSDB: Optionally use Direct IO for chunks writing. #15365 +* [PERF] TSDB: When fetching label values, stop work earlier if the limit is reached. #16158 +* [PERF] Labels: Simpler/faster stringlabels encoding. #16069 +* [PERF] Scraping: Reload scrape pools concurrently. #16595 #16783 +* [BUGFIX] Top-level: Update GOGC before loading TSDB. #16491 +* [BUGFIX] Config: Respect GOGC environment variable if no "runtime" block exists. #16558 +* [BUGFIX] PromQL: Fix native histogram `last_over_time`. #16744 +* [BUGFIX] PromQL: Fix reported parser position range in errors for aggregations wrapped in ParenExpr #16041 #16754 +* [BUGFIX] PromQL: Don't emit a value from `histogram_fraction` or `histogram_quantile` if classic and native histograms are present at the same timestamp. #16552 +* [BUGFIX] PromQL: Incorrect rounding of `[1001ms]` to `[1s]` and similar. #16478 +* [BUGFIX] PromQL: Fix inconsistent / sometimes negative `histogram_count` and `histogram_sum`. #16682 +* [BUGFIX] PromQL: Improve handling of NaNs in native histograms. #16724 +* [BUGFIX] PromQL: Fix unary operator precedence in duration expressions. #16713 +* [BUGFIX] PromQL: Improve consistency of `avg` aggregation and `avg_over_time`. #16569 #16773 +* [BUGFIX] UI: Add query warnings and info to graph view. #16753 #16759 +* [BUGFIX] API: Add HTTP `Vary: Origin` header to responses to avoid cache poisoning. #16008 +* [BUGFIX] Discovery: Avoid deadlocks by taking locks in consistent order. #16587 +* [BUGFIX] Remote-write: For Azure AD auth, allow empty `client_id` to suppport system assigned managed identity. #16421 +* [BUGFIX] Scraping: Fix rare memory corruption bug. #16623 +* [BUGFIX] Scraping: continue handling custom-bucket histograms after an exponential histogram is encountered. #16720 +* [BUGFIX] OTLP: Default config not respected when `otlp:` block is unset. #16693 + +## 3.4.2 / 2025-06-26 + +* [BUGFIX] OTLP receiver: Fix default configuration not being respected if the `otlp:` block is unset in the config file. #16693 + +## 3.4.1 / 2025-05-31 + +* [BUGFIX] Parser: Add reproducer for a dangling-reference issue in parsers. #16633 + +## 3.4.0 / 2025-05-17 + +* [CHANGE] Config: Make setting out-of-order native histograms feature (`--enable-feature=ooo-native-histograms`) a no-op. Out-of-order native histograms are now always enabled when `out_of_order_time_window` is greater than zero and `--enable-feature=native-histograms` is set. #16207 +* [FEATURE] OTLP translate: Add feature flag for optionally translating OTel explicit bucket histograms into native histograms with custom buckets. #15850 +* [FEATURE] OTLP translate: Add option to receive OTLP metrics without translating names or attributes. #16441 +* [FEATURE] PromQL: allow arithmetic operations in durations in PromQL parser. #16249 +* [FEATURE] OTLP receiver: Add primitive support for ingesting OTLP delta metrics as-is. #16360 +* [ENHANCEMENT] PromQL: histogram_fraction for bucket histograms. #16095 +* [ENHANCEMENT] TSDB: add `prometheus_tsdb_wal_replay_unknown_refs_total` and `prometheus_tsdb_wbl_replay_unknown_refs_total` metrics to track unknown series references during WAL/WBL replay. #16166 +* [ENHANCEMENT] Scraping: Add config option for escaping scheme request. #16066 +* [ENHANCEMENT] Config: Add global config option for convert_classic_histograms_to_nhcb. #16226 +* [ENHANCEMENT] Alerting: make batch size configurable (`--alertmanager.notification-batch-size`). #16254 +* [PERF] Kubernetes SD: make endpointSlice discovery more efficient. #16433 +* [BUGFIX] Config: Fix auto-reload on changes to rule and scrape config files. #16340 +* [BUGFIX] Scraping: Skip native histogram series if ingestion is disabled. #16218 +* [BUGFIX] TSDB: Handle metadata/tombstones/exemplars for duplicate series during WAL replay. #16231 +* [BUGFIX] TSDB: Avoid processing exemplars outside the valid time range during WAL replay. #16242 +* [BUGFIX] Promtool: Add feature flags for PromQL features. #16443 +* [BUGFIX] Rules: correct logging of alert name & template data. #15093 +* [BUGFIX] PromQL: Use arithmetic mean for `histogram_stddev()` and `histogram_stdvar()` . #16444 + +## 3.3.0 / 2025-04-15 + +* [FEATURE] PromQL: Implement `idelta()` and `irate()` for native histograms. #15853 +* [ENHANCEMENT] Scaleway SD: Add `__meta_scaleway_instance_public_ipv4_addresses` and `__meta_scaleway_instance_public_ipv6_addresses` labels. #14228 +* [ENHANCEMENT] TSDB: Reduce locking while reloading blocks. #12920 +* [ENHANCEMENT] PromQL: Allow UTF-8 labels in `label_replace()`. #15974 +* [ENHANCEMENT] Promtool: `tsdb create-blocks-from openmetrics` can now read from a Pipe. #16011 +* [ENHANCEMENT] Rules: Add support for anchors and aliases in rule files. #14957 +* [ENHANCEMENT] Dockerfile: Make `/prometheus` writable. #16073 +* [ENHANCEMENT] API: Include scrape pool name for dropped targets in `/api/v1/targets`. #16085 +* [ENHANCEMENT] UI: Improve time formatting and copying of selectors. #15999 #16165 +* [ENHANCEMENT] UI: Bring back vertical grid lines and graph legend series toggling instructions. #16163 #16164 +* [ENHANCEMENT] Mixin: The `cluster` label can be customized using `clusterLabel`. #15826 +* [PERF] TSDB: Optimize some operations on head chunks by taking shortcuts. #12659 +* [PERF] TSDB & Agent: Reduce memory footprint during WL replay. #15778 +* [PERF] Remote-Write: Reduce memory footprint during WAL replay. #16197 +* [PERF] API: Reduce memory footprint during header parsing. #16001 +* [PERF] Rules: Improve dependency evaluation, enabling better concurrency. #16039 +* [PERF] Scraping: Improve scraping performance for native histograms. #15731 +* [PERF] Scraping: Improve parsing of created timestamps. #16072 +* [BUGFIX] Scraping: Bump cache iteration after error to avoid false duplicate detections. #16174 +* [BUGFIX] Scraping: Skip native histograms series when ingestion is disabled. #16218 +* [BUGFIX] PromQL: Fix counter reset detection for native histograms. #15902 #15987 +* [BUGFIX] PromQL: Fix inconsistent behavior with an empty range. #15970 +* [BUGFIX] PromQL: Fix inconsistent annotation in `quantile_over_time()`. #16018 +* [BUGFIX] PromQL: Prevent `label_join()` from producing duplicates. #15975 +* [BUGFIX] PromQL: Ignore native histograms in `scalar()`, `sort()` and `sort_desc()`. #15964 +* [BUGFIX] PromQL: Fix annotations for binary operations between incompatible native histograms. #15895 +* [BUGFIX] Alerting: Consider alert relabeling when deciding whether alerts are dropped. #15979 +* [BUGFIX] Config: Set `GoGC` to the default value in case of an empty configuration. #16052 +* [BUGFIX] TSDB: Fix unknown series errors and potential data loss during WAL replay when inactive series are removed from the head and reappear before the next WAL checkpoint. #16060 +* [BUGFIX] Scaleway SD: The public IP will no longer be set to `__meta_meta_scaleway_instance_public_ipv4` if it is an IPv6 address. #14228 +* [BUGFIX] UI: Display the correct value of Alerting rules' `keep_firing_for`. #16211 + +## 3.2.1 / 2025-02-25 + +* [BUGFIX] Don't send `Accept` header `escape=allow-utf-8` when `metric_name_validation_scheme: legacy` is configured. #16061 + +## 3.2.0 / 2025-02-17 + +* [CHANGE] relabel: Replace actions can now use UTF-8 characters in `targetLabel` field. Note that `$` or `${}` will be expanded. This also apply to `replacement` field for `LabelMap` action. #15851 +* [CHANGE] rulefmt: Rule names can use UTF-8 characters, except `{` and `}` characters (due to common mistake checks). #15851 +* [FEATURE] remote/otlp: Add feature flag `otlp-deltatocumulative` to support conversion from delta to cumulative. #15165 +* [ENHANCEMENT] openstack SD: Discover Octavia loadbalancers. #15539 +* [ENHANCEMENT] scrape: Add metadata for automatic metrics to WAL for `metadata-wal-records` feature. #15837 +* [ENHANCEMENT] promtool: Support linting of scrape interval, through lint option `too-long-scrape-interval`. #15719 +* [ENHANCEMENT] promtool: Add --ignore-unknown-fields option. #15706 +* [ENHANCEMENT] ui: Make "hide empty rules" and "hide empty rules" persistent #15807 +* [ENHANCEMENT] web/api: Add a limit parameter to `/query` and `/query_range`. #15552 +* [ENHANCEMENT] api: Add fields Node and ServerTime to `/status`. #15784 +* [PERF] Scraping: defer computing labels for dropped targets until they are needed by the UI. #15261 +* [BUGFIX] remotewrite2: Fix invalid metadata bug for metrics without metadata. #15829 +* [BUGFIX] remotewrite2: Fix the unit field propagation. #15825 +* [BUGFIX] scrape: Fix WAL metadata for histograms and summaries. #15832 +* [BUGFIX] ui: Merge duplicate "Alerts page settings" sections. #15810 +* [BUGFIX] PromQL: Fix `` functions with histograms. #15711 + +## 3.1.0 / 2025-01-02 + + * [SECURITY] upgrade golang.org/x/crypto to address reported CVE-2024-45337. #15691 + * [CHANGE] Notifier: Increment prometheus_notifications_errors_total by the number of affected alerts rather than per batch. #15428 + * [CHANGE] API: list rules field "groupNextToken:omitempty" renamed to "groupNextToken". #15400 + * [ENHANCEMENT] OTLP translate: keep identifying attributes in target_info. #15448 + * [ENHANCEMENT] Paginate rule groups, add infinite scroll to rules within groups. #15677 + * [ENHANCEMENT] TSDB: Improve calculation of space used by labels. #13880 + * [ENHANCEMENT] Rules: new metric rule_group_last_rule_duration_sum_seconds. #15672 + * [ENHANCEMENT] Observability: Export 'go_sync_mutex_wait_total_seconds_total' metric. #15339 + * [ENHANCEMENT] Remote-Write: optionally use a DNS resolver that picks a random IP. #15329 + * [PERF] Optimize `l=~".+"` matcher. #15474, #15684 + * [PERF] TSDB: Cache all symbols for compaction. #15455 + * [PERF] TSDB: MemPostings: keep a map of label values slices. #15426 + * [PERF] Remote-Write: Remove interning hook. #15456 + * [PERF] Scrape: optimize string manipulation for experimental native histograms with custom buckets. #15453 + * [PERF] TSDB: reduce memory allocations. #15465, #15427 + * [PERF] Storage: Implement limit in mergeGenericQuerier. #14489 + * [PERF] TSDB: Optimize inverse matching. #14144 + * [PERF] Regex: use stack memory for lowercase copy of string. #15210 + * [PERF] TSDB: When deleting from postings index, pause to unlock and let readers read. #15242 + * [BUGFIX] Main: Avoid possible segfault at exit. (#15724) + * [BUGFIX] Rules: Do not run rules concurrently if uncertain about dependencies. #15560 + * [BUGFIX] PromQL: Adds test for `absent`, `absent_over_time` and `deriv` func with histograms. #15667 + * [BUGFIX] PromQL: Fix various bugs related to quoting UTF-8 characters. #15531 + * [BUGFIX] Scrape: fix nil panic after scrape loop reload. #15563 + * [BUGFIX] Remote-write: fix panic on repeated log message. #15562 + * [BUGFIX] Scrape: reload would ignore always_scrape_classic_histograms and convert_classic_histograms_to_nhcb configs. #15489 + * [BUGFIX] TSDB: fix data corruption in experimental native histograms. #15482 + * [BUGFIX] PromQL: Ignore histograms in all time related functions. #15479 + * [BUGFIX] OTLP receiver: Convert metric metadata. #15416 + * [BUGFIX] PromQL: Fix `resets` function for histograms. #15527 + * [BUGFIX] PromQL: Fix behaviour of `changes()` for mix of histograms and floats. #15469 + * [BUGFIX] PromQL: Fix behaviour of some aggregations with histograms. #15432 + * [BUGFIX] allow quoted exemplar keys in openmetrics text format. #15260 + * [BUGFIX] TSDB: fixes for rare conditions when loading write-behind-log (WBL). #15380 + * [BUGFIX] `round()` function did not remove `__name__` label. #15250 + * [BUGFIX] Promtool: analyze block shows metric name with 0 cardinality. #15438 + * [BUGFIX] PromQL: Fix `count_values` for histograms. #15422 + * [BUGFIX] PromQL: fix issues with comparison binary operations with `bool` modifier and native histograms. #15413 + * [BUGFIX] PromQL: fix incorrect "native histogram ignored in aggregation" annotations. #15414 + * [BUGFIX] PromQL: Corrects the behaviour of some operator and aggregators with Native Histograms. #15245 + * [BUGFIX] TSDB: Always return unknown hint for first sample in non-gauge histogram chunk. #15343 + * [BUGFIX] PromQL: Clamp functions: Ignore any points with native histograms. #15169 + * [BUGFIX] TSDB: Fix race on stale values in headAppender. #15322 + * [BUGFIX] UI: Fix selector / series formatting for empty metric names. #15340 + * [BUGFIX] OTLP receiver: Allow colons in non-standard units. #15710 + +## 3.0.1 / 2024-11-28 + +The first bug fix release for Prometheus 3. + +* [BUGFIX] Promql: Make subqueries left open. #15431 +* [BUGFIX] Fix memory leak when query log is enabled. #15434 +* [BUGFIX] Support utf8 names on /v1/label/:name/values endpoint. #15399 + +## 3.0.0 / 2024-11-14 + +This release includes new features such as a brand new UI and UTF-8 support enabled by default. As this marks the first new major version in seven years, several breaking changes are introduced. The breaking changes are mainly around the removal of deprecated feature flags and CLI arguments, and the full list can be found below. For users that want to upgrade we recommend to read through our [migration guide](https://prometheus.io/docs/prometheus/3.0/migration/). + +* [CHANGE] Set the `GOMAXPROCS` variable automatically to match the Linux CPU quota. Use `--no-auto-gomaxprocs` to disable it. The `auto-gomaxprocs` feature flag was removed. #15376 +* [CHANGE] Set the `GOMEMLIMIT` variable automatically to match the Linux container memory limit. Use `--no-auto-gomemlimit` to disable it. The `auto-gomemlimit` feature flag was removed. #15373 +* [CHANGE] Scraping: Remove implicit fallback to the Prometheus text format in case of invalid/missing Content-Type and fail the scrape instead. Add ability to specify a `fallback_scrape_protocol` in the scrape config. #15136 +* [CHANGE] Remote-write: default enable_http2 to false. #15219 +* [CHANGE] Scraping: normalize "le" and "quantile" label values upon ingestion. #15164 +* [CHANGE] Scraping: config `scrape_classic_histograms` was renamed to `always_scrape_classic_histograms`. #15178 +* [CHANGE] Config: remove expand-external-labels flag, expand external labels env vars by default. #14657 +* [CHANGE] Disallow configuring AM with the v1 api. #13883 +* [CHANGE] regexp `.` now matches all characters (performance improvement). #14505 +* [CHANGE] `holt_winters` is now called `double_exponential_smoothing` and moves behind the [experimental-promql-functions feature flag](https://prometheus.io/docs/prometheus/latest/feature_flags/#experimental-promql-functions). #14930 +* [CHANGE] API: The OTLP receiver endpoint can now be enabled using `--web.enable-otlp-receiver` instead of `--enable-feature=otlp-write-receiver`. #14894 +* [CHANGE] Prometheus will not add or remove port numbers from the target address. `no-default-scrape-port` feature flag removed. #14160 +* [CHANGE] Logging: the format of log lines has changed a little, along with the adoption of Go's Structured Logging package. #14906 +* [CHANGE] Don't create extra `_created` timeseries if feature-flag `created-timestamp-zero-ingestion` is enabled. #14738 +* [CHANGE] Float literals and time durations being the same is now a stable fetaure. #15111 +* [CHANGE] UI: The old web UI has been replaced by a completely new one that is less cluttered and adds a few new features (PromLens-style tree view, better metrics explorer, "Explain" tab). However, it is still missing some features of the old UI (notably, exemplar display and heatmaps). To switch back to the old UI, you can use the feature flag `--enable-feature=old-ui` for the time being. #14872 +* [CHANGE] PromQL: Range selectors and the lookback delta are now left-open, i.e. a sample coinciding with the lower time limit is excluded rather than included. #13904 +* [CHANGE] Kubernetes SD: Remove support for `discovery.k8s.io/v1beta1` API version of EndpointSlice. This version is no longer served as of Kubernetes v1.25. #14365 +* [CHANGE] Kubernetes SD: Remove support for `networking.k8s.io/v1beta1` API version of Ingress. This version is no longer served as of Kubernetes v1.22. #14365 +* [CHANGE] UTF-8: Enable UTF-8 support by default. Prometheus now allows all UTF-8 characters in metric and label names. The corresponding `utf8-name` feature flag has been removed. #14705, #15258 +* [CHANGE] Console: Remove example files for the console feature. Users can continue using the console feature by supplying their own JavaScript and templates. #14807 +* [CHANGE] SD: Enable the new service discovery manager by default. This SD manager does not restart unchanged discoveries upon reloading. This makes reloads faster and reduces pressure on service discoveries' sources. The corresponding `new-service-discovery-manager` feature flag has been removed. #14770 +* [CHANGE] Agent mode has been promoted to stable. The feature flag `agent` has been removed. To run Prometheus in Agent mode, use the new `--agent` cmdline arg instead. #14747 +* [CHANGE] Remove deprecated `remote-write-receiver`,`promql-at-modifier`, and `promql-negative-offset` feature flags. #13456, #14526 +* [CHANGE] Remove deprecated `storage.tsdb.allow-overlapping-blocks`, `alertmanager.timeout`, and `storage.tsdb.retention` flags. #14640, #14643 +* [FEATURE] OTLP receiver: Ability to skip UTF-8 normalization using `otlp.translation_strategy = NoUTF8EscapingWithSuffixes` configuration option. #15384 +* [FEATURE] Support config reload automatically - feature flag `auto-reload-config`. #14769, #15011 +* [ENHANCEMENT] Scraping, rules: handle targets reappearing, or rules moving group, when out-of-order is enabled. #14710 +* [ENHANCEMENT] Tools: add debug printouts to promtool rules unit testing #15196 +* [ENHANCEMENT] Scraping: support Created-Timestamp feature on native histograms. #14694 +* [ENHANCEMENT] UI: Many fixes and improvements. #14898, #14899, #14907, #14908, #14912, #14913, #14914, #14931, #14940, #14945, #14946, #14972, #14981, #14982, #14994, #15096 +* [ENHANCEMENT] UI: Web UI now displays notifications, e.g. when starting up and shutting down. #15082 +* [ENHANCEMENT] PromQL: Introduce exponential interpolation for native histograms. #14677 +* [ENHANCEMENT] TSDB: Add support for ingestion of out-of-order native histogram samples. #14850, #14546 +* [ENHANCEMENT] Alerts: remove metrics for removed Alertmanagers. #13909 +* [ENHANCEMENT] Kubernetes SD: Support sidecar containers in endpoint discovery. #14929 +* [ENHANCEMENT] Consul SD: Support catalog filters. #11224 +* [ENHANCEMENT] Move AM discovery page from "Monitoring status" to "Server status". #14875 +* [PERF] TSDB: Parallelize deletion of postings after head compaction. #14975 +* [PERF] TSDB: Chunk encoding: shorten some write sequences. #14932 +* [PERF] TSDB: Grow postings by doubling. #14721 +* [PERF] Relabeling: Optimize adding a constant label pair. #12180 +* [BUGFIX] Scraping: Don't log errors on empty scrapes. #15357 +* [BUGFIX] UI: fix selector / series formatting for empty metric names. #15341 +* [BUGFIX] PromQL: Fix stddev+stdvar aggregations to always ignore native histograms. #14941 +* [BUGFIX] PromQL: Fix stddev+stdvar aggregations to treat Infinity consistently. #14941 +* [BUGFIX] OTLP receiver: Preserve colons when generating metric names in suffix adding mode (this mode is always enabled, unless one uses Prometheus as a library). #15251 +* [BUGFIX] Scraping: Unit was missing when using protobuf format. #15095 +* [BUGFIX] PromQL: Only return "possible non-counter" annotation when `rate` returns points. #14910 +* [BUGFIX] TSDB: Chunks could have one unnecessary zero byte at the end. #14854 +* [BUGFIX] "superfluous response.WriteHeader call" messages in log. #14884 +* [BUGFIX] PromQL: Unary negation of native histograms. #14821 +* [BUGFIX] PromQL: Handle stale marker in native histogram series (e.g. if series goes away and comes back). #15025 +* [BUGFIX] Autoreload: Reload invalid yaml files. #14947 +* [BUGFIX] Scrape: Do not override target parameter labels with config params. #11029 + +## 2.53.5 / 2025-06-30 + +* [ENHANCEMENT] TSDB: Add backward compatibility with the upcoming TSDB block index v3 #16762 +* [BUGFIX] Top-level: Update GOGC before loading TSDB #16521 + +## 2.53.4 / 2025-03-18 + +* [BUGFIX] Runtime: fix GOGC is being set to 0 when installed with empty prometheus.yml file resulting high cpu usage. #16090 +* [BUGFIX] Scrape: fix dropping valid metrics after previous scrape failed. #16220 + +## 2.53.3 / 2024-11-04 + +* [BUGFIX] Scraping: allow multiple samples on same series, with explicit timestamps. #14685, #14740 + +## 2.53.2 / 2024-08-09 + +Fix a bug where Prometheus would crash with a segmentation fault if a remote-read +request accessed a block on disk at about the same time as TSDB created a new block. + +* [BUGFIX] Remote-Read: Resolve occasional segmentation fault on query. #14515,#14523 + +## 2.55.1 / 2024-11-04 * [BUGFIX] `round()` function did not remove `__name__` label. #15250 @@ -42,7 +705,7 @@ ## 2.54.1 / 2024-08-27 -* [BUGFIX] Scraping: allow multiple samples on same series, with explicit timestamps. #14685 +* [BUGFIX] Scraping: allow multiple samples on same series, with explicit timestamps (mixing samples of the same series with and without timestamps is still rejected). #14685 * [BUGFIX] Docker SD: fix crash in `match_first_network` mode when container is reconnected to a new network. #14654 * [BUGFIX] PromQL: fix experimental native histograms getting corrupted due to vector selector bug in range queries. #14538 * [BUGFIX] PromQL: fix experimental native histogram counter reset detection on stale samples. #14514 @@ -124,6 +787,7 @@ This release changes the default for GOGC, the Go runtime control for the trade- ## 2.52.0 / 2024-05-07 * [CHANGE] TSDB: Fix the predicate checking for blocks which are beyond the retention period to include the ones right at the retention boundary. #9633 +* [CHANGE] Scrape: Multiple samples (even with different timestamps) are treated as duplicates during one scrape. * [FEATURE] Kubernetes SD: Add a new metric `prometheus_sd_kubernetes_failures_total` to track failed requests to Kubernetes API. #13554 * [FEATURE] Kubernetes SD: Add node and zone metadata labels when using the endpointslice role. #13935 * [FEATURE] Azure SD/Remote Write: Allow usage of Azure authorization SDK. #13099 @@ -137,7 +801,7 @@ This release changes the default for GOGC, the Go runtime control for the trade- * [ENHANCEMENT] TSDB: Pause regular block compactions if the head needs to be compacted (prioritize head as it increases memory consumption). #13754 * [ENHANCEMENT] Observability: Improved logging during signal handling termination. #13772 * [ENHANCEMENT] Observability: All log lines for drop series use "num_dropped" key consistently. #13823 -* [ENHANCEMENT] Observability: Log chunk snapshot and mmaped chunk replay duration during WAL replay. #13838 +* [ENHANCEMENT] Observability: Log chunk snapshot and mmapped chunk replay duration during WAL replay. #13838 * [ENHANCEMENT] Observability: Log if the block is being created from WBL during compaction. #13846 * [BUGFIX] PromQL: Fix inaccurate sample number statistic when querying histograms. #13667 * [BUGFIX] PromQL: Fix `histogram_stddev` and `histogram_stdvar` for cases where the histogram has negative buckets. #13852 @@ -674,7 +1338,7 @@ The binaries published with this release are built with Go1.17.8 to avoid [CVE-2 ## 2.33.0 / 2022-01-29 -* [CHANGE] PromQL: Promote negative offset and `@` modifer to stable features. #10121 +* [CHANGE] PromQL: Promote negative offset and `@` modifier to stable features. #10121 * [CHANGE] Web: Promote remote-write-receiver to stable. #10119 * [FEATURE] Config: Add `stripPort` template function. #10002 * [FEATURE] Promtool: Add cardinality analysis to `check metrics`, enabled by flag `--extended`. #10045 @@ -911,7 +1575,7 @@ This vulnerability has been reported by Aaron Devaney from MDSec. * [ENHANCEMENT] Templating: Enable parsing strings in `humanize` functions. #8682 * [BUGFIX] UI: Provide errors instead of blank page on TSDB Status Page. #8654 #8659 * [BUGFIX] TSDB: Do not panic when writing very large records to the WAL. #8790 -* [BUGFIX] TSDB: Avoid panic when mmaped memory is referenced after the file is closed. #8723 +* [BUGFIX] TSDB: Avoid panic when mmapped memory is referenced after the file is closed. #8723 * [BUGFIX] Scaleway Discovery: Fix nil pointer dereference. #8737 * [BUGFIX] Consul Discovery: Restart no longer required after config update with no targets. #8766 @@ -1837,7 +2501,7 @@ information, read the announcement blog post and migration guide. ## 1.7.0 / 2017-06-06 * [CHANGE] Compress remote storage requests and responses with unframed/raw snappy. -* [CHANGE] Properly ellide secrets in config. +* [CHANGE] Properly elide secrets in config. * [FEATURE] Add OpenStack service discovery. * [FEATURE] Add ability to limit Kubernetes service discovery to certain namespaces. * [FEATURE] Add metric for discovered number of Alertmanagers. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..43c994c2d36 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000000..1822bf63d18 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,33 @@ +# +# Please keep this file in sync with the MAINTAINERS.md file! +# + +# Prometheus team members are members of the "default maintainers" github team. +# They are code owners by default for the whole repo. +* @prometheus/default-maintainers + +# Subsystems. +/Makefile @prometheus/default-maintainers @simonpasquier @SuperQ +/cmd/promtool @prometheus/default-maintainers @dgl +/documentation/prometheus-mixin @prometheus/default-maintainers @metalmatze +/model/histogram @prometheus/default-maintainers @beorn7 @krajorama +/web/ui @prometheus/default-maintainers @juliusv +/web/ui/module @prometheus/default-maintainers @juliusv @nexucis +/promql @prometheus/default-maintainers @roidelapluie +/storage/remote @prometheus/default-maintainers @cstyan @bwplotka @tomwilkie @alexgreenbank +/storage/remote/otlptranslator @prometheus/default-maintainers @aknuds1 @jesusvazquez @ArthurSens +/tsdb @prometheus/default-maintainers @jesusvazquez @codesome @bwplotka @krajorama +/tsdb/agent @prometheus/default-maintainers @jesusvazquez @codesome @bwplotka @krajorama @kgeckhart + +# Service discovery. +/discovery/kubernetes @prometheus/default-maintainers @brancz @rexagod +/discovery/stackit @prometheus/default-maintainers @jkroepke +/discovery/aws/ @prometheus/default-maintainers @matt-gp @sysadmind +/discovery/consul @prometheus/default-maintainers @mrvarmazyar +/discovery/hetzner @prometheus/default-maintainers @jooola @apricote +/discovery/scaleway @prometheus/default-maintainers @remyleone +# Pending +# https://github.com/prometheus/prometheus/pull/15212#issuecomment-3575225179 +# /discovery/aliyun @prometheus/default-maintainers @KeyOfSpectator +# https://github.com/prometheus/prometheus/pull/14108#issuecomment-2639515421 +# /discovery/nomad @prometheus/default-maintainers @jaloren @jrasell diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b1b286ccf4..cfb346e4d0d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ Prometheus uses GitHub to manage reviews of pull requests. of inspiration. Also please see our [non-goals issue](https://github.com/prometheus/docs/issues/149) on areas that the Prometheus community doesn't plan to work on. * Relevant coding style guidelines are the [Go Code Review - Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) + Comments](https://go.dev/wiki/CodeReviewComments) and the _Formatting and style_ section of Peter Bourgon's [Go: Best Practices for Production Environments](https://peter.bourgon.org/go-in-production/#formatting-and-style). @@ -78,8 +78,7 @@ go get example.com/some/module/pkg@vX.Y.Z Tidy up the `go.mod` and `go.sum` files: ```bash -# The GO111MODULE variable can be omitted when the code isn't located in GOPATH. -GO111MODULE=on go mod tidy +go mod tidy ``` You have to commit the changes to `go.mod` and `go.sum` before submitting the pull request. diff --git a/Dockerfile b/Dockerfile index b47f77dcd69..88a64ac490d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,27 +2,30 @@ ARG ARCH="amd64" ARG OS="linux" FROM quay.io/prometheus/busybox-${OS}-${ARCH}:latest LABEL maintainer="The Prometheus Authors " +LABEL org.opencontainers.image.authors="The Prometheus Authors" \ + org.opencontainers.image.vendor="Prometheus" \ + org.opencontainers.image.title="Prometheus" \ + org.opencontainers.image.description="The Prometheus monitoring system and time series database" \ + org.opencontainers.image.source="https://github.com/prometheus/prometheus" \ + org.opencontainers.image.url="https://github.com/prometheus/prometheus" \ + org.opencontainers.image.documentation="https://prometheus.io/docs" \ + org.opencontainers.image.licenses="Apache License 2.0" \ + io.prometheus.image.variant="busybox" ARG ARCH="amd64" ARG OS="linux" COPY .build/${OS}-${ARCH}/prometheus /bin/prometheus COPY .build/${OS}-${ARCH}/promtool /bin/promtool COPY documentation/examples/prometheus.yml /etc/prometheus/prometheus.yml -COPY console_libraries/ /usr/share/prometheus/console_libraries/ -COPY consoles/ /usr/share/prometheus/consoles/ COPY LICENSE /LICENSE COPY NOTICE /NOTICE -COPY npm_licenses.tar.bz2 /npm_licenses.tar.bz2 WORKDIR /prometheus -RUN ln -s /usr/share/prometheus/console_libraries /usr/share/prometheus/consoles/ /etc/prometheus/ && \ - chown -R nobody:nobody /etc/prometheus /prometheus +RUN chown -R nobody:nobody /etc/prometheus /prometheus && chmod g+w /prometheus USER nobody EXPOSE 9090 VOLUME [ "/prometheus" ] ENTRYPOINT [ "/bin/prometheus" ] CMD [ "--config.file=/etc/prometheus/prometheus.yml", \ - "--storage.tsdb.path=/prometheus", \ - "--web.console.libraries=/usr/share/prometheus/console_libraries", \ - "--web.console.templates=/usr/share/prometheus/consoles" ] + "--storage.tsdb.path=/prometheus" ] diff --git a/Dockerfile.distroless b/Dockerfile.distroless new file mode 100644 index 00000000000..7cf85012c28 --- /dev/null +++ b/Dockerfile.distroless @@ -0,0 +1,29 @@ +ARG DISTROLESS_ARCH="amd64" + +# Use DISTROLESS_ARCH for base image selection (handles armv7->arm mapping). +FROM gcr.io/distroless/static-debian13:nonroot-${DISTROLESS_ARCH} +# Base image sets USER to 65532:65532 (nonroot user). + +ARG ARCH="amd64" +ARG OS="linux" + +LABEL org.opencontainers.image.authors="The Prometheus Authors" +LABEL org.opencontainers.image.vendor="Prometheus" +LABEL org.opencontainers.image.title="Prometheus" +LABEL org.opencontainers.image.description="The Prometheus monitoring system and time series database" +LABEL org.opencontainers.image.source="https://github.com/prometheus/prometheus" +LABEL org.opencontainers.image.url="https://github.com/prometheus/prometheus" +LABEL org.opencontainers.image.documentation="https://prometheus.io/docs" +LABEL org.opencontainers.image.licenses="Apache License 2.0" +LABEL io.prometheus.image.variant="distroless" + +COPY documentation/examples/prometheus.yml /etc/prometheus/prometheus.yml +COPY LICENSE NOTICE / +COPY .build/${OS}-${ARCH}/prometheus /bin/prometheus +COPY .build/${OS}-${ARCH}/promtool /bin/promtool + +WORKDIR /prometheus +EXPOSE 9090 +ENTRYPOINT [ "/bin/prometheus" ] +CMD [ "--config.file=/etc/prometheus/prometheus.yml", \ + "--storage.tsdb.path=/prometheus" ] diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 3661ddaa0a1..57dc9811e9c 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,28 +1,34 @@ # Maintainers +## Please keep this file in sync with the CODEOWNERS file! + General maintainers: * Bryan Boreham (bjboreham@gmail.com / @bboreham) -* Levi Harrison (levi@leviharrison.dev / @LeviHarrison) * Ayoub Mrini (ayoubmrini424@gmail.com / @machine424) * Julien Pivotto (roidelapluie@prometheus.io / @roidelapluie) +* György Krajcsovits ( / @krajorama) +* Bartłomiej Płotka ( / @bwplotka) +* Arve Knudsen ( / @aknuds1) Maintainers for specific parts of the codebase: * `cmd` * `promtool`: David Leadbeater ( / @dgl) * `discovery` - * `k8s`: Frederic Branczyk ( / @brancz) + * `hetzner`: (@jooola) (@apricote) + * `k8s`: Frederic Branczyk ( / @brancz), Pranshu Srivastava ( / @rexagod) + * `stackit`: Jan-Otto Kröpke ( / @jkroepke) + * `consul`: Mohammad Varmazyar ( / @mrvarmazyar) + * `scaleway`: Rémy Léone (rleone@scaleway.com / @remyleone) * `documentation` * `prometheus-mixin`: Matthias Loibl ( / @metalmatze) -* `model/histogram` and other code related to native histograms: Björn Rabenstein ( / @beorn7), -George Krajcsovits ( / @krajorama) * `storage` - * `remote`: Callum Styan ( / @cstyan), Bartłomiej Płotka ( / @bwplotka), Tom Wilkie (tom.wilkie@gmail.com / @tomwilkie), Nicolás Pazos ( / @npazosmendez), Alex Greenbank ( / @alexgreenbank) - * `otlptranslator`: Arve Knudsen ( / @aknuds1), Jesús Vázquez ( / @jesusvazquez) -* `tsdb`: Ganesh Vernekar ( / @codesome), Bartłomiej Płotka ( / @bwplotka), Jesús Vázquez ( / @jesusvazquez) - * `agent`: Robert Fratto ( / @rfratto) + * `remote`: Callum Styan ( / @cstyan), Tom Wilkie (tom.wilkie@gmail.com / @tomwilkie), Alex Greenbank ( / @alexgreenbank) + * `otlptranslator`: Arthur Silva Sens ( / @ArthurSens, Jesús Vázquez ( / @jesusvazquez) +* `tsdb`: Ganesh Vernekar ( / @codesome), Jesús Vázquez ( / @jesusvazquez) + * `agent`: Kyle Eckhart ( / @kgeckhart) * `web` * `ui`: Julius Volz ( / @juliusv) - * `module`: Augustin Husson ( @nexucis) + * `module`: Augustin Husson ( / @nexucis) * `Makefile` and related build configuration: Simon Pasquier ( / @simonpasquier), Ben Kochie ( / @SuperQ) For the sake of brevity, not all subtrees are explicitly listed. Due to the @@ -32,7 +38,7 @@ incomplete and out of date. However the listed maintainer(s) should be able to direct a PR/question to the right person. v3 release coordinators: -* Alex Greenbank ( / @alexgreenbank) +* Alex Greenbank ( / @alexgreenbank) * Carrie Edwards ( / @carrieedwards) * Fiona Liao ( / @fionaliao) * Jan Fajerski ( / @jan--f) diff --git a/Makefile b/Makefile index f2bb3fcb7a5..d1033a2a582 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2018 The Prometheus Authors +# Copyright The Prometheus Authors # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,11 +12,9 @@ # limitations under the License. # Needs to be defined before including Makefile.common to auto-generate targets -DOCKER_ARCHS ?= amd64 armv7 arm64 ppc64le s390x - UI_PATH = web/ui +BUILD_UI ?= all UI_NODE_MODULES_PATH = $(UI_PATH)/node_modules -REACT_APP_NPM_LICENSES_TARBALL = "npm_licenses.tar.bz2" PROMTOOL = ./promtool TSDB_BENCHMARK_NUM_METRICS ?= 1000 @@ -28,8 +26,22 @@ GOYACC_VERSION ?= v0.6.0 include Makefile.common +ifeq (arm, $(GOHOSTARCH)) + PROTOC_ARCH ?= aarch_64 +else + PROTOC_ARCH ?= x86_64 +endif + +PROTOC_VERSION ?= 3.15.8 +PROTOC_URL ?= https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-$(PROTOC_ARCH).zip + DOCKER_IMAGE_NAME ?= prometheus +# Only build UI if PREBUILT_ASSETS_STATIC_DIR is not set +ifdef PREBUILT_ASSETS_STATIC_DIR + SKIP_UI_BUILD = true +endif + .PHONY: update-npm-deps update-npm-deps: @echo ">> updating npm dependencies" @@ -42,32 +54,64 @@ upgrade-npm-deps: .PHONY: ui-bump-version ui-bump-version: - version=$$(sed s/2/0/ < VERSION) && ./scripts/ui_release.sh --bump-version "$${version}" - cd web/ui && npm install - git add "./web/ui/package-lock.json" "./**/package.json" + version=$$(./scripts/get_module_version.sh) && ./scripts/ui_release.sh --bump-version "$${version}" + cd web/ui && pnpm install + cd web/ui/react-app && pnpm install + git add "./web/ui/pnpm-lock.yaml" "./web/ui/react-app/pnpm-lock.yaml" "./**/package.json" .PHONY: ui-install ui-install: - cd $(UI_PATH) && npm install + cd $(UI_PATH) && pnpm install + # The old React app has been separated from the npm workspaces setup to avoid + # issues with conflicting dependencies. This is a temporary solution until the + # new Mantine-based UI is fully integrated and the old app can be removed. + cd $(UI_PATH)/react-app && pnpm install .PHONY: ui-build ui-build: - cd $(UI_PATH) && CI="" npm run build +ifeq ($(BUILD_UI),mantine) + cd $(UI_PATH) && CI="" pnpm run build:mantine-ui +else + cd $(UI_PATH) && CI="" pnpm run build +endif .PHONY: ui-build-module ui-build-module: - cd $(UI_PATH) && npm run build:module + cd $(UI_PATH) && pnpm run build:module .PHONY: ui-test ui-test: - cd $(UI_PATH) && CI=true npm run test + cd $(UI_PATH) && CI=true pnpm run test .PHONY: ui-lint ui-lint: - cd $(UI_PATH) && npm run lint + cd $(UI_PATH) && pnpm run lint + # The old React app has been separated from the npm workspaces setup to avoid + # issues with conflicting dependencies. This is a temporary solution until the + # new Mantine-based UI is fully integrated and the old app can be removed. + cd $(UI_PATH)/react-app && pnpm run lint + +.PHONY: generate-promql-functions +generate-promql-functions: ui-install + @echo ">> generating PromQL function signatures" + @cd $(UI_PATH)/mantine-ui/src/promql/tools && $(GO) run ./gen_functions_list > ../functionSignatures.ts + @echo ">> generating PromQL function documentation" + @cd $(UI_PATH)/mantine-ui/src/promql/tools && $(GO) run ./gen_functions_docs $(CURDIR)/docs/querying/functions.md > ../functionDocs.tsx + @echo ">> formatting generated files" + @cd $(UI_PATH)/mantine-ui && pnpm exec prettier --write --print-width 120 src/promql/functionSignatures.ts src/promql/functionDocs.tsx + +.PHONY: check-generated-promql-functions +check-generated-promql-functions: generate-promql-functions + @echo ">> checking generated PromQL functions" + @git diff --exit-code -- $(UI_PATH)/mantine-ui/src/promql/functionSignatures.ts $(UI_PATH)/mantine-ui/src/promql/functionDocs.tsx || (echo "Generated PromQL function files are out of date. Please run 'make generate-promql-functions' and commit the changes." && false) .PHONY: assets -assets: ui-install ui-build +ifndef SKIP_UI_BUILD +assets: check-node-version ui-install ui-build +else +assets: + @echo '>> skipping assets build, pre-built assets provided' +endif .PHONY: assets-compress assets-compress: assets @@ -79,6 +123,15 @@ assets-tarball: assets @echo '>> packaging assets' scripts/package_assets.sh +.PHONY: protoc +protoc: + @echo ">> Installing protoc" + $(eval PROTOC_TMP := $(shell mktemp)) + curl -s -o $(PROTOC_TMP) -L $(PROTOC_URL) + unzip -u $(PROTOC_TMP) bin/protoc -d $(FIRST_GOPATH) + chmod +x $(FIRST_GOPATH)/bin/protoc + rm -v $(PROTOC_TMP) + .PHONY: parser parser: @echo ">> running goyacc to generate the .go file." @@ -96,7 +149,12 @@ promql/parser/generated_parser.y.go: promql/parser/generated_parser.y .PHONY: clean-parser clean-parser: @echo ">> cleaning generated parser" +ifeq (, $(shell command -v goyacc 2> /dev/null)) + @echo "goyacc not installed so skipping" + @echo "To install: \"go install golang.org/x/tools/cmd/goyacc@$(GOYACC_VERSION)\" or run \"make install-goyacc\"" +else @rm -f promql/parser/generated_parser.y.go +endif .PHONY: check-generated-parser check-generated-parser: clean-parser promql/parser/generated_parser.y.go @@ -114,32 +172,17 @@ install-goyacc: ifeq ($(GO_ONLY),1) test: common-test check-go-mod-version else -test: check-generated-parser common-test ui-build-module ui-test ui-lint check-go-mod-version +test: check-generated-parser common-test check-node-version ui-build-module ui-test ui-lint check-go-mod-version endif -.PHONY: npm_licenses -npm_licenses: ui-install - @echo ">> bundling npm licenses" - rm -f $(REACT_APP_NPM_LICENSES_TARBALL) npm_licenses - ln -s . npm_licenses - find npm_licenses/$(UI_NODE_MODULES_PATH) -iname "license*" | tar cfj $(REACT_APP_NPM_LICENSES_TARBALL) --files-from=- - rm -f npm_licenses - .PHONY: tarball -tarball: npm_licenses common-tarball +tarball: common-tarball .PHONY: docker -docker: npm_licenses common-docker - -plugins/plugins.go: plugins.yml plugins/generate.go - @echo ">> creating plugins list" - $(GO) generate -tags plugins ./plugins - -.PHONY: plugins -plugins: plugins/plugins.go +docker: common-docker .PHONY: build -build: assets npm_licenses assets-compress plugins common-build +build: assets assets-compress common-build .PHONY: bench_tsdb bench_tsdb: $(PROMU) @@ -163,11 +206,37 @@ check-go-mod-version: @echo ">> checking go.mod version matching" @./scripts/check-go-mod-version.sh +.PHONY: update-features-testdata +update-features-testdata: + @echo ">> updating features testdata" + @$(GO) test ./cmd/prometheus -run TestFeaturesAPI -update-features + +GO_SUBMODULE_DIRS := documentation/examples/remote_storage internal/tools web/ui/mantine-ui/src/promql/tools compliance + .PHONY: update-all-go-deps -update-all-go-deps: - @$(MAKE) update-go-deps - @echo ">> updating Go dependencies in ./documentation/examples/remote_storage/" - @cd ./documentation/examples/remote_storage/ && for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ - $(GO) get -d $$m; \ +update-all-go-deps: update-go-deps + $(foreach dir,$(GO_SUBMODULE_DIRS),$(MAKE) update-go-deps-in-dir DIR=$(dir);) + @echo ">> syncing Go workspace" + @$(GO) work sync + +.PHONY: update-go-deps-in-dir +update-go-deps-in-dir: + @echo ">> updating Go dependencies in ./$(DIR)/" + @cd ./$(DIR) && for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ + $(GO) get $$m; \ done - @cd ./documentation/examples/remote_storage/ && $(GO) mod tidy + @cd ./$(DIR) && $(GO) mod tidy + +.PHONY: check-node-version +check-node-version: + @./scripts/check-node-version.sh + +.PHONY: bump-go-version +bump-go-version: + @echo ">> bumping Go minor version" + @./scripts/bump_go_version.sh + +.PHONY: generate-fuzzing-seed-corpus +generate-fuzzing-seed-corpus: + @echo ">> Generating fuzzing seed corpus" + @$(GO) generate -tags fuzzing ./util/fuzzing/corpus_gen diff --git a/Makefile.common b/Makefile.common index 34d65bb56de..a7c5f553e12 100644 --- a/Makefile.common +++ b/Makefile.common @@ -1,4 +1,4 @@ -# Copyright 2018 The Prometheus Authors +# Copyright The Prometheus Authors # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -55,13 +55,14 @@ ifneq ($(shell command -v gotestsum 2> /dev/null),) endif endif -PROMU_VERSION ?= 0.17.0 +PROMU_VERSION ?= 0.20.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.60.2 +GOLANGCI_LINT_VERSION ?= v2.11.4 +GOLANGCI_FMT_OPTS ?= # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) @@ -81,11 +82,32 @@ endif PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) -DOCKERFILE_PATH ?= ./Dockerfile DOCKERBUILD_CONTEXT ?= ./ DOCKER_REPO ?= prom -DOCKER_ARCHS ?= amd64 +# Check if deprecated DOCKERFILE_PATH is set +ifdef DOCKERFILE_PATH +$(error DOCKERFILE_PATH is deprecated. Use DOCKERFILE_VARIANTS ?= $(DOCKERFILE_PATH) in the Makefile) +endif + +DOCKER_ARCHS ?= amd64 arm64 armv7 ppc64le riscv64 s390x +DOCKERFILE_VARIANTS ?= $(wildcard Dockerfile Dockerfile.*) + +# Function to extract variant from Dockerfile label. +# Returns the variant name from io.prometheus.image.variant label, or "default" if not found. +define dockerfile_variant +$(strip $(or $(shell sed -n 's/.*io\.prometheus\.image\.variant="\([^"]*\)".*/\1/p' $(1)),default)) +endef + +# Check for duplicate variant names (including default for Dockerfiles without labels). +DOCKERFILE_VARIANT_NAMES := $(foreach df,$(DOCKERFILE_VARIANTS),$(call dockerfile_variant,$(df))) +DOCKERFILE_VARIANT_NAMES_SORTED := $(sort $(DOCKERFILE_VARIANT_NAMES)) +ifneq ($(words $(DOCKERFILE_VARIANT_NAMES)),$(words $(DOCKERFILE_VARIANT_NAMES_SORTED))) +$(error Duplicate variant names found. Each Dockerfile must have a unique io.prometheus.image.variant label, and only one can be without a label (default)) +endif + +# Build variant:dockerfile pairs for shell iteration. +DOCKERFILE_VARIANTS_WITH_NAMES := $(foreach df,$(DOCKERFILE_VARIANTS),$(call dockerfile_variant,$(df)):$(df)) BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS)) PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS)) @@ -111,7 +133,7 @@ common-all: precheck style check_license lint yamllint unused build test .PHONY: common-style common-style: @echo ">> checking code style" - @fmtRes=$$($(GOFMT) -d $$(find . -path ./vendor -prune -o -name '*.go' -print)); \ + @fmtRes=$$($(GOFMT) -d $$(git ls-files '*.go' ':!:vendor/*' || find . -path ./vendor -prune -o -name '*.go' -print)); \ if [ -n "$${fmtRes}" ]; then \ echo "gofmt checking failed!"; echo "$${fmtRes}"; echo; \ echo "Please ensure you are using $$($(GO) version) for formatting code."; \ @@ -121,13 +143,19 @@ common-style: .PHONY: common-check_license common-check_license: @echo ">> checking license header" - @licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*') ; do \ + @licRes=$$(for file in $$(git ls-files '*.go' ':!:vendor/*' || find . -path ./vendor -prune -o -type f -iname '*.go' -print) ; do \ awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \ done); \ if [ -n "$${licRes}" ]; then \ echo "license header checking failed:"; echo "$${licRes}"; \ exit 1; \ fi + @echo ">> checking for copyright years 2026 or later" + @futureYearRes=$$(git grep -E 'Copyright (202[6-9]|20[3-9][0-9])' -- '*.go' ':!:vendor/*' || true); \ + if [ -n "$${futureYearRes}" ]; then \ + echo "Files with copyright year 2026 or later found (should use 'Copyright The Prometheus Authors'):"; echo "$${futureYearRes}"; \ + exit 1; \ + fi .PHONY: common-deps common-deps: @@ -138,7 +166,7 @@ common-deps: update-go-deps: @echo ">> updating Go dependencies" @for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ - $(GO) get -d $$m; \ + $(GO) get $$m; \ done $(GO) mod tidy @@ -156,9 +184,13 @@ $(GOTEST_DIR): @mkdir -p $@ .PHONY: common-format -common-format: +common-format: $(GOLANGCI_LINT) @echo ">> formatting code" $(GO) fmt $(pkgs) +ifdef GOLANGCI_LINT + @echo ">> formatting code with golangci-lint" + $(GOLANGCI_LINT) fmt $(GOLANGCI_FMT_OPTS) +endif .PHONY: common-vet common-vet: @@ -215,28 +247,142 @@ common-docker-repo-name: .PHONY: common-docker $(BUILD_DOCKER_ARCHS) common-docker: $(BUILD_DOCKER_ARCHS) $(BUILD_DOCKER_ARCHS): common-docker-%: - docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ - -f $(DOCKERFILE_PATH) \ - --build-arg ARCH="$*" \ - --build-arg OS="linux" \ - $(DOCKERBUILD_CONTEXT) + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + distroless_arch="$*"; \ + if [ "$*" = "armv7" ]; then \ + distroless_arch="arm"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Building default variant ($$variant_name) for linux-$* using $$dockerfile"; \ + docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ + -f $$dockerfile \ + --build-arg ARCH="$*" \ + --build-arg OS="linux" \ + --build-arg DISTROLESS_ARCH="$$distroless_arch" \ + $(DOCKERBUILD_CONTEXT); \ + if [ "$$variant_name" != "default" ]; then \ + echo "Tagging default variant with $$variant_name suffix"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ + "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ + fi; \ + else \ + echo "Building $$variant_name variant for linux-$* using $$dockerfile"; \ + docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" \ + -f $$dockerfile \ + --build-arg ARCH="$*" \ + --build-arg OS="linux" \ + --build-arg DISTROLESS_ARCH="$$distroless_arch" \ + $(DOCKERBUILD_CONTEXT); \ + fi; \ + done .PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS) common-docker-publish: $(PUBLISH_DOCKER_ARCHS) $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: - docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Pushing $$variant_name variant for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Pushing default variant ($$variant_name) for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)"; \ + fi; \ + if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Pushing $$variant_name variant version tags for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Pushing default variant version tag for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"; \ + fi; \ + fi; \ + done DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION))) .PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS) common-docker-tag-latest: $(TAG_DOCKER_ARCHS) $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: - docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest" - docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)" + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Tagging $$variant_name variant for linux-$* as latest"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest-$$variant_name"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Tagging default variant ($$variant_name) for linux-$* as latest"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"; \ + fi; \ + done .PHONY: common-docker-manifest common-docker-manifest: - DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)) - DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Creating manifest for $$variant_name variant"; \ + refs=""; \ + for arch in $(DOCKER_ARCHS); do \ + refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ + done; \ + if [ -z "$$refs" ]; then \ + echo "Skipping manifest for $$variant_name variant (no supported architectures)"; \ + continue; \ + fi; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" $$refs; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Creating default variant ($$variant_name) manifest"; \ + refs=""; \ + for arch in $(DOCKER_ARCHS); do \ + refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:$(SANITIZED_DOCKER_IMAGE_TAG)"; \ + done; \ + if [ -z "$$refs" ]; then \ + echo "Skipping default variant manifest (no supported architectures)"; \ + continue; \ + fi; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $$refs; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)"; \ + fi; \ + if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Creating manifest for $$variant_name variant version tag"; \ + refs=""; \ + for arch in $(DOCKER_ARCHS); do \ + refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ + done; \ + if [ -z "$$refs" ]; then \ + echo "Skipping version-tag manifest for $$variant_name variant (no supported architectures)"; \ + continue; \ + fi; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name" $$refs; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Creating default variant version tag manifest"; \ + refs=""; \ + for arch in $(DOCKER_ARCHS); do \ + refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:v$(DOCKER_MAJOR_VERSION_TAG)"; \ + done; \ + if [ -z "$$refs" ]; then \ + echo "Skipping default variant version-tag manifest (no supported architectures)"; \ + continue; \ + fi; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)" $$refs; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)"; \ + fi; \ + fi; \ + done .PHONY: promu promu: $(PROMU) @@ -248,8 +394,8 @@ $(PROMU): cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu rm -r $(PROMU_TMP) -.PHONY: proto -proto: +.PHONY: common-proto +common-proto: @echo ">> generating code from proto files" @./scripts/genproto.sh @@ -261,6 +407,10 @@ $(GOLANGCI_LINT): | sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION) endif +.PHONY: common-print-golangci-lint-version +common-print-golangci-lint-version: + @echo $(GOLANGCI_LINT_VERSION) + .PHONY: precheck precheck:: diff --git a/NOTICE b/NOTICE index 8605c258e32..b82d284fb19 100644 --- a/NOTICE +++ b/NOTICE @@ -101,8 +101,19 @@ https://github.com/microsoft/vscode-codicons Copyright (c) Microsoft Corporation and other contributors See https://github.com/microsoft/vscode-codicons/blob/main/LICENSE for license details. +Mantine UI +https://github.com/mantinedev/mantine +Copyright (c) 2021 Vitaly Rtishchev +See https://github.com/mantinedev/mantine/blob/master/LICENSE for license details. + We also use code from a large number of npm packages. For details, see: - https://github.com/prometheus/prometheus/blob/main/web/ui/react-app/package.json - https://github.com/prometheus/prometheus/blob/main/web/ui/react-app/package-lock.json -- The individual package licenses as copied from the node_modules directory can be found in - the npm_licenses.tar.bz2 archive in release tarballs and Docker images. +- https://github.com/prometheus/prometheus/blob/main/web/ui/mantine-ui/package.json +- https://github.com/prometheus/prometheus/blob/main/web/ui/pnpm-lock.yaml +- The individual licenses of the packages bundled into the new (mantine) web UI + are collected at build time, embedded in the Prometheus binary, and served by + the web UI at /assets/third-party-licenses.txt. +- The licenses of the packages bundled into the old (react-app) web UI, served + via --enable-feature=old-ui, are extracted at build time into a + *.LICENSE.txt file that is embedded alongside the old UI's JavaScript bundle. diff --git a/README.md b/README.md index df974e1097b..9f644d57dc4 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ examples and guides.

[![Docker Pulls](https://img.shields.io/docker/pulls/prom/prometheus.svg?maxAge=604800)][hub] [![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/prometheus)](https://goreportcard.com/report/github.com/prometheus/prometheus) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486) +[![govulncheck](https://github.com/prometheus/prometheus/actions/workflows/govulncheck.yml/badge.svg?event=schedule)](https://github.com/prometheus/prometheus/actions/workflows/govulncheck.yml) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus) [![CLOMonitor](https://img.shields.io/endpoint?url=https://clomonitor.io/api/projects/cncf/prometheus/badge)](https://clomonitor.io/projects/cncf/prometheus) -[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus) [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus) @@ -40,14 +40,14 @@ The features that distinguish Prometheus from other metrics and monitoring syste ## Install -There are various ways of installing Prometheus. +There are various ways to install Prometheus. ### Precompiled binaries Precompiled binaries for released versions are available in the [*download* section](https://prometheus.io/download/) on [prometheus.io](https://prometheus.io). Using the latest production release binary -is the recommended way of installing Prometheus. +is the recommended way to install Prometheus. See the [Installing](https://prometheus.io/docs/introduction/install/) chapter in the documentation for all the details. @@ -65,11 +65,11 @@ Prometheus will now be reachable at . ### Building from source -To build Prometheus from source code, You need: +To build Prometheus from source code, you need: -* Go [version 1.17 or greater](https://golang.org/doc/install). -* NodeJS [version 16 or greater](https://nodejs.org/). -* npm [version 7 or greater](https://www.npmjs.com/). +* Go: Version specified in [go.mod](./go.mod) or greater. +* NodeJS: Version specified in [.nvmrc](./web/ui/.nvmrc) or greater. +* npm: Version 10 or greater (check with `npm --version` and [here](https://www.npmjs.com/)). Start by cloning the repository: @@ -82,15 +82,15 @@ You can use the `go` tool to build and install the `prometheus` and `promtool` binaries into your `GOPATH`: ```bash -GO111MODULE=on go install github.com/prometheus/prometheus/cmd/... +go install github.com/prometheus/prometheus/cmd/... prometheus --config.file=your_config.yml ``` *However*, when using `go install` to build Prometheus, Prometheus will expect to be able to -read its web assets from local filesystem directories under `web/ui/static` and -`web/ui/templates`. In order for these assets to be found, you will have to run Prometheus -from the root of the cloned repository. Note also that these directories do not include the -React UI unless it has been built explicitly using `make assets` or `make build`. +read its web assets from local filesystem directories under `web/ui/static`. In order for +these assets to be found, you will have to run Prometheus from the root of the cloned +repository. Note also that this directory does not include the React UI unless it has been +built explicitly using `make assets` or `make build`. An example of the above configuration file can be found [here.](https://github.com/prometheus/prometheus/blob/main/documentation/examples/prometheus.yml) @@ -113,16 +113,31 @@ The Makefile provides several targets: ### Service discovery plugins -Prometheus is bundled with many service discovery plugins. -When building Prometheus from source, you can edit the [plugins.yml](./plugins.yml) -file to disable some service discoveries. The file is a yaml-formated list of go -import path that will be built into the Prometheus binary. +Prometheus is bundled with many service discovery plugins. You can customize +which service discoveries are included in your build using Go build tags. -After you have changed the file, you -need to run `make build` again. +To exclude service discoveries when building with `make build`, add the desired +tags to the `.promu.yml` file under `build.tags.all`: -If you are using another method to compile Prometheus, `make plugins` will -generate the plugins file accordingly. +```yaml +build: + tags: + all: + - netgo + - builtinassets + - remove_all_sd # Exclude all optional SDs + - enable_kubernetes_sd # Re-enable only kubernetes +``` + +Then run `make build` as usual. Alternatively, when using `go build` directly: + +```bash +go build -tags "remove_all_sd,enable_kubernetes_sd" ./cmd/prometheus +``` + +Available build tags: +* `remove_all_sd` - Exclude all optional service discoveries (keeps file_sd, static_sd, and http_sd) +* `enable__sd` - Re-enable a specific SD when using `remove_all_sd` If you add out-of-tree plugins, which we do not endorse at the moment, additional steps might be needed to adjust the `go.mod` and `go.sum` files. As @@ -130,18 +145,28 @@ always, be extra careful when loading third party code. ### Building the Docker image -The `make docker` target is designed for use in our CI system. You can build a docker image locally with the following commands: ```bash make promu promu crossbuild -p linux/amd64 -make npm_licenses make common-docker-amd64 ``` +The `make docker` target is intended only for use in our CI system and will not +produce a fully working image when run locally. + ## Using Prometheus as a Go Library +Within the Prometheus project, repositories such as [prometheus/common](https://github.com/prometheus/common) and +[prometheus/client-golang](https://github.com/prometheus/client-golang) are designed as re-usable libraries. + +The [prometheus/prometheus](https://github.com/prometheus/prometheus) repository builds a stand-alone program and is not +designed for use as a library. We are aware that people do use parts as such, +and we do not put any deliberate inconvenience in the way, but we want you to be +aware that no care has been taken to make it work well as a library. For instance, +you may encounter errors that only surface when used as a library. + ### Remote Write We are publishing our Remote Write protobuf independently at @@ -158,8 +183,19 @@ This is experimental. ### Prometheus code base In order to comply with [go mod](https://go.dev/ref/mod#versions) rules, -Prometheus release number do not exactly match Go module releases. For the -Prometheus v2.y.z releases, we are publishing equivalent v0.y.z tags. +Prometheus release number do not exactly match Go module releases. + +For the +Prometheus v3.y.z releases, we are publishing equivalent v0.3y.z tags. The y in v0.3y.z is always padded to two digits, with a leading zero if needed. + +Therefore, a user that would want to use Prometheus v3.0.0 as a library could do: + +```shell +go get github.com/prometheus/prometheus@v0.300.0 +``` + +For the +Prometheus v2.y.z releases, we published the equivalent v0.y.z tags. Therefore, a user that would want to use Prometheus v2.35.0 as a library could do: @@ -177,7 +213,7 @@ For more information on building, running, and developing on the React-based UI, ## More information -* Godoc documentation is available via [pkg.go.dev](https://pkg.go.dev/github.com/prometheus/prometheus). Due to peculiarities of Go Modules, v2.x.y will be displayed as v0.x.y. +* Godoc documentation is available via [pkg.go.dev](https://pkg.go.dev/github.com/prometheus/prometheus). Due to peculiarities of Go Modules, v3.y.z will be displayed as v0.3y.z (the y in v0.3y.z is always padded to two digits, with a leading zero if needed), while v2.y.z will be displayed as v0.y.z. * See the [Community page](https://prometheus.io/community) for how to reach the Prometheus developers and users on various communication channels. ## Contributing diff --git a/RELEASE.md b/RELEASE.md index b978a3c2261..cc309d1c3cd 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -5,61 +5,27 @@ This page describes the release process and the currently planned schedule for u ## Release schedule Release cadence of first pre-releases being cut is 6 weeks. - -| release series | date of first pre-release (year-month-day) | release shepherd | -|----------------|--------------------------------------------|---------------------------------------------| -| v2.4 | 2018-09-06 | Goutham Veeramachaneni (GitHub: @gouthamve) | -| v2.5 | 2018-10-24 | Frederic Branczyk (GitHub: @brancz) | -| v2.6 | 2018-12-05 | Simon Pasquier (GitHub: @simonpasquier) | -| v2.7 | 2019-01-16 | Goutham Veeramachaneni (GitHub: @gouthamve) | -| v2.8 | 2019-02-27 | Ganesh Vernekar (GitHub: @codesome) | -| v2.9 | 2019-04-10 | Brian Brazil (GitHub: @brian-brazil) | -| v2.10 | 2019-05-22 | Björn Rabenstein (GitHub: @beorn7) | -| v2.11 | 2019-07-03 | Frederic Branczyk (GitHub: @brancz) | -| v2.12 | 2019-08-14 | Julius Volz (GitHub: @juliusv) | -| v2.13 | 2019-09-25 | Krasi Georgiev (GitHub: @krasi-georgiev) | -| v2.14 | 2019-11-06 | Chris Marchbanks (GitHub: @csmarchbanks) | -| v2.15 | 2019-12-18 | Bartek Plotka (GitHub: @bwplotka) | -| v2.16 | 2020-01-29 | Callum Styan (GitHub: @cstyan) | -| v2.17 | 2020-03-11 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.18 | 2020-04-22 | Bartek Plotka (GitHub: @bwplotka) | -| v2.19 | 2020-06-03 | Ganesh Vernekar (GitHub: @codesome) | -| v2.20 | 2020-07-15 | Björn Rabenstein (GitHub: @beorn7) | -| v2.21 | 2020-08-26 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.22 | 2020-10-07 | Frederic Branczyk (GitHub: @brancz) | -| v2.23 | 2020-11-18 | Ganesh Vernekar (GitHub: @codesome) | -| v2.24 | 2020-12-30 | Björn Rabenstein (GitHub: @beorn7) | -| v2.25 | 2021-02-10 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.26 | 2021-03-24 | Bartek Plotka (GitHub: @bwplotka) | -| v2.27 | 2021-05-05 | Chris Marchbanks (GitHub: @csmarchbanks) | -| v2.28 | 2021-06-16 | Julius Volz (GitHub: @juliusv) | -| v2.29 | 2021-07-28 | Frederic Branczyk (GitHub: @brancz) | -| v2.30 | 2021-09-08 | Ganesh Vernekar (GitHub: @codesome) | -| v2.31 | 2021-10-20 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.32 | 2021-12-01 | Julius Volz (GitHub: @juliusv) | -| v2.33 | 2022-01-12 | Björn Rabenstein (GitHub: @beorn7) | -| v2.34 | 2022-02-23 | Chris Marchbanks (GitHub: @csmarchbanks) | -| v2.35 | 2022-04-06 | Augustin Husson (GitHub: @nexucis) | -| v2.36 | 2022-05-18 | Matthias Loibl (GitHub: @metalmatze) | -| v2.37 LTS | 2022-06-29 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.38 | 2022-08-10 | Julius Volz (GitHub: @juliusv) | -| v2.39 | 2022-09-21 | Ganesh Vernekar (GitHub: @codesome) | -| v2.40 | 2022-11-02 | Ganesh Vernekar (GitHub: @codesome) | -| v2.41 | 2022-12-14 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.42 | 2023-01-25 | Kemal Akkoyun (GitHub: @kakkoyun) | -| v2.43 | 2023-03-08 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.44 | 2023-04-19 | Bryan Boreham (GitHub: @bboreham) | -| v2.45 LTS | 2023-05-31 | Jesus Vazquez (Github: @jesusvazquez) | -| v2.46 | 2023-07-12 | Julien Pivotto (GitHub: @roidelapluie) | -| v2.47 | 2023-08-23 | Bryan Boreham (GitHub: @bboreham) | -| v2.48 | 2023-10-04 | Levi Harrison (GitHub: @LeviHarrison) | -| v2.49 | 2023-12-05 | Bartek Plotka (GitHub: @bwplotka) | -| v2.50 | 2024-01-16 | Augustin Husson (GitHub: @nexucis) | -| v2.51 | 2024-03-07 | Bryan Boreham (GitHub: @bboreham) | -| v2.52 | 2024-04-22 | Arthur Silva Sens (GitHub: @ArthurSens) | -| v2.53 LTS | 2024-06-03 | George Krajcsovits (GitHub: @krajorama) | -| v2.54 | 2024-07-17 | Bryan Boreham (GitHub: @bboreham) | -| v2.55 | 2024-09-17 | Bryan Boreham (GitHub: @bboreham) | +Please see [the v2.55 RELEASE.md](https://github.com/prometheus/prometheus/blob/release-2.55/RELEASE.md) for the v2 release series schedule. + +| release series | date of first pre-release (year-month-day) | release shepherd | +|----------------|--------------------------------------------|-------------------------------------------------------------------------| +| v3.0 | 2024-11-14 | Jan Fajerski (GitHub: @jan--f) | +| v3.1 | 2024-12-17 | Bryan Boreham (GitHub: @bboreham) | +| v3.2 | 2025-01-28 | Jan Fajerski (GitHub: @jan--f) | +| v3.3 | 2025-03-11 | Ayoub Mrini (Github: @machine424) | +| v3.4 | 2025-04-29 | Jan-Otto Kröpke (Github: @jkroepke) | +| v3.5 LTS | 2025-06-03 | Bryan Boreham (GitHub: @bboreham) | +| v3.6 | 2025-08-01 | Ayoub Mrini (Github: @machine424) | +| v3.7 | 2025-09-25 | Arthur Sens and György Krajcsovits (Github: @ArthurSens and @krajorama) | +| v3.8 | 2025-11-06 | Jan Fajerski (GitHub: @jan--f) | +| v3.9 | 2025-12-18 | Bryan Boreham (GitHub: @bboreham) | +| v3.10 | 2026-02-05 | Ganesh Vernekar (Github: @codesome) | +| v3.11 | 2026-03-25 | Julien Pivotto (GitHub: @roidelapluie) | +| v3.12 | 2026-05-06 | Bartek Plotka (GitHub: @bwplotka) | +| v3.13 | 2026-06-17 | György Krajcsovits (Github: @krajorama) | +| v3.14 | 2026-07-29 | **volunteer welcome** | +| v3.15 | 2026-09-09 | **volunteer welcome** | +| v3.16 | 2026-10-21 | **volunteer welcome** | If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice. @@ -82,7 +48,7 @@ These instructions are currently valid for the Prometheus server, i.e. the [prom We use [Semantic Versioning](https://semver.org/). -We maintain a separate branch for each minor release, named `release-.`, e.g. `release-1.1`, `release-2.0`. +We maintain a separate branch for each minor release, named `release-.`, e.g. `release-2.1`, `release-3.0`. Note that branch protection kicks in automatically for any branches whose name starts with `release-`. Never use names starting with `release-` for branches that are not release branches. @@ -145,25 +111,24 @@ though this may be done at convenient times (e.g. by the UI maintainers) that ar ### 1. Prepare your release -At the start of a new major or minor release cycle create the corresponding release branch based on the main branch. For example if we're releasing `2.17.0` and the previous stable release is `2.16.0` we need to create a `release-2.17` branch. Note that all releases are handled in protected release branches, see the above `Branch management and versioning` section. Release candidates and patch releases for any given major or minor release happen in the same `release-.` branch. Do not create `release-` for patch or release candidate releases. +At the start of a new major or minor release cycle create the corresponding release branch based on the main branch. For example if we're releasing `3.7.0` and the previous stable release is `3.6.0` we need to create a `release-3.7` branch. Note that all releases are handled in protected release branches, see the [Branch management and versioning strategy](#branch-management-and-versioning-strategy) section. Release candidates and patch releases for any given major or minor release happen in the same `release-.` branch. Do not create `release-` for patch or release candidate releases. Changes for a patch release or release candidate should be merged into the previously mentioned release branch via pull request. -Bump the version in the `VERSION` file and update `CHANGELOG.md`. Do this in a proper PR pointing to the release branch as this gives others the opportunity to chime in on the release in general and on the addition to the changelog in particular. For a release candidate, append something like `-rc.0` to the version (with the corresponding changes to the tag name, the release name etc.). +Bump the version in the `VERSION` file and update `CHANGELOG.md`. Do this in a proper PR pointing to the release branch as this gives others the opportunity to chime in on the release in general and on the addition to `CHANGELOG.md` in particular. For a release candidate, append something like `-rc.0` to the version (with the corresponding changes to the tag name, the release name etc.). -When updating the `CHANGELOG.md` look at all PRs included in the release since the last release and verify if they need a changelog entry. +Use [`scripts/generate_release_notes.sh`](scripts/generate_release_notes.sh) to produce a starting point for the `CHANGELOG.md` entries. It requires the [`release-notes`](https://github.com/kubernetes/release/tree/master/cmd/release-notes) tool and a `GITHUB_TOKEN` with read access to the repository: -Note that `CHANGELOG.md` should only document changes relevant to users of Prometheus, including external API changes, performance improvements, and new features. Do not document changes of internal interfaces, code refactorings and clean-ups, changes to the build process, etc. People interested in these are asked to refer to the git history. +Run it for the target release version (RC, final, or patch; for example `v3.11.0-rc.0`, `v3.11.0`, or `v3.11.1`): -For release candidates still update `CHANGELOG.md`, but when you cut the final release later, merge all the changes from the pre-releases into the one final update. +```bash +GITHUB_TOKEN= scripts/generate_release_notes.sh 3.Y_SET_ME.Z_SET_ME +``` -Entries in the `CHANGELOG.md` are meant to be in this order: +Review the output carefully before adding it to `CHANGELOG.md`: -* `[SECURITY]` - A bugfix that specifically fixes a security issue. -* `[CHANGE]` -* `[FEATURE]` -* `[ENHANCEMENT]` -* `[BUGFIX]` +- Remove entries already present in a previous release's Changelog. Duplicates can appear for fixes merged to the previous release branch. +- `CHANGELOG.md` should only document changes relevant to users of Prometheus, including external API changes, performance improvements, and new features. Do not document changes of internal interfaces, code refactorings and clean-ups, changes to the build process, etc. People interested in these are asked to refer to the git history. Then bump the UI module version: @@ -171,6 +136,9 @@ Then bump the UI module version: make ui-bump-version ``` +All of the above goes to make a PR, which you should create targeting the `release-x.y` branch, then wait for CI to complete and get an approval. +Once the PR is ready, merge it into the release branch and pull it down to your work area. Then you are ready for the next step: + ### 2. Draft the new release Tag the new release via the following commands: @@ -181,19 +149,7 @@ git tag -s "${tag}" -m "${tag}" git push origin "${tag}" ``` -Go modules versioning requires strict use of semver. Because we do not commit to -avoid code-level breaking changes for the libraries between minor releases of -the Prometheus server, we use major version zero releases for the libraries. - -Tag the new library release via the following commands: - -```bash -tag="v$(sed s/2/0/ < VERSION)" -git tag -s "${tag}" -m "${tag}" -git push origin "${tag}" -``` - -Optionally, you can use this handy `.gitconfig` alias. +Alternatively, you can use this handy `.gitconfig` alias. ```ini [alias] @@ -204,14 +160,29 @@ Then release with `git tag-release`. Signing a tag with a GPG key is appreciated, but in case you can't add a GPG key to your Github account using the following [procedure](https://help.github.com/articles/generating-a-gpg-key/), you can replace the `-s` flag by `-a` flag of the `git tag` command to only annotate the tag without signing. -Once a tag is created, the release process through CircleCI will be triggered for this tag and Circle CI will draft the GitHub release using the `prombot` account. +Once a tag is created, the release process through Github Actions will be triggered for this tag and Github Actions will draft the GitHub release using the `prombot` account. Finally, wait for the build step for the tag to finish. The point here is to wait for tarballs to be uploaded to the Github release and the container images to be pushed to the Docker Hub and Quay.io. Once that has happened, click _Publish release_, which will make the release publicly visible and create a GitHub notification. **Note:** for a release candidate version ensure the _This is a pre-release_ box is checked when drafting the release in the Github UI. The CI job should take care of this but it's a good idea to double check before clicking _Publish release_.` -### 3. Wrapping up +### 3. Tag the library release + +Go modules versioning requires strict use of semver. Because we do not commit to +avoid code-level breaking changes for the libraries between minor releases of +the Prometheus server, we use major version zero releases for the libraries. + +Tagging the new library release works similar to the normal release tagging, +but without the subsequent build and publish steps. Use the following commands: + +```bash +tag="v$(./scripts/get_module_version.sh)" +git tag -s "${tag}" -m "${tag}" +git push origin "${tag}" +``` + +### 4. Wrapping up -For release candidate versions (`v2.16.0-rc.0`), run the benchmark for 3 days using the `/prombench vX.Y.Z` command, `vX.Y.Z` being the latest stable patch release's tag of the previous minor release series, such as `v2.15.2`. +For release candidate versions (`v3.6.0-rc.0`), run the benchmark for 3 days using the `/prombench vX.Y.Z` command, `vX.Y.Z` being the latest stable patch release's tag of the previous minor release series, such as `v3.5.2`. If the release has happened in the latest release branch, merge the changes into main. diff --git a/SECURITY.md b/SECURITY.md index fed02d85c79..5e6f976dbff 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -3,4 +3,4 @@ The Prometheus security policy, including how to report vulnerabilities, can be found here: - +[https://prometheus.io/docs/operating/security/](https://prometheus.io/docs/operating/security/) diff --git a/VERSION b/VERSION index 0a756ea0a78..c10780c628a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.55.1 +3.13.1 diff --git a/cmd/prometheus/features_test.go b/cmd/prometheus/features_test.go new file mode 100644 index 00000000000..5907c87247c --- /dev/null +++ b/cmd/prometheus/features_test.go @@ -0,0 +1,125 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/util/testutil" +) + +var updateFeatures = flag.Bool("update-features", false, "update features.json golden file") + +func TestFeaturesAPI(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + t.Parallel() + + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "prometheus.yml") + require.NoError(t, os.WriteFile(configFile, []byte{}, 0o644)) + + port := testutil.RandomUnprivilegedPort(t) + prom := prometheusCommandWithLogging( + t, + configFile, + port, + fmt.Sprintf("--storage.tsdb.path=%s", tmpDir), + ) + require.NoError(t, prom.Start()) + + baseURL := fmt.Sprintf("http://127.0.0.1:%d", port) + + // Wait for Prometheus to be ready. + require.Eventually(t, func() bool { + resp, err := http.Get(baseURL + "/-/ready") + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK + }, 10*time.Second, 100*time.Millisecond, "Prometheus didn't become ready in time") + + // Fetch features from the API. + resp, err := http.Get(baseURL + "/api/v1/features") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + // Parse API response. + var apiResponse struct { + Status string `json:"status"` + Data map[string]map[string]bool `json:"data"` + } + require.NoError(t, json.Unmarshal(body, &apiResponse)) + require.Equal(t, "success", apiResponse.Status) + + goldenPath := filepath.Join("testdata", "features.json") + + // If update flag is set, write the current features to the golden file. + if *updateFeatures { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + require.NoError(t, encoder.Encode(apiResponse.Data)) + // Ensure testdata directory exists. + require.NoError(t, os.MkdirAll(filepath.Dir(goldenPath), 0o755)) + require.NoError(t, os.WriteFile(goldenPath, buf.Bytes(), 0o644)) + t.Logf("Updated golden file: %s", goldenPath) + return + } + + // Load golden file. + goldenData, err := os.ReadFile(goldenPath) + require.NoError(t, err, "Failed to read golden file %s. Run 'make update-features-testdata' to generate it.", goldenPath) + + var expectedFeatures map[string]map[string]bool + require.NoError(t, json.Unmarshal(goldenData, &expectedFeatures)) + + // The labels implementation depends on build tags (stringlabels, slicelabels, or dedupelabels). + // We need to update the expected features to match the current build. + if prometheusFeatures, ok := expectedFeatures["prometheus"]; ok { + // Remove all label implementation features from expected. + delete(prometheusFeatures, "stringlabels") + delete(prometheusFeatures, "slicelabels") + delete(prometheusFeatures, "dedupelabels") + // Add the current implementation. + if actualPrometheus, ok := apiResponse.Data["prometheus"]; ok { + for _, impl := range []string{"stringlabels", "slicelabels", "dedupelabels"} { + if actualPrometheus[impl] { + prometheusFeatures[impl] = true + } + } + } + } + + // Compare the features data with the golden file. + require.Equal(t, expectedFeatures, apiResponse.Data, "Features mismatch. Run 'make update-features-testdata' to update the golden file.") +} diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index b3bcb78b78b..6795c6bb4cd 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,19 +16,22 @@ package main import ( "context" + "encoding/json" "errors" "fmt" + "log/slog" "math" "math/bits" "net" "net/http" - _ "net/http/pprof" // Comment this line to disable pprof endpoint. "net/url" "os" "os/signal" "path/filepath" + goregexp "regexp" //nolint:depguard // The Prometheus client library requires us to pass a regexp from this package. "runtime" "runtime/debug" + "slices" "strconv" "strings" "sync" @@ -38,28 +41,26 @@ import ( "github.com/KimMachineGun/automemlimit/memlimit" "github.com/alecthomas/kingpin/v2" "github.com/alecthomas/units" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/grafana/regexp" "github.com/mwitkow/go-conntrack" "github.com/oklog/run" + remoteapi "github.com/prometheus/client_golang/exp/api/remote" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" versioncollector "github.com/prometheus/client_golang/prometheus/collectors/version" "github.com/prometheus/common/model" - "github.com/prometheus/common/promlog" - promlogflag "github.com/prometheus/common/promlog/flag" + "github.com/prometheus/common/promslog" + promslogflag "github.com/prometheus/common/promslog/flag" "github.com/prometheus/common/version" toolkit_web "github.com/prometheus/exporter-toolkit/web" "go.uber.org/atomic" "go.uber.org/automaxprocs/maxprocs" + "k8s.io/client-go/rest" "k8s.io/klog" klogv2 "k8s.io/klog/v2" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" - "github.com/prometheus/prometheus/discovery/legacymanager" - "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -73,16 +74,60 @@ import ( "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage/remote" + "github.com/prometheus/prometheus/template" "github.com/prometheus/prometheus/tracing" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/agent" - "github.com/prometheus/prometheus/tsdb/wlog" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/fileutil" + "github.com/prometheus/prometheus/util/compression" "github.com/prometheus/prometheus/util/documentcli" + "github.com/prometheus/prometheus/util/features" "github.com/prometheus/prometheus/util/logging" + "github.com/prometheus/prometheus/util/notifications" prom_runtime "github.com/prometheus/prometheus/util/runtime" "github.com/prometheus/prometheus/web" ) +// klogv1OutputCallDepth is the stack depth where we can find the origin of this call. +const klogv1OutputCallDepth = 6 + +// klogv1DefaultPrefixLength is the length of the log prefix that we have to strip out. +const klogv1DefaultPrefixLength = 53 + +// klogv1Writer is used in SetOutputBySeverity call below to redirect any calls +// to klogv1 to end up in klogv2. +// This is a hack to support klogv1 without use of go-kit/log. It is inspired +// by klog's upstream klogv1/v2 coexistence example: +// https://github.com/kubernetes/klog/blob/main/examples/coexist_klog_v1_and_v2/coexist_klog_v1_and_v2.go +type klogv1Writer struct{} + +// Write redirects klogv1 calls to klogv2. +// This is a hack to support klogv1 without use of go-kit/log. It is inspired +// by klog's upstream klogv1/v2 coexistence example: +// https://github.com/kubernetes/klog/blob/main/examples/coexist_klog_v1_and_v2/coexist_klog_v1_and_v2.go +func (klogv1Writer) Write(p []byte) (n int, err error) { + if len(p) < klogv1DefaultPrefixLength { + klogv2.InfoDepth(klogv1OutputCallDepth, string(p)) + return len(p), nil + } + + switch p[0] { + case 'I': + klogv2.InfoDepth(klogv1OutputCallDepth, string(p[klogv1DefaultPrefixLength:])) + case 'W': + klogv2.WarningDepth(klogv1OutputCallDepth, string(p[klogv1DefaultPrefixLength:])) + case 'E': + klogv2.ErrorDepth(klogv1OutputCallDepth, string(p[klogv1DefaultPrefixLength:])) + case 'F': + klogv2.FatalDepth(klogv1OutputCallDepth, string(p[klogv1DefaultPrefixLength:])) + default: + klogv2.InfoDepth(klogv1OutputCallDepth, string(p[klogv1DefaultPrefixLength:])) + } + + return len(p), nil +} + var ( appName = "prometheus" @@ -103,6 +148,10 @@ var ( ) func init() { + // This can be removed when the legacy global mode is fully deprecated. + //nolint:staticcheck + model.NameValidationScheme = model.UTF8Validation + prometheus.MustRegister(versioncollector.NewCollector(strings.ReplaceAll(appName, "-", "_"))) var err error @@ -115,7 +164,7 @@ func init() { // serverOnlyFlag creates server-only kingpin flag. func serverOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagClause { return app.Flag(name, fmt.Sprintf("%s Use with server mode only.", help)). - PreAction(func(parseContext *kingpin.ParseContext) error { + PreAction(func(*kingpin.ParseContext) error { // This will be invoked only if flag is actually provided by user. serverOnlyFlags = append(serverOnlyFlags, "--"+name) return nil @@ -125,7 +174,7 @@ func serverOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagCl // agentOnlyFlag creates agent-only kingpin flag. func agentOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagClause { return app.Flag(name, fmt.Sprintf("%s Use with agent mode only.", help)). - PreAction(func(parseContext *kingpin.ParseContext) error { + PreAction(func(*kingpin.ParseContext) error { // This will be invoked only if flag is actually provided by user. agentOnlyFlags = append(agentOnlyFlags, "--"+name) return nil @@ -135,141 +184,190 @@ func agentOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagCla type flagConfig struct { configFile string - agentStoragePath string - serverStoragePath string - notifier notifier.Options - forGracePeriod model.Duration - outageTolerance model.Duration - resendDelay model.Duration - maxConcurrentEvals int64 - web web.Options - scrape scrape.Options - tsdb tsdbOptions - agent agentOptions - lookbackDelta model.Duration - webTimeout model.Duration - queryTimeout model.Duration - queryConcurrency int - queryMaxSamples int - RemoteFlushDeadline model.Duration - nameEscapingScheme string - - featureList []string - memlimitRatio float64 + agentStoragePath string + serverStoragePath string + notifier notifier.Options + forGracePeriod model.Duration + outageTolerance model.Duration + resendDelay model.Duration + maxConcurrentEvals int64 + web web.Options + scrape scrape.Options + tsdb tsdbOptions + agent agentOptions + lookbackDelta model.Duration + webTimeout model.Duration + queryTimeout model.Duration + queryConcurrency int + queryMaxSamples int + RemoteFlushDeadline model.Duration + maxNotificationsSubscribers int + + enableAutoReload bool + autoReloadInterval model.Duration + + maxprocsEnable bool + memlimitEnable bool + memlimitRatio float64 + + featureList []string // These options are extracted from featureList // for ease of use. - enableExpandExternalLabels bool - enableNewSDManager bool - enablePerStepStats bool - enableAutoGOMAXPROCS bool - enableAutoGOMEMLIMIT bool - enableConcurrentRuleEval bool + enablePerStepStats bool + enableConcurrentRuleEval bool + useStartTimestamps bool prometheusURL string corsRegexString string - promlogConfig promlog.Config - promqlEnableDelayedNameRemoval bool + + parserOpts parser.Options + + promslogConfig promslog.Config } // setFeatureListOptions sets the corresponding options from the featureList. -func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { +func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error { for _, f := range c.featureList { - opts := strings.Split(f, ",") - for _, o := range opts { + for o := range strings.SplitSeq(f, ",") { switch o { - case "remote-write-receiver": - c.web.EnableRemoteWriteReceiver = true - level.Warn(logger).Log("msg", "Remote write receiver enabled via feature flag remote-write-receiver. This is DEPRECATED. Use --web.enable-remote-write-receiver.") - case "otlp-write-receiver": - c.web.EnableOTLPWriteReceiver = true - level.Info(logger).Log("msg", "Experimental OTLP write receiver enabled") - case "expand-external-labels": - c.enableExpandExternalLabels = true - level.Info(logger).Log("msg", "Experimental expand-external-labels enabled") case "exemplar-storage": c.tsdb.EnableExemplarStorage = true - level.Info(logger).Log("msg", "Experimental in-memory exemplar storage enabled") + logger.Info("Experimental in-memory exemplar storage enabled") case "memory-snapshot-on-shutdown": c.tsdb.EnableMemorySnapshotOnShutdown = true - level.Info(logger).Log("msg", "Experimental memory snapshot on shutdown enabled") + logger.Info("Experimental memory snapshot on shutdown enabled") case "extra-scrape-metrics": - c.scrape.ExtraMetrics = true - level.Info(logger).Log("msg", "Experimental additional scrape metrics enabled") + t := true + config.DefaultConfig.GlobalConfig.ExtraScrapeMetrics = &t + config.DefaultGlobalConfig.ExtraScrapeMetrics = &t + logger.Warn("This option for --enable-feature is being phased out. It currently changes the default for the extra_scrape_metrics config setting to true, but will become a no-op in a future version. Stop using this option and set extra_scrape_metrics in the config instead.", "option", o) case "metadata-wal-records": c.scrape.AppendMetadata = true - level.Info(logger).Log("msg", "Experimental metadata records in WAL enabled, required for remote write 2.0") - case "new-service-discovery-manager": - c.enableNewSDManager = true - level.Info(logger).Log("msg", "Experimental service discovery manager") - case "agent": - agentMode = true - level.Info(logger).Log("msg", "Experimental agent mode enabled.") + c.web.AppendMetadata = true + features.Enable(features.TSDB, "metadata_wal_records") + logger.Info("Experimental metadata records in WAL enabled") case "promql-per-step-stats": c.enablePerStepStats = true - level.Info(logger).Log("msg", "Experimental per-step statistics reporting") - case "auto-gomaxprocs": - c.enableAutoGOMAXPROCS = true - level.Info(logger).Log("msg", "Automatically set GOMAXPROCS to match Linux container CPU quota") - case "auto-gomemlimit": - c.enableAutoGOMEMLIMIT = true - level.Info(logger).Log("msg", "Automatically set GOMEMLIMIT to match Linux container or system memory limit") + logger.Info("Experimental per-step statistics reporting") + case "auto-reload-config": + c.enableAutoReload = true + logger.Warn("This option for --enable-feature is deprecated. Use --config.auto-reload instead.", "option", o) case "concurrent-rule-eval": c.enableConcurrentRuleEval = true - level.Info(logger).Log("msg", "Experimental concurrent rule evaluation enabled.") - case "no-default-scrape-port": - c.scrape.NoDefaultPort = true - level.Info(logger).Log("msg", "No default port will be appended to scrape targets' addresses.") + logger.Info("Experimental concurrent rule evaluation enabled.") case "promql-experimental-functions": - parser.EnableExperimentalFunctions = true - level.Info(logger).Log("msg", "Experimental PromQL functions enabled.") + c.parserOpts.EnableExperimentalFunctions = true + logger.Info("Experimental PromQL functions enabled.") + case "promql-duration-expr": + c.parserOpts.ExperimentalDurationExpr = true + logger.Info("Experimental duration expression parsing enabled.") case "native-histograms": - c.tsdb.EnableNativeHistograms = true - c.scrape.EnableNativeHistogramsIngestion = true + logger.Warn("This option for --enable-feature is a no-op. To scrape native histograms, set the scrape_native_histograms scrape config setting to true.", "option", o) + case "ooo-native-histograms": + logger.Warn("This option for --enable-feature is now permanently enabled and therefore a no-op.", "option", o) + case "created-timestamp-zero-ingestion": + // NOTE(bwplotka): Once AppendableV1 is removed, there will be only the TSDB and agent flags. + c.scrape.EnableStartTimestampZeroIngestion = true + c.scrape.ParseST = true + c.web.STZeroIngestionEnabled = true + c.tsdb.EnableSTAsZeroSample = true + c.agent.EnableSTAsZeroSample = true + // Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers. + // This is to widen the ST support surface. config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols - level.Info(logger).Log("msg", "Experimental native histogram support enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) - case "created-timestamp-zero-ingestion": - c.scrape.EnableCreatedTimestampZeroIngestion = true - // Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers. + logger.Info("Experimental start timestamp zero ingestion enabled. OpenMetrics 1.0 parsing will parse _created metrics as ST instead of normal sample. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) + case "xor2-encoding": + c.tsdb.FloatChunkEncoding = chunkenc.EncXOR2 + logger.Info("Experimental XOR2 chunk encoding enabled.") + case "st-synthesis": + // TODO(ridwanmsharif): Move this to scrape configuration once stable. + c.scrape.SynthesizeST = true + features.Enable(features.Scrape, "st-synthesis") + logger.Info("Experimental start timestamp synthesis enabled.") + case "st-storage": + c.scrape.ParseST = true + c.tsdb.EnableSTStorage = true + c.agent.EnableSTStorage = true + + // Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers. This is to widen the ST support surface. config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols - level.Info(logger).Log("msg", "Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) + logger.Info("Experimental start timestamp storage enabled. OpenMetrics 1.0 parsing will parse _created metrics as ST instead of normal sample. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) + case "use-start-timestamps": + c.useStartTimestamps = true + logger.Info("Experimental usage of start timestamps in PromQL engine is enabled.") case "delayed-compaction": c.tsdb.EnableDelayedCompaction = true - level.Info(logger).Log("msg", "Experimental delayed compaction is enabled.") + logger.Info("Experimental delayed compaction is enabled.") case "promql-delayed-name-removal": c.promqlEnableDelayedNameRemoval = true - level.Info(logger).Log("msg", "Experimental PromQL delayed name removal enabled.") - case "utf8-names": - model.NameValidationScheme = model.UTF8Validation - level.Info(logger).Log("msg", "Experimental UTF-8 support enabled") - case "": - continue - case "promql-at-modifier", "promql-negative-offset": - level.Warn(logger).Log("msg", "This option for --enable-feature is now permanently enabled and therefore a no-op.", "option", o) + logger.Info("Experimental PromQL delayed name removal enabled.") + case "promql-extended-range-selectors": + c.parserOpts.EnableExtendedRangeSelectors = true + logger.Info("Experimental PromQL extended range selectors enabled.") + case "promql-binop-fill-modifiers": + c.parserOpts.EnableBinopFillModifiers = true + logger.Info("Experimental PromQL binary operator fill modifiers enabled.") + case "old-ui": + c.web.UseOldUI = true + logger.Info("Serving previous version of the Prometheus web UI.") + case "otlp-deltatocumulative": + c.web.ConvertOTLPDelta = true + logger.Info("Converting delta OTLP metrics to cumulative") + case "otlp-native-delta-ingestion": + // Experimental OTLP native delta ingestion. + // This currently just stores the raw delta value as-is with unknown metric type. Better typing and + // type-aware functions may come later. + // See proposal: https://github.com/prometheus/proposals/pull/48 + c.web.NativeOTLPDeltaIngestion = true + logger.Info("Enabling native ingestion of delta OTLP metrics, storing the raw sample values without conversion. WARNING: Delta support is in an early stage of development. The ingestion and querying process is likely to change over time.") + case "type-and-unit-labels": + c.scrape.EnableTypeAndUnitLabels = true + c.web.EnableTypeAndUnitLabels = true + logger.Info("Experimental type and unit labels enabled") + case "use-uncached-io": + if !fileutil.UncachedIOSupported() { + return errors.New("experimental Uncached IO is not supported") + } + c.tsdb.UseUncachedIO = true + logger.Info("Experimental Uncached IO is enabled.") + case "fast-startup": + c.tsdb.EnableFastStartup = true + logger.Info("Experimental fast startup is enabled.") + case "search-api": + c.web.EnableSearch = true + logger.Info("Experimental search API enabled.") default: - level.Warn(logger).Log("msg", "Unknown option for --enable-feature", "option", o) + logger.Warn("Unknown option for --enable-feature", "option", o) } } } + if c.web.ConvertOTLPDelta && c.web.NativeOTLPDeltaIngestion { + return errors.New("cannot enable otlp-deltatocumulative and otlp-native-delta-ingestion features at the same time") + } + return nil } +// parseCompressionType parses the two compression-related configuration values and returns the CompressionType. +func parseCompressionType(compress bool, compressType compression.Type) compression.Type { + if compress { + return compressType + } + return compression.None +} + func main() { if os.Getenv("DEBUG") != "" { runtime.SetBlockProfileRate(20) runtime.SetMutexProfileFraction(20) } - var ( - oldFlagRetentionDuration model.Duration - newFlagRetentionDuration model.Duration - ) - // Unregister the default GoCollector, and reregister with our defaults. if prometheus.Unregister(collectors.NewGoCollector()) { prometheus.MustRegister( @@ -277,6 +375,7 @@ func main() { collectors.WithGoCollectorRuntimeMetrics( collectors.MetricsGC, collectors.MetricsScheduler, + collectors.GoRuntimeMetricsRule{Matcher: goregexp.MustCompile(`^/sync/mutex/wait/total:seconds$`)}, ), ), ) @@ -287,10 +386,18 @@ func main() { Registerer: prometheus.DefaultRegisterer, }, web: web.Options{ - Registerer: prometheus.DefaultRegisterer, - Gatherer: prometheus.DefaultGatherer, + Registerer: prometheus.DefaultRegisterer, + Gatherer: prometheus.DefaultGatherer, + FeatureRegistry: features.DefaultRegistry, + }, + promslogConfig: promslog.Config{}, + scrape: scrape.Options{ + FeatureRegistry: features.DefaultRegistry, + }, + tsdb: tsdbOptions{ + // Default to XOR encoding; xor2-encoding feature flag overrides this to EncXOR2. + FloatChunkEncoding: chunkenc.EncXOR, }, - promlogConfig: promlog.Config{}, } a := kingpin.New(filepath.Base(os.Args[0]), "The Prometheus monitoring server").UsageWriter(os.Stdout) @@ -302,9 +409,19 @@ func main() { a.Flag("config.file", "Prometheus configuration file path."). Default("prometheus.yml").StringVar(&cfg.configFile) + a.Flag("config.auto-reload", "Enable automatic configuration file reloading. See also --config.auto-reload-interval."). + Default("false").BoolVar(&cfg.enableAutoReload) + + a.Flag("config.auto-reload-interval", "Specifies the interval for checking and automatically reloading the Prometheus configuration file upon detecting changes. Only used when --config.auto-reload is set."). + Default("30s").SetValue(&cfg.autoReloadInterval) + a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry. Can be repeated."). Default("0.0.0.0:9090").StringsVar(&cfg.web.ListenAddresses) + a.Flag("auto-gomaxprocs", "Automatically set GOMAXPROCS to match Linux container CPU quota"). + Default("true").BoolVar(&cfg.maxprocsEnable) + a.Flag("auto-gomemlimit", "Automatically set GOMEMLIMIT to match Linux container or system memory limit"). + Default("true").BoolVar(&cfg.memlimitEnable) a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory"). Default("0.9").FloatVar(&cfg.memlimitRatio) @@ -320,6 +437,9 @@ func main() { a.Flag("web.max-connections", "Maximum number of simultaneous connections across all listeners."). Default("512").IntVar(&cfg.web.MaxConnections) + a.Flag("web.max-notifications-subscribers", "Limits the maximum number of subscribers that can concurrently receive live notifications. If the limit is reached, new subscription requests will be denied until existing connections close."). + Default("16").IntVar(&cfg.maxNotificationsSubscribers) + a.Flag("web.external-url", "The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for generating relative and absolute links back to Prometheus itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived automatically."). PlaceHolder("").StringVar(&cfg.prometheusURL) @@ -342,10 +462,13 @@ func main() { a.Flag("web.enable-remote-write-receiver", "Enable API endpoint accepting remote write requests."). Default("false").BoolVar(&cfg.web.EnableRemoteWriteReceiver) - supportedRemoteWriteProtoMsgs := config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV1, config.RemoteWriteProtoMsgV2} + supportedRemoteWriteProtoMsgs := remoteapi.MessageTypes{remoteapi.WriteV1MessageType, remoteapi.WriteV2MessageType} a.Flag("web.remote-write-receiver.accepted-protobuf-messages", fmt.Sprintf("List of the remote write protobuf messages to accept when receiving the remote writes. Supported values: %v", supportedRemoteWriteProtoMsgs.String())). Default(supportedRemoteWriteProtoMsgs.Strings()...).SetValue(rwProtoMsgFlagValue(&cfg.web.AcceptRemoteWriteProtoMsgs)) + a.Flag("web.enable-otlp-receiver", "Enable API endpoint accepting OTLP write requests."). + Default("false").BoolVar(&cfg.web.EnableOTLPWriteReceiver) + a.Flag("web.console.templates", "Path to the console template directory, available at /consoles."). Default("consoles").StringVar(&cfg.web.ConsoleTemplatesPath) @@ -376,31 +499,28 @@ func main() { "Size at which to split the tsdb WAL segment files. Example: 100MB"). Hidden().PlaceHolder("").BytesVar(&cfg.tsdb.WALSegmentSize) - serverOnlyFlag(a, "storage.tsdb.retention", "[DEPRECATED] How long to retain samples in storage. This flag has been deprecated, use \"storage.tsdb.retention.time\" instead."). - SetValue(&oldFlagRetentionDuration) - - serverOnlyFlag(a, "storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms."). - SetValue(&newFlagRetentionDuration) + serverOnlyFlag(a, "storage.tsdb.retention.time", "[DEPRECATED] How long to retain samples in storage. If neither this flag nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms. This flag has been deprecated, use the storage.tsdb.retention.time field in the config file instead."). + SetValue(&cfg.tsdb.RetentionDuration) - serverOnlyFlag(a, "storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". Based on powers-of-2, so 1KB is 1024B."). + serverOnlyFlag(a, "storage.tsdb.retention.size", "[DEPRECATED] Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". Based on powers-of-2, so 1KB is 1024B. This flag has been deprecated, use the storage.tsdb.retention.size field in the config file instead."). BytesVar(&cfg.tsdb.MaxBytes) serverOnlyFlag(a, "storage.tsdb.no-lockfile", "Do not create lockfile in data directory."). Default("false").BoolVar(&cfg.tsdb.NoLockfile) - // TODO: Remove in Prometheus 3.0. - var b bool - serverOnlyFlag(a, "storage.tsdb.allow-overlapping-blocks", "[DEPRECATED] This flag has no effect. Overlapping blocks are enabled by default now."). - Default("true").Hidden().BoolVar(&b) - serverOnlyFlag(a, "storage.tsdb.allow-overlapping-compaction", "Allow compaction of overlapping blocks. If set to false, TSDB stops vertical compaction and leaves overlapping blocks there. The use case is to let another component handle the compaction of overlapping blocks."). Default("true").Hidden().BoolVar(&cfg.tsdb.EnableOverlappingCompaction) - serverOnlyFlag(a, "storage.tsdb.wal-compression", "Compress the tsdb WAL."). - Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression) + var ( + tsdbWALCompression bool + tsdbWALCompressionType string + tsdbDelayCompactFilePath string + ) + serverOnlyFlag(a, "storage.tsdb.wal-compression", "Compress the tsdb WAL. If false, the --storage.tsdb.wal-compression-type flag is ignored."). + Hidden().Default("true").BoolVar(&tsdbWALCompression) - serverOnlyFlag(a, "storage.tsdb.wal-compression-type", "Compression algorithm for the tsdb WAL."). - Hidden().Default(string(wlog.CompressionSnappy)).EnumVar(&cfg.tsdb.WALCompressionType, string(wlog.CompressionSnappy), string(wlog.CompressionZstd)) + serverOnlyFlag(a, "storage.tsdb.wal-compression-type", "Compression algorithm for the tsdb WAL, used when --storage.tsdb.wal-compression is true."). + Hidden().Default(compression.Snappy).EnumVar(&tsdbWALCompressionType, compression.Snappy, compression.Zstd) serverOnlyFlag(a, "storage.tsdb.head-chunks-write-queue-size", "Size of the queue through which head chunks are written to the disk to be m-mapped, 0 disables the queue completely. Experimental."). Default("0").IntVar(&cfg.tsdb.HeadChunksWriteQueueSize) @@ -408,6 +528,15 @@ func main() { serverOnlyFlag(a, "storage.tsdb.samples-per-chunk", "Target number of samples per chunk."). Default("120").Hidden().IntVar(&cfg.tsdb.SamplesPerChunk) + serverOnlyFlag(a, "storage.tsdb.delayed-compaction.max-percent", "Sets the upper limit for the random compaction delay, specified as a percentage of the head chunk range. 100 means the compaction can be delayed by up to the entire head chunk range. Only effective when the delayed-compaction feature flag is enabled."). + Default("10").Hidden().IntVar(&cfg.tsdb.CompactionDelayMaxPercent) + + serverOnlyFlag(a, "storage.tsdb.delay-compact-file.path", "Path to a JSON file with uploaded TSDB blocks e.g. Thanos shipper meta file. If set TSDB will only compact 1 level blocks that are marked as uploaded in that file, improving external storage integrations e.g. with Thanos sidecar. 1+ level compactions won't be delayed."). + Default("").StringVar(&tsdbDelayCompactFilePath) + + serverOnlyFlag(a, "storage.tsdb.block-reload-interval", "Interval at which to check for new or removed blocks in storage. Users who manually backfill or drop blocks must wait up to this duration before changes become available."). + Default("1m").Hidden().SetValue(&cfg.tsdb.BlockReloadInterval) + agentOnlyFlag(a, "storage.agent.path", "Base path for metrics storage."). Default("data-agent/").StringVar(&cfg.agentStoragePath) @@ -415,23 +544,37 @@ func main() { "Size at which to split WAL segment files. Example: 100MB"). Hidden().PlaceHolder("").BytesVar(&cfg.agent.WALSegmentSize) - agentOnlyFlag(a, "storage.agent.wal-compression", "Compress the agent WAL."). - Default("true").BoolVar(&cfg.agent.WALCompression) + var ( + agentWALCompression bool + agentWALCompressionType string + ) + agentOnlyFlag(a, "storage.agent.wal-compression", "Compress the agent WAL. If false, the --storage.agent.wal-compression-type flag is ignored."). + Default("true").BoolVar(&agentWALCompression) - agentOnlyFlag(a, "storage.agent.wal-compression-type", "Compression algorithm for the agent WAL."). - Hidden().Default(string(wlog.CompressionSnappy)).EnumVar(&cfg.agent.WALCompressionType, string(wlog.CompressionSnappy), string(wlog.CompressionZstd)) + agentOnlyFlag(a, "storage.agent.wal-compression-type", "Compression algorithm for the agent WAL, used when --storage.agent.wal-compression is true."). + Hidden().Default(compression.Snappy).EnumVar(&agentWALCompressionType, compression.Snappy, compression.Zstd) agentOnlyFlag(a, "storage.agent.wal-truncate-frequency", "The frequency at which to truncate the WAL and remove old data."). Hidden().PlaceHolder("").SetValue(&cfg.agent.TruncateFrequency) + // Dynamically calculate the format strings from the tsdb/agent constants + agentDefaultMinWALTime := model.Duration(agent.DefaultMinWALTime * int64(time.Millisecond)).String() + agentDefaultMaxWALTime := model.Duration(agent.DefaultMaxWALTime * int64(time.Millisecond)).String() + agentOnlyFlag(a, "storage.agent.retention.min-time", "Minimum age samples may be before being considered for deletion when the WAL is truncated"). - SetValue(&cfg.agent.MinWALTime) + Default(agentDefaultMinWALTime).SetValue(&cfg.agent.MinWALTime) agentOnlyFlag(a, "storage.agent.retention.max-time", "Maximum age samples may be before being forcibly deleted when the WAL is truncated"). - SetValue(&cfg.agent.MaxWALTime) + Default(agentDefaultMaxWALTime).SetValue(&cfg.agent.MaxWALTime) + + agentOnlyFlag(a, "storage.agent.checkpoint-from-in-memory-series", "Use only in-memory series data when building a checkpoint."). + Default("false").BoolVar(&cfg.agent.CheckpointFromInMemorySeries) + + agentOnlyFlag(a, "storage.agent.checkpoint-batch-size", "Size of a single WAL log entry chunk to be flushed. Has no effect without --storage.agent.checkpoint-from-in-memory-series flag."). + Default("1000").IntVar(&cfg.agent.CheckpointBatchSize) agentOnlyFlag(a, "storage.agent.no-lockfile", "Do not create lockfile in data directory."). Default("false").BoolVar(&cfg.agent.NoLockfile) @@ -448,6 +591,9 @@ func main() { serverOnlyFlag(a, "storage.remote.read-max-bytes-in-frame", "Maximum number of bytes in a single frame for streaming remote read response types before marshalling. Note that client might have limit on frame size as well. 1MB as recommended by protobuf by default."). Default("1048576").IntVar(&cfg.web.RemoteReadBytesInFrame) + serverOnlyFlag(a, "web.search.max-limit", "Hard upper bound on the \"limit\" query parameter accepted by the experimental search API (--enable-feature=search-api). Requests with a higher limit are rejected with HTTP 400. 0 disables the cap."). + Default("10000").IntVar(&cfg.web.MaxSearchLimit) + serverOnlyFlag(a, "rules.alert.for-outage-tolerance", "Max time to tolerate prometheus outage for restoring \"for\" state of alert."). Default("1h").SetValue(&cfg.outageTolerance) @@ -469,12 +615,12 @@ func main() { serverOnlyFlag(a, "alertmanager.notification-queue-capacity", "The capacity of the queue for pending Alertmanager notifications."). Default("10000").IntVar(&cfg.notifier.QueueCapacity) + serverOnlyFlag(a, "alertmanager.notification-batch-size", "The maximum number of notifications per batch to send to the Alertmanager."). + Default(strconv.Itoa(notifier.DefaultMaxBatchSize)).IntVar(&cfg.notifier.MaxBatchSize) + serverOnlyFlag(a, "alertmanager.drain-notification-queue-on-shutdown", "Send any outstanding Alertmanager notifications when shutting down. If false, any outstanding Alertmanager notifications will be dropped when shutting down."). Default("true").BoolVar(&cfg.notifier.DrainOnShutdown) - // TODO: Remove in Prometheus 3.0. - alertmanagerTimeout := a.Flag("alertmanager.timeout", "[DEPRECATED] This flag has no effect.").Hidden().String() - serverOnlyFlag(a, "query.lookback-delta", "The maximum lookback duration for retrieving metrics during expression evaluations and federation."). Default("5m").SetValue(&cfg.lookbackDelta) @@ -490,14 +636,14 @@ func main() { a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates."). Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval) - a.Flag("scrape.name-escaping-scheme", `Method for escaping legacy invalid names when sending to Prometheus that does not support UTF-8. Can be one of "values", "underscores", or "dots".`).Default(scrape.DefaultNameEscapingScheme.String()).StringVar(&cfg.nameEscapingScheme) + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, extra-scrape-metrics, memory-snapshot-on-shutdown, metadata-wal-records, old-ui, otlp-deltatocumulative, otlp-native-delta-ingestion, promql-binop-fill-modifiers, promql-delayed-name-removal, promql-duration-expr, promql-experimental-functions, promql-extended-range-selectors, promql-per-step-stats, search-api, st-storage, st-synthesis, type-and-unit-labels, use-start-timestamps, use-uncached-io, xor2-encoding. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). + StringsVar(&cfg.featureList) - a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomaxprocs, auto-gomemlimit, concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, expand-external-labels, extra-scrape-metrics, memory-snapshot-on-shutdown, native-histograms, new-service-discovery-manager, no-default-scrape-port, otlp-write-receiver, promql-experimental-functions, promql-delayed-name-removal, promql-per-step-stats, remote-write-receiver (DEPRECATED), utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). - Default("").StringsVar(&cfg.featureList) + a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode) - promlogflag.AddFlags(a, &cfg.promlogConfig) + promslogflag.AddFlags(a, &cfg.promslogConfig) - a.Flag("write-documentation", "Generate command line documentation. Internal use.").Hidden().Action(func(ctx *kingpin.ParseContext) error { + a.Flag("write-documentation", "Generate command line documentation. Internal use.").Hidden().Action(func(*kingpin.ParseContext) error { if err := documentcli.GenerateMarkdown(a.Model(), os.Stdout); err != nil { os.Exit(1) return err @@ -508,27 +654,38 @@ func main() { _, err := a.Parse(os.Args[1:]) if err != nil { - fmt.Fprintln(os.Stderr, fmt.Errorf("Error parsing command line arguments: %w", err)) + fmt.Fprintf(os.Stderr, "Error parsing command line arguments: %s\n", err) a.Usage(os.Args[1:]) os.Exit(2) } - logger := promlog.New(&cfg.promlogConfig) + logger := promslog.New(&cfg.promslogConfig) + slog.SetDefault(logger) + + notifs := notifications.NewNotifications(cfg.maxNotificationsSubscribers, prometheus.DefaultRegisterer) + cfg.web.NotificationsSub = notifs.Sub + cfg.web.NotificationsGetter = notifs.Get + notifs.AddNotification(notifications.StartingUp) if err := cfg.setFeatureListOptions(logger); err != nil { - fmt.Fprintln(os.Stderr, fmt.Errorf("Error parsing feature list: %w", err)) + fmt.Fprintf(os.Stderr, "Error parsing feature list: %s\n", err) os.Exit(1) } - if cfg.nameEscapingScheme != "" { - scheme, err := model.ToEscapingScheme(cfg.nameEscapingScheme) - if err != nil { - fmt.Fprintf(os.Stderr, `Invalid name escaping scheme: %q; Needs to be one of "values", "underscores", or "dots"`, cfg.nameEscapingScheme) - os.Exit(1) + if cfg.enableAutoReload { + if s := time.Duration(cfg.autoReloadInterval).Seconds(); s < 1 { + cfg.autoReloadInterval, _ = model.ParseDuration("1s") } - model.NameEscapingScheme = scheme + logger.Info("Automatic configuration file reloading enabled", "interval", cfg.autoReloadInterval) } + if cfg.web.MaxSearchLimit < 0 { + fmt.Fprintf(os.Stderr, "--web.search.max-limit must be non-negative; got %d (use 0 to disable the cap)\n", cfg.web.MaxSearchLimit) + os.Exit(1) + } + + promqlParser := parser.NewParser(cfg.parserOpts) + if agentMode && len(serverOnlyFlags) > 0 { fmt.Fprintf(os.Stderr, "The following flag(s) can not be used in agent mode: %q", serverOnlyFlags) os.Exit(3) @@ -539,6 +696,11 @@ func main() { os.Exit(3) } + if agentMode && cfg.agent.CheckpointBatchSize <= 0 { + fmt.Fprintln(os.Stderr, "--storage.agent.checkpoint-batch-size must be greater than 0.") + os.Exit(1) + } + if cfg.memlimitRatio <= 0.0 || cfg.memlimitRatio > 1.0 { fmt.Fprintf(os.Stderr, "--auto-gomemlimit.ratio must be greater than 0 and less than or equal to 1.") os.Exit(1) @@ -561,36 +723,95 @@ func main() { os.Exit(2) } - if *alertmanagerTimeout != "" { - level.Warn(logger).Log("msg", "The flag --alertmanager.timeout has no effect and will be removed in the future.") + // Set TSDB retention defaults from CLI flags before any config file is loaded. + // This makes CLI flags act as the default when no retention section is present. + cliRetentionDuration := cfg.tsdb.RetentionDuration + cliMaxBytes := cfg.tsdb.MaxBytes + if cliRetentionDuration == 0 && cliMaxBytes == 0 { + cliRetentionDuration = defaultRetentionDuration + } + config.DefaultTSDBRetentionConfig = config.TSDBRetentionConfig{ + Time: cliRetentionDuration, + Size: cliMaxBytes, } // Throw error for invalid config before starting other components. var cfgFile *config.Config - if cfgFile, err = config.LoadFile(cfg.configFile, agentMode, false, log.NewNopLogger()); err != nil { + if cfgFile, err = config.LoadFile(cfg.configFile, agentMode, promslog.NewNopLogger()); err != nil { absPath, pathErr := filepath.Abs(cfg.configFile) if pathErr != nil { absPath = cfg.configFile } - level.Error(logger).Log("msg", fmt.Sprintf("Error loading config (--config.file=%s)", cfg.configFile), "file", absPath, "err", err) + logger.Error(fmt.Sprintf("Error loading config (--config.file=%s)", cfg.configFile), "file", absPath, "err", err) os.Exit(2) } + // Get scrape configs to validate dynamically loaded scrape_config_files. + // They can change over time, but do the extra validation on startup for better experience. if _, err := cfgFile.GetScrapeConfigs(); err != nil { absPath, pathErr := filepath.Abs(cfg.configFile) if pathErr != nil { absPath = cfg.configFile } - level.Error(logger).Log("msg", fmt.Sprintf("Error loading scrape config files from config (--config.file=%q)", cfg.configFile), "file", absPath, "err", err) + logger.Error(fmt.Sprintf("Error loading dynamic scrape config files from config (--config.file=%q)", cfg.configFile), "file", absPath, "err", err) os.Exit(2) } + + // Parse rule files to verify they exist and contain valid rules. + if err := rules.ParseFiles(cfgFile.RuleFiles, cfgFile.GlobalConfig.MetricNameValidationScheme, promqlParser, logger); err != nil { + absPath, pathErr := filepath.Abs(cfg.configFile) + if pathErr != nil { + absPath = cfg.configFile + } + logger.Error(fmt.Sprintf("Error loading rule file patterns from config (--config.file=%q)", cfg.configFile), "file", absPath, "err", err) + os.Exit(2) + } + if cfg.tsdb.EnableExemplarStorage { if cfgFile.StorageConfig.ExemplarsConfig == nil { cfgFile.StorageConfig.ExemplarsConfig = &config.DefaultExemplarsConfig } cfg.tsdb.MaxExemplars = cfgFile.StorageConfig.ExemplarsConfig.MaxExemplars } - if cfgFile.StorageConfig.TSDBConfig != nil { - cfg.tsdb.OutOfOrderTimeWindow = cfgFile.StorageConfig.TSDBConfig.OutOfOrderTimeWindow + if cfg.tsdb.BlockReloadInterval < model.Duration(1*time.Second) { + logger.Warn("The option --storage.tsdb.block-reload-interval is set to a value less than 1s. Setting it to 1s to avoid overload.") + cfg.tsdb.BlockReloadInterval = model.Duration(1 * time.Second) + } + cfg.tsdb.OutOfOrderTimeWindow = cfgFile.StorageConfig.TSDBConfig.OutOfOrderTimeWindow + cfg.tsdb.StaleSeriesCompactionThreshold = cfgFile.StorageConfig.TSDBConfig.StaleSeriesCompactionThreshold + cfg.tsdb.RetentionDuration = cfgFile.StorageConfig.TSDBConfig.Retention.Time + cfg.tsdb.MaxBytes = cfgFile.StorageConfig.TSDBConfig.Retention.Size + cfg.tsdb.MaxPercentage = cfgFile.StorageConfig.TSDBConfig.Retention.Percentage + + // Set Go runtime parameters before we get too far into initialization. + updateGoGC(cfgFile, logger) + if cfg.maxprocsEnable { + l := func(format string, a ...any) { + logger.Info(fmt.Sprintf(strings.TrimPrefix(format, "maxprocs: "), a...), "component", "automaxprocs") + } + if _, err := maxprocs.Set(maxprocs.Logger(l)); err != nil { + logger.Warn("Failed to set GOMAXPROCS automatically", "component", "automaxprocs", "err", err) + } + } + + if cfg.memlimitEnable { + if _, err := memlimit.SetGoMemLimitWithOpts( + memlimit.WithRatio(cfg.memlimitRatio), + memlimit.WithProvider( + memlimit.ApplyFallback( + memlimit.FromCgroup, + memlimit.FromSystem, + ), + ), + memlimit.WithLogger(logger.With("component", "automemlimit")), + ); err != nil { + logger.Warn("automemlimit", "msg", "Failed to set GOMEMLIMIT automatically", "err", err) + } + } + + if tsdbDelayCompactFilePath != "" { + logger.Info("Compactions will be delayed for blocks not marked as uploaded in the file tracking uploads", "path", tsdbDelayCompactFilePath) + cfg.tsdb.BlockCompactionExcludeFunc = exludeBlocksPendingUpload( + logger, tsdbDelayCompactFilePath) } // Now that the validity of the config is established, set the config @@ -611,22 +832,6 @@ func main() { cfg.web.RoutePrefix = "/" + strings.Trim(cfg.web.RoutePrefix, "/") if !agentMode { - // Time retention settings. - if oldFlagRetentionDuration != 0 { - level.Warn(logger).Log("deprecation_notice", "'storage.tsdb.retention' flag is deprecated use 'storage.tsdb.retention.time' instead.") - cfg.tsdb.RetentionDuration = oldFlagRetentionDuration - } - - // When the new flag is set it takes precedence. - if newFlagRetentionDuration != 0 { - cfg.tsdb.RetentionDuration = newFlagRetentionDuration - } - - if cfg.tsdb.RetentionDuration == 0 && cfg.tsdb.MaxBytes == 0 { - cfg.tsdb.RetentionDuration = defaultRetentionDuration - level.Info(logger).Log("msg", "No time or size retention was set so using the default time retention", "duration", defaultRetentionDuration) - } - // Check for overflows. This limits our max retention to 100y. if cfg.tsdb.RetentionDuration < 0 { y, err := model.ParseDuration("100y") @@ -634,7 +839,17 @@ func main() { panic(err) } cfg.tsdb.RetentionDuration = y - level.Warn(logger).Log("msg", "Time retention value is too high. Limiting to: "+y.String()) + logger.Warn("Time retention value is too high. Limiting to: " + y.String()) + } + + if cfg.tsdb.MaxPercentage > 0 { + if cfg.tsdb.MaxBytes > 0 { + logger.Warn("storage.tsdb.retention.size is ignored, because storage.tsdb.retention.percentage is specified") + } + if storagePathFsSize(localStoragePath) == 0 { + fmt.Fprintln(os.Stderr, fmt.Errorf("unable to detect total capacity of metric storage at %s, please disable retention percentage (%g%%)", localStoragePath, cfg.tsdb.MaxPercentage)) + os.Exit(2) + } } // Max block size settings. @@ -650,16 +865,26 @@ func main() { cfg.tsdb.MaxBlockDuration = maxBlockDuration } + + // Delayed compaction checks + if cfg.tsdb.EnableDelayedCompaction && (cfg.tsdb.CompactionDelayMaxPercent > 100 || cfg.tsdb.CompactionDelayMaxPercent <= 0) { + logger.Warn("The --storage.tsdb.delayed-compaction.max-percent should have a value between 1 and 100. Using default", "default", tsdb.DefaultCompactionDelayMaxPercent) + cfg.tsdb.CompactionDelayMaxPercent = tsdb.DefaultCompactionDelayMaxPercent + } + + cfg.tsdb.WALCompressionType = parseCompressionType(tsdbWALCompression, tsdbWALCompressionType) + } else { + cfg.agent.WALCompressionType = parseCompressionType(agentWALCompression, agentWALCompressionType) } noStepSubqueryInterval := &safePromQLNoStepSubqueryInterval{} noStepSubqueryInterval.Set(config.DefaultGlobalConfig.EvaluationInterval) - // Above level 6, the k8s client would log bearer tokens in clear-text. - klog.ClampLevel(6) - klog.SetLogger(log.With(logger, "component", "k8s_client_runtime")) - klogv2.ClampLevel(6) - klogv2.SetLogger(log.With(logger, "component", "k8s_client_runtime")) + klogv2.SetSlogLogger(logger.With("component", "k8s_client_runtime")) + klog.SetOutputBySeverity("INFO", klogv1Writer{}) + // Avoid duplicate API deprecation warnings (e.g., "v1 Endpoints is deprecated in v1.33+...") + // that can pollute the logs. + rest.SetDefaultWarningHandlerWithContext(logging.NewDedupDeprecationWarningLogger()) modeAppName := "Prometheus Server" mode := "server" @@ -668,20 +893,28 @@ func main() { mode = "agent" } - level.Info(logger).Log("msg", "Starting "+modeAppName, "mode", mode, "version", version.Info()) + logger.Info("Starting "+modeAppName, "mode", mode, "version", version.Info()) if bits.UintSize < 64 { - level.Warn(logger).Log("msg", "This Prometheus binary has not been compiled for a 64-bit architecture. Due to virtual memory constraints of 32-bit systems, it is highly recommended to switch to a 64-bit binary of Prometheus.", "GOARCH", runtime.GOARCH) + logger.Warn("This Prometheus binary has not been compiled for a 64-bit architecture. Due to virtual memory constraints of 32-bit systems, it is highly recommended to switch to a 64-bit binary of Prometheus.", "GOARCH", runtime.GOARCH) } - level.Info(logger).Log("build_context", version.BuildContext()) - level.Info(logger).Log("host_details", prom_runtime.Uname()) - level.Info(logger).Log("fd_limits", prom_runtime.FdLimits()) - level.Info(logger).Log("vm_limits", prom_runtime.VMLimits()) + logger.Info("operational information", + "build_context", version.BuildContext(), + "host_details", prom_runtime.Uname(), + "fd_limits", prom_runtime.FdLimits(), + "vm_limits", prom_runtime.VMLimits(), + ) + + features.Set(features.Prometheus, "agent_mode", agentMode) + features.Set(features.Prometheus, "server_mode", !agentMode) + features.Set(features.Prometheus, "auto_reload_config", cfg.enableAutoReload) + features.Enable(features.Prometheus, labels.ImplementationName) + template.RegisterFeatures(features.DefaultRegistry) var ( localStorage = &readyStorage{stats: tsdb.NewDBStats()} scraper = &readyScrapeManager{} - remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper, cfg.scrape.AppendMetadata) + remoteStorage = remote.NewStorage(logger.With("component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper, cfg.scrape.EnableTypeAndUnitLabels) fanoutStorage = storage.NewFanout(logger, localStorage, remoteStorage) ) @@ -689,12 +922,12 @@ func main() { ctxWeb, cancelWeb = context.WithCancel(context.Background()) ctxRule = context.Background() - notifierManager = notifier.NewManager(&cfg.notifier, log.With(logger, "component", "notifier")) + notifierManager = notifier.NewManager(&cfg.notifier, cfgFile.GlobalConfig.MetricNameValidationScheme, logger.With("component", "notifier")) ctxScrape, cancelScrape = context.WithCancel(context.Background()) ctxNotify, cancelNotify = context.WithCancel(context.Background()) - discoveryManagerScrape discoveryManager - discoveryManagerNotify discoveryManager + discoveryManagerScrape *discovery.Manager + discoveryManagerNotify *discovery.Manager ) // Kubernetes client metrics are used by Kubernetes SD. @@ -704,63 +937,37 @@ func main() { // they are not specific to an SD instance. err = discovery.RegisterK8sClientMetricsWithPrometheus(prometheus.DefaultRegisterer) if err != nil { - level.Error(logger).Log("msg", "failed to register Kubernetes client metrics", "err", err) + logger.Error("failed to register Kubernetes client metrics", "err", err) os.Exit(1) } sdMetrics, err := discovery.CreateAndRegisterSDMetrics(prometheus.DefaultRegisterer) if err != nil { - level.Error(logger).Log("msg", "failed to register service discovery metrics", "err", err) + logger.Error("failed to register service discovery metrics", "err", err) os.Exit(1) } - if cfg.enableNewSDManager { - { - discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape")) - if discMgr == nil { - level.Error(logger).Log("msg", "failed to create a discovery manager scrape") - os.Exit(1) - } - discoveryManagerScrape = discMgr - } - - { - discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify")) - if discMgr == nil { - level.Error(logger).Log("msg", "failed to create a discovery manager notify") - os.Exit(1) - } - discoveryManagerNotify = discMgr - } - } else { - { - discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("scrape")) - if discMgr == nil { - level.Error(logger).Log("msg", "failed to create a discovery manager scrape") - os.Exit(1) - } - discoveryManagerScrape = discMgr - } + discoveryManagerScrape = discovery.NewManager(ctxScrape, logger.With("component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape"), discovery.FeatureRegistry(features.DefaultRegistry)) + if discoveryManagerScrape == nil { + logger.Error("failed to create a discovery manager scrape") + os.Exit(1) + } - { - discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("notify")) - if discMgr == nil { - level.Error(logger).Log("msg", "failed to create a discovery manager notify") - os.Exit(1) - } - discoveryManagerNotify = discMgr - } + discoveryManagerNotify = discovery.NewManager(ctxNotify, logger.With("component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify"), discovery.FeatureRegistry(features.DefaultRegistry)) + if discoveryManagerNotify == nil { + logger.Error("failed to create a discovery manager notify") + os.Exit(1) } scrapeManager, err := scrape.NewManager( &cfg.scrape, - log.With(logger, "component", "scrape manager"), - func(s string) (log.Logger, error) { return logging.NewJSONFileLogger(s) }, - fanoutStorage, + logger.With("component", "scrape manager"), + logging.NewJSONFileLogger, + nil, fanoutStorage, prometheus.DefaultRegisterer, ) if err != nil { - level.Error(logger).Log("msg", "failed to create a scrape manager", "err", err) + logger.Error("failed to create a scrape manager", "err", err) os.Exit(1) } @@ -771,36 +978,13 @@ func main() { ruleManager *rules.Manager ) - if cfg.enableAutoGOMAXPROCS { - l := func(format string, a ...interface{}) { - level.Info(logger).Log("component", "automaxprocs", "msg", fmt.Sprintf(strings.TrimPrefix(format, "maxprocs: "), a...)) - } - if _, err := maxprocs.Set(maxprocs.Logger(l)); err != nil { - level.Warn(logger).Log("component", "automaxprocs", "msg", "Failed to set GOMAXPROCS automatically", "err", err) - } - } - - if cfg.enableAutoGOMEMLIMIT { - if _, err := memlimit.SetGoMemLimitWithOpts( - memlimit.WithRatio(cfg.memlimitRatio), - memlimit.WithProvider( - memlimit.ApplyFallback( - memlimit.FromCgroup, - memlimit.FromSystem, - ), - ), - ); err != nil { - level.Warn(logger).Log("component", "automemlimit", "msg", "Failed to set GOMEMLIMIT automatically", "err", err) - } - } - if !agentMode { opts := promql.EngineOpts{ - Logger: log.With(logger, "component", "query engine"), + Logger: logger.With("component", "query engine"), Reg: prometheus.DefaultRegisterer, MaxSamples: cfg.queryMaxSamples, Timeout: time.Duration(cfg.queryTimeout), - ActiveQueryTracker: promql.NewActiveQueryTracker(localStoragePath, cfg.queryConcurrency, log.With(logger, "component", "activeQueryTracker")), + ActiveQueryTracker: promql.NewActiveQueryTracker(localStoragePath, cfg.queryConcurrency, logger.With("component", "activeQueryTracker")), LookbackDelta: time.Duration(cfg.lookbackDelta), NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get, // EnableAtModifier and EnableNegativeOffset have to be @@ -809,11 +993,16 @@ func main() { EnableNegativeOffset: true, EnablePerStepStats: cfg.enablePerStepStats, EnableDelayedNameRemoval: cfg.promqlEnableDelayedNameRemoval, + EnableTypeAndUnitLabels: cfg.scrape.EnableTypeAndUnitLabels, + UseStartTimestamps: cfg.useStartTimestamps, + FeatureRegistry: features.DefaultRegistry, + Parser: promqlParser, } queryEngine = promql.NewEngine(opts) ruleManager = rules.NewManager(&rules.ManagerOptions{ + NameValidationScheme: cfgFile.GlobalConfig.MetricNameValidationScheme, Appendable: fanoutStorage, Queryable: localStorage, QueryFunc: rules.EngineQueryFunc(queryEngine, fanoutStorage), @@ -821,7 +1010,7 @@ func main() { Context: ctxRule, ExternalURL: cfg.web.ExternalURL, Registerer: prometheus.DefaultRegisterer, - Logger: log.With(logger, "component", "rule manager"), + Logger: logger.With("component", "rule manager"), OutageTolerance: time.Duration(cfg.outageTolerance), ForGracePeriod: time.Duration(cfg.forGracePeriod), ResendDelay: time.Duration(cfg.resendDelay), @@ -830,6 +1019,8 @@ func main() { DefaultRuleQueryOffset: func() time.Duration { return time.Duration(cfgFile.GlobalConfig.RuleQueryOffset) }, + FeatureRegistry: features.DefaultRegistry, + Parser: promqlParser, }) } @@ -838,6 +1029,7 @@ func main() { cfg.web.Context = ctxWeb cfg.web.TSDBRetentionDuration = cfg.tsdb.RetentionDuration cfg.web.TSDBMaxBytes = cfg.tsdb.MaxBytes + cfg.web.TSDBMaxPercentage = cfg.tsdb.MaxPercentage cfg.web.TSDBDir = localStoragePath cfg.web.LocalStorage = localStorage cfg.web.Storage = fanoutStorage @@ -849,6 +1041,7 @@ func main() { cfg.web.LookbackDelta = time.Duration(cfg.lookbackDelta) cfg.web.IsAgent = agentMode cfg.web.AppName = modeAppName + cfg.web.Parser = promqlParser cfg.web.Version = &web.PrometheusVersion{ Version: version.Version, @@ -872,7 +1065,7 @@ func main() { } // Depends on cfg.web.ScrapeManager so needs to be after cfg.web.ScrapeManager = scrapeManager. - webHandler := web.New(log.With(logger, "component", "web"), &cfg.web) + webHandler := web.New(logger.With("component", "web"), &cfg.web) // Monitor outgoing connections on default transport with conntrack. http.DefaultTransport.(*http.Transport).DialContext = conntrack.NewDialContextFunc( @@ -884,8 +1077,29 @@ func main() { reloaders := []reloader{ { - name: "db_storage", - reloader: localStorage.ApplyConfig, + name: "db_storage", + reloader: func() func(*config.Config) error { + lastTSDBRetention := config.TSDBRetentionConfig{} + return func(cfg *config.Config) error { + err := localStorage.ApplyConfig(cfg) + if err != nil || agentMode || cfg.StorageConfig.TSDBConfig == nil || cfg.StorageConfig.TSDBConfig.Retention == nil { + return err + } + + curr := cfg.StorageConfig.TSDBConfig.Retention + if *curr == lastTSDBRetention { + return nil + } + + logger.Info("TSDB retention updated", + "duration", curr.Time, + "size", curr.Size, + "percentage", curr.Percentage, + ) + lastTSDBRetention = *curr + return nil + } + }(), }, { name: "remote_storage", reloader: remoteStorage.ApplyConfig, @@ -999,19 +1213,13 @@ func main() { listeners, err := webHandler.Listeners() if err != nil { - level.Error(logger).Log("msg", "Unable to start web listeners", "err", err) - if err := queryEngine.Close(); err != nil { - level.Warn(logger).Log("msg", "Closing query engine failed", "err", err) - } + logger.Error("Unable to start web listener", "err", err) os.Exit(1) } err = toolkit_web.Validate(*webConfig) if err != nil { - level.Error(logger).Log("msg", "Unable to validate web configuration file", "err", err) - if err := queryEngine.Close(); err != nil { - level.Warn(logger).Log("msg", "Closing query engine failed", "err", err) - } + logger.Error("Unable to validate web configuration file", "err", err) os.Exit(1) } @@ -1026,21 +1234,19 @@ func main() { // Don't forget to release the reloadReady channel so that waiting blocks can exit normally. select { case sig := <-term: - level.Warn(logger).Log("msg", "Received an OS signal, exiting gracefully...", "signal", sig.String()) + logger.Warn("Received an OS signal, exiting gracefully...", "signal", sig.String()) reloadReady.Close() case <-webHandler.Quit(): - level.Warn(logger).Log("msg", "Received termination request via web service, exiting gracefully...") + logger.Warn("Received termination request via web service, exiting gracefully...") case <-cancel: reloadReady.Close() } - if err := queryEngine.Close(); err != nil { - level.Warn(logger).Log("msg", "Closing query engine failed", "err", err) - } return nil }, - func(err error) { + func(error) { close(cancel) - webHandler.SetReady(false) + webHandler.SetReady(web.Stopping) + notifs.AddNotification(notifications.ShuttingDown) }, ) } @@ -1049,11 +1255,11 @@ func main() { g.Add( func() error { err := discoveryManagerScrape.Run() - level.Info(logger).Log("msg", "Scrape discovery manager stopped") + logger.Info("Scrape discovery manager stopped") return err }, - func(err error) { - level.Info(logger).Log("msg", "Stopping scrape discovery manager...") + func(error) { + logger.Info("Stopping scrape discovery manager...") cancelScrape() }, ) @@ -1063,11 +1269,11 @@ func main() { g.Add( func() error { err := discoveryManagerNotify.Run() - level.Info(logger).Log("msg", "Notify discovery manager stopped") + logger.Info("Notify discovery manager stopped") return err }, - func(err error) { - level.Info(logger).Log("msg", "Stopping notify discovery manager...") + func(error) { + logger.Info("Stopping notify discovery manager...") cancelNotify() }, ) @@ -1078,9 +1284,11 @@ func main() { func() error { <-reloadReady.C ruleManager.Run() + logger.Info("Rule manager stopped") return nil }, - func(err error) { + func(error) { + logger.Info("Stopping rule manager manager...") ruleManager.Stop() }, ) @@ -1096,15 +1304,15 @@ func main() { <-reloadReady.C err := scrapeManager.Run(discoveryManagerScrape.SyncCh()) - level.Info(logger).Log("msg", "Scrape manager stopped") + logger.Info("Scrape manager stopped") return err }, - func(err error) { + func(error) { // Scrape manager needs to be stopped before closing the local TSDB // so that it doesn't try to write samples to a closed storage. // We should also wait for rule manager to be fully stopped to ensure // we don't trigger any false positive alerts for rules using absent(). - level.Info(logger).Log("msg", "Stopping scrape manager...") + logger.Info("Stopping scrape manager...") scrapeManager.Stop() }, ) @@ -1115,9 +1323,11 @@ func main() { func() error { <-reloadReady.C tracingManager.Run() + logger.Info("Tracing manager stopped") return nil }, - func(err error) { + func(error) { + logger.Info("Stopping tracing manager...") tracingManager.Stop() }, ) @@ -1130,6 +1340,23 @@ func main() { hup := make(chan os.Signal, 1) signal.Notify(hup, syscall.SIGHUP) cancel := make(chan struct{}) + + var checksum string + if cfg.enableAutoReload { + checksum, err = config.GenerateChecksum(cfg.configFile) + if err != nil { + logger.Error("Failed to generate initial checksum for configuration file", "err", err) + } + } + + callback := func(success bool) { + if success { + notifs.DeleteNotification(notifications.ConfigurationUnsuccessful) + return + } + notifs.AddNotification(notifications.ConfigurationUnsuccessful) + } + g.Add( func() error { <-reloadReady.C @@ -1137,24 +1364,55 @@ func main() { for { select { case <-hup: - if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil { - level.Error(logger).Log("msg", "Error reloading config", "err", err) + if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, callback, reloaders...); err != nil { + logger.Error("Error reloading config", "err", err) + } else if cfg.enableAutoReload { + checksum, err = config.GenerateChecksum(cfg.configFile) + if err != nil { + logger.Error("Failed to generate checksum during configuration reload", "err", err) + } } case rc := <-webHandler.Reload(): - if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil { - level.Error(logger).Log("msg", "Error reloading config", "err", err) + if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, callback, reloaders...); err != nil { + logger.Error("Error reloading config", "err", err) rc <- err } else { rc <- nil + if cfg.enableAutoReload { + checksum, err = config.GenerateChecksum(cfg.configFile) + if err != nil { + logger.Error("Failed to generate checksum during configuration reload", "err", err) + } + } + } + case <-time.Tick(time.Duration(cfg.autoReloadInterval)): + if !cfg.enableAutoReload { + continue + } + currentChecksum, err := config.GenerateChecksum(cfg.configFile) + if err != nil { + checksum = currentChecksum + logger.Error("Failed to generate checksum during configuration reload", "err", err) + } else if currentChecksum == checksum { + continue + } + logger.Info("Configuration file change detected, reloading the configuration.") + + if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, callback, reloaders...); err != nil { + logger.Error("Error reloading config", "err", err) + } else { + checksum = currentChecksum } case <-cancel: + logger.Info("Reloaders stopped") return nil } } }, - func(err error) { + func(error) { // Wait for any in-progress reloads to complete to avoid // reloading things after they have been shutdown. + logger.Info("Stopping reloaders...") cancel <- struct{}{} }, ) @@ -1172,18 +1430,19 @@ func main() { return nil } - if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil { + if err := reloadConfig(cfg.configFile, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, func(bool) {}, reloaders...); err != nil { return fmt.Errorf("error loading config from %q: %w", cfg.configFile, err) } reloadReady.Close() - webHandler.SetReady(true) - level.Info(logger).Log("msg", "Server is ready to receive web requests.") + webHandler.SetReady(web.Ready) + notifs.DeleteNotification(notifications.StartingUp) + logger.Info("Server is ready to receive web requests.") <-cancel return nil }, - func(err error) { + func(error) { close(cancel) }, ) @@ -1194,7 +1453,7 @@ func main() { cancel := make(chan struct{}) g.Add( func() error { - level.Info(logger).Log("msg", "Starting TSDB ...") + logger.Info("Starting TSDB ...") if cfg.tsdb.WALSegmentSize != 0 { if cfg.tsdb.WALSegmentSize < 10*1024*1024 || cfg.tsdb.WALSegmentSize > 256*1024*1024 { return errors.New("flag 'storage.tsdb.wal-segment-size' must be set between 10MB and 256MB") @@ -1211,22 +1470,26 @@ func main() { return fmt.Errorf("opening storage failed: %w", err) } - switch fsType := prom_runtime.Statfs(localStoragePath); fsType { + switch fsType := prom_runtime.FsType(localStoragePath); fsType { case "NFS_SUPER_MAGIC": - level.Warn(logger).Log("fs_type", fsType, "msg", "This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.") + logger.Warn("This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.", "fs_type", fsType) default: - level.Info(logger).Log("fs_type", fsType) + logger.Info("filesystem information", "fs_type", fsType) } - level.Info(logger).Log("msg", "TSDB started") - level.Debug(logger).Log("msg", "TSDB options", + logger.Info("TSDB started") + logger.Debug("TSDB options", "MinBlockDuration", cfg.tsdb.MinBlockDuration, "MaxBlockDuration", cfg.tsdb.MaxBlockDuration, "MaxBytes", cfg.tsdb.MaxBytes, + "MaxPercentage", cfg.tsdb.MaxPercentage, "NoLockfile", cfg.tsdb.NoLockfile, "RetentionDuration", cfg.tsdb.RetentionDuration, "WALSegmentSize", cfg.tsdb.WALSegmentSize, - "WALCompression", cfg.tsdb.WALCompression, + "WALCompressionType", cfg.tsdb.WALCompressionType, + "BlockReloadInterval", cfg.tsdb.BlockReloadInterval, + "EnableSTAsZeroSample", cfg.tsdb.EnableSTAsZeroSample, + "EnableSTStorage", cfg.tsdb.EnableSTStorage, ) startTimeMargin := int64(2 * time.Duration(cfg.tsdb.MinBlockDuration).Seconds() * 1000) @@ -1234,11 +1497,13 @@ func main() { db.SetWriteNotified(remoteStorage) close(dbOpen) <-cancel + logger.Info("TSDB stopped") return nil }, - func(err error) { + func(error) { + logger.Info("Stopping storage...") if err := fanoutStorage.Close(); err != nil { - level.Error(logger).Log("msg", "Error stopping storage", "err", err) + logger.Error("Error stopping storage", "err", err) } close(cancel) }, @@ -1250,7 +1515,7 @@ func main() { cancel := make(chan struct{}) g.Add( func() error { - level.Info(logger).Log("msg", "Starting WAL storage ...") + logger.Info("Starting WAL storage ...") if cfg.agent.WALSegmentSize != 0 { if cfg.agent.WALSegmentSize < 10*1024*1024 || cfg.agent.WALSegmentSize > 256*1024*1024 { return errors.New("flag 'storage.agent.wal-segment-size' must be set between 10MB and 256MB") @@ -1267,33 +1532,37 @@ func main() { return fmt.Errorf("opening storage failed: %w", err) } - switch fsType := prom_runtime.Statfs(localStoragePath); fsType { + switch fsType := prom_runtime.FsType(localStoragePath); fsType { case "NFS_SUPER_MAGIC": - level.Warn(logger).Log("fs_type", fsType, "msg", "This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.") + logger.Warn(fsType, "msg", "This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.") default: - level.Info(logger).Log("fs_type", fsType) + logger.Info(fsType) } - level.Info(logger).Log("msg", "Agent WAL storage started") - level.Debug(logger).Log("msg", "Agent WAL storage options", + logger.Info("Agent WAL storage started") + logger.Debug("Agent WAL storage options", "WALSegmentSize", cfg.agent.WALSegmentSize, - "WALCompression", cfg.agent.WALCompression, + "WALCompressionType", cfg.agent.WALCompressionType, "StripeSize", cfg.agent.StripeSize, "TruncateFrequency", cfg.agent.TruncateFrequency, "MinWALTime", cfg.agent.MinWALTime, "MaxWALTime", cfg.agent.MaxWALTime, "OutOfOrderTimeWindow", cfg.agent.OutOfOrderTimeWindow, + "EnableSTAsZeroSample", cfg.agent.EnableSTAsZeroSample, + "EnableSTStorage", cfg.tsdb.EnableSTStorage, ) localStorage.Set(db, 0) db.SetWriteNotified(remoteStorage) close(dbOpen) <-cancel + logger.Info("Agent WAL storage stopped") return nil }, - func(e error) { + func(error) { + logger.Info("Stopping agent WAL storage...") if err := fanoutStorage.Close(); err != nil { - level.Error(logger).Log("msg", "Error stopping storage", "err", err) + logger.Error("Error stopping storage", "err", err) } close(cancel) }, @@ -1306,9 +1575,11 @@ func main() { if err := webHandler.Run(ctxWeb, listeners, *webConfig); err != nil { return fmt.Errorf("error starting web server: %w", err) } + logger.Info("Web handler stopped") return nil }, - func(err error) { + func(error) { + logger.Info("Stopping web handler...") cancelWeb() }, ) @@ -1327,25 +1598,28 @@ func main() { <-reloadReady.C notifierManager.Run(discoveryManagerNotify.SyncCh()) - level.Info(logger).Log("msg", "Notifier manager stopped") + logger.Info("Notifier manager stopped") return nil }, - func(err error) { + func(error) { + logger.Info("Stopping notifier manager...") notifierManager.Stop() }, ) } - if err := g.Run(); err != nil { - level.Error(logger).Log("err", err) - os.Exit(1) - } - level.Info(logger).Log("msg", "See you next time!") + func() { // This function exists so the top of the stack is named 'main.main.funcxxx' and not 'oklog'. + if err := g.Run(); err != nil { + logger.Error("Fatal error", "err", err) + os.Exit(1) + } + }() + logger.Info("See you next time!") } -func openDBWithMetrics(dir string, logger log.Logger, reg prometheus.Registerer, opts *tsdb.Options, stats *tsdb.DBStats) (*tsdb.DB, error) { +func openDBWithMetrics(dir string, logger *slog.Logger, reg prometheus.Registerer, opts *tsdb.Options, stats *tsdb.DBStats) (*tsdb.DB, error) { db, err := tsdb.Open( dir, - log.With(logger, "component", "tsdb"), + logger.With("component", "tsdb"), reg, opts, stats, @@ -1398,21 +1672,23 @@ type reloader struct { reloader func(*config.Config) error } -func reloadConfig(filename string, expandExternalLabels, enableExemplarStorage bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) { +func reloadConfig(filename string, enableExemplarStorage bool, logger *slog.Logger, noStepSubqueryInterval *safePromQLNoStepSubqueryInterval, callback func(bool), rls ...reloader) (err error) { start := time.Now() - timings := []interface{}{} - level.Info(logger).Log("msg", "Loading configuration file", "filename", filename) + timingsLogger := logger + logger.Info("Loading configuration file", "filename", filename) defer func() { if err == nil { configSuccess.Set(1) configSuccessTime.SetToCurrentTime() + callback(true) } else { configSuccess.Set(0) + callback(false) } }() - conf, err := config.LoadFile(filename, agentMode, expandExternalLabels, logger) + conf, err := config.LoadFile(filename, agentMode, logger) if err != nil { return fmt.Errorf("couldn't load configuration (--config.file=%q): %w", filename, err) } @@ -1427,18 +1703,25 @@ func reloadConfig(filename string, expandExternalLabels, enableExemplarStorage b for _, rl := range rls { rstart := time.Now() if err := rl.reloader(conf); err != nil { - level.Error(logger).Log("msg", "Failed to apply configuration", "err", err) + logger.Error("Failed to apply configuration", "err", err) failed = true } - timings = append(timings, rl.name, time.Since(rstart)) + timingsLogger = timingsLogger.With(rl.name, time.Since(rstart)) } if failed { return fmt.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename) } + updateGoGC(conf, logger) + noStepSubqueryInterval.Set(conf.GlobalConfig.EvaluationInterval) + timingsLogger.Info("Completed loading of configuration file", "filename", filename, "totalDuration", time.Since(start)) + return nil +} + +func updateGoGC(conf *config.Config, logger *slog.Logger) { oldGoGC := debug.SetGCPercent(conf.Runtime.GoGC) if oldGoGC != conf.Runtime.GoGC { - level.Info(logger).Log("msg", "updated GOGC", "old", oldGoGC, "new", conf.Runtime.GoGC) + logger.Info("updated GOGC", "old", oldGoGC, "new", conf.Runtime.GoGC) } // Write the new setting out to the ENV var for runtime API output. if conf.Runtime.GoGC >= 0 { @@ -1446,11 +1729,6 @@ func reloadConfig(filename string, expandExternalLabels, enableExemplarStorage b } else { os.Setenv("GOGC", "off") } - - noStepSuqueryInterval.Set(conf.GlobalConfig.EvaluationInterval) - l := []interface{}{"msg", "Completed loading of configuration file", "filename", filename, "totalDuration", time.Since(start)} - level.Info(logger).Log(append(l, timings...)...) - return nil } func startsOrEndsWithQuote(s string) bool { @@ -1500,6 +1778,25 @@ func computeExternalURL(u, listenAddr string) (*url.URL, error) { return eu, nil } +// storagePathFsSize returns the filesystem size for path or its closest existing parent. +func storagePathFsSize(path string) uint64 { + for { + if size := prom_runtime.FsSize(path); size > 0 { + return size + } + + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + return 0 + } + + parent := filepath.Dir(path) + if parent == path { + return 0 + } + path = parent + } +} + // readyStorage implements the Storage interface while allowing to set the actual // storage at a later point in time. type readyStorage struct { @@ -1601,31 +1898,55 @@ func (s *readyStorage) Appender(ctx context.Context) storage.Appender { return notReadyAppender{} } +// AppenderV2 implements the Storage interface. +func (s *readyStorage) AppenderV2(ctx context.Context) storage.AppenderV2 { + if x := s.get(); x != nil { + return x.AppenderV2(ctx) + } + return notReadyAppenderV2{} +} + type notReadyAppender struct{} -func (n notReadyAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +// SetOptions does nothing in this appender implementation. +func (notReadyAppender) SetOptions(*storage.AppendOptions) {} + +func (notReadyAppender) Append(storage.SeriesRef, labels.Labels, int64, float64) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } -func (n notReadyAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { +func (notReadyAppender) AppendExemplar(storage.SeriesRef, labels.Labels, exemplar.Exemplar) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } -func (n notReadyAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (notReadyAppender) AppendHistogram(storage.SeriesRef, labels.Labels, int64, *histogram.Histogram, *histogram.FloatHistogram) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } -func (n notReadyAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { +func (notReadyAppender) AppendHistogramSTZeroSample(storage.SeriesRef, labels.Labels, int64, int64, *histogram.Histogram, *histogram.FloatHistogram) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } -func (n notReadyAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) { +func (notReadyAppender) UpdateMetadata(storage.SeriesRef, labels.Labels, metadata.Metadata) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } -func (n notReadyAppender) Commit() error { return tsdb.ErrNotReady } +func (notReadyAppender) AppendSTZeroSample(storage.SeriesRef, labels.Labels, int64, int64) (storage.SeriesRef, error) { + return 0, tsdb.ErrNotReady +} -func (n notReadyAppender) Rollback() error { return tsdb.ErrNotReady } +func (notReadyAppender) Commit() error { return tsdb.ErrNotReady } + +func (notReadyAppender) Rollback() error { return tsdb.ErrNotReady } + +type notReadyAppenderV2 struct{} + +func (notReadyAppenderV2) Append(storage.SeriesRef, labels.Labels, int64, int64, float64, *histogram.Histogram, *histogram.FloatHistogram, storage.AOptions) (storage.SeriesRef, error) { + return 0, tsdb.ErrNotReady +} +func (notReadyAppenderV2) Commit() error { return tsdb.ErrNotReady } + +func (notReadyAppenderV2) Rollback() error { return tsdb.ErrNotReady } // Close implements the Storage interface. func (s *readyStorage) Close() error { @@ -1650,6 +1971,21 @@ func (s *readyStorage) CleanTombstones() error { return tsdb.ErrNotReady } +// BlockMetas implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces. +func (s *readyStorage) BlockMetas() ([]tsdb.BlockMeta, error) { + if x := s.get(); x != nil { + switch db := x.(type) { + case *tsdb.DB: + return db.BlockMetas(), nil + case *agent.DB: + return nil, agent.ErrUnsupported + default: + panic(fmt.Sprintf("unknown storage type %T", db)) + } + } + return nil, tsdb.ErrNotReady +} + // Delete implements the api_v1.TSDBAdminStats and api_v2.TSDBAdmin interfaces. func (s *readyStorage) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Matcher) error { if x := s.get(); x != nil { @@ -1704,7 +2040,7 @@ func (s *readyStorage) WALReplayStatus() (tsdb.WALReplayStatus, error) { } // ErrNotReady is returned if the underlying scrape manager is not ready yet. -var ErrNotReady = errors.New("Scrape manager not ready") +var ErrNotReady = errors.New("scrape manager not ready") // ReadyScrapeManager allows a scrape manager to be retrieved. Even if it's set at a later point in time. type readyScrapeManager struct { @@ -1739,9 +2075,9 @@ type tsdbOptions struct { MaxBlockChunkSegmentSize units.Base2Bytes RetentionDuration model.Duration MaxBytes units.Base2Bytes + MaxPercentage float64 NoLockfile bool - WALCompression bool - WALCompressionType string + WALCompressionType compression.Type HeadChunksWriteQueueSize int SamplesPerChunk int StripeSize int @@ -1751,9 +2087,17 @@ type tsdbOptions struct { EnableExemplarStorage bool MaxExemplars int64 EnableMemorySnapshotOnShutdown bool - EnableNativeHistograms bool EnableDelayedCompaction bool + CompactionDelayMaxPercent int EnableOverlappingCompaction bool + UseUncachedIO bool + BlockCompactionExcludeFunc tsdb.BlockExcludeFilterFunc + BlockReloadInterval model.Duration + EnableSTAsZeroSample bool + EnableSTStorage bool + StaleSeriesCompactionThreshold float64 + EnableFastStartup bool + FloatChunkEncoding chunkenc.Encoding } func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { @@ -1762,8 +2106,9 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { MaxBlockChunkSegmentSize: int64(opts.MaxBlockChunkSegmentSize), RetentionDuration: int64(time.Duration(opts.RetentionDuration) / time.Millisecond), MaxBytes: int64(opts.MaxBytes), + MaxPercentage: opts.MaxPercentage, NoLockfile: opts.NoLockfile, - WALCompression: wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType), + WALCompression: opts.WALCompressionType, HeadChunksWriteQueueSize: opts.HeadChunksWriteQueueSize, SamplesPerChunk: opts.SamplesPerChunk, StripeSize: opts.StripeSize, @@ -1772,24 +2117,36 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { EnableExemplarStorage: opts.EnableExemplarStorage, MaxExemplars: opts.MaxExemplars, EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown, - EnableNativeHistograms: opts.EnableNativeHistograms, OutOfOrderTimeWindow: opts.OutOfOrderTimeWindow, EnableDelayedCompaction: opts.EnableDelayedCompaction, + CompactionDelayMaxPercent: opts.CompactionDelayMaxPercent, EnableOverlappingCompaction: opts.EnableOverlappingCompaction, + UseUncachedIO: opts.UseUncachedIO, + BlockCompactionExcludeFunc: opts.BlockCompactionExcludeFunc, + BlockReloadInterval: time.Duration(opts.BlockReloadInterval), + FeatureRegistry: features.DefaultRegistry, + EnableSTAsZeroSample: opts.EnableSTAsZeroSample, + EnableSTStorage: opts.EnableSTStorage, + StaleSeriesCompactionThreshold: opts.StaleSeriesCompactionThreshold, + EnableFastStartup: opts.EnableFastStartup, + FloatChunkEncoding: opts.FloatChunkEncoding, } } // agentOptions is a version of agent.Options with defined units. This is required // as agent.Option fields are unit agnostic (time). type agentOptions struct { - WALSegmentSize units.Base2Bytes - WALCompression bool - WALCompressionType string - StripeSize int - TruncateFrequency model.Duration - MinWALTime, MaxWALTime model.Duration - NoLockfile bool - OutOfOrderTimeWindow int64 + WALSegmentSize units.Base2Bytes + WALCompressionType compression.Type + StripeSize int + TruncateFrequency model.Duration + MinWALTime, MaxWALTime model.Duration + NoLockfile bool + OutOfOrderTimeWindow int64 // TODO(bwplotka): Unused option, fix it or remove. + EnableSTAsZeroSample bool + EnableSTStorage bool + CheckpointFromInMemorySeries bool + CheckpointBatchSize int } func (opts agentOptions) ToAgentOptions(outOfOrderTimeWindow int64) agent.Options { @@ -1797,37 +2154,32 @@ func (opts agentOptions) ToAgentOptions(outOfOrderTimeWindow int64) agent.Option outOfOrderTimeWindow = 0 } return agent.Options{ - WALSegmentSize: int(opts.WALSegmentSize), - WALCompression: wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType), - StripeSize: opts.StripeSize, - TruncateFrequency: time.Duration(opts.TruncateFrequency), - MinWALTime: durationToInt64Millis(time.Duration(opts.MinWALTime)), - MaxWALTime: durationToInt64Millis(time.Duration(opts.MaxWALTime)), - NoLockfile: opts.NoLockfile, - OutOfOrderTimeWindow: outOfOrderTimeWindow, + WALSegmentSize: int(opts.WALSegmentSize), + WALCompression: opts.WALCompressionType, + StripeSize: opts.StripeSize, + TruncateFrequency: time.Duration(opts.TruncateFrequency), + MinWALTime: durationToInt64Millis(time.Duration(opts.MinWALTime)), + MaxWALTime: durationToInt64Millis(time.Duration(opts.MaxWALTime)), + NoLockfile: opts.NoLockfile, + OutOfOrderTimeWindow: outOfOrderTimeWindow, + EnableSTAsZeroSample: opts.EnableSTAsZeroSample, + EnableSTStorage: opts.EnableSTStorage, + CheckpointFromInMemorySeries: opts.CheckpointFromInMemorySeries, + CheckpointBatchSize: opts.CheckpointBatchSize, } } -// discoveryManager interfaces the discovery manager. This is used to keep using -// the manager that restarts SD's on reload for a few releases until we feel -// the new manager can be enabled for all users. -type discoveryManager interface { - ApplyConfig(cfg map[string]discovery.Configs) error - Run() error - SyncCh() <-chan map[string][]*targetgroup.Group -} - -// rwProtoMsgFlagParser is a custom parser for config.RemoteWriteProtoMsg enum. +// rwProtoMsgFlagParser is a custom parser for remoteapi.WriteMessageType enum. type rwProtoMsgFlagParser struct { - msgs *[]config.RemoteWriteProtoMsg + msgs *remoteapi.MessageTypes } -func rwProtoMsgFlagValue(msgs *[]config.RemoteWriteProtoMsg) kingpin.Value { +func rwProtoMsgFlagValue(msgs *remoteapi.MessageTypes) kingpin.Value { return &rwProtoMsgFlagParser{msgs: msgs} } // IsCumulative is used by kingpin to tell if it's an array or not. -func (p *rwProtoMsgFlagParser) IsCumulative() bool { +func (*rwProtoMsgFlagParser) IsCumulative() bool { return true } @@ -1840,15 +2192,58 @@ func (p *rwProtoMsgFlagParser) String() string { } func (p *rwProtoMsgFlagParser) Set(opt string) error { - t := config.RemoteWriteProtoMsg(opt) + t := remoteapi.WriteMessageType(opt) if err := t.Validate(); err != nil { return err } - for _, prev := range *p.msgs { - if prev == t { - return fmt.Errorf("duplicated %v flag value, got %v already", t, *p.msgs) - } + if slices.Contains(*p.msgs, t) { + return fmt.Errorf("duplicated %v flag value, got %v already", t, *p.msgs) } *p.msgs = append(*p.msgs, t) return nil } + +type UploadMeta struct { + Uploaded []string `json:"uploaded"` +} + +// Cache the last read UploadMeta. +var ( + tsdbDelayCompactLastMeta *UploadMeta // The content of uploadMetaPath from the last time we've opened it. + tsdbDelayCompactLastMetaTime time.Time // The timestamp at which we stored tsdbDelayCompactLastMeta last time. +) + +func exludeBlocksPendingUpload(logger *slog.Logger, uploadMetaPath string) tsdb.BlockExcludeFilterFunc { + return func(meta *tsdb.BlockMeta) bool { + if meta.Compaction.Level > 1 { + // Blocks with level > 1 are assumed to be not uploaded, thus no need to delay those. + // See `storage.tsdb.delay-compact-file.path` flag for detail. + return false + } + + // If we have cached uploadMetaPath content that was stored in the last minute the use it. + if tsdbDelayCompactLastMeta != nil && + tsdbDelayCompactLastMetaTime.After(time.Now().UTC().Add(time.Minute*-1)) { + return !slices.Contains(tsdbDelayCompactLastMeta.Uploaded, meta.ULID.String()) + } + + // We don't have anything cached or it's older than a minute. Try to open and parse the uploadMetaPath path. + data, err := os.ReadFile(uploadMetaPath) + if err != nil { + logger.Warn("cannot open TSDB upload meta file", slog.String("path", uploadMetaPath), slog.Any("err", err)) + return false + } + + var uploadMeta UploadMeta + if err = json.Unmarshal(data, &uploadMeta); err != nil { + logger.Warn("cannot parse TSDB upload meta file", slog.String("path", uploadMetaPath), slog.Any("err", err)) + return false + } + + // We have parsed the uploadMetaPath file, cache it. + tsdbDelayCompactLastMeta = &uploadMeta + tsdbDelayCompactLastMetaTime = time.Now().UTC() + + return !slices.Contains(uploadMeta.Uploaded, meta.ULID.String()) + } +} diff --git a/cmd/prometheus/main_test.go b/cmd/prometheus/main_test.go index c827812e607..216bb882c80 100644 --- a/cmd/prometheus/main_test.go +++ b/cmd/prometheus/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -20,28 +20,41 @@ import ( "fmt" "io" "math" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" + "slices" "strconv" "strings" + "sync" "syscall" "testing" "time" "github.com/alecthomas/kingpin/v2" - "github.com/go-kit/log" + "github.com/grafana/regexp" + remoteapi "github.com/prometheus/client_golang/exp/api/remote" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/rules" + "github.com/prometheus/prometheus/util/testutil" ) +func init() { + // This can be removed when the legacy global mode is fully deprecated. + //nolint:staticcheck + model.NameValidationScheme = model.UTF8Validation +} + const startupTime = 10 * time.Second var ( @@ -120,6 +133,7 @@ func TestFailedStartupExitCode(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() fakeInputFile := "fake-input-file" expectedExitStatus := 2 @@ -134,6 +148,47 @@ func TestFailedStartupExitCode(t *testing.T) { require.Equal(t, expectedExitStatus, status.ExitStatus()) } +func TestRetentionPercentageStartsWithMissingStoragePath(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + t.Parallel() + + tmpDir := t.TempDir() + if storagePathFsSize(tmpDir) == 0 { + t.Skip("skipping test because filesystem size detection is unavailable.") + } + + configFile := filepath.Join(tmpDir, "prometheus.yml") + storagePath := filepath.Join(tmpDir, "missing", "data") + + require.NoError(t, os.WriteFile(configFile, []byte(` +storage: + tsdb: + retention: + percentage: 1.5 +`), 0o777)) + + port := testutil.RandomUnprivilegedPort(t) + prom := prometheusCommandWithLogging( + t, + configFile, + port, + "--storage.tsdb.path="+storagePath, + ) + require.NoError(t, prom.Start()) + + require.Eventually(t, func() bool { + r, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port)) + if err != nil { + return false + } + defer r.Body.Close() + return r.StatusCode == http.StatusOK + }, startupTime, 100*time.Millisecond) + require.DirExists(t, storagePath) +} + type senderFunc func(alerts ...*notifier.Alert) func (s senderFunc) Send(alerts ...*notifier.Alert) { @@ -191,7 +246,6 @@ func TestSendAlerts(t *testing.T) { } for i, tc := range testCases { - tc := tc t.Run(strconv.Itoa(i), func(t *testing.T) { senderFunc := senderFunc(func(alerts ...*notifier.Alert) { require.NotEmpty(t, tc.in, "sender called with 0 alert") @@ -206,83 +260,241 @@ func TestWALSegmentSizeBounds(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() - for size, expectedExitStatus := range map[string]int{"9MB": 1, "257MB": 1, "10": 2, "1GB": 1, "12MB": 0} { - prom := exec.Command(promPath, "-test.main", "--storage.tsdb.wal-segment-size="+size, "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig, "--storage.tsdb.path="+filepath.Join(t.TempDir(), "data")) + for _, tc := range []struct { + size string + exitCode int + }{ + { + size: "9MB", + exitCode: 1, + }, + { + size: "257MB", + exitCode: 1, + }, + { + size: "10", + exitCode: 2, + }, + { + size: "1GB", + exitCode: 1, + }, + { + size: "12MB", + exitCode: 0, + }, + } { + t.Run(tc.size, func(t *testing.T) { + t.Parallel() + prom := exec.Command(promPath, "-test.main", "--storage.tsdb.wal-segment-size="+tc.size, "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig, "--storage.tsdb.path="+filepath.Join(t.TempDir(), "data")) - // Log stderr in case of failure. - stderr, err := prom.StderrPipe() - require.NoError(t, err) - go func() { - slurp, _ := io.ReadAll(stderr) - t.Log(string(slurp)) - }() + // Log stderr in case of failure. + stderr, err := prom.StderrPipe() + require.NoError(t, err) - err = prom.Start() - require.NoError(t, err) + // WaitGroup is used to ensure that we don't call t.Log() after the test has finished. + var wg sync.WaitGroup + wg.Add(1) + defer wg.Wait() + + go func() { + defer wg.Done() + slurp, _ := io.ReadAll(stderr) + t.Log(string(slurp)) + }() + + err = prom.Start() + require.NoError(t, err) - if expectedExitStatus == 0 { - done := make(chan error, 1) - go func() { done <- prom.Wait() }() - select { - case err := <-done: - require.Fail(t, "prometheus should be still running: %v", err) - case <-time.After(startupTime): - prom.Process.Kill() - <-done + if tc.exitCode == 0 { + done := make(chan error, 1) + go func() { done <- prom.Wait() }() + select { + case err := <-done: + t.Fatalf("prometheus should be still running: %v", err) + case <-time.After(startupTime): + prom.Process.Kill() + <-done + } + return } - continue - } - err = prom.Wait() - require.Error(t, err) - var exitError *exec.ExitError - require.ErrorAs(t, err, &exitError) - status := exitError.Sys().(syscall.WaitStatus) - require.Equal(t, expectedExitStatus, status.ExitStatus()) + err = prom.Wait() + require.Error(t, err) + var exitError *exec.ExitError + require.ErrorAs(t, err, &exitError) + status := exitError.Sys().(syscall.WaitStatus) + require.Equal(t, tc.exitCode, status.ExitStatus()) + }) } } func TestMaxBlockChunkSegmentSizeBounds(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } t.Parallel() + for _, tc := range []struct { + size string + exitCode int + }{ + { + size: "512KB", + exitCode: 1, + }, + { + size: "1MB", + exitCode: 0, + }, + } { + t.Run(tc.size, func(t *testing.T) { + t.Parallel() + prom := exec.Command(promPath, "-test.main", "--storage.tsdb.max-block-chunk-segment-size="+tc.size, "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig, "--storage.tsdb.path="+filepath.Join(t.TempDir(), "data")) + + // Log stderr in case of failure. + stderr, err := prom.StderrPipe() + require.NoError(t, err) + + // WaitGroup is used to ensure that we don't call t.Log() after the test has finished. + var wg sync.WaitGroup + wg.Add(1) + defer wg.Wait() + + go func() { + defer wg.Done() + slurp, _ := io.ReadAll(stderr) + t.Log(string(slurp)) + }() + + err = prom.Start() + require.NoError(t, err) + + if tc.exitCode == 0 { + done := make(chan error, 1) + go func() { done <- prom.Wait() }() + select { + case err := <-done: + t.Fatalf("prometheus should be still running: %v", err) + case <-time.After(startupTime): + prom.Process.Kill() + <-done + } + return + } + + err = prom.Wait() + require.Error(t, err) + var exitError *exec.ExitError + require.ErrorAs(t, err, &exitError) + status := exitError.Sys().(syscall.WaitStatus) + require.Equal(t, tc.exitCode, status.ExitStatus()) + }) + } +} + +func TestChunkEncodingStartupValidation(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() - for size, expectedExitStatus := range map[string]int{"512KB": 1, "1MB": 0} { - prom := exec.Command(promPath, "-test.main", "--storage.tsdb.max-block-chunk-segment-size="+size, "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig, "--storage.tsdb.path="+filepath.Join(t.TempDir(), "data")) + for _, tc := range []struct { + name string + config string + features string + exitCode int + }{ + { + name: "xor2 without xor2-encoding feature", + config: ` +storage: + tsdb: + chunk_encoding: + floats: xor2`, + features: "", + exitCode: 1, + }, + { + name: "xor2 with xor2-encoding feature", + config: ` +storage: + tsdb: + chunk_encoding: + floats: xor2`, + features: "xor2-encoding", + exitCode: 0, + }, + { + name: "xor with st-storage and xor2-encoding features", + config: ` +storage: + tsdb: + chunk_encoding: + floats: xor`, + features: "st-storage,xor2-encoding", + exitCode: 1, + }, + { + name: "xor without st-storage feature", + config: ` +storage: + tsdb: + chunk_encoding: + floats: xor`, + features: "", + exitCode: 0, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "prometheus.yml") + require.NoError(t, os.WriteFile(configFile, []byte(tc.config), 0o777)) + + args := []string{"-test.main", "--web.listen-address=0.0.0.0:0", "--config.file=" + configFile, "--storage.tsdb.path=" + filepath.Join(tmpDir, "data")} + if tc.features != "" { + args = append(args, "--enable-feature="+tc.features) + } + prom := exec.Command(promPath, args...) - // Log stderr in case of failure. - stderr, err := prom.StderrPipe() - require.NoError(t, err) - go func() { - slurp, _ := io.ReadAll(stderr) - t.Log(string(slurp)) - }() + stderr, err := prom.StderrPipe() + require.NoError(t, err) - err = prom.Start() - require.NoError(t, err) + var wg sync.WaitGroup + wg.Add(1) + defer wg.Wait() + go func() { + defer wg.Done() + slurp, _ := io.ReadAll(stderr) + t.Log(string(slurp)) + }() - if expectedExitStatus == 0 { - done := make(chan error, 1) - go func() { done <- prom.Wait() }() - select { - case err := <-done: - require.Fail(t, "prometheus should be still running: %v", err) - case <-time.After(startupTime): - prom.Process.Kill() - <-done + require.NoError(t, prom.Start()) + + if tc.exitCode == 0 { + done := make(chan error, 1) + go func() { done <- prom.Wait() }() + select { + case err := <-done: + t.Fatalf("prometheus should be still running: %v", err) + case <-time.After(startupTime): + prom.Process.Kill() + <-done + } + return } - continue - } - err = prom.Wait() - require.Error(t, err) - var exitError *exec.ExitError - require.ErrorAs(t, err, &exitError) - status := exitError.Sys().(syscall.WaitStatus) - require.Equal(t, expectedExitStatus, status.ExitStatus()) + err = prom.Wait() + require.Error(t, err) + var exitError *exec.ExitError + require.ErrorAs(t, err, &exitError) + status := exitError.Sys().(syscall.WaitStatus) + require.Equal(t, tc.exitCode, status.ExitStatus()) + }) } } @@ -290,7 +502,7 @@ func TestTimeMetrics(t *testing.T) { tmpDir := t.TempDir() reg := prometheus.NewRegistry() - db, err := openDBWithMetrics(tmpDir, log.NewNopLogger(), reg, nil, nil) + db, err := openDBWithMetrics(tmpDir, promslog.NewNopLogger(), reg, nil, nil) require.NoError(t, err) defer func() { require.NoError(t, db.Close()) @@ -328,6 +540,7 @@ func TestTimeMetrics(t *testing.T) { } func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames ...string) map[string]float64 { + t.Helper() f, err := reg.Gather() require.NoError(t, err) @@ -348,7 +561,9 @@ func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames } func TestAgentSuccessfulStartup(t *testing.T) { - prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--web.listen-address=0.0.0.0:0", "--config.file="+agentConfig) + t.Parallel() + + prom := exec.Command(promPath, "-test.main", "--agent", "--web.listen-address=0.0.0.0:0", "--config.file="+agentConfig) require.NoError(t, prom.Start()) actualExitStatus := 0 @@ -357,7 +572,7 @@ func TestAgentSuccessfulStartup(t *testing.T) { go func() { done <- prom.Wait() }() select { case err := <-done: - t.Logf("prometheus agent should be still running: %v", err) + t.Logf("prometheus agent exited early: %v", err) actualExitStatus = prom.ProcessState.ExitCode() case <-time.After(startupTime): prom.Process.Kill() @@ -366,7 +581,9 @@ func TestAgentSuccessfulStartup(t *testing.T) { } func TestAgentFailedStartupWithServerFlag(t *testing.T) { - prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--storage.tsdb.path=.", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig) + t.Parallel() + + prom := exec.Command(promPath, "-test.main", "--agent", "--storage.tsdb.path=.", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig) output := bytes.Buffer{} prom.Stderr = &output @@ -393,7 +610,9 @@ func TestAgentFailedStartupWithServerFlag(t *testing.T) { } func TestAgentFailedStartupWithInvalidConfig(t *testing.T) { - prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig) + t.Parallel() + + prom := exec.Command(promPath, "-test.main", "--agent", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig) require.NoError(t, prom.Start()) actualExitStatus := 0 @@ -414,6 +633,7 @@ func TestModeSpecificFlags(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() testcases := []struct { mode string @@ -428,10 +648,11 @@ func TestModeSpecificFlags(t *testing.T) { for _, tc := range testcases { t.Run(fmt.Sprintf("%s mode with option %s", tc.mode, tc.arg), func(t *testing.T) { + t.Parallel() args := []string{"-test.main", tc.arg, t.TempDir(), "--web.listen-address=0.0.0.0:0"} if tc.mode == "agent" { - args = append(args, "--enable-feature=agent", "--config.file="+agentConfig) + args = append(args, "--agent", "--config.file="+agentConfig) } else { args = append(args, "--config.file="+promConfig) } @@ -441,7 +662,14 @@ func TestModeSpecificFlags(t *testing.T) { // Log stderr in case of failure. stderr, err := prom.StderrPipe() require.NoError(t, err) + + // WaitGroup is used to ensure that we don't call t.Log() after the test has finished. + var wg sync.WaitGroup + wg.Add(1) + defer wg.Wait() + go func() { + defer wg.Done() slurp, _ := io.ReadAll(stderr) t.Log(string(slurp)) }() @@ -479,6 +707,8 @@ func TestDocumentation(t *testing.T) { if runtime.GOOS == "windows" { t.SkipNow() } + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -487,12 +717,7 @@ func TestDocumentation(t *testing.T) { var stdout bytes.Buffer cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - var exitError *exec.ExitError - if errors.As(err, &exitError) && exitError.ExitCode() != 0 { - fmt.Println("Command failed with non-zero exit code") - } - } + require.NoError(t, cmd.Run(), "failed to generate CLI documentation via --write-documentation") generatedContent := strings.ReplaceAll(stdout.String(), filepath.Base(promPath), strings.TrimSuffix(filepath.Base(promPath), ".test")) @@ -503,13 +728,15 @@ func TestDocumentation(t *testing.T) { } func TestRwProtoMsgFlagParser(t *testing.T) { - defaultOpts := config.RemoteWriteProtoMsgs{ - config.RemoteWriteProtoMsgV1, config.RemoteWriteProtoMsgV2, + t.Parallel() + + defaultOpts := remoteapi.MessageTypes{ + remoteapi.WriteV1MessageType, remoteapi.WriteV2MessageType, } for _, tcase := range []struct { args []string - expected []config.RemoteWriteProtoMsg + expected remoteapi.MessageTypes expectedErr error }{ { @@ -518,25 +745,25 @@ func TestRwProtoMsgFlagParser(t *testing.T) { }, { args: []string{"--test-proto-msgs", "test"}, - expectedErr: errors.New("unknown remote write protobuf message test, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request"), + expectedErr: errors.New("unknown type for remote write protobuf message test, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request"), }, { args: []string{"--test-proto-msgs", "io.prometheus.write.v2.Request"}, - expected: config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV2}, + expected: remoteapi.MessageTypes{remoteapi.WriteV2MessageType}, }, { args: []string{ "--test-proto-msgs", "io.prometheus.write.v2.Request", "--test-proto-msgs", "io.prometheus.write.v2.Request", }, - expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got [io.prometheus.write.v2.Request] already"), + expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got io.prometheus.write.v2.Request already"), }, { args: []string{ "--test-proto-msgs", "io.prometheus.write.v2.Request", "--test-proto-msgs", "prometheus.WriteRequest", }, - expected: config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV2, config.RemoteWriteProtoMsgV1}, + expected: remoteapi.MessageTypes{remoteapi.WriteV2MessageType, remoteapi.WriteV1MessageType}, }, { args: []string{ @@ -544,12 +771,12 @@ func TestRwProtoMsgFlagParser(t *testing.T) { "--test-proto-msgs", "prometheus.WriteRequest", "--test-proto-msgs", "io.prometheus.write.v2.Request", }, - expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got [io.prometheus.write.v2.Request prometheus.WriteRequest] already"), + expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got io.prometheus.write.v2.Request, prometheus.WriteRequest already"), }, } { t.Run(strings.Join(tcase.args, ","), func(t *testing.T) { a := kingpin.New("test", "") - var opt []config.RemoteWriteProtoMsg + var opt remoteapi.MessageTypes a.Flag("test-proto-msgs", "").Default(defaultOpts.Strings()...).SetValue(rwProtoMsgFlagValue(&opt)) _, err := a.Parse(tcase.args) @@ -563,3 +790,447 @@ func TestRwProtoMsgFlagParser(t *testing.T) { }) } } + +// reloadPrometheusConfig sends a reload request to the Prometheus server to apply +// updated configurations. +func reloadPrometheusConfig(t *testing.T, reloadURL string) { + t.Helper() + + r, err := http.Post(reloadURL, "text/plain", nil) + require.NoError(t, err, "Failed to reload Prometheus") + require.Equal(t, http.StatusOK, r.StatusCode, "Unexpected status code when reloading Prometheus") +} + +func getMetricValue(t *testing.T, body io.Reader, metricType model.MetricType, metricName string) (float64, error) { + t.Helper() + + p := expfmt.NewTextParser(model.UTF8Validation) + metricFamilies, err := p.TextToMetricFamilies(body) + if err != nil { + return 0, err + } + metricFamily, ok := metricFamilies[metricName] + if !ok { + return 0, errors.New("metric family not found") + } + metric := metricFamily.GetMetric() + if len(metric) != 1 { + return 0, errors.New("metric not found") + } + switch metricType { + case model.MetricTypeGauge: + return metric[0].GetGauge().GetValue(), nil + case model.MetricTypeCounter: + return metric[0].GetCounter().GetValue(), nil + default: + t.Fatalf("metric type %s not supported", metricType) + } + + return 0, errors.New("cannot get value") +} + +func TestRuntimeGOGCConfig(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + t.Parallel() + + for _, tc := range []struct { + name string + config string + gogcEnvVar string + expectedGOGC float64 + }{ + { + name: "empty config file", + expectedGOGC: 75, + }, + { + name: "empty config file with GOGC env var set", + gogcEnvVar: "66", + expectedGOGC: 66, + }, + { + name: "gogc set through config", + config: ` +runtime: + gogc: 77`, + expectedGOGC: 77.0, + }, + { + name: "gogc set through config and env var", + config: ` +runtime: + gogc: 77`, + gogcEnvVar: "88", + expectedGOGC: 77.0, + }, + { + name: "incomplete runtime block", + config: ` +runtime:`, + expectedGOGC: 75.0, + }, + { + name: "incomplete runtime block and GOGC env var set", + config: ` +runtime:`, + gogcEnvVar: "88", + expectedGOGC: 88.0, + }, + { + name: "unrelated config and GOGC env var set", + config: ` +global: + scrape_interval: 500ms`, + gogcEnvVar: "80", + expectedGOGC: 80, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "prometheus.yml") + + port := testutil.RandomUnprivilegedPort(t) + require.NoError(t, os.WriteFile(configFile, []byte(tc.config), 0o777)) + prom := prometheusCommandWithLogging( + t, + configFile, + port, + fmt.Sprintf("--storage.tsdb.path=%s", tmpDir), + "--web.enable-lifecycle", + ) + // Inject GOGC when set. + prom.Env = os.Environ() + if tc.gogcEnvVar != "" { + prom.Env = append(prom.Env, fmt.Sprintf("GOGC=%s", tc.gogcEnvVar)) + } + require.NoError(t, prom.Start()) + + ensureGOGCValue := func(val float64) { + var ( + r *http.Response + err error + ) + // Wait for the /metrics endpoint to be ready. + require.Eventually(t, func() bool { + r, err = http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port)) + if err != nil { + return false + } + return r.StatusCode == http.StatusOK + }, 5*time.Second, 50*time.Millisecond) + defer r.Body.Close() + + // Check the final GOGC that's set, consider go_gc_gogc_percent from /metrics as source of truth. + gogc, err := getMetricValue(t, r.Body, model.MetricTypeGauge, "go_gc_gogc_percent") + require.NoError(t, err) + require.Equal(t, val, gogc) + } + + // The value is applied on startup. + ensureGOGCValue(tc.expectedGOGC) + + // After a reload with the same config, the value stays the same. + reloadURL := fmt.Sprintf("http://127.0.0.1:%d/-/reload", port) + reloadPrometheusConfig(t, reloadURL) + ensureGOGCValue(tc.expectedGOGC) + + // After a reload with different config, the value gets updated. + newConfig := ` +runtime: + gogc: 99` + require.NoError(t, os.WriteFile(configFile, []byte(newConfig), 0o777)) + reloadPrometheusConfig(t, reloadURL) + ensureGOGCValue(99.0) + }) + } +} + +// TestHeadCompactionWhileScraping verifies that running a head compaction +// concurrently with a scrape does not trigger the data race described in +// https://github.com/prometheus/prometheus/issues/16490. +func TestHeadCompactionWhileScraping(t *testing.T) { + t.Parallel() + + // To increase the chance of reproducing the data race + for i := range 5 { + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "prometheus.yml") + + port := testutil.RandomUnprivilegedPort(t) + config := fmt.Sprintf(` +scrape_configs: + - job_name: 'self1' + scrape_interval: 61ms + static_configs: + - targets: ['localhost:%d'] + - job_name: 'self2' + scrape_interval: 67ms + static_configs: + - targets: ['localhost:%d'] +`, port, port) + require.NoError(t, os.WriteFile(configFile, []byte(config), 0o777)) + + prom := prometheusCommandWithLogging( + t, + configFile, + port, + fmt.Sprintf("--storage.tsdb.path=%s", tmpDir), + "--storage.tsdb.min-block-duration=100ms", + ) + require.NoError(t, prom.Start()) + + require.Eventually(t, func() bool { + r, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port)) + if err != nil { + return false + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return false + } + metrics, err := io.ReadAll(r.Body) + if err != nil { + return false + } + + // Wait for some compactions to run + compactions, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeCounter, "prometheus_tsdb_compactions_total") + if err != nil { + return false + } + if compactions < 3 { + return false + } + + // Sanity check: Some actual scraping was done. + series, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeCounter, "prometheus_tsdb_head_series_created_total") + require.NoError(t, err) + require.NotZero(t, series) + + // No compaction must have failed + failures, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeCounter, "prometheus_tsdb_compactions_failed_total") + require.NoError(t, err) + require.Zero(t, failures) + return true + }, 15*time.Second, 500*time.Millisecond) + }) + } +} + +// This test verifies that metrics for the highest timestamps per queue account for relabelling. +// See: https://github.com/prometheus/prometheus/pull/17065. +func TestRemoteWrite_PerQueueMetricsAfterRelabeling(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "prometheus.yml") + + port := testutil.RandomUnprivilegedPort(t) + targetPort := testutil.RandomUnprivilegedPort(t) + + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.Fail(t, "should never be reached because the remote write relabeling shouldn't yield anything", "header: %v, body: %s", r.Header, body) + })) + t.Cleanup(server.Close) + + // Simulate a remote write relabeling that doesn't yield any series. + config := fmt.Sprintf(` +global: + scrape_interval: 1s +scrape_configs: + - job_name: 'self' + static_configs: + - targets: ['localhost:%d'] + - job_name: 'target' + static_configs: + - targets: ['localhost:%d'] + +remote_write: + - url: %s + write_relabel_configs: + - source_labels: [job,__name__] + regex: 'target,special_metric' + action: keep +`, port, targetPort, server.URL) + require.NoError(t, os.WriteFile(configFile, []byte(config), 0o777)) + + prom := prometheusCommandWithLogging( + t, + configFile, + port, + fmt.Sprintf("--storage.tsdb.path=%s", tmpDir), + ) + require.NoError(t, prom.Start()) + + require.Eventually(t, func() bool { + r, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port)) + if err != nil { + return false + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return false + } + + metrics, err := io.ReadAll(r.Body) + if err != nil { + return false + } + + gHighestTimestamp, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeGauge, "prometheus_remote_storage_highest_timestamp_in_seconds") + // The highest timestamp at storage level sees all samples, it should also consider the ones that are filtered out by relabeling. + if err != nil || gHighestTimestamp == 0 { + return false + } + + // The queue shouldn't see and send any sample, all samples are dropped due to relabeling, the metrics should reflect that. + droppedSamples, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeCounter, "prometheus_remote_storage_samples_dropped_total") + if err != nil || droppedSamples == 0 { + return false + } + + highestTimestamp, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeGauge, "prometheus_remote_storage_queue_highest_timestamp_seconds") + require.NoError(t, err) + require.Zero(t, highestTimestamp) + + highestSentTimestamp, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeGauge, "prometheus_remote_storage_queue_highest_sent_timestamp_seconds") + require.NoError(t, err) + require.Zero(t, highestSentTimestamp) + return true + }, 10*time.Second, 100*time.Millisecond) +} + +// TestRemoteWrite_ReshardingWithoutDeadlock ensures that resharding (scaling up) doesn't block when the shards are full. +// See: https://github.com/prometheus/prometheus/issues/17384. +// +// The following shows key resharding metrics before and after the fix. +// In v3.7.0, the deadlock prevented the resharding logic from observing the true incoming data rate. +// +// | Metric | v3.7.0 | after the fix | +// |---------------------|---------------|---------------------| +// | dataInRate | 0.6 | 307.2 | +// | dataPendingRate | 0.2 | 306.8 | +// | dataPending | 0 | 1228.8 | +// | desiredShards | 0.6 | 369.2 |. +func TestRemoteWrite_ReshardingWithoutDeadlock(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "prometheus.yml") + + port := testutil.RandomUnprivilegedPort(t) + + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + time.Sleep(time.Second) + })) + t.Cleanup(server.Close) + + config := fmt.Sprintf(` +global: + # Using a smaller interval may cause the scrape to time out. + scrape_interval: 1s +scrape_configs: + - job_name: 'self' + static_configs: + - targets: ['localhost:%d'] + +remote_write: + - url: %s + queue_config: + # Speed up the queue being full. + capacity: 1 + # Helps keep the “time to send one sample” low so it doesn’t influence the resharding logic. + max_samples_per_send: 1 +`, port, server.URL) + require.NoError(t, os.WriteFile(configFile, []byte(config), 0o777)) + + prom := prometheusCommandWithLogging( + t, + configFile, + port, + fmt.Sprintf("--storage.tsdb.path=%s", tmpDir), + "--log.level=debug", + ) + require.NoError(t, prom.Start()) + + const desiredShardsMetric = "prometheus_remote_storage_shards_desired" + getMetrics := func() ([]byte, error) { + r, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port)) + if err != nil { + return nil, err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", r.StatusCode) + } + + metrics, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + return metrics, nil + } + + // Ensure the initial desired shards is 1. + require.Eventually(t, func() bool { + metrics, err := getMetrics() + if err != nil { + return false + } + initialDesiredShards, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeGauge, desiredShardsMetric) + if err != nil { + return false + } + return initialDesiredShards == 1.0 + }, 10*time.Second, 100*time.Millisecond) + + // Ensure scaling up is triggered after some time. + require.Eventually(t, func() bool { + metrics, err := getMetrics() + if err != nil { + return false + } + desiredShards, err := getMetricValue(t, bytes.NewReader(metrics), model.MetricTypeGauge, desiredShardsMetric) + if err != nil || desiredShards <= 1.0 { + return false + } + return true + // 3*shardUpdateDuration to allow for the resharding logic to run. + }, 30*time.Second, time.Second) +} + +// TestFeatureFlagsDocumented ensures the --enable-feature help text in main.go +// and the documented flags in docs/feature_flags.md list the same set of flags. +func TestFeatureFlagsDocumented(t *testing.T) { + if runtime.GOOS == "windows" { + t.SkipNow() + } + h, err := os.ReadFile(filepath.Join("..", "..", "cmd", "prometheus", "main.go")) + require.NoError(t, err) + m := regexp.MustCompile(`a\.Flag\("enable-feature", "Comma separated feature names to enable. Valid options: (.+?)\.`).FindSubmatch(h) + require.NotNil(t, m) + var helpFlags []string + for f := range strings.SplitSeq(string(m[1]), ",") { + helpFlags = append(helpFlags, strings.TrimSpace(f)) + } + require.NotEmpty(t, helpFlags) + + d, err := os.ReadFile(filepath.Join("..", "..", "docs", "feature_flags.md")) + require.NoError(t, err) + var docFlags []string + for _, dm := range regexp.MustCompile("(?m)^`--enable-feature=(.+)`$").FindAllSubmatch(d, -1) { + docFlags = append(docFlags, string(dm[1])) + } + require.NotEmpty(t, docFlags) + require.True(t, slices.IsSorted(helpFlags)) + require.ElementsMatch(t, helpFlags, docFlags) +} diff --git a/cmd/prometheus/main_unix_test.go b/cmd/prometheus/main_unix_test.go index 2011fb123f3..ea130b3bf9d 100644 --- a/cmd/prometheus/main_unix_test.go +++ b/cmd/prometheus/main_unix_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -34,6 +34,7 @@ func TestStartupInterrupt(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() port := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t)) @@ -52,7 +53,7 @@ func TestStartupInterrupt(t *testing.T) { url := "http://localhost" + port + "/graph" Loop: - for x := 0; x < 10; x++ { + for range 10 { // error=nil means prometheus has started, so we can send the interrupt // signal and wait for the graceful shutdown. if _, err := http.Get(url); err == nil { diff --git a/cmd/prometheus/main_upgrade_test.go b/cmd/prometheus/main_upgrade_test.go new file mode 100644 index 00000000000..65840b36589 --- /dev/null +++ b/cmd/prometheus/main_upgrade_test.go @@ -0,0 +1,413 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/util/testutil" +) + +const prometheusDownloadManifestURL = "https://prometheus.io/download.json" + +// ltsRelease identifies an LTS release and where to download it for the current OS/arch. +type ltsRelease struct { + version string + assetURL string +} + +// fetchLTSReleases fetches all current LTS releases and their download URLs for the +// current OS/arch from the Prometheus download manifest. There may be more than one +// LTS release active at the same time. +func fetchLTSReleases(t *testing.T) []ltsRelease { + t.Helper() + + resp, err := http.Get(prometheusDownloadManifestURL) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + var manifest struct { + Prometheus []struct { + Version string `json:"version"` + LTS bool `json:"lts"` + Files []struct { + URL string `json:"url"` + OS string `json:"os"` + Arch string `json:"arch"` + Kind string `json:"kind"` + } `json:"files"` + } `json:"prometheus"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&manifest)) + + var releases []ltsRelease + for _, r := range manifest.Prometheus { + if !r.LTS { + continue + } + var found bool + for _, f := range r.Files { + if f.Kind == "archive" && f.OS == runtime.GOOS && f.Arch == runtime.GOARCH { + releases = append(releases, ltsRelease{ + version: strings.TrimPrefix(r.Version, "v"), + assetURL: f.URL, + }) + found = true + break + } + } + require.True(t, found, "no archive found for current OS/arch in LTS release %s (os=%s arch=%s)", r.Version, runtime.GOOS, runtime.GOARCH) + } + require.NotEmpty(t, releases, "no LTS release found in download manifest %s", prometheusDownloadManifestURL) + return releases +} + +func getPrometheusMetricValue(t *testing.T, port int, metricType model.MetricType, metricName string) (float64, error) { + t.Helper() + + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port)) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("bad status code %d", resp.StatusCode) + } + return getMetricValue(t, resp.Body, metricType, metricName) +} + +type versionChangeTest struct { + start time.Time + + ltsAssetURL string + ltsVersionBinPath string + + prometheusPort int + prometheusDataPath string + prometheusConfigFilePath string + rulesFilePath string + remoteWriteURL string +} + +func (c versionChangeTest) downloadAndExtractLatestLTS(t *testing.T) { + const prometheusBinName = "prometheus" + + resp, err := http.Get(c.ltsAssetURL) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + gzReader, err := gzip.NewReader(resp.Body) + require.NoError(t, err) + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + + if header.Typeflag == tar.TypeReg && filepath.Base(header.Name) == prometheusBinName { + out, err := os.OpenFile(c.ltsVersionBinPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) + require.NoError(t, err) + defer out.Close() + + _, err = io.Copy(out, tarReader) + require.NoError(t, err) + return + } + } + require.FailNow(t, "prometheus binary not found in LTS tarball", "URL", c.ltsAssetURL) +} + +// ensureHealthyMetrics polls metrics until all health invariants are satisfied. It checks +// both liveness (expected operations completed) and safety (no failures). +func (c versionChangeTest) ensureHealthyMetrics(t *testing.T) { + t.Helper() + checkStartTime := time.Now() + + for _, mc := range []struct { + mType model.MetricType + mName string + check func(float64) bool + }{ + {model.MetricTypeGauge, "prometheus_ready", func(v float64) bool { return v == 1 }}, + {model.MetricTypeGauge, "prometheus_config_last_reload_successful", func(v float64) bool { return v == 1 }}, + + {model.MetricTypeCounter, "prometheus_target_scrape_pools_total", func(v float64) bool { return v == 3 }}, + {model.MetricTypeCounter, "prometheus_target_scrape_pools_failed_total", func(v float64) bool { return v == 0 }}, + {model.MetricTypeCounter, "prometheus_target_scrape_pool_reloads_failed_total", func(v float64) bool { return v == 0 }}, + + {model.MetricTypeGauge, "prometheus_remote_storage_highest_timestamp_in_seconds", func(v float64) bool { return v > float64(checkStartTime.Unix()) }}, + + {model.MetricTypeGauge, "prometheus_rule_group_rules", func(v float64) bool { return v == 2 }}, + {model.MetricTypeCounter, "prometheus_rule_evaluations_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_rule_evaluation_failures_total", func(v float64) bool { return v == 0 }}, + + {model.MetricTypeCounter, "prometheus_tsdb_compactions_triggered_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_compactions_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_compactions_failed_total", func(v float64) bool { return v == 0 }}, + + {model.MetricTypeGauge, "prometheus_tsdb_head_series", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeGauge, "prometheus_tsdb_head_chunks", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeGauge, "prometheus_tsdb_head_chunks_storage_size_bytes", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_head_series_created_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeGauge, "prometheus_tsdb_blocks_loaded", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeGauge, "prometheus_tsdb_storage_blocks_bytes", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_reloads_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_reloads_failures_total", func(v float64) bool { return v == 0 }}, + + {model.MetricTypeCounter, "prometheus_tsdb_head_chunks_created_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_head_chunks_removed_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_mmap_chunks_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_mmap_chunk_corruptions_total", func(v float64) bool { return v == 0 }}, + + {model.MetricTypeCounter, "prometheus_tsdb_checkpoint_creations_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_checkpoint_creations_failed_total", func(v float64) bool { return v == 0 }}, + {model.MetricTypeCounter, "prometheus_tsdb_checkpoint_deletions_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_checkpoint_deletions_failed_total", func(v float64) bool { return v == 0 }}, + + {model.MetricTypeCounter, "prometheus_tsdb_head_truncations_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_head_truncations_failed_total", func(v float64) bool { return v == 0 }}, + + {model.MetricTypeGauge, "prometheus_tsdb_wal_storage_size_bytes", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_wal_completed_pages_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_wal_page_flushes_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_wal_truncations_total", func(v float64) bool { return v >= 1 }}, + {model.MetricTypeCounter, "prometheus_tsdb_wal_writes_failed_total", func(v float64) bool { return v == 0 }}, + {model.MetricTypeCounter, "prometheus_tsdb_wal_corruptions_total", func(v float64) bool { return v == 0 }}, + {model.MetricTypeCounter, "prometheus_tsdb_wal_truncations_failed_total", func(v float64) bool { return v == 0 }}, + } { + require.Eventually(t, func() bool { + val, err := getPrometheusMetricValue(t, c.prometheusPort, mc.mType, mc.mName) + if err != nil { + return false + } + if mc.check(val) { + t.Logf("[%s] metric %s=%g is healthy", time.Since(c.start), mc.mName, val) + return true + } + return false + }, 7*time.Minute, 1*time.Second) + } +} + +func (c versionChangeTest) generatePrometheusConfig(t *testing.T) { + t.Helper() + + rulesContent := ` +groups: + - name: upgrade_test + interval: 3s + rules: + - record: upgrade_test:up:sum + expr: sum(up) + - alert: UpgradeTestAlwaysFiring + expr: vector(1) + for: 1s + labels: + severity: foo +` + require.NoError(t, os.WriteFile(c.rulesFilePath, []byte(rulesContent), 0o644)) + + promConfig := fmt.Sprintf(` +rule_files: + - %s + +scrape_configs: + - job_name: 'self1' + scrape_interval: 1s + static_configs: + - targets: ['localhost:%d'] + - job_name: 'self2' + scrape_interval: 2s + static_configs: + - targets: ['localhost:%d'] + - job_name: 'self3' + scrape_interval: 3s + static_configs: + - targets: ['localhost:%d'] + +remote_write: + - url: %s`, c.rulesFilePath, c.prometheusPort, c.prometheusPort, c.prometheusPort, c.remoteWriteURL) + + require.NoError(t, os.WriteFile(c.prometheusConfigFilePath, []byte(promConfig), 0o644)) +} + +// snapshotQueryRange queries all metrics over [c.start, end]. +// The result is deterministic and can be compared across Prometheus versions to detect data loss or corruption. +func (c versionChangeTest) snapshotQueryRange(t *testing.T, end time.Time) string { + t.Helper() + + u := fmt.Sprintf("http://127.0.0.1:%d/api/v1/query_range?query=%s&start=%d&end=%d&step=15", + c.prometheusPort, url.QueryEscape(`{__name__=~".+"}`), c.start.Unix(), end.Unix()) + resp, err := http.Get(u) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + var parsed struct { + Data struct { + Result json.RawMessage `json:"result"` + } `json:"data"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&parsed)) + require.NotEqual(t, "[]", string(parsed.Data.Result)) + return string(parsed.Data.Result) +} + +func ensureHealthyLogs(t *testing.T, r io.Reader) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + t.Log(line) + require.NotContains(t, line, "level=ERROR") + } + require.NoError(t, scanner.Err()) +} + +var ( + testVersionUpgrade = flag.Bool("test.version-upgrade", false, "run resource-intensive and probably slow version upgrade tests") + testLTSVersion = flag.String("test.lts-version", "", "restrict the version upgrade test to a single LTS version (e.g. 3.5.0); if empty, all active LTS releases are tested") +) + +// TestVersionUpgrade_UpgradeDowngradeLatestLTS verifies that Prometheus can +// upgrade from each current LTS release to the current build and then downgrade +// back without errors (data loss, corruption, etc.). There may be more than one +// active LTS release, in which case each is tested independently. +// +// NOTE: If this test is renamed, update the corresponding invocation in CI. +func TestVersionUpgrade_UpgradeDowngradeLatestLTS(t *testing.T) { + if !*testVersionUpgrade { + t.Skip("test can be slow, resource-intensive or requires internet access") + } + + start := time.Now() + + ltsReleases := fetchLTSReleases(t) + t.Logf("[%s] found %d LTS release(s) from %s", time.Since(start), len(ltsReleases), prometheusDownloadManifestURL) + + if *testLTSVersion != "" { + want := strings.TrimPrefix(*testLTSVersion, "v") + filtered := ltsReleases[:0] + for _, lts := range ltsReleases { + if lts.version == want { + filtered = append(filtered, lts) + } + } + require.NotEmpty(t, filtered, "LTS version %q not found among active LTS releases in %s", want, prometheusDownloadManifestURL) + ltsReleases = filtered + } + + for _, lts := range ltsReleases { + t.Run(lts.version, func(t *testing.T) { + testUpgradeDowngradeLTS(t, start, lts) + }) + } +} + +func testUpgradeDowngradeLTS(t *testing.T, start time.Time, lts ltsRelease) { + rootDir := t.TempDir() + + t.Logf("[%s] using LTS %s from %s", time.Since(start), lts.version, lts.assetURL) + + rwServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + t.Cleanup(rwServer.Close) + + c := versionChangeTest{ + start: start, + + ltsAssetURL: lts.assetURL, + + ltsVersionBinPath: filepath.Join(rootDir, fmt.Sprintf("prometheus-%s", lts.version)), + + prometheusPort: testutil.RandomUnprivilegedPort(t), + prometheusDataPath: filepath.Join(rootDir, "data"), + prometheusConfigFilePath: filepath.Join(rootDir, "prometheus.yml"), + rulesFilePath: filepath.Join(rootDir, "rules.yml"), + remoteWriteURL: rwServer.URL, + } + + t.Logf("[%s] downloading and preparing LTS %s", time.Since(start), lts.version) + // TODO: Cache if downloads are expensive in CI. + c.downloadAndExtractLatestLTS(t) + t.Logf("[%s] downloaded and prepared LTS %s at %s", time.Since(start), lts.version, c.ltsVersionBinPath) + + c.generatePrometheusConfig(t) + + commonArgs := []string{ + fmt.Sprintf("--config.file=%s", c.prometheusConfigFilePath), + fmt.Sprintf("--web.listen-address=0.0.0.0:%d", c.prometheusPort), + fmt.Sprintf("--storage.tsdb.path=%s", c.prometheusDataPath), + // Accelerate compaction. + "--storage.tsdb.min-block-duration=65s", + // Accelerate chunks mmapping. + "--storage.tsdb.samples-per-chunk=10", + "--log.level=debug", + } + + runLTS := append([]string{c.ltsVersionBinPath}, commonArgs...) + runCurrent := append([]string{os.Args[0], "-test.main"}, commonArgs...) + + var ( + queryRangeSnapshotEnd time.Time + queryRangeBaseline string + ) + + t.Run("Running the LTS version", func(t *testing.T) { + cmd := commandWithLogging(t, ensureHealthyLogs, runLTS[0], runLTS[1:]...) + require.NoError(t, cmd.Start()) + c.ensureHealthyMetrics(t) + queryRangeSnapshotEnd = time.Now().Add(-3 * time.Second) + queryRangeBaseline = c.snapshotQueryRange(t, queryRangeSnapshotEnd) + }) + + t.Run("Upgrading to the current build", func(t *testing.T) { + cmd := commandWithLogging(t, ensureHealthyLogs, runCurrent[0], runCurrent[1:]...) + require.NoError(t, cmd.Start()) + c.ensureHealthyMetrics(t) + require.Equal(t, queryRangeBaseline, c.snapshotQueryRange(t, queryRangeSnapshotEnd)) + // Take a new snapshot after the upgrade. + queryRangeSnapshotEnd = time.Now().Add(-3 * time.Second) + queryRangeBaseline = c.snapshotQueryRange(t, queryRangeSnapshotEnd) + }) + + t.Run("Downgrading to the LTS version", func(t *testing.T) { + cmd := commandWithLogging(t, ensureHealthyLogs, runLTS[0], runLTS[1:]...) + require.NoError(t, cmd.Start()) + c.ensureHealthyMetrics(t) + require.Equal(t, queryRangeBaseline, c.snapshotQueryRange(t, queryRangeSnapshotEnd)) + }) +} diff --git a/cmd/prometheus/query_log_test.go b/cmd/prometheus/query_log_test.go index 62e317bf8b1..b3fd5e1bdb3 100644 --- a/cmd/prometheus/query_log_test.go +++ b/cmd/prometheus/query_log_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -68,16 +68,16 @@ func (p *queryLogTest) skip(t *testing.T) { } // waitForPrometheus waits for Prometheus to be ready. -func (p *queryLogTest) waitForPrometheus() error { - var err error - for x := 0; x < 20; x++ { - var r *http.Response - if r, err = http.Get(fmt.Sprintf("http://%s:%d%s/-/ready", p.host, p.port, p.prefix)); err == nil && r.StatusCode == http.StatusOK { - break +func (p *queryLogTest) waitForPrometheus(t *testing.T) { + t.Helper() + require.Eventually(t, func() bool { + r, err := http.Get(fmt.Sprintf("http://%s:%d%s/-/ready", p.host, p.port, p.prefix)) + if err != nil { + return false } - time.Sleep(500 * time.Millisecond) - } - return err + r.Body.Close() + return r.StatusCode == http.StatusOK + }, 20*time.Second, 500*time.Millisecond, "prometheus at %s:%d did not become ready in time", p.host, p.port) } // setQueryLog alters the configuration file to enable or disable the query log, @@ -88,20 +88,13 @@ func (p *queryLogTest) setQueryLog(t *testing.T, queryLogFile string) { _, err = p.configFile.Seek(0, 0) require.NoError(t, err) if queryLogFile != "" { - _, err = p.configFile.Write([]byte(fmt.Sprintf("global:\n query_log_file: %s\n", queryLogFile))) + _, err = fmt.Fprintf(p.configFile, "global:\n query_log_file: %s\n", queryLogFile) require.NoError(t, err) } _, err = p.configFile.Write([]byte(p.configuration())) require.NoError(t, err) } -// reloadConfig reloads the configuration using POST. -func (p *queryLogTest) reloadConfig(t *testing.T) { - r, err := http.Post(fmt.Sprintf("http://%s:%d%s/-/reload", p.host, p.port, p.prefix), "text/plain", nil) - require.NoError(t, err) - require.Equal(t, 200, r.StatusCode) -} - // query runs a query according to the test origin. func (p *queryLogTest) query(t *testing.T) { switch p.origin { @@ -125,12 +118,61 @@ func (p *queryLogTest) query(t *testing.T) { require.NoError(t, err) require.Equal(t, 200, r.StatusCode) case ruleOrigin: - time.Sleep(2 * time.Second) + // Poll the /api/v1/rules endpoint until a new rule evaluation is detected. + var lastEvalTime time.Time + for { + r, err := http.Get(fmt.Sprintf("http://%s:%d/api/v1/rules", p.host, p.port)) + require.NoError(t, err) + + rulesBody, err := io.ReadAll(r.Body) + require.NoError(t, err) + defer r.Body.Close() + + // Parse the rules response to find the last evaluation time. + newEvalTime := parseLastEvaluation(rulesBody) + if newEvalTime.After(lastEvalTime) { + if !lastEvalTime.IsZero() { + break + } + lastEvalTime = newEvalTime + } + + time.Sleep(100 * time.Millisecond) + } default: panic("can't query this origin") } } +// parseLastEvaluation extracts the last evaluation timestamp from the /api/v1/rules response. +func parseLastEvaluation(rulesBody []byte) time.Time { + var ruleResponse struct { + Status string `json:"status"` + Data struct { + Groups []struct { + Rules []struct { + LastEvaluation string `json:"lastEvaluation"` + } `json:"rules"` + } `json:"groups"` + } `json:"data"` + } + + err := json.Unmarshal(rulesBody, &ruleResponse) + if err != nil { + return time.Time{} + } + + for _, group := range ruleResponse.Data.Groups { + for _, rule := range group.Rules { + if evalTime, err := time.Parse(time.RFC3339Nano, rule.LastEvaluation); err == nil { + return evalTime + } + } + } + + return time.Time{} +} + // queryString returns the expected queryString of a this test. func (p *queryLogTest) queryString() string { switch p.origin { @@ -208,7 +250,7 @@ func (p *queryLogTest) params() []string { s = append(s, "--web.route-prefix="+p.prefix) } if p.origin == consoleOrigin { - s = append(s, "--web.console.templates="+filepath.Join("testdata", "consoles")) + s = append(s, "--web.console.templates="+filepath.Join(p.cwd, "testdata", "consoles")) } return s } @@ -259,6 +301,7 @@ func (p *queryLogTest) run(t *testing.T) { }, p.params()...) prom := exec.Command(promPath, params...) + reloadURL := fmt.Sprintf("http://%s:%d%s/-/reload", p.host, p.port, p.prefix) // Log stderr in case of failure. stderr, err := prom.StderrPipe() @@ -280,18 +323,19 @@ func (p *queryLogTest) run(t *testing.T) { prom.Process.Kill() prom.Wait() }() - require.NoError(t, p.waitForPrometheus()) + p.waitForPrometheus(t) if !p.enabledAtStart { p.query(t) require.Empty(t, readQueryLog(t, queryLogFile.Name())) p.setQueryLog(t, queryLogFile.Name()) - p.reloadConfig(t) + reloadPrometheusConfig(t, reloadURL) } p.query(t) - ql := readQueryLog(t, queryLogFile.Name()) + // Wait for query log entry to be written (avoid race with file I/O). + ql := waitForQueryLog(t, queryLogFile.Name(), 1) qc := len(ql) if p.exactQueryCount() { require.Equal(t, 1, qc) @@ -301,7 +345,7 @@ func (p *queryLogTest) run(t *testing.T) { p.validateLastQuery(t, ql) p.setQueryLog(t, "") - p.reloadConfig(t) + reloadPrometheusConfig(t, reloadURL) if !p.exactQueryCount() { qc = len(readQueryLog(t, queryLogFile.Name())) } @@ -313,16 +357,17 @@ func (p *queryLogTest) run(t *testing.T) { qc = len(ql) p.setQueryLog(t, queryLogFile.Name()) - p.reloadConfig(t) + reloadPrometheusConfig(t, reloadURL) p.query(t) qc++ - ql = readQueryLog(t, queryLogFile.Name()) + // Wait for query log entry to be written (avoid race with file I/O). + ql = waitForQueryLog(t, queryLogFile.Name(), qc) if p.exactQueryCount() { require.Len(t, ql, qc) } else { - require.Greater(t, len(ql), qc, "no queries logged") + require.GreaterOrEqual(t, len(ql), qc, "no queries logged") } p.validateLastQuery(t, ql) qc = len(ql) @@ -349,19 +394,21 @@ func (p *queryLogTest) run(t *testing.T) { qc++ - ql = readQueryLog(t, newFile.Name()) + // Wait for query log entry to be written (avoid race with file I/O). + ql = waitForQueryLog(t, newFile.Name(), qc) if p.exactQueryCount() { require.Len(t, ql, qc) } else { - require.Greater(t, len(ql), qc, "no queries logged") + require.GreaterOrEqual(t, len(ql), qc, "no queries logged") } p.validateLastQuery(t, ql) - p.reloadConfig(t) + reloadPrometheusConfig(t, reloadURL) p.query(t) - ql = readQueryLog(t, queryLogFile.Name()) + // Wait for query log entry to be written (avoid race with file I/O). + ql = waitForQueryLog(t, queryLogFile.Name(), 1) qc = len(ql) if p.exactQueryCount() { require.Equal(t, 1, qc) @@ -393,6 +440,7 @@ func readQueryLog(t *testing.T, path string) []queryLogLine { file, err := os.Open(path) require.NoError(t, err) defer file.Close() + scanner := bufio.NewScanner(file) for scanner.Scan() { var q queryLogLine @@ -402,10 +450,23 @@ func readQueryLog(t *testing.T, path string) []queryLogLine { return ql } +// waitForQueryLog waits for the query log to contain at least minEntries entries, +// polling at regular intervals until the timeout is reached. +func waitForQueryLog(t *testing.T, path string, minEntries int) []queryLogLine { + t.Helper() + var ql []queryLogLine + require.Eventually(t, func() bool { + ql = readQueryLog(t, path) + return len(ql) >= minEntries + }, 5*time.Second, 100*time.Millisecond, "timed out waiting for query log to have at least %d entries, got %d", minEntries, len(ql)) + return ql +} + func TestQueryLog(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() cwd, err := os.Getwd() require.NoError(t, err) @@ -424,6 +485,7 @@ func TestQueryLog(t *testing.T) { } t.Run(p.String(), func(t *testing.T) { + t.Parallel() p.run(t) }) } diff --git a/cmd/prometheus/reload_test.go b/cmd/prometheus/reload_test.go new file mode 100644 index 00000000000..1a75b410cf3 --- /dev/null +++ b/cmd/prometheus/reload_test.go @@ -0,0 +1,243 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/json" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/util/testutil" +) + +const configReloadMetric = "prometheus_config_last_reload_successful" + +func TestAutoReloadConfig_ValidToValid(t *testing.T) { + steps := []struct { + configText string + expectedInterval string + expectedMetric float64 + }{ + { + configText: ` +global: + scrape_interval: 30s +`, + expectedInterval: "30s", + expectedMetric: 1, + }, + { + configText: ` +global: + scrape_interval: 15s +`, + expectedInterval: "15s", + expectedMetric: 1, + }, + { + configText: ` +global: + scrape_interval: 30s +`, + expectedInterval: "30s", + expectedMetric: 1, + }, + } + + runTestSteps(t, steps) +} + +func TestAutoReloadConfig_ValidToInvalidToValid(t *testing.T) { + steps := []struct { + configText string + expectedInterval string + expectedMetric float64 + }{ + { + configText: ` +global: + scrape_interval: 30s +`, + expectedInterval: "30s", + expectedMetric: 1, + }, + { + configText: ` +global: + scrape_interval: 15s +invalid_syntax +`, + expectedInterval: "30s", + expectedMetric: 0, + }, + { + configText: ` +global: + scrape_interval: 30s +`, + expectedInterval: "30s", + expectedMetric: 1, + }, + } + + runTestSteps(t, steps) +} + +func runTestSteps(t *testing.T, steps []struct { + configText string + expectedInterval string + expectedMetric float64 +}, +) { + configDir := t.TempDir() + configFilePath := filepath.Join(configDir, "prometheus.yml") + + t.Logf("Config file path: %s", configFilePath) + + require.NoError(t, os.WriteFile(configFilePath, []byte(steps[0].configText), 0o644), "Failed to write initial config file") + + port := testutil.RandomUnprivilegedPort(t) + prom := prometheusCommandWithLogging(t, configFilePath, port, "--config.auto-reload", "--config.auto-reload-interval=1s") + require.NoError(t, prom.Start()) + + baseURL := "http://localhost:" + strconv.Itoa(port) + require.Eventually(t, func() bool { + resp, err := http.Get(baseURL + "/-/ready") + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK + }, 5*time.Second, 100*time.Millisecond, "Prometheus didn't become ready in time") + + for i, step := range steps { + t.Logf("Step %d", i) + require.NoError(t, os.WriteFile(configFilePath, []byte(step.configText), 0o644), "Failed to write config file for step") + + require.Eventually(t, func() bool { + return verifyScrapeInterval(t, baseURL, step.expectedInterval) && + verifyConfigReloadMetric(t, baseURL, step.expectedMetric) + }, 10*time.Second, 500*time.Millisecond, "Prometheus config reload didn't happen in time") + } +} + +func verifyScrapeInterval(t *testing.T, baseURL, expectedInterval string) bool { + resp, err := http.Get(baseURL + "/api/v1/status/config") + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + config := struct { + Data struct { + YAML string `json:"yaml"` + } `json:"data"` + }{} + + require.NoError(t, json.Unmarshal(body, &config)) + return strings.Contains(config.Data.YAML, "scrape_interval: "+expectedInterval) +} + +func verifyConfigReloadMetric(t *testing.T, baseURL string, expectedValue float64) bool { + resp, err := http.Get(baseURL + "/metrics") + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + lines := string(body) + var actualValue float64 + found := false + + for line := range strings.SplitSeq(lines, "\n") { + if strings.HasPrefix(line, configReloadMetric) { + parts := strings.Fields(line) + if len(parts) >= 2 { + actualValue, err = strconv.ParseFloat(parts[1], 64) + require.NoError(t, err) + found = true + break + } + } + } + + return found && actualValue == expectedValue +} + +func captureLogsToTLog(t *testing.T, r io.Reader) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + t.Log(scanner.Text()) + } + if err := scanner.Err(); err != nil { + t.Logf("Error reading logs: %v", err) + } +} + +func commandWithLogging(t *testing.T, logProcessor func(*testing.T, io.Reader), name string, args ...string) *exec.Cmd { + if logProcessor == nil { + logProcessor = captureLogsToTLog + } + + stdoutPipe, stdoutWriter := io.Pipe() + stderrPipe, stderrWriter := io.Pipe() + + var wg sync.WaitGroup + wg.Add(2) + + cmd := exec.Command(name, args...) + cmd.Stdout = stdoutWriter + cmd.Stderr = stderrWriter + + go func() { + defer wg.Done() + logProcessor(t, stdoutPipe) + }() + go func() { + defer wg.Done() + logProcessor(t, stderrPipe) + }() + + t.Cleanup(func() { + cmd.Process.Kill() + cmd.Wait() + stdoutWriter.Close() + stderrWriter.Close() + wg.Wait() + }) + return cmd +} + +func prometheusCommandWithLogging(t *testing.T, configFilePath string, port int, extraArgs ...string) *exec.Cmd { + args := []string{ + "-test.main", + "--config.file=" + configFilePath, + "--web.listen-address=0.0.0.0:" + strconv.Itoa(port), + } + args = append(args, extraArgs...) + return commandWithLogging(t, nil, promPath, args...) +} diff --git a/cmd/prometheus/scrape_failure_log_test.go b/cmd/prometheus/scrape_failure_log_test.go index 8d86d719f98..c3f459f6014 100644 --- a/cmd/prometheus/scrape_failure_log_test.go +++ b/cmd/prometheus/scrape_failure_log_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -112,7 +112,8 @@ scrape_configs: require.NoError(t, err, "Failed to update Prometheus configuration file") // Reload Prometheus with the updated configuration. - reloadPrometheus(t, port) + reloadURL := fmt.Sprintf("http://127.0.0.1:%d/-/reload", port) + reloadPrometheusConfig(t, reloadURL) // Count the number of lines in the scrape failure log file before any // further requests. @@ -146,7 +147,7 @@ scrape_configs: require.NoError(t, err, "Failed to update Prometheus configuration file") // Reload Prometheus with the updated configuration. - reloadPrometheus(t, port) + reloadPrometheusConfig(t, reloadURL) // Wait for at least two more requests to the mock server and verify that // new log entries are created. @@ -160,18 +161,10 @@ scrape_configs: require.Greater(t, countLinesInFile(scrapeFailureLogFile), postReloadLogLineCount, "New lines should be added to the scrape failure log file after re-adding the log setting") } -// reloadPrometheus sends a reload request to the Prometheus server to apply -// updated configurations. -func reloadPrometheus(t *testing.T, port int) { - resp, err := http.Post(fmt.Sprintf("http://127.0.0.1:%d/-/reload", port), "", nil) - require.NoError(t, err, "Failed to reload Prometheus") - require.Equal(t, http.StatusOK, resp.StatusCode, "Unexpected status code when reloading Prometheus") -} - // startGarbageServer sets up a mock server that returns a 500 Internal Server Error // for all requests. It also increments the request count each time it's hit. func startGarbageServer(t *testing.T, requestCount *atomic.Int32) string { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { requestCount.Inc() w.WriteHeader(http.StatusInternalServerError) })) diff --git a/cmd/prometheus/testdata/features.json b/cmd/prometheus/testdata/features.json new file mode 100644 index 00000000000..ae7490c5b7d --- /dev/null +++ b/cmd/prometheus/testdata/features.json @@ -0,0 +1,272 @@ +{ + "api": { + "admin": false, + "exclude_alerts": true, + "label_values_match": true, + "lifecycle": false, + "openapi_3.1": true, + "openapi_3.2": true, + "otlp_write_receiver": false, + "query_stats": true, + "query_warnings": true, + "remote_write_receiver": false, + "search": false, + "search_fuzz_alg_jarowinkler": true, + "search_fuzz_alg_subsequence": true, + "time_range_labels": true, + "time_range_series": true + }, + "otlp_receiver": { + "delta_conversion": false, + "native_delta_ingestion": false + }, + "prometheus": { + "agent_mode": false, + "auto_reload_config": false, + "server_mode": true, + "stringlabels": true + }, + "promql": { + "anchored": false, + "at_modifier": true, + "bool": true, + "by": true, + "delayed_name_removal": false, + "duration_expr": false, + "fill": false, + "fill_left": false, + "fill_right": false, + "group_left": true, + "group_right": true, + "ignoring": true, + "negative_offset": true, + "offset": true, + "on": true, + "per_query_lookback_delta": true, + "per_step_stats": false, + "smoothed": false, + "subqueries": true, + "type_and_unit_labels": false, + "without": true + }, + "promql_functions": { + "abs": true, + "absent": true, + "absent_over_time": true, + "acos": true, + "acosh": true, + "asin": true, + "asinh": true, + "atan": true, + "atanh": true, + "avg_over_time": true, + "ceil": true, + "changes": true, + "clamp": true, + "clamp_max": true, + "clamp_min": true, + "cos": true, + "cosh": true, + "count_over_time": true, + "day_of_month": true, + "day_of_week": true, + "day_of_year": true, + "days_in_month": true, + "deg": true, + "delta": true, + "deriv": true, + "double_exponential_smoothing": false, + "end": false, + "exp": true, + "first_over_time": false, + "floor": true, + "histogram_avg": true, + "histogram_count": true, + "histogram_fraction": true, + "histogram_quantile": true, + "histogram_quantiles": false, + "histogram_stddev": true, + "histogram_stdvar": true, + "histogram_sum": true, + "hour": true, + "idelta": true, + "increase": true, + "info": false, + "irate": true, + "label_join": true, + "label_replace": true, + "last_over_time": true, + "ln": true, + "log10": true, + "log2": true, + "mad_over_time": false, + "max_of": false, + "max_over_time": true, + "min_of": false, + "min_over_time": true, + "minute": true, + "month": true, + "pi": true, + "predict_linear": true, + "present_over_time": true, + "quantile_over_time": true, + "rad": true, + "range": false, + "rate": true, + "resets": true, + "round": true, + "scalar": true, + "sgn": true, + "sin": true, + "sinh": true, + "sort": true, + "sort_by_label": false, + "sort_by_label_desc": false, + "sort_desc": true, + "sqrt": true, + "start": false, + "stddev_over_time": true, + "stdvar_over_time": true, + "step": false, + "sum_over_time": true, + "tan": true, + "tanh": true, + "time": true, + "timestamp": true, + "ts_of_first_over_time": false, + "ts_of_last_over_time": false, + "ts_of_max_over_time": false, + "ts_of_min_over_time": false, + "vector": true, + "year": true + }, + "promql_operators": { + "!=": true, + "!~": true, + "%": true, + "*": true, + "+": true, + "-": true, + "/": true, + "<": true, + "": true, + ">/": true, + ">=": true, + "@": true, + "^": true, + "and": true, + "atan2": true, + "avg": true, + "bottomk": true, + "count": true, + "count_values": true, + "group": true, + "limit_ratio": false, + "limitk": false, + "max": true, + "min": true, + "or": true, + "quantile": true, + "stddev": true, + "stdvar": true, + "sum": true, + "topk": true, + "unless": true + }, + "rules": { + "concurrent_rule_eval": false, + "keep_firing_for": true, + "query_offset": true + }, + "scrape": { + "extra_scrape_metrics": true, + "start_timestamp_zero_ingestion": false, + "type_and_unit_labels": false + }, + "service_discovery_providers": { + "aws": true, + "azure": true, + "consul": true, + "digitalocean": true, + "dns": true, + "docker": true, + "dockerswarm": true, + "ec2": true, + "ecs": true, + "elasticache": true, + "eureka": true, + "file": true, + "gce": true, + "hetzner": true, + "http": true, + "ionos": true, + "kubernetes": true, + "kuma": true, + "lightsail": true, + "linode": true, + "marathon": true, + "msk": true, + "nerve": true, + "nomad": true, + "openstack": true, + "outscale": true, + "ovhcloud": true, + "puppetdb": true, + "rds": true, + "scaleway": true, + "serverset": true, + "stackit": true, + "static": true, + "triton": true, + "uyuni": true, + "vultr": true + }, + "templating_functions": { + "args": true, + "externalURL": true, + "first": true, + "graphLink": true, + "humanize": true, + "humanize1024": true, + "humanizeDuration": true, + "humanizePercentage": true, + "humanizeTimestamp": true, + "label": true, + "match": true, + "now": true, + "parseDuration": true, + "pathPrefix": true, + "query": true, + "reReplaceAll": true, + "safeHtml": true, + "sortByLabel": true, + "stripDomain": true, + "stripPort": true, + "strvalue": true, + "tableLink": true, + "title": true, + "toDuration": true, + "toLower": true, + "toTime": true, + "toUpper": true, + "urlQueryEscape": true, + "value": true + }, + "tsdb": { + "delayed_compaction": false, + "exemplar_storage": false, + "isolation": true, + "native_histograms": true, + "st_storage": false, + "xor2_encoding": false, + "use_uncached_io": false + }, + "ui": { + "ui_v2": false, + "ui_v3": true + } +} diff --git a/cmd/prometheus/upload_test.go b/cmd/prometheus/upload_test.go new file mode 100644 index 00000000000..97a98351a70 --- /dev/null +++ b/cmd/prometheus/upload_test.go @@ -0,0 +1,144 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "os" + "path" + "testing" + "time" + + "github.com/oklog/ulid/v2" + "github.com/prometheus/common/promslog" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/tsdb" +) + +func TestBlockExcludeFilter(t *testing.T) { + for _, test := range []struct { + summary string // Description of the test case. + uploaded []ulid.ULID // List of blocks marked as uploaded inside the shipper file. + setupFn func(string) // Optional function to run before the test, takes the path to the shipper file. + meta tsdb.BlockMeta // Meta of the block we're checking. + isExcluded bool // What do we expect to be returned. + }{ + { + summary: "missing file", + setupFn: func(path string) { + // Delete shipper file to test error handling. + os.Remove(path) + }, + meta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil)}, + isExcluded: false, + }, + { + summary: "corrupt file", + setupFn: func(path string) { + // Overwrite the shipper file content with invalid JSON. + os.WriteFile(path, []byte("{["), 0o644) + }, + meta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil)}, + isExcluded: false, + }, + { + summary: "empty uploaded list", + uploaded: []ulid.ULID{}, + meta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil)}, + isExcluded: true, + }, + { + summary: "block meta not present in the uploaded list, level=1", + uploaded: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(3, nil)}, + meta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + Compaction: tsdb.BlockMetaCompaction{Level: 1}, + }, + isExcluded: true, + }, + { + summary: "block meta not present in the uploaded list, level=2", + uploaded: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(3, nil)}, + meta: tsdb.BlockMeta{ + ULID: ulid.MustNew(2, nil), + Compaction: tsdb.BlockMetaCompaction{Level: 2}, + }, + isExcluded: false, + }, + { + summary: "block meta present in the uploaded list", + uploaded: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil), ulid.MustNew(3, nil)}, + meta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil)}, + isExcluded: false, + }, + { + summary: "don't read the file if there's valid cache", + setupFn: func(path string) { + // Remove the shipper file, cache should be used instead. + require.NoError(t, os.Remove(path)) + // Set cached values + tsdbDelayCompactLastMeta = &UploadMeta{ + Uploaded: []string{ + ulid.MustNew(1, nil).String(), + ulid.MustNew(2, nil).String(), + ulid.MustNew(3, nil).String(), + }, + } + tsdbDelayCompactLastMetaTime = time.Now().UTC().Add(time.Second * -1) + }, + uploaded: []ulid.ULID{}, + meta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil)}, + isExcluded: false, + }, + { + summary: "read the file if there's cache but expired", + setupFn: func(_ string) { + // Set the cache but make it too old + tsdbDelayCompactLastMeta = &UploadMeta{ + Uploaded: []string{}, + } + tsdbDelayCompactLastMetaTime = time.Now().UTC().Add(time.Second * -61) + }, + uploaded: []ulid.ULID{ulid.MustNew(1, nil), ulid.MustNew(2, nil), ulid.MustNew(3, nil)}, + meta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil)}, + isExcluded: false, + }, + } { + t.Run(test.summary, func(t *testing.T) { + dir := t.TempDir() + shipperPath := path.Join(dir, "shipper.json") + + uploaded := make([]string, 0, len(test.uploaded)) + for _, ul := range test.uploaded { + uploaded = append(uploaded, ul.String()) + } + ts := UploadMeta{Uploaded: uploaded} + data, err := json.Marshal(ts) + require.NoError(t, err, "failed to marshall upload meta file") + require.NoError(t, os.WriteFile(shipperPath, data, 0o644), "failed to write upload meta file") + + tsdbDelayCompactLastMeta = nil + tsdbDelayCompactLastMetaTime = time.Time{} + + if test.setupFn != nil { + test.setupFn(shipperPath) + } + + fn := exludeBlocksPendingUpload(promslog.NewNopLogger(), shipperPath) + isExcluded := fn(&test.meta) + require.Equal(t, test.isExcluded, isExcluded) + }) + } +} diff --git a/cmd/promtool/analyze.go b/cmd/promtool/analyze.go index c1f523de525..a725772f5dc 100644 --- a/cmd/promtool/analyze.go +++ b/cmd/promtool/analyze.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -34,8 +34,8 @@ import ( ) var ( - errNotNativeHistogram = fmt.Errorf("not a native histogram") - errNotEnoughData = fmt.Errorf("not enough data") + errNotNativeHistogram = errors.New("not a native histogram") + errNotEnoughData = errors.New("not enough data") outputHeader = `Bucket stats for each histogram series over time ------------------------------------------------ @@ -169,7 +169,7 @@ func querySamples(ctx context.Context, api v1.API, query string, end time.Time) matrix, ok := values.(model.Matrix) if !ok { - return nil, fmt.Errorf("query of buckets resulted in non-Matrix") + return nil, errors.New("query of buckets resulted in non-Matrix") } return matrix, nil @@ -207,7 +207,7 @@ func calcClassicBucketStatistics(matrix model.Matrix) (*statistics, error) { sortMatrix(matrix) totalPop := 0 - for timeIdx := 0; timeIdx < numSamples; timeIdx++ { + for timeIdx := range numSamples { curr, err := getBucketCountsAtTime(matrix, numBuckets, timeIdx) if err != nil { return stats, err @@ -259,7 +259,7 @@ func getBucketCountsAtTime(matrix model.Matrix, numBuckets, timeIdx int) ([]int, prev := matrix[i].Values[timeIdx] // Assume the results are nicely aligned. if curr.Timestamp != prev.Timestamp { - return counts, fmt.Errorf("matrix result is not time aligned") + return counts, errors.New("matrix result is not time aligned") } counts[i+1] = int(curr.Value - prev.Value) } diff --git a/cmd/promtool/analyze_test.go b/cmd/promtool/analyze_test.go index 83d2ac4a3db..d2e81da2c8c 100644 --- a/cmd/promtool/analyze_test.go +++ b/cmd/promtool/analyze_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,9 +17,8 @@ import ( "fmt" "testing" - "github.com/stretchr/testify/require" - "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" ) var ( @@ -109,6 +108,7 @@ func init() { } func TestGetBucketCountsAtTime(t *testing.T) { + t.Parallel() cases := []struct { matrix model.Matrix length int @@ -137,6 +137,7 @@ func TestGetBucketCountsAtTime(t *testing.T) { for _, c := range cases { t.Run(fmt.Sprintf("exampleMatrix@%d", c.timeIdx), func(t *testing.T) { + t.Parallel() res, err := getBucketCountsAtTime(c.matrix, c.length, c.timeIdx) require.NoError(t, err) require.Equal(t, c.expected, res) @@ -145,6 +146,7 @@ func TestGetBucketCountsAtTime(t *testing.T) { } func TestCalcClassicBucketStatistics(t *testing.T) { + t.Parallel() cases := []struct { matrix model.Matrix expected *statistics @@ -162,6 +164,7 @@ func TestCalcClassicBucketStatistics(t *testing.T) { for i, c := range cases { t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { + t.Parallel() res, err := calcClassicBucketStatistics(c.matrix) require.NoError(t, err) require.Equal(t, c.expected, res) diff --git a/cmd/promtool/archive.go b/cmd/promtool/archive.go index 7b565c57cc2..23baea27003 100644 --- a/cmd/promtool/archive.go +++ b/cmd/promtool/archive.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/cmd/promtool/backfill.go b/cmd/promtool/backfill.go index 16491f0416f..e7a9a7f18ab 100644 --- a/cmd/promtool/backfill.go +++ b/cmd/promtool/backfill.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,13 +21,12 @@ import ( "math" "time" - "github.com/go-kit/log" - "github.com/oklog/ulid" + "github.com/oklog/ulid/v2" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/textparse" "github.com/prometheus/prometheus/tsdb" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" ) func getMinAndMaxTimestamps(p textparse.Parser) (int64, int64, error) { @@ -48,7 +47,7 @@ func getMinAndMaxTimestamps(p textparse.Parser) (int64, int64, error) { _, ts, _ := p.Series() if ts == nil { - return 0, 0, fmt.Errorf("expected timestamp for series got none") + return 0, 0, errors.New("expected timestamp for series got none") } if *ts > maxt { @@ -94,7 +93,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn return err } defer func() { - returnErr = tsdb_errors.NewMulti(returnErr, db.Close()).Err() + returnErr = errors.Join(returnErr, db.Close()) }() var ( @@ -120,12 +119,12 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn // also need to append samples throughout the whole block range. To allow that, we // pretend that the block is twice as large here, but only really add sample in the // original interval later. - w, err := tsdb.NewBlockWriter(log.NewNopLogger(), outputDir, 2*blockDuration) + w, err := tsdb.NewBlockWriter(promslog.NewNopLogger(), outputDir, 2*blockDuration) if err != nil { return fmt.Errorf("block writer: %w", err) } defer func() { - err = tsdb_errors.NewMulti(err, w.Close()).Err() + err = errors.Join(err, w.Close()) }() ctx := context.Background() @@ -148,7 +147,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn _, ts, v := p.Series() if ts == nil { l := labels.Labels{} - p.Metric(&l) + p.Labels(&l) return fmt.Errorf("expected timestamp for series %v, got none", l) } if *ts < t { @@ -162,7 +161,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn } l := labels.Labels{} - p.Metric(&l) + p.Labels(&l) lb.Reset(l) for name, value := range customLabels { diff --git a/cmd/promtool/backfill_test.go b/cmd/promtool/backfill_test.go index b818194e865..499b90e99aa 100644 --- a/cmd/promtool/backfill_test.go +++ b/cmd/promtool/backfill_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -45,7 +45,7 @@ func sortSamples(samples []backfillSample) { }) } -func queryAllSeries(t testing.TB, q storage.Querier, expectedMinTime, expectedMaxTime int64) []backfillSample { +func queryAllSeries(t testing.TB, q storage.Querier, _, _ int64) []backfillSample { ss := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*")) samples := []backfillSample{} for ss.Next() { @@ -86,6 +86,7 @@ func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime, exp } func TestBackfill(t *testing.T) { + t.Parallel() tests := []struct { ToParse string IsOk bool @@ -729,6 +730,7 @@ after_eof 1 2 } for _, test := range tests { t.Run(test.Description, func(t *testing.T) { + t.Parallel() t.Logf("Test:%s", test.Description) outputDir := t.TempDir() diff --git a/cmd/promtool/debug.go b/cmd/promtool/debug.go index 6383aaface0..b6e82ef981f 100644 --- a/cmd/promtool/debug.go +++ b/cmd/promtool/debug.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index d1b6c0fcd98..a6dd4e0c05b 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -20,6 +20,7 @@ import ( "errors" "fmt" "io" + "log/slog" "math" "net/http" "net/url" @@ -32,20 +33,18 @@ import ( "time" "github.com/alecthomas/kingpin/v2" - "github.com/go-kit/log" "github.com/google/pprof/profile" "github.com/prometheus/client_golang/api" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil/promlint" - config_util "github.com/prometheus/common/config" - "github.com/prometheus/common/model" - "github.com/prometheus/common/version" - "github.com/prometheus/exporter-toolkit/web" - "gopkg.in/yaml.v2" - dto "github.com/prometheus/client_model/go" promconfig "github.com/prometheus/common/config" "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "github.com/prometheus/common/version" + "github.com/prometheus/exporter-toolkit/web" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" @@ -58,24 +57,44 @@ import ( _ "github.com/prometheus/prometheus/plugins" // Register plugins. "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/promql/promqltest" + "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/util/documentcli" ) +var ( + promqlEnableDelayedNameRemoval = false + promtoolParserOpts parser.Options + logger = promslog.New(&promslog.Config{}) +) + +func init() { + // This can be removed when the legacy global mode is fully deprecated. + //nolint:staticcheck + model.NameValidationScheme = model.UTF8Validation +} + const ( successExitCode = 0 failureExitCode = 1 // Exit code 3 is used for "one or more lint issues detected". lintErrExitCode = 3 - lintOptionAll = "all" - lintOptionDuplicateRules = "duplicate-rules" - lintOptionNone = "none" - checkHealth = "/-/healthy" - checkReadiness = "/-/ready" + lintOptionAll = "all" + lintOptionDuplicateRules = "duplicate-rules" + lintOptionTooLongScrapeInterval = "too-long-scrape-interval" + lintOptionNone = "none" + checkHealth = "/-/healthy" + checkReadiness = "/-/ready" ) -var lintOptions = []string{lintOptionAll, lintOptionDuplicateRules, lintOptionNone} +var ( + lintRulesOptions = []string{lintOptionAll, lintOptionDuplicateRules, lintOptionNone} + // Same as lintRulesOptions, but including scrape config linting options as well. + lintConfigOptions = append(append([]string{}, lintRulesOptions...), lintOptionTooLongScrapeInterval) +) + +const httpConfigFileDescription = "HTTP client configuration file, see details at https://prometheus.io/docs/prometheus/latest/configuration/promtool" func main() { var ( @@ -92,6 +111,10 @@ func main() { app.HelpFlag.Short('h') checkCmd := app.Command("check", "Check the resources for validity.") + checkLookbackDelta := checkCmd.Flag( + "query.lookback-delta", + "The server's maximum query lookback duration.", + ).Default("5m").Duration() experimental := app.Flag("experimental", "Enable experimental commands.").Bool() @@ -108,11 +131,12 @@ func main() { checkConfigSyntaxOnly := checkConfigCmd.Flag("syntax-only", "Only check the config file syntax, ignoring file and content validation referenced in the config").Bool() checkConfigLint := checkConfigCmd.Flag( "lint", - "Linting checks to apply to the rules specified in the config. Available options are: "+strings.Join(lintOptions, ", ")+". Use --lint=none to disable linting", + "Linting checks to apply to the rules/scrape configs specified in the config. Available options are: "+strings.Join(lintConfigOptions, ", ")+". Use --lint=none to disable linting", ).Default(lintOptionDuplicateRules).String() checkConfigLintFatal := checkConfigCmd.Flag( "lint-fatal", "Make lint errors exit with exit code 3.").Default("false").Bool() + checkConfigIgnoreUnknownFields := checkConfigCmd.Flag("ignore-unknown-fields", "Ignore unknown fields in the rule groups read by the config files. This is useful when you want to extend rule files with custom metadata. Ensure that those fields are removed before loading them into the Prometheus server as it performs strict checks by default.").Default("false").Bool() checkWebConfigCmd := checkCmd.Command("web-config", "Check if the web config files are valid or not.") webConfigFiles := checkWebConfigCmd.Arg( @@ -121,11 +145,11 @@ func main() { ).Required().ExistingFiles() checkServerHealthCmd := checkCmd.Command("healthy", "Check if the Prometheus server is healthy.") - checkServerHealthCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("").ExistingFileVar(&httpConfigFilePath) + checkServerHealthCmd.Flag("http.config.file", httpConfigFileDescription).PlaceHolder("").ExistingFileVar(&httpConfigFilePath) checkServerHealthCmd.Flag("url", "The URL for the Prometheus server.").Default("http://localhost:9090").URLVar(&serverURL) checkServerReadyCmd := checkCmd.Command("ready", "Check if the Prometheus server is ready.") - checkServerReadyCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("").ExistingFileVar(&httpConfigFilePath) + checkServerReadyCmd.Flag("http.config.file", httpConfigFileDescription).PlaceHolder("").ExistingFileVar(&httpConfigFilePath) checkServerReadyCmd.Flag("url", "The URL for the Prometheus server.").Default("http://localhost:9090").URLVar(&serverURL) checkRulesCmd := checkCmd.Command("rules", "Check if the rule files are valid or not.") @@ -135,24 +159,30 @@ func main() { ).ExistingFiles() checkRulesLint := checkRulesCmd.Flag( "lint", - "Linting checks to apply. Available options are: "+strings.Join(lintOptions, ", ")+". Use --lint=none to disable linting", + "Linting checks to apply. Available options are: "+strings.Join(lintRulesOptions, ", ")+". Use --lint=none to disable linting", ).Default(lintOptionDuplicateRules).String() checkRulesLintFatal := checkRulesCmd.Flag( "lint-fatal", "Make lint errors exit with exit code 3.").Default("false").Bool() + checkRulesIgnoreUnknownFields := checkRulesCmd.Flag("ignore-unknown-fields", "Ignore unknown fields in the rule files. This is useful when you want to extend rule files with custom metadata. Ensure that those fields are removed before loading them into the Prometheus server as it performs strict checks by default.").Default("false").Bool() checkMetricsCmd := checkCmd.Command("metrics", checkMetricsUsage) - checkMetricsExtended := checkCmd.Flag("extended", "Print extended information related to the cardinality of the metrics.").Bool() + checkMetricsExtended := checkMetricsCmd.Flag("extended", "Print extended information related to the cardinality of the metrics.").Bool() + checkMetricsLint := checkMetricsCmd.Flag( + "lint", + "Linting checks to apply for metrics. Available options are: all, none. Use --lint=none to disable metrics linting.", + ).Default(lintOptionAll).String() agentMode := checkConfigCmd.Flag("agent", "Check config file for Prometheus in Agent mode.").Bool() queryCmd := app.Command("query", "Run query against a Prometheus server.") queryCmdFmt := queryCmd.Flag("format", "Output format of the query.").Short('o').Default("promql").Enum("promql", "json") - queryCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("").ExistingFileVar(&httpConfigFilePath) + queryCmd.Flag("http.config.file", httpConfigFileDescription).PlaceHolder("").ExistingFileVar(&httpConfigFilePath) queryInstantCmd := queryCmd.Command("instant", "Run instant query.") queryInstantCmd.Arg("server", "Prometheus server to query.").Required().URLVar(&serverURL) queryInstantExpr := queryInstantCmd.Arg("expr", "PromQL query expression.").Required().String() queryInstantTime := queryInstantCmd.Flag("time", "Query evaluation time (RFC3339 or Unix timestamp).").String() + queryInstantHeaders := queryInstantCmd.Flag("header", "Extra headers to send to server.").StringMap() queryRangeCmd := queryCmd.Command("range", "Run range query.") queryRangeCmd.Arg("server", "Prometheus server to query.").Required().URLVar(&serverURL) @@ -192,7 +222,7 @@ func main() { queryAnalyzeCmd.Flag("match", "Series selector. Can be specified multiple times.").Required().StringsVar(&queryAnalyzeCfg.matchers) pushCmd := app.Command("push", "Push to a Prometheus server.") - pushCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("").ExistingFileVar(&httpConfigFilePath) + pushCmd.Flag("http.config.file", httpConfigFileDescription).PlaceHolder("").ExistingFileVar(&httpConfigFilePath) pushMetricsCmd := pushCmd.Command("metrics", "Push metrics to a prometheus remote write (for testing purpose only).") pushMetricsCmd.Arg("remote-write-url", "Prometheus remote write url to push metrics.").Required().URLVar(&remoteWriteURL) metricFiles := pushMetricsCmd.Arg( @@ -202,6 +232,7 @@ func main() { pushMetricsLabels := pushMetricsCmd.Flag("label", "Label to attach to metrics. Can be specified multiple times.").Default("job=promtool").StringMap() pushMetricsTimeout := pushMetricsCmd.Flag("timeout", "The time to wait for pushing metrics.").Default("30s").Duration() pushMetricsHeaders := pushMetricsCmd.Flag("header", "Prometheus remote write header.").StringMap() + pushMetricsProtoMsg := pushMetricsCmd.Flag("protobuf_message", "Protobuf message to use when writing (prometheus.WriteRequest or io.prometheus.write.v2.Request).").Default("prometheus.WriteRequest").String() testCmd := app.Command("test", "Unit testing.") junitOutFile := testCmd.Flag("junit", "File path to store JUnit XML test results.").OpenFile(os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) @@ -211,7 +242,9 @@ func main() { "test-rule-file", "The unit test file.", ).Required().ExistingFiles() + testRulesDebug := testRulesCmd.Flag("debug", "Enable unit test debugging.").Default("false").Bool() testRulesDiff := testRulesCmd.Flag("diff", "[Experimental] Print colored differential output between expected & received output.").Default("false").Bool() + testRulesIgnoreUnknownFields := testRulesCmd.Flag("ignore-unknown-fields", "Ignore unknown fields in the test files. This is useful when you want to extend rule files with custom metadata. Ensure that those fields are removed before loading them into the Prometheus server as it performs strict checks by default.").Default("false").Bool() defaultDBPath := "data/" tsdbCmd := app.Command("tsdb", "Run tsdb commands.") @@ -234,18 +267,19 @@ func main() { listHumanReadable := tsdbListCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool() listPath := tsdbListCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String() - tsdbDumpCmd := tsdbCmd.Command("dump", "Dump samples from a TSDB.") + tsdbDumpCmd := tsdbCmd.Command("dump", "Dump data (series+samples or optionally just series) from a TSDB.") dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String() dumpSandboxDirRoot := tsdbDumpCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory will be created, this sandbox is used in case WAL replay generates chunks (default is the database path). The sandbox is cleaned up at the end.").String() - dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64() - dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64() + dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump, in milliseconds since the Unix epoch.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64() + dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump, in milliseconds since the Unix epoch.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64() dumpMatch := tsdbDumpCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings() + dumpFormat := tsdbDumpCmd.Flag("format", "Output format of the dump (prom (default) or seriesjson).").Default("prom").Enum("prom", "seriesjson") tsdbDumpOpenMetricsCmd := tsdbCmd.Command("dump-openmetrics", "[Experimental] Dump samples from a TSDB into OpenMetrics text format, excluding native histograms and staleness markers, which are not representable in OpenMetrics.") dumpOpenMetricsPath := tsdbDumpOpenMetricsCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String() dumpOpenMetricsSandboxDirRoot := tsdbDumpOpenMetricsCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory will be created, this sandbox is used in case WAL replay generates chunks (default is the database path). The sandbox is cleaned up at the end.").String() - dumpOpenMetricsMinTime := tsdbDumpOpenMetricsCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64() - dumpOpenMetricsMaxTime := tsdbDumpOpenMetricsCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64() + dumpOpenMetricsMinTime := tsdbDumpOpenMetricsCmd.Flag("min-time", "Minimum timestamp to dump, in milliseconds since the Unix epoch.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64() + dumpOpenMetricsMaxTime := tsdbDumpOpenMetricsCmd.Flag("max-time", "Maximum timestamp to dump, in milliseconds since the Unix epoch.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64() dumpOpenMetricsMatch := tsdbDumpOpenMetricsCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings() importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.") @@ -257,7 +291,7 @@ func main() { importFilePath := openMetricsImportCmd.Arg("input file", "OpenMetrics file to read samples from.").Required().String() importDBPath := openMetricsImportCmd.Arg("output directory", "Output directory for generated blocks.").Default(defaultDBPath).String() importRulesCmd := importCmd.Command("rules", "Create blocks of data for new recording rules.") - importRulesCmd.Flag("http.config.file", "HTTP client configuration file for promtool to connect to Prometheus.").PlaceHolder("").ExistingFileVar(&httpConfigFilePath) + importRulesCmd.Flag("http.config.file", httpConfigFileDescription).PlaceHolder("").ExistingFileVar(&httpConfigFilePath) importRulesCmd.Flag("url", "The URL for the Prometheus API with the data where the rule will be backfilled from.").Default("http://localhost:9090").URLVar(&serverURL) importRulesStart := importRulesCmd.Flag("start", "The time to start backfilling the new rule from. Must be a RFC3339 formatted date or Unix timestamp. Required."). Required().String() @@ -286,7 +320,7 @@ func main() { promQLLabelsDeleteQuery := promQLLabelsDeleteCmd.Arg("query", "PromQL query.").Required().String() promQLLabelsDeleteName := promQLLabelsDeleteCmd.Arg("name", "Name of the label to delete.").Required().String() - featureList := app.Flag("enable-feature", "Comma separated feature names to enable (only PromQL related and no-default-scrape-port). See https://prometheus.io/docs/prometheus/latest/feature_flags/ for the options and more details.").Default("").Strings() + featureList := app.Flag("enable-feature", "Comma separated feature names to enable. Valid options: promql-experimental-functions, promql-delayed-name-removal, promql-duration-expr, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details").Default("").Strings() documentationCmd := app.Command("write-documentation", "Generate command line documentation. Internal use.").Hidden() @@ -305,40 +339,43 @@ func main() { kingpin.Fatalf("Cannot set base auth in the server URL and use a http.config.file at the same time") } var err error - httpConfig, _, err := config_util.LoadHTTPConfigFile(httpConfigFilePath) + httpConfig, _, err := promconfig.LoadHTTPConfigFile(httpConfigFilePath) if err != nil { kingpin.Fatalf("Failed to load HTTP config file: %v", err) } - httpRoundTripper, err = promconfig.NewRoundTripperFromConfig(*httpConfig, "promtool", config_util.WithUserAgent("promtool/"+version.Version)) + httpRoundTripper, err = promconfig.NewRoundTripperFromConfig(*httpConfig, "promtool", promconfig.WithUserAgent(version.ComponentUserAgent("promtool"))) if err != nil { kingpin.Fatalf("Failed to create a new HTTP round tripper: %v", err) } } - var noDefaultScrapePort bool for _, f := range *featureList { - opts := strings.Split(f, ",") - for _, o := range opts { + for o := range strings.SplitSeq(f, ",") { switch o { - case "no-default-scrape-port": - noDefaultScrapePort = true + case "promql-experimental-functions": + promtoolParserOpts.EnableExperimentalFunctions = true + case "promql-delayed-name-removal": + promqlEnableDelayedNameRemoval = true + case "promql-duration-expr": + promtoolParserOpts.ExperimentalDurationExpr = true + case "promql-extended-range-selectors": + promtoolParserOpts.EnableExtendedRangeSelectors = true case "": continue - case "promql-at-modifier", "promql-negative-offset": - fmt.Printf(" WARNING: Option for --enable-feature is a no-op after promotion to a stable feature: %q\n", o) default: - fmt.Printf(" WARNING: Unknown option for --enable-feature: %q\n", o) + fmt.Printf(" WARNING: Unknown feature passed to --enable-feature: %s", o) } } } + promtoolParser := parser.NewParser(promtoolParserOpts) switch parsedCmd { case sdCheckCmd.FullCommand(): - os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort, prometheus.DefaultRegisterer)) + os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, prometheus.DefaultRegisterer)) case checkConfigCmd.FullCommand(): - os.Exit(CheckConfig(*agentMode, *checkConfigSyntaxOnly, newLintConfig(*checkConfigLint, *checkConfigLintFatal), *configFiles...)) + os.Exit(CheckConfig(*agentMode, *checkConfigSyntaxOnly, newConfigLintConfig(*checkConfigLint, *checkConfigLintFatal, *checkConfigIgnoreUnknownFields, model.UTF8Validation, model.Duration(*checkLookbackDelta)), promtoolParser, *configFiles...)) case checkServerHealthCmd.FullCommand(): os.Exit(checkErr(CheckServerStatus(serverURL, checkHealth, httpRoundTripper))) @@ -350,16 +387,16 @@ func main() { os.Exit(CheckWebConfig(*webConfigFiles...)) case checkRulesCmd.FullCommand(): - os.Exit(CheckRules(newLintConfig(*checkRulesLint, *checkRulesLintFatal), *ruleFiles...)) + os.Exit(CheckRules(newRulesLintConfig(*checkRulesLint, *checkRulesLintFatal, *checkRulesIgnoreUnknownFields, model.UTF8Validation), promtoolParser, *ruleFiles...)) case checkMetricsCmd.FullCommand(): - os.Exit(CheckMetrics(*checkMetricsExtended)) + os.Exit(CheckMetrics(*checkMetricsExtended, *checkMetricsLint)) case pushMetricsCmd.FullCommand(): - os.Exit(PushMetrics(remoteWriteURL, httpRoundTripper, *pushMetricsHeaders, *pushMetricsTimeout, *pushMetricsLabels, *metricFiles...)) + os.Exit(PushMetrics(remoteWriteURL, httpRoundTripper, *pushMetricsHeaders, *pushMetricsTimeout, *pushMetricsProtoMsg, *pushMetricsLabels, *metricFiles...)) case queryInstantCmd.FullCommand(): - os.Exit(QueryInstant(serverURL, httpRoundTripper, *queryInstantExpr, *queryInstantTime, p)) + os.Exit(QueryInstant(serverURL, httpRoundTripper, *queryInstantHeaders, *queryInstantExpr, *queryInstantTime, p)) case queryRangeCmd.FullCommand(): os.Exit(QueryRange(serverURL, httpRoundTripper, *queryRangeHeaders, *queryRangeExpr, *queryRangeBegin, *queryRangeEnd, *queryRangeStep, p)) @@ -386,11 +423,15 @@ func main() { } os.Exit(RulesUnitTestResult(results, promqltest.LazyLoaderOpts{ - EnableAtModifier: true, - EnableNegativeOffset: true, + EnableAtModifier: true, + EnableNegativeOffset: true, + EnableDelayedNameRemoval: promqlEnableDelayedNameRemoval, }, + promtoolParser, *testRulesRun, *testRulesDiff, + *testRulesDebug, + *testRulesIgnoreUnknownFields, *testRulesFiles...), ) @@ -398,21 +439,26 @@ func main() { os.Exit(checkErr(benchmarkWrite(*benchWriteOutPath, *benchSamplesFile, *benchWriteNumMetrics, *benchWriteNumScrapes))) case tsdbAnalyzeCmd.FullCommand(): - os.Exit(checkErr(analyzeBlock(ctx, *analyzePath, *analyzeBlockID, *analyzeLimit, *analyzeRunExtended, *analyzeMatchers))) + os.Exit(checkErr(analyzeBlock(ctx, *analyzePath, *analyzeBlockID, *analyzeLimit, *analyzeRunExtended, *analyzeMatchers, promtoolParser))) case tsdbListCmd.FullCommand(): os.Exit(checkErr(listBlocks(*listPath, *listHumanReadable))) case tsdbDumpCmd.FullCommand(): - os.Exit(checkErr(dumpSamples(ctx, *dumpPath, *dumpSandboxDirRoot, *dumpMinTime, *dumpMaxTime, *dumpMatch, formatSeriesSet))) + format := formatSeriesSet + if *dumpFormat == "seriesjson" { + format = formatSeriesSetLabelsToJSON + } + os.Exit(checkErr(dumpTSDBData(ctx, *dumpPath, *dumpSandboxDirRoot, *dumpMinTime, *dumpMaxTime, *dumpMatch, format, promtoolParser))) + case tsdbDumpOpenMetricsCmd.FullCommand(): - os.Exit(checkErr(dumpSamples(ctx, *dumpOpenMetricsPath, *dumpOpenMetricsSandboxDirRoot, *dumpOpenMetricsMinTime, *dumpOpenMetricsMaxTime, *dumpOpenMetricsMatch, formatSeriesSetOpenMetrics))) + os.Exit(checkErr(dumpTSDBData(ctx, *dumpOpenMetricsPath, *dumpOpenMetricsSandboxDirRoot, *dumpOpenMetricsMinTime, *dumpOpenMetricsMaxTime, *dumpOpenMetricsMatch, formatSeriesSetOpenMetrics, promtoolParser))) // TODO(aSquare14): Work on adding support for custom block size. case openMetricsImportCmd.FullCommand(): os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet, *maxBlockDuration, *openMetricsLabels)) case importRulesCmd.FullCommand(): - os.Exit(checkErr(importRules(serverURL, httpRoundTripper, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *maxBlockDuration, *importRulesFiles...))) + os.Exit(checkErr(importRules(serverURL, httpRoundTripper, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *maxBlockDuration, model.UTF8Validation, *importRulesFiles...))) case queryAnalyzeCmd.FullCommand(): os.Exit(checkErr(queryAnalyzeCfg.run(serverURL, httpRoundTripper))) @@ -422,15 +468,15 @@ func main() { case promQLFormatCmd.FullCommand(): checkExperimental(*experimental) - os.Exit(checkErr(formatPromQL(*promQLFormatQuery))) + os.Exit(checkErr(formatPromQL(*promQLFormatQuery, promtoolParser))) case promQLLabelsSetCmd.FullCommand(): checkExperimental(*experimental) - os.Exit(checkErr(labelsSetPromQL(*promQLLabelsSetQuery, *promQLLabelsSetType, *promQLLabelsSetName, *promQLLabelsSetValue))) + os.Exit(checkErr(labelsSetPromQL(*promQLLabelsSetQuery, *promQLLabelsSetType, *promQLLabelsSetName, *promQLLabelsSetValue, promtoolParser))) case promQLLabelsDeleteCmd.FullCommand(): checkExperimental(*experimental) - os.Exit(checkErr(labelsDeletePromQL(*promQLLabelsDeleteQuery, *promQLLabelsDeleteName))) + os.Exit(checkErr(labelsDeletePromQL(*promQLLabelsDeleteQuery, *promQLLabelsDeleteName, promtoolParser))) } } @@ -441,20 +487,26 @@ func checkExperimental(f bool) { } } -var errLint = fmt.Errorf("lint error") +var errLint = errors.New("lint error") -type lintConfig struct { - all bool - duplicateRules bool - fatal bool +type rulesLintConfig struct { + all bool + duplicateRules bool + fatal bool + ignoreUnknownFields bool + nameValidationScheme model.ValidationScheme } -func newLintConfig(stringVal string, fatal bool) lintConfig { - items := strings.Split(stringVal, ",") - ls := lintConfig{ - fatal: fatal, +func newRulesLintConfig(stringVal string, fatal, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme) rulesLintConfig { + ls := rulesLintConfig{ + fatal: fatal, + ignoreUnknownFields: ignoreUnknownFields, + nameValidationScheme: nameValidationScheme, } - for _, setting := range items { + if stringVal == "" { + return ls + } + for setting := range strings.SplitSeq(stringVal, ",") { switch setting { case lintOptionAll: ls.all = true @@ -462,24 +514,66 @@ func newLintConfig(stringVal string, fatal bool) lintConfig { ls.duplicateRules = true case lintOptionNone: default: - fmt.Printf("WARNING: unknown lint option %s\n", setting) + fmt.Printf("WARNING: unknown lint option: %q\n", setting) } } return ls } -func (ls lintConfig) lintDuplicateRules() bool { +func (ls rulesLintConfig) lintDuplicateRules() bool { return ls.all || ls.duplicateRules } +type configLintConfig struct { + rulesLintConfig + + lookbackDelta model.Duration +} + +func newConfigLintConfig(optionsStr string, fatal, ignoreUnknownFields bool, nameValidationScheme model.ValidationScheme, lookbackDelta model.Duration) configLintConfig { + c := configLintConfig{ + rulesLintConfig: rulesLintConfig{ + fatal: fatal, + }, + } + + lintNone := false + var rulesOptions []string + for option := range strings.SplitSeq(optionsStr, ",") { + switch option { + case lintOptionAll, lintOptionTooLongScrapeInterval: + c.lookbackDelta = lookbackDelta + if option == lintOptionAll { + rulesOptions = append(rulesOptions, lintOptionAll) + } + case lintOptionNone: + lintNone = true + default: + rulesOptions = append(rulesOptions, option) + } + } + + if lintNone { + c.lookbackDelta = 0 + rulesOptions = nil + } + + c.rulesLintConfig = newRulesLintConfig(strings.Join(rulesOptions, ","), fatal, ignoreUnknownFields, nameValidationScheme) + + return c +} + // CheckServerStatus - healthy & ready. func CheckServerStatus(serverURL *url.URL, checkEndpoint string, roundTripper http.RoundTripper) error { if serverURL.Scheme == "" { serverURL.Scheme = "http" } + // Join the endpoint onto the server URL path so that a trailing slash in + // the user-provided URL does not result in a doubled slash (e.g. + // "//-/healthy"), which the Prometheus router does not match. config := api.Config{ - Address: serverURL.String() + checkEndpoint, + Address: serverURL.JoinPath(checkEndpoint).String(), RoundTripper: roundTripper, } @@ -490,7 +584,7 @@ func CheckServerStatus(serverURL *url.URL, checkEndpoint string, roundTripper ht return err } - request, err := http.NewRequest(http.MethodGet, config.Address, nil) + request, err := http.NewRequest(http.MethodGet, config.Address, http.NoBody) if err != nil { return err } @@ -510,12 +604,12 @@ func CheckServerStatus(serverURL *url.URL, checkEndpoint string, roundTripper ht } // CheckConfig validates configuration files. -func CheckConfig(agentMode, checkSyntaxOnly bool, lintSettings lintConfig, files ...string) int { +func CheckConfig(agentMode, checkSyntaxOnly bool, lintSettings configLintConfig, p parser.Parser, files ...string) int { failed := false hasErrors := false for _, f := range files { - ruleFiles, err := checkConfig(agentMode, f, checkSyntaxOnly) + ruleFiles, scrapeConfigs, err := checkConfig(agentMode, f, checkSyntaxOnly) if err != nil { fmt.Fprintln(os.Stderr, " FAILED:", err) hasErrors = true @@ -528,12 +622,12 @@ func CheckConfig(agentMode, checkSyntaxOnly bool, lintSettings lintConfig, files } fmt.Println() - rulesFailed, rulesHasErrors := checkRules(ruleFiles, lintSettings) - if rulesFailed { - failed = rulesFailed - } - if rulesHasErrors { - hasErrors = rulesHasErrors + if !checkSyntaxOnly { + scrapeConfigsFailed := lintScrapeConfigs(scrapeConfigs, lintSettings) + failed = failed || scrapeConfigsFailed + rulesFailed, rulesHaveErrors := checkRules(ruleFiles, lintSettings.rulesLintConfig, p, logger) + failed = failed || rulesFailed + hasErrors = hasErrors || rulesHaveErrors } } if failed && hasErrors { @@ -572,12 +666,12 @@ func checkFileExists(fn string) error { return err } -func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]string, error) { +func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]string, []*config.ScrapeConfig, error) { fmt.Println("Checking", filename) - cfg, err := config.LoadFile(filename, agentMode, false, log.NewNopLogger()) + cfg, err := config.LoadFile(filename, agentMode, logger) if err != nil { - return nil, err + return nil, nil, err } var ruleFiles []string @@ -585,15 +679,15 @@ func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]strin for _, rf := range cfg.RuleFiles { rfs, err := filepath.Glob(rf) if err != nil { - return nil, err + return nil, nil, err } // If an explicit file was given, error if it is not accessible. if !strings.Contains(rf, "*") { if len(rfs) == 0 { - return nil, fmt.Errorf("%q does not point to an existing file", rf) + return nil, nil, fmt.Errorf("%q does not point to an existing file", rf) } if err := checkFileExists(rfs[0]); err != nil { - return nil, fmt.Errorf("error checking rule file %q: %w", rfs[0], err) + return nil, nil, fmt.Errorf("error checking rule file %q: %w", rfs[0], err) } } ruleFiles = append(ruleFiles, rfs...) @@ -607,26 +701,26 @@ func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]strin var err error scfgs, err = cfg.GetScrapeConfigs() if err != nil { - return nil, fmt.Errorf("error loading scrape configs: %w", err) + return nil, nil, fmt.Errorf("error loading scrape configs: %w", err) } } for _, scfg := range scfgs { if !checkSyntaxOnly && scfg.HTTPClientConfig.Authorization != nil { if err := checkFileExists(scfg.HTTPClientConfig.Authorization.CredentialsFile); err != nil { - return nil, fmt.Errorf("error checking authorization credentials or bearer token file %q: %w", scfg.HTTPClientConfig.Authorization.CredentialsFile, err) + return nil, nil, fmt.Errorf("error checking authorization credentials or bearer token file %q: %w", scfg.HTTPClientConfig.Authorization.CredentialsFile, err) } } if err := checkTLSConfig(scfg.HTTPClientConfig.TLSConfig, checkSyntaxOnly); err != nil { - return nil, err + return nil, nil, err } for _, c := range scfg.ServiceDiscoveryConfigs { switch c := c.(type) { case *kubernetes.SDConfig: if err := checkTLSConfig(c.HTTPClientConfig.TLSConfig, checkSyntaxOnly); err != nil { - return nil, err + return nil, nil, err } case *file.SDConfig: if checkSyntaxOnly { @@ -635,17 +729,17 @@ func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]strin for _, file := range c.Files { files, err := filepath.Glob(file) if err != nil { - return nil, err + return nil, nil, err } if len(files) != 0 { for _, f := range files { var targetGroups []*targetgroup.Group targetGroups, err = checkSDFile(f) if err != nil { - return nil, fmt.Errorf("checking SD file %q: %w", file, err) + return nil, nil, fmt.Errorf("checking SD file %q: %w", file, err) } if err := checkTargetGroupsForScrapeConfig(targetGroups, scfg); err != nil { - return nil, err + return nil, nil, err } } continue @@ -654,7 +748,7 @@ func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]strin } case discovery.StaticConfig: if err := checkTargetGroupsForScrapeConfig(c, scfg); err != nil { - return nil, err + return nil, nil, err } } } @@ -671,18 +765,18 @@ func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]strin for _, file := range c.Files { files, err := filepath.Glob(file) if err != nil { - return nil, err + return nil, nil, err } if len(files) != 0 { for _, f := range files { var targetGroups []*targetgroup.Group targetGroups, err = checkSDFile(f) if err != nil { - return nil, fmt.Errorf("checking SD file %q: %w", file, err) + return nil, nil, fmt.Errorf("checking SD file %q: %w", file, err) } if err := checkTargetGroupsForAlertmanager(targetGroups, amcfg); err != nil { - return nil, err + return nil, nil, err } } continue @@ -691,19 +785,19 @@ func checkConfig(agentMode bool, filename string, checkSyntaxOnly bool) ([]strin } case discovery.StaticConfig: if err := checkTargetGroupsForAlertmanager(c, amcfg); err != nil { - return nil, err + return nil, nil, err } } } } - return ruleFiles, nil + return ruleFiles, scfgs, nil } -func checkTLSConfig(tlsConfig config_util.TLSConfig, checkSyntaxOnly bool) error { - if len(tlsConfig.CertFile) > 0 && len(tlsConfig.KeyFile) == 0 { +func checkTLSConfig(tlsConfig promconfig.TLSConfig, checkSyntaxOnly bool) error { + if tlsConfig.CertFile != "" && tlsConfig.KeyFile == "" { return fmt.Errorf("client cert file %q specified without client key file", tlsConfig.CertFile) } - if len(tlsConfig.KeyFile) > 0 && len(tlsConfig.CertFile) == 0 { + if tlsConfig.KeyFile != "" && tlsConfig.CertFile == "" { return fmt.Errorf("client key file %q specified without client cert file", tlsConfig.KeyFile) } @@ -758,13 +852,13 @@ func checkSDFile(filename string) ([]*targetgroup.Group, error) { } // CheckRules validates rule files. -func CheckRules(ls lintConfig, files ...string) int { +func CheckRules(ls rulesLintConfig, p parser.Parser, files ...string) int { failed := false hasErrors := false if len(files) == 0 { - failed, hasErrors = checkRulesFromStdin(ls) + failed, hasErrors = checkRulesFromStdin(ls, p, logger) } else { - failed, hasErrors = checkRules(files, ls) + failed, hasErrors = checkRules(files, ls, p, logger) } if failed && hasErrors { @@ -778,7 +872,7 @@ func CheckRules(ls lintConfig, files ...string) int { } // checkRulesFromStdin validates rule from stdin. -func checkRulesFromStdin(ls lintConfig) (bool, bool) { +func checkRulesFromStdin(ls rulesLintConfig, p parser.Parser, logger *slog.Logger) (bool, bool) { failed := false hasErrors := false fmt.Println("Checking standard input") @@ -787,7 +881,7 @@ func checkRulesFromStdin(ls lintConfig) (bool, bool) { fmt.Fprintln(os.Stderr, " FAILED:", err) return true, true } - rgs, errs := rulefmt.Parse(data) + rgs, errs := rulefmt.Parse(data, ls.ignoreUnknownFields, ls.nameValidationScheme, p, logger) if errs != nil { failed = true fmt.Fprintln(os.Stderr, " FAILED:") @@ -816,12 +910,12 @@ func checkRulesFromStdin(ls lintConfig) (bool, bool) { } // checkRules validates rule files. -func checkRules(files []string, ls lintConfig) (bool, bool) { +func checkRules(files []string, ls rulesLintConfig, p parser.Parser, logger *slog.Logger) (bool, bool) { failed := false hasErrors := false for _, f := range files { fmt.Println("Checking", f) - rgs, errs := rulefmt.ParseFile(f) + rgs, errs := rulefmt.ParseFile(f, ls.ignoreUnknownFields, ls.nameValidationScheme, p, logger) if errs != nil { failed = true fmt.Fprintln(os.Stderr, " FAILED:") @@ -850,7 +944,7 @@ func checkRules(files []string, ls lintConfig) (bool, bool) { return failed, hasErrors } -func checkRuleGroups(rgs *rulefmt.RuleGroups, lintSettings lintConfig) (int, []error) { +func checkRuleGroups(rgs *rulefmt.RuleGroups, lintSettings rulesLintConfig) (int, []error) { numRules := 0 for _, rg := range rgs.Groups { numRules += len(rg.Rules) @@ -859,21 +953,32 @@ func checkRuleGroups(rgs *rulefmt.RuleGroups, lintSettings lintConfig) (int, []e if lintSettings.lintDuplicateRules() { dRules := checkDuplicates(rgs.Groups) if len(dRules) != 0 { - errMessage := fmt.Sprintf("%d duplicate rule(s) found.\n", len(dRules)) + var errMessage strings.Builder + fmt.Fprintf(&errMessage, "%d duplicate rule(s) found.\n", len(dRules)) for _, n := range dRules { - errMessage += fmt.Sprintf("Metric: %s\nLabel(s):\n", n.metric) + fmt.Fprintf(&errMessage, "Metric: %s\nLabel(s):\n", n.metric) n.label.Range(func(l labels.Label) { - errMessage += fmt.Sprintf("\t%s: %s\n", l.Name, l.Value) + fmt.Fprintf(&errMessage, "\t%s: %s\n", l.Name, l.Value) }) } - errMessage += "Might cause inconsistency while recording expressions" - return 0, []error{fmt.Errorf("%w %s", errLint, errMessage)} + errMessage.WriteString("Might cause inconsistency while recording expressions") + return 0, []error{fmt.Errorf("%w %s", errLint, errMessage.String())} } } return numRules, nil } +func lintScrapeConfigs(scrapeConfigs []*config.ScrapeConfig, lintSettings configLintConfig) bool { + for _, scfg := range scrapeConfigs { + if lintSettings.lookbackDelta > 0 && scfg.ScrapeInterval >= lintSettings.lookbackDelta { + fmt.Fprintf(os.Stderr, " FAILED: too long scrape interval found, data point will be marked as stale - job: %s, interval: %s\n", scfg.JobName, scfg.ScrapeInterval) + return true + } + } + return false +} + type compareRuleType struct { metric string label labels.Labels @@ -895,73 +1000,90 @@ func compare(a, b compareRuleType) int { func checkDuplicates(groups []rulefmt.RuleGroup) []compareRuleType { var duplicates []compareRuleType - var rules compareRuleTypes + var cRules compareRuleTypes for _, group := range groups { for _, rule := range group.Rules { - rules = append(rules, compareRuleType{ + cRules = append(cRules, compareRuleType{ metric: ruleMetric(rule), - label: labels.FromMap(rule.Labels), + label: rules.FromMaps(group.Labels, rule.Labels), }) } } - if len(rules) < 2 { + if len(cRules) < 2 { return duplicates } - sort.Sort(rules) + sort.Sort(cRules) - last := rules[0] - for i := 1; i < len(rules); i++ { - if compare(last, rules[i]) == 0 { + last := cRules[0] + for i := 1; i < len(cRules); i++ { + if compare(last, cRules[i]) == 0 { // Don't add a duplicated rule multiple times. if len(duplicates) == 0 || compare(last, duplicates[len(duplicates)-1]) != 0 { - duplicates = append(duplicates, rules[i]) + duplicates = append(duplicates, cRules[i]) } } - last = rules[i] + last = cRules[i] } return duplicates } -func ruleMetric(rule rulefmt.RuleNode) string { - if rule.Alert.Value != "" { - return rule.Alert.Value +func ruleMetric(rule rulefmt.Rule) string { + if rule.Alert != "" { + return rule.Alert } - return rule.Record.Value + return rule.Record } var checkMetricsUsage = strings.TrimSpace(` -Pass Prometheus metrics over stdin to lint them for consistency and correctness. +Pass Prometheus metrics over stdin to lint them for consistency and correctness, and optionally perform cardinality analysis. examples: $ cat metrics.prom | promtool check metrics -$ curl -s http://localhost:9090/metrics | promtool check metrics +$ curl -s http://localhost:9090/metrics | promtool check metrics --extended + +$ curl -s http://localhost:9100/metrics | promtool check metrics --extended --lint=none `) // CheckMetrics performs a linting pass on input metrics. -func CheckMetrics(extended bool) int { - var buf bytes.Buffer - tee := io.TeeReader(os.Stdin, &buf) - l := promlint.New(tee) - problems, err := l.Lint() - if err != nil { - fmt.Fprintln(os.Stderr, "error while linting:", err) +func CheckMetrics(extended bool, lint string) int { + // Validate that at least one feature is enabled. + if !extended && lint == lintOptionNone { + fmt.Fprintln(os.Stderr, "error: at least one of --extended or linting must be enabled") + fmt.Fprintln(os.Stderr, "Use --extended for cardinality analysis, or remove --lint=none to enable linting") return failureExitCode } - for _, p := range problems { - fmt.Fprintln(os.Stderr, p.Metric, p.Text) - } + var buf bytes.Buffer + var ( + problems []promlint.Problem + reader io.Reader + err error + ) - if len(problems) > 0 { - return lintErrExitCode + if lint != lintOptionNone { + tee := io.TeeReader(os.Stdin, &buf) + l := promlint.New(tee) + problems, err = l.Lint() + if err != nil { + fmt.Fprintln(os.Stderr, "error while linting:", err) + return failureExitCode + } + for _, p := range problems { + fmt.Fprintln(os.Stderr, p.Metric, p.Text) + } + reader = &buf + } else { + reader = os.Stdin } + hasLintProblems := len(problems) > 0 + if extended { - stats, total, err := checkMetricsExtended(&buf) + stats, total, err := checkMetricsExtended(reader) if err != nil { fmt.Fprintln(os.Stderr, err) return failureExitCode @@ -975,6 +1097,10 @@ func CheckMetrics(extended bool) int { w.Flush() } + if hasLintProblems { + return lintErrExitCode + } + return successExitCode } @@ -985,7 +1111,10 @@ type metricStat struct { } func checkMetricsExtended(r io.Reader) ([]metricStat, int, error) { - p := expfmt.TextParser{} + // Lacking information about what the intended validation scheme is, use the + // deprecated library bool. + //nolint:staticcheck + p := expfmt.NewTextParser(model.NameValidationScheme) metricFamilies, err := p.TextToMetricFamilies(r) if err != nil { return nil, 0, fmt.Errorf("error while parsing text to metric families: %w", err) @@ -1113,17 +1242,17 @@ type printer interface { type promqlPrinter struct{} -func (p *promqlPrinter) printValue(v model.Value) { +func (*promqlPrinter) printValue(v model.Value) { fmt.Println(v) } -func (p *promqlPrinter) printSeries(val []model.LabelSet) { +func (*promqlPrinter) printSeries(val []model.LabelSet) { for _, v := range val { fmt.Println(v) } } -func (p *promqlPrinter) printLabelValues(val model.LabelValues) { +func (*promqlPrinter) printLabelValues(val model.LabelValues) { for _, v := range val { fmt.Println(v) } @@ -1131,24 +1260,24 @@ func (p *promqlPrinter) printLabelValues(val model.LabelValues) { type jsonPrinter struct{} -func (j *jsonPrinter) printValue(v model.Value) { +func (*jsonPrinter) printValue(v model.Value) { //nolint:errcheck json.NewEncoder(os.Stdout).Encode(v) } -func (j *jsonPrinter) printSeries(v []model.LabelSet) { +func (*jsonPrinter) printSeries(v []model.LabelSet) { //nolint:errcheck json.NewEncoder(os.Stdout).Encode(v) } -func (j *jsonPrinter) printLabelValues(v model.LabelValues) { +func (*jsonPrinter) printLabelValues(v model.LabelValues) { //nolint:errcheck json.NewEncoder(os.Stdout).Encode(v) } // importRules backfills recording rules from the files provided. The output are blocks of data // at the outputDir location. -func importRules(url *url.URL, roundTripper http.RoundTripper, start, end, outputDir string, evalInterval, maxBlockDuration time.Duration, files ...string) error { +func importRules(url *url.URL, roundTripper http.RoundTripper, start, end, outputDir string, evalInterval, maxBlockDuration time.Duration, nameValidationScheme model.ValidationScheme, files ...string) error { ctx := context.Background() var stime, etime time.Time var err error @@ -1171,18 +1300,19 @@ func importRules(url *url.URL, roundTripper http.RoundTripper, start, end, outpu } cfg := ruleImporterConfig{ - outputDir: outputDir, - start: stime, - end: etime, - evalInterval: evalInterval, - maxBlockDuration: maxBlockDuration, + outputDir: outputDir, + start: stime, + end: etime, + evalInterval: evalInterval, + maxBlockDuration: maxBlockDuration, + nameValidationScheme: nameValidationScheme, } api, err := newAPI(url, roundTripper, nil) if err != nil { return fmt.Errorf("new api client error: %w", err) } - ruleImporter := newRuleImporter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), cfg, api) + ruleImporter := newRuleImporter(logger, cfg, api) errs := ruleImporter.loadGroups(ctx, files) for _, err := range errs { if err != nil { @@ -1216,7 +1346,7 @@ func checkTargetGroupsForScrapeConfig(targetGroups []*targetgroup.Group, scfg *c lb := labels.NewBuilder(labels.EmptyLabels()) for _, tg := range targetGroups { var failures []error - targets, failures = scrape.TargetsFromGroup(tg, scfg, false, targets, lb) + targets, failures = scrape.TargetsFromGroup(tg, scfg, targets, lb) if len(failures) > 0 { first := failures[0] return first @@ -1226,8 +1356,8 @@ func checkTargetGroupsForScrapeConfig(targetGroups []*targetgroup.Group, scfg *c return nil } -func formatPromQL(query string) error { - expr, err := parser.ParseExpr(query) +func formatPromQL(query string, p parser.Parser) error { + expr, err := p.ParseExpr(query) if err != nil { return err } @@ -1236,8 +1366,8 @@ func formatPromQL(query string) error { return nil } -func labelsSetPromQL(query, labelMatchType, name, value string) error { - expr, err := parser.ParseExpr(query) +func labelsSetPromQL(query, labelMatchType, name, value string, p parser.Parser) error { + expr, err := p.ParseExpr(query) if err != nil { return err } @@ -1256,7 +1386,7 @@ func labelsSetPromQL(query, labelMatchType, name, value string) error { return fmt.Errorf("invalid label match type: %s", labelMatchType) } - parser.Inspect(expr, func(node parser.Node, path []parser.Node) error { + parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error { if n, ok := node.(*parser.VectorSelector); ok { var found bool for i, l := range n.LabelMatchers { @@ -1281,13 +1411,13 @@ func labelsSetPromQL(query, labelMatchType, name, value string) error { return nil } -func labelsDeletePromQL(query, name string) error { - expr, err := parser.ParseExpr(query) +func labelsDeletePromQL(query, name string, p parser.Parser) error { + expr, err := p.ParseExpr(query) if err != nil { return err } - parser.Inspect(expr, func(node parser.Node, path []parser.Node) error { + parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error { if n, ok := node.(*parser.VectorSelector); ok { for i, l := range n.LabelMatchers { if l.Name == name { diff --git a/cmd/promtool/main_test.go b/cmd/promtool/main_test.go index 2b086851f36..e42ca69441b 100644 --- a/cmd/promtool/main_test.go +++ b/cmd/promtool/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "io" "net/http" "net/http/httptest" "net/url" @@ -31,13 +32,22 @@ import ( "testing" "time" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" + "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/promql/promqltest" ) +func init() { + // This can be removed when the legacy global mode is fully deprecated. + //nolint:staticcheck + model.NameValidationScheme = model.UTF8Validation +} + var promtoolPath = os.Args[0] func TestMain(m *testing.M) { @@ -54,6 +64,7 @@ func TestMain(m *testing.M) { } func TestQueryRange(t *testing.T) { + t.Parallel() s, getRequest := mockServer(200, `{"status": "success", "data": {"resultType": "matrix", "result": []}}`) defer s.Close() @@ -77,6 +88,7 @@ func TestQueryRange(t *testing.T) { } func TestQueryInstant(t *testing.T) { + t.Parallel() s, getRequest := mockServer(200, `{"status": "success", "data": {"resultType": "vector", "result": []}}`) defer s.Close() @@ -84,7 +96,7 @@ func TestQueryInstant(t *testing.T) { require.NoError(t, err) p := &promqlPrinter{} - exitCode := QueryInstant(urlObject, http.DefaultTransport, "up", "300", p) + exitCode := QueryInstant(urlObject, http.DefaultTransport, map[string]string{}, "up", "300", p) require.Equal(t, "/api/v1/query", getRequest().URL.Path) form := getRequest().Form require.Equal(t, "up", form.Get("query")) @@ -92,6 +104,51 @@ func TestQueryInstant(t *testing.T) { require.Equal(t, 0, exitCode) } +func TestQueryInstantHeaders(t *testing.T) { + t.Parallel() + s, getRequest := mockServer(200, `{"status": "success", "data": {"resultType": "vector", "result": []}}`) + defer s.Close() + + urlObject, err := url.Parse(s.URL) + require.NoError(t, err) + + p := &promqlPrinter{} + headers := map[string]string{"X-Scope-OrgID": "prom", "X-Custom": "value"} + exitCode := QueryInstant(urlObject, http.DefaultTransport, headers, "up", "300", p) + require.Equal(t, 0, exitCode) + require.Equal(t, "prom", getRequest().Header.Get("X-Scope-OrgID")) + require.Equal(t, "value", getRequest().Header.Get("X-Custom")) +} + +func TestCheckServerStatus(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + urlSuffix string + }{ + {name: "no trailing slash", urlSuffix: ""}, + {name: "trailing slash", urlSuffix: "/"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + s, getRequest := mockServer(200, "Prometheus is Healthy.\n") + defer s.Close() + + urlObject, err := url.Parse(s.URL + tc.urlSuffix) + require.NoError(t, err) + + err = CheckServerStatus(urlObject, checkHealth, http.DefaultTransport) + require.NoError(t, err) + // The endpoint path must be requested verbatim regardless of a + // trailing slash in the user-provided URL. A naive string + // concatenation would request "//-/healthy" here, which the + // Prometheus router does not match. + require.Equal(t, checkHealth, getRequest().URL.Path) + }) + } +} + func mockServer(code int, body string) (*httptest.Server, func() *http.Request) { var req *http.Request server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -108,6 +165,7 @@ func mockServer(code int, body string) (*httptest.Server, func() *http.Request) } func TestCheckSDFile(t *testing.T) { + t.Parallel() cases := []struct { name string file string @@ -127,8 +185,8 @@ func TestCheckSDFile(t *testing.T) { }, { name: "bad file extension", - file: "./testdata/bad-sd-file-extension.nonexistant", - err: "invalid file extension: \".nonexistant\"", + file: "./testdata/bad-sd-file-extension.nonexistent", + err: "invalid file extension: \".nonexistent\"", }, { name: "bad format", @@ -138,9 +196,10 @@ func TestCheckSDFile(t *testing.T) { } for _, test := range cases { t.Run(test.name, func(t *testing.T) { + t.Parallel() _, err := checkSDFile(test.file) if test.err != "" { - require.Equalf(t, test.err, err.Error(), "Expected error %q, got %q", test.err, err.Error()) + require.EqualErrorf(t, err, test.err, "Expected error %q, got %q", test.err, err.Error()) return } require.NoError(t, err) @@ -149,6 +208,7 @@ func TestCheckSDFile(t *testing.T) { } func TestCheckDuplicates(t *testing.T) { + t.Parallel() cases := []struct { name string ruleFile string @@ -173,7 +233,8 @@ func TestCheckDuplicates(t *testing.T) { for _, test := range cases { c := test t.Run(c.name, func(t *testing.T) { - rgs, err := rulefmt.ParseFile(c.ruleFile) + t.Parallel() + rgs, err := rulefmt.ParseFile(c.ruleFile, false, model.UTF8Validation, parser.NewParser(parser.Options{}), promslog.NewNopLogger()) require.Empty(t, err) dups := checkDuplicates(rgs.Groups) require.Equal(t, c.expectedDups, dups) @@ -182,16 +243,16 @@ func TestCheckDuplicates(t *testing.T) { } func BenchmarkCheckDuplicates(b *testing.B) { - rgs, err := rulefmt.ParseFile("./testdata/rules_large.yml") + rgs, err := rulefmt.ParseFile("./testdata/rules_large.yml", false, model.UTF8Validation, parser.NewParser(parser.Options{}), promslog.NewNopLogger()) require.Empty(b, err) - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { checkDuplicates(rgs.Groups) } } func TestCheckTargetConfig(t *testing.T) { + t.Parallel() cases := []struct { name string file string @@ -220,9 +281,10 @@ func TestCheckTargetConfig(t *testing.T) { } for _, test := range cases { t.Run(test.name, func(t *testing.T) { - _, err := checkConfig(false, "testdata/"+test.file, false) + t.Parallel() + _, _, err := checkConfig(false, "testdata/"+test.file, false) if test.err != "" { - require.Equalf(t, test.err, err.Error(), "Expected error %q, got %q", test.err, err.Error()) + require.EqualErrorf(t, err, test.err, "Expected error %q, got %q", test.err, err.Error()) return } require.NoError(t, err) @@ -231,6 +293,7 @@ func TestCheckTargetConfig(t *testing.T) { } func TestCheckConfigSyntax(t *testing.T) { + t.Parallel() cases := []struct { name string file string @@ -282,7 +345,7 @@ func TestCheckConfigSyntax(t *testing.T) { err: "error checking client cert file \"testdata/nonexistent_cert_file.yml\": " + "stat testdata/nonexistent_cert_file.yml: no such file or directory", errWindows: "error checking client cert file \"testdata\\\\nonexistent_cert_file.yml\": " + - "CreateFile testdata\\nonexistent_cert_file.yml: The system cannot find the file specified.", + "GetFileAttributesEx testdata\\nonexistent_cert_file.yml: The system cannot find the file specified.", }, { name: "check with syntax only succeeds with nonexistent credentials file", @@ -298,18 +361,19 @@ func TestCheckConfigSyntax(t *testing.T) { err: "error checking authorization credentials or bearer token file \"/random/file/which/does/not/exist.yml\": " + "stat /random/file/which/does/not/exist.yml: no such file or directory", errWindows: "error checking authorization credentials or bearer token file \"testdata\\\\random\\\\file\\\\which\\\\does\\\\not\\\\exist.yml\": " + - "CreateFile testdata\\random\\file\\which\\does\\not\\exist.yml: The system cannot find the path specified.", + "GetFileAttributesEx testdata\\random\\file\\which\\does\\not\\exist.yml: The system cannot find the path specified.", }, } for _, test := range cases { t.Run(test.name, func(t *testing.T) { - _, err := checkConfig(false, "testdata/"+test.file, test.syntaxOnly) + t.Parallel() + _, _, err := checkConfig(false, "testdata/"+test.file, test.syntaxOnly) expectedErrMsg := test.err if strings.Contains(runtime.GOOS, "windows") { expectedErrMsg = test.errWindows } if expectedErrMsg != "" { - require.Equalf(t, expectedErrMsg, err.Error(), "Expected error %q, got %q", test.err, err.Error()) + require.EqualErrorf(t, err, expectedErrMsg, "Expected error %q, got %q", test.err, err.Error()) return } require.NoError(t, err) @@ -318,6 +382,7 @@ func TestCheckConfigSyntax(t *testing.T) { } func TestAuthorizationConfig(t *testing.T) { + t.Parallel() cases := []struct { name string file string @@ -337,9 +402,10 @@ func TestAuthorizationConfig(t *testing.T) { for _, test := range cases { t.Run(test.name, func(t *testing.T) { - _, err := checkConfig(false, "testdata/"+test.file, false) + t.Parallel() + _, _, err := checkConfig(false, "testdata/"+test.file, false) if test.err != "" { - require.Contains(t, err.Error(), test.err, "Expected error to contain %q, got %q", test.err, err.Error()) + require.ErrorContains(t, err, test.err, "Expected error to contain %q, got %q", test.err, err.Error()) return } require.NoError(t, err) @@ -351,6 +417,7 @@ func TestCheckMetricsExtended(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Skipping on windows") } + t.Parallel() f, err := os.Open("testdata/metrics-test.prom") require.NoError(t, err) @@ -383,10 +450,104 @@ func TestCheckMetricsExtended(t *testing.T) { }, stats) } +func TestCheckMetricsLintOptions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping on windows") + } + + const testMetrics = ` +# HELP testMetric_CamelCase A test metric with camelCase +# TYPE testMetric_CamelCase gauge +testMetric_CamelCase{label="value1"} 1 +` + + tests := []struct { + name string + lint string + extended bool + wantErrCode int + wantLint bool + wantCard bool + }{ + { + name: "default_all_with_extended", + lint: lintOptionAll, + extended: true, + wantErrCode: lintErrExitCode, + wantLint: true, + wantCard: true, + }, + { + name: "lint_none_with_extended", + lint: lintOptionNone, + extended: true, + wantErrCode: successExitCode, + wantLint: false, + wantCard: true, + }, + { + name: "both_disabled_fails", + lint: lintOptionNone, + extended: false, + wantErrCode: failureExitCode, + wantLint: false, + wantCard: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, w, err := os.Pipe() + require.NoError(t, err) + _, err = w.WriteString(testMetrics) + require.NoError(t, err) + w.Close() + + oldStdin := os.Stdin + os.Stdin = r + defer func() { os.Stdin = oldStdin }() + + oldStdout := os.Stdout + oldStderr := os.Stderr + rOut, wOut, err := os.Pipe() + require.NoError(t, err) + rErr, wErr, err := os.Pipe() + require.NoError(t, err) + os.Stdout = wOut + os.Stderr = wErr + + code := CheckMetrics(tt.extended, tt.lint) + + wOut.Close() + wErr.Close() + os.Stdout = oldStdout + os.Stderr = oldStderr + + var outBuf, errBuf bytes.Buffer + _, _ = io.Copy(&outBuf, rOut) + _, _ = io.Copy(&errBuf, rErr) + + require.Equal(t, tt.wantErrCode, code) + if tt.wantLint { + require.Contains(t, errBuf.String(), "testMetric_CamelCase") + } else { + require.NotContains(t, errBuf.String(), "testMetric_CamelCase") + } + + if tt.wantCard { + require.Contains(t, outBuf.String(), "Cardinality") + } else { + require.NotContains(t, outBuf.String(), "Cardinality") + } + }) + } +} + func TestExitCodes(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() for _, c := range []struct { file string @@ -411,8 +572,10 @@ func TestExitCodes(t *testing.T) { }, } { t.Run(c.file, func(t *testing.T) { + t.Parallel() for _, lintFatal := range []bool{true, false} { t.Run(strconv.FormatBool(lintFatal), func(t *testing.T) { + t.Parallel() args := []string{"-test.main", "check", "config", "testdata/" + c.file} if lintFatal { args = append(args, "--lint-fatal") @@ -443,6 +606,7 @@ func TestDocumentation(t *testing.T) { if runtime.GOOS == "windows" { t.SkipNow() } + t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -485,8 +649,8 @@ func TestCheckRules(t *testing.T) { defer func(v *os.File) { os.Stdin = v }(os.Stdin) os.Stdin = r - exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false)) - require.Equal(t, successExitCode, exitCode, "") + exitCode := CheckRules(newRulesLintConfig(lintOptionDuplicateRules, false, false, model.UTF8Validation), parser.NewParser(parser.Options{})) + require.Equal(t, successExitCode, exitCode) }) t.Run("rules-bad", func(t *testing.T) { @@ -507,8 +671,8 @@ func TestCheckRules(t *testing.T) { defer func(v *os.File) { os.Stdin = v }(os.Stdin) os.Stdin = r - exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false)) - require.Equal(t, failureExitCode, exitCode, "") + exitCode := CheckRules(newRulesLintConfig(lintOptionDuplicateRules, false, false, model.UTF8Validation), parser.NewParser(parser.Options{})) + require.Equal(t, failureExitCode, exitCode) }) t.Run("rules-lint-fatal", func(t *testing.T) { @@ -529,38 +693,95 @@ func TestCheckRules(t *testing.T) { defer func(v *os.File) { os.Stdin = v }(os.Stdin) os.Stdin = r - exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, true)) - require.Equal(t, lintErrExitCode, exitCode, "") + exitCode := CheckRules(newRulesLintConfig(lintOptionDuplicateRules, true, false, model.UTF8Validation), parser.NewParser(parser.Options{})) + require.Equal(t, lintErrExitCode, exitCode) }) } +func TestCheckRulesWithFeatureFlag(t *testing.T) { + // As opposed to TestCheckRules calling CheckRules directly we run promtool + // so the feature flag parsing can be tested. + + args := []string{"-test.main", "--enable-feature=promql-experimental-functions", "--enable-feature=promql-duration-expr", "--enable-feature=promql-extended-range-selectors", "check", "rules", "testdata/features.yml"} + tool := exec.Command(promtoolPath, args...) + err := tool.Run() + require.NoError(t, err) +} + func TestCheckRulesWithRuleFiles(t *testing.T) { t.Run("rules-good", func(t *testing.T) { - exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false), "./testdata/rules.yml") - require.Equal(t, successExitCode, exitCode, "") + t.Parallel() + exitCode := CheckRules(newRulesLintConfig(lintOptionDuplicateRules, false, false, model.UTF8Validation), parser.NewParser(parser.Options{}), "./testdata/rules.yml") + require.Equal(t, successExitCode, exitCode) }) t.Run("rules-bad", func(t *testing.T) { - exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false), "./testdata/rules-bad.yml") - require.Equal(t, failureExitCode, exitCode, "") + t.Parallel() + exitCode := CheckRules(newRulesLintConfig(lintOptionDuplicateRules, false, false, model.UTF8Validation), parser.NewParser(parser.Options{}), "./testdata/rules-bad.yml") + require.Equal(t, failureExitCode, exitCode) }) t.Run("rules-lint-fatal", func(t *testing.T) { - exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, true), "./testdata/prometheus-rules.lint.yml") - require.Equal(t, lintErrExitCode, exitCode, "") + t.Parallel() + exitCode := CheckRules(newRulesLintConfig(lintOptionDuplicateRules, true, false, model.UTF8Validation), parser.NewParser(parser.Options{}), "./testdata/prometheus-rules.lint.yml") + require.Equal(t, lintErrExitCode, exitCode) }) } +func TestCheckScrapeConfigs(t *testing.T) { + for _, tc := range []struct { + name string + lookbackDelta model.Duration + expectError bool + }{ + { + name: "scrape interval less than lookback delta", + lookbackDelta: model.Duration(11 * time.Minute), + expectError: false, + }, + { + name: "scrape interval greater than lookback delta", + lookbackDelta: model.Duration(5 * time.Minute), + expectError: true, + }, + { + name: "scrape interval same as lookback delta", + lookbackDelta: model.Duration(10 * time.Minute), + expectError: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + // Non-fatal linting. + p := parser.NewParser(parser.Options{}) + code := CheckConfig(false, false, newConfigLintConfig(lintOptionTooLongScrapeInterval, false, false, model.UTF8Validation, tc.lookbackDelta), p, "./testdata/prometheus-config.lint.too_long_scrape_interval.yml") + require.Equal(t, successExitCode, code, "Non-fatal linting should return success") + // Fatal linting. + code = CheckConfig(false, false, newConfigLintConfig(lintOptionTooLongScrapeInterval, true, false, model.UTF8Validation, tc.lookbackDelta), p, "./testdata/prometheus-config.lint.too_long_scrape_interval.yml") + if tc.expectError { + require.Equal(t, lintErrExitCode, code, "Fatal linting should return error") + } else { + require.Equal(t, successExitCode, code, "Fatal linting should return success when there are no problems") + } + // Check syntax only, no linting. + code = CheckConfig(false, true, newConfigLintConfig(lintOptionTooLongScrapeInterval, true, false, model.UTF8Validation, tc.lookbackDelta), p, "./testdata/prometheus-config.lint.too_long_scrape_interval.yml") + require.Equal(t, successExitCode, code, "Fatal linting should return success when checking syntax only") + // Lint option "none" should disable linting. + code = CheckConfig(false, false, newConfigLintConfig(lintOptionNone+","+lintOptionTooLongScrapeInterval, true, false, model.UTF8Validation, tc.lookbackDelta), p, "./testdata/prometheus-config.lint.too_long_scrape_interval.yml") + require.Equal(t, successExitCode, code, `Fatal linting should return success when lint option "none" is specified`) + }) + } +} + func TestTSDBDumpCommand(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + t.Parallel() storage := promqltest.LoadedStorage(t, ` load 1m metric{foo="bar"} 1 2 3 `) - t.Cleanup(func() { storage.Close() }) for _, c := range []struct { name string @@ -587,6 +808,7 @@ func TestTSDBDumpCommand(t *testing.T) { }, } { t.Run(c.name, func(t *testing.T) { + t.Parallel() args := []string{"-test.main", "tsdb", c.subCmd, storage.Dir()} cmd := exec.Command(promtoolPath, args...) require.NoError(t, cmd.Run()) diff --git a/cmd/promtool/metrics.go b/cmd/promtool/metrics.go index 4c91d1d6fea..b1a2beb72ef 100644 --- a/cmd/promtool/metrics.go +++ b/cmd/promtool/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,43 +23,46 @@ import ( "os" "time" - "github.com/golang/snappy" - config_util "github.com/prometheus/common/config" - "github.com/prometheus/common/model" + remoteapi "github.com/prometheus/client_golang/exp/api/remote" - "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/util/fmtutil" ) // PushMetrics to a prometheus remote write (for testing purpose only). -func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, timeout time.Duration, labels map[string]string, files ...string) int { +func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, timeout time.Duration, protoMsg string, labels map[string]string, files ...string) int { addressURL, err := url.Parse(url.String()) if err != nil { fmt.Fprintln(os.Stderr, err) return failureExitCode } - // build remote write client - writeClient, err := remote.NewWriteClient("remote-write", &remote.ClientConfig{ - URL: &config_util.URL{URL: addressURL}, - Timeout: model.Duration(timeout), - }) + // Build HTTP client with custom transport for headers. + httpClient := &http.Client{ + Transport: &setHeadersTransport{ + RoundTripper: roundTripper, + headers: headers, + }, + Timeout: timeout, + } + + // Create remote write API client. + writeAPI, err := remoteapi.NewAPI(addressURL.String(), remoteapi.WithAPIHTTPClient(httpClient)) if err != nil { fmt.Fprintln(os.Stderr, err) return failureExitCode } - // set custom tls config from httpConfigFilePath - // set custom headers to every request - client, ok := writeClient.(*remote.Client) - if !ok { - fmt.Fprintln(os.Stderr, fmt.Errorf("unexpected type %T", writeClient)) + // Determine message type based on protobuf_message flag. + var messageType remoteapi.WriteMessageType + switch protoMsg { + case "io.prometheus.write.v2.Request": + messageType = remoteapi.WriteV2MessageType + case "prometheus.WriteRequest": + messageType = remoteapi.WriteV1MessageType + default: + fmt.Fprintf(os.Stderr, " FAILED: invalid protobuf message %q, must be prometheus.WriteRequest or io.prometheus.write.v2.Request\n", protoMsg) return failureExitCode } - client.Client.Transport = &setHeadersTransport{ - RoundTripper: roundTripper, - headers: headers, - } var data []byte var failed bool @@ -71,7 +74,7 @@ func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[strin return failureExitCode } fmt.Printf("Parsing standard input\n") - if parseAndPushMetrics(client, data, labels) { + if parseAndPushMetrics(writeAPI, messageType, data, labels) { fmt.Printf(" SUCCESS: metrics pushed to remote write.\n") return successExitCode } @@ -87,7 +90,7 @@ func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[strin } fmt.Printf("Parsing metrics file %s\n", file) - if parseAndPushMetrics(client, data, labels) { + if parseAndPushMetrics(writeAPI, messageType, data, labels) { fmt.Printf(" SUCCESS: metrics file %s pushed to remote write.\n", file) continue } @@ -101,23 +104,15 @@ func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[strin return successExitCode } -// TODO(bwplotka): Add PRW 2.0 support. -func parseAndPushMetrics(client *remote.Client, data []byte, labels map[string]string) bool { +func parseAndPushMetrics(writeAPI *remoteapi.API, messageType remoteapi.WriteMessageType, data []byte, labels map[string]string) bool { metricsData, err := fmtutil.MetricTextToWriteRequest(bytes.NewReader(data), labels) if err != nil { fmt.Fprintln(os.Stderr, " FAILED:", err) return false } - raw, err := metricsData.Marshal() - if err != nil { - fmt.Fprintln(os.Stderr, " FAILED:", err) - return false - } - - // Encode the request body into snappy encoding. - compressed := snappy.Encode(nil, raw) - _, err = client.Store(context.Background(), compressed, 0) + // Use remoteapi.Write which handles marshaling and compression internally. + _, err = writeAPI.Write(context.Background(), messageType, metricsData) if err != nil { fmt.Fprintln(os.Stderr, " FAILED:", err) return false diff --git a/cmd/promtool/metrics_test.go b/cmd/promtool/metrics_test.go new file mode 100644 index 00000000000..d5a3bf63cc0 --- /dev/null +++ b/cmd/promtool/metrics_test.go @@ -0,0 +1,162 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + "time" + + remoteapi "github.com/prometheus/client_golang/exp/api/remote" + "github.com/stretchr/testify/require" +) + +// setupTestServer creates a test server with a mock storage for remote write testing. +func setupTestServer(t *testing.T, supportedTypes remoteapi.MessageTypes) (*httptest.Server, *mockStorage, *url.URL) { + t.Helper() + + store := &mockStorage{} + handler := remoteapi.NewWriteHandler(store, supportedTypes) + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + + return server, store, serverURL +} + +// createMetricsFile creates a temporary file with the given metrics data. +func createMetricsFile(t *testing.T, metricsData string) string { + t.Helper() + + tmpFile := t.TempDir() + "/metrics.txt" + err := os.WriteFile(tmpFile, []byte(metricsData), 0o644) + require.NoError(t, err) + + return tmpFile +} + +func TestPushMetrics(t *testing.T) { + tests := []struct { + name string + metricsData string + protoMsg string + serverTypes remoteapi.MessageTypes + }{ + { + name: "successful push with gauge metrics (V1)", + metricsData: `# HELP test_metric A test metric +# TYPE test_metric gauge +test_metric{label="value1"} 42.0 +test_metric{label="value2"} 43.0 +`, + protoMsg: "prometheus.WriteRequest", + serverTypes: remoteapi.MessageTypes{remoteapi.WriteV1MessageType}, + }, + { + name: "successful push with counter metrics (V1)", + metricsData: `# HELP test_counter A test counter +# TYPE test_counter counter +test_counter 100 +`, + protoMsg: "prometheus.WriteRequest", + serverTypes: remoteapi.MessageTypes{remoteapi.WriteV1MessageType}, + }, + { + name: "successful push with gauge metrics (V2)", + metricsData: `# HELP test_metric A test metric +# TYPE test_metric gauge +test_metric{label="value1"} 42.0 +test_metric{label="value2"} 43.0 +`, + protoMsg: "io.prometheus.write.v2.Request", + serverTypes: remoteapi.MessageTypes{remoteapi.WriteV2MessageType}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, store, serverURL := setupTestServer(t, tc.serverTypes) + tmpFile := createMetricsFile(t, tc.metricsData) + + status := PushMetrics( + serverURL, + http.DefaultTransport, + map[string]string{}, + 30*time.Second, + tc.protoMsg, + map[string]string{"job": "test"}, + tmpFile, + ) + + require.Equal(t, successExitCode, status) + require.True(t, store.called, "Handler should have been called") + require.NoError(t, store.lastErr, "Handler should not have returned an error") + require.NotEmpty(t, store.receivedData, "Request should contain data (compression and decompression successful)") + require.Contains(t, store.receivedContentType, "application/x-protobuf", "Content-Type should be protobuf") + }) + } +} + +// mockStorage is a simple mock for testing the remote write handler. +type mockStorage struct { + called bool + lastErr error + receivedData []byte + receivedContentType string +} + +func (m *mockStorage) Store(req *http.Request, _ remoteapi.WriteMessageType) (*remoteapi.WriteResponse, error) { + m.called = true + + // Capture content-type header. + m.receivedContentType = req.Header.Get("Content-Type") + + if req.Body != nil { + data, err := io.ReadAll(req.Body) + if err == nil { + m.receivedData = data + } + } + + if m.lastErr != nil { + return nil, m.lastErr + } + resp := remoteapi.NewWriteResponse() + resp.SetStatusCode(http.StatusNoContent) + return resp, nil +} + +func TestPushMetricsInvalidProtoMsg(t *testing.T) { + _, store, serverURL := setupTestServer(t, remoteapi.MessageTypes{remoteapi.WriteV1MessageType}) + tmpFile := createMetricsFile(t, "test_metric 1\n") + + status := PushMetrics( + serverURL, + http.DefaultTransport, + map[string]string{}, + 30*time.Second, + "blablalba", // Invalid protoMsg. + map[string]string{"job": "test"}, + tmpFile, + ) + + require.Equal(t, failureExitCode, status) + require.False(t, store.called, "Handler should not have been called for invalid protoMsg") +} diff --git a/cmd/promtool/query.go b/cmd/promtool/query.go index 0d7cb12cf42..ef0a2368ef3 100644 --- a/cmd/promtool/query.go +++ b/cmd/promtool/query.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -61,8 +61,8 @@ func newAPI(url *url.URL, roundTripper http.RoundTripper, headers map[string]str } // QueryInstant performs an instant query against a Prometheus server. -func QueryInstant(url *url.URL, roundTripper http.RoundTripper, query, evalTime string, p printer) int { - api, err := newAPI(url, roundTripper, nil) +func QueryInstant(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, query, evalTime string, p printer) int { + api, err := newAPI(url, roundTripper, headers) if err != nil { fmt.Fprintln(os.Stderr, "error creating API client:", err) return failureExitCode diff --git a/cmd/promtool/rules.go b/cmd/promtool/rules.go index 5a18644842b..bb45178e9c8 100644 --- a/cmd/promtool/rules.go +++ b/cmd/promtool/rules.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,20 +15,20 @@ package main import ( "context" + "errors" "fmt" + "log/slog" "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" ) const maxSamplesInMemory = 5000 @@ -38,7 +38,7 @@ type queryRangeAPI interface { } type ruleImporter struct { - logger log.Logger + logger *slog.Logger config ruleImporterConfig apiClient queryRangeAPI @@ -48,28 +48,31 @@ type ruleImporter struct { } type ruleImporterConfig struct { - outputDir string - start time.Time - end time.Time - evalInterval time.Duration - maxBlockDuration time.Duration + outputDir string + start time.Time + end time.Time + evalInterval time.Duration + maxBlockDuration time.Duration + nameValidationScheme model.ValidationScheme } // newRuleImporter creates a new rule importer that can be used to parse and evaluate recording rule files and create new series // written to disk in blocks. -func newRuleImporter(logger log.Logger, config ruleImporterConfig, apiClient queryRangeAPI) *ruleImporter { - level.Info(logger).Log("backfiller", "new rule importer", "start", config.start.Format(time.RFC822), "end", config.end.Format(time.RFC822)) +func newRuleImporter(logger *slog.Logger, config ruleImporterConfig, apiClient queryRangeAPI) *ruleImporter { + logger.Info("new rule importer", "component", "backfiller", "start", config.start.Format(time.RFC822), "end", config.end.Format(time.RFC822)) return &ruleImporter{ - logger: logger, - config: config, - apiClient: apiClient, - ruleManager: rules.NewManager(&rules.ManagerOptions{}), + logger: logger, + config: config, + apiClient: apiClient, + ruleManager: rules.NewManager(&rules.ManagerOptions{ + NameValidationScheme: config.nameValidationScheme, + }), } } // loadGroups parses groups from a list of recording rule files. func (importer *ruleImporter) loadGroups(_ context.Context, filenames []string) (errs []error) { - groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, "", nil, filenames...) + groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, "", nil, false, filenames...) if errs != nil { return errs } @@ -80,10 +83,10 @@ func (importer *ruleImporter) loadGroups(_ context.Context, filenames []string) // importAll evaluates all the recording rules and creates new time series and writes them to disk in blocks. func (importer *ruleImporter) importAll(ctx context.Context) (errs []error) { for name, group := range importer.groups { - level.Info(importer.logger).Log("backfiller", "processing group", "name", name) + importer.logger.Info("processing group", "component", "backfiller", "name", name) for i, r := range group.Rules() { - level.Info(importer.logger).Log("backfiller", "processing rule", "id", i, "name", r.Name()) + importer.logger.Info("processing rule", "component", "backfiller", "id", i, "name", r.Name()) if err := importer.importRule(ctx, r.Query().String(), r.Name(), r.Labels(), importer.config.start, importer.config.end, int64(importer.config.maxBlockDuration/time.Millisecond), group); err != nil { errs = append(errs, err) } @@ -124,7 +127,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName return fmt.Errorf("query range: %w", err) } if warnings != nil { - level.Warn(importer.logger).Log("msg", "Range query returned warnings.", "warnings", warnings) + importer.logger.Warn("Range query returned warnings.", "warnings", warnings) } // To prevent races with compaction, a block writer only allows appending samples @@ -133,14 +136,14 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName // also need to append samples throughout the whole block range. To allow that, we // pretend that the block is twice as large here, but only really add sample in the // original interval later. - w, err := tsdb.NewBlockWriter(log.NewNopLogger(), importer.config.outputDir, 2*blockDuration) + w, err := tsdb.NewBlockWriter(promslog.NewNopLogger(), importer.config.outputDir, 2*blockDuration) if err != nil { return fmt.Errorf("new block writer: %w", err) } var closed bool defer func() { if !closed { - err = tsdb_errors.NewMulti(err, w.Close()).Err() + err = errors.Join(err, w.Close()) } }() app := newMultipleAppender(ctx, w) @@ -178,7 +181,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName if err := app.flushAndCommit(ctx); err != nil { return fmt.Errorf("flush and commit: %w", err) } - err = tsdb_errors.NewMulti(err, w.Close()).Err() + err = errors.Join(err, w.Close()) closed = true } diff --git a/cmd/promtool/rules_test.go b/cmd/promtool/rules_test.go index d55fb0c8963..678e2b4d506 100644 --- a/cmd/promtool/rules_test.go +++ b/cmd/promtool/rules_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/go-kit/log" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/labels" @@ -35,7 +35,7 @@ type mockQueryRangeAPI struct { samples model.Matrix } -func (mockAPI mockQueryRangeAPI) QueryRange(_ context.Context, query string, r v1.Range, opts ...v1.Option) (model.Value, v1.Warnings, error) { +func (mockAPI mockQueryRangeAPI) QueryRange(context.Context, string, v1.Range, ...v1.Option) (model.Value, v1.Warnings, error) { return mockAPI.samples, v1.Warnings{}, nil } @@ -43,6 +43,7 @@ const defaultBlockDuration = time.Duration(tsdb.DefaultBlockDuration) * time.Mil // TestBackfillRuleIntegration is an integration test that runs all the rule importer code to confirm the parts work together. func TestBackfillRuleIntegration(t *testing.T) { + t.Parallel() const ( testMaxSampleCount = 50 testValue = 123 @@ -72,6 +73,7 @@ func TestBackfillRuleIntegration(t *testing.T) { } for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { + t.Parallel() tmpDir := t.TempDir() ctx := context.Background() @@ -161,13 +163,14 @@ func TestBackfillRuleIntegration(t *testing.T) { } func newTestRuleImporter(_ context.Context, start time.Time, tmpDir string, testSamples model.Matrix, maxBlockDuration time.Duration) (*ruleImporter, error) { - logger := log.NewNopLogger() + logger := promslog.NewNopLogger() cfg := ruleImporterConfig{ - outputDir: tmpDir, - start: start.Add(-10 * time.Hour), - end: start.Add(-7 * time.Hour), - evalInterval: 60 * time.Second, - maxBlockDuration: maxBlockDuration, + outputDir: tmpDir, + start: start.Add(-10 * time.Hour), + end: start.Add(-7 * time.Hour), + evalInterval: 60 * time.Second, + maxBlockDuration: maxBlockDuration, + nameValidationScheme: model.UTF8Validation, } return newRuleImporter(logger, cfg, mockQueryRangeAPI{ @@ -210,6 +213,7 @@ func createMultiRuleTestFiles(path string) error { // TestBackfillLabels confirms that the labels in the rule file override the labels from the metrics // received from Prometheus Query API, including the __name__ label. func TestBackfillLabels(t *testing.T) { + t.Parallel() tmpDir := t.TempDir() ctx := context.Background() @@ -251,6 +255,7 @@ func TestBackfillLabels(t *testing.T) { require.NoError(t, err) t.Run("correct-labels", func(t *testing.T) { + t.Parallel() selectedSeries := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*")) for selectedSeries.Next() { series := selectedSeries.At() diff --git a/cmd/promtool/sd.go b/cmd/promtool/sd.go index e65262d439f..6b844c699ab 100644 --- a/cmd/promtool/sd.go +++ b/cmd/promtool/sd.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -20,9 +20,9 @@ import ( "os" "time" - "github.com/go-kit/log" "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" @@ -38,10 +38,10 @@ type sdCheckResult struct { } // CheckSD performs service discovery for the given job name and reports the results. -func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool, registerer prometheus.Registerer) int { - logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) +func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, _ prometheus.Registerer) int { + logger := promslog.New(&promslog.Config{}) - cfg, err := config.LoadFile(sdConfigFiles, false, false, logger) + cfg, err := config.LoadFile(sdConfigFiles, false, logger) if err != nil { fmt.Fprintln(os.Stderr, "Cannot load config", err) return failureExitCode @@ -114,7 +114,7 @@ outerLoop: } results := []sdCheckResult{} for _, tgs := range sdCheckResults { - results = append(results, getSDCheckResult(tgs, scrapeConfig, noDefaultScrapePort)...) + results = append(results, getSDCheckResult(tgs, scrapeConfig)...) } res, err := json.MarshalIndent(results, "", " ") @@ -127,7 +127,7 @@ outerLoop: return successExitCode } -func getSDCheckResult(targetGroups []*targetgroup.Group, scrapeConfig *config.ScrapeConfig, noDefaultScrapePort bool) []sdCheckResult { +func getSDCheckResult(targetGroups []*targetgroup.Group, scrapeConfig *config.ScrapeConfig) []sdCheckResult { sdCheckResults := []sdCheckResult{} lb := labels.NewBuilder(labels.EmptyLabels()) for _, targetGroup := range targetGroups { @@ -144,7 +144,9 @@ func getSDCheckResult(targetGroups []*targetgroup.Group, scrapeConfig *config.Sc } } - res, orig, err := scrape.PopulateLabels(lb, scrapeConfig, noDefaultScrapePort) + scrape.PopulateDiscoveredLabels(lb, scrapeConfig, target, targetGroup.Labels) + orig := lb.Labels() + res, err := scrape.PopulateLabels(lb, scrapeConfig, target, targetGroup.Labels) result := sdCheckResult{ DiscoveredLabels: orig, Labels: res, diff --git a/cmd/promtool/sd_test.go b/cmd/promtool/sd_test.go index cb65ee72aa6..3d561aeae3f 100644 --- a/cmd/promtool/sd_test.go +++ b/cmd/promtool/sd_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,17 +18,17 @@ import ( "time" "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/util/testutil" - - "github.com/stretchr/testify/require" ) func TestSDCheckResult(t *testing.T) { + t.Parallel() targetGroups := []*targetgroup.Group{{ Targets: []model.LabelSet{ map[model.LabelName]model.LabelValue{"__address__": "localhost:8080", "foo": "bar"}, @@ -42,11 +42,12 @@ func TestSDCheckResult(t *testing.T) { ScrapeInterval: model.Duration(1 * time.Minute), ScrapeTimeout: model.Duration(10 * time.Second), RelabelConfigs: []*relabel.Config{{ - SourceLabels: model.LabelNames{"foo"}, - Action: relabel.Replace, - TargetLabel: "newfoo", - Regex: reg, - Replacement: "$1", + SourceLabels: model.LabelNames{"foo"}, + Action: relabel.Replace, + TargetLabel: "newfoo", + Regex: reg, + Replacement: "$1", + NameValidationScheme: model.UTF8Validation, }}, } @@ -54,13 +55,19 @@ func TestSDCheckResult(t *testing.T) { { DiscoveredLabels: labels.FromStrings( "__address__", "localhost:8080", + "__always_scrape_classic_histograms__", "false", + "__convert_classic_histograms_to_nhcb__", "false", "__scrape_interval__", "1m", + "__scrape_native_histograms__", "false", "__scrape_timeout__", "10s", "foo", "bar", ), Labels: labels.FromStrings( "__address__", "localhost:8080", + "__always_scrape_classic_histograms__", "false", + "__convert_classic_histograms_to_nhcb__", "false", "__scrape_interval__", "1m", + "__scrape_native_histograms__", "false", "__scrape_timeout__", "10s", "foo", "bar", "instance", "localhost:8080", @@ -70,5 +77,5 @@ func TestSDCheckResult(t *testing.T) { }, } - testutil.RequireEqual(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig, true)) + testutil.RequireEqual(t, expectedSDCheckResult, getSDCheckResult(targetGroups, scrapeConfig)) } diff --git a/cmd/promtool/testdata/bad-sd-file-extension.nonexistant b/cmd/promtool/testdata/bad-sd-file-extension.nonexistent similarity index 100% rename from cmd/promtool/testdata/bad-sd-file-extension.nonexistant rename to cmd/promtool/testdata/bad-sd-file-extension.nonexistent diff --git a/cmd/promtool/testdata/config_with_service_discovery_files.yml b/cmd/promtool/testdata/config_with_service_discovery_files.yml index 13b6d7faffb..6a550a84034 100644 --- a/cmd/promtool/testdata/config_with_service_discovery_files.yml +++ b/cmd/promtool/testdata/config_with_service_discovery_files.yml @@ -6,7 +6,7 @@ scrape_configs: alerting: alertmanagers: - scheme: http - api_version: v1 + api_version: v2 file_sd_configs: - files: - nonexistent_file.yml diff --git a/cmd/promtool/testdata/dump-series-1.prom b/cmd/promtool/testdata/dump-series-1.prom new file mode 100644 index 00000000000..5e44c0bf1b4 --- /dev/null +++ b/cmd/promtool/testdata/dump-series-1.prom @@ -0,0 +1,3 @@ +{"__name__":"heavy_metric","foo":"bar"} +{"__name__":"heavy_metric","foo":"foo"} +{"__name__":"metric","baz":"abc","foo":"bar"} diff --git a/cmd/promtool/testdata/dump-series-2.prom b/cmd/promtool/testdata/dump-series-2.prom new file mode 100644 index 00000000000..fefefa6d1b8 --- /dev/null +++ b/cmd/promtool/testdata/dump-series-2.prom @@ -0,0 +1,2 @@ +{"__name__":"heavy_metric","foo":"foo"} +{"__name__":"metric","baz":"abc","foo":"bar"} diff --git a/cmd/promtool/testdata/dump-series-3.prom b/cmd/promtool/testdata/dump-series-3.prom new file mode 100644 index 00000000000..dd98e8707d8 --- /dev/null +++ b/cmd/promtool/testdata/dump-series-3.prom @@ -0,0 +1 @@ +{"__name__":"metric","baz":"abc","foo":"bar"} diff --git a/cmd/promtool/testdata/features.yml b/cmd/promtool/testdata/features.yml new file mode 100644 index 00000000000..946e07d0d76 --- /dev/null +++ b/cmd/promtool/testdata/features.yml @@ -0,0 +1,10 @@ +groups: + - name: features + rules: + # We don't expect anything from these, just want to check the syntax parses. + - record: promql-experimental-functions + expr: sort_by_label(up, "instance") + - record: promql-duration-expr + expr: rate(up[1m * 2]) + - record: promql-extended-range-selectors + expr: rate(up[1m] anchored) diff --git a/cmd/promtool/testdata/prometheus-config.lint.too_long_scrape_interval.yml b/cmd/promtool/testdata/prometheus-config.lint.too_long_scrape_interval.yml new file mode 100644 index 00000000000..fe21d21707f --- /dev/null +++ b/cmd/promtool/testdata/prometheus-config.lint.too_long_scrape_interval.yml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: too_long_scrape_interval_test + scrape_interval: 10m +rule_files: + - prometheus-config.rules.good.yml diff --git a/cmd/promtool/testdata/prometheus-config.rules.good.yml b/cmd/promtool/testdata/prometheus-config.rules.good.yml new file mode 100644 index 00000000000..4529a3ec865 --- /dev/null +++ b/cmd/promtool/testdata/prometheus-config.rules.good.yml @@ -0,0 +1,3 @@ +groups: + - name: rules + rules: [] diff --git a/cmd/promtool/testdata/rules_extrafields.yml b/cmd/promtool/testdata/rules_extrafields.yml new file mode 100644 index 00000000000..85ef079bb83 --- /dev/null +++ b/cmd/promtool/testdata/rules_extrafields.yml @@ -0,0 +1,33 @@ +# This is the rules file. It has an extra "ownership" +# field in the second group. promtool should ignore this field +# and not return an error with --ignore-unknown-fields. + +groups: + - name: alerts + namespace: "foobar" + rules: + - alert: InstanceDown + expr: up == 0 + for: 5m + labels: + severity: page + annotations: + summary: "Instance {{ $labels.instance }} down" + description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." + - alert: AlwaysFiring + expr: 1 + + - name: rules + ownership: + service: "test" + rules: + - record: job:test:count_over_time1m + expr: sum without(instance) (count_over_time(test[1m])) + + # A recording rule that doesn't depend on input series. + - record: fixed_data + expr: 1 + + # Subquery with default resolution test. + - record: suquery_interval_test + expr: count_over_time(up[5m:]) diff --git a/cmd/promtool/testdata/rules_run_extrafields.yml b/cmd/promtool/testdata/rules_run_extrafields.yml new file mode 100644 index 00000000000..86879fc396b --- /dev/null +++ b/cmd/promtool/testdata/rules_run_extrafields.yml @@ -0,0 +1,21 @@ +# Minimal test case to see that --ignore-unknown-fields +# is working as expected. It should not return an error +# when any extra fields are present in the rules file. +rule_files: + - rules_extrafields.yml + +evaluation_interval: 1m + + +tests: + - name: extra ownership field test + input_series: + - series: test + values: 1 + + promql_expr_test: + - expr: test + eval_time: 0 + exp_samples: + - value: 1 + labels: test diff --git a/cmd/promtool/testdata/rules_run_fuzzy.yml b/cmd/promtool/testdata/rules_run_fuzzy.yml new file mode 100644 index 00000000000..3bf4e47a45b --- /dev/null +++ b/cmd/promtool/testdata/rules_run_fuzzy.yml @@ -0,0 +1,43 @@ +# Minimal test case to see that fuzzy compare is working as expected. +# It should allow slight floating point differences through. Larger +# floating point differences should still fail. + +evaluation_interval: 1m +fuzzy_compare: true + +tests: + - name: correct fuzzy match + input_series: + - series: test_low + values: 2.9999999999999996 + - series: test_high + values: 3.0000000000000004 + promql_expr_test: + - expr: test_low + eval_time: 0 + exp_samples: + - labels: test_low + value: 3 + - expr: test_high + eval_time: 0 + exp_samples: + - labels: test_high + value: 3 + + - name: wrong fuzzy match + input_series: + - series: test_low + values: 2.9999999999999987 + - series: test_high + values: 3.0000000000000013 + promql_expr_test: + - expr: test_low + eval_time: 0 + exp_samples: + - labels: test_low + value: 3 + - expr: test_high + eval_time: 0 + exp_samples: + - labels: test_high + value: 3 diff --git a/cmd/promtool/testdata/rules_run_no_fuzzy.yml b/cmd/promtool/testdata/rules_run_no_fuzzy.yml new file mode 100644 index 00000000000..eba201a28c1 --- /dev/null +++ b/cmd/promtool/testdata/rules_run_no_fuzzy.yml @@ -0,0 +1,24 @@ +# Minimal test case to see that fuzzy compare can be turned off, +# and slight floating point differences fail matching. + +evaluation_interval: 1m +fuzzy_compare: false + +tests: + - name: correct fuzzy match + input_series: + - series: test_low + values: 2.9999999999999996 + - series: test_high + values: 3.0000000000000004 + promql_expr_test: + - expr: test_low + eval_time: 0 + exp_samples: + - labels: test_low + value: 3 + - expr: test_high + eval_time: 0 + exp_samples: + - labels: test_high + value: 3 diff --git a/cmd/promtool/testdata/start-time-test.yml b/cmd/promtool/testdata/start-time-test.yml new file mode 100644 index 00000000000..b7365366f47 --- /dev/null +++ b/cmd/promtool/testdata/start-time-test.yml @@ -0,0 +1,76 @@ +rule_files: + - rules.yml + +evaluation_interval: 1m + +tests: + # Test with default start_time (0 / Unix epoch). + - name: default_start_time + interval: 1m + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 0 + - expr: time() + eval_time: 5m + exp_samples: + - value: 300 + + # Test with RFC3339 start_timestamp. + - name: rfc3339_start_timestamp + interval: 1m + start_timestamp: "2024-01-01T00:00:00Z" + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 1704067200 + - expr: time() + eval_time: 5m + exp_samples: + - value: 1704067500 + + # Test with Unix timestamp start_timestamp. + - name: unix_timestamp_start_timestamp + interval: 1m + start_timestamp: 1609459200 + input_series: + - series: test_metric + values: "1 1 1" + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 1609459200 + - expr: time() + eval_time: 10m + exp_samples: + - value: 1609459800 + + # Test that input series samples are correctly timestamped with custom start_timestamp. + - name: samples_with_start_timestamp + interval: 1m + start_timestamp: "2024-01-01T00:00:00Z" + input_series: + - series: 'my_metric{label="test"}' + values: "10+10x15" + promql_expr_test: + # Query at absolute timestamp (start_timestamp = 1704067200). + - expr: my_metric@1704067200 + eval_time: 5m + exp_samples: + - labels: 'my_metric{label="test"}' + value: 10 + # Query at 2 minutes after start_timestamp (1704067200 + 120 = 1704067320). + - expr: my_metric@1704067320 + eval_time: 5m + exp_samples: + - labels: 'my_metric{label="test"}' + value: 30 + # Verify timestamp() function returns the absolute timestamp. + - expr: timestamp(my_metric) + eval_time: 5m + exp_samples: + - labels: '{label="test"}' + value: 1704067500 diff --git a/cmd/promtool/testdata/unittest.yml b/cmd/promtool/testdata/unittest.yml index ff511729ba3..e2a8230902e 100644 --- a/cmd/promtool/testdata/unittest.yml +++ b/cmd/promtool/testdata/unittest.yml @@ -69,13 +69,13 @@ tests: eval_time: 2m exp_samples: - labels: "test_histogram_repeat" - histogram: "{{count:2 sum:3 buckets:[2]}}" + histogram: "{{count:2 sum:3 counter_reset_hint:not_reset buckets:[2]}}" - expr: test_histogram_increase eval_time: 2m exp_samples: - labels: "test_histogram_increase" - histogram: "{{count:4 sum:5.6 buckets:[4]}}" + histogram: "{{count:4 sum:5.6 counter_reset_hint:not_reset buckets:[4]}}" # Ensure a value is stale as soon as it is marked as such. - expr: test_stale @@ -89,11 +89,11 @@ tests: # Ensure lookback delta is respected, when a value is missing. - expr: timestamp(test_missing) - eval_time: 5m + eval_time: 4m59s exp_samples: - value: 0 - expr: timestamp(test_missing) - eval_time: 5m1s + eval_time: 5m exp_samples: [] # Minimal test case to check edge case of a single sample. @@ -113,7 +113,7 @@ tests: - expr: count_over_time(fixed_data[1h]) eval_time: 1h exp_samples: - - value: 61 + - value: 60 - expr: timestamp(fixed_data) eval_time: 1h exp_samples: @@ -183,7 +183,7 @@ tests: - expr: job:test:count_over_time1m eval_time: 1m exp_samples: - - value: 61 + - value: 60 labels: 'job:test:count_over_time1m{job="test"}' - expr: timestamp(job:test:count_over_time1m) eval_time: 1m10s @@ -194,7 +194,7 @@ tests: - expr: job:test:count_over_time1m eval_time: 2m exp_samples: - - value: 61 + - value: 60 labels: 'job:test:count_over_time1m{job="test"}' - expr: timestamp(job:test:count_over_time1m) eval_time: 2m59s999ms diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index 971ea8ab000..13c544d5afc 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,9 +17,11 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "io" + "log/slog" "os" "path/filepath" "runtime" @@ -32,7 +34,7 @@ import ( "time" "github.com/alecthomas/units" - "github.com/go-kit/log" + "github.com/prometheus/common/promslog" "go.uber.org/atomic" "github.com/prometheus/prometheus/model/labels" @@ -41,7 +43,6 @@ import ( "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" ) @@ -60,7 +61,7 @@ type writeBenchmark struct { memprof *os.File blockprof *os.File mtxprof *os.File - logger log.Logger + logger *slog.Logger } func benchmarkWrite(outPath, samplesFile string, numMetrics, numScrapes int) error { @@ -68,7 +69,7 @@ func benchmarkWrite(outPath, samplesFile string, numMetrics, numScrapes int) err outPath: outPath, samplesFile: samplesFile, numMetrics: numMetrics, - logger: log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), + logger: promslog.New(&promslog.Config{}), } if b.outPath == "" { dir, err := os.MkdirTemp("", "tsdb_bench") @@ -87,9 +88,7 @@ func benchmarkWrite(outPath, samplesFile string, numMetrics, numScrapes int) err dir := filepath.Join(b.outPath, "storage") - l := log.With(b.logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) - - st, err := tsdb.Open(dir, l, nil, &tsdb.Options{ + st, err := tsdb.Open(dir, b.logger, nil, &tsdb.Options{ RetentionDuration: int64(15 * 24 * time.Hour / time.Millisecond), MinBlockDuration: int64(2 * time.Hour / time.Millisecond), }, tsdb.NewDBStats()) @@ -156,24 +155,18 @@ func (b *writeBenchmark) ingestScrapes(lbls []labels.Labels, scrapeCount int) (u var wg sync.WaitGroup lbls := lbls for len(lbls) > 0 { - l := 1000 - if len(lbls) < 1000 { - l = len(lbls) - } + l := min(len(lbls), 1000) batch := lbls[:l] lbls = lbls[l:] - wg.Add(1) - go func() { - defer wg.Done() - + wg.Go(func() { n, err := b.ingestScrapesShard(batch, 100, int64(timeDelta*i)) if err != nil { // exitWithError(err) fmt.Println(" err", err) } total.Add(n) - }() + }) } wg.Wait() } @@ -201,7 +194,7 @@ func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount in } total := uint64(0) - for i := 0; i < scrapeCount; i++ { + for range scrapeCount { app := b.storage.Appender(context.TODO()) ts += timeDelta @@ -315,12 +308,11 @@ func readPrometheusLabels(r io.Reader, n int) ([]labels.Labels, error) { i := 0 for scanner.Scan() && i < n { - m := make([]labels.Label, 0, 10) - r := strings.NewReplacer("\"", "", "{", "", "}", "") s := r.Replace(scanner.Text()) labelChunks := strings.Split(s, ",") + m := make([]labels.Label, 0, len(labelChunks)) for _, labelChunk := range labelChunks { split := strings.Split(labelChunk, ":") m = append(m, labels.Label{Name: split[0], Value: split[1]}) @@ -343,7 +335,7 @@ func listBlocks(path string, humanReadable bool) error { return err } defer func() { - err = tsdb_errors.NewMulti(err, db.Close()).Err() + err = errors.Join(err, db.Close()) }() blocks, err := db.Blocks() if err != nil { @@ -367,25 +359,25 @@ func printBlocks(blocks []tsdb.BlockReader, writeHeader, humanReadable bool) { fmt.Fprintf(tw, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", meta.ULID, - getFormatedTime(meta.MinTime, humanReadable), - getFormatedTime(meta.MaxTime, humanReadable), + getFormattedTime(meta.MinTime, humanReadable), + getFormattedTime(meta.MaxTime, humanReadable), time.Duration(meta.MaxTime-meta.MinTime)*time.Millisecond, meta.Stats.NumSamples, meta.Stats.NumChunks, meta.Stats.NumSeries, - getFormatedBytes(b.Size(), humanReadable), + getFormattedBytes(b.Size(), humanReadable), ) } } -func getFormatedTime(timestamp int64, humanReadable bool) string { +func getFormattedTime(timestamp int64, humanReadable bool) string { if humanReadable { return time.Unix(timestamp/1000, 0).UTC().String() } return strconv.FormatInt(timestamp, 10) } -func getFormatedBytes(bytes int64, humanReadable bool) string { +func getFormattedBytes(bytes int64, humanReadable bool) string { if humanReadable { return units.Base2Bytes(bytes).String() } @@ -405,7 +397,7 @@ func openBlock(path, blockID string) (*tsdb.DBReadOnly, tsdb.BlockReader, error) } } - b, err := db.Block(blockID) + b, err := db.Block(blockID, tsdb.DefaultPostingsDecoderFactory) if err != nil { return nil, nil, err } @@ -413,13 +405,13 @@ func openBlock(path, blockID string) (*tsdb.DBReadOnly, tsdb.BlockReader, error) return db, b, nil } -func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExtended bool, matchers string) error { +func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExtended bool, matchers string, p parser.Parser) error { var ( selectors []*labels.Matcher err error ) - if len(matchers) > 0 { - selectors, err = parser.ParseMetricSelector(matchers) + if matchers != "" { + selectors, err = p.ParseMetricSelector(matchers) if err != nil { return err } @@ -429,7 +421,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten return err } defer func() { - err = tsdb_errors.NewMulti(err, db.Close()).Err() + err = errors.Join(err, db.Close()) }() meta := block.Meta() @@ -437,7 +429,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten // Presume 1ms resolution that Prometheus uses. fmt.Printf("Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String()) fmt.Printf("Total Series: %d\n", meta.Stats.NumSeries) - if len(matchers) > 0 { + if matchers != "" { fmt.Printf("Matcher: %s\n", matchers) } ir, err := block.Index() @@ -483,24 +475,24 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten labelpairsCount := map[string]uint64{} entries := 0 var ( - p index.Postings - refs []storage.SeriesRef + postings index.Postings + refs []storage.SeriesRef ) - if len(matchers) > 0 { - p, err = tsdb.PostingsForMatchers(ctx, ir, selectors...) + if matchers != "" { + postings, err = tsdb.PostingsForMatchers(ctx, ir, selectors...) if err != nil { return err } // Expand refs first and cache in memory. // So later we don't have to expand again. - refs, err = index.ExpandPostings(p) + refs, err = index.ExpandPostings(postings) if err != nil { return err } fmt.Printf("Matched series: %d\n", len(refs)) - p = index.NewListPostings(refs) + postings = index.NewListPostings(refs) } else { - p, err = ir.Postings(ctx, "", "") // The special all key. + postings, err = ir.Postings(ctx, "", "") // The special all key. if err != nil { return err } @@ -508,8 +500,8 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten chks := []chunks.Meta{} builder := labels.ScratchBuilder{} - for p.Next() { - if err = ir.Series(p.At(), &builder, &chks); err != nil { + for postings.Next() { + if err = ir.Series(postings.At(), &builder, &chks); err != nil { return err } // Amount of the block time range not covered by this series. @@ -522,8 +514,8 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten entries++ }) } - if p.Err() != nil { - return p.Err() + if postings.Err() != nil { + return postings.Err() } fmt.Printf("Postings (unique label pairs): %d\n", len(labelpairsUncovered)) fmt.Printf("Postings entries (total label pairs): %d\n", entries) @@ -554,7 +546,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten postingInfos = postingInfos[:0] for _, n := range allLabelNames { - values, err := ir.SortedLabelValues(ctx, n, selectors...) + values, err := ir.SortedLabelValues(ctx, n, nil, selectors...) if err != nil { return err } @@ -570,7 +562,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten postingInfos = postingInfos[:0] for _, n := range allLabelNames { - lv, err := ir.SortedLabelValues(ctx, n, selectors...) + lv, err := ir.SortedLabelValues(ctx, n, nil, selectors...) if err != nil { return err } @@ -580,7 +572,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten printInfo(postingInfos) postingInfos = postingInfos[:0] - lv, err := ir.SortedLabelValues(ctx, "__name__", selectors...) + lv, err := ir.SortedLabelValues(ctx, "__name__", nil, selectors...) if err != nil { return err } @@ -589,7 +581,10 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten if err != nil { return err } - postings = index.Intersect(postings, index.NewListPostings(refs)) + // Only intersect postings if matchers are specified. + if matchers != "" { + postings = index.Intersect(postings, index.NewListPostings(refs)) + } count := 0 for postings.Next() { count++ @@ -626,7 +621,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb. return err } defer func() { - err = tsdb_errors.NewMulti(err, chunkr.Close()).Err() + err = errors.Join(err, chunkr.Close()) }() totalChunks := 0 @@ -662,7 +657,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb. histogramChunkSize = append(histogramChunkSize, len(chk.Bytes())) fhchk, ok := chk.(*chunkenc.FloatHistogramChunk) if !ok { - return fmt.Errorf("chunk is not FloatHistogramChunk") + return errors.New("chunk is not FloatHistogramChunk") } it := fhchk.Iterator(nil) bucketCount := 0 @@ -677,7 +672,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb. histogramChunkSize = append(histogramChunkSize, len(chk.Bytes())) hchk, ok := chk.(*chunkenc.HistogramChunk) if !ok { - return fmt.Errorf("chunk is not HistogramChunk") + return errors.New("chunk is not HistogramChunk") } it := hchk.Iterator(nil) bucketCount := 0 @@ -708,13 +703,13 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb. type SeriesSetFormatter func(series storage.SeriesSet) error -func dumpSamples(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt int64, match []string, formatter SeriesSetFormatter) (err error) { +func dumpTSDBData(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt int64, match []string, formatter SeriesSetFormatter, p parser.Parser) (err error) { db, err := tsdb.OpenDBReadOnly(dbDir, sandboxDirRoot, nil) if err != nil { return err } defer func() { - err = tsdb_errors.NewMulti(err, db.Close()).Err() + err = errors.Join(err, db.Close()) }() q, err := db.Querier(mint, maxt) if err != nil { @@ -722,7 +717,7 @@ func dumpSamples(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt i } defer q.Close() - matcherSets, err := parser.ParseMetricSelectors(match) + matcherSets, err := p.ParseMetricSelectors(match) if err != nil { return err } @@ -733,7 +728,7 @@ func dumpSamples(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt i for _, mset := range matcherSets { sets = append(sets, q.Select(ctx, true, nil, mset...)) } - ss = storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) + ss = storage.NewMergeSeriesSet(sets, 0, storage.ChainedSeriesMerge) } else { ss = q.Select(ctx, false, nil, matcherSets[0]...) } @@ -744,7 +739,7 @@ func dumpSamples(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt i } if ws := ss.Warnings(); len(ws) > 0 { - return tsdb_errors.NewMulti(ws.AsErrors()...).Err() + return errors.Join(ws.AsErrors()...) } if ss.Err() != nil { @@ -796,12 +791,36 @@ func CondensedString(ls labels.Labels) string { return b.String() } +func formatSeriesSetLabelsToJSON(ss storage.SeriesSet) error { + seriesCache := make(map[string]struct{}) + for ss.Next() { + series := ss.At() + lbs := series.Labels() + + b, err := json.Marshal(lbs) + if err != nil { + return err + } + + if len(b) == 0 { + continue + } + + s := string(b) + if _, ok := seriesCache[s]; !ok { + fmt.Println(s) + seriesCache[s] = struct{}{} + } + } + return nil +} + func formatSeriesSetOpenMetrics(ss storage.SeriesSet) error { for ss.Next() { series := ss.At() lbs := series.Labels() metricName := lbs.Get(labels.MetricName) - lbs = lbs.DropMetricName() + lbs = lbs.DropReserved(func(n string) bool { return n == labels.MetricName }) it := series.Iterator(nil) for it.Next() == chunkenc.ValFloat { ts, val := it.At() @@ -824,17 +843,31 @@ func checkErr(err error) int { } func backfillOpenMetrics(path, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration, customLabels map[string]string) int { - inputFile, err := fileutil.OpenMmapFile(path) + var buf []byte + info, err := os.Stat(path) if err != nil { return checkErr(err) } - defer inputFile.Close() + if info.Mode()&(os.ModeNamedPipe|os.ModeCharDevice) != 0 { + // Read the pipe chunks by chunks as it cannot be mmap-ed + buf, err = os.ReadFile(path) + if err != nil { + return checkErr(err) + } + } else { + inputFile, err := fileutil.OpenMmapFile(path) + if err != nil { + return checkErr(err) + } + defer inputFile.Close() + buf = inputFile.Bytes() + } if err := os.MkdirAll(outputDir, 0o777); err != nil { return checkErr(fmt.Errorf("create output dir: %w", err)) } - return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet, maxBlockDuration, customLabels)) + return checkErr(backfill(5000, buf, outputDir, humanReadable, quiet, maxBlockDuration, customLabels)) } func displayHistogram(dataType string, datas []int, total int) { @@ -877,5 +910,5 @@ func generateBucket(minVal, maxVal int) (start, end, step int) { start = minVal - minVal%step end = maxVal - maxVal%step + step - return + return start, end, step } diff --git a/cmd/promtool/tsdb_posix_test.go b/cmd/promtool/tsdb_posix_test.go new file mode 100644 index 00000000000..9d0034844f7 --- /dev/null +++ b/cmd/promtool/tsdb_posix_test.go @@ -0,0 +1,69 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package main + +import ( + "bytes" + "io" + "math" + "os" + "path" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/tsdb" +) + +func TestTSDBDumpOpenMetricsRoundTripPipe(t *testing.T) { + initialMetrics, err := os.ReadFile("testdata/dump-openmetrics-roundtrip-test.prom") + require.NoError(t, err) + initialMetrics = normalizeNewLine(initialMetrics) + + pipeDir := t.TempDir() + dbDir := t.TempDir() + + // create pipe + pipe := path.Join(pipeDir, "pipe") + err = syscall.Mkfifo(pipe, 0o666) + require.NoError(t, err) + + go func() { + // open pipe to write + in, err := os.OpenFile(pipe, os.O_WRONLY, os.ModeNamedPipe) + require.NoError(t, err) + defer func() { require.NoError(t, in.Close()) }() + _, err = io.Copy(in, bytes.NewReader(initialMetrics)) + require.NoError(t, err) + }() + + // Import samples from OM format + code := backfillOpenMetrics(pipe, dbDir, false, false, 2*time.Hour, map[string]string{}) + require.Equal(t, 0, code) + db, err := tsdb.Open(dbDir, nil, nil, tsdb.DefaultOptions(), nil) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + + // Dump the blocks into OM format + dumpedMetrics := getDumpedSamples(t, dbDir, "", math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics) + + // Should get back the initial metrics. + require.Equal(t, string(initialMetrics), dumpedMetrics) +} diff --git a/cmd/promtool/tsdb_test.go b/cmd/promtool/tsdb_test.go index 90192e31ab9..86d7c67d774 100644 --- a/cmd/promtool/tsdb_test.go +++ b/cmd/promtool/tsdb_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -27,11 +27,13 @@ import ( "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/promql/promqltest" "github.com/prometheus/prometheus/tsdb" ) func TestGenerateBucket(t *testing.T) { + t.Parallel() tcs := []struct { min, max int start, end, step int @@ -62,7 +64,7 @@ func getDumpedSamples(t *testing.T, databasePath, sandboxDirRoot string, mint, m r, w, _ := os.Pipe() os.Stdout = w - err := dumpSamples( + err := dumpTSDBData( context.Background(), databasePath, sandboxDirRoot, @@ -70,6 +72,7 @@ func getDumpedSamples(t *testing.T, databasePath, sandboxDirRoot string, mint, m maxt, match, formatter, + parser.NewParser(parser.Options{}), ) require.NoError(t, err) @@ -96,7 +99,6 @@ func TestTSDBDump(t *testing.T) { heavy_metric{foo="bar"} 5 4 3 2 1 heavy_metric{foo="foo"} 5 4 3 2 1 `) - t.Cleanup(func() { storage.Close() }) tests := []struct { name string @@ -105,13 +107,15 @@ func TestTSDBDump(t *testing.T) { sandboxDirRoot string match []string expectedDump string + expectedSeries string }{ { - name: "default match", - mint: math.MinInt64, - maxt: math.MaxInt64, - match: []string{"{__name__=~'(?s:.*)'}"}, - expectedDump: "testdata/dump-test-1.prom", + name: "default match", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__=~'(?s:.*)'}"}, + expectedDump: "testdata/dump-test-1.prom", + expectedSeries: "testdata/dump-series-1.prom", }, { name: "default match with sandbox dir root set", @@ -120,41 +124,47 @@ func TestTSDBDump(t *testing.T) { sandboxDirRoot: t.TempDir(), match: []string{"{__name__=~'(?s:.*)'}"}, expectedDump: "testdata/dump-test-1.prom", + expectedSeries: "testdata/dump-series-1.prom", }, { - name: "same matcher twice", - mint: math.MinInt64, - maxt: math.MaxInt64, - match: []string{"{foo=~'.+'}", "{foo=~'.+'}"}, - expectedDump: "testdata/dump-test-1.prom", + name: "same matcher twice", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{foo=~'.+'}", "{foo=~'.+'}"}, + expectedDump: "testdata/dump-test-1.prom", + expectedSeries: "testdata/dump-series-1.prom", }, { - name: "no duplication", - mint: math.MinInt64, - maxt: math.MaxInt64, - match: []string{"{__name__=~'(?s:.*)'}", "{baz='abc'}"}, - expectedDump: "testdata/dump-test-1.prom", + name: "no duplication", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__=~'(?s:.*)'}", "{baz='abc'}"}, + expectedDump: "testdata/dump-test-1.prom", + expectedSeries: "testdata/dump-series-1.prom", }, { - name: "well merged", - mint: math.MinInt64, - maxt: math.MaxInt64, - match: []string{"{__name__='heavy_metric'}", "{baz='abc'}"}, - expectedDump: "testdata/dump-test-1.prom", + name: "well merged", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__='heavy_metric'}", "{baz='abc'}"}, + expectedDump: "testdata/dump-test-1.prom", + expectedSeries: "testdata/dump-series-1.prom", }, { - name: "multi matchers", - mint: math.MinInt64, - maxt: math.MaxInt64, - match: []string{"{__name__='heavy_metric',foo='foo'}", "{__name__='metric'}"}, - expectedDump: "testdata/dump-test-2.prom", + name: "multi matchers", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__='heavy_metric',foo='foo'}", "{__name__='metric'}"}, + expectedDump: "testdata/dump-test-2.prom", + expectedSeries: "testdata/dump-series-2.prom", }, { - name: "with reduced mint and maxt", - mint: int64(60000), - maxt: int64(120000), - match: []string{"{__name__='metric'}"}, - expectedDump: "testdata/dump-test-3.prom", + name: "with reduced mint and maxt", + mint: int64(60000), + maxt: int64(120000), + match: []string{"{__name__='metric'}"}, + expectedDump: "testdata/dump-test-3.prom", + expectedSeries: "testdata/dump-series-3.prom", }, } for _, tt := range tests { @@ -165,6 +175,12 @@ func TestTSDBDump(t *testing.T) { expectedMetrics = normalizeNewLine(expectedMetrics) // Sort both, because Prometheus does not guarantee the output order. require.Equal(t, sortLines(string(expectedMetrics)), sortLines(dumpedMetrics)) + + dumpedSeries := getDumpedSamples(t, storage.Dir(), tt.sandboxDirRoot, tt.mint, tt.maxt, tt.match, formatSeriesSetLabelsToJSON) + expectedSeries, err := os.ReadFile(tt.expectedSeries) + require.NoError(t, err) + expectedSeries = normalizeNewLine(expectedSeries) + require.Equal(t, sortLines(string(expectedSeries)), sortLines(dumpedSeries)) }) } } @@ -181,7 +197,6 @@ func TestTSDBDumpOpenMetrics(t *testing.T) { my_counter{foo="bar", baz="abc"} 1 2 3 4 5 my_gauge{bar="foo", abc="baz"} 9 8 0 4 7 `) - t.Cleanup(func() { storage.Close() }) tests := []struct { name string diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 7030635d1c0..d63908efc6a 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,20 +19,21 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" + "slices" "sort" "strconv" "strings" "time" - "github.com/go-kit/log" "github.com/google/go-cmp/cmp" "github.com/grafana/regexp" "github.com/nsf/jsondiff" - "gopkg.in/yaml.v2" - "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -46,11 +47,11 @@ import ( // RulesUnitTest does unit testing of rules based on the unit testing files provided. // More info about the file format can be found in the docs. -func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int { - return RulesUnitTestResult(io.Discard, queryOpts, runStrings, diffFlag, files...) +func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, p parser.Parser, runStrings []string, diffFlag, debug, ignoreUnknownFields bool, files ...string) int { + return RulesUnitTestResult(io.Discard, queryOpts, p, runStrings, diffFlag, debug, ignoreUnknownFields, files...) } -func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int { +func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, p parser.Parser, runStrings []string, diffFlag, debug, ignoreUnknownFields bool, files ...string) int { failed := false junit := &junitxml.JUnitXML{} @@ -60,7 +61,7 @@ func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, } for _, f := range files { - if errs := ruleUnitTest(f, queryOpts, run, diffFlag, junit.Suite(f)); errs != nil { + if errs := ruleUnitTest(f, queryOpts, p, run, diffFlag, debug, ignoreUnknownFields, junit.Suite(f)); errs != nil { fmt.Fprintln(os.Stderr, " FAILED:") for _, e := range errs { fmt.Fprintln(os.Stderr, e.Error()) @@ -82,7 +83,7 @@ func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, return successExitCode } -func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool, ts *junitxml.TestSuite) []error { +func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, p parser.Parser, run *regexp.Regexp, diffFlag, debug, ignoreUnknownFields bool, ts *junitxml.TestSuite) []error { b, err := os.ReadFile(filename) if err != nil { ts.Abort(err) @@ -131,7 +132,8 @@ func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *reg if t.Interval == 0 { t.Interval = unitTestInp.EvaluationInterval } - ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...) + t.parser = p + ers := t.test(testname, evalInterval, groupOrderMap, queryOpts, diffFlag, debug, ignoreUnknownFields, unitTestInp.FuzzyCompare, unitTestInp.RuleFiles...) if ers != nil { for _, e := range ers { tc.Fail(e.Error()) @@ -160,6 +162,7 @@ type unitTestFile struct { EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"` GroupEvalOrder []string `yaml:"group_eval_order"` Tests []testGroup `yaml:"tests"` + FuzzyCompare bool `yaml:"fuzzy_compare,omitempty"` } // resolveAndGlobFilepaths joins all relative paths in a configuration @@ -186,20 +189,53 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error { return nil } +// testStartTimestamp wraps time.Time to support custom YAML unmarshaling. +// It can parse both RFC3339 timestamps and Unix timestamps. +type testStartTimestamp struct { + time.Time +} + +// UnmarshalYAML implements custom YAML unmarshaling for testStartTimestamp. +// It accepts both RFC3339 formatted strings and numeric Unix timestamps. +func (t *testStartTimestamp) UnmarshalYAML(unmarshal func(any) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + parsed, err := parseTime(s) + if err != nil { + return err + } + t.Time = parsed + return nil +} + // testGroup is a group of input series and tests associated with it. type testGroup struct { - Interval model.Duration `yaml:"interval"` - InputSeries []series `yaml:"input_series"` - AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` - PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` - ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` - ExternalURL string `yaml:"external_url,omitempty"` - TestGroupName string `yaml:"name,omitempty"` + Interval model.Duration `yaml:"interval"` + InputSeries []series `yaml:"input_series"` + AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` + PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` + ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` + ExternalURL string `yaml:"external_url,omitempty"` + TestGroupName string `yaml:"name,omitempty"` + StartTimestamp testStartTimestamp `yaml:"start_timestamp,omitempty"` + + parser parser.Parser `yaml:"-"` } // test performs the unit tests. -func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) (outErr []error) { +func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag, debug, ignoreUnknownFields, fuzzyCompare bool, ruleFiles ...string) (outErr []error) { + if debug { + testStart := time.Now() + fmt.Fprintf(os.Stderr, "DEBUG: Starting test %s\n", testname) + defer func() { + fmt.Fprintf(os.Stderr, "DEBUG: Test %s finished, took %v\n", testname, time.Since(testStart)) + }() + } // Setup testing suite. + // Set the start time from the test group. + queryOpts.StartTime = tg.StartTimestamp.Time suite, err := promqltest.NewLazyLoader(tg.seriesLoadingString(), queryOpts) if err != nil { return []error{err} @@ -217,20 +253,34 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i QueryFunc: rules.EngineQueryFunc(suite.QueryEngine(), suite.Storage()), Appendable: suite.Storage(), Context: context.Background(), - NotifyFunc: func(ctx context.Context, expr string, alerts ...*rules.Alert) {}, - Logger: log.NewNopLogger(), + NotifyFunc: func(context.Context, string, ...*rules.Alert) {}, + Logger: promslog.NewNopLogger(), + Parser: tg.parser, } m := rules.NewManager(opts) - groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, tg.ExternalURL, nil, ruleFiles...) + groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, tg.ExternalURL, nil, ignoreUnknownFields, ruleFiles...) if ers != nil { return ers } groups := orderedGroups(groupsMap, groupOrderMap) // Bounds for evaluating the rules. - mint := time.Unix(0, 0).UTC() + var mint time.Time + if tg.StartTimestamp.IsZero() { + mint = time.Unix(0, 0).UTC() + } else { + mint = tg.StartTimestamp.Time + } maxt := mint.Add(tg.maxEvalTime()) + // Optional floating point compare fuzzing. + var compareFloat64 cmp.Option = cmp.Options{} + if fuzzyCompare { + compareFloat64 = cmp.Comparer(func(x, y float64) bool { + return x == y || math.Nextafter(x, math.Inf(-1)) == y || math.Nextafter(x, math.Inf(1)) == y + }) + } + // Pre-processing some data for testing alerts. // All this preparation is so that we can test alerts as we evaluate the rules. // This avoids storing them in memory, as the number of evals might be high. @@ -262,9 +312,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i for k := range alertEvalTimesMap { alertEvalTimes = append(alertEvalTimes, k) } - sort.Slice(alertEvalTimes, func(i, j int) bool { - return alertEvalTimes[i] < alertEvalTimes[j] - }) + slices.Sort(alertEvalTimes) // Current index in alertEvalTimes what we are looking at. curr := 0 @@ -305,12 +353,8 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i return errs } - for { - if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) && - time.Duration(alertEvalTimes[curr]) < ts.Add(evalInterval).Sub(mint)) { - break - } - + for curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) && + time.Duration(alertEvalTimes[curr]) < ts.Add(evalInterval).Sub(mint) { // We need to check alerts for this time. // If 'ts <= `eval_time=alertEvalTimes[curr]` < ts+evalInterval' // then we compare alerts with the Eval at `ts`. @@ -368,7 +412,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i sort.Sort(gotAlerts) sort.Sort(expAlerts) - if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal)) { + if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal), compareFloat64) { var testName string if tg.TestGroupName != "" { testName = fmt.Sprintf(" name: %s,\n", tg.TestGroupName) @@ -442,10 +486,10 @@ Outer: var expSamples []parsedSample for _, s := range testCase.ExpSamples { - lb, err := parser.ParseMetric(s.Labels) + lb, err := tg.parser.ParseMetric(s.Labels) var hist *histogram.FloatHistogram if err == nil && s.Histogram != "" { - _, values, parseErr := parser.ParseSeriesDesc("{} " + s.Histogram) + _, values, parseErr := tg.parser.ParseSeriesDesc("{} " + s.Histogram) switch { case parseErr != nil: err = parseErr @@ -476,12 +520,38 @@ Outer: sort.Slice(gotSamples, func(i, j int) bool { return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0 }) - if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) { + if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal), compareFloat64) { errs = append(errs, fmt.Errorf(" expr: %q, time: %s,\n exp: %v\n got: %v", testCase.Expr, testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples))) } } + if debug { + ts := tg.maxEvalTime() + // Potentially a test can be specified at a time with fractional seconds, + // which PromQL cannot represent, so round up to the next whole second. + ts = (ts + time.Second).Truncate(time.Second) + expr := fmt.Sprintf(`{__name__=~".+"}[%v]`, ts) + q, err := suite.QueryEngine().NewInstantQuery(context.Background(), suite.Queryable(), nil, expr, mint.Add(ts)) + if err != nil { + fmt.Fprintf(os.Stderr, "DEBUG: Failed querying, expr: %q, err: %v\n", expr, err) + return errs + } + res := q.Exec(suite.Context()) + if res.Err != nil { + fmt.Fprintf(os.Stderr, "DEBUG: Failed query exec, expr: %q, err: %v\n", expr, res.Err) + return errs + } + switch v := res.Value.(type) { + case promql.Matrix: + fmt.Fprintf(os.Stderr, "DEBUG: Dump of all data (input_series and rules) at %v:\n", ts) + fmt.Fprintln(os.Stderr, v.String()) + default: + fmt.Fprintf(os.Stderr, "DEBUG: Got unexpected type %T\n", v) + return errs + } + } + if len(errs) > 0 { return errs } @@ -490,11 +560,12 @@ Outer: // seriesLoadingString returns the input series in PromQL notation. func (tg *testGroup) seriesLoadingString() string { - result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval)) + var result strings.Builder + fmt.Fprintf(&result, "load %v\n", shortDuration(tg.Interval)) for _, is := range tg.InputSeries { - result += fmt.Sprintf(" %v %v\n", is.Series, is.Values) + fmt.Fprintf(&result, " %v %v\n", is.Series, is.Values) } - return result + return result.String() } func shortDuration(d model.Duration) string { @@ -593,13 +664,14 @@ func (la labelsAndAnnotations) String() string { if len(la) == 0 { return "[]" } - s := "[\n0:" + indentLines("\n"+la[0].String(), " ") + var s strings.Builder + s.WriteString("[\n0:" + indentLines("\n"+la[0].String(), " ")) for i, l := range la[1:] { - s += ",\n" + strconv.Itoa(i+1) + ":" + indentLines("\n"+l.String(), " ") + s.WriteString(",\n" + strconv.Itoa(i+1) + ":" + indentLines("\n"+l.String(), " ")) } - s += "\n]" + s.WriteString("\n]") - return s + return s.String() } type labelAndAnnotation struct { @@ -650,11 +722,12 @@ func parsedSamplesString(pss []parsedSample) string { if len(pss) == 0 { return "nil" } - s := pss[0].String() + var s strings.Builder + s.WriteString(pss[0].String()) for _, ps := range pss[1:] { - s += ", " + ps.String() + s.WriteString(", " + ps.String()) } - return s + return s.String() } func (ps *parsedSample) String() string { diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go index 9bbac28e9fb..ce317e5e414 100644 --- a/cmd/promtool/unittest_test.go +++ b/cmd/promtool/unittest_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,11 +21,13 @@ import ( "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/promql/promqltest" "github.com/prometheus/prometheus/util/junitxml" ) func TestRulesUnitTest(t *testing.T) { + t.Parallel() type args struct { files []string } @@ -128,6 +130,16 @@ func TestRulesUnitTest(t *testing.T) { }, want: 0, }, + { + name: "Start time tests", + args: args{ + files: []string{"./testdata/start-time-test.yml"}, + }, + queryOpts: promqltest.LazyLoaderOpts{ + EnableAtModifier: true, + }, + want: 0, + }, } reuseFiles := []string{} reuseCount := [2]int{} @@ -141,14 +153,16 @@ func TestRulesUnitTest(t *testing.T) { reuseCount[tt.want] += len(tt.args.files) } t.Run(tt.name, func(t *testing.T) { - if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want { + t.Parallel() + if got := RulesUnitTest(tt.queryOpts, parser.NewParser(parser.Options{}), nil, false, false, false, tt.args.files...); got != tt.want { t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want) } }) } t.Run("Junit xml output ", func(t *testing.T) { + t.Parallel() var buf bytes.Buffer - if got := RulesUnitTestResult(&buf, promqltest.LazyLoaderOpts{}, nil, false, reuseFiles...); got != 1 { + if got := RulesUnitTestResult(&buf, promqltest.LazyLoaderOpts{}, parser.NewParser(parser.Options{}), nil, false, false, false, reuseFiles...); got != 1 { t.Errorf("RulesUnitTestResults() = %v, want 1", got) } var test junitxml.JUnitXML @@ -185,15 +199,17 @@ func TestRulesUnitTest(t *testing.T) { } func TestRulesUnitTestRun(t *testing.T) { + t.Parallel() type args struct { run []string files []string } tests := []struct { - name string - args args - queryOpts promqltest.LazyLoaderOpts - want int + name string + args args + queryOpts promqltest.LazyLoaderOpts + want int + ignoreUnknownFields bool }{ { name: "Test all without run arg", @@ -227,10 +243,42 @@ func TestRulesUnitTestRun(t *testing.T) { }, want: 1, }, + { + name: "Test all with extra fields", + args: args{ + files: []string{"./testdata/rules_run_extrafields.yml"}, + }, + ignoreUnknownFields: true, + want: 0, + }, + { + name: "Test precise floating point comparison expected failure", + args: args{ + files: []string{"./testdata/rules_run_no_fuzzy.yml"}, + }, + want: 1, + }, + { + name: "Test fuzzy floating point comparison correct match", + args: args{ + run: []string{"correct"}, + files: []string{"./testdata/rules_run_fuzzy.yml"}, + }, + want: 0, + }, + { + name: "Test fuzzy floating point comparison wrong match", + args: args{ + run: []string{"wrong"}, + files: []string{"./testdata/rules_run_fuzzy.yml"}, + }, + want: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...) + t.Parallel() + got := RulesUnitTest(tt.queryOpts, parser.NewParser(parser.Options{}), tt.args.run, false, false, tt.ignoreUnknownFields, tt.args.files...) require.Equal(t, tt.want, got) }) } diff --git a/compliance/go.mod b/compliance/go.mod new file mode 100644 index 00000000000..7006b63344e --- /dev/null +++ b/compliance/go.mod @@ -0,0 +1,26 @@ +module compliance + +go 1.25.0 + +require github.com/prometheus/compliance/remotewrite v0.0.0-20260223092825-818283e1171e + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/oklog/run v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang/exp v0.0.0-20260602051030-3537b20ac86b // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.69.0 // indirect + github.com/prometheus/prometheus v0.307.4-0.20251119130332-1174b0ce4f1f // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/stretchr/testify v1.11.1 // indirect + golang.org/x/text v0.38.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/compliance/go.sum b/compliance/go.sum new file mode 100644 index 00000000000..b0f18f5f791 --- /dev/null +++ b/compliance/go.sum @@ -0,0 +1,79 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= +github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang/exp v0.0.0-20260602051030-3537b20ac86b h1:633sracZPrB7O7T6r5skFtwqXDOrXlQkE9Wr5DnYVJE= +github.com/prometheus/client_golang/exp v0.0.0-20260602051030-3537b20ac86b/go.mod h1:7hAEIbflIgnK0HubVroVy6UgJYYKryF6p3mP/dcyay8= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.69.0 h1:OA85nJQS/T/MaYh/Q2CcgDKSGWqNIgrBDvDH85CuiNk= +github.com/prometheus/common v0.69.0/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= +github.com/prometheus/compliance/remotewrite v0.0.0-20260223092825-818283e1171e h1:tT/KBv0aSFq4AElo/bSVvUd+yNKj72hkRsyiKU45nIQ= +github.com/prometheus/compliance/remotewrite v0.0.0-20260223092825-818283e1171e/go.mod h1:VEPZGvpSBbzTKc5acnBj9ng4gfo1DZ4qBsCQnoNFiSc= +github.com/prometheus/prometheus v0.307.4-0.20251119130332-1174b0ce4f1f h1:ERPCnBglv9Z4IjkEBTNbcHmZPlryMldXVWLkk7TeBIY= +github.com/prometheus/prometheus v0.307.4-0.20251119130332-1174b0ce4f1f/go.mod h1:7hcXiGf9AXIKW2ehWWzxkvRYJTGmc2StUIJ8mprfxjg= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/compliance/remote_write_sender_test.go b/compliance/remote_write_sender_test.go new file mode 100644 index 00000000000..38c3bc3531c --- /dev/null +++ b/compliance/remote_write_sender_test.go @@ -0,0 +1,111 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compliance + +import ( + "bytes" + "context" + "fmt" + "html/template" + "os" + "path/filepath" + "testing" + + "github.com/prometheus/compliance/remotewrite/sender" +) + +const ( + scrapeConfigTemplate = ` +global: + scrape_interval: 1s + +remote_write: + - url: "{{.RemoteWriteEndpointURL}}" + protobuf_message: "{{.RemoteWriteMessage}}" + send_exemplars: true + queue_config: + retry_on_http_429: true + metadata_config: + send: true + +scrape_configs: + - job_name: "{{.ScrapeTargetJobName}}" + scrape_interval: 1s + scrape_protocols: + - PrometheusProto + - OpenMetricsText1.0.0 + - PrometheusText0.0.4 + static_configs: + - targets: ["{{.ScrapeTargetHostPort}}"] +` +) + +var scrapeConfigTmpl = template.Must(template.New("config").Parse(scrapeConfigTemplate)) + +type internalPrometheus struct { + agentMode bool +} + +func (internalPrometheus) Name() string { return "internal-prometheus" } + +// Run runs a cmd/prometheus main package as a test sender target, until ctx is done. +func (p internalPrometheus) Run(ctx context.Context, opts sender.Options) error { + var buf bytes.Buffer + if err := scrapeConfigTmpl.Execute(&buf, opts); err != nil { + return fmt.Errorf("failed to execute config template: %w", err) + } + + dir, err := os.MkdirTemp("", "test-*") + if err != nil { + return err + } + configFile := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(configFile, buf.Bytes(), 0o600); err != nil { + return err + } + defer os.RemoveAll(dir) + + args := []string{ + "run", ".", + "--web.listen-address=0.0.0.0:0", + fmt.Sprintf("--config.file=%s", configFile), + } + if p.agentMode { + // Agent mode: st-storage only (agent WAL does not use XOR2 chunks). + args = append(args, "--enable-feature=st-storage") + args = append(args, fmt.Sprintf("--storage.agent.path=%v", dir), "--agent") + } else { + // Server mode: st-storage requires XOR2 chunk encoding so that start + // timestamps can be stored in float chunks. + args = append(args, "--enable-feature=st-storage,xor2-encoding") + args = append(args, fmt.Sprintf("--storage.tsdb.path=%v", dir)) + } + return sender.RunCommand(ctx, "../cmd/prometheus", nil, "go", args...) +} + +var _ sender.Sender = internalPrometheus{} + +// TestRemoteWriteSender runs remote write sender compliance tests defined in +// https://github.com/prometheus/compliance/tree/main/remotewrite/sender against +// both agent and server modes. +func TestRemoteWriteSender(t *testing.T) { + t.Run("mode=server", func(t *testing.T) { + t.Parallel() + sender.RunTests(t, internalPrometheus{}, sender.ComplianceTests()) + }) + t.Run("mode=agent", func(t *testing.T) { + t.Parallel() + sender.RunTests(t, internalPrometheus{agentMode: true}, sender.ComplianceTests()) + }) +} diff --git a/config/config.go b/config/config.go index 4f80b551bc6..cdf914b891c 100644 --- a/config/config.go +++ b/config/config.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,22 +16,25 @@ package config import ( "errors" "fmt" + "log/slog" + "mime" "net/url" "os" "path/filepath" + "slices" "sort" "strconv" "strings" "time" "github.com/alecthomas/units" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/grafana/regexp" + remoteapi "github.com/prometheus/client_golang/exp/api/remote" "github.com/prometheus/common/config" "github.com/prometheus/common/model" - "github.com/prometheus/common/sigv4" - "gopkg.in/yaml.v2" + "github.com/prometheus/otlptranslator" + "github.com/prometheus/sigv4" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/model/labels" @@ -67,13 +70,8 @@ var ( } ) -const ( - LegacyValidationConfig = "legacy" - UTF8ValidationConfig = "utf8" -) - // Load parses the YAML input s into a Config. -func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, error) { +func Load(s string, logger *slog.Logger) (*Config, error) { cfg := &Config{} // If the entire config body is empty the UnmarshalYAML method is // never called. We thus have to set the DefaultConfig at the entry @@ -85,8 +83,11 @@ func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, erro return nil, err } - if !expandExternalLabels { - return cfg, nil + // When the config body is empty, UnmarshalYAML is never called, so + // TSDBConfig may still be nil. + if cfg.StorageConfig.TSDBConfig == nil { + retention := DefaultTSDBRetentionConfig + cfg.StorageConfig.TSDBConfig = &TSDBConfig{Retention: &retention} } b := labels.NewScratchBuilder(0) @@ -98,26 +99,41 @@ func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, erro if v := os.Getenv(s); v != "" { return v } - level.Warn(logger).Log("msg", "Empty environment variable", "name", s) + logger.Warn("Empty environment variable", "name", s) return "" }) if newV != v.Value { - level.Debug(logger).Log("msg", "External label replaced", "label", v.Name, "input", v.Value, "output", newV) + logger.Debug("External label replaced", "label", v.Name, "input", v.Value, "output", newV) } // Note newV can be blank. https://github.com/prometheus/prometheus/issues/11024 b.Add(v.Name, newV) }) - cfg.GlobalConfig.ExternalLabels = b.Labels() + if !b.Labels().IsEmpty() { + cfg.GlobalConfig.ExternalLabels = b.Labels() + } + + switch cfg.OTLPConfig.TranslationStrategy { + case otlptranslator.UnderscoreEscapingWithSuffixes, otlptranslator.UnderscoreEscapingWithoutSuffixes: + case "": + case otlptranslator.NoTranslation, otlptranslator.NoUTF8EscapingWithSuffixes: + if cfg.GlobalConfig.MetricNameValidationScheme == model.LegacyValidation { + return nil, fmt.Errorf("OTLP translation strategy %q is not allowed when UTF8 is disabled", cfg.OTLPConfig.TranslationStrategy) + } + default: + return nil, fmt.Errorf("unsupported OTLP translation strategy %q", cfg.OTLPConfig.TranslationStrategy) + } + cfg.loaded = true return cfg, nil } -// LoadFile parses the given YAML file into a Config. -func LoadFile(filename string, agentMode, expandExternalLabels bool, logger log.Logger) (*Config, error) { +// LoadFile parses and validates the given YAML file into a read-only Config. +// Callers should never write to or shallow copy the returned Config. +func LoadFile(filename string, agentMode bool, logger *slog.Logger) (*Config, error) { content, err := os.ReadFile(filename) if err != nil { return nil, err } - cfg, err := Load(string(content), expandExternalLabels, logger) + cfg, err := Load(string(content), logger) if err != nil { return nil, fmt.Errorf("parsing YAML file %s: %w", filename, err) } @@ -140,11 +156,17 @@ func LoadFile(filename string, agentMode, expandExternalLabels bool, logger log. return cfg, nil } +func boolPtr(b bool) *bool { + return &b +} + // The defaults applied before parsing the respective config sections. var ( // DefaultConfig is the default top-level configuration. DefaultConfig = Config{ GlobalConfig: DefaultGlobalConfig, + Runtime: DefaultRuntimeConfig, + OTLPConfig: DefaultOTLPConfig, } // DefaultGlobalConfig is the default global configuration. @@ -153,26 +175,39 @@ var ( ScrapeTimeout: model.Duration(10 * time.Second), EvaluationInterval: model.Duration(1 * time.Minute), RuleQueryOffset: model.Duration(0 * time.Minute), - // When native histogram feature flag is enabled, ScrapeProtocols default - // changes to DefaultNativeHistogramScrapeProtocols. - ScrapeProtocols: DefaultScrapeProtocols, + // This is nil to be able to distinguish between the case when + // the normal default should be used and the case when a + // new default is needed due to an enabled feature flag. + // E.g. set to `DefaultProtoFirstScrapeProtocols` when + // the feature flag `created-timestamp-zero-ingestion` is set. + ScrapeProtocols: nil, + // When the native histogram feature flag is enabled, + // ScrapeNativeHistograms default changes to true. + ScrapeNativeHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: false, + AlwaysScrapeClassicHistograms: false, + ExtraScrapeMetrics: boolPtr(false), + MetricNameValidationScheme: model.UTF8Validation, + MetricNameEscapingScheme: model.AllowUTF8, } DefaultRuntimeConfig = RuntimeConfig{ // Go runtime tuning. - GoGC: 75, + GoGC: getGoGC(), } - // DefaultScrapeConfig is the default scrape configuration. + // DefaultScrapeConfig is the default scrape configuration. Users of this + // default MUST call Validate() on the config after creation, even if it's + // used unaltered, to check for parameter correctness and fill out default + // values that can't be set inline in this declaration. DefaultScrapeConfig = ScrapeConfig{ - // ScrapeTimeout, ScrapeInterval and ScrapeProtocols default to the configured globals. - ScrapeClassicHistograms: false, - MetricsPath: "/metrics", - Scheme: "http", - HonorLabels: false, - HonorTimestamps: true, - HTTPClientConfig: config.DefaultHTTPClientConfig, - EnableCompression: true, + // ScrapeTimeout, ScrapeInterval, ScrapeProtocols, AlwaysScrapeClassicHistograms, and ConvertClassicHistogramsToNHCB default to the configured globals. + MetricsPath: "/metrics", + Scheme: "http", + HonorLabels: false, + HonorTimestamps: true, + HTTPClientConfig: config.DefaultHTTPClientConfig, + EnableCompression: true, } // DefaultAlertmanagerConfig is the default alertmanager configuration. @@ -183,13 +218,18 @@ var ( HTTPClientConfig: config.DefaultHTTPClientConfig, } + DefaultRemoteWriteHTTPClientConfig = config.HTTPClientConfig{ + FollowRedirects: true, + EnableHTTP2: false, + } + // DefaultRemoteWriteConfig is the default remote write configuration. DefaultRemoteWriteConfig = RemoteWriteConfig{ RemoteTimeout: model.Duration(30 * time.Second), - ProtobufMessage: RemoteWriteProtoMsgV1, + ProtobufMessage: remoteapi.WriteV1MessageType, QueueConfig: DefaultQueueConfig, MetadataConfig: DefaultMetadataConfig, - HTTPClientConfig: config.DefaultHTTPClientConfig, + HTTPClientConfig: DefaultRemoteWriteHTTPClientConfig, } // DefaultQueueConfig is the default remote queue configuration. @@ -236,7 +276,16 @@ var ( } // DefaultOTLPConfig is the default OTLP configuration. - DefaultOTLPConfig = OTLPConfig{} + DefaultOTLPConfig = OTLPConfig{ + TranslationStrategy: otlptranslator.UnderscoreEscapingWithSuffixes, + // For backwards compatibility. + LabelNameUnderscoreSanitization: true, + // For backwards compatibility. + LabelNamePreserveMultipleUnderscores: true, + } + + // DefaultTSDBRetentionConfig is the default TSDB retention configuration. + DefaultTSDBRetentionConfig TSDBRetentionConfig ) // Config is the top-level configuration for Prometheus's config files. @@ -253,9 +302,12 @@ type Config struct { RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"` RemoteReadConfigs []*RemoteReadConfig `yaml:"remote_read,omitempty"` OTLPConfig OTLPConfig `yaml:"otlp,omitempty"` + + loaded bool // Certain methods require configuration to use Load validation. } // SetDirectory joins any relative file paths with dir. +// This method writes to config, and it's not concurrency safe. func (c *Config) SetDirectory(dir string) { c.GlobalConfig.SetDirectory(dir) c.AlertingConfig.SetDirectory(dir) @@ -285,24 +337,26 @@ func (c Config) String() string { return string(b) } -// GetScrapeConfigs returns the scrape configurations. +// GetScrapeConfigs returns the read-only, validated scrape configurations including +// the ones from the scrape_config_files. +// This method does not write to config, and it's concurrency safe (the pointer receiver is for efficiency). +// This method also assumes the Config was created by Load or LoadFile function, it returns error +// if it was not. We can't re-validate or apply globals here due to races, +// read more https://github.com/prometheus/prometheus/issues/15538. func (c *Config) GetScrapeConfigs() ([]*ScrapeConfig, error) { - scfgs := make([]*ScrapeConfig, len(c.ScrapeConfigs)) + if !c.loaded { + // Programmatic error, we warn before more confusing errors would happen due to lack of the globalization. + return nil, errors.New("scrape config cannot be fetched, main config was not validated and loaded correctly; should not happen") + } + scfgs := make([]*ScrapeConfig, len(c.ScrapeConfigs)) jobNames := map[string]string{} for i, scfg := range c.ScrapeConfigs { - // We do these checks for library users that would not call validate in - // Unmarshal. - if err := scfg.Validate(c.GlobalConfig); err != nil { - return nil, err - } - - if _, ok := jobNames[scfg.JobName]; ok { - return nil, fmt.Errorf("found multiple scrape configs with job name %q", scfg.JobName) - } jobNames[scfg.JobName] = "main config file" scfgs[i] = scfg } + + // Re-read and validate the dynamic scrape config rules. for _, pat := range c.ScrapeConfigFiles { fs, err := filepath.Glob(pat) if err != nil { @@ -338,7 +392,8 @@ func (c *Config) GetScrapeConfigs() ([]*ScrapeConfig, error) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { +// NOTE: This method should not be used outside of this package. Use Load or LoadFile instead. +func (c *Config) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultConfig // We want to set c to the defaults and then overwrite it with the input. // To make unmarshal fill the plain data struct rather than calling UnmarshalYAML @@ -358,8 +413,13 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { // We have to restore it here. if c.Runtime.isZero() { c.Runtime = DefaultRuntimeConfig - // Use the GOGC env var value if the runtime section is empty. - c.Runtime.GoGC = getGoGCEnv() + } + + // If no storage.tsdb section is present, TSDBConfig is nil and its + // UnmarshalYAML never runs. Inject the default retention here. + if c.StorageConfig.TSDBConfig == nil { + retention := DefaultTSDBRetentionConfig + c.StorageConfig.TSDBConfig = &TSDBConfig{Retention: &retention} } for _, rf := range c.RuleFiles { @@ -374,18 +434,22 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { } } - // Do global overrides and validate unique names. + // Do global overrides and validation. jobNames := map[string]struct{}{} for _, scfg := range c.ScrapeConfigs { if err := scfg.Validate(c.GlobalConfig); err != nil { return err } - if _, ok := jobNames[scfg.JobName]; ok { return fmt.Errorf("found multiple scrape configs with job name %q", scfg.JobName) } jobNames[scfg.JobName] = struct{}{} } + + if err := c.AlertingConfig.Validate(c.GlobalConfig.MetricNameValidationScheme); err != nil { + return err + } + rwNames := map[string]struct{}{} for _, rwcfg := range c.RemoteWriteConfigs { if rwcfg == nil { @@ -395,6 +459,9 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { if _, ok := rwNames[rwcfg.Name]; ok && rwcfg.Name != "" { return fmt.Errorf("found multiple remote write configs with job name %q", rwcfg.Name) } + if err := rwcfg.Validate(c.GlobalConfig.MetricNameValidationScheme); err != nil { + return err + } rwNames[rwcfg.Name] = struct{}{} } rrNames := map[string]struct{}{} @@ -421,7 +488,7 @@ type GlobalConfig struct { // The protocols to negotiate during a scrape. It tells clients what // protocol are accepted by Prometheus and with what weight (most wanted is first). // Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, - // OpenMetricsText1.0.0, PrometheusText0.0.4. + // OpenMetricsText1.0.0, PrometheusText1.0.0, PrometheusText0.0.4 ScrapeProtocols []ScrapeProtocol `yaml:"scrape_protocols,omitempty"` // How frequently to evaluate rules by default. EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"` @@ -454,8 +521,23 @@ type GlobalConfig struct { // Keep no more than this many dropped targets per job. // 0 means no limit. KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"` - // Allow UTF8 Metric and Label Names. - MetricNameValidationScheme string `yaml:"metric_name_validation_scheme,omitempty"` + // Allow UTF8 Metric and Label Names. Can be blank in config files but must + // have a value if a GlobalConfig is created programmatically. + MetricNameValidationScheme model.ValidationScheme `yaml:"metric_name_validation_scheme,omitempty"` + // Metric name escaping mode to request through content negotiation. Can be + // blank in config files but must have a value if a ScrapeConfig is created + // programmatically. + MetricNameEscapingScheme string `yaml:"metric_name_escaping_scheme,omitempty"` + // Whether to scrape native histograms. + ScrapeNativeHistograms *bool `yaml:"scrape_native_histograms,omitempty"` + // Whether to convert all scraped classic histograms into native histograms with custom buckets. + ConvertClassicHistogramsToNHCB bool `yaml:"convert_classic_histograms_to_nhcb,omitempty"` + // Whether to scrape a classic histogram, even if it is also exposed as a native histogram. + AlwaysScrapeClassicHistograms bool `yaml:"always_scrape_classic_histograms,omitempty"` + // Whether to enable additional scrape metrics. + // When enabled, Prometheus stores samples for scrape_timeout_seconds, + // scrape_sample_limit, and scrape_body_size_bytes. + ExtraScrapeMetrics *bool `yaml:"extra_scrape_metrics,omitempty"` } // ScrapeProtocol represents supported protocol for scraping metrics. @@ -476,9 +558,22 @@ func (s ScrapeProtocol) Validate() error { return nil } +// HeaderMediaType returns the MIME mediaType for a particular ScrapeProtocol. +func (s ScrapeProtocol) HeaderMediaType() string { + if _, ok := ScrapeProtocolsHeaders[s]; !ok { + return "" + } + mediaType, _, err := mime.ParseMediaType(ScrapeProtocolsHeaders[s]) + if err != nil { + return "" + } + return mediaType +} + var ( PrometheusProto ScrapeProtocol = "PrometheusProto" PrometheusText0_0_4 ScrapeProtocol = "PrometheusText0.0.4" + PrometheusText1_0_0 ScrapeProtocol = "PrometheusText1.0.0" OpenMetricsText0_0_1 ScrapeProtocol = "OpenMetricsText0.0.1" OpenMetricsText1_0_0 ScrapeProtocol = "OpenMetricsText1.0.0" UTF8NamesHeader string = model.EscapingKey + "=" + model.AllowUTF8 @@ -486,6 +581,7 @@ var ( ScrapeProtocolsHeaders = map[ScrapeProtocol]string{ PrometheusProto: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", PrometheusText0_0_4: "text/plain;version=0.0.4", + PrometheusText1_0_0: "text/plain;version=1.0.0", OpenMetricsText0_0_1: "application/openmetrics-text;version=0.0.1", OpenMetricsText1_0_0: "application/openmetrics-text;version=1.0.0", } @@ -495,17 +591,19 @@ var ( DefaultScrapeProtocols = []ScrapeProtocol{ OpenMetricsText1_0_0, OpenMetricsText0_0_1, + PrometheusText1_0_0, PrometheusText0_0_4, } // DefaultProtoFirstScrapeProtocols is like DefaultScrapeProtocols, but it // favors protobuf Prometheus exposition format. - // Used by default for certain feature-flags like - // "native-histograms" and "created-timestamp-zero-ingestion". + // Used by default by the "scrape_native_histograms" option and for certain + // feature-flags like "created-timestamp-zero-ingestion". DefaultProtoFirstScrapeProtocols = []ScrapeProtocol{ PrometheusProto, OpenMetricsText1_0_0, OpenMetricsText0_0_1, + PrometheusText1_0_0, PrometheusText0_0_4, } ) @@ -535,7 +633,7 @@ func (c *GlobalConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *GlobalConfig) UnmarshalYAML(unmarshal func(any) error) error { // Create a clean global config as the previous one was already populated // by the default due to the YAML parser behavior for empty blocks. gc := &GlobalConfig{} @@ -544,8 +642,14 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } + switch gc.MetricNameValidationScheme { + case model.UTF8Validation, model.LegacyValidation: + default: + gc.MetricNameValidationScheme = DefaultGlobalConfig.MetricNameValidationScheme + } + if err := gc.ExternalLabels.Validate(func(l labels.Label) error { - if !model.LabelName(l.Name).IsValid() { + if !gc.MetricNameValidationScheme.IsValidLabelName(l.Name) { return fmt.Errorf("%q is not a valid label name", l.Name) } if !model.LabelValue(l.Value).IsValid() { @@ -565,21 +669,34 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return errors.New("global scrape timeout greater than scrape interval") } if gc.ScrapeTimeout == 0 { - if DefaultGlobalConfig.ScrapeTimeout > gc.ScrapeInterval { - gc.ScrapeTimeout = gc.ScrapeInterval - } else { - gc.ScrapeTimeout = DefaultGlobalConfig.ScrapeTimeout - } + gc.ScrapeTimeout = min(DefaultGlobalConfig.ScrapeTimeout, gc.ScrapeInterval) } if gc.EvaluationInterval == 0 { gc.EvaluationInterval = DefaultGlobalConfig.EvaluationInterval } - - if gc.ScrapeProtocols == nil { - gc.ScrapeProtocols = DefaultGlobalConfig.ScrapeProtocols + if gc.ScrapeNativeHistograms == nil { + gc.ScrapeNativeHistograms = DefaultGlobalConfig.ScrapeNativeHistograms + } + if gc.ExtraScrapeMetrics == nil { + gc.ExtraScrapeMetrics = DefaultGlobalConfig.ExtraScrapeMetrics } - if err := validateAcceptScrapeProtocols(gc.ScrapeProtocols); err != nil { - return fmt.Errorf("%w for global config", err) + if gc.ScrapeProtocols == nil { + if DefaultGlobalConfig.ScrapeProtocols != nil { + // This is the case where the defaults are set due to a feature flag. + // E.g. if the created-timestamp-zero-ingestion feature flag is + // used. + gc.ScrapeProtocols = DefaultGlobalConfig.ScrapeProtocols + } + // Otherwise, we leave ScrapeProtocols at nil for now. In the + // per-job scrape config, we have to recognize the unset case to + // correctly set the default depending on the local value of + // ScrapeNativeHistograms. + } + if gc.ScrapeProtocols != nil { + // Only validate if not-nil at this point. + if err := validateAcceptScrapeProtocols(gc.ScrapeProtocols); err != nil { + return fmt.Errorf("%w for global config", err) + } } *c = *gc @@ -595,13 +712,43 @@ func (c *GlobalConfig) isZero() bool { c.RuleQueryOffset == 0 && c.QueryLogFile == "" && c.ScrapeFailureLogFile == "" && - c.ScrapeProtocols == nil + c.ScrapeProtocols == nil && + c.ScrapeNativeHistograms == nil && + !c.ConvertClassicHistogramsToNHCB && + !c.AlwaysScrapeClassicHistograms && + c.BodySizeLimit == 0 && + c.SampleLimit == 0 && + c.TargetLimit == 0 && + c.LabelLimit == 0 && + c.LabelNameLengthLimit == 0 && + c.LabelValueLengthLimit == 0 && + c.KeepDroppedTargets == 0 && + c.MetricNameValidationScheme == model.UnsetValidation && + c.MetricNameEscapingScheme == "" && + c.ExtraScrapeMetrics == nil } +const DefaultGoGCPercentage = 75 + // RuntimeConfig configures the values for the process behavior. type RuntimeConfig struct { // The Go garbage collection target percentage. GoGC int `yaml:"gogc,omitempty"` + + // Below are guidelines for adding a new field: + // + // For config that shouldn't change after startup, you might want to use + // flags https://prometheus.io/docs/prometheus/latest/command-line/prometheus/. + // + // Consider when the new field is first applied: at the very beginning of instance + // startup, after the TSDB is loaded etc. See https://github.com/prometheus/prometheus/pull/16491 + // for an example. + // + // Provide a test covering various scenarios: empty config file, empty or incomplete runtime + // config block, precedence over other inputs (e.g., env vars, if applicable) etc. + // See TestRuntimeGOGCConfig (or https://github.com/prometheus/prometheus/pull/15238). + // The test should also verify behavior on reloads, since this config should be + // adjustable at runtime. } // isZero returns true iff the global config is the zero value. @@ -632,10 +779,19 @@ type ScrapeConfig struct { // The protocols to negotiate during a scrape. It tells clients what // protocol are accepted by Prometheus and with what preference (most wanted is first). // Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, - // OpenMetricsText1.0.0, PrometheusText0.0.4. + // OpenMetricsText1.0.0, PrometheusText1.0.0, PrometheusText0.0.4. ScrapeProtocols []ScrapeProtocol `yaml:"scrape_protocols,omitempty"` - // Whether to scrape a classic histogram that is also exposed as a native histogram. - ScrapeClassicHistograms bool `yaml:"scrape_classic_histograms,omitempty"` + // The fallback protocol to use if the Content-Type provided by the target + // is not provided, blank, or not one of the expected values. + // Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, + // OpenMetricsText1.0.0, PrometheusText1.0.0, PrometheusText0.0.4. + ScrapeFallbackProtocol ScrapeProtocol `yaml:"fallback_scrape_protocol,omitempty"` + // Whether to scrape native histograms. + ScrapeNativeHistograms *bool `yaml:"scrape_native_histograms,omitempty"` + // Whether to scrape a classic histogram, even if it is also exposed as a native histogram. + AlwaysScrapeClassicHistograms *bool `yaml:"always_scrape_classic_histograms,omitempty"` + // Whether to convert all scraped classic histograms into a native histogram with custom buckets. + ConvertClassicHistogramsToNHCB *bool `yaml:"convert_classic_histograms_to_nhcb,omitempty"` // File to which scrape failures are logged. ScrapeFailureLogFile string `yaml:"scrape_failure_log_file,omitempty"` // The HTTP resource path on which to fetch metrics from targets. @@ -671,8 +827,18 @@ type ScrapeConfig struct { // Keep no more than this many dropped targets per job. // 0 means no limit. KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"` - // Allow UTF8 Metric and Label Names. - MetricNameValidationScheme string `yaml:"metric_name_validation_scheme,omitempty"` + // Allow UTF8 Metric and Label Names. Can be blank in config files but must + // have a value if a ScrapeConfig is created programmatically. + MetricNameValidationScheme model.ValidationScheme `yaml:"metric_name_validation_scheme,omitempty"` + // Metric name escaping mode to request through content negotiation. Can be + // blank in config files but must have a value if a ScrapeConfig is created + // programmatically. + MetricNameEscapingScheme string `yaml:"metric_name_escaping_scheme,omitempty"` + // Whether to enable additional scrape metrics. + // When enabled, Prometheus stores samples for scrape_timeout_seconds, + // scrape_sample_limit, and scrape_body_size_bytes. + // If not set (nil), inherits the value from the global configuration. + ExtraScrapeMetrics *bool `yaml:"extra_scrape_metrics,omitempty"` // We cannot do proper Go type embedding below as the parser will then parse // values arbitrarily into the overflow maps of further-down types. @@ -694,12 +860,12 @@ func (c *ScrapeConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultScrapeConfig if err := discovery.UnmarshalYAMLWithInlineConfigs(c, unmarshal); err != nil { return err } - if len(c.JobName) == 0 { + if c.JobName == "" { return errors.New("job_name is empty") } @@ -745,11 +911,7 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error { return fmt.Errorf("scrape timeout greater than scrape interval for scrape config with job name %q", c.JobName) } if c.ScrapeTimeout == 0 { - if globalConfig.ScrapeTimeout > c.ScrapeInterval { - c.ScrapeTimeout = c.ScrapeInterval - } else { - c.ScrapeTimeout = globalConfig.ScrapeTimeout - } + c.ScrapeTimeout = min(globalConfig.ScrapeTimeout, c.ScrapeInterval) } if c.BodySizeLimit == 0 { c.BodySizeLimit = globalConfig.BodySizeLimit @@ -775,41 +937,214 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error { if c.ScrapeFailureLogFile == "" { c.ScrapeFailureLogFile = globalConfig.ScrapeFailureLogFile } + if c.ScrapeNativeHistograms == nil { + c.ScrapeNativeHistograms = globalConfig.ScrapeNativeHistograms + } + if c.ExtraScrapeMetrics == nil { + c.ExtraScrapeMetrics = globalConfig.ExtraScrapeMetrics + } if c.ScrapeProtocols == nil { - c.ScrapeProtocols = globalConfig.ScrapeProtocols + switch { + case globalConfig.ScrapeProtocols != nil: + // global ScrapeProtocols either set explicitly or via a + // default triggered by a feature flag. This overrides + // the selection based on locally active scraping of + // native histograms. + c.ScrapeProtocols = globalConfig.ScrapeProtocols + case c.ScrapeNativeHistogramsEnabled(): + c.ScrapeProtocols = DefaultProtoFirstScrapeProtocols + default: + c.ScrapeProtocols = DefaultScrapeProtocols + } } if err := validateAcceptScrapeProtocols(c.ScrapeProtocols); err != nil { return fmt.Errorf("%w for scrape config with job name %q", err, c.JobName) } - switch globalConfig.MetricNameValidationScheme { - case "", LegacyValidationConfig: - case UTF8ValidationConfig: - if model.NameValidationScheme != model.UTF8Validation { - return fmt.Errorf("utf8 name validation requested but feature not enabled via --enable-feature=utf8-names") + if c.ScrapeFallbackProtocol != "" { + if err := c.ScrapeFallbackProtocol.Validate(); err != nil { + return fmt.Errorf("invalid fallback_scrape_protocol for scrape config with job name %q: %w", c.JobName, err) } + } + + //nolint:staticcheck + if model.NameValidationScheme != model.UTF8Validation { + return errors.New("model.NameValidationScheme must be set to UTF8") + } + + switch globalConfig.MetricNameValidationScheme { + case model.LegacyValidation, model.UTF8Validation: default: - return fmt.Errorf("unknown name validation method specified, must be either 'legacy' or 'utf8', got %s", globalConfig.MetricNameValidationScheme) + return errors.New("global name validation method must be set") } - if c.MetricNameValidationScheme == "" { + // Scrapeconfig validation scheme matches global if left blank. + localValidationUnset := false + switch c.MetricNameValidationScheme { + case model.UnsetValidation: + localValidationUnset = true c.MetricNameValidationScheme = globalConfig.MetricNameValidationScheme + case model.LegacyValidation, model.UTF8Validation: + default: + return fmt.Errorf("unknown scrape config name validation method specified, must be either '', 'legacy' or 'utf8', got %s", c.MetricNameValidationScheme) + } + + // Escaping scheme is based on the validation scheme if left blank. + switch globalConfig.MetricNameEscapingScheme { + case "": + if globalConfig.MetricNameValidationScheme == model.LegacyValidation { + globalConfig.MetricNameEscapingScheme = model.EscapeUnderscores + } else { + globalConfig.MetricNameEscapingScheme = model.AllowUTF8 + } + case model.AllowUTF8, model.EscapeUnderscores, model.EscapeDots, model.EscapeValues: + default: + return fmt.Errorf("unknown global name escaping method specified, must be one of '%s', '%s', '%s', or '%s', got %q", model.AllowUTF8, model.EscapeUnderscores, model.EscapeDots, model.EscapeValues, globalConfig.MetricNameEscapingScheme) + } + + // Similarly, if ScrapeConfig escaping scheme is blank, infer it from the + // ScrapeConfig validation scheme if that was set, or the Global validation + // scheme if the ScrapeConfig validation scheme was also not set. This ensures + // that local ScrapeConfigs that only specify Legacy validation do not inherit + // the global AllowUTF8 escaping setting, which is an error. + if c.MetricNameEscapingScheme == "" { + //nolint:gocritic + if localValidationUnset { + c.MetricNameEscapingScheme = globalConfig.MetricNameEscapingScheme + } else if c.MetricNameValidationScheme == model.LegacyValidation { + c.MetricNameEscapingScheme = model.EscapeUnderscores + } else { + c.MetricNameEscapingScheme = model.AllowUTF8 + } + } + + switch c.MetricNameEscapingScheme { + case model.AllowUTF8: + if c.MetricNameValidationScheme != model.UTF8Validation { + return errors.New("utf8 metric names requested but validation scheme is not set to UTF8") + } + case model.EscapeUnderscores, model.EscapeDots, model.EscapeValues: + default: + return fmt.Errorf("unknown scrape config name escaping method specified, must be one of '%s', '%s', '%s', or '%s', got %q", model.AllowUTF8, model.EscapeUnderscores, model.EscapeDots, model.EscapeValues, c.MetricNameEscapingScheme) + } + + if c.ConvertClassicHistogramsToNHCB == nil { + global := globalConfig.ConvertClassicHistogramsToNHCB + c.ConvertClassicHistogramsToNHCB = &global + } + + if c.AlwaysScrapeClassicHistograms == nil { + global := globalConfig.AlwaysScrapeClassicHistograms + c.AlwaysScrapeClassicHistograms = &global + } + + for _, rc := range c.RelabelConfigs { + if err := rc.Validate(c.MetricNameValidationScheme); err != nil { + return err + } + } + for _, rc := range c.MetricRelabelConfigs { + if err := rc.Validate(c.MetricNameValidationScheme); err != nil { + return err + } } return nil } // MarshalYAML implements the yaml.Marshaler interface. -func (c *ScrapeConfig) MarshalYAML() (interface{}, error) { +func (c *ScrapeConfig) MarshalYAML() (any, error) { return discovery.MarshalYAMLWithInlineConfigs(c) } +// ToEscapingScheme wraps the equivalent common library function with the +// desired default behavior based on the given validation scheme. This is a +// workaround for third party exporters that don't set the escaping scheme. +func ToEscapingScheme(s string, v model.ValidationScheme) (model.EscapingScheme, error) { + if s == "" { + switch v { + case model.UTF8Validation: + return model.NoEscaping, nil + case model.LegacyValidation: + return model.UnderscoreEscaping, nil + case model.UnsetValidation: + return model.NoEscaping, fmt.Errorf("ValidationScheme is unset: %s", v) + default: + panic(fmt.Errorf("unhandled validation scheme: %s", v)) + } + } + return model.ToEscapingScheme(s) +} + +// ScrapeNativeHistogramsEnabled returns whether to scrape native histograms. +func (c *ScrapeConfig) ScrapeNativeHistogramsEnabled() bool { + return c.ScrapeNativeHistograms != nil && *c.ScrapeNativeHistograms +} + +// ConvertClassicHistogramsToNHCBEnabled returns whether to convert classic histograms to NHCB. +func (c *ScrapeConfig) ConvertClassicHistogramsToNHCBEnabled() bool { + return c.ConvertClassicHistogramsToNHCB != nil && *c.ConvertClassicHistogramsToNHCB +} + +// AlwaysScrapeClassicHistogramsEnabled returns whether to always scrape classic histograms. +func (c *ScrapeConfig) AlwaysScrapeClassicHistogramsEnabled() bool { + return c.AlwaysScrapeClassicHistograms != nil && *c.AlwaysScrapeClassicHistograms +} + +// ExtraScrapeMetricsEnabled returns whether to enable extra scrape metrics. +func (c *ScrapeConfig) ExtraScrapeMetricsEnabled() bool { + return c.ExtraScrapeMetrics != nil && *c.ExtraScrapeMetrics +} + // StorageConfig configures runtime reloadable configuration options. type StorageConfig struct { TSDBConfig *TSDBConfig `yaml:"tsdb,omitempty"` ExemplarsConfig *ExemplarsConfig `yaml:"exemplars,omitempty"` } +// TSDBRetentionConfig holds the configuration retention of data in storage storage. +type TSDBRetentionConfig struct { + // How long to retain samples in storage. + Time model.Duration `yaml:"time,omitempty"` + + // Maximum number of bytes that can be stored for blocks. + Size units.Base2Bytes `yaml:"size,omitempty"` + + // Maximum percentage of disk used for TSDB storage. + Percentage float64 `yaml:"percentage,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (t *TSDBRetentionConfig) UnmarshalYAML(unmarshal func(any) error) error { + *t = TSDBRetentionConfig{} + type plain TSDBRetentionConfig + if err := unmarshal((*plain)(t)); err != nil { + return err + } + if t.Size < 0 { + return fmt.Errorf("'storage.tsdb.retention.size' must be greater than or equal to 0, got %v", t.Size) + } + if t.Percentage < 0 || t.Percentage > 100 { + return fmt.Errorf("'storage.tsdb.retention.percentage' must be in the range [0, 100], got %v", t.Percentage) + } + return nil +} + +const ( + // FloatChunkEncodingXOR selects standard XOR encoding for float chunks. + FloatChunkEncodingXOR = "xor" + // FloatChunkEncodingXOR2 selects XOR2 encoding for float chunks; requires --enable-feature=xor2-encoding. + FloatChunkEncodingXOR2 = "xor2" +) + +// ChunkEncodingConfig configures per-chunk-type encoding overrides. +type ChunkEncodingConfig struct { + // Floats selects the encoding used for float chunks. + // Valid values are "xor", "xor2", and "" (empty/absent). When empty, the encoding + // follows the --enable-feature=xor2-encoding flag. Setting "xor2" requires the flag to be enabled. + Floats string `yaml:"floats,omitempty"` +} + // TSDBConfig configures runtime reloadable configuration options. type TSDBConfig struct { // OutOfOrderTimeWindow sets how long back in time an out-of-order sample can be inserted @@ -822,10 +1157,19 @@ type TSDBConfig struct { // During unmarshall, this is converted into milliseconds and stored in OutOfOrderTimeWindow. // This should not be used directly and must be converted into OutOfOrderTimeWindow. OutOfOrderTimeWindowFlag model.Duration `yaml:"out_of_order_time_window,omitempty"` + + // StaleSeriesCompactionThreshold is a number between 0.0-1.0 indicating the % of stale series in + // the in-memory Head block. If the % of stale series crosses this threshold, stale series compaction is run immediately. + StaleSeriesCompactionThreshold float64 `yaml:"stale_series_compaction_threshold,omitempty"` + + // ChunkEncoding configures per-chunk-type encoding overrides. + ChunkEncoding ChunkEncodingConfig `yaml:"chunk_encoding,omitempty"` + + Retention *TSDBRetentionConfig `yaml:"retention,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (t *TSDBConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (t *TSDBConfig) UnmarshalYAML(unmarshal func(any) error) error { *t = TSDBConfig{} type plain TSDBConfig if err := unmarshal((*plain)(t)); err != nil { @@ -834,6 +1178,18 @@ func (t *TSDBConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { t.OutOfOrderTimeWindow = time.Duration(t.OutOfOrderTimeWindowFlag).Milliseconds() + switch t.ChunkEncoding.Floats { + case "", FloatChunkEncodingXOR, FloatChunkEncodingXOR2: + // Valid; no action required. + default: + return fmt.Errorf("'storage.tsdb.chunk_encoding.floats' must be 'xor' or 'xor2', or the field must be omitted entirely, got %q", t.ChunkEncoding.Floats) + } + + if t.Retention == nil { + retention := DefaultTSDBRetentionConfig + t.Retention = &retention + } + return nil } @@ -847,7 +1203,7 @@ const ( ) // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (t *TracingClientType) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (t *TracingClientType) UnmarshalYAML(unmarshal func(any) error) error { *t = TracingClientType("") type plain TracingClientType if err := unmarshal((*plain)(t)); err != nil { @@ -881,7 +1237,7 @@ func (t *TracingConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (t *TracingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (t *TracingConfig) UnmarshalYAML(unmarshal func(any) error) error { *t = TracingConfig{ ClientType: TracingClientGRPC, } @@ -919,6 +1275,20 @@ type AlertingConfig struct { AlertmanagerConfigs AlertmanagerConfigs `yaml:"alertmanagers,omitempty"` } +func (c *AlertingConfig) Validate(nameValidationScheme model.ValidationScheme) error { + for _, rc := range c.AlertRelabelConfigs { + if err := rc.Validate(nameValidationScheme); err != nil { + return err + } + } + for _, rc := range c.AlertmanagerConfigs { + if err := rc.Validate(nameValidationScheme); err != nil { + return err + } + } + return nil +} + // SetDirectory joins any relative file paths with dir. func (c *AlertingConfig) SetDirectory(dir string) { for _, c := range c.AlertmanagerConfigs { @@ -927,7 +1297,7 @@ func (c *AlertingConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *AlertingConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *AlertingConfig) UnmarshalYAML(unmarshal func(any) error) error { // Create a clean global config as the previous one was already populated // by the default due to the YAML parser behavior for empty blocks. *c = AlertingConfig{} @@ -958,23 +1328,22 @@ func (a AlertmanagerConfigs) ToMap() map[string]*AlertmanagerConfig { // AlertmanagerAPIVersion represents a version of the // github.com/prometheus/alertmanager/api, e.g. 'v1' or 'v2'. +// 'v1' is no longer supported. type AlertmanagerAPIVersion string // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (v *AlertmanagerAPIVersion) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (v *AlertmanagerAPIVersion) UnmarshalYAML(unmarshal func(any) error) error { *v = AlertmanagerAPIVersion("") type plain AlertmanagerAPIVersion if err := unmarshal((*plain)(v)); err != nil { return err } - for _, supportedVersion := range SupportedAlertmanagerAPIVersions { - if *v == supportedVersion { - return nil - } + if !slices.Contains(SupportedAlertmanagerAPIVersions, *v) { + return fmt.Errorf("expected Alertmanager api version to be one of %v but got %v", SupportedAlertmanagerAPIVersions, *v) } - return fmt.Errorf("expected Alertmanager api version to be one of %v but got %v", SupportedAlertmanagerAPIVersions, *v) + return nil } const ( @@ -987,7 +1356,7 @@ const ( ) var SupportedAlertmanagerAPIVersions = []AlertmanagerAPIVersion{ - AlertmanagerAPIVersionV1, AlertmanagerAPIVersionV2, + AlertmanagerAPIVersionV2, } // AlertmanagerConfig configures how Alertmanagers can be discovered and communicated with. @@ -1022,7 +1391,7 @@ func (c *AlertmanagerConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultAlertmanagerConfig if err := discovery.UnmarshalYAMLWithInlineConfigs(c, unmarshal); err != nil { return err @@ -1039,7 +1408,7 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil if httpClientConfigAuthEnabled && c.SigV4Config != nil { - return fmt.Errorf("at most one of basic_auth, authorization, oauth2, & sigv4 must be configured") + return errors.New("at most one of basic_auth, authorization, oauth2, & sigv4 must be configured") } // Check for users putting URLs in target groups. @@ -1064,8 +1433,22 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er return nil } +func (c *AlertmanagerConfig) Validate(nameValidationScheme model.ValidationScheme) error { + for _, rc := range c.AlertRelabelConfigs { + if err := rc.Validate(nameValidationScheme); err != nil { + return err + } + } + for _, rc := range c.RelabelConfigs { + if err := rc.Validate(nameValidationScheme); err != nil { + return err + } + } + return nil +} + // MarshalYAML implements the yaml.Marshaler interface. -func (c *AlertmanagerConfig) MarshalYAML() (interface{}, error) { +func (c *AlertmanagerConfig) MarshalYAML() (any, error) { return discovery.MarshalYAMLWithInlineConfigs(c) } @@ -1095,50 +1478,6 @@ func CheckTargetAddress(address model.LabelValue) error { return nil } -// RemoteWriteProtoMsg represents the known protobuf message for the remote write -// 1.0 and 2.0 specs. -type RemoteWriteProtoMsg string - -// Validate returns error if the given reference for the protobuf message is not supported. -func (s RemoteWriteProtoMsg) Validate() error { - switch s { - case RemoteWriteProtoMsgV1, RemoteWriteProtoMsgV2: - return nil - default: - return fmt.Errorf("unknown remote write protobuf message %v, supported: %v", s, RemoteWriteProtoMsgs{RemoteWriteProtoMsgV1, RemoteWriteProtoMsgV2}.String()) - } -} - -type RemoteWriteProtoMsgs []RemoteWriteProtoMsg - -func (m RemoteWriteProtoMsgs) Strings() []string { - ret := make([]string, 0, len(m)) - for _, typ := range m { - ret = append(ret, string(typ)) - } - return ret -} - -func (m RemoteWriteProtoMsgs) String() string { - return strings.Join(m.Strings(), ", ") -} - -var ( - // RemoteWriteProtoMsgV1 represents the `prometheus.WriteRequest` protobuf - // message introduced in the https://prometheus.io/docs/specs/remote_write_spec/, - // which will eventually be deprecated. - // - // NOTE: This string is used for both HTTP header values and config value, so don't change - // this reference. - RemoteWriteProtoMsgV1 RemoteWriteProtoMsg = "prometheus.WriteRequest" - // RemoteWriteProtoMsgV2 represents the `io.prometheus.write.v2.Request` protobuf - // message introduced in https://prometheus.io/docs/specs/remote_write_spec_2_0/ - // - // NOTE: This string is used for both HTTP header values and config value, so don't change - // this reference. - RemoteWriteProtoMsgV2 RemoteWriteProtoMsg = "io.prometheus.write.v2.Request" -) - // RemoteWriteConfig is the configuration for writing to remote storage. type RemoteWriteConfig struct { URL *config.URL `yaml:"url"` @@ -1148,9 +1487,10 @@ type RemoteWriteConfig struct { Name string `yaml:"name,omitempty"` SendExemplars bool `yaml:"send_exemplars,omitempty"` SendNativeHistograms bool `yaml:"send_native_histograms,omitempty"` + RoundRobinDNS bool `yaml:"round_robin_dns,omitempty"` // ProtobufMessage specifies the protobuf message to use against the remote // receiver as specified in https://prometheus.io/docs/specs/remote_write_spec_2_0/ - ProtobufMessage RemoteWriteProtoMsg `yaml:"protobuf_message,omitempty"` + ProtobufMessage remoteapi.WriteMessageType `yaml:"protobuf_message,omitempty"` // We cannot do proper Go type embedding below as the parser will then parse // values arbitrarily into the overflow maps of further-down types. @@ -1168,7 +1508,7 @@ func (c *RemoteWriteConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultRemoteWriteConfig type plain RemoteWriteConfig if err := unmarshal((*plain)(c)); err != nil { @@ -1197,9 +1537,23 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err return err } + if err := c.QueueConfig.Validate(); err != nil { + return err + } + return validateAuthConfigs(c) } +func (c *RemoteWriteConfig) Validate(nameValidationScheme model.ValidationScheme) error { + for _, rc := range c.WriteRelabelConfigs { + if err := rc.Validate(nameValidationScheme); err != nil { + return err + } + } + + return nil +} + // validateAuthConfigs validates that at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured. func validateAuthConfigs(c *RemoteWriteConfig) error { var authConfigured []string @@ -1279,6 +1633,29 @@ type QueueConfig struct { SampleAgeLimit model.Duration `yaml:"sample_age_limit,omitempty"` } +// Validate checks QueueConfig fields for invalid values. +func (c *QueueConfig) Validate() error { + if c.MaxShards <= 0 { + return errors.New("remote write queue max_shards must be positive") + } + if c.MinShards <= 0 { + return errors.New("remote write queue min_shards must be positive") + } + if c.MinShards > c.MaxShards { + return errors.New("remote write queue min_shards must not be greater than max_shards") + } + if c.MaxSamplesPerSend <= 0 { + return errors.New("remote write queue max_samples_per_send must be positive") + } + if c.Capacity <= 0 { + return errors.New("remote write queue capacity must be positive") + } + if c.MaxBackoff < c.MinBackoff { + return errors.New("remote write queue max_backoff must not be less than min_backoff") + } + return nil +} + // MetadataConfig is the configuration for sending metadata to remote // storage. type MetadataConfig struct { @@ -1323,7 +1700,7 @@ func (c *RemoteReadConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *RemoteReadConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *RemoteReadConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultRemoteReadConfig type plain RemoteReadConfig if err := unmarshal((*plain)(c)); err != nil { @@ -1353,7 +1730,7 @@ func fileErr(filename string, err error) error { return fmt.Errorf("%q: %w", filePath(filename), err) } -func getGoGCEnv() int { +func getGoGC() int { goGCEnv := os.Getenv("GOGC") // If the GOGC env var is set, use the same logic as upstream Go. if goGCEnv != "" { @@ -1366,37 +1743,73 @@ func getGoGCEnv() int { return i } } - return DefaultRuntimeConfig.GoGC + return DefaultGoGCPercentage } // OTLPConfig is the configuration for writing to the OTLP endpoint. type OTLPConfig struct { - PromoteResourceAttributes []string `yaml:"promote_resource_attributes,omitempty"` + PromoteAllResourceAttributes bool `yaml:"promote_all_resource_attributes,omitempty"` + PromoteResourceAttributes []string `yaml:"promote_resource_attributes,omitempty"` + IgnoreResourceAttributes []string `yaml:"ignore_resource_attributes,omitempty"` + TranslationStrategy otlptranslator.TranslationStrategyOption `yaml:"translation_strategy,omitempty"` + KeepIdentifyingResourceAttributes bool `yaml:"keep_identifying_resource_attributes,omitempty"` + ConvertHistogramsToNHCB bool `yaml:"convert_histograms_to_nhcb,omitempty"` + // PromoteScopeMetadata controls whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels. + // As per OTel spec, the aforementioned scope metadata should be identifying, i.e. made into metric labels. + PromoteScopeMetadata bool `yaml:"promote_scope_metadata,omitempty"` + // LabelNameUnderscoreSanitization controls whether to enable prepending of 'key_' to labels + // starting with '_'. Reserved labels starting with `__` are not modified. + // This is only relevant when AllowUTF8 is false (i.e., when using underscore escaping). + LabelNameUnderscoreSanitization bool `yaml:"label_name_underscore_sanitization,omitempty"` + // LabelNamePreserveMultipleUnderscores enables preserving of multiple consecutive underscores + // in label names when AllowUTF8 is false. When false, multiple consecutive underscores are + // collapsed to a single underscore during label name sanitization. + LabelNamePreserveMultipleUnderscores bool `yaml:"label_name_preserve_multiple_underscores,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *OTLPConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *OTLPConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultOTLPConfig type plain OTLPConfig if err := unmarshal((*plain)(c)); err != nil { return err } + if c.PromoteAllResourceAttributes { + if len(c.PromoteResourceAttributes) > 0 { + return errors.New("'promote_all_resource_attributes' and 'promote_resource_attributes' cannot be configured simultaneously") + } + if err := sanitizeAttributes(c.IgnoreResourceAttributes, "ignored"); err != nil { + return fmt.Errorf("invalid 'ignore_resource_attributes': %w", err) + } + } else { + if len(c.IgnoreResourceAttributes) > 0 { + return errors.New("'ignore_resource_attributes' cannot be configured unless 'promote_all_resource_attributes' is true") + } + if err := sanitizeAttributes(c.PromoteResourceAttributes, "promoted"); err != nil { + return fmt.Errorf("invalid 'promote_resource_attributes': %w", err) + } + } + + return nil +} + +func sanitizeAttributes(attributes []string, adjective string) error { seen := map[string]struct{}{} var err error - for i, attr := range c.PromoteResourceAttributes { + for i, attr := range attributes { attr = strings.TrimSpace(attr) if attr == "" { - err = errors.Join(err, fmt.Errorf("empty promoted OTel resource attribute")) + err = errors.Join(err, fmt.Errorf("empty %s OTel resource attribute", adjective)) continue } if _, exists := seen[attr]; exists { - err = errors.Join(err, fmt.Errorf("duplicated promoted OTel resource attribute %q", attr)) + err = errors.Join(err, fmt.Errorf("duplicated %s OTel resource attribute %q", adjective, attr)) continue } seen[attr] = struct{}{} - c.PromoteResourceAttributes[i] = attr + attributes[i] = attr } return err } diff --git a/config/config_default_test.go b/config/config_default_test.go index 31133f1e04b..ec7a1128241 100644 --- a/config/config_default_test.go +++ b/config/config_default_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,8 +18,12 @@ package config const ruleFilesConfigFile = "testdata/rules_abs_path.good.yml" var ruleFilesExpectedConf = &Config{ - GlobalConfig: DefaultGlobalConfig, - Runtime: DefaultRuntimeConfig, + loaded: true, + + GlobalConfig: DefaultGlobalConfig, + Runtime: DefaultRuntimeConfig, + OTLPConfig: DefaultOTLPConfig, + StorageConfig: StorageConfig{TSDBConfig: &TSDBConfig{Retention: &TSDBRetentionConfig{}}}, RuleFiles: []string{ "testdata/first.rules", "testdata/rules/second.rules", diff --git a/config/config_test.go b/config/config_test.go index ea1d3b11c99..79b8334d537 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,7 +15,6 @@ package config import ( "crypto/tls" - "encoding/json" "fmt" "net/url" "os" @@ -24,12 +23,16 @@ import ( "time" "github.com/alecthomas/units" - "github.com/go-kit/log" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/grafana/regexp" + remoteapi "github.com/prometheus/client_golang/exp/api/remote" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "github.com/prometheus/otlptranslator" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/aws" @@ -48,9 +51,11 @@ import ( "github.com/prometheus/prometheus/discovery/moby" "github.com/prometheus/prometheus/discovery/nomad" "github.com/prometheus/prometheus/discovery/openstack" + "github.com/prometheus/prometheus/discovery/outscale" "github.com/prometheus/prometheus/discovery/ovhcloud" "github.com/prometheus/prometheus/discovery/puppetdb" "github.com/prometheus/prometheus/discovery/scaleway" + "github.com/prometheus/prometheus/discovery/stackit" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/discovery/triton" "github.com/prometheus/prometheus/discovery/uyuni" @@ -82,6 +87,7 @@ const ( ) var expectedConf = &Config{ + loaded: true, GlobalConfig: GlobalConfig{ ScrapeInterval: model.Duration(15 * time.Second), ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, @@ -91,13 +97,17 @@ var expectedConf = &Config{ ExternalLabels: labels.FromStrings("foo", "bar", "monitor", "codelab"), - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: false, + ConvertClassicHistogramsToNHCB: false, + ExtraScrapeMetrics: boolPtr(false), + MetricNameValidationScheme: model.UTF8Validation, }, Runtime: RuntimeConfig{ @@ -112,16 +122,17 @@ var expectedConf = &Config{ RemoteWriteConfigs: []*RemoteWriteConfig{ { URL: mustParseURL("http://remote1/push"), - ProtobufMessage: RemoteWriteProtoMsgV1, + ProtobufMessage: remoteapi.WriteV1MessageType, RemoteTimeout: model.Duration(30 * time.Second), Name: "drop_expensive", WriteRelabelConfigs: []*relabel.Config{ { - SourceLabels: model.LabelNames{"__name__"}, - Separator: ";", - Regex: relabel.MustNewRegexp("expensive.*"), - Replacement: "$1", - Action: relabel.Drop, + SourceLabels: model.LabelNames{"__name__"}, + Separator: ";", + Regex: relabel.MustNewRegexp("expensive.*"), + Replacement: "$1", + Action: relabel.Drop, + NameValidationScheme: model.UTF8Validation, }, }, QueueConfig: DefaultQueueConfig, @@ -137,12 +148,12 @@ var expectedConf = &Config{ }, }, FollowRedirects: true, - EnableHTTP2: true, + EnableHTTP2: false, }, }, { URL: mustParseURL("http://remote2/push"), - ProtobufMessage: RemoteWriteProtoMsgV2, + ProtobufMessage: remoteapi.WriteV2MessageType, RemoteTimeout: model.Duration(30 * time.Second), QueueConfig: DefaultQueueConfig, MetadataConfig: DefaultMetadataConfig, @@ -153,7 +164,7 @@ var expectedConf = &Config{ KeyFile: filepath.FromSlash("testdata/valid_key_file"), }, FollowRedirects: true, - EnableHTTP2: true, + EnableHTTP2: false, }, Headers: map[string]string{"name": "value"}, }, @@ -163,6 +174,9 @@ var expectedConf = &Config{ PromoteResourceAttributes: []string{ "k8s.cluster.name", "k8s.job.name", "k8s.namespace.name", }, + TranslationStrategy: otlptranslator.UnderscoreEscapingWithSuffixes, + LabelNameUnderscoreSanitization: true, + LabelNamePreserveMultipleUnderscores: true, }, RemoteReadConfigs: []*RemoteReadConfig{ @@ -201,19 +215,26 @@ var expectedConf = &Config{ { JobName: "prometheus", - HonorLabels: true, - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: "testdata/fail_prom.log", + HonorLabels: true, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFallbackProtocol: PrometheusText0_0_4, + ScrapeFailureLogFile: "testdata/fail_prom.log", + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -265,68 +286,80 @@ var expectedConf = &Config{ RelabelConfigs: []*relabel.Config{ { - SourceLabels: model.LabelNames{"job", "__meta_dns_name"}, - TargetLabel: "job", - Separator: ";", - Regex: relabel.MustNewRegexp("(.*)some-[regex]"), - Replacement: "foo-${1}", - Action: relabel.Replace, + SourceLabels: model.LabelNames{"job", "__meta_dns_name"}, + TargetLabel: "job", + Separator: ";", + Regex: relabel.MustNewRegexp("(.*)some-[regex]"), + Replacement: "foo-${1}", + Action: relabel.Replace, + NameValidationScheme: model.UTF8Validation, }, { - SourceLabels: model.LabelNames{"abc"}, - TargetLabel: "cde", - Separator: ";", - Regex: relabel.DefaultRelabelConfig.Regex, - Replacement: relabel.DefaultRelabelConfig.Replacement, - Action: relabel.Replace, + SourceLabels: model.LabelNames{"abc"}, + TargetLabel: "cde", + Separator: ";", + Regex: relabel.DefaultRelabelConfig.Regex, + Replacement: relabel.DefaultRelabelConfig.Replacement, + Action: relabel.Replace, + NameValidationScheme: model.UTF8Validation, }, { - TargetLabel: "abc", - Separator: ";", - Regex: relabel.DefaultRelabelConfig.Regex, - Replacement: "static", - Action: relabel.Replace, + TargetLabel: "abc", + Separator: ";", + Regex: relabel.DefaultRelabelConfig.Regex, + Replacement: "static", + Action: relabel.Replace, + NameValidationScheme: model.UTF8Validation, }, { - TargetLabel: "abc", - Separator: ";", - Regex: relabel.MustNewRegexp(""), - Replacement: "static", - Action: relabel.Replace, + TargetLabel: "abc", + Separator: ";", + Regex: relabel.MustNewRegexp(""), + Replacement: "static", + Action: relabel.Replace, + NameValidationScheme: model.UTF8Validation, }, { - SourceLabels: model.LabelNames{"foo"}, - TargetLabel: "abc", - Action: relabel.KeepEqual, - Regex: relabel.DefaultRelabelConfig.Regex, - Replacement: relabel.DefaultRelabelConfig.Replacement, - Separator: relabel.DefaultRelabelConfig.Separator, + SourceLabels: model.LabelNames{"foo"}, + TargetLabel: "abc", + Action: relabel.KeepEqual, + Regex: relabel.DefaultRelabelConfig.Regex, + Replacement: relabel.DefaultRelabelConfig.Replacement, + Separator: relabel.DefaultRelabelConfig.Separator, + NameValidationScheme: model.UTF8Validation, }, { - SourceLabels: model.LabelNames{"foo"}, - TargetLabel: "abc", - Action: relabel.DropEqual, - Regex: relabel.DefaultRelabelConfig.Regex, - Replacement: relabel.DefaultRelabelConfig.Replacement, - Separator: relabel.DefaultRelabelConfig.Separator, + SourceLabels: model.LabelNames{"foo"}, + TargetLabel: "abc", + Action: relabel.DropEqual, + Regex: relabel.DefaultRelabelConfig.Regex, + Replacement: relabel.DefaultRelabelConfig.Replacement, + Separator: relabel.DefaultRelabelConfig.Separator, + NameValidationScheme: model.UTF8Validation, }, }, }, { JobName: "service-x", - HonorTimestamps: true, - ScrapeInterval: model.Duration(50 * time.Second), - ScrapeTimeout: model.Duration(5 * time.Second), - EnableCompression: true, - BodySizeLimit: 10 * units.MiB, - SampleLimit: 1000, - TargetLimit: 35, - LabelLimit: 35, - LabelNameLengthLimit: 210, - LabelValueLengthLimit: 210, - ScrapeProtocols: []ScrapeProtocol{PrometheusText0_0_4}, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(50 * time.Second), + ScrapeTimeout: model.Duration(5 * time.Second), + EnableCompression: true, + BodySizeLimit: 10 * units.MiB, + SampleLimit: 1000, + TargetLimit: 35, + LabelLimit: 35, + LabelNameLengthLimit: 210, + LabelValueLengthLimit: 210, + ScrapeProtocols: []ScrapeProtocol{PrometheusText0_0_4}, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), HTTPClientConfig: config.HTTPClientConfig{ BasicAuth: &config.BasicAuth{ @@ -359,72 +392,85 @@ var expectedConf = &Config{ RelabelConfigs: []*relabel.Config{ { - SourceLabels: model.LabelNames{"job"}, - Regex: relabel.MustNewRegexp("(.*)some-[regex]"), - Separator: ";", - Replacement: relabel.DefaultRelabelConfig.Replacement, - Action: relabel.Drop, + SourceLabels: model.LabelNames{"job"}, + Regex: relabel.MustNewRegexp("(.*)some-[regex]"), + Separator: ";", + Replacement: relabel.DefaultRelabelConfig.Replacement, + Action: relabel.Drop, + NameValidationScheme: model.UTF8Validation, }, { - SourceLabels: model.LabelNames{"__address__"}, - TargetLabel: "__tmp_hash", - Regex: relabel.DefaultRelabelConfig.Regex, - Replacement: relabel.DefaultRelabelConfig.Replacement, - Modulus: 8, - Separator: ";", - Action: relabel.HashMod, + SourceLabels: model.LabelNames{"__address__"}, + TargetLabel: "__tmp_hash", + Regex: relabel.DefaultRelabelConfig.Regex, + Replacement: relabel.DefaultRelabelConfig.Replacement, + Modulus: 8, + Separator: ";", + Action: relabel.HashMod, + NameValidationScheme: model.UTF8Validation, }, { - SourceLabels: model.LabelNames{"__tmp_hash"}, - Regex: relabel.MustNewRegexp("1"), - Separator: ";", - Replacement: relabel.DefaultRelabelConfig.Replacement, - Action: relabel.Keep, + SourceLabels: model.LabelNames{"__tmp_hash"}, + Regex: relabel.MustNewRegexp("1"), + Separator: ";", + Replacement: relabel.DefaultRelabelConfig.Replacement, + Action: relabel.Keep, + NameValidationScheme: model.UTF8Validation, }, { - Regex: relabel.MustNewRegexp("1"), - Separator: ";", - Replacement: relabel.DefaultRelabelConfig.Replacement, - Action: relabel.LabelMap, + Regex: relabel.MustNewRegexp("1"), + Separator: ";", + Replacement: relabel.DefaultRelabelConfig.Replacement, + Action: relabel.LabelMap, + NameValidationScheme: model.UTF8Validation, }, { - Regex: relabel.MustNewRegexp("d"), - Separator: ";", - Replacement: relabel.DefaultRelabelConfig.Replacement, - Action: relabel.LabelDrop, + Regex: relabel.MustNewRegexp("d"), + Separator: ";", + Replacement: relabel.DefaultRelabelConfig.Replacement, + Action: relabel.LabelDrop, + NameValidationScheme: model.UTF8Validation, }, { - Regex: relabel.MustNewRegexp("k"), - Separator: ";", - Replacement: relabel.DefaultRelabelConfig.Replacement, - Action: relabel.LabelKeep, + Regex: relabel.MustNewRegexp("k"), + Separator: ";", + Replacement: relabel.DefaultRelabelConfig.Replacement, + Action: relabel.LabelKeep, + NameValidationScheme: model.UTF8Validation, }, }, MetricRelabelConfigs: []*relabel.Config{ { - SourceLabels: model.LabelNames{"__name__"}, - Regex: relabel.MustNewRegexp("expensive_metric.*"), - Separator: ";", - Replacement: relabel.DefaultRelabelConfig.Replacement, - Action: relabel.Drop, + SourceLabels: model.LabelNames{"__name__"}, + Regex: relabel.MustNewRegexp("expensive_metric.*"), + Separator: ";", + Replacement: relabel.DefaultRelabelConfig.Replacement, + Action: relabel.Drop, + NameValidationScheme: model.UTF8Validation, }, }, }, { JobName: "service-y", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -436,6 +482,7 @@ var expectedConf = &Config{ PathPrefix: "/consul", Token: "mysecret", Services: []string{"nginx", "cache", "mysql"}, + HealthFilter: `Service.Tags contains "canary"`, ServiceTags: []string{"canary", "v1"}, NodeMeta: map[string]string{"rack": "123"}, TagSeparator: consul.DefaultSDConfig.TagSeparator, @@ -457,30 +504,37 @@ var expectedConf = &Config{ RelabelConfigs: []*relabel.Config{ { - SourceLabels: model.LabelNames{"__meta_sd_consul_tags"}, - Regex: relabel.MustNewRegexp("label:([^=]+)=([^,]+)"), - Separator: ",", - TargetLabel: "${1}", - Replacement: "${2}", - Action: relabel.Replace, + SourceLabels: model.LabelNames{"__meta_sd_consul_tags"}, + Regex: relabel.MustNewRegexp("label:([^=]+)=([^,]+)"), + Separator: ",", + TargetLabel: "${1}", + Replacement: "${2}", + Action: relabel.Replace, + NameValidationScheme: model.UTF8Validation, }, }, }, { JobName: "service-z", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: model.Duration(10 * time.Second), - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: model.Duration(10 * time.Second), + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: "/metrics", Scheme: "http", @@ -503,18 +557,24 @@ var expectedConf = &Config{ { JobName: "service-kubernetes", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -543,18 +603,24 @@ var expectedConf = &Config{ { JobName: "service-kubernetes-namespaces", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -583,18 +649,24 @@ var expectedConf = &Config{ { JobName: "service-kuma", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -613,18 +685,24 @@ var expectedConf = &Config{ { JobName: "service-marathon", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -651,18 +729,24 @@ var expectedConf = &Config{ { JobName: "service-nomad", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -686,18 +770,24 @@ var expectedConf = &Config{ { JobName: "service-ec2", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -711,7 +801,7 @@ var expectedConf = &Config{ Profile: "profile", RefreshInterval: model.Duration(60 * time.Second), Port: 80, - Filters: []*aws.EC2Filter{ + Filters: []*aws.Filter{ { Name: "tag:environment", Values: []string{"prod"}, @@ -728,18 +818,24 @@ var expectedConf = &Config{ { JobName: "service-lightsail", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -760,18 +856,24 @@ var expectedConf = &Config{ { JobName: "service-azure", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -795,18 +897,24 @@ var expectedConf = &Config{ { JobName: "service-nerve", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -823,18 +931,24 @@ var expectedConf = &Config{ { JobName: "0123service-xxx", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -854,18 +968,24 @@ var expectedConf = &Config{ { JobName: "badfederation", - HonorTimestamps: false, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: false, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: "/federate", Scheme: DefaultScrapeConfig.Scheme, @@ -885,18 +1005,24 @@ var expectedConf = &Config{ { JobName: "測試", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -916,18 +1042,24 @@ var expectedConf = &Config{ { JobName: "httpsd", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -944,18 +1076,24 @@ var expectedConf = &Config{ { JobName: "service-triton", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -980,18 +1118,24 @@ var expectedConf = &Config{ { JobName: "digitalocean-droplets", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1009,24 +1153,31 @@ var expectedConf = &Config{ }, Port: 80, RefreshInterval: model.Duration(60 * time.Second), + Role: "droplets", }, }, }, { JobName: "docker", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1047,18 +1198,24 @@ var expectedConf = &Config{ { JobName: "dockerswarm", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1078,18 +1235,24 @@ var expectedConf = &Config{ { JobName: "service-openstack", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1113,18 +1276,24 @@ var expectedConf = &Config{ { JobName: "service-puppetdb", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1150,19 +1319,25 @@ var expectedConf = &Config{ }, }, { - JobName: "hetzner", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + JobName: "hetzner", + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultProtoFirstScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(true), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1170,12 +1345,13 @@ var expectedConf = &Config{ RelabelConfigs: []*relabel.Config{ { - Action: relabel.Uppercase, - Regex: relabel.DefaultRelabelConfig.Regex, - Replacement: relabel.DefaultRelabelConfig.Replacement, - Separator: relabel.DefaultRelabelConfig.Separator, - SourceLabels: model.LabelNames{"instance"}, - TargetLabel: "instance", + Action: relabel.Uppercase, + Regex: relabel.DefaultRelabelConfig.Regex, + Replacement: relabel.DefaultRelabelConfig.Replacement, + Separator: relabel.DefaultRelabelConfig.Separator, + SourceLabels: model.LabelNames{"instance"}, + TargetLabel: "instance", + NameValidationScheme: model.UTF8Validation, }, }, @@ -1208,18 +1384,24 @@ var expectedConf = &Config{ { JobName: "service-eureka", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1236,18 +1418,24 @@ var expectedConf = &Config{ { JobName: "ovhcloud", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), HTTPClientConfig: config.DefaultHTTPClientConfig, MetricsPath: DefaultScrapeConfig.MetricsPath, @@ -1275,18 +1463,24 @@ var expectedConf = &Config{ { JobName: "scaleway", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), HTTPClientConfig: config.DefaultHTTPClientConfig, MetricsPath: DefaultScrapeConfig.MetricsPath, @@ -1320,18 +1514,24 @@ var expectedConf = &Config{ { JobName: "linode-instances", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1353,21 +1553,70 @@ var expectedConf = &Config{ }, }, }, + { + JobName: "stackit-servers", + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + HTTPClientConfig: config.DefaultHTTPClientConfig, + ServiceDiscoveryConfigs: discovery.Configs{ + &stackit.SDConfig{ + Project: "11111111-1111-1111-1111-111111111111", + ServiceAccountKey: "mysecret_sa_key", + PrivateKey: "mysecret_private_key", + Region: "eu01", + HTTPClientConfig: config.HTTPClientConfig{ + Authorization: &config.Authorization{ + Type: "Bearer", + Credentials: "abcdef", + }, + FollowRedirects: true, + EnableHTTP2: true, + }, + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + }, + }, + }, { JobName: "uyuni", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), HTTPClientConfig: config.DefaultHTTPClientConfig, MetricsPath: DefaultScrapeConfig.MetricsPath, @@ -1385,19 +1634,26 @@ var expectedConf = &Config{ }, }, { - JobName: "ionos", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + JobName: "ionos", + + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1419,18 +1675,24 @@ var expectedConf = &Config{ { JobName: "vultr", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - EnableCompression: true, - BodySizeLimit: globBodySizeLimit, - SampleLimit: globSampleLimit, - TargetLimit: globTargetLimit, - LabelLimit: globLabelLimit, - LabelNameLengthLimit: globLabelNameLengthLimit, - LabelValueLengthLimit: globLabelValueLengthLimit, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, - ScrapeFailureLogFile: globScrapeFailureLogFile, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -1451,6 +1713,44 @@ var expectedConf = &Config{ }, }, }, + { + JobName: "outscale", + + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: DefaultGlobalConfig.MetricNameValidationScheme, + MetricNameEscapingScheme: DefaultGlobalConfig.MetricNameEscapingScheme, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + HTTPClientConfig: config.DefaultHTTPClientConfig, + + ServiceDiscoveryConfigs: discovery.Configs{ + &outscale.SDConfig{ + Endpoint: "https://api.eu-west-2.outscale.com/api/v1", + Region: "eu-west-2", + AccessKey: "A1B2C3D4E5F6G7H8I9J0", + SecretKey: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, + }, + }, + }, }, AlertingConfig: AlertingConfig{ AlertmanagerConfigs: []*AlertmanagerConfig{ @@ -1476,8 +1776,14 @@ var expectedConf = &Config{ }, StorageConfig: StorageConfig{ TSDBConfig: &TSDBConfig{ - OutOfOrderTimeWindow: 30 * time.Minute.Milliseconds(), - OutOfOrderTimeWindowFlag: model.Duration(30 * time.Minute), + OutOfOrderTimeWindow: 30 * time.Minute.Milliseconds(), + OutOfOrderTimeWindowFlag: model.Duration(30 * time.Minute), + StaleSeriesCompactionThreshold: 0.5, + Retention: &TSDBRetentionConfig{ + Time: model.Duration(24 * time.Hour), + Size: 1 * units.GiB, + Percentage: 28, + }, }, }, TracingConfig: TracingConfig{ @@ -1495,36 +1801,41 @@ var expectedConf = &Config{ }, } +func TestYAMLNotLongerSupportedAMApi(t *testing.T) { + _, err := LoadFile("testdata/config_with_no_longer_supported_am_api_config.yml", false, promslog.NewNopLogger()) + require.Error(t, err) +} + func TestYAMLRoundtrip(t *testing.T) { - want, err := LoadFile("testdata/roundtrip.good.yml", false, false, log.NewNopLogger()) + want, err := LoadFile("testdata/roundtrip.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) + require.NoError(t, err) + got, err := Load(string(out), promslog.NewNopLogger()) require.NoError(t, err) - got := &Config{} - require.NoError(t, yaml.UnmarshalStrict(out, got)) require.Equal(t, want, got) } func TestRemoteWriteRetryOnRateLimit(t *testing.T) { - want, err := LoadFile("testdata/remote_write_retry_on_rate_limit.good.yml", false, false, log.NewNopLogger()) + want, err := LoadFile("testdata/remote_write_retry_on_rate_limit.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) + require.NoError(t, err) + got, err := Load(string(out), promslog.NewNopLogger()) require.NoError(t, err) - got := &Config{} - require.NoError(t, yaml.UnmarshalStrict(out, got)) require.True(t, got.RemoteWriteConfigs[0].QueueConfig.RetryOnRateLimit) require.False(t, got.RemoteWriteConfigs[1].QueueConfig.RetryOnRateLimit) } func TestOTLPSanitizeResourceAttributes(t *testing.T) { - t.Run("good config", func(t *testing.T) { - want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.good.yml"), false, false, log.NewNopLogger()) + t.Run("good config - default resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_default_resource_attributes.good.yml"), false, promslog.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) @@ -1532,29 +1843,303 @@ func TestOTLPSanitizeResourceAttributes(t *testing.T) { var got Config require.NoError(t, yaml.UnmarshalStrict(out, &got)) + require.False(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes) + require.Empty(t, got.OTLPConfig.PromoteResourceAttributes) + }) + + t.Run("good config - promote resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_promote_resource_attributes.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + + require.False(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes) require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.PromoteResourceAttributes) }) - t.Run("bad config", func(t *testing.T) { - _, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.bad.yml"), false, false, log.NewNopLogger()) + t.Run("bad config - promote resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_promote_resource_attributes.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `invalid 'promote_resource_attributes'`) require.ErrorContains(t, err, `duplicated promoted OTel resource attribute "k8s.job.name"`) require.ErrorContains(t, err, `empty promoted OTel resource attribute`) }) + + t.Run("good config - promote all resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes_promote_all.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + require.True(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.PromoteResourceAttributes) + require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes) + }) + + t.Run("good config - ignore resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_ignore_resource_attributes.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + require.True(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.PromoteResourceAttributes) + require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.IgnoreResourceAttributes) + }) + + t.Run("bad config - ignore resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_ignore_resource_attributes.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `invalid 'ignore_resource_attributes'`) + require.ErrorContains(t, err, `duplicated ignored OTel resource attribute "k8s.job.name"`) + require.ErrorContains(t, err, `empty ignored OTel resource attribute`) + }) + + t.Run("bad config - conflict between promote all and promote specific resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_promote_all_resource_attributes.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `'promote_all_resource_attributes' and 'promote_resource_attributes' cannot be configured simultaneously`) + }) + + t.Run("bad config - configuring ignoring of resource attributes without also enabling promotion of all resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_ignore_resource_attributes_without_promote_all.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `'ignore_resource_attributes' cannot be configured unless 'promote_all_resource_attributes' is true`) + }) +} + +func TestOTLPAllowServiceNameInTargetInfo(t *testing.T) { + t.Run("good config", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_allow_keep_identifying_resource_attributes.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + + require.True(t, got.OTLPConfig.KeepIdentifyingResourceAttributes) + }) +} + +func TestOTLPConvertHistogramsToNHCB(t *testing.T) { + t.Run("good config", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_convert_histograms_to_nhcb.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + + require.True(t, got.OTLPConfig.ConvertHistogramsToNHCB) + }) +} + +func TestOTLPPromoteScopeMetadata(t *testing.T) { + t.Run("good config", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_promote_scope_metadata.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + + require.True(t, got.OTLPConfig.PromoteScopeMetadata) + }) +} + +func TestOTLPLabelUnderscoreSanitization(t *testing.T) { + t.Run("defaults to true", func(t *testing.T) { + conf, err := LoadFile(filepath.Join("testdata", "otlp_label_underscore_sanitization_defaults.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + // Test that default values are true + require.True(t, conf.OTLPConfig.LabelNameUnderscoreSanitization) + require.True(t, conf.OTLPConfig.LabelNamePreserveMultipleUnderscores) + }) + + t.Run("explicitly enabled", func(t *testing.T) { + conf, err := LoadFile(filepath.Join("testdata", "otlp_label_underscore_sanitization_enabled.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(conf) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + + require.True(t, got.OTLPConfig.LabelNameUnderscoreSanitization) + require.True(t, got.OTLPConfig.LabelNamePreserveMultipleUnderscores) + }) + + t.Run("explicitly disabled", func(t *testing.T) { + conf, err := LoadFile(filepath.Join("testdata", "otlp_label_underscore_sanitization_disabled.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + // When explicitly set to false, they should be false + require.False(t, conf.OTLPConfig.LabelNameUnderscoreSanitization) + require.False(t, conf.OTLPConfig.LabelNamePreserveMultipleUnderscores) + }) + + t.Run("empty config uses defaults", func(t *testing.T) { + conf, err := LoadFile(filepath.Join("testdata", "otlp_empty.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + // Empty config should use default values (true) + require.True(t, conf.OTLPConfig.LabelNameUnderscoreSanitization) + require.True(t, conf.OTLPConfig.LabelNamePreserveMultipleUnderscores) + }) +} + +func TestOTLPAllowUTF8(t *testing.T) { + t.Run("good config - NoUTF8EscapingWithSuffixes", func(t *testing.T) { + fpath := filepath.Join("testdata", "otlp_allow_utf8.good.yml") + verify := func(t *testing.T, conf *Config, err error) { + t.Helper() + require.NoError(t, err) + require.Equal(t, otlptranslator.NoUTF8EscapingWithSuffixes, conf.OTLPConfig.TranslationStrategy) + } + + t.Run("LoadFile", func(t *testing.T) { + conf, err := LoadFile(fpath, false, promslog.NewNopLogger()) + verify(t, conf, err) + }) + t.Run("Load", func(t *testing.T) { + content, err := os.ReadFile(fpath) + require.NoError(t, err) + conf, err := Load(string(content), promslog.NewNopLogger()) + verify(t, conf, err) + }) + }) + + t.Run("incompatible config - NoUTF8EscapingWithSuffixes", func(t *testing.T) { + fpath := filepath.Join("testdata", "otlp_allow_utf8.incompatible.yml") + verify := func(t *testing.T, err error) { + t.Helper() + require.ErrorContains(t, err, `OTLP translation strategy "NoUTF8EscapingWithSuffixes" is not allowed when UTF8 is disabled`) + } + + t.Run("LoadFile", func(t *testing.T) { + _, err := LoadFile(fpath, false, promslog.NewNopLogger()) + verify(t, err) + }) + t.Run("Load", func(t *testing.T) { + content, err := os.ReadFile(fpath) + require.NoError(t, err) + _, err = Load(string(content), promslog.NewNopLogger()) + t.Log("err", err) + verify(t, err) + }) + }) + + t.Run("good config - NoTranslation", func(t *testing.T) { + fpath := filepath.Join("testdata", "otlp_no_translation.good.yml") + verify := func(t *testing.T, conf *Config, err error) { + t.Helper() + require.NoError(t, err) + require.Equal(t, otlptranslator.NoTranslation, conf.OTLPConfig.TranslationStrategy) + } + + t.Run("LoadFile", func(t *testing.T) { + conf, err := LoadFile(fpath, false, promslog.NewNopLogger()) + verify(t, conf, err) + }) + t.Run("Load", func(t *testing.T) { + content, err := os.ReadFile(fpath) + require.NoError(t, err) + conf, err := Load(string(content), promslog.NewNopLogger()) + verify(t, conf, err) + }) + }) + + t.Run("incompatible config - NoTranslation", func(t *testing.T) { + fpath := filepath.Join("testdata", "otlp_no_translation.incompatible.yml") + verify := func(t *testing.T, err error) { + t.Helper() + require.ErrorContains(t, err, `OTLP translation strategy "NoTranslation" is not allowed when UTF8 is disabled`) + } + + t.Run("LoadFile", func(t *testing.T) { + _, err := LoadFile(fpath, false, promslog.NewNopLogger()) + verify(t, err) + }) + t.Run("Load", func(t *testing.T) { + content, err := os.ReadFile(fpath) + require.NoError(t, err) + _, err = Load(string(content), promslog.NewNopLogger()) + t.Log("err", err) + verify(t, err) + }) + }) + + t.Run("bad config", func(t *testing.T) { + fpath := filepath.Join("testdata", "otlp_allow_utf8.bad.yml") + verify := func(t *testing.T, err error) { + t.Helper() + require.ErrorContains(t, err, `unsupported OTLP translation strategy "Invalid"`) + } + + t.Run("LoadFile", func(t *testing.T) { + _, err := LoadFile(fpath, false, promslog.NewNopLogger()) + verify(t, err) + }) + t.Run("Load", func(t *testing.T) { + content, err := os.ReadFile(fpath) + require.NoError(t, err) + _, err = Load(string(content), promslog.NewNopLogger()) + verify(t, err) + }) + }) + + t.Run("good config - missing otlp config uses default", func(t *testing.T) { + fpath := filepath.Join("testdata", "otlp_empty.yml") + verify := func(t *testing.T, conf *Config, err error) { + t.Helper() + require.NoError(t, err) + require.Equal(t, otlptranslator.UnderscoreEscapingWithSuffixes, conf.OTLPConfig.TranslationStrategy) + } + + t.Run("LoadFile", func(t *testing.T) { + conf, err := LoadFile(fpath, false, promslog.NewNopLogger()) + verify(t, conf, err) + }) + t.Run("Load", func(t *testing.T) { + content, err := os.ReadFile(fpath) + require.NoError(t, err) + conf, err := Load(string(content), promslog.NewNopLogger()) + verify(t, conf, err) + }) + }) } func TestLoadConfig(t *testing.T) { // Parse a valid file that sets a global scrape timeout. This tests whether parsing // an overwritten default field in the global config permanently changes the default. - _, err := LoadFile("testdata/global_timeout.good.yml", false, false, log.NewNopLogger()) + _, err := LoadFile("testdata/global_timeout.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) - c, err := LoadFile("testdata/conf.good.yml", false, false, log.NewNopLogger()) + c, err := LoadFile("testdata/conf.good.yml", false, promslog.NewNopLogger()) + require.NoError(t, err) - require.Equal(t, expectedConf, c) + testutil.RequireEqualWithOptions(t, expectedConf, c, []cmp.Option{ + cmpopts.IgnoreUnexported(config.ProxyConfig{}), + cmpopts.IgnoreUnexported(ionos.SDConfig{}), + cmpopts.IgnoreUnexported(outscale.SDConfig{}), + cmpopts.IgnoreUnexported(stackit.SDConfig{}), + cmpopts.IgnoreUnexported(regexp.Regexp{}), + cmpopts.IgnoreUnexported(hetzner.SDConfig{}), + cmpopts.IgnoreUnexported(Config{}), + }) } func TestScrapeIntervalLarger(t *testing.T) { - c, err := LoadFile("testdata/scrape_interval_larger.good.yml", false, false, log.NewNopLogger()) + c, err := LoadFile("testdata/scrape_interval_larger.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) require.Len(t, c.ScrapeConfigs, 1) for _, sc := range c.ScrapeConfigs { @@ -1564,7 +2149,7 @@ func TestScrapeIntervalLarger(t *testing.T) { // YAML marshaling must not reveal authentication credentials. func TestElideSecrets(t *testing.T) { - c, err := LoadFile("testdata/conf.good.yml", false, false, log.NewNopLogger()) + c, err := LoadFile("testdata/conf.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) secretRe := regexp.MustCompile(`\\u003csecret\\u003e|`) @@ -1574,38 +2159,38 @@ func TestElideSecrets(t *testing.T) { yamlConfig := string(config) matches := secretRe.FindAllStringIndex(yamlConfig, -1) - require.Len(t, matches, 24, "wrong number of secret matches found") + require.Len(t, matches, 28, "wrong number of secret matches found") require.NotContains(t, yamlConfig, "mysecret", "yaml marshal reveals authentication credentials.") } func TestLoadConfigRuleFilesAbsolutePath(t *testing.T) { // Parse a valid file that sets a rule files with an absolute path - c, err := LoadFile(ruleFilesConfigFile, false, false, log.NewNopLogger()) + c, err := LoadFile(ruleFilesConfigFile, false, promslog.NewNopLogger()) require.NoError(t, err) require.Equal(t, ruleFilesExpectedConf, c) } func TestKubernetesEmptyAPIServer(t *testing.T) { - _, err := LoadFile("testdata/kubernetes_empty_apiserver.good.yml", false, false, log.NewNopLogger()) + _, err := LoadFile("testdata/kubernetes_empty_apiserver.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) } func TestKubernetesWithKubeConfig(t *testing.T) { - _, err := LoadFile("testdata/kubernetes_kubeconfig_without_apiserver.good.yml", false, false, log.NewNopLogger()) + _, err := LoadFile("testdata/kubernetes_kubeconfig_without_apiserver.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) } func TestKubernetesSelectors(t *testing.T) { - _, err := LoadFile("testdata/kubernetes_selectors_endpoints.good.yml", false, false, log.NewNopLogger()) + _, err := LoadFile("testdata/kubernetes_selectors_endpoints.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_node.good.yml", false, false, log.NewNopLogger()) + _, err = LoadFile("testdata/kubernetes_selectors_node.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_ingress.good.yml", false, false, log.NewNopLogger()) + _, err = LoadFile("testdata/kubernetes_selectors_ingress.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_pod.good.yml", false, false, log.NewNopLogger()) + _, err = LoadFile("testdata/kubernetes_selectors_pod.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) - _, err = LoadFile("testdata/kubernetes_selectors_service.good.yml", false, false, log.NewNopLogger()) + _, err = LoadFile("testdata/kubernetes_selectors_service.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) } @@ -1627,11 +2212,7 @@ var expectedErrors = []struct { }, { filename: "labelname.bad.yml", - errMsg: `"not$allowed" is not a valid label name`, - }, - { - filename: "labelname2.bad.yml", - errMsg: `"not:allowed" is not a valid label name`, + errMsg: `"\xff" is not a valid label name`, }, { filename: "labelvalue.bad.yml", @@ -1703,16 +2284,12 @@ var expectedErrors = []struct { }, { filename: "labelmap.bad.yml", - errMsg: "\"l-$1\" is invalid 'replacement' for labelmap action", + errMsg: "!!binary value contains invalid base64 data", }, { filename: "lowercase.bad.yml", errMsg: "relabel configuration for lowercase action requires 'target_label' value", }, - { - filename: "lowercase2.bad.yml", - errMsg: "\"42lab\" is invalid 'target_label' for lowercase action", - }, { filename: "lowercase3.bad.yml", errMsg: "'replacement' can not be set for lowercase action", @@ -1721,10 +2298,6 @@ var expectedErrors = []struct { filename: "uppercase.bad.yml", errMsg: "relabel configuration for uppercase action requires 'target_label' value", }, - { - filename: "uppercase2.bad.yml", - errMsg: "\"42lab\" is invalid 'target_label' for uppercase action", - }, { filename: "uppercase3.bad.yml", errMsg: "'replacement' can not be set for uppercase action", @@ -1787,7 +2360,7 @@ var expectedErrors = []struct { }, { filename: "kubernetes_selectors_pod.bad.yml", - errMsg: "pod role supports only pod selectors", + errMsg: "pod role supports only pod, node selectors", }, { filename: "kubernetes_selectors_service.bad.yml", @@ -1875,7 +2448,7 @@ var expectedErrors = []struct { }, { filename: "remote_write_wrong_msg.bad.yml", - errMsg: `invalid protobuf_message value: unknown remote write protobuf message io.prometheus.writet.v2.Request, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request`, + errMsg: `invalid protobuf_message value: unknown type for remote write protobuf message io.prometheus.writet.v2.Request, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request`, }, { filename: "remote_write_url_missing.bad.yml", @@ -1885,6 +2458,30 @@ var expectedErrors = []struct { filename: "remote_write_dup.bad.yml", errMsg: `found multiple remote write configs with job name "queue1"`, }, + { + filename: "remote_write_queue_max_samples_per_send_zero.bad.yml", + errMsg: `remote write queue max_samples_per_send must be positive`, + }, + { + filename: "remote_write_queue_max_shards_zero.bad.yml", + errMsg: `remote write queue max_shards must be positive`, + }, + { + filename: "remote_write_queue_min_shards_zero.bad.yml", + errMsg: `remote write queue min_shards must be positive`, + }, + { + filename: "remote_write_queue_capacity_zero.bad.yml", + errMsg: `remote write queue capacity must be positive`, + }, + { + filename: "remote_write_queue_min_shards_greater_than_max.bad.yml", + errMsg: `remote write queue min_shards must not be greater than max_shards`, + }, + { + filename: "remote_write_queue_max_backoff_less_than_min.bad.yml", + errMsg: `remote write queue max_backoff must not be less than min_backoff`, + }, { filename: "remote_read_dup.bad.yml", errMsg: `found multiple remote read configs with job name "queue1"`, @@ -1923,7 +2520,7 @@ var expectedErrors = []struct { }, { filename: "azure_authentication_method.bad.yml", - errMsg: "unknown authentication_type \"invalid\". Supported types are \"OAuth\", \"ManagedIdentity\" or \"SDK\"", + errMsg: "unknown authentication_type \"invalid\". Supported types are \"OAuth\", \"ManagedIdentity\", \"SDK\" or \"WorkloadIdentity\"", }, { filename: "azure_bearertoken_basicauth.bad.yml", @@ -2009,6 +2606,18 @@ var expectedErrors = []struct { filename: "scaleway_two_secrets.bad.yml", errMsg: "at most one of secret_key & secret_key_file must be configured", }, + { + filename: "outscale_no_region.bad.yml", + errMsg: "outscale SD configuration requires a region", + }, + { + filename: "outscale_no_secret.bad.yml", + errMsg: "one of secret_key & secret_key_file must be configured", + }, + { + filename: "outscale_two_secrets.bad.yml", + errMsg: "at most one of secret_key & secret_key_file must be configured", + }, { filename: "scrape_body_size_limit.bad.yml", errMsg: "units: unknown unit in 100", @@ -2075,29 +2684,91 @@ var expectedErrors = []struct { }, { filename: "scrape_config_files_scrape_protocols.bad.yml", - errMsg: `parsing YAML file testdata/scrape_config_files_scrape_protocols.bad.yml: scrape_protocols: unknown scrape protocol prometheusproto, supported: [OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4] for scrape config with job name "node"`, + errMsg: `parsing YAML file testdata/scrape_config_files_scrape_protocols.bad.yml: scrape_protocols: unknown scrape protocol prometheusproto, supported: [OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4 PrometheusText1.0.0] for scrape config with job name "node"`, }, { filename: "scrape_config_files_scrape_protocols2.bad.yml", errMsg: `parsing YAML file testdata/scrape_config_files_scrape_protocols2.bad.yml: duplicated protocol in scrape_protocols, got [OpenMetricsText1.0.0 PrometheusProto OpenMetricsText1.0.0] for scrape config with job name "node"`, }, + { + filename: "scrape_config_files_fallback_scrape_protocol1.bad.yml", + errMsg: `parsing YAML file testdata/scrape_config_files_fallback_scrape_protocol1.bad.yml: invalid fallback_scrape_protocol for scrape config with job name "node": unknown scrape protocol prometheusproto, supported: [OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4 PrometheusText1.0.0]`, + }, + { + filename: "scrape_config_files_fallback_scrape_protocol2.bad.yml", + errMsg: `unmarshal errors`, + }, + { + filename: "scrape_config_utf8_conflicting.bad.yml", + errMsg: `utf8 metric names requested but validation scheme is not set to UTF8`, + }, + { + filename: "stackit_endpoint.bad.yml", + errMsg: "invalid endpoint", + }, + { + filename: "tsdb_retention_time.bad.yml", + errMsg: `not a valid duration string: "-1h"`, + }, + { + filename: "tsdb_retention_size.bad.yml", + errMsg: `'storage.tsdb.retention.size' must be greater than or equal to 0`, + }, + { + filename: "tsdb_retention_percentage.bad.yml", + errMsg: `'storage.tsdb.retention.percentage' must be in the range [0, 100]`, + }, + { + filename: "tsdb_retention_percentage_negative.bad.yml", + errMsg: "'storage.tsdb.retention.percentage' must be in the range [0, 100]", + }, + { + filename: "tsdb_chunk_encoding_floats.bad.yml", + errMsg: `'storage.tsdb.chunk_encoding.floats' must be 'xor' or 'xor2', or the field must be omitted entirely, got "xor3"`, + }, + { + filename: "tsdb_chunk_encoding_floats_wrong_case.bad.yml", + errMsg: `'storage.tsdb.chunk_encoding.floats' must be 'xor' or 'xor2', or the field must be omitted entirely, got "XOR"`, + }, + { + filename: "tsdb_chunk_encoding_floats_wrong_case_xor2.bad.yml", + errMsg: `'storage.tsdb.chunk_encoding.floats' must be 'xor' or 'xor2', or the field must be omitted entirely, got "XOR2"`, + }, } func TestBadConfigs(t *testing.T) { for _, ee := range expectedErrors { - _, err := LoadFile("testdata/"+ee.filename, false, false, log.NewNopLogger()) - require.Error(t, err, "%s", ee.filename) - require.Contains(t, err.Error(), ee.errMsg, + _, err := LoadFile("testdata/"+ee.filename, false, promslog.NewNopLogger()) + require.ErrorContains(t, err, ee.errMsg, "Expected error for %s to contain %q but got: %s", ee.filename, ee.errMsg, err) } } -func TestBadStaticConfigsJSON(t *testing.T) { - content, err := os.ReadFile("testdata/static_config.bad.json") +func TestTSDBRetentionPercentageFloat(t *testing.T) { + c, err := LoadFile("testdata/tsdb_retention_percentage_float.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) - var tg targetgroup.Group - err = json.Unmarshal(content, &tg) - require.Error(t, err) + require.Equal(t, 0.5, c.StorageConfig.TSDBConfig.Retention.Percentage) +} + +func TestTSDBChunkEncoding(t *testing.T) { + for _, tc := range []struct { + filename string + encoding string + }{ + {filename: "tsdb_chunk_encoding_floats_xor.good.yml", encoding: FloatChunkEncodingXOR}, + {filename: "tsdb_chunk_encoding_floats_xor2.good.yml", encoding: FloatChunkEncodingXOR2}, + } { + t.Run(tc.encoding, func(t *testing.T) { + c, err := LoadFile("testdata/"+tc.filename, false, promslog.NewNopLogger()) + require.NoError(t, err) + require.Equal(t, tc.encoding, c.StorageConfig.TSDBConfig.ChunkEncoding.Floats) + }) + } + + // Empty/absent value is also valid. + c, err := Load("", promslog.NewNopLogger()) + require.NoError(t, err) + require.Empty(t, c.StorageConfig.TSDBConfig.ChunkEncoding.Floats) } func TestBadStaticConfigsYML(t *testing.T) { @@ -2109,48 +2780,48 @@ func TestBadStaticConfigsYML(t *testing.T) { } func TestEmptyConfig(t *testing.T) { - c, err := Load("", false, log.NewNopLogger()) + c, err := Load("", promslog.NewNopLogger()) require.NoError(t, err) exp := DefaultConfig + exp.loaded = true + retention := DefaultTSDBRetentionConfig + exp.StorageConfig.TSDBConfig = &TSDBConfig{Retention: &retention} require.Equal(t, exp, *c) + require.Equal(t, 75, c.Runtime.GoGC) } func TestExpandExternalLabels(t *testing.T) { // Cleanup ant TEST env variable that could exist on the system. os.Setenv("TEST", "") - c, err := LoadFile("testdata/external_labels.good.yml", false, false, log.NewNopLogger()) - require.NoError(t, err) - testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foo${TEST}bar", "foo", "${TEST}", "qux", "foo$${TEST}", "xyz", "foo$$bar"), c.GlobalConfig.ExternalLabels) - - c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger()) + c, err := LoadFile("testdata/external_labels.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "foobar", "foo", "", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels) os.Setenv("TEST", "TestValue") - c, err = LoadFile("testdata/external_labels.good.yml", false, true, log.NewNopLogger()) + c, err = LoadFile("testdata/external_labels.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) testutil.RequireEqual(t, labels.FromStrings("bar", "foo", "baz", "fooTestValuebar", "foo", "TestValue", "qux", "foo${TEST}", "xyz", "foo$bar"), c.GlobalConfig.ExternalLabels) } func TestAgentMode(t *testing.T) { - _, err := LoadFile("testdata/agent_mode.with_alert_manager.yml", true, false, log.NewNopLogger()) + _, err := LoadFile("testdata/agent_mode.with_alert_manager.yml", true, promslog.NewNopLogger()) require.ErrorContains(t, err, "field alerting is not allowed in agent mode") - _, err = LoadFile("testdata/agent_mode.with_alert_relabels.yml", true, false, log.NewNopLogger()) + _, err = LoadFile("testdata/agent_mode.with_alert_relabels.yml", true, promslog.NewNopLogger()) require.ErrorContains(t, err, "field alerting is not allowed in agent mode") - _, err = LoadFile("testdata/agent_mode.with_rule_files.yml", true, false, log.NewNopLogger()) + _, err = LoadFile("testdata/agent_mode.with_rule_files.yml", true, promslog.NewNopLogger()) require.ErrorContains(t, err, "field rule_files is not allowed in agent mode") - _, err = LoadFile("testdata/agent_mode.with_remote_reads.yml", true, false, log.NewNopLogger()) + _, err = LoadFile("testdata/agent_mode.with_remote_reads.yml", true, promslog.NewNopLogger()) require.ErrorContains(t, err, "field remote_read is not allowed in agent mode") - c, err := LoadFile("testdata/agent_mode.without_remote_writes.yml", true, false, log.NewNopLogger()) + c, err := LoadFile("testdata/agent_mode.without_remote_writes.yml", true, promslog.NewNopLogger()) require.NoError(t, err) require.Empty(t, c.RemoteWriteConfigs) - c, err = LoadFile("testdata/agent_mode.good.yml", true, false, log.NewNopLogger()) + c, err = LoadFile("testdata/agent_mode.good.yml", true, promslog.NewNopLogger()) require.NoError(t, err) require.Len(t, c.RemoteWriteConfigs, 1) require.Equal( @@ -2160,22 +2831,116 @@ func TestAgentMode(t *testing.T) { ) } -func TestEmptyGlobalBlock(t *testing.T) { - c, err := Load("global:\n", false, log.NewNopLogger()) - require.NoError(t, err) - exp := DefaultConfig - exp.Runtime = DefaultRuntimeConfig - require.Equal(t, exp, *c) +func TestGlobalConfig(t *testing.T) { + t.Run("empty block restores defaults", func(t *testing.T) { + c, err := Load("global:\n", promslog.NewNopLogger()) + require.NoError(t, err) + exp := DefaultConfig + exp.loaded = true + // TSDBConfig is always injected by Config.UnmarshalYAML even when no + // storage.tsdb section is present, so the expected config must include it. + retention := DefaultTSDBRetentionConfig + exp.StorageConfig.TSDBConfig = &TSDBConfig{Retention: &retention} + require.Equal(t, exp, *c) + }) + + // Verify that isZero() correctly identifies non-zero configurations for all + // fields in GlobalConfig. This is important because isZero() is used during + // YAML unmarshaling to detect empty global blocks that should be replaced + // with defaults. + t.Run("isZero", func(t *testing.T) { + for _, tc := range []struct { + name string + config GlobalConfig + expectZero bool + }{ + { + name: "empty GlobalConfig", + config: GlobalConfig{}, + expectZero: true, + }, + { + name: "ScrapeInterval set", + config: GlobalConfig{ScrapeInterval: model.Duration(30 * time.Second)}, + expectZero: false, + }, + { + name: "BodySizeLimit set", + config: GlobalConfig{BodySizeLimit: 1 * units.MiB}, + expectZero: false, + }, + { + name: "SampleLimit set", + config: GlobalConfig{SampleLimit: 1000}, + expectZero: false, + }, + { + name: "TargetLimit set", + config: GlobalConfig{TargetLimit: 500}, + expectZero: false, + }, + { + name: "LabelLimit set", + config: GlobalConfig{LabelLimit: 100}, + expectZero: false, + }, + { + name: "LabelNameLengthLimit set", + config: GlobalConfig{LabelNameLengthLimit: 50}, + expectZero: false, + }, + { + name: "LabelValueLengthLimit set", + config: GlobalConfig{LabelValueLengthLimit: 200}, + expectZero: false, + }, + { + name: "KeepDroppedTargets set", + config: GlobalConfig{KeepDroppedTargets: 10}, + expectZero: false, + }, + { + name: "MetricNameValidationScheme set", + config: GlobalConfig{MetricNameValidationScheme: model.LegacyValidation}, + expectZero: false, + }, + { + name: "MetricNameEscapingScheme set", + config: GlobalConfig{MetricNameEscapingScheme: model.EscapeUnderscores}, + expectZero: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + result := tc.config.isZero() + require.Equal(t, tc.expectZero, result) + }) + } + }) +} + +// ScrapeConfigOptions contains options for creating a scrape config. +type ScrapeConfigOptions struct { + JobName string + ScrapeInterval model.Duration + ScrapeTimeout model.Duration + ScrapeProtocols []ScrapeProtocol // Set to DefaultScrapeProtocols by default. + ScrapeNativeHistograms bool + AlwaysScrapeClassicHistograms bool + ConvertClassicHistToNHCB bool + ExtraScrapeMetrics bool } func TestGetScrapeConfigs(t *testing.T) { - sc := func(jobName string, scrapeInterval, scrapeTimeout model.Duration) *ScrapeConfig { - return &ScrapeConfig{ - JobName: jobName, - HonorTimestamps: true, - ScrapeInterval: scrapeInterval, - ScrapeTimeout: scrapeTimeout, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, + // Helper function to create a scrape config with the given options. + sc := func(opts ScrapeConfigOptions) *ScrapeConfig { + sc := ScrapeConfig{ + JobName: opts.JobName, + HonorTimestamps: true, + ScrapeInterval: opts.ScrapeInterval, + ScrapeTimeout: opts.ScrapeTimeout, + ScrapeProtocols: opts.ScrapeProtocols, + MetricNameValidationScheme: model.UTF8Validation, + MetricNameEscapingScheme: model.AllowUTF8, MetricsPath: "/metrics", Scheme: "http", @@ -2193,7 +2958,15 @@ func TestGetScrapeConfigs(t *testing.T) { }, }, }, + ScrapeNativeHistograms: boolPtr(opts.ScrapeNativeHistograms), + AlwaysScrapeClassicHistograms: boolPtr(opts.AlwaysScrapeClassicHistograms), + ConvertClassicHistogramsToNHCB: boolPtr(opts.ConvertClassicHistToNHCB), + ExtraScrapeMetrics: boolPtr(opts.ExtraScrapeMetrics), + } + if opts.ScrapeProtocols == nil { + sc.ScrapeProtocols = DefaultScrapeProtocols } + return &sc } testCases := []struct { @@ -2203,35 +2976,76 @@ func TestGetScrapeConfigs(t *testing.T) { expectedError string }{ { - name: "An included config file should be a valid global config.", - configFile: "testdata/scrape_config_files.good.yml", - expectedResult: []*ScrapeConfig{sc("prometheus", model.Duration(60*time.Second), model.Duration(10*time.Second))}, + name: "An included config file should be a valid global config.", + configFile: "testdata/scrape_config_files.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{ + JobName: "prometheus", + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeTimeout: model.Duration(10 * time.Second), + ScrapeNativeHistograms: false, + AlwaysScrapeClassicHistograms: false, + ConvertClassicHistToNHCB: false, + })}, }, { - name: "An global config that only include a scrape config file.", - configFile: "testdata/scrape_config_files_only.good.yml", - expectedResult: []*ScrapeConfig{sc("prometheus", model.Duration(60*time.Second), model.Duration(10*time.Second))}, + name: "A global config that only include a scrape config file.", + configFile: "testdata/scrape_config_files_only.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{ + JobName: "prometheus", + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeTimeout: model.Duration(10 * time.Second), + ScrapeNativeHistograms: false, + AlwaysScrapeClassicHistograms: false, + ConvertClassicHistToNHCB: false, + })}, }, { - name: "An global config that combine scrape config files and scrape configs.", + name: "A global config that combine scrape config files and scrape configs.", configFile: "testdata/scrape_config_files_combined.good.yml", expectedResult: []*ScrapeConfig{ - sc("node", model.Duration(60*time.Second), model.Duration(10*time.Second)), - sc("prometheus", model.Duration(60*time.Second), model.Duration(10*time.Second)), - sc("alertmanager", model.Duration(60*time.Second), model.Duration(10*time.Second)), + sc(ScrapeConfigOptions{ + JobName: "node", + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeTimeout: model.Duration(10 * time.Second), + ScrapeNativeHistograms: false, + AlwaysScrapeClassicHistograms: false, + ConvertClassicHistToNHCB: false, + }), + sc(ScrapeConfigOptions{ + JobName: "prometheus", + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeTimeout: model.Duration(10 * time.Second), + ScrapeNativeHistograms: false, + AlwaysScrapeClassicHistograms: false, + ConvertClassicHistToNHCB: false, + }), + sc(ScrapeConfigOptions{ + JobName: "alertmanager", + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeTimeout: model.Duration(10 * time.Second), + ScrapeNativeHistograms: false, + AlwaysScrapeClassicHistograms: false, + ConvertClassicHistToNHCB: false, + }), }, }, { - name: "An global config that includes a scrape config file with globs", + name: "A global config that includes a scrape config file with globs", configFile: "testdata/scrape_config_files_glob.good.yml", expectedResult: []*ScrapeConfig{ { JobName: "prometheus", - HonorTimestamps: true, - ScrapeInterval: model.Duration(60 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, + HonorTimestamps: true, + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + ScrapeProtocols: DefaultScrapeProtocols, + MetricNameValidationScheme: model.UTF8Validation, + MetricNameEscapingScheme: model.AllowUTF8, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -2261,10 +3075,16 @@ func TestGetScrapeConfigs(t *testing.T) { { JobName: "node", - HonorTimestamps: true, - ScrapeInterval: model.Duration(15 * time.Second), - ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, - ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + ScrapeProtocols: DefaultScrapeProtocols, + MetricNameValidationScheme: model.UTF8Validation, + MetricNameEscapingScheme: model.AllowUTF8, + ScrapeNativeHistograms: boolPtr(false), + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + ExtraScrapeMetrics: boolPtr(false), HTTPClientConfig: config.HTTPClientConfig{ TLSConfig: config.TLSConfig{ @@ -2298,29 +3118,114 @@ func TestGetScrapeConfigs(t *testing.T) { }, }, { - name: "An global config that includes twice the same scrape configs.", + name: "A global config that includes twice the same scrape configs.", configFile: "testdata/scrape_config_files_double_import.bad.yml", expectedError: `found multiple scrape configs with job name "prometheus"`, }, { - name: "An global config that includes a scrape config identical to a scrape config in the main file.", + name: "A global config that includes a scrape config identical to a scrape config in the main file.", configFile: "testdata/scrape_config_files_duplicate.bad.yml", expectedError: `found multiple scrape configs with job name "prometheus"`, }, { - name: "An global config that includes a scrape config file with errors.", + name: "A global config that includes a scrape config file with errors.", configFile: "testdata/scrape_config_files_global.bad.yml", expectedError: `scrape timeout greater than scrape interval for scrape config with job name "prometheus"`, }, + { + name: "A global config that enables convert classic histograms to nhcb.", + configFile: "testdata/global_convert_classic_hist_to_nhcb.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), AlwaysScrapeClassicHistograms: false, ConvertClassicHistToNHCB: true})}, + }, + { + name: "A global config that enables convert classic histograms to nhcb and scrape config that disables the conversion", + configFile: "testdata/local_disable_convert_classic_hist_to_nhcb.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), AlwaysScrapeClassicHistograms: false, ConvertClassicHistToNHCB: false})}, + }, + { + name: "A global config that disables convert classic histograms to nhcb and scrape config that enables the conversion", + configFile: "testdata/local_convert_classic_hist_to_nhcb.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), AlwaysScrapeClassicHistograms: false, ConvertClassicHistToNHCB: true})}, + }, + { + name: "A global config that enables always scrape classic histograms", + configFile: "testdata/global_enable_always_scrape_classic_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), AlwaysScrapeClassicHistograms: true, ConvertClassicHistToNHCB: false})}, + }, + { + name: "A global config that disables always scrape classic histograms", + configFile: "testdata/global_disable_always_scrape_classic_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), AlwaysScrapeClassicHistograms: false, ConvertClassicHistToNHCB: false})}, + }, + { + name: "A global config that disables always scrape classic histograms and scrape config that enables it", + configFile: "testdata/local_enable_always_scrape_classic_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), AlwaysScrapeClassicHistograms: true, ConvertClassicHistToNHCB: false})}, + }, + { + name: "A global config that enables always scrape classic histograms and scrape config that disables it", + configFile: "testdata/local_disable_always_scrape_classic_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), AlwaysScrapeClassicHistograms: false, ConvertClassicHistToNHCB: false})}, + }, + { + name: "A global config that enables scrape native histograms", + configFile: "testdata/global_enable_scrape_native_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ScrapeNativeHistograms: true, ScrapeProtocols: DefaultProtoFirstScrapeProtocols})}, + }, + { + name: "A global config that enables scrape native histograms and sets scrape protocols explicitly", + configFile: "testdata/global_enable_scrape_native_hist_and_scrape_protocols.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ScrapeNativeHistograms: true, ScrapeProtocols: []ScrapeProtocol{PrometheusText0_0_4}})}, + }, + { + name: "A local config that enables scrape native histograms", + configFile: "testdata/local_enable_scrape_native_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ScrapeNativeHistograms: true, ScrapeProtocols: DefaultProtoFirstScrapeProtocols})}, + }, + { + name: "A local config that enables scrape native histograms and sets scrape protocols explicitly", + configFile: "testdata/local_enable_scrape_native_hist_and_scrape_protocols.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ScrapeNativeHistograms: true, ScrapeProtocols: []ScrapeProtocol{PrometheusText0_0_4}})}, + }, + { + name: "A global config that enables scrape native histograms and scrape config that disables it", + configFile: "testdata/local_disable_scrape_native_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ScrapeNativeHistograms: false, ScrapeProtocols: DefaultScrapeProtocols})}, + }, + { + name: "A global config that enables scrape native histograms and scrape protocols and scrape config that disables scrape native histograms but does not change scrape protocols", + configFile: "testdata/global_scrape_protocols_and_local_disable_scrape_native_hist.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ScrapeNativeHistograms: false, ScrapeProtocols: []ScrapeProtocol{PrometheusText0_0_4}})}, + }, + { + name: "A global config that enables extra scrape metrics", + configFile: "testdata/global_enable_extra_scrape_metrics.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ExtraScrapeMetrics: true})}, + }, + { + name: "A global config that disables extra scrape metrics", + configFile: "testdata/global_disable_extra_scrape_metrics.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ExtraScrapeMetrics: false})}, + }, + { + name: "A global config that disables extra scrape metrics and scrape config that enables it", + configFile: "testdata/local_enable_extra_scrape_metrics.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ExtraScrapeMetrics: true})}, + }, + { + name: "A global config that enables extra scrape metrics and scrape config that disables it", + configFile: "testdata/local_disable_extra_scrape_metrics.good.yml", + expectedResult: []*ScrapeConfig{sc(ScrapeConfigOptions{JobName: "prometheus", ScrapeInterval: model.Duration(60 * time.Second), ScrapeTimeout: model.Duration(10 * time.Second), ExtraScrapeMetrics: false})}, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c, err := LoadFile(tc.configFile, false, false, log.NewNopLogger()) + c, err := LoadFile(tc.configFile, false, promslog.NewNopLogger()) require.NoError(t, err) scfgs, err := c.GetScrapeConfigs() - if len(tc.expectedError) > 0 { + if tc.expectedError != "" { require.ErrorContains(t, err, tc.expectedError) } require.Equal(t, tc.expectedResult, scfgs) @@ -2328,13 +3233,106 @@ func TestGetScrapeConfigs(t *testing.T) { } } +func TestExtraScrapeMetrics(t *testing.T) { + tests := []struct { + name string + config string + expectGlobal *bool + expectEnabled bool + }{ + { + name: "default values (not set)", + config: ` +scrape_configs: + - job_name: test + static_configs: + - targets: ['localhost:9090'] +`, + expectGlobal: boolPtr(false), // inherits from DefaultGlobalConfig + expectEnabled: false, + }, + { + name: "global enabled", + config: ` +global: + extra_scrape_metrics: true +scrape_configs: + - job_name: test + static_configs: + - targets: ['localhost:9090'] +`, + expectGlobal: boolPtr(true), + expectEnabled: true, + }, + { + name: "global disabled", + config: ` +global: + extra_scrape_metrics: false +scrape_configs: + - job_name: test + static_configs: + - targets: ['localhost:9090'] +`, + expectGlobal: boolPtr(false), + expectEnabled: false, + }, + { + name: "scrape override enabled", + config: ` +global: + extra_scrape_metrics: false +scrape_configs: + - job_name: test + extra_scrape_metrics: true + static_configs: + - targets: ['localhost:9090'] +`, + expectGlobal: boolPtr(false), + expectEnabled: true, + }, + { + name: "scrape override disabled", + config: ` +global: + extra_scrape_metrics: true +scrape_configs: + - job_name: test + extra_scrape_metrics: false + static_configs: + - targets: ['localhost:9090'] +`, + expectGlobal: boolPtr(true), + expectEnabled: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg, err := Load(tc.config, promslog.NewNopLogger()) + require.NoError(t, err) + + // Check global config + require.Equal(t, tc.expectGlobal, cfg.GlobalConfig.ExtraScrapeMetrics) + + // Check scrape config + scfgs, err := cfg.GetScrapeConfigs() + require.NoError(t, err) + require.Len(t, scfgs, 1) + + // Check the effective value via the helper method + require.Equal(t, tc.expectEnabled, scfgs[0].ExtraScrapeMetricsEnabled()) + }) + } +} + func kubernetesSDHostURL() config.URL { tURL, _ := url.Parse("https://localhost:1234") return config.URL{URL: tURL} } func TestScrapeConfigDisableCompression(t *testing.T) { - want, err := LoadFile("testdata/scrape_config_disable_compression.good.yml", false, false, log.NewNopLogger()) + want, err := LoadFile("testdata/scrape_config_disable_compression.good.yml", false, promslog.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) @@ -2347,41 +3345,47 @@ func TestScrapeConfigDisableCompression(t *testing.T) { } func TestScrapeConfigNameValidationSettings(t *testing.T) { - model.NameValidationScheme = model.UTF8Validation - defer func() { - model.NameValidationScheme = model.LegacyValidation - }() - tests := []struct { - name string - inputFile string - expectScheme string + name string + inputFile string + expectScheme model.ValidationScheme + expectEscaping model.EscapingScheme }{ { - name: "blank config implies default", - inputFile: "scrape_config_default_validation_mode", - expectScheme: "", + name: "blank config implies default", + inputFile: "scrape_config_default_validation_mode", + expectScheme: model.UTF8Validation, + expectEscaping: model.NoEscaping, + }, + { + name: "global setting implies local settings", + inputFile: "scrape_config_global_validation_mode", + expectScheme: model.LegacyValidation, + expectEscaping: model.DotsEscaping, }, { - name: "global setting implies local settings", - inputFile: "scrape_config_global_validation_mode", - expectScheme: "utf8", + name: "local setting", + inputFile: "scrape_config_local_validation_mode", + expectScheme: model.LegacyValidation, + expectEscaping: model.ValueEncodingEscaping, }, { - name: "local setting", - inputFile: "scrape_config_local_validation_mode", - expectScheme: "utf8", + name: "local setting overrides global setting", + inputFile: "scrape_config_local_global_validation_mode", + expectScheme: model.UTF8Validation, + expectEscaping: model.DotsEscaping, }, { - name: "local setting overrides global setting", - inputFile: "scrape_config_local_global_validation_mode", - expectScheme: "legacy", + name: "local validation implies underscores escaping", + inputFile: "scrape_config_local_infer_escaping", + expectScheme: model.LegacyValidation, + expectEscaping: model.UnderscoreEscaping, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - want, err := LoadFile(fmt.Sprintf("testdata/%s.yml", tc.inputFile), false, false, log.NewNopLogger()) + want, err := LoadFile(fmt.Sprintf("testdata/%s.yml", tc.inputFile), false, promslog.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) @@ -2391,6 +3395,126 @@ func TestScrapeConfigNameValidationSettings(t *testing.T) { require.NoError(t, yaml.UnmarshalStrict(out, got)) require.Equal(t, tc.expectScheme, got.ScrapeConfigs[0].MetricNameValidationScheme) + + escaping, err := model.ToEscapingScheme(got.ScrapeConfigs[0].MetricNameEscapingScheme) + require.NoError(t, err) + require.Equal(t, tc.expectEscaping, escaping) }) } } + +func TestScrapeConfigNameEscapingSettings(t *testing.T) { + tests := []struct { + name string + inputFile string + expectValidationScheme model.ValidationScheme + expectEscapingScheme string + }{ + { + name: "blank config implies default", + inputFile: "scrape_config_default_validation_mode", + expectValidationScheme: model.UTF8Validation, + expectEscapingScheme: "allow-utf-8", + }, + { + name: "global setting implies local settings", + inputFile: "scrape_config_global_validation_mode", + expectValidationScheme: model.LegacyValidation, + expectEscapingScheme: "dots", + }, + { + name: "local setting", + inputFile: "scrape_config_local_validation_mode", + expectValidationScheme: model.LegacyValidation, + expectEscapingScheme: "values", + }, + { + name: "local setting overrides global setting", + inputFile: "scrape_config_local_global_validation_mode", + expectValidationScheme: model.UTF8Validation, + expectEscapingScheme: "dots", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + want, err := LoadFile(fmt.Sprintf("testdata/%s.yml", tc.inputFile), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + + require.NoError(t, err) + got := &Config{} + require.NoError(t, yaml.UnmarshalStrict(out, got)) + + require.Equal(t, tc.expectValidationScheme, got.ScrapeConfigs[0].MetricNameValidationScheme) + require.Equal(t, tc.expectEscapingScheme, got.ScrapeConfigs[0].MetricNameEscapingScheme) + }) + } +} + +func TestScrapeProtocolHeader(t *testing.T) { + tests := []struct { + name string + proto ScrapeProtocol + expectedValue string + }{ + { + name: "blank", + proto: ScrapeProtocol(""), + expectedValue: "", + }, + { + name: "invalid", + proto: ScrapeProtocol("invalid"), + expectedValue: "", + }, + { + name: "prometheus protobuf", + proto: PrometheusProto, + expectedValue: "application/vnd.google.protobuf", + }, + { + name: "prometheus text 0.0.4", + proto: PrometheusText0_0_4, + expectedValue: "text/plain", + }, + { + name: "prometheus text 1.0.0", + proto: PrometheusText1_0_0, + expectedValue: "text/plain", + }, + { + name: "openmetrics 0.0.1", + proto: OpenMetricsText0_0_1, + expectedValue: "application/openmetrics-text", + }, + { + name: "openmetrics 1.0.0", + proto: OpenMetricsText1_0_0, + expectedValue: "application/openmetrics-text", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mediaType := tc.proto.HeaderMediaType() + + require.Equal(t, tc.expectedValue, mediaType) + }) + } +} + +// Regression test against https://github.com/prometheus/prometheus/issues/15538 +func TestGetScrapeConfigs_Loaded(t *testing.T) { + t.Run("without load", func(t *testing.T) { + c := &Config{} + _, err := c.GetScrapeConfigs() + require.EqualError(t, err, "scrape config cannot be fetched, main config was not validated and loaded correctly; should not happen") + }) + t.Run("with load", func(t *testing.T) { + c, err := Load("", promslog.NewNopLogger()) + require.NoError(t, err) + _, err = c.GetScrapeConfigs() + require.NoError(t, err) + }) +} diff --git a/config/config_windows_test.go b/config/config_windows_test.go index db4d46ef13a..e7627f562a2 100644 --- a/config/config_windows_test.go +++ b/config/config_windows_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,8 +16,11 @@ package config const ruleFilesConfigFile = "testdata/rules_abs_path_windows.good.yml" var ruleFilesExpectedConf = &Config{ - GlobalConfig: DefaultGlobalConfig, - Runtime: DefaultRuntimeConfig, + loaded: true, + + GlobalConfig: DefaultGlobalConfig, + Runtime: DefaultRuntimeConfig, + StorageConfig: StorageConfig{TSDBConfig: &TSDBConfig{Retention: &TSDBRetentionConfig{}}}, RuleFiles: []string{ "testdata\\first.rules", "testdata\\rules\\second.rules", diff --git a/config/reload.go b/config/reload.go new file mode 100644 index 00000000000..a250693169c --- /dev/null +++ b/config/reload.go @@ -0,0 +1,93 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + + promconfig "github.com/prometheus/common/config" + "go.yaml.in/yaml/v2" +) + +type ExternalFilesConfig struct { + RuleFiles []string `yaml:"rule_files"` + ScrapeConfigFiles []string `yaml:"scrape_config_files"` +} + +// GenerateChecksum generates a checksum of the YAML file and the files it references. +func GenerateChecksum(yamlFilePath string) (string, error) { + hash := sha256.New() + + yamlContent, err := os.ReadFile(yamlFilePath) + if err != nil { + return "", fmt.Errorf("error reading YAML file: %w", err) + } + _, err = hash.Write(yamlContent) + if err != nil { + return "", fmt.Errorf("error writing YAML file to hash: %w", err) + } + + var config ExternalFilesConfig + if err := yaml.Unmarshal(yamlContent, &config); err != nil { + return "", fmt.Errorf("error unmarshalling YAML: %w", err) + } + + dir := filepath.Dir(yamlFilePath) + + for i, file := range config.RuleFiles { + config.RuleFiles[i] = promconfig.JoinDir(dir, file) + } + for i, file := range config.ScrapeConfigFiles { + config.ScrapeConfigFiles[i] = promconfig.JoinDir(dir, file) + } + + files := map[string][]string{ + "r": config.RuleFiles, // "r" for rule files + "s": config.ScrapeConfigFiles, // "s" for scrape config files + } + + for _, prefix := range []string{"r", "s"} { + for _, pattern := range files[prefix] { + matchingFiles, err := filepath.Glob(pattern) + if err != nil { + return "", fmt.Errorf("error finding files with pattern %q: %w", pattern, err) + } + + for _, file := range matchingFiles { + // Write prefix to the hash ("r" or "s") followed by \0, then + // the file path. + _, err = hash.Write([]byte(prefix + "\x00" + file + "\x00")) + if err != nil { + return "", fmt.Errorf("error writing %q path to hash: %w", file, err) + } + + // Read and hash the content of the file. + content, err := os.ReadFile(file) + if err != nil { + return "", fmt.Errorf("error reading file %s: %w", file, err) + } + _, err = hash.Write(append(content, []byte("\x00")...)) + if err != nil { + return "", fmt.Errorf("error writing %q content to hash: %w", file, err) + } + } + } + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} diff --git a/config/reload_test.go b/config/reload_test.go new file mode 100644 index 00000000000..cb60d47651b --- /dev/null +++ b/config/reload_test.go @@ -0,0 +1,246 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGenerateChecksum(t *testing.T) { + tmpDir := t.TempDir() + + // Define paths for the temporary files. + yamlFilePath := filepath.Join(tmpDir, "test.yml") + ruleFile := "rule_file.yml" + ruleFilePath := filepath.Join(tmpDir, ruleFile) + scrapeConfigFile := "scrape_config.yml" + scrapeConfigFilePath := filepath.Join(tmpDir, scrapeConfigFile) + + // Define initial and modified content for the files. + originalRuleContent := "groups:\n- name: example\n rules:\n - alert: ExampleAlert" + modifiedRuleContent := "groups:\n- name: example\n rules:\n - alert: ModifiedAlert" + + originalScrapeConfigContent := "scrape_configs:\n- job_name: example" + modifiedScrapeConfigContent := "scrape_configs:\n- job_name: modified_example" + + testCases := []struct { + name string + ruleFilePath string + scrapeConfigFilePath string + }{ + { + name: "Auto reload using relative path.", + ruleFilePath: ruleFile, + scrapeConfigFilePath: scrapeConfigFile, + }, + { + name: "Auto reload using absolute path.", + ruleFilePath: ruleFilePath, + scrapeConfigFilePath: scrapeConfigFilePath, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Define YAML content referencing the rule and scrape config files. + yamlContent := fmt.Sprintf(` +rule_files: + - %s +scrape_config_files: + - %s +`, tc.ruleFilePath, tc.scrapeConfigFilePath) + + // Write initial content to files. + require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644)) + require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644)) + require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644)) + + // Generate the original checksum. + originalChecksum := calculateChecksum(t, yamlFilePath) + + t.Run("Rule File Change", func(t *testing.T) { + // Modify the rule file. + require.NoError(t, os.WriteFile(ruleFilePath, []byte(modifiedRuleContent), 0o644)) + + // Checksum should change. + modifiedChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, modifiedChecksum) + + // Revert the rule file. + require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Scrape Config Change", func(t *testing.T) { + // Modify the scrape config file. + require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(modifiedScrapeConfigContent), 0o644)) + + // Checksum should change. + modifiedChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, modifiedChecksum) + + // Revert the scrape config file. + require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Rule File Deletion", func(t *testing.T) { + // Delete the rule file. + require.NoError(t, os.Remove(ruleFilePath)) + + // Checksum should change. + deletedChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, deletedChecksum) + + // Restore the rule file. + require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Scrape Config Deletion", func(t *testing.T) { + // Delete the scrape config file. + require.NoError(t, os.Remove(scrapeConfigFilePath)) + + // Checksum should change. + deletedChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, deletedChecksum) + + // Restore the scrape config file. + require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Main File Change", func(t *testing.T) { + // Modify the main YAML file. + modifiedYamlContent := fmt.Sprintf(` +global: + scrape_interval: 3s +rule_files: + - %s +scrape_config_files: + - %s +`, tc.ruleFilePath, tc.scrapeConfigFilePath) + require.NoError(t, os.WriteFile(yamlFilePath, []byte(modifiedYamlContent), 0o644)) + + // Checksum should change. + modifiedChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, modifiedChecksum) + + // Revert the main YAML file. + require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Rule File Removed from YAML Config", func(t *testing.T) { + // Modify the YAML content to remove the rule file. + modifiedYamlContent := fmt.Sprintf(` +scrape_config_files: + - %s +`, tc.scrapeConfigFilePath) + require.NoError(t, os.WriteFile(yamlFilePath, []byte(modifiedYamlContent), 0o644)) + + // Checksum should change. + modifiedChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, modifiedChecksum) + + // Revert the YAML content. + require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Scrape Config Removed from YAML Config", func(t *testing.T) { + // Modify the YAML content to remove the scrape config file. + modifiedYamlContent := fmt.Sprintf(` +rule_files: + - %s +`, tc.ruleFilePath) + require.NoError(t, os.WriteFile(yamlFilePath, []byte(modifiedYamlContent), 0o644)) + + // Checksum should change. + modifiedChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, modifiedChecksum) + + // Revert the YAML content. + require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Empty Rule File", func(t *testing.T) { + // Write an empty rule file. + require.NoError(t, os.WriteFile(ruleFilePath, []byte(""), 0o644)) + + // Checksum should change. + emptyChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, emptyChecksum) + + // Restore the rule file. + require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + + t.Run("Empty Scrape Config File", func(t *testing.T) { + // Write an empty scrape config file. + require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(""), 0o644)) + + // Checksum should change. + emptyChecksum := calculateChecksum(t, yamlFilePath) + require.NotEqual(t, originalChecksum, emptyChecksum) + + // Restore the scrape config file. + require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644)) + + // Checksum should return to the original. + revertedChecksum := calculateChecksum(t, yamlFilePath) + require.Equal(t, originalChecksum, revertedChecksum) + }) + }) + } +} + +// calculateChecksum generates a checksum for the given YAML file path. +func calculateChecksum(t *testing.T, yamlFilePath string) string { + checksum, err := GenerateChecksum(yamlFilePath) + require.NoError(t, err) + require.NotEmpty(t, checksum) + return checksum +} diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 9eb79954325..a2f8459b24d 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -74,6 +74,8 @@ scrape_configs: # metrics_path defaults to '/metrics' # scheme defaults to 'http'. + fallback_scrape_protocol: PrometheusText0.0.4 + scrape_failure_log_file: fail_prom.log file_sd_configs: - files: @@ -180,6 +182,7 @@ scrape_configs: token: mysecret path_prefix: /consul services: ["nginx", "cache", "mysql"] + health_filter: 'Service.Tags contains "canary"' tags: ["canary", "v1"] node_meta: rack: "123" @@ -367,6 +370,7 @@ scrape_configs: key_file: valid_key_file - job_name: hetzner + scrape_native_histograms: true relabel_configs: - action: uppercase source_labels: [instance] @@ -415,6 +419,14 @@ scrape_configs: - authorization: credentials: abcdef + - job_name: stackit-servers + stackit_sd_configs: + - project: 11111111-1111-1111-1111-111111111111 + service_account_key: mysecret_sa_key + private_key: mysecret_private_key + authorization: + credentials: abcdef + - job_name: uyuni uyuni_sd_configs: - server: https://localhost:1234 @@ -432,6 +444,12 @@ scrape_configs: - authorization: credentials: abcdef + - job_name: outscale + outscale_sd_configs: + - region: eu-west-2 + access_key: A1B2C3D4E5F6G7H8I9J0 + secret_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + alerting: alertmanagers: - scheme: https @@ -444,6 +462,11 @@ alerting: storage: tsdb: out_of_order_time_window: 30m + stale_series_compaction_threshold: 0.5 + retention: + time: 1d + size: 1GB + percentage: 28 tracing: endpoint: "localhost:4317" diff --git a/config/testdata/config_with_deprecated_am_api_config.yml b/config/testdata/config_with_deprecated_am_api_config.yml new file mode 100644 index 00000000000..ac89537ff1b --- /dev/null +++ b/config/testdata/config_with_deprecated_am_api_config.yml @@ -0,0 +1,7 @@ +alerting: + alertmanagers: + - scheme: http + api_version: v1 + file_sd_configs: + - files: + - nonexistent_file.yml diff --git a/config/testdata/global_convert_classic_hist_to_nhcb.good.yml b/config/testdata/global_convert_classic_hist_to_nhcb.good.yml new file mode 100644 index 00000000000..c97b7597afc --- /dev/null +++ b/config/testdata/global_convert_classic_hist_to_nhcb.good.yml @@ -0,0 +1,6 @@ +global: + convert_classic_histograms_to_nhcb: true +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/global_disable_always_scrape_classic_hist.good.yml b/config/testdata/global_disable_always_scrape_classic_hist.good.yml new file mode 100644 index 00000000000..de28f1357aa --- /dev/null +++ b/config/testdata/global_disable_always_scrape_classic_hist.good.yml @@ -0,0 +1,6 @@ +global: + always_scrape_classic_histograms: false +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/global_disable_extra_scrape_metrics.good.yml b/config/testdata/global_disable_extra_scrape_metrics.good.yml new file mode 100644 index 00000000000..26c6e4b8b54 --- /dev/null +++ b/config/testdata/global_disable_extra_scrape_metrics.good.yml @@ -0,0 +1,6 @@ +global: + extra_scrape_metrics: false +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/global_enable_always_scrape_classic_hist.good.yml b/config/testdata/global_enable_always_scrape_classic_hist.good.yml new file mode 100644 index 00000000000..d42cf69cb69 --- /dev/null +++ b/config/testdata/global_enable_always_scrape_classic_hist.good.yml @@ -0,0 +1,6 @@ +global: + always_scrape_classic_histograms: true +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/global_enable_extra_scrape_metrics.good.yml b/config/testdata/global_enable_extra_scrape_metrics.good.yml new file mode 100644 index 00000000000..1d7ea2db1c6 --- /dev/null +++ b/config/testdata/global_enable_extra_scrape_metrics.good.yml @@ -0,0 +1,6 @@ +global: + extra_scrape_metrics: true +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/global_enable_scrape_native_hist.good.yml b/config/testdata/global_enable_scrape_native_hist.good.yml new file mode 100644 index 00000000000..fe727f8018e --- /dev/null +++ b/config/testdata/global_enable_scrape_native_hist.good.yml @@ -0,0 +1,6 @@ +global: + scrape_native_histograms: true +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/global_enable_scrape_native_hist_and_scrape_protocols.good.yml b/config/testdata/global_enable_scrape_native_hist_and_scrape_protocols.good.yml new file mode 100644 index 00000000000..04474c53e3c --- /dev/null +++ b/config/testdata/global_enable_scrape_native_hist_and_scrape_protocols.good.yml @@ -0,0 +1,7 @@ +global: + scrape_native_histograms: true + scrape_protocols: ['PrometheusText0.0.4'] +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/global_scrape_protocols_and_local_disable_scrape_native_hist.good.yml b/config/testdata/global_scrape_protocols_and_local_disable_scrape_native_hist.good.yml new file mode 100644 index 00000000000..6cad07e3b0f --- /dev/null +++ b/config/testdata/global_scrape_protocols_and_local_disable_scrape_native_hist.good.yml @@ -0,0 +1,8 @@ +global: + scrape_native_histograms: true + scrape_protocols: ['PrometheusText0.0.4'] +scrape_configs: + - job_name: prometheus + scrape_native_histograms: false + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/jobname_dup.bad.yml b/config/testdata/jobname_dup.bad.yml index 0265493c30c..d03cb0cf979 100644 --- a/config/testdata/jobname_dup.bad.yml +++ b/config/testdata/jobname_dup.bad.yml @@ -1,4 +1,6 @@ # Two scrape configs with the same job names are not allowed. +global: + metric_name_validation_scheme: legacy scrape_configs: - job_name: prometheus - job_name: service-x diff --git a/config/testdata/kubernetes_selectors_pod.bad.yml b/config/testdata/kubernetes_selectors_pod.bad.yml index 3a1a83abd2d..24285c0ce0a 100644 --- a/config/testdata/kubernetes_selectors_pod.bad.yml +++ b/config/testdata/kubernetes_selectors_pod.bad.yml @@ -6,3 +6,6 @@ scrape_configs: - role: "node" label: "foo=bar" field: "metadata.status=Running" + - role: "service" + label: "baz=que" + field: "metadata.status=Running" diff --git a/config/testdata/kubernetes_selectors_pod.good.yml b/config/testdata/kubernetes_selectors_pod.good.yml index 91da6ada177..49c17d72cee 100644 --- a/config/testdata/kubernetes_selectors_pod.good.yml +++ b/config/testdata/kubernetes_selectors_pod.good.yml @@ -11,3 +11,8 @@ scrape_configs: - role: "pod" label: "foo in (bar,baz)" field: "metadata.status=Running" + - role: pod + selectors: + - role: "node" + label: "foo=bar" + field: "metadata.status=Running" diff --git a/config/testdata/labelmap.bad.yml b/config/testdata/labelmap.bad.yml index 29d2653990c..b8aa117acf8 100644 --- a/config/testdata/labelmap.bad.yml +++ b/config/testdata/labelmap.bad.yml @@ -2,4 +2,4 @@ scrape_configs: - job_name: prometheus relabel_configs: - action: labelmap - replacement: l-$1 + replacement: !!binary "/w==$1" diff --git a/config/testdata/labelname.bad.yml b/config/testdata/labelname.bad.yml index c06853a26bc..0c9c5ef3b2a 100644 --- a/config/testdata/labelname.bad.yml +++ b/config/testdata/labelname.bad.yml @@ -1,3 +1,3 @@ global: external_labels: - not$allowed: value + !!binary "/w==": value diff --git a/config/testdata/labelname2.bad.yml b/config/testdata/labelname2.bad.yml deleted file mode 100644 index 7afcd6bcfc4..00000000000 --- a/config/testdata/labelname2.bad.yml +++ /dev/null @@ -1,3 +0,0 @@ -global: - external_labels: - 'not:allowed': value diff --git a/config/testdata/local_convert_classic_hist_to_nhcb.good.yml b/config/testdata/local_convert_classic_hist_to_nhcb.good.yml new file mode 100644 index 00000000000..0fd31727eb2 --- /dev/null +++ b/config/testdata/local_convert_classic_hist_to_nhcb.good.yml @@ -0,0 +1,7 @@ +global: + convert_classic_histograms_to_nhcb: false +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] + convert_classic_histograms_to_nhcb: true diff --git a/config/testdata/local_disable_always_scrape_classic_hist.good.yml b/config/testdata/local_disable_always_scrape_classic_hist.good.yml new file mode 100644 index 00000000000..9e668340dc9 --- /dev/null +++ b/config/testdata/local_disable_always_scrape_classic_hist.good.yml @@ -0,0 +1,7 @@ +global: + always_scrape_classic_histograms: true +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] + always_scrape_classic_histograms: false diff --git a/config/testdata/local_disable_convert_classic_hist_to_nhcb.good.yml b/config/testdata/local_disable_convert_classic_hist_to_nhcb.good.yml new file mode 100644 index 00000000000..b41af7e0a54 --- /dev/null +++ b/config/testdata/local_disable_convert_classic_hist_to_nhcb.good.yml @@ -0,0 +1,7 @@ +global: + convert_classic_histograms_to_nhcb: true +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] + convert_classic_histograms_to_nhcb: false diff --git a/config/testdata/local_disable_extra_scrape_metrics.good.yml b/config/testdata/local_disable_extra_scrape_metrics.good.yml new file mode 100644 index 00000000000..a1b7c646fa6 --- /dev/null +++ b/config/testdata/local_disable_extra_scrape_metrics.good.yml @@ -0,0 +1,7 @@ +global: + extra_scrape_metrics: true +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] + extra_scrape_metrics: false diff --git a/config/testdata/local_disable_scrape_native_hist.good.yml b/config/testdata/local_disable_scrape_native_hist.good.yml new file mode 100644 index 00000000000..9cff5e74477 --- /dev/null +++ b/config/testdata/local_disable_scrape_native_hist.good.yml @@ -0,0 +1,7 @@ +global: + scrape_native_histograms: true +scrape_configs: + - job_name: prometheus + scrape_native_histograms: false + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/local_enable_always_scrape_classic_hist.good.yml b/config/testdata/local_enable_always_scrape_classic_hist.good.yml new file mode 100644 index 00000000000..165be077544 --- /dev/null +++ b/config/testdata/local_enable_always_scrape_classic_hist.good.yml @@ -0,0 +1,7 @@ +global: + always_scrape_classic_histograms: false +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] + always_scrape_classic_histograms: true diff --git a/config/testdata/local_enable_extra_scrape_metrics.good.yml b/config/testdata/local_enable_extra_scrape_metrics.good.yml new file mode 100644 index 00000000000..a1c8b2808ee --- /dev/null +++ b/config/testdata/local_enable_extra_scrape_metrics.good.yml @@ -0,0 +1,7 @@ +global: + extra_scrape_metrics: false +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:8080'] + extra_scrape_metrics: true diff --git a/config/testdata/local_enable_scrape_native_hist.good.yml b/config/testdata/local_enable_scrape_native_hist.good.yml new file mode 100644 index 00000000000..21d3a008562 --- /dev/null +++ b/config/testdata/local_enable_scrape_native_hist.good.yml @@ -0,0 +1,6 @@ +global: +scrape_configs: + - job_name: prometheus + scrape_native_histograms: true + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/local_enable_scrape_native_hist_and_scrape_protocols.good.yml b/config/testdata/local_enable_scrape_native_hist_and_scrape_protocols.good.yml new file mode 100644 index 00000000000..00e10304a8d --- /dev/null +++ b/config/testdata/local_enable_scrape_native_hist_and_scrape_protocols.good.yml @@ -0,0 +1,7 @@ +global: +scrape_configs: + - job_name: prometheus + scrape_native_histograms: true + scrape_protocols: ['PrometheusText0.0.4'] + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/lowercase.bad.yml b/config/testdata/lowercase.bad.yml index 9bc95833417..6dd72e6476a 100644 --- a/config/testdata/lowercase.bad.yml +++ b/config/testdata/lowercase.bad.yml @@ -1,3 +1,5 @@ +global: + metric_name_validation_scheme: legacy scrape_configs: - job_name: prometheus relabel_configs: diff --git a/config/testdata/lowercase2.bad.yml b/config/testdata/lowercase2.bad.yml deleted file mode 100644 index bde8862c66c..00000000000 --- a/config/testdata/lowercase2.bad.yml +++ /dev/null @@ -1,6 +0,0 @@ -scrape_configs: - - job_name: prometheus - relabel_configs: - - action: lowercase - source_labels: [__name__] - target_label: 42lab diff --git a/config/testdata/otlp_allow_keep_identifying_resource_attributes.good.yml b/config/testdata/otlp_allow_keep_identifying_resource_attributes.good.yml new file mode 100644 index 00000000000..63151e2a77a --- /dev/null +++ b/config/testdata/otlp_allow_keep_identifying_resource_attributes.good.yml @@ -0,0 +1,2 @@ +otlp: + keep_identifying_resource_attributes: true diff --git a/config/testdata/otlp_allow_utf8.bad.yml b/config/testdata/otlp_allow_utf8.bad.yml new file mode 100644 index 00000000000..488e4b05584 --- /dev/null +++ b/config/testdata/otlp_allow_utf8.bad.yml @@ -0,0 +1,4 @@ +global: + metric_name_validation_scheme: legacy +otlp: + translation_strategy: Invalid diff --git a/config/testdata/otlp_allow_utf8.good.yml b/config/testdata/otlp_allow_utf8.good.yml new file mode 100644 index 00000000000..f3069d2fddb --- /dev/null +++ b/config/testdata/otlp_allow_utf8.good.yml @@ -0,0 +1,2 @@ +otlp: + translation_strategy: NoUTF8EscapingWithSuffixes diff --git a/config/testdata/otlp_allow_utf8.incompatible.yml b/config/testdata/otlp_allow_utf8.incompatible.yml new file mode 100644 index 00000000000..2625c24131e --- /dev/null +++ b/config/testdata/otlp_allow_utf8.incompatible.yml @@ -0,0 +1,4 @@ +global: + metric_name_validation_scheme: legacy +otlp: + translation_strategy: NoUTF8EscapingWithSuffixes diff --git a/config/testdata/otlp_convert_histograms_to_nhcb.good.yml b/config/testdata/otlp_convert_histograms_to_nhcb.good.yml new file mode 100644 index 00000000000..1462cafe9b3 --- /dev/null +++ b/config/testdata/otlp_convert_histograms_to_nhcb.good.yml @@ -0,0 +1,2 @@ +otlp: + convert_histograms_to_nhcb: true diff --git a/config/testdata/otlp_empty.yml b/config/testdata/otlp_empty.yml new file mode 100644 index 00000000000..7085e9246bd --- /dev/null +++ b/config/testdata/otlp_empty.yml @@ -0,0 +1 @@ +global: diff --git a/config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml b/config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml new file mode 100644 index 00000000000..be4ee60f2ab --- /dev/null +++ b/config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml @@ -0,0 +1,2 @@ +otlp: + ignore_resource_attributes: ["k8s.job.name"] diff --git a/config/testdata/otlp_label_underscore_sanitization_defaults.good.yml b/config/testdata/otlp_label_underscore_sanitization_defaults.good.yml new file mode 100644 index 00000000000..3b1e9796d39 --- /dev/null +++ b/config/testdata/otlp_label_underscore_sanitization_defaults.good.yml @@ -0,0 +1,2 @@ +otlp: + promote_resource_attributes: ["service.name"] diff --git a/config/testdata/otlp_label_underscore_sanitization_disabled.good.yml b/config/testdata/otlp_label_underscore_sanitization_disabled.good.yml new file mode 100644 index 00000000000..f8fe4ea6699 --- /dev/null +++ b/config/testdata/otlp_label_underscore_sanitization_disabled.good.yml @@ -0,0 +1,3 @@ +otlp: + label_name_underscore_sanitization: false + label_name_preserve_multiple_underscores: false diff --git a/config/testdata/otlp_label_underscore_sanitization_enabled.good.yml b/config/testdata/otlp_label_underscore_sanitization_enabled.good.yml new file mode 100644 index 00000000000..8ce347495ea --- /dev/null +++ b/config/testdata/otlp_label_underscore_sanitization_enabled.good.yml @@ -0,0 +1,3 @@ +otlp: + label_name_underscore_sanitization: true + label_name_preserve_multiple_underscores: true diff --git a/config/testdata/otlp_no_translation.good.yml b/config/testdata/otlp_no_translation.good.yml new file mode 100644 index 00000000000..e5c44608425 --- /dev/null +++ b/config/testdata/otlp_no_translation.good.yml @@ -0,0 +1,2 @@ +otlp: + translation_strategy: NoTranslation diff --git a/config/testdata/otlp_no_translation.incompatible.yml b/config/testdata/otlp_no_translation.incompatible.yml new file mode 100644 index 00000000000..33c5a756f53 --- /dev/null +++ b/config/testdata/otlp_no_translation.incompatible.yml @@ -0,0 +1,4 @@ +global: + metric_name_validation_scheme: legacy +otlp: + translation_strategy: NoTranslation diff --git a/config/testdata/otlp_promote_all_resource_attributes.bad.yml b/config/testdata/otlp_promote_all_resource_attributes.bad.yml new file mode 100644 index 00000000000..2be54ec155b --- /dev/null +++ b/config/testdata/otlp_promote_all_resource_attributes.bad.yml @@ -0,0 +1,3 @@ +otlp: + promote_all_resource_attributes: true + promote_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name"] diff --git a/config/testdata/otlp_promote_scope_metadata.good.yml b/config/testdata/otlp_promote_scope_metadata.good.yml new file mode 100644 index 00000000000..d88f657abd8 --- /dev/null +++ b/config/testdata/otlp_promote_scope_metadata.good.yml @@ -0,0 +1,2 @@ +otlp: + promote_scope_metadata: true diff --git a/config/testdata/otlp_sanitize_default_resource_attributes.good.yml b/config/testdata/otlp_sanitize_default_resource_attributes.good.yml new file mode 100644 index 00000000000..abdd98dc7a2 --- /dev/null +++ b/config/testdata/otlp_sanitize_default_resource_attributes.good.yml @@ -0,0 +1 @@ +otlp: diff --git a/config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml b/config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml new file mode 100644 index 00000000000..57ce0efac09 --- /dev/null +++ b/config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml @@ -0,0 +1,3 @@ +otlp: + promote_all_resource_attributes: true + ignore_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name", ""] diff --git a/config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml b/config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml new file mode 100644 index 00000000000..7a50fef4051 --- /dev/null +++ b/config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml @@ -0,0 +1,3 @@ +otlp: + promote_all_resource_attributes: true + ignore_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name"] diff --git a/config/testdata/otlp_sanitize_resource_attributes.bad.yml b/config/testdata/otlp_sanitize_promote_resource_attributes.bad.yml similarity index 100% rename from config/testdata/otlp_sanitize_resource_attributes.bad.yml rename to config/testdata/otlp_sanitize_promote_resource_attributes.bad.yml diff --git a/config/testdata/otlp_sanitize_resource_attributes.good.yml b/config/testdata/otlp_sanitize_promote_resource_attributes.good.yml similarity index 100% rename from config/testdata/otlp_sanitize_resource_attributes.good.yml rename to config/testdata/otlp_sanitize_promote_resource_attributes.good.yml diff --git a/config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml b/config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml new file mode 100644 index 00000000000..2c8360011bc --- /dev/null +++ b/config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml @@ -0,0 +1,2 @@ +otlp: + promote_all_resource_attributes: true diff --git a/config/testdata/outscale_no_region.bad.yml b/config/testdata/outscale_no_region.bad.yml new file mode 100644 index 00000000000..f49102bb3d0 --- /dev/null +++ b/config/testdata/outscale_no_region.bad.yml @@ -0,0 +1,6 @@ +scrape_configs: + - job_name: outscale + outscale_sd_configs: + - region: "" + access_key: A1B2C3D4E5F6G7H8I9J0 + secret_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/config/testdata/outscale_no_secret.bad.yml b/config/testdata/outscale_no_secret.bad.yml new file mode 100644 index 00000000000..30602b44a99 --- /dev/null +++ b/config/testdata/outscale_no_secret.bad.yml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: prometheus + outscale_sd_configs: + - region: eu-west-2 + access_key: A1B2C3D4E5F6G7H8I9J0 diff --git a/config/testdata/outscale_two_secrets.bad.yml b/config/testdata/outscale_two_secrets.bad.yml new file mode 100644 index 00000000000..4e8e76c3529 --- /dev/null +++ b/config/testdata/outscale_two_secrets.bad.yml @@ -0,0 +1,7 @@ +scrape_configs: + - job_name: outscale + outscale_sd_configs: + - region: eu-west-2 + access_key: A1B2C3D4E5F6G7H8I9J0 + secret_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + secret_key_file: /tmp/outscale-secret-key diff --git a/config/testdata/remote_write_queue_capacity_zero.bad.yml b/config/testdata/remote_write_queue_capacity_zero.bad.yml new file mode 100644 index 00000000000..775b1e21a9a --- /dev/null +++ b/config/testdata/remote_write_queue_capacity_zero.bad.yml @@ -0,0 +1,4 @@ +remote_write: + - url: http://localhost:9090/api/v1/write + queue_config: + capacity: 0 diff --git a/config/testdata/remote_write_queue_max_backoff_less_than_min.bad.yml b/config/testdata/remote_write_queue_max_backoff_less_than_min.bad.yml new file mode 100644 index 00000000000..e0629b206ee --- /dev/null +++ b/config/testdata/remote_write_queue_max_backoff_less_than_min.bad.yml @@ -0,0 +1,5 @@ +remote_write: + - url: http://localhost:9090/api/v1/write + queue_config: + min_backoff: 10s + max_backoff: 1s diff --git a/config/testdata/remote_write_queue_max_samples_per_send_zero.bad.yml b/config/testdata/remote_write_queue_max_samples_per_send_zero.bad.yml new file mode 100644 index 00000000000..301e733632a --- /dev/null +++ b/config/testdata/remote_write_queue_max_samples_per_send_zero.bad.yml @@ -0,0 +1,4 @@ +remote_write: + - url: http://localhost:9090/api/v1/write + queue_config: + max_samples_per_send: 0 diff --git a/config/testdata/remote_write_queue_max_shards_zero.bad.yml b/config/testdata/remote_write_queue_max_shards_zero.bad.yml new file mode 100644 index 00000000000..5fd362e6923 --- /dev/null +++ b/config/testdata/remote_write_queue_max_shards_zero.bad.yml @@ -0,0 +1,4 @@ +remote_write: + - url: http://localhost:9090/api/v1/write + queue_config: + max_shards: 0 diff --git a/config/testdata/remote_write_queue_min_shards_greater_than_max.bad.yml b/config/testdata/remote_write_queue_min_shards_greater_than_max.bad.yml new file mode 100644 index 00000000000..020ecde766f --- /dev/null +++ b/config/testdata/remote_write_queue_min_shards_greater_than_max.bad.yml @@ -0,0 +1,5 @@ +remote_write: + - url: http://localhost:9090/api/v1/write + queue_config: + min_shards: 100 + max_shards: 10 diff --git a/config/testdata/remote_write_queue_min_shards_zero.bad.yml b/config/testdata/remote_write_queue_min_shards_zero.bad.yml new file mode 100644 index 00000000000..a925449c4af --- /dev/null +++ b/config/testdata/remote_write_queue_min_shards_zero.bad.yml @@ -0,0 +1,4 @@ +remote_write: + - url: http://localhost:9090/api/v1/write + queue_config: + min_shards: 0 diff --git a/config/testdata/roundtrip.good.yml b/config/testdata/roundtrip.good.yml index 24ab7d25927..d999dafd320 100644 --- a/config/testdata/roundtrip.good.yml +++ b/config/testdata/roundtrip.good.yml @@ -41,6 +41,7 @@ scrape_configs: - server: localhost:1234 token: services: [nginx, cache, mysql] + health_filter: 'Service.Tags contains "canary"' tags: [canary, v1] node_meta: rack: "123" diff --git a/config/testdata/scrape_config_files_fallback_scrape_protocol1.bad.yml b/config/testdata/scrape_config_files_fallback_scrape_protocol1.bad.yml new file mode 100644 index 00000000000..07cfe47594e --- /dev/null +++ b/config/testdata/scrape_config_files_fallback_scrape_protocol1.bad.yml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: node + fallback_scrape_protocol: "prometheusproto" + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/scrape_config_files_fallback_scrape_protocol2.bad.yml b/config/testdata/scrape_config_files_fallback_scrape_protocol2.bad.yml new file mode 100644 index 00000000000..c5d133f9c46 --- /dev/null +++ b/config/testdata/scrape_config_files_fallback_scrape_protocol2.bad.yml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: node + fallback_scrape_protocol: ["OpenMetricsText1.0.0", "PrometheusText0.0.4"] + static_configs: + - targets: ['localhost:8080'] diff --git a/config/testdata/scrape_config_global_validation_mode.yml b/config/testdata/scrape_config_global_validation_mode.yml index 1548554397c..fb4baf7b07c 100644 --- a/config/testdata/scrape_config_global_validation_mode.yml +++ b/config/testdata/scrape_config_global_validation_mode.yml @@ -1,4 +1,5 @@ global: - metric_name_validation_scheme: utf8 + metric_name_validation_scheme: legacy + metric_name_escaping_scheme: dots scrape_configs: - job_name: prometheus diff --git a/config/testdata/scrape_config_local_global_validation_mode.yml b/config/testdata/scrape_config_local_global_validation_mode.yml index d13605e21d9..29cd2b41409 100644 --- a/config/testdata/scrape_config_local_global_validation_mode.yml +++ b/config/testdata/scrape_config_local_global_validation_mode.yml @@ -1,5 +1,7 @@ global: - metric_name_validation_scheme: utf8 + metric_name_validation_scheme: legacy + metric_name_escaping_scheme: values scrape_configs: - job_name: prometheus - metric_name_validation_scheme: legacy + metric_name_validation_scheme: utf8 + metric_name_escaping_scheme: dots diff --git a/config/testdata/scrape_config_local_infer_escaping.yml b/config/testdata/scrape_config_local_infer_escaping.yml new file mode 100644 index 00000000000..90279ff0818 --- /dev/null +++ b/config/testdata/scrape_config_local_infer_escaping.yml @@ -0,0 +1,3 @@ +scrape_configs: + - job_name: prometheus + metric_name_validation_scheme: legacy diff --git a/config/testdata/scrape_config_local_validation_mode.yml b/config/testdata/scrape_config_local_validation_mode.yml index fad4235806a..b4d1ff05df9 100644 --- a/config/testdata/scrape_config_local_validation_mode.yml +++ b/config/testdata/scrape_config_local_validation_mode.yml @@ -1,3 +1,4 @@ scrape_configs: - job_name: prometheus - metric_name_validation_scheme: utf8 + metric_name_validation_scheme: legacy + metric_name_escaping_scheme: values diff --git a/config/testdata/scrape_config_utf8_conflicting.bad.yml b/config/testdata/scrape_config_utf8_conflicting.bad.yml new file mode 100644 index 00000000000..3f1b8f87ac3 --- /dev/null +++ b/config/testdata/scrape_config_utf8_conflicting.bad.yml @@ -0,0 +1,5 @@ +global: + metric_name_validation_scheme: legacy + metric_name_escaping_scheme: allow-utf-8 +scrape_configs: + - job_name: prometheus diff --git a/config/testdata/stackit_endpoint.bad.yml b/config/testdata/stackit_endpoint.bad.yml new file mode 100644 index 00000000000..ecb0fefe9ef --- /dev/null +++ b/config/testdata/stackit_endpoint.bad.yml @@ -0,0 +1,4 @@ +scrape_configs: + - job_name: stackit + stackit_sd_configs: + - endpoint: "://invalid" diff --git a/config/testdata/static_config.bad.json b/config/testdata/static_config.bad.json deleted file mode 100644 index 6050ed9c509..00000000000 --- a/config/testdata/static_config.bad.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "targets": ["1.2.3.4:9100"], - "labels": { - "some_valid_label": "foo", - "oops:this-label-is-invalid": "bar" - } -} diff --git a/config/testdata/static_config.bad.yml b/config/testdata/static_config.bad.yml index 1d229ec5e73..7e9003dcbf4 100644 --- a/config/testdata/static_config.bad.yml +++ b/config/testdata/static_config.bad.yml @@ -1,4 +1,4 @@ targets: ['1.2.3.4:9001', '1.2.3.5:9090'] labels: valid_label: foo - not:valid_label: bar + !!binary "/w==": bar diff --git a/config/testdata/tsdb_chunk_encoding_floats.bad.yml b/config/testdata/tsdb_chunk_encoding_floats.bad.yml new file mode 100644 index 00000000000..4ca1cadb58d --- /dev/null +++ b/config/testdata/tsdb_chunk_encoding_floats.bad.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + chunk_encoding: + floats: xor3 diff --git a/config/testdata/tsdb_chunk_encoding_floats_wrong_case.bad.yml b/config/testdata/tsdb_chunk_encoding_floats_wrong_case.bad.yml new file mode 100644 index 00000000000..d38459d1ac0 --- /dev/null +++ b/config/testdata/tsdb_chunk_encoding_floats_wrong_case.bad.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + chunk_encoding: + floats: XOR diff --git a/config/testdata/tsdb_chunk_encoding_floats_wrong_case_xor2.bad.yml b/config/testdata/tsdb_chunk_encoding_floats_wrong_case_xor2.bad.yml new file mode 100644 index 00000000000..58965a9506b --- /dev/null +++ b/config/testdata/tsdb_chunk_encoding_floats_wrong_case_xor2.bad.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + chunk_encoding: + floats: XOR2 diff --git a/config/testdata/tsdb_chunk_encoding_floats_xor.good.yml b/config/testdata/tsdb_chunk_encoding_floats_xor.good.yml new file mode 100644 index 00000000000..71d71a42ead --- /dev/null +++ b/config/testdata/tsdb_chunk_encoding_floats_xor.good.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + chunk_encoding: + floats: xor diff --git a/config/testdata/tsdb_chunk_encoding_floats_xor2.good.yml b/config/testdata/tsdb_chunk_encoding_floats_xor2.good.yml new file mode 100644 index 00000000000..e17c2cc753c --- /dev/null +++ b/config/testdata/tsdb_chunk_encoding_floats_xor2.good.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + chunk_encoding: + floats: xor2 diff --git a/config/testdata/tsdb_retention_percentage.bad.yml b/config/testdata/tsdb_retention_percentage.bad.yml new file mode 100644 index 00000000000..cb57abe0c02 --- /dev/null +++ b/config/testdata/tsdb_retention_percentage.bad.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + retention: + percentage: 101 diff --git a/config/testdata/tsdb_retention_percentage_float.good.yml b/config/testdata/tsdb_retention_percentage_float.good.yml new file mode 100644 index 00000000000..cf82ebf5faf --- /dev/null +++ b/config/testdata/tsdb_retention_percentage_float.good.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + retention: + percentage: 0.5 diff --git a/config/testdata/tsdb_retention_percentage_negative.bad.yml b/config/testdata/tsdb_retention_percentage_negative.bad.yml new file mode 100644 index 00000000000..2eeb60c091f --- /dev/null +++ b/config/testdata/tsdb_retention_percentage_negative.bad.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + retention: + percentage: -1 diff --git a/config/testdata/tsdb_retention_size.bad.yml b/config/testdata/tsdb_retention_size.bad.yml new file mode 100644 index 00000000000..ecae64aae6b --- /dev/null +++ b/config/testdata/tsdb_retention_size.bad.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + retention: + size: -1GB diff --git a/config/testdata/tsdb_retention_time.bad.yml b/config/testdata/tsdb_retention_time.bad.yml new file mode 100644 index 00000000000..465b3cf5da5 --- /dev/null +++ b/config/testdata/tsdb_retention_time.bad.yml @@ -0,0 +1,4 @@ +storage: + tsdb: + retention: + time: -1h diff --git a/config/testdata/uppercase2.bad.yml b/config/testdata/uppercase2.bad.yml deleted file mode 100644 index 330b9aceb68..00000000000 --- a/config/testdata/uppercase2.bad.yml +++ /dev/null @@ -1,6 +0,0 @@ -scrape_configs: - - job_name: prometheus - relabel_configs: - - action: uppercase - source_labels: [__name__] - target_label: 42lab diff --git a/console_libraries/menu.lib b/console_libraries/menu.lib deleted file mode 100644 index 199ebf9f480..00000000000 --- a/console_libraries/menu.lib +++ /dev/null @@ -1,82 +0,0 @@ -{{/* vim: set ft=html: */}} - -{{/* Navbar, should be passed . */}} -{{ define "navbar" }} - -{{ end }} - -{{/* LHS menu, should be passed . */}} -{{ define "menu" }} -
- -
-{{ end }} - -{{/* Helper, pass (args . path name) */}} -{{ define "_menuItem" }} -
  • {{ .arg2 }}
    • -{{ end }} - diff --git a/console_libraries/prom.lib b/console_libraries/prom.lib deleted file mode 100644 index d7d436f9474..00000000000 --- a/console_libraries/prom.lib +++ /dev/null @@ -1,138 +0,0 @@ -{{/* vim: set ft=html: */}} -{{/* Load Prometheus console library JS/CSS. Should go in */}} -{{ define "prom_console_head" }} - - - - - - - - - - - - - -{{ end }} - -{{/* Top of all pages. */}} -{{ define "head" -}} - - - -{{ template "prom_console_head" }} - - -{{ template "navbar" . }} - -{{ template "menu" . }} -{{ end }} - -{{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }} -{{ define "humanize" }}{{ humanize . }}{{ end }} -{{ define "humanizeNoSmallPrefix" }}{{ if and (lt . 1.0) (gt . -1.0) }}{{ printf "%.3g" . }}{{ else }}{{ humanize . }}{{ end }}{{ end }} -{{ define "humanize1024" }}{{ humanize1024 . }}{{ end }} -{{ define "humanizeDuration" }}{{ humanizeDuration . }}{{ end }} -{{ define "humanizePercentage" }}{{ humanizePercentage . }}{{ end }} -{{ define "humanizeTimestamp" }}{{ humanizeTimestamp . }}{{ end }} -{{ define "printf.1f" }}{{ printf "%.1f" . }}{{ end }} -{{ define "printf.3g" }}{{ printf "%.3g" . }}{{ end }} - -{{/* prom_query_drilldown (args expr suffix? renderTemplate?) -Displays the result of the expression, with a link to /graph for it. - -renderTemplate is the name of the template to use to render the value. -*/}} -{{ define "prom_query_drilldown" }} -{{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }} -{{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }} -{{ end }} - -{{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&{{ end }}{{ end }}" - -{{ define "prom_right_table_head" }} -
    - -{{ end }} -{{ define "prom_right_table_tail" }} -
    -
    -{{ end }} - -{{/* RHS table head, pass job name. Should be used after prom_right_table_head. */}} -{{ define "prom_right_table_job_head" }} - - {{ . }} - {{ template "prom_query_drilldown" (args (printf "sum(up{job='%s'})" .)) }} / {{ template "prom_query_drilldown" (args (printf "count(up{job='%s'})" .)) }} - - - CPU - {{ template "prom_query_drilldown" (args (printf "avg by(job)(irate(process_cpu_seconds_total{job='%s'}[5m]))" .) "s/s" "humanizeNoSmallPrefix") }} - - - Memory - {{ template "prom_query_drilldown" (args (printf "avg by(job)(process_resident_memory_bytes{job='%s'})" .) "B" "humanize1024") }} - -{{ end }} - - -{{ define "prom_content_head" }} -
    -
    -{{ template "prom_graph_timecontrol" . }} -{{ end }} -{{ define "prom_content_tail" }} -
    -
    -{{ end }} - -{{ define "prom_graph_timecontrol" }} -
    -
    -
    - -
    -
    - -
    -
    -
    - - - -
    -
    -
    - -
    -{{ end }} - -{{/* Bottom of all pages. */}} -{{ define "tail" }} - - -{{ end }} diff --git a/consoles/index.html.example b/consoles/index.html.example deleted file mode 100644 index c725d30dea3..00000000000 --- a/consoles/index.html.example +++ /dev/null @@ -1,28 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Overview

    -

    These are example consoles for Prometheus.

    - -

    These consoles expect exporters to have the following job labels:

    - - - - - - - - - - - - - -
    ExporterJob label
    Node Exporternode
    Prometheusprometheus
    - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/consoles/node-cpu.html b/consoles/node-cpu.html deleted file mode 100644 index 284ad738f2b..00000000000 --- a/consoles/node-cpu.html +++ /dev/null @@ -1,60 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - CPU(s): {{ template "prom_query_drilldown" (args (printf "scalar(count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'})))" .Params.instance)) }} - -{{ range printf "sum by (mode)(irate(node_cpu_seconds_total{job='node',instance='%s'}[5m])) * 100 / scalar(count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'})))" .Params.instance .Params.instance | query | sortByLabel "mode" }} - - {{ .Labels.mode | title }} CPU - {{ .Value | printf "%.1f" }}% - -{{ end }} - Misc - - Processes Running - {{ template "prom_query_drilldown" (args (printf "node_procs_running{job='node',instance='%s'}" .Params.instance) "" "humanize") }} - - - Processes Blocked - {{ template "prom_query_drilldown" (args (printf "node_procs_blocked{job='node',instance='%s'}" .Params.instance) "" "humanize") }} - - - Forks - {{ template "prom_query_drilldown" (args (printf "irate(node_forks_total{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} - - - Context Switches - {{ template "prom_query_drilldown" (args (printf "irate(node_context_switches_total{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} - - - Interrupts - {{ template "prom_query_drilldown" (args (printf "irate(node_intr_total{job='node',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} - - - 1m Loadavg - {{ template "prom_query_drilldown" (args (printf "node_load1{job='node',instance='%s'}" .Params.instance)) }} - - - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Node CPU - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    - -

    CPU Usage

    -
    - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/consoles/node-disk.html b/consoles/node-disk.html deleted file mode 100644 index ffff41b7978..00000000000 --- a/consoles/node-disk.html +++ /dev/null @@ -1,78 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - Disks - -{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }} - {{ .Labels.device }} - - Utilization - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} - - - Throughput - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} - - - Avg Read Time - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} - - - Avg Write Time - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} - -{{ end }} - - Filesystem Fullness - -{{ define "roughlyNearZero" }} -{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} -{{ end }} -{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} - - {{ .Labels.mountpoint }} - {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} - -{{ end }} - - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Node Disk - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    - -

    Disk I/O Utilization

    -
    - -

    Filesystem Usage

    -
    - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/consoles/node-overview.html b/consoles/node-overview.html deleted file mode 100644 index 4ae8984b99a..00000000000 --- a/consoles/node-overview.html +++ /dev/null @@ -1,121 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - Overview - - User CPU - {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} - - - System CPU - {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} - - - Memory Total - {{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Memory Free - {{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Network - -{{ range printf "node_network_receive_bytes_total{job='node',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Received - {{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} - - - {{ .Labels.device }} Transmitted - {{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} - -{{ end }} - - Disks - -{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Utilization - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} - -{{ end }} -{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Throughput - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} - -{{ end }} - - Filesystem Fullness - -{{ define "roughlyNearZero" }} -{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} -{{ end }} -{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} - - {{ .Labels.mountpoint }} - {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} - -{{ end }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Node Overview - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    - -

    CPU Usage

    -
    - - -

    Disk I/O Utilization

    -
    - - -

    Memory

    -
    - - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/consoles/node.html b/consoles/node.html deleted file mode 100644 index c1dfc1a8913..00000000000 --- a/consoles/node.html +++ /dev/null @@ -1,35 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - Node - {{ template "prom_query_drilldown" (args "sum(up{job='node'})") }} / {{ template "prom_query_drilldown" (args "count(up{job='node'})") }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Node

    - - - - - - - - -{{ range query "up{job='node'}" | sortByLabel "instance" }} - - - Yes{{ else }} class="alert-danger">No{{ end }} - - - -{{ else }} - -{{ end }} -
    NodeUpCPU
    Used    		Memory
    Available
    {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Labels.instance }}{{ template "prom_query_drilldown" (args (printf "100 * (1 - avg by(instance) (sum without(mode) (irate(node_cpu_seconds_total{job='node',mode=~'idle|iowait|steal',instance='%s'}[5m]))))" .Labels.instance) "%" "printf.1f") }}{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'} + node_memory_Cached_bytes{job='node',instance='%s'} + node_memory_Buffers_bytes{job='node',instance='%s'}" .Labels.instance .Labels.instance .Labels.instance) "B" "humanize1024") }}
    No nodes found.
    - - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/consoles/prometheus-overview.html b/consoles/prometheus-overview.html deleted file mode 100644 index 08e027de066..00000000000 --- a/consoles/prometheus-overview.html +++ /dev/null @@ -1,96 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - Overview - - - CPU - {{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }} - - - Memory - {{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Version - {{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}} - - - - Storage - - - Ingested Samples - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }} - - - Head Series - {{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }} - - - Blocks Loaded - {{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }} - - - Rules - - - Evaluation Duration - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }} - - - Notification Latency - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }} - - - Notification Queue - {{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }} - - - HTTP Server - -{{ range printf "prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s'}" .Params.instance | query | sortByLabel "handler" }} - - {{ .Labels.handler }} - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s',handler='%s'}[5m])" .Labels.instance .Labels.handler) "/s" "humanizeNoSmallPrefix") }} - -{{ end }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -
    -

    Prometheus Overview - {{ .Params.instance }}

    - -

    Ingested Samples

    -
    - - -

    HTTP Server

    -
    - -
    -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/consoles/prometheus.html b/consoles/prometheus.html deleted file mode 100644 index e0d026376d1..00000000000 --- a/consoles/prometheus.html +++ /dev/null @@ -1,34 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - Prometheus - {{ template "prom_query_drilldown" (args "sum(up{job='prometheus'})") }} / {{ template "prom_query_drilldown" (args "count(up{job='prometheus'})") }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Prometheus

    - - - - - - - - -{{ range query "up{job='prometheus'}" | sortByLabel "instance" }} - - - - - - -{{ else }} - -{{ end }} -
    PrometheusUpIngested SamplesMemory
    {{ .Labels.instance }}Yes{{ else }} class="alert-danger">No{{ end }}{{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Labels.instance) "/s" "humanizeNoSmallPrefix") }}{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Labels.instance) "B" "humanize1024")}}
    No devices found.
    - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/discovery/README.md b/discovery/README.md index 4c066086256..5d1adcf1452 100644 --- a/discovery/README.md +++ b/discovery/README.md @@ -50,7 +50,7 @@ file for use with `file_sd`. The general principle with SD is to extract all the potentially useful information we can out of the SD, and let the user choose what they need of it using -[relabelling](https://prometheus.io/docs/operating/configuration/#). +[relabelling](https://prometheus.io/docs/operating/configuration/#relabel_config). This information is generally termed metadata. Metadata is exposed as a set of key/value pairs (labels) per target. The keys @@ -233,7 +233,7 @@ type Config interface { } type DiscovererOptions struct { - Logger log.Logger + Logger *slog.Logger // A registerer for the Discoverer's metrics. Registerer prometheus.Registerer diff --git a/discovery/aws/aws.go b/discovery/aws/aws.go new file mode 100644 index 00000000000..89cae23f028 --- /dev/null +++ b/discovery/aws/aws.go @@ -0,0 +1,430 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "errors" + "fmt" + "time" + + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + + "github.com/prometheus/prometheus/discovery" +) + +// DefaultSDConfig is the default AWS SD configuration. +var DefaultSDConfig = SDConfig{ + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +func init() { + discovery.RegisterConfig(&SDConfig{}) +} + +// Role is role of the service in AWS. +type Role string + +// The valid options for Role. +const ( + RoleEC2 Role = "ec2" + RoleECS Role = "ecs" + RoleElasticache Role = "elasticache" + RoleLightsail Role = "lightsail" + RoleMSK Role = "msk" + RoleRDS Role = "rds" +) + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *Role) UnmarshalYAML(unmarshal func(any) error) error { + if err := unmarshal((*string)(c)); err != nil { + return err + } + switch *c { + case RoleEC2, RoleECS, RoleElasticache, RoleLightsail, RoleMSK, RoleRDS: + return nil + default: + return fmt.Errorf("unknown AWS SD role %q", *c) + } +} + +func (c Role) String() string { + return string(c) +} + +// Filter is the configuration for filtering AWS resources. +type Filter struct { + Name string `yaml:"name"` + Values []string `yaml:"values"` +} + +// SDConfig is the configuration for AWS service discovery. +type SDConfig struct { + Role Role `yaml:"role"` + Region string `yaml:"region,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + ExternalID string `yaml:"external_id,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port,omitempty"` + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` + + // ec2, rds specific + Filters []*Filter `yaml:"filters,omitempty"` + + // ecs, msk specific + Clusters []string `yaml:"clusters,omitempty"` + + // Embedded sub-configs (internal use only, not serialized) + *EC2SDConfig `yaml:"-"` + *ECSSDConfig `yaml:"-"` + *ElasticacheSDConfig `yaml:"-"` + *LightsailSDConfig `yaml:"-"` + *MSKSDConfig `yaml:"-"` + *RDSSDConfig `yaml:"-"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for SDConfig. +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { + // Alias to avoid recursion + type plain SDConfig + var aux plain + // Unmarshal into aux + if err := unmarshal(&aux); err != nil { + return err + } + *c = SDConfig(aux) + + var err error + c.Region, err = loadRegion(context.Background(), c.Region) + if err != nil { + return fmt.Errorf("could not determine AWS region: %w", err) + } + + switch c.Role { + case RoleEC2: + if c.EC2SDConfig == nil { + ec2Config := DefaultEC2SDConfig + c.EC2SDConfig = &ec2Config + } + c.EC2SDConfig.HTTPClientConfig = c.HTTPClientConfig + c.EC2SDConfig.Region = c.Region + if c.Endpoint != "" { + c.EC2SDConfig.Endpoint = c.Endpoint + } + if c.AccessKey != "" { + c.EC2SDConfig.AccessKey = c.AccessKey + } + if c.SecretKey != "" { + c.EC2SDConfig.SecretKey = c.SecretKey + } + if c.Profile != "" { + c.EC2SDConfig.Profile = c.Profile + } + if c.RoleARN != "" { + c.EC2SDConfig.RoleARN = c.RoleARN + } + if c.ExternalID != "" { + c.EC2SDConfig.ExternalID = c.ExternalID + } + if c.Port != 0 { + c.EC2SDConfig.Port = c.Port + } + if c.RefreshInterval != 0 { + c.EC2SDConfig.RefreshInterval = c.RefreshInterval + } + if c.Filters != nil { + c.EC2SDConfig.Filters = c.Filters + } + case RoleECS: + if c.ECSSDConfig == nil { + ecsConfig := DefaultECSSDConfig + c.ECSSDConfig = &ecsConfig + } + c.ECSSDConfig.HTTPClientConfig = c.HTTPClientConfig + c.ECSSDConfig.Region = c.Region + if c.Endpoint != "" { + c.ECSSDConfig.Endpoint = c.Endpoint + } + if c.AccessKey != "" { + c.ECSSDConfig.AccessKey = c.AccessKey + } + if c.SecretKey != "" { + c.ECSSDConfig.SecretKey = c.SecretKey + } + if c.Profile != "" { + c.ECSSDConfig.Profile = c.Profile + } + if c.RoleARN != "" { + c.ECSSDConfig.RoleARN = c.RoleARN + } + if c.ExternalID != "" { + c.ECSSDConfig.ExternalID = c.ExternalID + } + if c.Port != 0 { + c.ECSSDConfig.Port = c.Port + } + if c.RefreshInterval != 0 { + c.ECSSDConfig.RefreshInterval = c.RefreshInterval + } + if c.Clusters != nil { + c.ECSSDConfig.Clusters = c.Clusters + } + case RoleElasticache: + if c.ElasticacheSDConfig == nil { + elasticacheConfig := DefaultElasticacheSDConfig + c.ElasticacheSDConfig = &elasticacheConfig + } + c.ElasticacheSDConfig.HTTPClientConfig = c.HTTPClientConfig + c.ElasticacheSDConfig.Region = c.Region + if c.Endpoint != "" { + c.ElasticacheSDConfig.Endpoint = c.Endpoint + } + if c.AccessKey != "" { + c.ElasticacheSDConfig.AccessKey = c.AccessKey + } + if c.SecretKey != "" { + c.ElasticacheSDConfig.SecretKey = c.SecretKey + } + if c.Profile != "" { + c.ElasticacheSDConfig.Profile = c.Profile + } + if c.RoleARN != "" { + c.ElasticacheSDConfig.RoleARN = c.RoleARN + } + if c.ExternalID != "" { + c.ElasticacheSDConfig.ExternalID = c.ExternalID + } + if c.Port != 0 { + c.ElasticacheSDConfig.Port = c.Port + } + if c.RefreshInterval != 0 { + c.ElasticacheSDConfig.RefreshInterval = c.RefreshInterval + } + if c.Clusters != nil { + c.ElasticacheSDConfig.Clusters = c.Clusters + } + case RoleLightsail: + if c.LightsailSDConfig == nil { + lightsailConfig := DefaultLightsailSDConfig + c.LightsailSDConfig = &lightsailConfig + } + c.LightsailSDConfig.HTTPClientConfig = c.HTTPClientConfig + c.LightsailSDConfig.Region = c.Region + if c.Endpoint != "" { + c.LightsailSDConfig.Endpoint = c.Endpoint + } + if c.AccessKey != "" { + c.LightsailSDConfig.AccessKey = c.AccessKey + } + if c.SecretKey != "" { + c.LightsailSDConfig.SecretKey = c.SecretKey + } + if c.Profile != "" { + c.LightsailSDConfig.Profile = c.Profile + } + if c.RoleARN != "" { + c.LightsailSDConfig.RoleARN = c.RoleARN + } + if c.ExternalID != "" { + c.LightsailSDConfig.ExternalID = c.ExternalID + } + if c.Port != 0 { + c.LightsailSDConfig.Port = c.Port + } + if c.RefreshInterval != 0 { + c.LightsailSDConfig.RefreshInterval = c.RefreshInterval + } + case RoleMSK: + if c.MSKSDConfig == nil { + mskConfig := DefaultMSKSDConfig + c.MSKSDConfig = &mskConfig + } + c.MSKSDConfig.HTTPClientConfig = c.HTTPClientConfig + c.MSKSDConfig.Region = c.Region + if c.Endpoint != "" { + c.MSKSDConfig.Endpoint = c.Endpoint + } + if c.AccessKey != "" { + c.MSKSDConfig.AccessKey = c.AccessKey + } + if c.SecretKey != "" { + c.MSKSDConfig.SecretKey = c.SecretKey + } + if c.Profile != "" { + c.MSKSDConfig.Profile = c.Profile + } + if c.RoleARN != "" { + c.MSKSDConfig.RoleARN = c.RoleARN + } + if c.ExternalID != "" { + c.MSKSDConfig.ExternalID = c.ExternalID + } + if c.Port != 0 { + c.MSKSDConfig.Port = c.Port + } + if c.RefreshInterval != 0 { + c.MSKSDConfig.RefreshInterval = c.RefreshInterval + } + if c.Clusters != nil { + c.MSKSDConfig.Clusters = c.Clusters + } + case RoleRDS: + if c.RDSSDConfig == nil { + rdsConfig := DefaultRDSSDConfig + c.RDSSDConfig = &rdsConfig + } + c.RDSSDConfig.HTTPClientConfig = c.HTTPClientConfig + c.RDSSDConfig.Region = c.Region + if c.Endpoint != "" { + c.RDSSDConfig.Endpoint = c.Endpoint + } + if c.AccessKey != "" { + c.RDSSDConfig.AccessKey = c.AccessKey + } + if c.SecretKey != "" { + c.RDSSDConfig.SecretKey = c.SecretKey + } + if c.Profile != "" { + c.RDSSDConfig.Profile = c.Profile + } + if c.RoleARN != "" { + c.RDSSDConfig.RoleARN = c.RoleARN + } + if c.ExternalID != "" { + c.RDSSDConfig.ExternalID = c.ExternalID + } + if c.Port != 0 { + c.RDSSDConfig.Port = c.Port + } + if c.RefreshInterval != 0 { + c.RDSSDConfig.RefreshInterval = c.RefreshInterval + } + if c.Filters != nil { + c.RDSSDConfig.Filters = c.Filters + } + if c.Clusters != nil { + c.RDSSDConfig.Clusters = c.Clusters + } + default: + return fmt.Errorf("unknown AWS SD role %q", c.Role) + } + return nil +} + +// Name returns the name of the AWS Config. +func (*SDConfig) Name() string { return "aws" } + +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &awsMetrics{refreshMetrics: rmi} +} + +// NewDiscoverer returns a Discoverer for the AWS Config. +func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + awsMetrics, ok := opts.Metrics.(*awsMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type for AWS SD") + } + + switch c.Role { + case RoleEC2: + opts.Metrics = &ec2Metrics{refreshMetrics: awsMetrics.refreshMetrics} + return NewEC2Discovery(c.EC2SDConfig, opts) + case RoleECS: + opts.Metrics = &ecsMetrics{refreshMetrics: awsMetrics.refreshMetrics} + return NewECSDiscovery(c.ECSSDConfig, opts) + case RoleElasticache: + opts.Metrics = &elasticacheMetrics{refreshMetrics: awsMetrics.refreshMetrics} + return NewElasticacheDiscovery(c.ElasticacheSDConfig, opts) + case RoleLightsail: + opts.Metrics = &lightsailMetrics{refreshMetrics: awsMetrics.refreshMetrics} + return NewLightsailDiscovery(c.LightsailSDConfig, opts) + case RoleMSK: + opts.Metrics = &mskMetrics{refreshMetrics: awsMetrics.refreshMetrics} + return NewMSKDiscovery(c.MSKSDConfig, opts) + case RoleRDS: + opts.Metrics = &rdsMetrics{refreshMetrics: awsMetrics.refreshMetrics} + return NewRDSDiscovery(c.RDSSDConfig, opts) + default: + return nil, fmt.Errorf("unknown AWS SD role %q", c.Role) + } +} + +// SetDirectory joins any relative file paths with dir. +func (c *SDConfig) SetDirectory(dir string) { + switch c.Role { + case RoleEC2: + if c.EC2SDConfig != nil { + c.EC2SDConfig.SetDirectory(dir) + } + case RoleECS: + if c.ECSSDConfig != nil { + c.ECSSDConfig.SetDirectory(dir) + } + case RoleElasticache: + if c.ElasticacheSDConfig != nil { + c.ElasticacheSDConfig.SetDirectory(dir) + } + case RoleLightsail: + if c.LightsailSDConfig != nil { + c.LightsailSDConfig.SetDirectory(dir) + } + case RoleMSK: + if c.MSKSDConfig != nil { + c.MSKSDConfig.SetDirectory(dir) + } + case RoleRDS: + if c.RDSSDConfig != nil { + c.RDSSDConfig.SetDirectory(dir) + } + } +} + +// loadRegion finds the region in order: AWS config/env vars ->IMDS. +func loadRegion(ctx context.Context, specifiedRegion string) (string, error) { + if specifiedRegion != "" { + return specifiedRegion, nil + } + + cfg, err := awsConfig.LoadDefaultConfig(ctx) + if err != nil { + return "", fmt.Errorf("failed to load AWS config: %w", err) + } + + if cfg.Region != "" { + return cfg.Region, nil + } + + // Fallback (may fail in non-AWS environments) + imdsClient := imds.NewFromConfig(cfg) + region, err := imdsClient.GetRegion(ctx, &imds.GetRegionInput{}) + if err != nil { + return "", fmt.Errorf("failed to get region from IMDS: %w", err) + } + + if region.Region == "" { + return "", errors.New("region not found in AWS config or IMDS") + } + + return region.Region, nil +} diff --git a/discovery/aws/aws_test.go b/discovery/aws/aws_test.go new file mode 100644 index 00000000000..e135515cc17 --- /dev/null +++ b/discovery/aws/aws_test.go @@ -0,0 +1,600 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "errors" + "math/rand/v2" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +func TestRoleUnmarshalYAML(t *testing.T) { + t.Parallel() + tests := []struct { + name string + input string + expected Role + wantErr bool + }{ + { + name: "EC2Role", + input: "ec2", + expected: RoleEC2, + wantErr: false, + }, + { + name: "LightsailRole", + input: "lightsail", + expected: RoleLightsail, + wantErr: false, + }, + { + name: "ECSRole", + input: "ecs", + expected: RoleECS, + wantErr: false, + }, + { + name: "InvalidRole", + input: "invalid", + expected: "invalid", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var r Role + err := r.UnmarshalYAML(func(v any) error { + ptr, ok := v.(*string) + if !ok { + return errors.New("not a string pointer") + } + *ptr = tt.input + return nil + }) + if tt.wantErr { + require.Error(t, err, "expected error for input %q", tt.input) + } else { + require.NoError(t, err, "unexpected error for input %q", tt.input) + require.Equal(t, tt.expected, r, "unexpected role for input %q", tt.input) + } + }) + } +} + +func TestRoleString(t *testing.T) { + t.Parallel() + tests := []struct { + name string + role Role + expected string + }{ + { + name: "EC2", + role: RoleEC2, + expected: "ec2", + }, + { + name: "Lightsail", + role: RoleLightsail, + expected: "lightsail", + }, + { + name: "ECS", + role: RoleECS, + expected: "ecs", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, tt.role.String()) + }) + } +} + +func TestSDConfigName(t *testing.T) { + t.Parallel() + cfg := &SDConfig{} + require.Equal(t, "aws", cfg.Name()) +} + +func TestDefaultSDConfig(t *testing.T) { + t.Parallel() + require.Equal(t, Role(""), DefaultSDConfig.Role) + require.Equal(t, model.Duration(60*time.Second), DefaultSDConfig.RefreshInterval) +} + +func TestSDConfigUnmarshalYAML(t *testing.T) { + t.Parallel() + tests := []struct { + name string + yaml string + validateFunc func(t *testing.T, cfg *SDConfig) + }{ + { + name: "EC2WithFlatFields", + yaml: `role: ec2 +region: us-west-2 +port: 9100 +filters: + - name: instance-state-name + values: [running]`, + validateFunc: func(t *testing.T, cfg *SDConfig) { + require.Equal(t, RoleEC2, cfg.Role) + require.NotNil(t, cfg.EC2SDConfig) + require.Equal(t, "us-west-2", cfg.EC2SDConfig.Region) + require.Equal(t, 9100, cfg.EC2SDConfig.Port) + require.Len(t, cfg.EC2SDConfig.Filters, 1) + require.Equal(t, "instance-state-name", cfg.EC2SDConfig.Filters[0].Name) + require.Equal(t, []string{"running"}, cfg.EC2SDConfig.Filters[0].Values) + }, + }, + { + name: "ECSWithFlatFields", + yaml: `role: ecs +region: us-east-1 +port: 9200 +clusters: ["some-cluster"]`, + validateFunc: func(t *testing.T, cfg *SDConfig) { + require.Equal(t, RoleECS, cfg.Role) + require.NotNil(t, cfg.ECSSDConfig) + require.Equal(t, "us-east-1", cfg.ECSSDConfig.Region) + require.Equal(t, 9200, cfg.ECSSDConfig.Port) + require.Equal(t, []string{"some-cluster"}, cfg.ECSSDConfig.Clusters) + }, + }, + { + name: "LightsailWithFlatFields", + yaml: `role: lightsail +region: eu-central-1 +port: 9300`, + validateFunc: func(t *testing.T, cfg *SDConfig) { + require.Equal(t, RoleLightsail, cfg.Role) + require.NotNil(t, cfg.LightsailSDConfig) + require.Equal(t, "eu-central-1", cfg.LightsailSDConfig.Region) + require.Equal(t, 9300, cfg.LightsailSDConfig.Port) + }, + }, + { + name: "RDSWithFlatFields", + yaml: `role: rds +region: us-east-1 +port: 9400 +filters: + - name: engine + values: [aurora-postgresql]`, + validateFunc: func(t *testing.T, cfg *SDConfig) { + require.Equal(t, RoleRDS, cfg.Role) + require.NotNil(t, cfg.RDSSDConfig) + require.Equal(t, "us-east-1", cfg.RDSSDConfig.Region) + require.Equal(t, 9400, cfg.RDSSDConfig.Port) + require.Len(t, cfg.RDSSDConfig.Filters, 1) + require.Equal(t, "engine", cfg.RDSSDConfig.Filters[0].Name) + require.Equal(t, []string{"aurora-postgresql"}, cfg.RDSSDConfig.Filters[0].Values) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var cfg SDConfig + require.NoError(t, yaml.Unmarshal([]byte(tt.yaml), &cfg)) + tt.validateFunc(t, &cfg) + }) + } +} + +// TestMultipleSDConfigsDoNotShareState verifies that multiple AWS SD configs +// don't share the same underlying configuration object. This was a bug where +// all configs pointed to the same global default, causing port and other +// settings from one job to overwrite settings in another job. +func TestMultipleSDConfigsDoNotShareState(t *testing.T) { + t.Parallel() + tests := []struct { + name string + yaml string + validateFunc func(t *testing.T, cfg1, cfg2 *SDConfig) + }{ + { + name: "EC2MultipleJobsDifferentPorts", + yaml: ` +- role: ec2 + region: us-west-2 + port: 9100 + filters: + - name: tag:Name + values: [host-1] +- role: ec2 + region: us-west-2 + port: 9101 + filters: + - name: tag:Name + values: [host-2]`, + validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) { + require.Equal(t, RoleEC2, cfg1.Role) + require.Equal(t, RoleEC2, cfg2.Role) + require.NotNil(t, cfg1.EC2SDConfig) + require.NotNil(t, cfg2.EC2SDConfig) + + // Verify ports are different and not shared + require.Equal(t, 9100, cfg1.EC2SDConfig.Port) + require.Equal(t, 9101, cfg2.EC2SDConfig.Port) + + // Verify filters are different and not shared + require.Len(t, cfg1.EC2SDConfig.Filters, 1) + require.Len(t, cfg2.EC2SDConfig.Filters, 1) + require.Equal(t, []string{"host-1"}, cfg1.EC2SDConfig.Filters[0].Values) + require.Equal(t, []string{"host-2"}, cfg2.EC2SDConfig.Filters[0].Values) + + // Most importantly: verify they're not the same pointer + require.NotSame(t, cfg1.EC2SDConfig, cfg2.EC2SDConfig, + "EC2SDConfig objects should not share the same memory address") + }, + }, + { + name: "ECSMultipleJobsDifferentPorts", + yaml: ` +- role: ecs + region: us-east-1 + port: 8080 + clusters: [cluster-a] +- role: ecs + region: us-east-1 + port: 8081 + clusters: [cluster-b]`, + validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) { + require.Equal(t, RoleECS, cfg1.Role) + require.Equal(t, RoleECS, cfg2.Role) + require.NotNil(t, cfg1.ECSSDConfig) + require.NotNil(t, cfg2.ECSSDConfig) + + require.Equal(t, 8080, cfg1.ECSSDConfig.Port) + require.Equal(t, 8081, cfg2.ECSSDConfig.Port) + require.Equal(t, []string{"cluster-a"}, cfg1.ECSSDConfig.Clusters) + require.Equal(t, []string{"cluster-b"}, cfg2.ECSSDConfig.Clusters) + + require.NotSame(t, cfg1.ECSSDConfig, cfg2.ECSSDConfig, + "ECSSDConfig objects should not share the same memory address") + }, + }, + { + name: "LightsailMultipleJobsDifferentPorts", + yaml: ` +- role: lightsail + region: eu-west-1 + port: 7070 +- role: lightsail + region: eu-west-1 + port: 7071`, + validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) { + require.Equal(t, RoleLightsail, cfg1.Role) + require.Equal(t, RoleLightsail, cfg2.Role) + require.NotNil(t, cfg1.LightsailSDConfig) + require.NotNil(t, cfg2.LightsailSDConfig) + + require.Equal(t, 7070, cfg1.LightsailSDConfig.Port) + require.Equal(t, 7071, cfg2.LightsailSDConfig.Port) + + require.NotSame(t, cfg1.LightsailSDConfig, cfg2.LightsailSDConfig, + "LightsailSDConfig objects should not share the same memory address") + }, + }, + { + name: "MSKMultipleJobsDifferentPorts", + yaml: ` +- role: msk + region: ap-south-1 + port: 6060 + clusters: ["cluster-1"] +- role: msk + region: ap-south-1 + port: 6061 + clusters: ["cluster-2"]`, + validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) { + require.Equal(t, RoleMSK, cfg1.Role) + require.Equal(t, RoleMSK, cfg2.Role) + require.NotNil(t, cfg1.MSKSDConfig) + require.NotNil(t, cfg2.MSKSDConfig) + + require.Equal(t, 6060, cfg1.MSKSDConfig.Port) + require.Equal(t, []string{"cluster-1"}, cfg1.MSKSDConfig.Clusters) + require.Equal(t, 6061, cfg2.MSKSDConfig.Port) + require.Equal(t, []string{"cluster-2"}, cfg2.MSKSDConfig.Clusters) + + require.NotSame(t, cfg1.MSKSDConfig, cfg2.MSKSDConfig, + "MSKSDConfig objects should not share the same memory address") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var configs []SDConfig + require.NoError(t, yaml.Unmarshal([]byte(tt.yaml), &configs)) + require.Len(t, configs, 2) + tt.validateFunc(t, &configs[0], &configs[1]) + }) + } +} + +// getRandomRegion is a helper to return a pseudo-random AWS region for testing. +func getRandomRegion() string { + regions := []string{ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "eu-west-1", + "eu-west-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-northeast-1", + "ap-northeast-2", + } + + return regions[rand.IntN(len(regions))] +} + +func TestLoadRegion(t *testing.T) { + t.Run("with_env_region", func(t *testing.T) { + randomRegion := getRandomRegion() + t.Setenv("AWS_REGION", randomRegion) + t.Setenv("AWS_ACCESS_KEY_ID", "dummy") + t.Setenv("AWS_SECRET_ACCESS_KEY", "dummy") + t.Setenv("AWS_CONFIG_FILE", "") // Ensure no config file is used + t.Setenv("AWS_PROFILE", "") // Ensure no profile file is used + + region, err := loadRegion(context.Background(), "") + require.NoError(t, err) + require.Equal(t, randomRegion, region) + }) + + t.Run("with_config_file_default_profile", func(t *testing.T) { + randomRegion := getRandomRegion() + + // Create a temporary AWS config file + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "config") + + configContent := `[default] +region = ` + randomRegion + ` +` + + err := os.WriteFile(configFile, []byte(configContent), 0o644) + require.NoError(t, err) + defer os.Remove(configFile) + + // Set up environment to use the config file + t.Setenv("AWS_CONFIG_FILE", configFile) + t.Setenv("AWS_ACCESS_KEY_ID", "dummy") + t.Setenv("AWS_SECRET_ACCESS_KEY", "dummy") + // Clear any region environment variables to force config file usage + t.Setenv("AWS_REGION", "") + t.Setenv("AWS_PROFILE", "") // Ensure no profile file is used + t.Setenv("AWS_DEFAULT_REGION", "") + + region, err := loadRegion(context.Background(), "") + require.NoError(t, err) + require.Equal(t, randomRegion, region) + }) + + t.Run("with_config_file_named_profile", func(t *testing.T) { + randomRegion := getRandomRegion() + + // Create a temporary AWS config file + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "config") + + configContent := `[default] +region = ` + getRandomRegion() + ` + +[profile ` + randomRegion + `-profile] +region = ` + randomRegion + ` +` + + err := os.WriteFile(configFile, []byte(configContent), 0o644) + require.NoError(t, err) + defer os.Remove(configFile) + + // Set up environment to use the config file + t.Setenv("AWS_CONFIG_FILE", configFile) + t.Setenv("AWS_PROFILE", randomRegion+"-profile") + t.Setenv("AWS_ACCESS_KEY_ID", "dummy") + t.Setenv("AWS_SECRET_ACCESS_KEY", "dummy") + // Clear any region environment variables to force config file usage + t.Setenv("AWS_REGION", "") + t.Setenv("AWS_DEFAULT_REGION", "") + + region, err := loadRegion(context.Background(), "") + require.NoError(t, err) + require.Equal(t, randomRegion, region) + }) + + t.Run("with_specified_region", func(t *testing.T) { + specifiedRegion := getRandomRegion() + + // Even with environment region set differently, specified region should take precedence + t.Setenv("AWS_REGION", getRandomRegion()) + t.Setenv("AWS_ACCESS_KEY_ID", "dummy") + t.Setenv("AWS_SECRET_ACCESS_KEY", "dummy") + + region, err := loadRegion(context.Background(), specifiedRegion) + require.NoError(t, err) + require.Equal(t, specifiedRegion, region) + }) + + t.Run("imds_fallback", func(t *testing.T) { + randomRegion := getRandomRegion() + + // Mock IMDS server that returns a region + mockIMDS := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Handle instance identity document (contains region info) + if r.URL.Path == "/latest/dynamic/instance-identity/document" { + imdsPayload := `{"region": "` + randomRegion + `"}` + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(imdsPayload)) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer mockIMDS.Close() + + // Set up environment with no region but valid credentials + // This will force fallback to IMDS + t.Setenv("AWS_ACCESS_KEY_ID", "dummy") + t.Setenv("AWS_SECRET_ACCESS_KEY", "dummy") + // Unset any existing region + t.Setenv("AWS_REGION", "") + t.Setenv("AWS_DEFAULT_REGION", "") + t.Setenv("AWS_CONFIG_FILE", "") // Ensure no config file is used + t.Setenv("AWS_PROFILE", "") // Ensure no profile file is used + // Point IMDS to our mock server + t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", mockIMDS.URL) + + region, err := loadRegion(context.Background(), "") + require.NoError(t, err) + require.Equal(t, randomRegion, region) + }) + + t.Run("imds_empty_region", func(t *testing.T) { + // Mock IMDS server that returns empty region + mockIMDS := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Handle instance identity document with empty region + if r.URL.Path == "/latest/dynamic/instance-identity/document" { + imdsPayload := `{"region": ""}` + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(imdsPayload)) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer mockIMDS.Close() + + // Set up environment with no region but valid credentials + t.Setenv("AWS_ACCESS_KEY_ID", "dummy") + t.Setenv("AWS_SECRET_ACCESS_KEY", "dummy") + // Unset any existing region + t.Setenv("AWS_REGION", "") + t.Setenv("AWS_DEFAULT_REGION", "") + t.Setenv("AWS_CONFIG_FILE", "") // Ensure no config file is used + t.Setenv("AWS_PROFILE", "") // Ensure no profile file is used + // Point IMDS to our mock server + t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", mockIMDS.URL) + + _, err := loadRegion(context.Background(), "") + require.Error(t, err) + require.Contains(t, err.Error(), "failed to get region from IMDS") + }) +} + +func TestSDConfigSetDirectory(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + tests := []struct { + name string + yaml string + role Role + }{ + { + name: "EC2", + yaml: ` +role: ec2 +region: us-east-1 +`, + role: RoleEC2, + }, + { + name: "ECS", + yaml: ` +role: ecs +region: us-west-2 +clusters: [test-cluster] +`, + role: RoleECS, + }, + { + name: "Elasticache", + yaml: ` +role: elasticache +region: eu-west-1 +`, + role: RoleElasticache, + }, + { + name: "Lightsail", + yaml: ` +role: lightsail +region: ap-south-1 +`, + role: RoleLightsail, + }, + { + name: "MSK", + yaml: ` +role: msk +region: us-east-2 +`, + role: RoleMSK, + }, + { + name: "RDS", + yaml: ` +role: rds +region: us-west-1 +`, + role: RoleRDS, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var cfg SDConfig + err := yaml.Unmarshal([]byte(tt.yaml), &cfg) + require.NoError(t, err) + require.Equal(t, tt.role, cfg.Role) + + // Call SetDirectory - should not panic + require.NotPanics(t, func() { + cfg.SetDirectory(tmpDir) + }) + }) + } + + t.Run("SetDirectoryWithNilConfigs", func(t *testing.T) { + // Test that SetDirectory doesn't panic when called on an SDConfig + // where the role-specific config might be nil (this was the original bug) + cfg := SDConfig{ + Role: RoleEC2, + // EC2SDConfig is nil - this would have caused a panic before the fix + } + // This should not panic + require.NotPanics(t, func() { + cfg.SetDirectory(tmpDir) + }) + }) +} diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index a44912481a8..e6e3e7bb00f 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,23 +17,24 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/go-kit/log" - "github.com/go-kit/log/level" + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/refresh" @@ -54,6 +55,7 @@ const ( ec2LabelInstanceType = ec2Label + "instance_type" ec2LabelOwnerID = ec2Label + "owner_id" ec2LabelPlatform = ec2Label + "platform" + ec2LabelDefaultIPv6Address = ec2Label + "default_ipv6_address" ec2LabelPrimaryIPv6Addresses = ec2Label + "primary_ipv6_addresses" ec2LabelPrimarySubnetID = ec2Label + "primary_subnet_id" ec2LabelPrivateDNS = ec2Label + "private_dns_name" @@ -78,12 +80,6 @@ func init() { discovery.RegisterConfig(&EC2SDConfig{}) } -// EC2Filter is the configuration for filtering EC2 instances. -type EC2Filter struct { - Name string `yaml:"name"` - Values []string `yaml:"values"` -} - // EC2SDConfig is the configuration for EC2 based service discovery. type EC2SDConfig struct { Endpoint string `yaml:"endpoint"` @@ -92,15 +88,16 @@ type EC2SDConfig struct { SecretKey config.Secret `yaml:"secret_key,omitempty"` Profile string `yaml:"profile,omitempty"` RoleARN string `yaml:"role_arn,omitempty"` + ExternalID string `yaml:"external_id,omitempty"` RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` Port int `yaml:"port"` - Filters []*EC2Filter `yaml:"filters"` + Filters []*Filter `yaml:"filters"` HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } // NewDiscovererMetrics implements discovery.Config. -func (*EC2SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*EC2SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &ec2Metrics{ refreshMetrics: rmi, } @@ -111,29 +108,29 @@ func (*EC2SDConfig) Name() string { return "ec2" } // NewDiscoverer returns a Discoverer for the EC2 Config. func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewEC2Discovery(c, opts.Logger, opts.Metrics) + return NewEC2Discovery(c, opts) +} + +// SetDirectory joins any relative file paths with dir. +func (c *EC2SDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) } // UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config. -func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultEC2SDConfig type plain EC2SDConfig err := unmarshal((*plain)(c)) if err != nil { return err } - if c.Region == "" { - sess, err := session.NewSession() - if err != nil { - return err - } - metadata := ec2metadata.New(sess) - region, err := metadata.Region() - if err != nil { - return errors.New("EC2 SD configuration requires a region") - } - c.Region = region + + // Check if the region is set, if not attempt to load it from the AWS SDK. + c.Region, err = loadRegion(context.Background(), c.Region) + if err != nil { + return fmt.Errorf("could not determine AWS region: %w", err) } + for _, f := range c.Filters { if len(f.Values) == 0 { return errors.New("EC2 SD configuration filter values cannot be empty") @@ -142,13 +139,59 @@ func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return c.HTTPClientConfig.Validate() } +type ec2Client interface { + DescribeAvailabilityZones(ctx context.Context, params *ec2.DescribeAvailabilityZonesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) + DescribeInstances(ctx context.Context, params *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) +} + +// ec2ClientAdapter holds the EC2 API calls that AWS discovery actually uses as +// method-value closures over the concrete *ec2.Client. +// +// It exists purely to keep the binary small. The Go linker, once reflection +// (reflect.Value.Method/Call plus struct-field traversal, both reachable via +// the YAML/config machinery) is live, conservatively retains every exported +// method of any concrete type that is reachable through an interface — and a +// type stored as a field of an interface-boxed struct counts. *ec2.Client has +// ~470 operation methods; retaining all of them pulls in ~1,500 serializers and +// roughly 21 MB. By capturing only the needed methods as func values, the +// concrete *ec2.Client is hidden inside closure contexts (which reflection +// cannot traverse) and never appears as a field of a boxed type, so dead-code +// elimination drops the unused operations. +type ec2ClientAdapter struct { + describeAvailabilityZones func(ctx context.Context, params *ec2.DescribeAvailabilityZonesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) + describeInstances func(ctx context.Context, params *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) + describeNetworkInterfaces func(ctx context.Context, params *ec2.DescribeNetworkInterfacesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeNetworkInterfacesOutput, error) +} + +// newEC2ClientAdapter wraps a concrete *ec2.Client, capturing only the API +// calls AWS discovery needs. See the ec2ClientAdapter doc comment for why. +func newEC2ClientAdapter(c *ec2.Client) ec2ClientAdapter { + return ec2ClientAdapter{ + describeAvailabilityZones: c.DescribeAvailabilityZones, + describeInstances: c.DescribeInstances, + describeNetworkInterfaces: c.DescribeNetworkInterfaces, + } +} + +func (a ec2ClientAdapter) DescribeAvailabilityZones(ctx context.Context, params *ec2.DescribeAvailabilityZonesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) { + return a.describeAvailabilityZones(ctx, params, optFns...) +} + +func (a ec2ClientAdapter) DescribeInstances(ctx context.Context, params *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { + return a.describeInstances(ctx, params, optFns...) +} + +func (a ec2ClientAdapter) DescribeNetworkInterfaces(ctx context.Context, params *ec2.DescribeNetworkInterfacesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeNetworkInterfacesOutput, error) { + return a.describeNetworkInterfaces(ctx, params, optFns...) +} + // EC2Discovery periodically performs EC2-SD requests. It implements // the Discoverer interface. type EC2Discovery struct { *refresh.Discovery - logger log.Logger + logger *slog.Logger cfg *EC2SDConfig - ec2 *ec2.EC2 + ec2 ec2Client // azToAZID maps this account's availability zones to their underlying AZ // ID, e.g. eu-west-2a -> euw2-az2. Refreshes are performed sequentially, so @@ -157,23 +200,24 @@ type EC2Discovery struct { } // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets. -func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*EC2Discovery, error) { - m, ok := metrics.(*ec2Metrics) +func NewEC2Discovery(conf *EC2SDConfig, opts discovery.DiscovererOptions) (*EC2Discovery, error) { + m, ok := opts.Metrics.(*ec2Metrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - if logger == nil { - logger = log.NewNopLogger() + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() } d := &EC2Discovery{ - logger: logger, + logger: opts.Logger, cfg: conf, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "ec2", + SetName: opts.SetName, Interval: time.Duration(d.cfg.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -182,51 +226,74 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, metrics discovery.Dis return d, nil } -func (d *EC2Discovery) ec2Client(context.Context) (*ec2.EC2, error) { +func (d *EC2Discovery) ec2Client(ctx context.Context) (ec2Client, error) { if d.ec2 != nil { return d.ec2, nil } - creds := credentials.NewStaticCredentials(d.cfg.AccessKey, string(d.cfg.SecretKey), "") - if d.cfg.AccessKey == "" && d.cfg.SecretKey == "" { - creds = nil - } - - client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "ec2_sd") + // Build the HTTP client from the provided HTTPClientConfig. + httpClient, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "ec2_sd") if err != nil { return nil, err } - sess, err := session.NewSessionWithOptions(session.Options{ - Config: aws.Config{ - Endpoint: &d.cfg.Endpoint, - Region: &d.cfg.Region, - Credentials: creds, - HTTPClient: client, - }, - Profile: d.cfg.Profile, - }) + // Build the AWS config with the provided region. + configOptions := []func(*awsConfig.LoadOptions) error{ + awsConfig.WithRegion(d.cfg.Region), + awsConfig.WithHTTPClient(httpClient), + } + + // Only set static credentials if both access key and secret key are provided. + // Otherwise, let the AWS SDK use its default credential chain (environment variables, IAM role, etc.). + if d.cfg.AccessKey != "" && d.cfg.SecretKey != "" { + credProvider := credentials.NewStaticCredentialsProvider(d.cfg.AccessKey, string(d.cfg.SecretKey), "") + configOptions = append(configOptions, awsConfig.WithCredentialsProvider(credProvider)) + } + + // Set the profile if provided. + if d.cfg.Profile != "" { + configOptions = append(configOptions, awsConfig.WithSharedConfigProfile(d.cfg.Profile)) + } + + cfg, err := awsConfig.LoadDefaultConfig(ctx, configOptions...) if err != nil { - return nil, fmt.Errorf("could not create aws session: %w", err) + return nil, fmt.Errorf("could not create aws config: %w", err) } + // If the role ARN is set, assume the role to get credentials and set the credentials provider in the config. if d.cfg.RoleARN != "" { - creds := stscreds.NewCredentials(sess, d.cfg.RoleARN) - d.ec2 = ec2.New(sess, &aws.Config{Credentials: creds}) - } else { - d.ec2 = ec2.New(sess) + assumeProvider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), d.cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { + if d.cfg.ExternalID != "" { + o.ExternalID = aws.String(d.cfg.ExternalID) + } + }) + cfg.Credentials = aws.NewCredentialsCache(assumeProvider) } + d.ec2 = newEC2ClientAdapter(ec2.NewFromConfig(cfg, func(options *ec2.Options) { + if d.cfg.Endpoint != "" { + options.BaseEndpoint = &d.cfg.Endpoint + } + options.HTTPClient = httpClient + })) + return d.ec2, nil } func (d *EC2Discovery) refreshAZIDs(ctx context.Context) error { - azs, err := d.ec2.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{}) + azs, err := d.ec2.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{}) if err != nil { return err } + if azs.AvailabilityZones == nil { + d.azToAZID = make(map[string]string) + return nil + } d.azToAZID = make(map[string]string, len(azs.AvailabilityZones)) for _, az := range azs.AvailabilityZones { + if az.ZoneName == nil || az.ZoneId == nil { + continue + } d.azToAZID[*az.ZoneName] = *az.ZoneId } return nil @@ -242,11 +309,11 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error Source: d.cfg.Region, } - var filters []*ec2.Filter + var filters []ec2Types.Filter for _, f := range d.cfg.Filters { - filters = append(filters, &ec2.Filter{ + filters = append(filters, ec2Types.Filter{ Name: aws.String(f.Name), - Values: aws.StringSlice(f.Values), + Values: f.Values, }) } @@ -254,17 +321,30 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error // Prometheus requires a reload if AWS adds a new AZ to the region. if d.azToAZID == nil { if err := d.refreshAZIDs(ctx); err != nil { - level.Debug(d.logger).Log( - "msg", "Unable to describe availability zones", + d.logger.Debug( + "Unable to describe availability zones", "err", err) } } input := &ec2.DescribeInstancesInput{Filters: filters} - if err := ec2Client.DescribeInstancesPagesWithContext(ctx, input, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool { + paginator := ec2.NewDescribeInstancesPaginator(ec2Client, input) + + for paginator.HasMorePages() { + p, err := paginator.NextPage(ctx) + if err != nil { + var awsErr smithy.APIError + if errors.As(err, &awsErr) && (awsErr.ErrorCode() == "AuthFailure" || awsErr.ErrorCode() == "UnauthorizedOperation") { + d.ec2 = nil + } + return nil, fmt.Errorf("could not describe instances: %w", err) + } + for _, r := range p.Reservations { for _, inst := range r.Instances { - if inst.PrivateIpAddress == nil { + defaultIPv6Addr, primaryIPv6Addrs, ipv6Addrs := getInstanceIPv6Addresses(&inst) + + if inst.PrivateIpAddress == nil && defaultIPv6Addr == nil { continue } @@ -277,39 +357,62 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error labels[ec2LabelOwnerID] = model.LabelValue(*r.OwnerId) } - labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress) + if defaultIPv6Addr != nil { + labels[ec2LabelDefaultIPv6Address] = model.LabelValue(*defaultIPv6Addr) + } + + if inst.PrivateIpAddress != nil { + labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress) + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(*inst.PrivateIpAddress, strconv.Itoa(d.cfg.Port))) + } else { + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(*defaultIPv6Addr, strconv.Itoa(d.cfg.Port))) + } + if inst.PrivateDnsName != nil { labels[ec2LabelPrivateDNS] = model.LabelValue(*inst.PrivateDnsName) } - addr := net.JoinHostPort(*inst.PrivateIpAddress, strconv.Itoa(d.cfg.Port)) - labels[model.AddressLabel] = model.LabelValue(addr) - if inst.Platform != nil { - labels[ec2LabelPlatform] = model.LabelValue(*inst.Platform) + if inst.Platform != "" { + labels[ec2LabelPlatform] = model.LabelValue(inst.Platform) } if inst.PublicIpAddress != nil { labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress) labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName) } + + if primaryIPv6Addrs != nil { + labels[ec2LabelPrimaryIPv6Addresses] = model.LabelValue( + ec2LabelSeparator + + strings.Join(primaryIPv6Addrs, ec2LabelSeparator) + + ec2LabelSeparator) + } + + if ipv6Addrs != nil { + labels[ec2LabelIPv6Addresses] = model.LabelValue( + ec2LabelSeparator + + strings.Join(ipv6Addrs, ec2LabelSeparator) + + ec2LabelSeparator) + } + labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId) labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone) azID, ok := d.azToAZID[*inst.Placement.AvailabilityZone] if !ok && d.azToAZID != nil { - level.Debug(d.logger).Log( - "msg", "Availability zone ID not found", + d.logger.Debug( + "Availability zone ID not found", "az", *inst.Placement.AvailabilityZone) } labels[ec2LabelAZID] = model.LabelValue(azID) - labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name) - labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType) + labels[ec2LabelInstanceState] = model.LabelValue(inst.State.Name) + labels[ec2LabelInstanceType] = model.LabelValue(inst.InstanceType) - if inst.InstanceLifecycle != nil { - labels[ec2LabelInstanceLifecycle] = model.LabelValue(*inst.InstanceLifecycle) + if inst.InstanceLifecycle != "" { + labels[ec2LabelInstanceLifecycle] = model.LabelValue(inst.InstanceLifecycle) } - if inst.Architecture != nil { - labels[ec2LabelArch] = model.LabelValue(*inst.Architecture) + if inst.Architecture != "" { + labels[ec2LabelArch] = model.LabelValue(inst.Architecture) } if inst.VpcId != nil { @@ -317,8 +420,6 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error labels[ec2LabelPrimarySubnetID] = model.LabelValue(*inst.SubnetId) var subnets []string - var ipv6addrs []string - var primaryipv6addrs []string subnetsMap := make(map[string]struct{}) for _, eni := range inst.NetworkInterfaces { if eni.SubnetId == nil { @@ -329,40 +430,15 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error subnetsMap[*eni.SubnetId] = struct{}{} subnets = append(subnets, *eni.SubnetId) } - - for _, ipv6addr := range eni.Ipv6Addresses { - ipv6addrs = append(ipv6addrs, *ipv6addr.Ipv6Address) - if *ipv6addr.IsPrimaryIpv6 { - // we might have to extend the slice with more than one element - // that could leave empty strings in the list which is intentional - // to keep the position/device index information - for int64(len(primaryipv6addrs)) <= *eni.Attachment.DeviceIndex { - primaryipv6addrs = append(primaryipv6addrs, "") - } - primaryipv6addrs[*eni.Attachment.DeviceIndex] = *ipv6addr.Ipv6Address - } - } } labels[ec2LabelSubnetID] = model.LabelValue( ec2LabelSeparator + strings.Join(subnets, ec2LabelSeparator) + ec2LabelSeparator) - if len(ipv6addrs) > 0 { - labels[ec2LabelIPv6Addresses] = model.LabelValue( - ec2LabelSeparator + - strings.Join(ipv6addrs, ec2LabelSeparator) + - ec2LabelSeparator) - } - if len(primaryipv6addrs) > 0 { - labels[ec2LabelPrimaryIPv6Addresses] = model.LabelValue( - ec2LabelSeparator + - strings.Join(primaryipv6addrs, ec2LabelSeparator) + - ec2LabelSeparator) - } } for _, t := range inst.Tags { - if t == nil || t.Key == nil || t.Value == nil { + if t.Key == nil || t.Value == nil { continue } name := strutil.SanitizeLabelName(*t.Key) @@ -371,13 +447,43 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error tg.Targets = append(tg.Targets, labels) } } - return true - }); err != nil { - var awsErr awserr.Error - if errors.As(err, &awsErr) && (awsErr.Code() == "AuthFailure" || awsErr.Code() == "UnauthorizedOperation") { - d.ec2 = nil - } - return nil, fmt.Errorf("could not describe instances: %w", err) } + return []*targetgroup.Group{tg}, nil } + +func getInstanceIPv6Addresses(i *ec2Types.Instance) (*string, []string, []string) { + var primaryIPv6Addrs []string + var ipv6Addrs []string + + if i.VpcId != nil { + for _, eni := range i.NetworkInterfaces { + if eni.SubnetId == nil { + continue + } + + for _, ipv6addr := range eni.Ipv6Addresses { + ipv6Addrs = append(ipv6Addrs, *ipv6addr.Ipv6Address) + if *ipv6addr.IsPrimaryIpv6 { + // we might have to extend the slice with more than one element + // that could leave empty strings in the list which is intentional + // to keep the position/device index information + for int32(len(primaryIPv6Addrs)) <= *eni.Attachment.DeviceIndex { + primaryIPv6Addrs = append(primaryIPv6Addrs, "") + } + primaryIPv6Addrs[*eni.Attachment.DeviceIndex] = *ipv6addr.Ipv6Address + } + } + } + + // Find an IPv6 address we can use by default. Pick the first primary one if + // there is any available, if not then pick the first non-primary address. + for _, ipv6addr := range append(primaryIPv6Addrs, ipv6Addrs...) { + if ipv6addr != "" { + return &ipv6addr, primaryIPv6Addrs, ipv6Addrs + } + } + } + + return nil, primaryIPv6Addrs, ipv6Addrs +} diff --git a/discovery/aws/ec2_test.go b/discovery/aws/ec2_test.go new file mode 100644 index 00000000000..48d4849c412 --- /dev/null +++ b/discovery/aws/ec2_test.go @@ -0,0 +1,592 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +// Helper function to get pointers on literals. +// NOTE: this is common between a few tests. In the future it might worth to move this out into a separate package. +func strptr(str string) *string { + return &str +} + +func boolptr(b bool) *bool { + return &b +} + +// Struct for test data. +type ec2DataStore struct { + region string + + azToAZID map[string]string + + ownerID string + + instances []ec2Types.Instance +} + +// The tests itself. +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + +func TestEC2DiscoveryRefreshAZIDs(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // iterate through the test cases + for _, tt := range []struct { + name string + shouldFail bool + ec2Data *ec2DataStore + }{ + { + name: "Normal", + shouldFail: false, + ec2Data: &ec2DataStore{ + azToAZID: map[string]string{ + "azname-a": "azid-1", + "azname-b": "azid-2", + "azname-c": "azid-3", + }, + }, + }, + { + name: "HandleError", + shouldFail: true, + ec2Data: &ec2DataStore{}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockEC2Client(tt.ec2Data) + + d := &EC2Discovery{ + ec2: client, + } + + err := d.refreshAZIDs(ctx) + if tt.shouldFail { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, client.ec2Data.azToAZID, d.azToAZID) + } + }) + } +} + +func TestEC2DiscoveryRefresh(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // iterate through the test cases + for _, tt := range []struct { + name string + ec2Data *ec2DataStore + filters []*Filter + expected []*targetgroup.Group + }{ + { + name: "NoPrivateIpOrIpv6", + ec2Data: &ec2DataStore{ + region: "region-noprivateip", + azToAZID: map[string]string{ + "azname-a": "azid-1", + "azname-b": "azid-2", + "azname-c": "azid-3", + }, + instances: []ec2Types.Instance{ + { + InstanceId: strptr("instance-id-noprivateip"), + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "region-noprivateip", + }, + }, + }, + { + name: "NoVpc", + ec2Data: &ec2DataStore{ + region: "region-novpc", + azToAZID: map[string]string{ + "azname-a": "azid-1", + "azname-b": "azid-2", + "azname-c": "azid-3", + }, + ownerID: "owner-id-novpc", + instances: []ec2Types.Instance{ + { + // set every possible options and test them here + Architecture: "architecture-novpc", + ImageId: strptr("ami-novpc"), + InstanceId: strptr("instance-id-novpc"), + InstanceLifecycle: "instance-lifecycle-novpc", + InstanceType: "instance-type-novpc", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-b")}, + Platform: "platform-novpc", + PrivateDnsName: strptr("private-dns-novpc"), + PrivateIpAddress: strptr("1.2.3.4"), + PublicDnsName: strptr("public-dns-novpc"), + PublicIpAddress: strptr("42.42.42.2"), + State: &ec2Types.InstanceState{Name: "running"}, + // test tags once and for all + Tags: []ec2Types.Tag{ + {Key: strptr("tag-1-key"), Value: strptr("tag-1-value")}, + {Key: strptr("tag-2-key"), Value: strptr("tag-2-value")}, + {}, + {Value: strptr("tag-4-value")}, + {Key: strptr("tag-5-key")}, + }, + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "region-novpc", + Targets: []model.LabelSet{ + { + "__address__": model.LabelValue("1.2.3.4:4242"), + "__meta_ec2_ami": model.LabelValue("ami-novpc"), + "__meta_ec2_architecture": model.LabelValue("architecture-novpc"), + "__meta_ec2_availability_zone": model.LabelValue("azname-b"), + "__meta_ec2_availability_zone_id": model.LabelValue("azid-2"), + "__meta_ec2_instance_id": model.LabelValue("instance-id-novpc"), + "__meta_ec2_instance_lifecycle": model.LabelValue("instance-lifecycle-novpc"), + "__meta_ec2_instance_type": model.LabelValue("instance-type-novpc"), + "__meta_ec2_instance_state": model.LabelValue("running"), + "__meta_ec2_owner_id": model.LabelValue("owner-id-novpc"), + "__meta_ec2_platform": model.LabelValue("platform-novpc"), + "__meta_ec2_private_dns_name": model.LabelValue("private-dns-novpc"), + "__meta_ec2_private_ip": model.LabelValue("1.2.3.4"), + "__meta_ec2_public_dns_name": model.LabelValue("public-dns-novpc"), + "__meta_ec2_public_ip": model.LabelValue("42.42.42.2"), + "__meta_ec2_region": model.LabelValue("region-novpc"), + "__meta_ec2_tag_tag_1_key": model.LabelValue("tag-1-value"), + "__meta_ec2_tag_tag_2_key": model.LabelValue("tag-2-value"), + }, + }, + }, + }, + }, + { + name: "Ipv4", + ec2Data: &ec2DataStore{ + region: "region-ipv4", + azToAZID: map[string]string{ + "azname-a": "azid-1", + "azname-b": "azid-2", + "azname-c": "azid-3", + }, + instances: []ec2Types.Instance{ + { + // just the minimum needed for the refresh work + ImageId: strptr("ami-ipv4"), + InstanceId: strptr("instance-id-ipv4"), + InstanceType: "instance-type-ipv4", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-c")}, + PrivateIpAddress: strptr("5.6.7.8"), + State: &ec2Types.InstanceState{Name: "running"}, + SubnetId: strptr("azid-3"), + VpcId: strptr("vpc-ipv4"), + // network interfaces + NetworkInterfaces: []ec2Types.InstanceNetworkInterface{ + // interface without subnet -> should be ignored + { + Ipv6Addresses: []ec2Types.InstanceIpv6Address{ + { + Ipv6Address: strptr("2001:db8:1::1"), + IsPrimaryIpv6: boolptr(true), + }, + }, + }, + // interface with subnet, no IPv6 + { + Ipv6Addresses: []ec2Types.InstanceIpv6Address{}, + SubnetId: strptr("azid-3"), + }, + // interface with another subnet, no IPv6 + { + Ipv6Addresses: []ec2Types.InstanceIpv6Address{}, + SubnetId: strptr("azid-1"), + }, + }, + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "region-ipv4", + Targets: []model.LabelSet{ + { + "__address__": model.LabelValue("5.6.7.8:4242"), + "__meta_ec2_ami": model.LabelValue("ami-ipv4"), + "__meta_ec2_availability_zone": model.LabelValue("azname-c"), + "__meta_ec2_availability_zone_id": model.LabelValue("azid-3"), + "__meta_ec2_instance_id": model.LabelValue("instance-id-ipv4"), + "__meta_ec2_instance_state": model.LabelValue("running"), + "__meta_ec2_instance_type": model.LabelValue("instance-type-ipv4"), + "__meta_ec2_owner_id": model.LabelValue(""), + "__meta_ec2_primary_subnet_id": model.LabelValue("azid-3"), + "__meta_ec2_private_ip": model.LabelValue("5.6.7.8"), + "__meta_ec2_region": model.LabelValue("region-ipv4"), + "__meta_ec2_subnet_id": model.LabelValue(",azid-3,azid-1,"), + "__meta_ec2_vpc_id": model.LabelValue("vpc-ipv4"), + }, + }, + }, + }, + }, + { + name: "Ipv6", + ec2Data: &ec2DataStore{ + region: "region-ipv6", + azToAZID: map[string]string{ + "azname-a": "azid-1", + "azname-b": "azid-2", + "azname-c": "azid-3", + }, + instances: []ec2Types.Instance{ + { + // just the minimum needed for the refresh work + ImageId: strptr("ami-ipv6"), + InstanceId: strptr("instance-id-ipv6"), + InstanceType: "instance-type-ipv6", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-b")}, + PrivateIpAddress: strptr("9.10.11.12"), + State: &ec2Types.InstanceState{Name: "running"}, + SubnetId: strptr("azid-2"), + VpcId: strptr("vpc-ipv6"), + // network interfaces + NetworkInterfaces: []ec2Types.InstanceNetworkInterface{ + // interface without primary IPv6, index 2 + { + Attachment: &ec2Types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(3), + }, + Ipv6Addresses: []ec2Types.InstanceIpv6Address{ + { + Ipv6Address: strptr("2001:db8:2::1:1"), + IsPrimaryIpv6: boolptr(false), + }, + }, + SubnetId: strptr("azid-2"), + }, + // interface with primary IPv6, index 1 + { + Attachment: &ec2Types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(1), + }, + Ipv6Addresses: []ec2Types.InstanceIpv6Address{ + { + Ipv6Address: strptr("2001:db8:2::2:1"), + IsPrimaryIpv6: boolptr(false), + }, + { + Ipv6Address: strptr("2001:db8:2::2:2"), + IsPrimaryIpv6: boolptr(true), + }, + }, + SubnetId: strptr("azid-2"), + }, + // interface with primary IPv6, index 3 + { + Attachment: &ec2Types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(3), + }, + Ipv6Addresses: []ec2Types.InstanceIpv6Address{ + { + Ipv6Address: strptr("2001:db8:2::3:1"), + IsPrimaryIpv6: boolptr(true), + }, + }, + SubnetId: strptr("azid-1"), + }, + // interface without primary IPv6, index 0 + { + Attachment: &ec2Types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(0), + }, + Ipv6Addresses: []ec2Types.InstanceIpv6Address{}, + SubnetId: strptr("azid-3"), + }, + }, + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "region-ipv6", + Targets: []model.LabelSet{ + { + "__address__": model.LabelValue("9.10.11.12:4242"), + "__meta_ec2_ami": model.LabelValue("ami-ipv6"), + "__meta_ec2_availability_zone": model.LabelValue("azname-b"), + "__meta_ec2_availability_zone_id": model.LabelValue("azid-2"), + "__meta_ec2_instance_id": model.LabelValue("instance-id-ipv6"), + "__meta_ec2_instance_state": model.LabelValue("running"), + "__meta_ec2_instance_type": model.LabelValue("instance-type-ipv6"), + "__meta_ec2_ipv6_addresses": model.LabelValue(",2001:db8:2::1:1,2001:db8:2::2:1,2001:db8:2::2:2,2001:db8:2::3:1,"), + "__meta_ec2_owner_id": model.LabelValue(""), + "__meta_ec2_default_ipv6_address": model.LabelValue("2001:db8:2::2:2"), + "__meta_ec2_primary_ipv6_addresses": model.LabelValue(",,2001:db8:2::2:2,,2001:db8:2::3:1,"), + "__meta_ec2_primary_subnet_id": model.LabelValue("azid-2"), + "__meta_ec2_private_ip": model.LabelValue("9.10.11.12"), + "__meta_ec2_region": model.LabelValue("region-ipv6"), + "__meta_ec2_subnet_id": model.LabelValue(",azid-2,azid-1,azid-3,"), + "__meta_ec2_vpc_id": model.LabelValue("vpc-ipv6"), + }, + }, + }, + }, + }, + { + name: "Ipv6-Only", + ec2Data: &ec2DataStore{ + region: "region-ipv6-only", + azToAZID: map[string]string{ + "azname-a": "azid-1", + "azname-b": "azid-2", + "azname-c": "azid-3", + }, + + instances: []ec2Types.Instance{ + { + // just the minimum needed for the refresh work + ImageId: strptr("ami-ipv6-only"), + InstanceId: strptr("instance-id-ipv6-only"), + InstanceType: "instance-type-ipv6-only", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-b")}, + State: &ec2Types.InstanceState{Name: "running"}, + SubnetId: strptr("azid-2"), + VpcId: strptr("vpc-ipv6-only"), + // network interfaces + NetworkInterfaces: []ec2Types.InstanceNetworkInterface{ + // interface without primary IPv6, index 0 + { + Attachment: &ec2Types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(0), + }, + Ipv6Addresses: []ec2Types.InstanceIpv6Address{ + { + Ipv6Address: strptr("2001:db8:2::1:1"), + IsPrimaryIpv6: boolptr(false), + }, + }, + SubnetId: strptr("azid-2"), + }, + }, + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "region-ipv6-only", + Targets: []model.LabelSet{ + { + "__address__": model.LabelValue("[2001:db8:2::1:1]:4242"), + "__meta_ec2_ami": model.LabelValue("ami-ipv6-only"), + "__meta_ec2_availability_zone": model.LabelValue("azname-b"), + "__meta_ec2_availability_zone_id": model.LabelValue("azid-2"), + "__meta_ec2_instance_id": model.LabelValue("instance-id-ipv6-only"), + "__meta_ec2_instance_state": model.LabelValue("running"), + "__meta_ec2_instance_type": model.LabelValue("instance-type-ipv6-only"), + "__meta_ec2_ipv6_addresses": model.LabelValue(",2001:db8:2::1:1,"), + "__meta_ec2_owner_id": model.LabelValue(""), + "__meta_ec2_default_ipv6_address": model.LabelValue("2001:db8:2::1:1"), + "__meta_ec2_primary_subnet_id": model.LabelValue("azid-2"), + "__meta_ec2_region": model.LabelValue("region-ipv6-only"), + "__meta_ec2_subnet_id": model.LabelValue(",azid-2,"), + "__meta_ec2_vpc_id": model.LabelValue("vpc-ipv6-only"), + }, + }, + }, + }, + }, + { + name: "FiltersMatchSingleInstance", + ec2Data: &ec2DataStore{ + region: "region-filter-match", + azToAZID: map[string]string{ + "azname-a": "azid-1", + }, + instances: []ec2Types.Instance{ + { + ImageId: strptr("ami-filter-1"), + InstanceId: strptr("instance-filter-1"), + InstanceType: "instance-type-filter", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-a")}, + PrivateIpAddress: strptr("10.0.0.1"), + State: &ec2Types.InstanceState{Name: "running"}, + }, + { + ImageId: strptr("ami-filter-2"), + InstanceId: strptr("instance-filter-2"), + InstanceType: "instance-type-filter", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-a")}, + PrivateIpAddress: strptr("10.0.0.2"), + State: &ec2Types.InstanceState{Name: "stopped"}, + }, + }, + }, + filters: []*Filter{{Name: "instance-state-name", Values: []string{"running"}}}, + expected: []*targetgroup.Group{ + { + Source: "region-filter-match", + Targets: []model.LabelSet{ + { + "__address__": model.LabelValue("10.0.0.1:4242"), + "__meta_ec2_ami": model.LabelValue("ami-filter-1"), + "__meta_ec2_availability_zone": model.LabelValue("azname-a"), + "__meta_ec2_availability_zone_id": model.LabelValue("azid-1"), + "__meta_ec2_instance_id": model.LabelValue("instance-filter-1"), + "__meta_ec2_instance_state": model.LabelValue("running"), + "__meta_ec2_instance_type": model.LabelValue("instance-type-filter"), + "__meta_ec2_owner_id": model.LabelValue(""), + "__meta_ec2_private_ip": model.LabelValue("10.0.0.1"), + "__meta_ec2_region": model.LabelValue("region-filter-match"), + }, + }, + }, + }, + }, + { + name: "FiltersMatchNoInstances", + ec2Data: &ec2DataStore{ + region: "region-filter-none", + azToAZID: map[string]string{ + "azname-a": "azid-1", + }, + instances: []ec2Types.Instance{ + { + ImageId: strptr("ami-filter-1"), + InstanceId: strptr("instance-filter-1"), + InstanceType: "instance-type-filter", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-a")}, + PrivateIpAddress: strptr("10.0.1.1"), + State: &ec2Types.InstanceState{Name: "running"}, + }, + { + ImageId: strptr("ami-filter-2"), + InstanceId: strptr("instance-filter-2"), + InstanceType: "instance-type-filter", + Placement: &ec2Types.Placement{AvailabilityZone: strptr("azname-a")}, + PrivateIpAddress: strptr("10.0.1.2"), + State: &ec2Types.InstanceState{Name: "stopped"}, + }, + }, + }, + filters: []*Filter{{Name: "instance-state-name", Values: []string{"terminated"}}}, + expected: []*targetgroup.Group{{Source: "region-filter-none"}}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockEC2Client(tt.ec2Data) + + d := &EC2Discovery{ + ec2: client, + cfg: &EC2SDConfig{ + Port: 4242, + Region: client.ec2Data.region, + Filters: tt.filters, + }, + } + + g, err := d.refresh(ctx) + require.NoError(t, err) + require.Equal(t, tt.expected, g) + }) + } +} + +// EC2 client mock. +type mockEC2Client struct { + ec2Data ec2DataStore +} + +func newMockEC2Client(ec2Data *ec2DataStore) *mockEC2Client { + client := mockEC2Client{ + ec2Data: *ec2Data, + } + return &client +} + +func (m *mockEC2Client) DescribeAvailabilityZones(context.Context, *ec2.DescribeAvailabilityZonesInput, ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) { + if len(m.ec2Data.azToAZID) == 0 { + return nil, errors.New("No AZs found") + } + + azs := make([]ec2Types.AvailabilityZone, len(m.ec2Data.azToAZID)) + + i := 0 + for k, v := range m.ec2Data.azToAZID { + azs[i] = ec2Types.AvailabilityZone{ + ZoneName: strptr(k), + ZoneId: strptr(v), + } + i++ + } + + return &ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: azs, + }, nil +} + +func (m *mockEC2Client) DescribeInstances(_ context.Context, input *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { + allowedStates := map[string]struct{}{} + hasStateFilter := false + for _, f := range input.Filters { + if f.Name == nil || *f.Name != "instance-state-name" { + continue + } + hasStateFilter = true + for _, v := range f.Values { + allowedStates[v] = struct{}{} + } + } + + r := ec2Types.Reservation{} + for _, inst := range m.ec2Data.instances { + if hasStateFilter { + if inst.State == nil { + continue + } + if _, ok := allowedStates[string(inst.State.Name)]; !ok { + continue + } + } + r.Instances = append(r.Instances, inst) + } + r.OwnerId = aws.String(m.ec2Data.ownerID) + + o := ec2.DescribeInstancesOutput{} + o.Reservations = []ec2Types.Reservation{r} + + return &o, nil +} diff --git a/discovery/aws/ecs.go b/discovery/aws/ecs.go new file mode 100644 index 00000000000..def9c88e6e5 --- /dev/null +++ b/discovery/aws/ecs.go @@ -0,0 +1,1059 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ecs" + "github.com/aws/aws-sdk-go-v2/service/ecs/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "golang.org/x/sync/errgroup" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + ecsLabel = model.MetaLabelPrefix + "ecs_" + ecsLabelCluster = ecsLabel + "cluster" + ecsLabelClusterARN = ecsLabel + "cluster_arn" + ecsLabelService = ecsLabel + "service" + ecsLabelServiceARN = ecsLabel + "service_arn" + ecsLabelServiceStatus = ecsLabel + "service_status" + ecsLabelTaskGroup = ecsLabel + "task_group" + ecsLabelTaskARN = ecsLabel + "task_arn" + ecsLabelTaskDefinition = ecsLabel + "task_definition" + ecsLabelRegion = ecsLabel + "region" + ecsLabelAvailabilityZone = ecsLabel + "availability_zone" + ecsLabelSubnetID = ecsLabel + "subnet_id" + ecsLabelIPAddress = ecsLabel + "ip_address" + ecsLabelLaunchType = ecsLabel + "launch_type" + ecsLabelDesiredStatus = ecsLabel + "desired_status" + ecsLabelLastStatus = ecsLabel + "last_status" + ecsLabelHealthStatus = ecsLabel + "health_status" + ecsLabelPlatformFamily = ecsLabel + "platform_family" + ecsLabelPlatformVersion = ecsLabel + "platform_version" + ecsLabelTag = ecsLabel + "tag_" + ecsLabelTagCluster = ecsLabelTag + "cluster_" + ecsLabelTagService = ecsLabelTag + "service_" + ecsLabelTagTask = ecsLabelTag + "task_" + ecsLabelTagEC2 = ecsLabelTag + "ec2_" + ecsLabelNetworkMode = ecsLabel + "network_mode" + ecsLabelContainerInstanceARN = ecsLabel + "container_instance_arn" + ecsLabelEC2InstanceID = ecsLabel + "ec2_instance_id" + ecsLabelEC2InstanceType = ecsLabel + "ec2_instance_type" + ecsLabelEC2InstancePrivateIP = ecsLabel + "ec2_instance_private_ip" + ecsLabelEC2InstancePublicIP = ecsLabel + "ec2_instance_public_ip" + ecsLabelPublicIP = ecsLabel + "public_ip" +) + +// DefaultECSSDConfig is the default ECS SD configuration. +var DefaultECSSDConfig = ECSSDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + RequestConcurrency: 20, // Aligned with AWS ECS API sustained rate limits (20 req/sec) + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +func init() { + discovery.RegisterConfig(&ECSSDConfig{}) +} + +// ECSSDConfig is the configuration for ECS based service discovery. +type ECSSDConfig struct { + Region string `yaml:"region"` + Endpoint string `yaml:"endpoint"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + ExternalID string `yaml:"external_id,omitempty"` + Clusters []string `yaml:"clusters,omitempty"` + Port int `yaml:"port"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + + // RequestConcurrency controls the maximum number of concurrent ECS API requests. + // Default is 20, which aligns with AWS ECS sustained rate limits: + // - Cluster read actions (DescribeClusters, ListClusters): 20 req/sec sustained + // - Service read actions (DescribeServices, ListServices): 20 req/sec sustained + // - Cluster resource read actions (DescribeTasks, ListTasks): 20 req/sec sustained + // See: https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html + RequestConcurrency int `yaml:"request_concurrency,omitempty"` + + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` +} + +// NewDiscovererMetrics implements discovery.Config. +func (*ECSSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &ecsMetrics{ + refreshMetrics: rmi, + } +} + +// Name returns the name of the ECS Config. +func (*ECSSDConfig) Name() string { return "ecs" } + +// NewDiscoverer returns a Discoverer for the EC2 Config. +func (c *ECSSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewECSDiscovery(c, opts) +} + +// SetDirectory joins any relative file paths with dir. +func (c *ECSSDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for the ECS Config. +func (c *ECSSDConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultECSSDConfig + type plain ECSSDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + + c.Region, err = loadRegion(context.Background(), c.Region) + if err != nil { + return fmt.Errorf("could not determine AWS region: %w", err) + } + + return c.HTTPClientConfig.Validate() +} + +type ecsClient interface { + ListClusters(context.Context, *ecs.ListClustersInput, ...func(*ecs.Options)) (*ecs.ListClustersOutput, error) + DescribeClusters(context.Context, *ecs.DescribeClustersInput, ...func(*ecs.Options)) (*ecs.DescribeClustersOutput, error) + ListServices(context.Context, *ecs.ListServicesInput, ...func(*ecs.Options)) (*ecs.ListServicesOutput, error) + DescribeServices(context.Context, *ecs.DescribeServicesInput, ...func(*ecs.Options)) (*ecs.DescribeServicesOutput, error) + ListTasks(context.Context, *ecs.ListTasksInput, ...func(*ecs.Options)) (*ecs.ListTasksOutput, error) + DescribeTasks(context.Context, *ecs.DescribeTasksInput, ...func(*ecs.Options)) (*ecs.DescribeTasksOutput, error) + DescribeContainerInstances(context.Context, *ecs.DescribeContainerInstancesInput, ...func(*ecs.Options)) (*ecs.DescribeContainerInstancesOutput, error) +} + +// ecsClientAdapter captures only the ECS API calls AWS discovery uses as +// method-value closures, keeping the concrete *ecs.Client out of any +// interface-boxed struct field. See ec2ClientAdapter for the full rationale: +// this stops the linker from retaining the entire ECS API surface (~2 MB). +type ecsClientAdapter struct { + listClusters func(context.Context, *ecs.ListClustersInput, ...func(*ecs.Options)) (*ecs.ListClustersOutput, error) + describeClusters func(context.Context, *ecs.DescribeClustersInput, ...func(*ecs.Options)) (*ecs.DescribeClustersOutput, error) + listServices func(context.Context, *ecs.ListServicesInput, ...func(*ecs.Options)) (*ecs.ListServicesOutput, error) + describeServices func(context.Context, *ecs.DescribeServicesInput, ...func(*ecs.Options)) (*ecs.DescribeServicesOutput, error) + listTasks func(context.Context, *ecs.ListTasksInput, ...func(*ecs.Options)) (*ecs.ListTasksOutput, error) + describeTasks func(context.Context, *ecs.DescribeTasksInput, ...func(*ecs.Options)) (*ecs.DescribeTasksOutput, error) + describeContainerInstances func(context.Context, *ecs.DescribeContainerInstancesInput, ...func(*ecs.Options)) (*ecs.DescribeContainerInstancesOutput, error) +} + +func newECSClientAdapter(c *ecs.Client) ecsClientAdapter { + return ecsClientAdapter{ + listClusters: c.ListClusters, + describeClusters: c.DescribeClusters, + listServices: c.ListServices, + describeServices: c.DescribeServices, + listTasks: c.ListTasks, + describeTasks: c.DescribeTasks, + describeContainerInstances: c.DescribeContainerInstances, + } +} + +func (a ecsClientAdapter) ListClusters(ctx context.Context, params *ecs.ListClustersInput, optFns ...func(*ecs.Options)) (*ecs.ListClustersOutput, error) { + return a.listClusters(ctx, params, optFns...) +} + +func (a ecsClientAdapter) DescribeClusters(ctx context.Context, params *ecs.DescribeClustersInput, optFns ...func(*ecs.Options)) (*ecs.DescribeClustersOutput, error) { + return a.describeClusters(ctx, params, optFns...) +} + +func (a ecsClientAdapter) ListServices(ctx context.Context, params *ecs.ListServicesInput, optFns ...func(*ecs.Options)) (*ecs.ListServicesOutput, error) { + return a.listServices(ctx, params, optFns...) +} + +func (a ecsClientAdapter) DescribeServices(ctx context.Context, params *ecs.DescribeServicesInput, optFns ...func(*ecs.Options)) (*ecs.DescribeServicesOutput, error) { + return a.describeServices(ctx, params, optFns...) +} + +func (a ecsClientAdapter) ListTasks(ctx context.Context, params *ecs.ListTasksInput, optFns ...func(*ecs.Options)) (*ecs.ListTasksOutput, error) { + return a.listTasks(ctx, params, optFns...) +} + +func (a ecsClientAdapter) DescribeTasks(ctx context.Context, params *ecs.DescribeTasksInput, optFns ...func(*ecs.Options)) (*ecs.DescribeTasksOutput, error) { + return a.describeTasks(ctx, params, optFns...) +} + +func (a ecsClientAdapter) DescribeContainerInstances(ctx context.Context, params *ecs.DescribeContainerInstancesInput, optFns ...func(*ecs.Options)) (*ecs.DescribeContainerInstancesOutput, error) { + return a.describeContainerInstances(ctx, params, optFns...) +} + +type ecsEC2Client interface { + DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) + DescribeNetworkInterfaces(context.Context, *ec2.DescribeNetworkInterfacesInput, ...func(*ec2.Options)) (*ec2.DescribeNetworkInterfacesOutput, error) +} + +// ECSDiscovery periodically performs ECS-SD requests. It implements +// the Discoverer interface. +type ECSDiscovery struct { + *refresh.Discovery + logger *slog.Logger + cfg *ECSSDConfig + ecs ecsClient + ec2 ecsEC2Client +} + +// NewECSDiscovery returns a new ECSDiscovery which periodically refreshes its targets. +func NewECSDiscovery(conf *ECSSDConfig, opts discovery.DiscovererOptions) (*ECSDiscovery, error) { + m, ok := opts.Metrics.(*ecsMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type") + } + + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() + } + d := &ECSDiscovery{ + logger: opts.Logger, + cfg: conf, + } + d.Discovery = refresh.NewDiscovery( + refresh.Options{ + Logger: opts.Logger, + Mech: "ecs", + Interval: time.Duration(d.cfg.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ) + return d, nil +} + +func (d *ECSDiscovery) initEcsClient(ctx context.Context) error { + if d.ecs != nil && d.ec2 != nil { + return nil + } + + if d.cfg.Region == "" { + return errors.New("region must be set for ECS service discovery") + } + + // Build the HTTP client from the provided HTTPClientConfig. + client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "ecs_sd") + if err != nil { + return err + } + + // Build the AWS config with the provided region. + var configOptions []func(*awsConfig.LoadOptions) error + configOptions = append(configOptions, awsConfig.WithRegion(d.cfg.Region)) + configOptions = append(configOptions, awsConfig.WithHTTPClient(client)) + + // Only set static credentials if both access key and secret key are provided + // Otherwise, let AWS SDK use its default credential chain + if d.cfg.AccessKey != "" && d.cfg.SecretKey != "" { + credProvider := credentials.NewStaticCredentialsProvider(d.cfg.AccessKey, string(d.cfg.SecretKey), "") + configOptions = append(configOptions, awsConfig.WithCredentialsProvider(credProvider)) + } + + if d.cfg.Profile != "" { + configOptions = append(configOptions, awsConfig.WithSharedConfigProfile(d.cfg.Profile)) + } + + cfg, err := awsConfig.LoadDefaultConfig(ctx, configOptions...) + if err != nil { + d.logger.Error("Failed to create AWS config", "error", err) + return fmt.Errorf("could not create aws config: %w", err) + } + + // If the role ARN is set, assume the role to get credentials and set the credentials provider in the config. + if d.cfg.RoleARN != "" { + assumeProvider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), d.cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { + if d.cfg.ExternalID != "" { + o.ExternalID = aws.String(d.cfg.ExternalID) + } + }) + cfg.Credentials = aws.NewCredentialsCache(assumeProvider) + } + + d.ecs = newECSClientAdapter(ecs.NewFromConfig(cfg, func(options *ecs.Options) { + if d.cfg.Endpoint != "" { + options.BaseEndpoint = &d.cfg.Endpoint + } + options.HTTPClient = client + })) + + d.ec2 = newEC2ClientAdapter(ec2.NewFromConfig(cfg, func(options *ec2.Options) { + options.HTTPClient = client + })) + + // Test credentials by making a simple API call + testCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + _, err = d.ecs.DescribeClusters(testCtx, &ecs.DescribeClustersInput{}) + if err != nil { + d.logger.Error("Failed to test ECS credentials", "error", err) + return fmt.Errorf("ECS credential test failed: %w", err) + } + + return nil +} + +// listClusterARNs returns a slice of cluster arns. +// This method does not use concurrency as it's a simple paginated call. +func (d *ECSDiscovery) listClusterARNs(ctx context.Context) ([]string, error) { + var ( + clusterARNs []string + nextToken *string + ) + for { + resp, err := d.ecs.ListClusters(ctx, &ecs.ListClustersInput{ + NextToken: nextToken, + MaxResults: aws.Int32(100), + }) + if err != nil { + return nil, fmt.Errorf("could not list clusters: %w", err) + } + + clusterARNs = append(clusterARNs, resp.ClusterArns...) + + if resp.NextToken == nil { + break + } + nextToken = resp.NextToken + } + + return clusterARNs, nil +} + +// describeClusters returns a map of cluster ARN to a slice of clusters. +// Uses concurrent requests limited by RequestConcurrency to respect AWS API throttling. +// Clusters are described in batches of 100 to respect AWS API limits (DescribeClusters allows up to 100 clusters per call). +func (d *ECSDiscovery) describeClusters(ctx context.Context, clusters []string) (map[string]types.Cluster, error) { + mu := sync.Mutex{} + clusterMap := make(map[string]types.Cluster) + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for batch := range slices.Chunk(clusters, 100) { + errg.Go(func() error { + resp, err := d.ecs.DescribeClusters(ectx, &ecs.DescribeClustersInput{ + Clusters: batch, + Include: []types.ClusterField{"TAGS"}, + }) + if err != nil { + d.logger.Error("Failed to describe clusters", "clusters", batch, "error", err) + return fmt.Errorf("could not describe clusters %v: %w", batch, err) + } + + for _, cluster := range resp.Clusters { + if cluster.ClusterArn != nil { + mu.Lock() + clusterMap[*cluster.ClusterArn] = cluster + mu.Unlock() + } + } + return nil + }) + } + + return clusterMap, errg.Wait() +} + +// listServiceARNs returns a map of cluster ARN to a slice of service ARNs. +// Uses concurrent requests limited by RequestConcurrency to respect AWS API throttling. +// Services are listed in batches of 100 to respect AWS API limits (ListServices allows up to 100 services per call). +func (d *ECSDiscovery) listServiceARNs(ctx context.Context, clusters []string) (map[string][]string, error) { + mu := sync.Mutex{} + services := make(map[string][]string) + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for _, clusterARN := range clusters { + errg.Go(func() error { + var nextToken *string + var serviceARNs []string + for { + resp, err := d.ecs.ListServices(ectx, &ecs.ListServicesInput{ + Cluster: aws.String(clusterARN), + NextToken: nextToken, + MaxResults: aws.Int32(100), + }) + if err != nil { + return fmt.Errorf("could not list services for cluster %q: %w", clusterARN, err) + } + + serviceARNs = append(serviceARNs, resp.ServiceArns...) + + if resp.NextToken == nil { + break + } + nextToken = resp.NextToken + } + + mu.Lock() + services[clusterARN] = serviceARNs + mu.Unlock() + return nil + }) + } + + return services, errg.Wait() +} + +// describeServices returns a map of service name to service. +// Uses concurrent requests limited by RequestConcurrency to respect AWS API throttling. +// Services are described in batches of 10 to respect AWS API limits (DescribeServices allows up to 10 services per call). +func (d *ECSDiscovery) describeServices(ctx context.Context, clusterARN string, serviceARNS []string) (map[string]types.Service, error) { + mu := sync.Mutex{} + services := make(map[string]types.Service) + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for batch := range slices.Chunk(serviceARNS, 10) { + errg.Go(func() error { + resp, err := d.ecs.DescribeServices(ectx, &ecs.DescribeServicesInput{ + Cluster: aws.String(clusterARN), + Services: batch, + Include: []types.ServiceField{"TAGS"}, + }) + if err != nil { + d.logger.Error("Failed to describe services", "cluster", clusterARN, "batch", batch, "error", err) + return fmt.Errorf("could not describe services for cluster %q: batch %v: %w", clusterARN, batch, err) + } + + for _, service := range resp.Services { + if service.ServiceArn != nil { + mu.Lock() + services[*service.ServiceName] = service + mu.Unlock() + } + } + return nil + }) + } + + return services, errg.Wait() +} + +// listTaskARNs returns a map of clustersARN to a slice of task ARNs. +// Uses concurrent requests limited by RequestConcurrency to respect AWS API throttling. +// Tasks are listed in batches of 100 to respect AWS API limits (ListTasks allows up to 100 tasks per call). +// This method also uses pagination to handle cases where there are more than 100 tasks in a cluster. +func (d *ECSDiscovery) listTaskARNs(ctx context.Context, clusterARNs []string) (map[string][]string, error) { + mu := sync.Mutex{} + tasks := make(map[string][]string) + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for _, clusterARN := range clusterARNs { + errg.Go(func() error { + var ( + nextToken *string + taskARNs []string + ) + for { + resp, err := d.ecs.ListTasks(ectx, &ecs.ListTasksInput{ + Cluster: aws.String(clusterARN), + NextToken: nextToken, + MaxResults: aws.Int32(100), + }) + if err != nil { + return fmt.Errorf("could not list tasks for cluster %q: %w", clusterARN, err) + } + + taskARNs = append(taskARNs, resp.TaskArns...) + + if resp.NextToken == nil { + break + } + nextToken = resp.NextToken + } + + mu.Lock() + tasks[clusterARN] = taskARNs + mu.Unlock() + return nil + }) + } + + return tasks, errg.Wait() +} + +// describeTasks returns a slice of tasks. +// Uses concurrent requests limited by RequestConcurrency to respect AWS API throttling. +// Tasks are described in batches of 100 to respect AWS API limits (DescribeTasks allows up to 100 tasks per call). +func (d *ECSDiscovery) describeTasks(ctx context.Context, clusterARN string, taskARNs []string) ([]types.Task, error) { + mu := sync.Mutex{} + var tasks []types.Task + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for batch := range slices.Chunk(taskARNs, 100) { + errg.Go(func() error { + resp, err := d.ecs.DescribeTasks(ectx, &ecs.DescribeTasksInput{ + Cluster: aws.String(clusterARN), + Tasks: batch, + Include: []types.TaskField{"TAGS"}, + }) + if err != nil { + d.logger.Error("Failed to describe tasks", "cluster", clusterARN, "batch", batch, "error", err) + return fmt.Errorf("could not describe tasks in cluster %q: batch %v: %w", clusterARN, batch, err) + } + + mu.Lock() + tasks = append(tasks, resp.Tasks...) + mu.Unlock() + return nil + }) + } + + return tasks, errg.Wait() +} + +// describeContainerInstances returns a map of container instance ARN to EC2 instance ID +// Uses concurrent requests limited by RequestConcurrency to respect AWS API throttling. +// Container instances are described in batches of 100 to respect AWS API limits (DescribeContainerInstances allows up to 100 container instances per call). +func (d *ECSDiscovery) describeContainerInstances(ctx context.Context, clusterARN string, tasks []types.Task) (map[string]string, error) { + containerInstanceARNs := make([]string, 0, len(tasks)) + for _, task := range tasks { + if task.ContainerInstanceArn != nil { + containerInstanceARNs = append(containerInstanceARNs, *task.ContainerInstanceArn) + } + } + + if len(containerInstanceARNs) == 0 { + return make(map[string]string), nil + } + + mu := sync.Mutex{} + containerInstToEC2 := make(map[string]string) + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for batch := range slices.Chunk(containerInstanceARNs, 100) { + errg.Go(func() error { + resp, err := d.ecs.DescribeContainerInstances(ectx, &ecs.DescribeContainerInstancesInput{ + Cluster: aws.String(clusterARN), + ContainerInstances: batch, + }) + if err != nil { + return fmt.Errorf("could not describe container instances: %w", err) + } + + for _, ci := range resp.ContainerInstances { + if ci.ContainerInstanceArn != nil && ci.Ec2InstanceId != nil { + mu.Lock() + containerInstToEC2[*ci.ContainerInstanceArn] = *ci.Ec2InstanceId + mu.Unlock() + } + } + return nil + }) + } + + return containerInstToEC2, errg.Wait() +} + +// ec2InstanceInfo holds information retrieved from EC2 DescribeInstances. +type ec2InstanceInfo struct { + privateIP string + publicIP string + subnetID string + instanceType string + tags map[string]string +} + +// describeEC2Instances returns a map of EC2 instance ID to instance information. +// Uses concurrent requests limited by RequestConcurrency to respect AWS API throttling. +// This method does not use concurrency as it's a simple paginated call. +func (d *ECSDiscovery) describeEC2Instances(ctx context.Context, instanceIDs []string) (map[string]ec2InstanceInfo, error) { + if len(instanceIDs) == 0 { + return make(map[string]ec2InstanceInfo), nil + } + + instanceInfo := make(map[string]ec2InstanceInfo) + var nextToken *string + + for { + resp, err := d.ec2.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: instanceIDs, + NextToken: nextToken, + }) + if err != nil { + return nil, fmt.Errorf("could not describe EC2 instances: %w", err) + } + + for _, reservation := range resp.Reservations { + for _, instance := range reservation.Instances { + if instance.InstanceId != nil && instance.PrivateIpAddress != nil { + info := ec2InstanceInfo{ + privateIP: *instance.PrivateIpAddress, + tags: make(map[string]string), + } + if instance.PublicIpAddress != nil { + info.publicIP = *instance.PublicIpAddress + } + if instance.SubnetId != nil { + info.subnetID = *instance.SubnetId + } + if instance.InstanceType != "" { + info.instanceType = string(instance.InstanceType) + } + // Collect EC2 instance tags + for _, tag := range instance.Tags { + if tag.Key != nil && tag.Value != nil { + info.tags[*tag.Key] = *tag.Value + } + } + instanceInfo[*instance.InstanceId] = info + } + } + } + + if resp.NextToken == nil { + break + } + nextToken = resp.NextToken + } + + return instanceInfo, nil +} + +// describeNetworkInterfaces returns a map of ENI ID to public IP address. +// This is needed to get the public IP for tasks using awsvpc network mode, as the ENI is what gets the public IP, not the EC2 instance. +// This method does not use concurrency as it's a simple paginated call. +func (d *ECSDiscovery) describeNetworkInterfaces(ctx context.Context, tasks []types.Task) (map[string]string, error) { + eniIDs := make([]string, 0, len(tasks)) + + for _, task := range tasks { + for _, attachment := range task.Attachments { + if attachment.Type != nil && *attachment.Type == "ElasticNetworkInterface" { + for _, detail := range attachment.Details { + if detail.Name != nil && *detail.Name == "networkInterfaceId" && detail.Value != nil { + eniIDs = append(eniIDs, *detail.Value) + break + } + } + break + } + } + } + + if len(eniIDs) == 0 { + return make(map[string]string), nil + } + + eniToPublicIP := make(map[string]string) + var nextToken *string + + for { + resp, err := d.ec2.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ + NetworkInterfaceIds: eniIDs, + NextToken: nextToken, + }) + if err != nil { + return nil, fmt.Errorf("could not describe network interfaces: %w", err) + } + + for _, eni := range resp.NetworkInterfaces { + if eni.NetworkInterfaceId != nil && eni.Association != nil && eni.Association.PublicIp != nil { + eniToPublicIP[*eni.NetworkInterfaceId] = *eni.Association.PublicIp + } + } + + if resp.NextToken == nil { + break + } + nextToken = resp.NextToken + } + + return eniToPublicIP, nil +} + +func (d *ECSDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + err := d.initEcsClient(ctx) + if err != nil { + return nil, err + } + + var clusters []string + if len(d.cfg.Clusters) == 0 { + clusters, err = d.listClusterARNs(ctx) + if err != nil { + return nil, err + } + } else { + clusters = d.cfg.Clusters + } + + if len(clusters) == 0 { + return []*targetgroup.Group{ + { + Source: d.cfg.Region, + }, + }, nil + } + + tg := &targetgroup.Group{ + Source: d.cfg.Region, + } + + // Fetch cluster details, service ARNs, and task ARNs in parallel + var ( + clusterMap map[string]types.Cluster + serviceMap map[string][]string + taskMap map[string][]string + ) + + clusterErrg, clusterCtx := errgroup.WithContext(ctx) + clusterErrg.Go(func() error { + var err error + clusterMap, err = d.describeClusters(clusterCtx, clusters) + return err + }) + clusterErrg.Go(func() error { + var err error + serviceMap, err = d.listServiceARNs(clusterCtx, clusters) + return err + }) + clusterErrg.Go(func() error { + var err error + taskMap, err = d.listTaskARNs(clusterCtx, clusters) + return err + }) + + if err := clusterErrg.Wait(); err != nil { + return nil, err + } + + // Use goroutines to process clusters in parallel + var ( + clusterWg sync.WaitGroup + clusterMu sync.Mutex + clusterTargets []model.LabelSet + ) + + for clusterARN, taskARNs := range taskMap { + if len(taskARNs) == 0 { + continue + } + + clusterWg.Add(1) + + go func(cluster types.Cluster, serviceARNs, taskARNs []string) { + defer clusterWg.Done() + + // Fetch services and tasks in parallel (they're independent) + var ( + services map[string]types.Service + tasks []types.Task + ) + + resourceErrg, resourceCtx := errgroup.WithContext(ctx) + resourceErrg.Go(func() error { + var err error + services, err = d.describeServices(resourceCtx, *cluster.ClusterArn, serviceARNs) + if err != nil { + d.logger.Error("Failed to describe services for cluster", "cluster", *cluster.ClusterArn, "error", err) + } + return err + }) + resourceErrg.Go(func() error { + var err error + tasks, err = d.describeTasks(resourceCtx, *cluster.ClusterArn, taskARNs) + if err != nil { + d.logger.Error("Failed to describe tasks for cluster", "cluster", *cluster.ClusterArn, "error", err) + } + return err + }) + + if err := resourceErrg.Wait(); err != nil { + return + } + + // Fetch container instances and network interfaces in parallel (both depend on tasks) + var ( + containerInstances map[string]string + eniToPublicIP map[string]string + ) + + instanceErrg, instanceCtx := errgroup.WithContext(ctx) + instanceErrg.Go(func() error { + var err error + containerInstances, err = d.describeContainerInstances(instanceCtx, *cluster.ClusterArn, tasks) + if err != nil { + d.logger.Error("Failed to describe container instances for cluster", "cluster", *cluster.ClusterArn, "error", err) + } + return err + }) + instanceErrg.Go(func() error { + var err error + eniToPublicIP, err = d.describeNetworkInterfaces(instanceCtx, tasks) + if err != nil { + d.logger.Error("Failed to describe network interfaces for cluster", "cluster", *cluster.ClusterArn, "error", err) + } + return err + }) + + if err := instanceErrg.Wait(); err != nil { + return + } + + ec2Instances := make(map[string]ec2InstanceInfo) + if len(containerInstances) > 0 { + // Deduplicate EC2 instance IDs (multiple tasks can share the same instance) + ec2InstanceIDSet := make(map[string]struct{}) + for _, ec2ID := range containerInstances { + ec2InstanceIDSet[ec2ID] = struct{}{} + } + ec2InstanceIDs := make([]string, 0, len(ec2InstanceIDSet)) + for ec2ID := range ec2InstanceIDSet { + ec2InstanceIDs = append(ec2InstanceIDs, ec2ID) + } + ec2Instances, err = d.describeEC2Instances(ctx, ec2InstanceIDs) + if err != nil { + d.logger.Error("Failed to describe EC2 instances for cluster", "cluster", *cluster.ClusterArn, "error", err) + return + } + } + + var ( + taskWg sync.WaitGroup + taskMu sync.Mutex + taskTargets []model.LabelSet + ) + + for _, task := range tasks { + taskWg.Add(1) + + go func(cluster types.Cluster, services map[string]types.Service, task types.Task, containerInstances map[string]string, ec2Instances map[string]ec2InstanceInfo, eniToPublicIP map[string]string) { + defer taskWg.Done() + + var ( + ipAddress, subnetID, publicIP string + networkMode string + ec2InstanceID, ec2InstanceType, ec2InstancePrivateIP, ec2InstancePublicIP string + ) + + // Try to get IP from ENI attachment (awsvpc mode) + var eniAttachment *types.Attachment + for _, attachment := range task.Attachments { + if attachment.Type != nil && *attachment.Type == "ElasticNetworkInterface" { + eniAttachment = &attachment + break + } + } + + if eniAttachment != nil { + // awsvpc networking mode - get IP from ENI + networkMode = "awsvpc" + var eniID string + for _, detail := range eniAttachment.Details { + switch *detail.Name { + case "privateIPv4Address": + ipAddress = *detail.Value + case "subnetId": + subnetID = *detail.Value + case "networkInterfaceId": + eniID = *detail.Value + } + } + // Get public IP from ENI if available + if eniID != "" { + if pub, ok := eniToPublicIP[eniID]; ok { + publicIP = pub + } + } + } else if task.ContainerInstanceArn != nil { + // bridge/host networking mode - need to get EC2 instance IP and subnet + networkMode = "bridge" + var ok bool + ec2InstanceID, ok = containerInstances[*task.ContainerInstanceArn] + if ok { + info, ok := ec2Instances[ec2InstanceID] + if ok { + ipAddress = info.privateIP + publicIP = info.publicIP + subnetID = info.subnetID + ec2InstanceType = info.instanceType + ec2InstancePrivateIP = info.privateIP + ec2InstancePublicIP = info.publicIP + } else { + d.logger.Debug("EC2 instance info not found", "instance", ec2InstanceID, "task", *task.TaskArn) + } + } else { + d.logger.Debug("Container instance not found in map", "arn", *task.ContainerInstanceArn, "task", *task.TaskArn) + } + } + + // Get EC2 instance metadata for awsvpc tasks running on EC2 + // We want the instance type and the host IPs for advanced use cases + if networkMode == "awsvpc" && task.ContainerInstanceArn != nil { + var ok bool + ec2InstanceID, ok = containerInstances[*task.ContainerInstanceArn] + if ok { + info, ok := ec2Instances[ec2InstanceID] + if ok { + ec2InstanceType = info.instanceType + ec2InstancePrivateIP = info.privateIP + ec2InstancePublicIP = info.publicIP + } + } + } + + if ipAddress == "" { + return + } + + labels := model.LabelSet{ + ecsLabelClusterARN: model.LabelValue(*cluster.ClusterArn), + ecsLabelCluster: model.LabelValue(*cluster.ClusterName), + ecsLabelTaskGroup: model.LabelValue(*task.Group), + ecsLabelTaskARN: model.LabelValue(*task.TaskArn), + ecsLabelTaskDefinition: model.LabelValue(*task.TaskDefinitionArn), + ecsLabelIPAddress: model.LabelValue(ipAddress), + ecsLabelRegion: model.LabelValue(d.cfg.Region), + ecsLabelLaunchType: model.LabelValue(task.LaunchType), + ecsLabelAvailabilityZone: model.LabelValue(*task.AvailabilityZone), + ecsLabelDesiredStatus: model.LabelValue(*task.DesiredStatus), + ecsLabelLastStatus: model.LabelValue(*task.LastStatus), + ecsLabelHealthStatus: model.LabelValue(task.HealthStatus), + ecsLabelNetworkMode: model.LabelValue(networkMode), + } + + // Add subnet ID when available (awsvpc mode from ENI, bridge/host from EC2 instance) + if subnetID != "" { + labels[ecsLabelSubnetID] = model.LabelValue(subnetID) + } + + // Add container instance and EC2 instance info for EC2 launch type + if task.ContainerInstanceArn != nil { + labels[ecsLabelContainerInstanceARN] = model.LabelValue(*task.ContainerInstanceArn) + } + if ec2InstanceID != "" { + labels[ecsLabelEC2InstanceID] = model.LabelValue(ec2InstanceID) + } + if ec2InstanceType != "" { + labels[ecsLabelEC2InstanceType] = model.LabelValue(ec2InstanceType) + } + if ec2InstancePrivateIP != "" { + labels[ecsLabelEC2InstancePrivateIP] = model.LabelValue(ec2InstancePrivateIP) + } + if ec2InstancePublicIP != "" { + labels[ecsLabelEC2InstancePublicIP] = model.LabelValue(ec2InstancePublicIP) + } + if publicIP != "" { + labels[ecsLabelPublicIP] = model.LabelValue(publicIP) + } + + if task.PlatformFamily != nil { + labels[ecsLabelPlatformFamily] = model.LabelValue(*task.PlatformFamily) + } + if task.PlatformVersion != nil { + labels[ecsLabelPlatformVersion] = model.LabelValue(*task.PlatformVersion) + } + + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(ipAddress, strconv.Itoa(d.cfg.Port))) + + // Add cluster tags + for _, clusterTag := range cluster.Tags { + if clusterTag.Key != nil && clusterTag.Value != nil { + labels[model.LabelName(ecsLabelTagCluster+strutil.SanitizeLabelName(*clusterTag.Key))] = model.LabelValue(*clusterTag.Value) + } + } + + // If this is not a standalone task, add service information and tags + if !isStandaloneTask(task) { + service, ok := services[getServiceNameFromTaskGroup(task)] + if !ok { + d.logger.Debug("Service not found for task", "task", *task.TaskArn, "service", getServiceNameFromTaskGroup(task)) + } + if service.ServiceName != nil { + labels[ecsLabelService] = model.LabelValue(*service.ServiceName) + } + if service.ServiceArn != nil { + labels[ecsLabelServiceARN] = model.LabelValue(*service.ServiceArn) + } + if service.Status != nil { + labels[ecsLabelServiceStatus] = model.LabelValue(*service.Status) + } + + // Add service tags + for _, serviceTag := range service.Tags { + if serviceTag.Key != nil && serviceTag.Value != nil { + labels[model.LabelName(ecsLabelTagService+strutil.SanitizeLabelName(*serviceTag.Key))] = model.LabelValue(*serviceTag.Value) + } + } + } + + // Add task tags + for _, taskTag := range task.Tags { + if taskTag.Key != nil && taskTag.Value != nil { + labels[model.LabelName(ecsLabelTagTask+strutil.SanitizeLabelName(*taskTag.Key))] = model.LabelValue(*taskTag.Value) + } + } + + // Add EC2 instance tags (if running on EC2) + if ec2InstanceID != "" { + if info, ok := ec2Instances[ec2InstanceID]; ok { + for tagKey, tagValue := range info.tags { + labels[model.LabelName(ecsLabelTagEC2+strutil.SanitizeLabelName(tagKey))] = model.LabelValue(tagValue) + } + } + } + + taskMu.Lock() + taskTargets = append(taskTargets, labels) + taskMu.Unlock() + }(cluster, services, task, containerInstances, ec2Instances, eniToPublicIP) + } + + taskWg.Wait() + + // Add this cluster's task targets to the overall collection + clusterMu.Lock() + clusterTargets = append(clusterTargets, taskTargets...) + clusterMu.Unlock() + }(clusterMap[clusterARN], serviceMap[clusterARN], taskARNs) + } + + clusterWg.Wait() + + // Set all targets to the target group + tg.Targets = clusterTargets + + return []*targetgroup.Group{tg}, nil +} + +func isStandaloneTask(task types.Task) bool { + // A standalone task will have a group of "family:task-def-name" + return task.Group != nil && strings.HasPrefix(*task.Group, "family:") +} + +func getServiceNameFromTaskGroup(task types.Task) string { + return strings.Split(*task.Group, ":")[1] +} diff --git a/discovery/aws/ecs_test.go b/discovery/aws/ecs_test.go new file mode 100644 index 00000000000..e1e494196e1 --- /dev/null +++ b/discovery/aws/ecs_test.go @@ -0,0 +1,1819 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/ecs" + ecsTypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +// Struct for test data. +type ecsDataStore struct { + region string + + clusters []ecsTypes.Cluster + services []ecsTypes.Service + tasks []ecsTypes.Task + containerInstances []ecsTypes.ContainerInstance + ec2Instances map[string]ec2InstanceInfo // EC2 instance ID to instance info + eniPublicIPs map[string]string // ENI ID to public IP +} + +func TestECSDiscoveryListClusterARNs(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // iterate through the test cases + for _, tt := range []struct { + name string + ecsData *ecsDataStore + expected []string + }{ + { + name: "MultipleClusters", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + }, + { + ClusterName: strptr("prod-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/prod-cluster"), + Status: strptr("ACTIVE"), + }, + }, + }, + expected: []string{ + "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + "arn:aws:ecs:us-west-2:123456789012:cluster/prod-cluster", + }, + }, + { + name: "SingleCluster", + ecsData: &ecsDataStore{ + region: "us-east-1", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("single-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-east-1:123456789012:cluster/single-cluster"), + Status: strptr("ACTIVE"), + }, + }, + }, + expected: []string{ + "arn:aws:ecs:us-east-1:123456789012:cluster/single-cluster", + }, + }, + { + name: "NoClusters", + ecsData: &ecsDataStore{ + region: "us-east-1", + clusters: []ecsTypes.Cluster{}, + }, + expected: nil, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockECSClient(tt.ecsData) + + d := &ECSDiscovery{ + ecs: client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 10, + }, + } + + clusters, err := d.listClusterARNs(ctx) + require.NoError(t, err) + require.Equal(t, tt.expected, clusters) + }) + } +} + +func TestECSDiscoveryDescribeClusters(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // iterate through the test cases + for _, tt := range []struct { + name string + ecsData *ecsDataStore + clusterARNs []string + expected map[string]ecsTypes.Cluster + }{ + { + name: "SingleClusterWithTags", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("test")}, + {Key: strptr("Team"), Value: strptr("backend")}, + }, + }, + }, + }, + clusterARNs: []string{"arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"}, + expected: map[string]ecsTypes.Cluster{ + "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster": { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("test")}, + {Key: strptr("Team"), Value: strptr("backend")}, + }, + }, + }, + }, + { + name: "MultipleClusters", + ecsData: &ecsDataStore{ + region: "us-east-1", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("cluster-1"), + ClusterArn: strptr("arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1"), + Status: strptr("ACTIVE"), + }, + { + ClusterName: strptr("cluster-2"), + ClusterArn: strptr("arn:aws:ecs:us-east-1:123456789012:cluster/cluster-2"), + Status: strptr("DRAINING"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Stage"), Value: strptr("prod")}, + }, + }, + }, + }, + clusterARNs: []string{ + "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1", + "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-2", + }, + expected: map[string]ecsTypes.Cluster{ + "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1": { + ClusterName: strptr("cluster-1"), + ClusterArn: strptr("arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1"), + Status: strptr("ACTIVE"), + }, + "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-2": { + ClusterName: strptr("cluster-2"), + ClusterArn: strptr("arn:aws:ecs:us-east-1:123456789012:cluster/cluster-2"), + Status: strptr("DRAINING"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Stage"), Value: strptr("prod")}, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockECSClient(tt.ecsData) + + d := &ECSDiscovery{ + ecs: client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 10, + }, + } + + clusterMap, err := d.describeClusters(ctx, tt.clusterARNs) + require.NoError(t, err) + require.Equal(t, tt.expected, clusterMap) + }) + } +} + +func TestECSDiscoveryListServiceARNs(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + ecsData *ecsDataStore + clusterARNs []string + expected map[string][]string + }{ + { + name: "SingleClusterWithServices", + ecsData: &ecsDataStore{ + region: "us-west-2", + services: []ecsTypes.Service{ + { + ServiceName: strptr("web-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/web-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + }, + { + ServiceName: strptr("api-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/api-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + }, + }, + }, + clusterARNs: []string{"arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"}, + expected: map[string][]string{ + "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster": { + "arn:aws:ecs:us-west-2:123456789012:service/test-cluster/web-service", + "arn:aws:ecs:us-west-2:123456789012:service/test-cluster/api-service", + }, + }, + }, + { + name: "MultipleClusters", + ecsData: &ecsDataStore{ + region: "us-west-2", + services: []ecsTypes.Service{ + { + ServiceName: strptr("web-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/cluster-1/web-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/cluster-1"), + Status: strptr("ACTIVE"), + }, + { + ServiceName: strptr("api-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/cluster-2/api-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/cluster-2"), + Status: strptr("ACTIVE"), + }, + }, + }, + clusterARNs: []string{ + "arn:aws:ecs:us-west-2:123456789012:cluster/cluster-1", + "arn:aws:ecs:us-west-2:123456789012:cluster/cluster-2", + }, + expected: map[string][]string{ + "arn:aws:ecs:us-west-2:123456789012:cluster/cluster-1": { + "arn:aws:ecs:us-west-2:123456789012:service/cluster-1/web-service", + }, + "arn:aws:ecs:us-west-2:123456789012:cluster/cluster-2": { + "arn:aws:ecs:us-west-2:123456789012:service/cluster-2/api-service", + }, + }, + }, + { + name: "EmptyCluster", + ecsData: &ecsDataStore{ + region: "us-west-2", + services: []ecsTypes.Service{}, + }, + clusterARNs: []string{"arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"}, + expected: map[string][]string{ + "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster": nil, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockECSClient(tt.ecsData) + + d := &ECSDiscovery{ + ecs: client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 2, + }, + } + + serviceMap, err := d.listServiceARNs(ctx, tt.clusterARNs) + require.NoError(t, err) + require.Equal(t, tt.expected, serviceMap) + }) + } +} + +func TestECSDiscoveryDescribeServices(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + ecsData *ecsDataStore + clusterARN string + serviceARNs []string + expected map[string]ecsTypes.Service + }{ + { + name: "ServicesWithTags", + ecsData: &ecsDataStore{ + region: "us-west-2", + services: []ecsTypes.Service{ + { + ServiceName: strptr("web-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/web-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("production")}, + {Key: strptr("Team"), Value: strptr("platform")}, + }, + }, + { + ServiceName: strptr("api-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/api-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("staging")}, + }, + }, + }, + }, + clusterARN: "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + serviceARNs: []string{ + "arn:aws:ecs:us-west-2:123456789012:service/test-cluster/web-service", + "arn:aws:ecs:us-west-2:123456789012:service/test-cluster/api-service", + }, + expected: map[string]ecsTypes.Service{ + "web-service": { + ServiceName: strptr("web-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/web-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("production")}, + {Key: strptr("Team"), Value: strptr("platform")}, + }, + }, + "api-service": { + ServiceName: strptr("api-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/api-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("staging")}, + }, + }, + }, + }, + { + name: "EmptyServiceList", + ecsData: &ecsDataStore{ + region: "us-west-2", + services: []ecsTypes.Service{}, + }, + clusterARN: "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + serviceARNs: []string{}, + expected: map[string]ecsTypes.Service{}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockECSClient(tt.ecsData) + + d := &ECSDiscovery{ + ecs: client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 2, + }, + } + + services, err := d.describeServices(ctx, tt.clusterARN, tt.serviceARNs) + require.NoError(t, err) + require.Equal(t, tt.expected, services) + }) + } +} + +func TestECSDiscoveryDescribeContainerInstances(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + ecsData *ecsDataStore + clusterARN string + tasks []ecsTypes.Task + expected map[string]string + }{ + { + name: "EC2Tasks", + ecsData: &ecsDataStore{ + region: "us-west-2", + containerInstances: []ecsTypes.ContainerInstance{ + { + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123"), + Ec2InstanceId: strptr("i-1234567890abcdef0"), + }, + { + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/xyz789"), + Ec2InstanceId: strptr("i-0987654321fedcba0"), + }, + }, + }, + clusterARN: "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123"), + LaunchType: ecsTypes.LaunchTypeEc2, + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-2"), + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/xyz789"), + LaunchType: ecsTypes.LaunchTypeEc2, + }, + }, + expected: map[string]string{ + "arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123": "i-1234567890abcdef0", + "arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/xyz789": "i-0987654321fedcba0", + }, + }, + { + name: "FargateTasks", + ecsData: &ecsDataStore{ + region: "us-west-2", + containerInstances: []ecsTypes.ContainerInstance{}, + }, + clusterARN: "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + LaunchType: ecsTypes.LaunchTypeFargate, + }, + }, + expected: map[string]string{}, + }, + { + name: "MixedTasks", + ecsData: &ecsDataStore{ + region: "us-west-2", + containerInstances: []ecsTypes.ContainerInstance{ + { + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123"), + Ec2InstanceId: strptr("i-1234567890abcdef0"), + }, + }, + }, + clusterARN: "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-ec2"), + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123"), + LaunchType: ecsTypes.LaunchTypeEc2, + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-fargate"), + LaunchType: ecsTypes.LaunchTypeFargate, + }, + }, + expected: map[string]string{ + "arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123": "i-1234567890abcdef0", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockECSClient(tt.ecsData) + + d := &ECSDiscovery{ + ecs: client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 2, + }, + } + + containerInstances, err := d.describeContainerInstances(ctx, tt.clusterARN, tt.tasks) + require.NoError(t, err) + require.Equal(t, tt.expected, containerInstances) + }) + } +} + +func TestECSDiscoveryDescribeEC2Instances(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + ecsData *ecsDataStore + instanceIDs []string + expected map[string]ec2InstanceInfo + }{ + { + name: "InstancesWithTags", + ecsData: &ecsDataStore{ + region: "us-west-2", + ec2Instances: map[string]ec2InstanceInfo{ + "i-1234567890abcdef0": { + privateIP: "10.0.1.50", + publicIP: "54.1.2.3", + subnetID: "subnet-12345", + instanceType: "t3.medium", + tags: map[string]string{ + "Name": "ecs-host-1", + "Environment": "production", + }, + }, + "i-0987654321fedcba0": { + privateIP: "10.0.1.75", + publicIP: "54.2.3.4", + subnetID: "subnet-67890", + instanceType: "t3.large", + tags: map[string]string{ + "Name": "ecs-host-2", + "Team": "platform", + }, + }, + }, + }, + instanceIDs: []string{"i-1234567890abcdef0", "i-0987654321fedcba0"}, + expected: map[string]ec2InstanceInfo{ + "i-1234567890abcdef0": { + privateIP: "10.0.1.50", + publicIP: "54.1.2.3", + subnetID: "subnet-12345", + instanceType: "t3.medium", + tags: map[string]string{ + "Name": "ecs-host-1", + "Environment": "production", + }, + }, + "i-0987654321fedcba0": { + privateIP: "10.0.1.75", + publicIP: "54.2.3.4", + subnetID: "subnet-67890", + instanceType: "t3.large", + tags: map[string]string{ + "Name": "ecs-host-2", + "Team": "platform", + }, + }, + }, + }, + { + name: "EmptyList", + ecsData: &ecsDataStore{ + region: "us-west-2", + ec2Instances: map[string]ec2InstanceInfo{}, + }, + instanceIDs: []string{}, + expected: map[string]ec2InstanceInfo{}, + }, + { + name: "InstanceWithoutPublicIP", + ecsData: &ecsDataStore{ + region: "us-west-2", + ec2Instances: map[string]ec2InstanceInfo{ + "i-privateonly": { + privateIP: "10.0.1.100", + publicIP: "", + subnetID: "subnet-private", + instanceType: "t3.micro", + tags: map[string]string{}, + }, + }, + }, + instanceIDs: []string{"i-privateonly"}, + expected: map[string]ec2InstanceInfo{ + "i-privateonly": { + privateIP: "10.0.1.100", + publicIP: "", + subnetID: "subnet-private", + instanceType: "t3.micro", + tags: map[string]string{}, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ec2Client := newMockECSEC2Client(tt.ecsData.ec2Instances, nil) + + d := &ECSDiscovery{ + ec2: ec2Client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 2, + }, + } + + instances, err := d.describeEC2Instances(ctx, tt.instanceIDs) + require.NoError(t, err) + require.Equal(t, tt.expected, instances) + }) + } +} + +func TestECSDiscoveryDescribeNetworkInterfaces(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + ecsData *ecsDataStore + tasks []ecsTypes.Task + expected map[string]string + }{ + { + name: "AwsvpcTasksWithPublicIPs", + ecsData: &ecsDataStore{ + region: "us-west-2", + eniPublicIPs: map[string]string{ + "eni-12345": "52.1.2.3", + "eni-67890": "52.2.3.4", + }, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + LaunchType: ecsTypes.LaunchTypeFargate, + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("networkInterfaceId"), Value: strptr("eni-12345")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.1.100")}, + }, + }, + }, + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-2"), + LaunchType: ecsTypes.LaunchTypeFargate, + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("networkInterfaceId"), Value: strptr("eni-67890")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.1.200")}, + }, + }, + }, + }, + }, + expected: map[string]string{ + "eni-12345": "52.1.2.3", + "eni-67890": "52.2.3.4", + }, + }, + { + name: "AwsvpcTasksWithoutPublicIPs", + ecsData: &ecsDataStore{ + region: "us-west-2", + eniPublicIPs: map[string]string{}, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + LaunchType: ecsTypes.LaunchTypeFargate, + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("networkInterfaceId"), Value: strptr("eni-private")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.1.100")}, + }, + }, + }, + }, + }, + expected: map[string]string{}, + }, + { + name: "BridgeTasksNoENI", + ecsData: &ecsDataStore{ + region: "us-west-2", + eniPublicIPs: map[string]string{}, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + LaunchType: ecsTypes.LaunchTypeEc2, + // No ENI attachment for bridge networking + Attachments: []ecsTypes.Attachment{}, + }, + }, + expected: map[string]string{}, + }, + { + name: "MixedTasks", + ecsData: &ecsDataStore{ + region: "us-west-2", + eniPublicIPs: map[string]string{ + "eni-fargate": "52.1.2.3", + }, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-fargate"), + LaunchType: ecsTypes.LaunchTypeFargate, + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("networkInterfaceId"), Value: strptr("eni-fargate")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.1.100")}, + }, + }, + }, + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-bridge"), + LaunchType: ecsTypes.LaunchTypeEc2, + Attachments: []ecsTypes.Attachment{}, + }, + }, + expected: map[string]string{ + "eni-fargate": "52.1.2.3", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ec2Client := newMockECSEC2Client(nil, tt.ecsData.eniPublicIPs) + + d := &ECSDiscovery{ + ec2: ec2Client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 2, + }, + } + + eniMap, err := d.describeNetworkInterfaces(ctx, tt.tasks) + require.NoError(t, err) + require.Equal(t, tt.expected, eniMap) + }) + } +} + +func TestECSDiscoveryListTaskARNs(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // iterate through the test cases + for _, tt := range []struct { + name string + ecsData *ecsDataStore + clusterARNs []string + expected map[string][]string + }{ + { + name: "TasksInCluster", + ecsData: &ecsDataStore{ + region: "us-west-2", + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Group: strptr("service:web-service"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/web-task:1"), + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-2"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Group: strptr("service:web-service"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/web-task:1"), + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-3"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Group: strptr("service:api-service"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/api-task:2"), + }, + }, + }, + clusterARNs: []string{"arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"}, + expected: map[string][]string{ + "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster": { + "arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1", + "arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-2", + "arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-3", + }, + }, + }, + { + name: "EmptyCluster", + ecsData: &ecsDataStore{ + region: "us-west-2", + tasks: []ecsTypes.Task{}, + }, + clusterARNs: []string{"arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"}, + expected: map[string][]string{ + "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster": nil, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockECSClient(tt.ecsData) + + d := &ECSDiscovery{ + ecs: client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 1, + }, + } + + taskMap, err := d.listTaskARNs(ctx, tt.clusterARNs) + require.NoError(t, err) + require.Equal(t, tt.expected, taskMap) + }) + } +} + +func TestECSDiscoveryDescribeTasks(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // iterate through the test cases + for _, tt := range []struct { + name string + ecsData *ecsDataStore + clusterARN string + taskARNs []string + expected []ecsTypes.Task + }{ + { + name: "TasksInCluster", + ecsData: &ecsDataStore{ + region: "us-west-2", + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Group: strptr("service:web-service"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/web-task:1"), + LastStatus: strptr("RUNNING"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("production")}, + }, + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-2"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Group: strptr("service:api-service"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/api-task:2"), + LastStatus: strptr("RUNNING"), + }, + }, + }, + clusterARN: "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + taskARNs: []string{ + "arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1", + "arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-2", + }, + expected: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Group: strptr("service:web-service"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/web-task:1"), + LastStatus: strptr("RUNNING"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("production")}, + }, + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-2"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Group: strptr("service:api-service"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/api-task:2"), + LastStatus: strptr("RUNNING"), + }, + }, + }, + { + name: "EmptyTaskList", + ecsData: &ecsDataStore{ + region: "us-west-2", + tasks: []ecsTypes.Task{}, + }, + clusterARN: "arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster", + taskARNs: []string{}, + expected: nil, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockECSClient(tt.ecsData) + + d := &ECSDiscovery{ + ecs: client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + RequestConcurrency: 1, + }, + } + + tasks, err := d.describeTasks(ctx, tt.clusterARN, tt.taskARNs) + require.NoError(t, err) + require.Equal(t, tt.expected, tasks) + }) + } +} + +func TestECSDiscoveryRefresh(t *testing.T) { + t.Parallel() + ctx := context.Background() + + tests := []struct { + name string + ecsData *ecsDataStore + expected []*targetgroup.Group + }{ + { + name: "SingleClusterWithTasks", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("Environment"), Value: strptr("test")}, + }, + }, + }, + services: []ecsTypes.Service{ + { + ServiceName: strptr("web-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/web-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + Tags: []ecsTypes.Tag{ + {Key: strptr("App"), Value: strptr("web")}, + }, + }, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/web-task:1"), + Group: strptr("service:web-service"), + LaunchType: ecsTypes.LaunchTypeFargate, + LastStatus: strptr("RUNNING"), + DesiredStatus: strptr("RUNNING"), + HealthStatus: ecsTypes.HealthStatusHealthy, + AvailabilityZone: strptr("us-west-2a"), + PlatformFamily: strptr("Linux"), + PlatformVersion: strptr("1.4.0"), + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("subnetId"), Value: strptr("subnet-12345")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.1.100")}, + {Name: strptr("networkInterfaceId"), Value: strptr("eni-fargate-123")}, + }, + }, + }, + Tags: []ecsTypes.Tag{ + {Key: strptr("Version"), Value: strptr("v1.0")}, + }, + }, + }, + eniPublicIPs: map[string]string{ + "eni-fargate-123": "52.1.2.3", + }, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("10.0.1.100:80"), + "__meta_ecs_cluster": model.LabelValue("test-cluster"), + "__meta_ecs_cluster_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + "__meta_ecs_service": model.LabelValue("web-service"), + "__meta_ecs_service_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/web-service"), + "__meta_ecs_service_status": model.LabelValue("ACTIVE"), + "__meta_ecs_task_group": model.LabelValue("service:web-service"), + "__meta_ecs_task_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + "__meta_ecs_task_definition": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task-definition/web-task:1"), + "__meta_ecs_region": model.LabelValue("us-west-2"), + "__meta_ecs_availability_zone": model.LabelValue("us-west-2a"), + "__meta_ecs_subnet_id": model.LabelValue("subnet-12345"), + "__meta_ecs_ip_address": model.LabelValue("10.0.1.100"), + "__meta_ecs_launch_type": model.LabelValue("FARGATE"), + "__meta_ecs_desired_status": model.LabelValue("RUNNING"), + "__meta_ecs_last_status": model.LabelValue("RUNNING"), + "__meta_ecs_health_status": model.LabelValue("HEALTHY"), + "__meta_ecs_platform_family": model.LabelValue("Linux"), + "__meta_ecs_platform_version": model.LabelValue("1.4.0"), + "__meta_ecs_network_mode": model.LabelValue("awsvpc"), + "__meta_ecs_public_ip": model.LabelValue("52.1.2.3"), + "__meta_ecs_tag_cluster_Environment": model.LabelValue("test"), + "__meta_ecs_tag_service_App": model.LabelValue("web"), + "__meta_ecs_tag_task_Version": model.LabelValue("v1.0"), + }, + }, + }, + }, + }, + { + name: "NoTasks", + ecsData: &ecsDataStore{ + region: "us-east-1", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("empty-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-east-1:123456789012:cluster/empty-cluster"), + Status: strptr("ACTIVE"), + }, + }, + services: []ecsTypes.Service{ + { + ServiceName: strptr("empty-service"), + ServiceArn: strptr("arn:aws:ecs:us-east-1:123456789012:service/empty-cluster/empty-service"), + ClusterArn: strptr("arn:aws:ecs:us-east-1:123456789012:cluster/empty-cluster"), + Status: strptr("ACTIVE"), + }, + }, + tasks: []ecsTypes.Task{}, + }, + expected: []*targetgroup.Group{ + { + Source: "us-east-1", + }, + }, + }, + { + name: "TaskWithoutENI", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + }, + }, + services: []ecsTypes.Service{ + { + ServiceName: strptr("service-1"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/service-1"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + }, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-1"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/task-def:1"), + Group: strptr("service:service-1"), + LaunchType: ecsTypes.LaunchTypeEc2, + LastStatus: strptr("RUNNING"), + DesiredStatus: strptr("RUNNING"), + HealthStatus: ecsTypes.HealthStatusHealthy, + AvailabilityZone: strptr("us-west-2a"), + // No attachments - should be skipped + Attachments: []ecsTypes.Attachment{}, + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + }, + }, + }, + { + name: "StandaloneTaskNoService", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("standalone-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/standalone-cluster"), + Status: strptr("ACTIVE"), + }, + }, + services: []ecsTypes.Service{}, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/standalone-cluster/task-standalone"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/standalone-cluster"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/standalone-task:1"), + Group: strptr("family:standalone-task"), + LaunchType: ecsTypes.LaunchTypeFargate, + LastStatus: strptr("RUNNING"), + DesiredStatus: strptr("RUNNING"), + HealthStatus: ecsTypes.HealthStatusHealthy, + AvailabilityZone: strptr("us-west-2a"), + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("subnetId"), Value: strptr("subnet-standalone-1")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.4.10")}, + {Name: strptr("networkInterfaceId"), Value: strptr("eni-standalone-123")}, + }, + }, + }, + Tags: []ecsTypes.Tag{ + {Key: strptr("Role"), Value: strptr("batch")}, + }, + }, + }, + eniPublicIPs: map[string]string{ + "eni-standalone-123": "52.4.5.6", + }, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("10.0.4.10:80"), + "__meta_ecs_cluster": model.LabelValue("standalone-cluster"), + "__meta_ecs_cluster_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:cluster/standalone-cluster"), + "__meta_ecs_task_group": model.LabelValue("family:standalone-task"), + "__meta_ecs_task_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task/standalone-cluster/task-standalone"), + "__meta_ecs_task_definition": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task-definition/standalone-task:1"), + "__meta_ecs_region": model.LabelValue("us-west-2"), + "__meta_ecs_availability_zone": model.LabelValue("us-west-2a"), + "__meta_ecs_subnet_id": model.LabelValue("subnet-standalone-1"), + "__meta_ecs_ip_address": model.LabelValue("10.0.4.10"), + "__meta_ecs_launch_type": model.LabelValue("FARGATE"), + "__meta_ecs_desired_status": model.LabelValue("RUNNING"), + "__meta_ecs_last_status": model.LabelValue("RUNNING"), + "__meta_ecs_health_status": model.LabelValue("HEALTHY"), + "__meta_ecs_network_mode": model.LabelValue("awsvpc"), + "__meta_ecs_public_ip": model.LabelValue("52.4.5.6"), + "__meta_ecs_tag_task_Role": model.LabelValue("batch"), + }, + }, + }, + }, + }, + { + name: "TaskWithBridgeNetworking", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + }, + }, + services: []ecsTypes.Service{ + { + ServiceName: strptr("bridge-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/bridge-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + Status: strptr("ACTIVE"), + }, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-bridge"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/bridge-task:1"), + Group: strptr("service:bridge-service"), + LaunchType: ecsTypes.LaunchTypeEc2, + LastStatus: strptr("RUNNING"), + DesiredStatus: strptr("RUNNING"), + HealthStatus: ecsTypes.HealthStatusHealthy, + AvailabilityZone: strptr("us-west-2a"), + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123"), + Attachments: []ecsTypes.Attachment{}, + }, + }, + containerInstances: []ecsTypes.ContainerInstance{ + { + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123"), + Ec2InstanceId: strptr("i-1234567890abcdef0"), + Status: strptr("ACTIVE"), + }, + }, + ec2Instances: map[string]ec2InstanceInfo{ + "i-1234567890abcdef0": { + privateIP: "10.0.1.50", + publicIP: "54.1.2.3", + subnetID: "subnet-bridge-1", + instanceType: "t3.medium", + tags: map[string]string{ + "Name": "ecs-host-1", + "Environment": "production", + }, + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("10.0.1.50:80"), + "__meta_ecs_cluster": model.LabelValue("test-cluster"), + "__meta_ecs_cluster_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:cluster/test-cluster"), + "__meta_ecs_service": model.LabelValue("bridge-service"), + "__meta_ecs_service_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:service/test-cluster/bridge-service"), + "__meta_ecs_service_status": model.LabelValue("ACTIVE"), + "__meta_ecs_task_group": model.LabelValue("service:bridge-service"), + "__meta_ecs_task_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task/test-cluster/task-bridge"), + "__meta_ecs_task_definition": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task-definition/bridge-task:1"), + "__meta_ecs_region": model.LabelValue("us-west-2"), + "__meta_ecs_availability_zone": model.LabelValue("us-west-2a"), + "__meta_ecs_ip_address": model.LabelValue("10.0.1.50"), + "__meta_ecs_subnet_id": model.LabelValue("subnet-bridge-1"), + "__meta_ecs_launch_type": model.LabelValue("EC2"), + "__meta_ecs_desired_status": model.LabelValue("RUNNING"), + "__meta_ecs_last_status": model.LabelValue("RUNNING"), + "__meta_ecs_health_status": model.LabelValue("HEALTHY"), + "__meta_ecs_network_mode": model.LabelValue("bridge"), + "__meta_ecs_container_instance_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:container-instance/test-cluster/abc123"), + "__meta_ecs_ec2_instance_id": model.LabelValue("i-1234567890abcdef0"), + "__meta_ecs_ec2_instance_type": model.LabelValue("t3.medium"), + "__meta_ecs_ec2_instance_private_ip": model.LabelValue("10.0.1.50"), + "__meta_ecs_ec2_instance_public_ip": model.LabelValue("54.1.2.3"), + "__meta_ecs_public_ip": model.LabelValue("54.1.2.3"), + "__meta_ecs_tag_ec2_Name": model.LabelValue("ecs-host-1"), + "__meta_ecs_tag_ec2_Environment": model.LabelValue("production"), + }, + }, + }, + }, + }, + { + name: "MixedNetworkingModes", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("mixed-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/mixed-cluster"), + Status: strptr("ACTIVE"), + }, + }, + services: []ecsTypes.Service{ + { + ServiceName: strptr("mixed-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/mixed-cluster/mixed-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/mixed-cluster"), + Status: strptr("ACTIVE"), + }, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/mixed-cluster/task-awsvpc"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/mixed-cluster"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/awsvpc-task:1"), + Group: strptr("service:mixed-service"), + LaunchType: ecsTypes.LaunchTypeFargate, + LastStatus: strptr("RUNNING"), + DesiredStatus: strptr("RUNNING"), + HealthStatus: ecsTypes.HealthStatusHealthy, + AvailabilityZone: strptr("us-west-2a"), + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("subnetId"), Value: strptr("subnet-12345")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.2.100")}, + {Name: strptr("networkInterfaceId"), Value: strptr("eni-mixed-awsvpc")}, + }, + }, + }, + }, + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/mixed-cluster/task-bridge"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/mixed-cluster"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/bridge-task:1"), + Group: strptr("service:mixed-service"), + LaunchType: ecsTypes.LaunchTypeEc2, + LastStatus: strptr("RUNNING"), + DesiredStatus: strptr("RUNNING"), + HealthStatus: ecsTypes.HealthStatusHealthy, + AvailabilityZone: strptr("us-west-2b"), + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/mixed-cluster/xyz789"), + Attachments: []ecsTypes.Attachment{}, + }, + }, + containerInstances: []ecsTypes.ContainerInstance{ + { + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/mixed-cluster/xyz789"), + Ec2InstanceId: strptr("i-0987654321fedcba0"), + Status: strptr("ACTIVE"), + }, + }, + ec2Instances: map[string]ec2InstanceInfo{ + "i-0987654321fedcba0": { + privateIP: "10.0.1.75", + publicIP: "54.2.3.4", + subnetID: "subnet-bridge-2", + instanceType: "t3.large", + tags: map[string]string{ + "Name": "mixed-host", + "Team": "platform", + }, + }, + }, + eniPublicIPs: map[string]string{ + "eni-mixed-awsvpc": "52.2.3.4", + }, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("10.0.2.100:80"), + "__meta_ecs_cluster": model.LabelValue("mixed-cluster"), + "__meta_ecs_cluster_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:cluster/mixed-cluster"), + "__meta_ecs_service": model.LabelValue("mixed-service"), + "__meta_ecs_service_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:service/mixed-cluster/mixed-service"), + "__meta_ecs_service_status": model.LabelValue("ACTIVE"), + "__meta_ecs_task_group": model.LabelValue("service:mixed-service"), + "__meta_ecs_task_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task/mixed-cluster/task-awsvpc"), + "__meta_ecs_task_definition": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task-definition/awsvpc-task:1"), + "__meta_ecs_region": model.LabelValue("us-west-2"), + "__meta_ecs_availability_zone": model.LabelValue("us-west-2a"), + "__meta_ecs_ip_address": model.LabelValue("10.0.2.100"), + "__meta_ecs_subnet_id": model.LabelValue("subnet-12345"), + "__meta_ecs_launch_type": model.LabelValue("FARGATE"), + "__meta_ecs_desired_status": model.LabelValue("RUNNING"), + "__meta_ecs_last_status": model.LabelValue("RUNNING"), + "__meta_ecs_health_status": model.LabelValue("HEALTHY"), + "__meta_ecs_network_mode": model.LabelValue("awsvpc"), + "__meta_ecs_public_ip": model.LabelValue("52.2.3.4"), + }, + { + model.AddressLabel: model.LabelValue("10.0.1.75:80"), + "__meta_ecs_cluster": model.LabelValue("mixed-cluster"), + "__meta_ecs_cluster_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:cluster/mixed-cluster"), + "__meta_ecs_service": model.LabelValue("mixed-service"), + "__meta_ecs_service_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:service/mixed-cluster/mixed-service"), + "__meta_ecs_service_status": model.LabelValue("ACTIVE"), + "__meta_ecs_task_group": model.LabelValue("service:mixed-service"), + "__meta_ecs_task_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task/mixed-cluster/task-bridge"), + "__meta_ecs_task_definition": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task-definition/bridge-task:1"), + "__meta_ecs_region": model.LabelValue("us-west-2"), + "__meta_ecs_availability_zone": model.LabelValue("us-west-2b"), + "__meta_ecs_ip_address": model.LabelValue("10.0.1.75"), + "__meta_ecs_subnet_id": model.LabelValue("subnet-bridge-2"), + "__meta_ecs_launch_type": model.LabelValue("EC2"), + "__meta_ecs_desired_status": model.LabelValue("RUNNING"), + "__meta_ecs_last_status": model.LabelValue("RUNNING"), + "__meta_ecs_health_status": model.LabelValue("HEALTHY"), + "__meta_ecs_network_mode": model.LabelValue("bridge"), + "__meta_ecs_container_instance_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:container-instance/mixed-cluster/xyz789"), + "__meta_ecs_ec2_instance_id": model.LabelValue("i-0987654321fedcba0"), + "__meta_ecs_ec2_instance_type": model.LabelValue("t3.large"), + "__meta_ecs_ec2_instance_private_ip": model.LabelValue("10.0.1.75"), + "__meta_ecs_ec2_instance_public_ip": model.LabelValue("54.2.3.4"), + "__meta_ecs_public_ip": model.LabelValue("54.2.3.4"), + "__meta_ecs_tag_ec2_Name": model.LabelValue("mixed-host"), + "__meta_ecs_tag_ec2_Team": model.LabelValue("platform"), + }, + }, + }, + }, + }, + { + name: "EC2WithAwsvpcNetworking", + ecsData: &ecsDataStore{ + region: "us-west-2", + clusters: []ecsTypes.Cluster{ + { + ClusterName: strptr("ec2-awsvpc-cluster"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/ec2-awsvpc-cluster"), + Status: strptr("ACTIVE"), + }, + }, + services: []ecsTypes.Service{ + { + ServiceName: strptr("ec2-awsvpc-service"), + ServiceArn: strptr("arn:aws:ecs:us-west-2:123456789012:service/ec2-awsvpc-cluster/ec2-awsvpc-service"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/ec2-awsvpc-cluster"), + Status: strptr("ACTIVE"), + }, + }, + tasks: []ecsTypes.Task{ + { + TaskArn: strptr("arn:aws:ecs:us-west-2:123456789012:task/ec2-awsvpc-cluster/task-ec2-awsvpc"), + ClusterArn: strptr("arn:aws:ecs:us-west-2:123456789012:cluster/ec2-awsvpc-cluster"), + TaskDefinitionArn: strptr("arn:aws:ecs:us-west-2:123456789012:task-definition/ec2-awsvpc-task:1"), + Group: strptr("service:ec2-awsvpc-service"), + LaunchType: ecsTypes.LaunchTypeEc2, + LastStatus: strptr("RUNNING"), + DesiredStatus: strptr("RUNNING"), + HealthStatus: ecsTypes.HealthStatusHealthy, + AvailabilityZone: strptr("us-west-2c"), + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/ec2-awsvpc-cluster/def456"), + // Has BOTH ENI attachment AND container instance ARN - should use ENI + Attachments: []ecsTypes.Attachment{ + { + Type: strptr("ElasticNetworkInterface"), + Details: []ecsTypes.KeyValuePair{ + {Name: strptr("subnetId"), Value: strptr("subnet-99999")}, + {Name: strptr("privateIPv4Address"), Value: strptr("10.0.3.200")}, + {Name: strptr("networkInterfaceId"), Value: strptr("eni-ec2-awsvpc")}, + }, + }, + }, + }, + }, + eniPublicIPs: map[string]string{ + "eni-ec2-awsvpc": "52.3.4.5", + }, + // Container instance data - IP should NOT be used, but instance type SHOULD be used + containerInstances: []ecsTypes.ContainerInstance{ + { + ContainerInstanceArn: strptr("arn:aws:ecs:us-west-2:123456789012:container-instance/ec2-awsvpc-cluster/def456"), + Ec2InstanceId: strptr("i-ec2awsvpcinstance"), + Status: strptr("ACTIVE"), + }, + }, + ec2Instances: map[string]ec2InstanceInfo{ + "i-ec2awsvpcinstance": { + privateIP: "10.0.9.99", // This IP should NOT be used (ENI IP is used instead) + publicIP: "54.3.4.5", // This public IP SHOULD be exposed + subnetID: "subnet-wrong", // This subnet should NOT be used (ENI subnet is used instead) + instanceType: "c5.2xlarge", // This instance type SHOULD be used + tags: map[string]string{ + "Name": "ec2-awsvpc-host", + "Owner": "team-a", + }, + }, + }, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("10.0.3.200:80"), + "__meta_ecs_cluster": model.LabelValue("ec2-awsvpc-cluster"), + "__meta_ecs_cluster_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:cluster/ec2-awsvpc-cluster"), + "__meta_ecs_service": model.LabelValue("ec2-awsvpc-service"), + "__meta_ecs_service_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:service/ec2-awsvpc-cluster/ec2-awsvpc-service"), + "__meta_ecs_service_status": model.LabelValue("ACTIVE"), + "__meta_ecs_task_group": model.LabelValue("service:ec2-awsvpc-service"), + "__meta_ecs_task_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task/ec2-awsvpc-cluster/task-ec2-awsvpc"), + "__meta_ecs_task_definition": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:task-definition/ec2-awsvpc-task:1"), + "__meta_ecs_region": model.LabelValue("us-west-2"), + "__meta_ecs_availability_zone": model.LabelValue("us-west-2c"), + "__meta_ecs_ip_address": model.LabelValue("10.0.3.200"), + "__meta_ecs_subnet_id": model.LabelValue("subnet-99999"), + "__meta_ecs_launch_type": model.LabelValue("EC2"), + "__meta_ecs_desired_status": model.LabelValue("RUNNING"), + "__meta_ecs_last_status": model.LabelValue("RUNNING"), + "__meta_ecs_health_status": model.LabelValue("HEALTHY"), + "__meta_ecs_network_mode": model.LabelValue("awsvpc"), + "__meta_ecs_container_instance_arn": model.LabelValue("arn:aws:ecs:us-west-2:123456789012:container-instance/ec2-awsvpc-cluster/def456"), + "__meta_ecs_ec2_instance_id": model.LabelValue("i-ec2awsvpcinstance"), + "__meta_ecs_ec2_instance_type": model.LabelValue("c5.2xlarge"), + "__meta_ecs_ec2_instance_private_ip": model.LabelValue("10.0.9.99"), + "__meta_ecs_ec2_instance_public_ip": model.LabelValue("54.3.4.5"), + "__meta_ecs_public_ip": model.LabelValue("52.3.4.5"), + "__meta_ecs_tag_ec2_Name": model.LabelValue("ec2-awsvpc-host"), + "__meta_ecs_tag_ec2_Owner": model.LabelValue("team-a"), + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ecsClient := newMockECSClient(tt.ecsData) + ec2Client := newMockECSEC2Client(tt.ecsData.ec2Instances, tt.ecsData.eniPublicIPs) + + d := &ECSDiscovery{ + ecs: ecsClient, + ec2: ec2Client, + cfg: &ECSSDConfig{ + Region: tt.ecsData.region, + Port: 80, + RequestConcurrency: 1, + }, + } + + groups, err := d.refresh(ctx) + require.NoError(t, err) + if tt.name == "MixedNetworkingModes" { + // Use ElementsMatch for tests with multiple tasks as goroutines can affect order + require.Len(t, groups, len(tt.expected)) + require.Equal(t, tt.expected[0].Source, groups[0].Source) + require.ElementsMatch(t, tt.expected[0].Targets, groups[0].Targets) + } else { + require.Equal(t, tt.expected, groups) + } + }) + } +} + +// ECS client mock. +type mockECSClient struct { + ecsData ecsDataStore +} + +func newMockECSClient(ecsData *ecsDataStore) *mockECSClient { + client := mockECSClient{ + ecsData: *ecsData, + } + return &client +} + +func (m *mockECSClient) ListClusters(_ context.Context, _ *ecs.ListClustersInput, _ ...func(*ecs.Options)) (*ecs.ListClustersOutput, error) { + clusterArns := make([]string, 0, len(m.ecsData.clusters)) + for _, cluster := range m.ecsData.clusters { + clusterArns = append(clusterArns, *cluster.ClusterArn) + } + + return &ecs.ListClustersOutput{ + ClusterArns: clusterArns, + }, nil +} + +func (m *mockECSClient) DescribeClusters(_ context.Context, input *ecs.DescribeClustersInput, _ ...func(*ecs.Options)) (*ecs.DescribeClustersOutput, error) { + var clusters []ecsTypes.Cluster + for _, clusterArn := range input.Clusters { + for _, cluster := range m.ecsData.clusters { + if *cluster.ClusterArn == clusterArn { + clusters = append(clusters, cluster) + break + } + } + } + + return &ecs.DescribeClustersOutput{ + Clusters: clusters, + }, nil +} + +func (m *mockECSClient) ListServices(_ context.Context, input *ecs.ListServicesInput, _ ...func(*ecs.Options)) (*ecs.ListServicesOutput, error) { + var serviceArns []string + for _, service := range m.ecsData.services { + if *service.ClusterArn == *input.Cluster { + serviceArns = append(serviceArns, *service.ServiceArn) + } + } + + return &ecs.ListServicesOutput{ + ServiceArns: serviceArns, + }, nil +} + +func (m *mockECSClient) DescribeServices(_ context.Context, input *ecs.DescribeServicesInput, _ ...func(*ecs.Options)) (*ecs.DescribeServicesOutput, error) { + var services []ecsTypes.Service + for _, serviceArn := range input.Services { + for _, service := range m.ecsData.services { + if *service.ServiceArn == serviceArn && *service.ClusterArn == *input.Cluster { + services = append(services, service) + break + } + } + } + + return &ecs.DescribeServicesOutput{ + Services: services, + }, nil +} + +func (m *mockECSClient) ListTasks(_ context.Context, input *ecs.ListTasksInput, _ ...func(*ecs.Options)) (*ecs.ListTasksOutput, error) { + var taskArns []string + for _, task := range m.ecsData.tasks { + if *task.ClusterArn == *input.Cluster { + // If ServiceName is specified, filter by service + if input.ServiceName != nil { + expectedGroup := "service:" + *input.ServiceName + if task.Group != nil && *task.Group == expectedGroup { + taskArns = append(taskArns, *task.TaskArn) + } + } else { + taskArns = append(taskArns, *task.TaskArn) + } + } + } + + return &ecs.ListTasksOutput{ + TaskArns: taskArns, + }, nil +} + +func (m *mockECSClient) DescribeTasks(_ context.Context, input *ecs.DescribeTasksInput, _ ...func(*ecs.Options)) (*ecs.DescribeTasksOutput, error) { + var tasks []ecsTypes.Task + for _, taskArn := range input.Tasks { + for _, task := range m.ecsData.tasks { + if *task.TaskArn == taskArn && *task.ClusterArn == *input.Cluster { + tasks = append(tasks, task) + break + } + } + } + + return &ecs.DescribeTasksOutput{ + Tasks: tasks, + }, nil +} + +func (m *mockECSClient) DescribeContainerInstances(_ context.Context, input *ecs.DescribeContainerInstancesInput, _ ...func(*ecs.Options)) (*ecs.DescribeContainerInstancesOutput, error) { + var containerInstances []ecsTypes.ContainerInstance + for _, ciArn := range input.ContainerInstances { + for _, ci := range m.ecsData.containerInstances { + if *ci.ContainerInstanceArn == ciArn { + containerInstances = append(containerInstances, ci) + break + } + } + } + + return &ecs.DescribeContainerInstancesOutput{ + ContainerInstances: containerInstances, + }, nil +} + +// Mock EC2 client wrapper for ECS tests. +type mockECSEC2Client struct { + ec2Instances map[string]ec2InstanceInfo + eniPublicIPs map[string]string +} + +func newMockECSEC2Client(ec2Instances map[string]ec2InstanceInfo, eniPublicIPs map[string]string) *mockECSEC2Client { + return &mockECSEC2Client{ + ec2Instances: ec2Instances, + eniPublicIPs: eniPublicIPs, + } +} + +func (m *mockECSEC2Client) DescribeInstances(_ context.Context, input *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { + var reservations []ec2Types.Reservation + + for _, instanceID := range input.InstanceIds { + if info, ok := m.ec2Instances[instanceID]; ok { + instance := ec2Types.Instance{ + InstanceId: &instanceID, + PrivateIpAddress: &info.privateIP, + } + if info.publicIP != "" { + instance.PublicIpAddress = &info.publicIP + } + if info.subnetID != "" { + instance.SubnetId = &info.subnetID + } + if info.instanceType != "" { + instance.InstanceType = ec2Types.InstanceType(info.instanceType) + } + // Add tags + for tagKey, tagValue := range info.tags { + instance.Tags = append(instance.Tags, ec2Types.Tag{ + Key: &tagKey, + Value: &tagValue, + }) + } + reservation := ec2Types.Reservation{ + Instances: []ec2Types.Instance{instance}, + } + reservations = append(reservations, reservation) + } + } + + return &ec2.DescribeInstancesOutput{ + Reservations: reservations, + }, nil +} + +func (m *mockECSEC2Client) DescribeNetworkInterfaces(_ context.Context, input *ec2.DescribeNetworkInterfacesInput, _ ...func(*ec2.Options)) (*ec2.DescribeNetworkInterfacesOutput, error) { + var networkInterfaces []ec2Types.NetworkInterface + + for _, eniID := range input.NetworkInterfaceIds { + if publicIP, ok := m.eniPublicIPs[eniID]; ok { + eni := ec2Types.NetworkInterface{ + NetworkInterfaceId: &eniID, + } + if publicIP != "" { + eni.Association = &ec2Types.NetworkInterfaceAssociation{ + PublicIp: &publicIP, + } + } + networkInterfaces = append(networkInterfaces, eni) + } + } + + return &ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: networkInterfaces, + }, nil +} + +func TestIsStandaloneTask(t *testing.T) { + t.Parallel() + tests := []struct { + name string + task ecsTypes.Task + expected bool + }{ + { + name: "StandaloneTask", + task: ecsTypes.Task{ + Group: strptr("family:my-task-definition"), + }, + expected: true, + }, + { + name: "ServiceTask", + task: ecsTypes.Task{ + Group: strptr("service:my-service"), + }, + expected: false, + }, + { + name: "ServiceTaskWithColon", + task: ecsTypes.Task{ + Group: strptr("service:my:service:name"), + }, + expected: false, + }, + { + name: "NilGroup", + task: ecsTypes.Task{ + Group: nil, + }, + expected: false, + }, + { + name: "EmptyGroup", + task: ecsTypes.Task{ + Group: strptr(""), + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isStandaloneTask(tt.task) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestGetServiceNameFromTaskGroup(t *testing.T) { + t.Parallel() + tests := []struct { + name string + task ecsTypes.Task + expected string + }{ + { + name: "SimpleServiceName", + task: ecsTypes.Task{ + Group: strptr("service:my-service"), + }, + expected: "my-service", + }, + { + name: "ServiceNameWithHyphens", + task: ecsTypes.Task{ + Group: strptr("service:web-api-service"), + }, + expected: "web-api-service", + }, + { + name: "ServiceNameWithColons", + task: ecsTypes.Task{ + Group: strptr("service:my:service:name"), + }, + expected: "my", + }, + { + name: "FamilyGroup", + task: ecsTypes.Task{ + Group: strptr("family:my-task-def"), + }, + expected: "my-task-def", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := getServiceNameFromTaskGroup(tt.task) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/discovery/aws/elasticache.go b/discovery/aws/elasticache.go new file mode 100644 index 00000000000..722a46085a0 --- /dev/null +++ b/discovery/aws/elasticache.go @@ -0,0 +1,948 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "net" + "strconv" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/elasticache" + "github.com/aws/aws-sdk-go-v2/service/elasticache/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "golang.org/x/sync/errgroup" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + elasticacheLabel = model.MetaLabelPrefix + "elasticache_" + elasticacheLabelDeploymentOption = elasticacheLabel + "deployment_option" + + // cache cluster. + elasticacheLabelCacheCluster = elasticacheLabel + "cache_cluster_" + elasticacheLabelCacheClusterARN = elasticacheLabelCacheCluster + "arn" + elasticacheLabelCacheClusterAtRestEncryptionEnabled = elasticacheLabelCacheCluster + "at_rest_encryption_enabled" + elasticacheLabelCacheClusterAuthTokenEnabled = elasticacheLabelCacheCluster + "auth_token_enabled" + elasticacheLabelCacheClusterAuthTokenLastModified = elasticacheLabelCacheCluster + "auth_token_last_modified" + elasticacheLabelCacheClusterAutoMinorVersionUpgrade = elasticacheLabelCacheCluster + "auto_minor_version_upgrade" + elasticacheLabelCacheClusterCreateTime = elasticacheLabelCacheCluster + "cache_cluster_create_time" + elasticacheLabelCacheClusterID = elasticacheLabelCacheCluster + "cache_cluster_id" + elasticacheLabelCacheClusterStatus = elasticacheLabelCacheCluster + "cache_cluster_status" + elasticacheLabelCacheClusterNodeType = elasticacheLabelCacheCluster + "cache_node_type" + elasticacheLabelCacheClusterParameterGroup = elasticacheLabelCacheCluster + "cache_parameter_group" + elasticacheLabelCacheClusterSubnetGroupName = elasticacheLabelCacheCluster + "cache_subnet_group_name" + elasticacheLabelCacheClusterClientDownloadLandingPage = elasticacheLabelCacheCluster + "client_download_landing_page" + elasticacheLabelCacheClusterEngine = elasticacheLabelCacheCluster + "engine" + elasticacheLabelCacheClusterEngineVersion = elasticacheLabelCacheCluster + "engine_version" + elasticacheLabelCacheClusterIPDiscovery = elasticacheLabelCacheCluster + "ip_discovery" + elasticacheLabelCacheClusterNetworkType = elasticacheLabelCacheCluster + "network_type" + elasticacheLabelCacheClusterNumCacheNodes = elasticacheLabelCacheCluster + "num_cache_nodes" + elasticacheLabelCacheClusterPreferredAvailabilityZone = elasticacheLabelCacheCluster + "preferred_availability_zone" + elasticacheLabelCacheClusterPreferredMaintenanceWindow = elasticacheLabelCacheCluster + "preferred_maintenance_window" + elasticacheLabelCacheClusterPreferredOutpostARN = elasticacheLabelCacheCluster + "preferred_outpost_arn" + elasticacheLabelCacheClusterReplicationGroupID = elasticacheLabelCacheCluster + "replication_group_id" + elasticacheLabelCacheClusterReplicationGroupLogDeliveryEnabled = elasticacheLabelCacheCluster + "replication_group_log_delivery_enabled" + elasticacheLabelCacheClusterSnapshotRetentionLimit = elasticacheLabelCacheCluster + "snapshot_retention_limit" + elasticacheLabelCacheClusterSnapshotWindow = elasticacheLabelCacheCluster + "snapshot_window" + elasticacheLabelCacheClusterTransitEncryptionEnabled = elasticacheLabelCacheCluster + "transit_encryption_enabled" + elasticacheLabelCacheClusterTransitEncryptionMode = elasticacheLabelCacheCluster + "transit_encryption_mode" + + // configuration endpoint. + elasticacheLabelCacheClusterConfigurationEndpoint = elasticacheLabelCacheCluster + "configuration_endpoint_" + elasticacheLabelCacheClusterConfigurationEndpointAddress = elasticacheLabelCacheClusterConfigurationEndpoint + "address" + elasticacheLabelCacheClusterConfigurationEndpointPort = elasticacheLabelCacheClusterConfigurationEndpoint + "port" + + // notification. + elasticacheLabelCacheClusterNotification = elasticacheLabelCacheCluster + "notification_" + elasticacheLabelCacheClusterNotificationTopicARN = elasticacheLabelCacheClusterNotification + "topic_arn" + elasticacheLabelCacheClusterNotificationTopicStatus = elasticacheLabelCacheClusterNotification + "topic_status" + + // log delivery configuration (slice - use with index). + elasticacheLabelCacheClusterLogDeliveryConfiguration = elasticacheLabelCacheCluster + "log_delivery_configuration_" + elasticacheLabelCacheClusterLogDeliveryConfigurationDestinationType = elasticacheLabelCacheClusterLogDeliveryConfiguration + "destination_type" + elasticacheLabelCacheClusterLogDeliveryConfigurationLogFormat = elasticacheLabelCacheClusterLogDeliveryConfiguration + "log_format" + elasticacheLabelCacheClusterLogDeliveryConfigurationLogType = elasticacheLabelCacheClusterLogDeliveryConfiguration + "log_type" + elasticacheLabelCacheClusterLogDeliveryConfigurationStatus = elasticacheLabelCacheClusterLogDeliveryConfiguration + "status" + elasticacheLabelCacheClusterLogDeliveryConfigurationMessage = elasticacheLabelCacheClusterLogDeliveryConfiguration + "message" + elasticacheLabelCacheClusterLogDeliveryConfigurationLogGroup = elasticacheLabelCacheClusterLogDeliveryConfiguration + "log_group" + elasticacheLabelCacheClusterLogDeliveryConfigurationDeliveryStream = elasticacheLabelCacheClusterLogDeliveryConfiguration + "delivery_stream" + + // pending modified values. + elasticacheLabelCacheClusterPendingModifiedValues = elasticacheLabelCacheCluster + "pending_modified_values_" + elasticacheLabelCacheClusterPendingModifiedValuesAuthTokenStatus = elasticacheLabelCacheClusterPendingModifiedValues + "auth_token_status" + elasticacheLabelCacheClusterPendingModifiedValuesCacheNodeType = elasticacheLabelCacheClusterPendingModifiedValues + "cache_node_type" + elasticacheLabelCacheClusterPendingModifiedValuesEngineVersion = elasticacheLabelCacheClusterPendingModifiedValues + "engine_version" + elasticacheLabelCacheClusterPendingModifiedValuesNumCacheNodes = elasticacheLabelCacheClusterPendingModifiedValues + "num_cache_nodes" + elasticacheLabelCacheClusterPendingModifiedValuesTransitEncryptionEnabled = elasticacheLabelCacheClusterPendingModifiedValues + "transit_encryption_enabled" + elasticacheLabelCacheClusterPendingModifiedValuesTransitEncryptionMode = elasticacheLabelCacheClusterPendingModifiedValues + "transit_encryption_mode" + elasticacheLabelCacheClusterPendingModifiedValuesCacheNodeIDsToRemove = elasticacheLabelCacheClusterPendingModifiedValues + "cache_node_ids_to_remove" + + // security group membership (slice - use with index). + elasticacheLabelCacheClusterSecurityGroupMembership = elasticacheLabelCacheCluster + "security_group_membership_" + elasticacheLabelCacheClusterSecurityGroupMembershipID = elasticacheLabelCacheClusterSecurityGroupMembership + "id" + elasticacheLabelCacheClusterSecurityGroupMembershipStatus = elasticacheLabelCacheClusterSecurityGroupMembership + "status" + + // tags - create one label per tag key, with the format: elasticache_cache_cluster_tag_. + elasticacheLabelCacheClusterTag = elasticacheLabelCacheCluster + "tag_" + + // node. + elasticacheLabelCacheClusterNode = elasticacheLabelCacheCluster + "node_" + elasticacheLabelCacheClusterNodeCreateTime = elasticacheLabelCacheClusterNode + "create_time" + elasticacheLabelCacheClusterNodeID = elasticacheLabelCacheClusterNode + "id" + elasticacheLabelCacheClusterNodeStatus = elasticacheLabelCacheClusterNode + "status" + elasticacheLabelCacheClusterNodeAZ = elasticacheLabelCacheClusterNode + "availability_zone" + elasticacheLabelCacheClusterNodeCustomerOutpostARN = elasticacheLabelCacheClusterNode + "customer_outpost_arn" + elasticacheLabelCacheClusterNodeSourceCacheNodeID = elasticacheLabelCacheClusterNode + "source_cache_node_id" + elasticacheLabelCacheClusterNodeParameterGroupStatus = elasticacheLabelCacheClusterNode + "parameter_group_status" + + // endpoint. + elasticacheLabelCacheClusterNodeEndpoint = elasticacheLabelCacheClusterNode + "endpoint_" + elasticacheLabelCacheClusterNodeEndpointAddress = elasticacheLabelCacheClusterNodeEndpoint + "address" + elasticacheLabelCacheClusterNodeEndpointPort = elasticacheLabelCacheClusterNodeEndpoint + "port" + + // serverless cache. + elasticacheLabelServerlessCache = elasticacheLabel + "serverless_cache_" + elasticacheLabelServerlessCacheARN = elasticacheLabelServerlessCache + "arn" + elasticacheLabelServerlessCacheName = elasticacheLabelServerlessCache + "name" + elasticacheLabelServerlessCacheCreateTime = elasticacheLabelServerlessCache + "create_time" + elasticacheLabelServerlessCacheDescription = elasticacheLabelServerlessCache + "description" + elasticacheLabelServerlessCacheEngine = elasticacheLabelServerlessCache + "engine" + elasticacheLabelServerlessCacheFullEngineVersion = elasticacheLabelServerlessCache + "full_engine_version" + elasticacheLabelServerlessCacheMajorEngineVersion = elasticacheLabelServerlessCache + "major_engine_version" + elasticacheLabelServerlessCacheStatus = elasticacheLabelServerlessCache + "status" + elasticacheLabelServerlessCacheKmsKeyID = elasticacheLabelServerlessCache + "kms_key_id" + elasticacheLabelServerlessCacheUserGroupID = elasticacheLabelServerlessCache + "user_group_id" + elasticacheLabelServerlessCacheDailySnapshotTime = elasticacheLabelServerlessCache + "daily_snapshot_time" + elasticacheLabelServerlessCacheSnapshotRetentionLimit = elasticacheLabelServerlessCache + "snapshot_retention_limit" + + // endpoint. + elasticacheLabelServerlessCacheEndpoint = elasticacheLabelServerlessCache + "endpoint_" + elasticacheLabelServerlessCacheEndpointAddress = elasticacheLabelServerlessCacheEndpoint + "address" + elasticacheLabelServerlessCacheEndpointPort = elasticacheLabelServerlessCacheEndpoint + "port" + elasticacheLabelServerlessCacheReaderEndpointAddress = elasticacheLabelServerlessCacheEndpoint + "reader_address" + elasticacheLabelServerlessCacheReaderEndpointPort = elasticacheLabelServerlessCacheEndpoint + "reader_port" + + // security group membership (slice - use with index). + elasticacheLabelServerlessCacheSecurityGroupID = elasticacheLabelServerlessCache + "security_group_id" + + // Subnet group membership (slice - use with index). + elasticacheLabelServerlessCacheSubnetID = elasticacheLabelServerlessCache + "subnet_id" + + // cache usage limits. + elasticacheLabelServerlessCacheCacheUsageLimit = elasticacheLabelServerlessCache + "cache_usage_limit_" + elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorage = elasticacheLabelServerlessCacheCacheUsageLimit + "data_storage" + elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorageMaximum = elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorage + "maximum" + elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorageMinimum = elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorage + "minimum" + elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorageUnit = elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorage + "unit" + elasticacheLabelServerlessCacheCacheUsageLimitECPUPerSecond = elasticacheLabelServerlessCacheCacheUsageLimit + "ecpu_per_second" + elasticacheLabelServerlessCacheCacheUsageLimitECPUPerSecondMaximum = elasticacheLabelServerlessCacheCacheUsageLimitECPUPerSecond + "maximum" + elasticacheLabelServerlessCacheCacheUsageLimitECPUPerSecondMinimum = elasticacheLabelServerlessCacheCacheUsageLimitECPUPerSecond + "minimum" + + // tags - create one label per tag key, with the format: elasticache_serverless_cache_tag_. + elasticacheLabelServerlessCacheTag = elasticacheLabelServerlessCache + "tag_" +) + +// DefaultElasticacheSDConfig is the default Elasticache SD configuration. +var DefaultElasticacheSDConfig = ElasticacheSDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + RequestConcurrency: 10, + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +func init() { + discovery.RegisterConfig(&ElasticacheSDConfig{}) +} + +// ElasticacheSDConfig is the configuration for Elasticache based service discovery. +type ElasticacheSDConfig struct { + Region string `yaml:"region"` + Endpoint string `yaml:"endpoint"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + ExternalID string `yaml:"external_id,omitempty"` + Clusters []string `yaml:"clusters,omitempty"` + Port int `yaml:"port"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + + // RequestConcurrency controls the maximum number of concurrent Elasticache API requests. + RequestConcurrency int `yaml:"request_concurrency,omitempty"` + + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` +} + +// NewDiscovererMetrics implements discovery.Config. +func (*ElasticacheSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &elasticacheMetrics{ + refreshMetrics: rmi, + } +} + +// Name returns the name of the Elasticache Config. +func (*ElasticacheSDConfig) Name() string { return "elasticache" } + +// NewDiscoverer returns a Discoverer for the Elasticache Config. +func (c *ElasticacheSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewElasticacheDiscovery(c, opts) +} + +// SetDirectory joins any relative file paths with dir. +func (c *ElasticacheSDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for the Elasticache Config. +func (c *ElasticacheSDConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultElasticacheSDConfig + type plain ElasticacheSDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + + c.Region, err = loadRegion(context.Background(), c.Region) + if err != nil { + return fmt.Errorf("could not determine AWS region: %w", err) + } + + return c.HTTPClientConfig.Validate() +} + +type elasticacheClient interface { + DescribeServerlessCaches(ctx context.Context, params *elasticache.DescribeServerlessCachesInput, optFns ...func(*elasticache.Options)) (*elasticache.DescribeServerlessCachesOutput, error) + DescribeCacheClusters(ctx context.Context, params *elasticache.DescribeCacheClustersInput, optFns ...func(*elasticache.Options)) (*elasticache.DescribeCacheClustersOutput, error) + ListTagsForResource(ctx context.Context, params *elasticache.ListTagsForResourceInput, optFns ...func(*elasticache.Options)) (*elasticache.ListTagsForResourceOutput, error) +} + +// elasticacheClientAdapter captures only the ElastiCache API calls AWS +// discovery uses as method-value closures, keeping the concrete +// *elasticache.Client out of any interface-boxed struct field. See +// ec2ClientAdapter for the full rationale: this stops the linker from retaining +// the entire ElastiCache API surface (~2.5 MB). +type elasticacheClientAdapter struct { + describeServerlessCaches func(ctx context.Context, params *elasticache.DescribeServerlessCachesInput, optFns ...func(*elasticache.Options)) (*elasticache.DescribeServerlessCachesOutput, error) + describeCacheClusters func(ctx context.Context, params *elasticache.DescribeCacheClustersInput, optFns ...func(*elasticache.Options)) (*elasticache.DescribeCacheClustersOutput, error) + listTagsForResource func(ctx context.Context, params *elasticache.ListTagsForResourceInput, optFns ...func(*elasticache.Options)) (*elasticache.ListTagsForResourceOutput, error) +} + +func newElastiCacheClientAdapter(c *elasticache.Client) elasticacheClientAdapter { + return elasticacheClientAdapter{ + describeServerlessCaches: c.DescribeServerlessCaches, + describeCacheClusters: c.DescribeCacheClusters, + listTagsForResource: c.ListTagsForResource, + } +} + +func (a elasticacheClientAdapter) DescribeServerlessCaches(ctx context.Context, params *elasticache.DescribeServerlessCachesInput, optFns ...func(*elasticache.Options)) (*elasticache.DescribeServerlessCachesOutput, error) { + return a.describeServerlessCaches(ctx, params, optFns...) +} + +func (a elasticacheClientAdapter) DescribeCacheClusters(ctx context.Context, params *elasticache.DescribeCacheClustersInput, optFns ...func(*elasticache.Options)) (*elasticache.DescribeCacheClustersOutput, error) { + return a.describeCacheClusters(ctx, params, optFns...) +} + +func (a elasticacheClientAdapter) ListTagsForResource(ctx context.Context, params *elasticache.ListTagsForResourceInput, optFns ...func(*elasticache.Options)) (*elasticache.ListTagsForResourceOutput, error) { + return a.listTagsForResource(ctx, params, optFns...) +} + +// ElasticacheDiscovery periodically performs Elasticache-SD requests. +// It implements the Discoverer interface. +type ElasticacheDiscovery struct { + *refresh.Discovery + logger *slog.Logger + cfg *ElasticacheSDConfig + elasticacheClient elasticacheClient +} + +// NewElasticacheDiscovery returns a new ElasticacheDiscovery which periodically refreshes its targets. +func NewElasticacheDiscovery(conf *ElasticacheSDConfig, opts discovery.DiscovererOptions) (*ElasticacheDiscovery, error) { + m, ok := opts.Metrics.(*elasticacheMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type") + } + + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() + } + d := &ElasticacheDiscovery{ + logger: opts.Logger, + cfg: conf, + } + d.Discovery = refresh.NewDiscovery( + refresh.Options{ + Logger: opts.Logger, + Mech: "elasticache", + Interval: time.Duration(d.cfg.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ) + return d, nil +} + +func (d *ElasticacheDiscovery) initElasticacheClient(ctx context.Context) error { + if d.elasticacheClient != nil { + return nil + } + + if d.cfg.Region == "" { + return errors.New("region must be set for Elasticache service discovery") + } + + // Build the HTTP client from the provided HTTPClientConfig. + client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "elasticache_sd") + if err != nil { + return err + } + + // Build the AWS config with the provided region. + var configOptions []func(*awsConfig.LoadOptions) error + configOptions = append(configOptions, awsConfig.WithRegion(d.cfg.Region)) + configOptions = append(configOptions, awsConfig.WithHTTPClient(client)) + + // Only set static credentials if both access key and secret key are provided + // Otherwise, let AWS SDK use its default credential chain + if d.cfg.AccessKey != "" && d.cfg.SecretKey != "" { + credProvider := credentials.NewStaticCredentialsProvider(d.cfg.AccessKey, string(d.cfg.SecretKey), "") + configOptions = append(configOptions, awsConfig.WithCredentialsProvider(credProvider)) + } + + if d.cfg.Profile != "" { + configOptions = append(configOptions, awsConfig.WithSharedConfigProfile(d.cfg.Profile)) + } + + cfg, err := awsConfig.LoadDefaultConfig(ctx, configOptions...) + if err != nil { + d.logger.Error("Failed to create AWS config", "error", err) + return fmt.Errorf("could not create aws config: %w", err) + } + + // If the role ARN is set, assume the role to get credentials and set the credentials provider in the config. + if d.cfg.RoleARN != "" { + assumeProvider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), d.cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { + if d.cfg.ExternalID != "" { + o.ExternalID = aws.String(d.cfg.ExternalID) + } + }) + cfg.Credentials = aws.NewCredentialsCache(assumeProvider) + } + + d.elasticacheClient = newElastiCacheClientAdapter(elasticache.NewFromConfig(cfg, func(options *elasticache.Options) { + if d.cfg.Endpoint != "" { + options.BaseEndpoint = &d.cfg.Endpoint + } + options.HTTPClient = client + })) + + // Test credentials by making a simple API call + testCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + _, err = d.elasticacheClient.DescribeCacheClusters(testCtx, &elasticache.DescribeCacheClustersInput{}) + if err != nil { + d.logger.Error("Failed to test Elasticache credentials", "error", err) + return fmt.Errorf("elasticache credential test failed: %w", err) + } + + return nil +} + +// describeServerlessCaches calls DescribeServerlessCaches API for the given cache IDs (or all caches if no IDs are provided) and returns the list of serverless caches. +func (d *ElasticacheDiscovery) describeServerlessCaches(ctx context.Context, caches []string) ([]types.ServerlessCache, error) { + mu := &sync.Mutex{} + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + var serverlessCaches []types.ServerlessCache + if len(caches) == 0 { + errg.Go(func() error { + var nextToken *string + for { + output, err := d.elasticacheClient.DescribeServerlessCaches(ectx, &elasticache.DescribeServerlessCachesInput{ + MaxResults: aws.Int32(50), + NextToken: nextToken, + }) + if err != nil { + return fmt.Errorf("failed to describe serverless caches: %w", err) + } + mu.Lock() + serverlessCaches = append(serverlessCaches, output.ServerlessCaches...) + mu.Unlock() + if output.NextToken == nil { + break + } + nextToken = output.NextToken + } + return nil + }) + } else { + for _, cacheID := range caches { + errg.Go(func() error { + output, err := d.elasticacheClient.DescribeServerlessCaches(ectx, &elasticache.DescribeServerlessCachesInput{ + MaxResults: aws.Int32(50), + NextToken: nil, + ServerlessCacheName: aws.String(cacheID), + }) + if err != nil { + return fmt.Errorf("failed to describe serverless cache %s: %w", cacheID, err) + } + mu.Lock() + serverlessCaches = append(serverlessCaches, output.ServerlessCaches...) + mu.Unlock() + return nil + }) + } + } + + return serverlessCaches, errg.Wait() +} + +// describeCacheClusters calls DescribeCacheClusters API for the given cache cluster IDs (or all cache clusters if no IDs are provided) and returns the list of cache clusters. +func (d *ElasticacheDiscovery) describeCacheClusters(ctx context.Context, caches []string) ([]types.CacheCluster, error) { + mu := &sync.Mutex{} + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + showCacheClustersNotInReplicationGroupsBools := []bool{false, true} + var cacheClusters []types.CacheCluster + if len(caches) == 0 { + for _, showCacheClustersNotInReplicationGroupsBool := range showCacheClustersNotInReplicationGroupsBools { + errg.Go(func() error { + var nextToken *string + for { + output, err := d.elasticacheClient.DescribeCacheClusters(ectx, &elasticache.DescribeCacheClustersInput{ + MaxRecords: aws.Int32(100), + Marker: nextToken, + ShowCacheNodeInfo: aws.Bool(true), + ShowCacheClustersNotInReplicationGroups: aws.Bool(showCacheClustersNotInReplicationGroupsBool), + }) + if err != nil { + return fmt.Errorf("failed to describe cache clusters: %w", err) + } + mu.Lock() + cacheClusters = append(cacheClusters, output.CacheClusters...) + mu.Unlock() + if output.Marker == nil { + break + } + nextToken = output.Marker + } + return nil + }) + } + } else { + for _, cacheID := range caches { + for _, showCacheClustersNotInReplicationGroupsBool := range showCacheClustersNotInReplicationGroupsBools { + errg.Go(func() error { + output, err := d.elasticacheClient.DescribeCacheClusters(ectx, &elasticache.DescribeCacheClustersInput{ + MaxRecords: aws.Int32(100), + Marker: nil, + ShowCacheNodeInfo: aws.Bool(true), + ShowCacheClustersNotInReplicationGroups: aws.Bool(showCacheClustersNotInReplicationGroupsBool), + CacheClusterId: aws.String(cacheID), + }) + if err != nil { + return fmt.Errorf("failed to describe cache cluster %s: %w", cacheID, err) + } + mu.Lock() + cacheClusters = append(cacheClusters, output.CacheClusters...) + mu.Unlock() + return nil + }) + } + } + } + + return cacheClusters, errg.Wait() +} + +// listTagsForResource calls ListTagsForResource API for the given resource ARNs and returns a map of resource ARN to list of tags. +func (d *ElasticacheDiscovery) listTagsForResource(ctx context.Context, resourceARNs []string) (map[string][]types.Tag, error) { + mu := &sync.Mutex{} + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + tagsByResourceARN := make(map[string][]types.Tag) + for _, resourceARN := range resourceARNs { + errg.Go(func() error { + output, err := d.elasticacheClient.ListTagsForResource(ectx, &elasticache.ListTagsForResourceInput{ + ResourceName: aws.String(resourceARN), + }) + if err != nil { + return fmt.Errorf("failed to list tags for resource %s: %w", resourceARN, err) + } + mu.Lock() + tagsByResourceARN[resourceARN] = output.TagList + mu.Unlock() + return nil + }) + } + return tagsByResourceARN, errg.Wait() +} + +func (d *ElasticacheDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + err := d.initElasticacheClient(ctx) + if err != nil { + return nil, err + } + + var clusters []string + clustersMu := sync.Mutex{} + serverlessCacheIDs, cacheClusterIDs := splitCacheDeploymentOptions(d.cfg.Clusters) + + clusterErrg, clusterCtx := errgroup.WithContext(ctx) + clusterErrg.Go(func() error { + caches, err := d.describeServerlessCaches(clusterCtx, serverlessCacheIDs) + if err != nil { + return fmt.Errorf("failed to describe serverless caches: %w", err) + } + for _, cache := range caches { + clustersMu.Lock() + clusters = append(clusters, *cache.ARN) + clustersMu.Unlock() + } + return nil + }) + + clusterErrg.Go(func() error { + cacheClusters, err := d.describeCacheClusters(clusterCtx, cacheClusterIDs) + if err != nil { + return fmt.Errorf("failed to describe cache clusters: %w", err) + } + for _, cluster := range cacheClusters { + clustersMu.Lock() + clusters = append(clusters, *cluster.ARN) + clustersMu.Unlock() + } + return nil + }) + + if err := clusterErrg.Wait(); err != nil { + return nil, err + } + + tagsByResourceARN, err := d.listTagsForResource(ctx, clusters) + if err != nil { + return nil, fmt.Errorf("failed to list tags for resources: %w", err) + } + + tg := &targetgroup.Group{ + Source: d.cfg.Region, + } + + errg, ectx := errgroup.WithContext(ctx) + errg.Go(func() error { + caches, err := d.describeServerlessCaches(ectx, serverlessCacheIDs) + if err != nil { + return fmt.Errorf("failed to describe serverless caches: %w", err) + } + for _, cache := range caches { + addServerlessCacheTargets(tg, &cache, tagsByResourceARN[*cache.ARN]) + } + return nil + }) + + errg.Go(func() error { + cacheClusters, err := d.describeCacheClusters(ectx, cacheClusterIDs) + if err != nil { + return fmt.Errorf("failed to describe cache clusters: %w", err) + } + for _, cluster := range cacheClusters { + addCacheClusterTargets(tg, &cluster, tagsByResourceARN[*cluster.ARN]) + } + return nil + }) + + if err := errg.Wait(); err != nil { + return nil, err + } + + return []*targetgroup.Group{tg}, nil +} + +// splitCacheTypes takes a list of cache ARNs and splits them into serverless cache IDs and cache cluster IDs based on their format. +// Serverless caches are in the format arn:aws:elasticache:::serverlesscache: +// Cache clusters are in the format arn:aws:elasticache:::replicationgroup:. +func splitCacheDeploymentOptions(caches []string) (serverlessCacheIDs, cacheClusterIDs []string) { + for _, cacheARN := range caches { + if cacheARN == "" { + continue + } + parts := strings.Split(cacheARN, ":") + if len(parts) < 6 { + continue + } + resourceType := parts[5] + resourceID := parts[6] + switch resourceType { + case "serverlesscache": + serverlessCacheIDs = append(serverlessCacheIDs, resourceID) + case "replicationgroup": + cacheClusterIDs = append(cacheClusterIDs, resourceID) + default: + continue + } + } + return serverlessCacheIDs, cacheClusterIDs +} + +// addServerlessCacheTargets adds targets for a serverless cache to the target group. +func addServerlessCacheTargets(tg *targetgroup.Group, cache *types.ServerlessCache, tags []types.Tag) { + labels := model.LabelSet{ + elasticacheLabelDeploymentOption: model.LabelValue("serverless"), + elasticacheLabelServerlessCacheARN: model.LabelValue(*cache.ARN), + elasticacheLabelServerlessCacheName: model.LabelValue(*cache.ServerlessCacheName), + elasticacheLabelServerlessCacheStatus: model.LabelValue(*cache.Status), + elasticacheLabelServerlessCacheEngine: model.LabelValue(*cache.Engine), + elasticacheLabelServerlessCacheFullEngineVersion: model.LabelValue(*cache.FullEngineVersion), + elasticacheLabelServerlessCacheMajorEngineVersion: model.LabelValue(*cache.MajorEngineVersion), + } + + if cache.Description != nil { + labels[elasticacheLabelServerlessCacheDescription] = model.LabelValue(*cache.Description) + } + + if cache.CreateTime != nil { + labels[elasticacheLabelServerlessCacheCreateTime] = model.LabelValue(cache.CreateTime.Format(time.RFC3339)) + } + + if cache.KmsKeyId != nil { + labels[elasticacheLabelServerlessCacheKmsKeyID] = model.LabelValue(*cache.KmsKeyId) + } + + if cache.UserGroupId != nil { + labels[elasticacheLabelServerlessCacheUserGroupID] = model.LabelValue(*cache.UserGroupId) + } + + if cache.DailySnapshotTime != nil { + labels[elasticacheLabelServerlessCacheDailySnapshotTime] = model.LabelValue(*cache.DailySnapshotTime) + } + + if cache.SnapshotRetentionLimit != nil { + labels[elasticacheLabelServerlessCacheSnapshotRetentionLimit] = model.LabelValue(strconv.Itoa(int(*cache.SnapshotRetentionLimit))) + } + + if cache.Endpoint != nil { + if cache.Endpoint.Address != nil { + labels[elasticacheLabelServerlessCacheEndpointAddress] = model.LabelValue(*cache.Endpoint.Address) + } + if cache.Endpoint.Port != nil { + labels[elasticacheLabelServerlessCacheEndpointPort] = model.LabelValue(strconv.Itoa(int(*cache.Endpoint.Port))) + } + } + + if cache.ReaderEndpoint != nil { + if cache.ReaderEndpoint.Address != nil { + labels[elasticacheLabelServerlessCacheReaderEndpointAddress] = model.LabelValue(*cache.ReaderEndpoint.Address) + } + if cache.ReaderEndpoint.Port != nil { + labels[elasticacheLabelServerlessCacheReaderEndpointPort] = model.LabelValue(strconv.Itoa(int(*cache.ReaderEndpoint.Port))) + } + } + + for i, sgID := range cache.SecurityGroupIds { + labels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelServerlessCacheSecurityGroupID, i))] = model.LabelValue(sgID) + } + + for i, subnetID := range cache.SubnetIds { + labels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelServerlessCacheSubnetID, i))] = model.LabelValue(subnetID) + } + + if cache.CacheUsageLimits != nil { + if cache.CacheUsageLimits.DataStorage != nil { + if cache.CacheUsageLimits.DataStorage.Maximum != nil { + labels[elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorageMaximum] = model.LabelValue(strconv.Itoa(int(*cache.CacheUsageLimits.DataStorage.Maximum))) + } + if cache.CacheUsageLimits.DataStorage.Minimum != nil { + labels[elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorageMinimum] = model.LabelValue(strconv.Itoa(int(*cache.CacheUsageLimits.DataStorage.Minimum))) + } + labels[elasticacheLabelServerlessCacheCacheUsageLimitCacheDataStorageUnit] = model.LabelValue(cache.CacheUsageLimits.DataStorage.Unit) + } + if cache.CacheUsageLimits.ECPUPerSecond != nil { + if cache.CacheUsageLimits.ECPUPerSecond.Maximum != nil { + labels[elasticacheLabelServerlessCacheCacheUsageLimitECPUPerSecondMaximum] = model.LabelValue(strconv.Itoa(int(*cache.CacheUsageLimits.ECPUPerSecond.Maximum))) + } + if cache.CacheUsageLimits.ECPUPerSecond.Minimum != nil { + labels[elasticacheLabelServerlessCacheCacheUsageLimitECPUPerSecondMinimum] = model.LabelValue(strconv.Itoa(int(*cache.CacheUsageLimits.ECPUPerSecond.Minimum))) + } + } + } + + for _, tag := range tags { + if tag.Key != nil && tag.Value != nil { + labels[model.LabelName(elasticacheLabelServerlessCacheTag+strutil.SanitizeLabelName(*tag.Key))] = model.LabelValue(*tag.Value) + } + } + + // Set the address label using the endpoint + if cache.Endpoint != nil && cache.Endpoint.Address != nil && cache.Endpoint.Port != nil { + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(*cache.Endpoint.Address, strconv.Itoa(int(*cache.Endpoint.Port)))) + } + + tg.Targets = append(tg.Targets, labels) +} + +// addCacheClusterTargets adds targets for a cache cluster to the target group. +// Creates one target per cache node for individual scraping. +func addCacheClusterTargets(tg *targetgroup.Group, cluster *types.CacheCluster, tags []types.Tag) { + // Build common labels that apply to all nodes in this cluster + commonLabels := model.LabelSet{ + elasticacheLabelDeploymentOption: model.LabelValue("node"), + elasticacheLabelCacheClusterARN: model.LabelValue(*cluster.ARN), + elasticacheLabelCacheClusterID: model.LabelValue(*cluster.CacheClusterId), + elasticacheLabelCacheClusterStatus: model.LabelValue(*cluster.CacheClusterStatus), + } + + if cluster.AtRestEncryptionEnabled != nil { + commonLabels[elasticacheLabelCacheClusterAtRestEncryptionEnabled] = model.LabelValue(strconv.FormatBool(*cluster.AtRestEncryptionEnabled)) + } + + if cluster.AuthTokenEnabled != nil { + commonLabels[elasticacheLabelCacheClusterAuthTokenEnabled] = model.LabelValue(strconv.FormatBool(*cluster.AuthTokenEnabled)) + } + + if cluster.AuthTokenLastModifiedDate != nil { + commonLabels[elasticacheLabelCacheClusterAuthTokenLastModified] = model.LabelValue(cluster.AuthTokenLastModifiedDate.Format(time.RFC3339)) + } + + if cluster.AutoMinorVersionUpgrade != nil { + commonLabels[elasticacheLabelCacheClusterAutoMinorVersionUpgrade] = model.LabelValue(strconv.FormatBool(*cluster.AutoMinorVersionUpgrade)) + } + + if cluster.CacheClusterCreateTime != nil { + commonLabels[elasticacheLabelCacheClusterCreateTime] = model.LabelValue(cluster.CacheClusterCreateTime.Format(time.RFC3339)) + } + + if cluster.CacheNodeType != nil { + commonLabels[elasticacheLabelCacheClusterNodeType] = model.LabelValue(*cluster.CacheNodeType) + } + + if cluster.CacheParameterGroup != nil && cluster.CacheParameterGroup.CacheParameterGroupName != nil { + commonLabels[elasticacheLabelCacheClusterParameterGroup] = model.LabelValue(*cluster.CacheParameterGroup.CacheParameterGroupName) + } + + if cluster.CacheSubnetGroupName != nil { + commonLabels[elasticacheLabelCacheClusterSubnetGroupName] = model.LabelValue(*cluster.CacheSubnetGroupName) + } + + if cluster.ClientDownloadLandingPage != nil { + commonLabels[elasticacheLabelCacheClusterClientDownloadLandingPage] = model.LabelValue(*cluster.ClientDownloadLandingPage) + } + + if cluster.ConfigurationEndpoint != nil { + if cluster.ConfigurationEndpoint.Address != nil { + commonLabels[elasticacheLabelCacheClusterConfigurationEndpointAddress] = model.LabelValue(*cluster.ConfigurationEndpoint.Address) + } + if cluster.ConfigurationEndpoint.Port != nil { + commonLabels[elasticacheLabelCacheClusterConfigurationEndpointPort] = model.LabelValue(strconv.Itoa(int(*cluster.ConfigurationEndpoint.Port))) + } + } + + if cluster.Engine != nil { + commonLabels[elasticacheLabelCacheClusterEngine] = model.LabelValue(*cluster.Engine) + } + + if cluster.EngineVersion != nil { + commonLabels[elasticacheLabelCacheClusterEngineVersion] = model.LabelValue(*cluster.EngineVersion) + } + + if len(cluster.IpDiscovery) > 0 { + commonLabels[elasticacheLabelCacheClusterIPDiscovery] = model.LabelValue(cluster.IpDiscovery) + } + + if len(cluster.NetworkType) > 0 { + commonLabels[elasticacheLabelCacheClusterNetworkType] = model.LabelValue(cluster.NetworkType) + } + + if cluster.NotificationConfiguration != nil { + if cluster.NotificationConfiguration.TopicArn != nil { + commonLabels[elasticacheLabelCacheClusterNotificationTopicARN] = model.LabelValue(*cluster.NotificationConfiguration.TopicArn) + } + if cluster.NotificationConfiguration.TopicStatus != nil { + commonLabels[elasticacheLabelCacheClusterNotificationTopicStatus] = model.LabelValue(*cluster.NotificationConfiguration.TopicStatus) + } + } + + if cluster.NumCacheNodes != nil { + commonLabels[elasticacheLabelCacheClusterNumCacheNodes] = model.LabelValue(strconv.Itoa(int(*cluster.NumCacheNodes))) + } + + if cluster.PreferredAvailabilityZone != nil { + commonLabels[elasticacheLabelCacheClusterPreferredAvailabilityZone] = model.LabelValue(*cluster.PreferredAvailabilityZone) + } + + if cluster.PreferredMaintenanceWindow != nil { + commonLabels[elasticacheLabelCacheClusterPreferredMaintenanceWindow] = model.LabelValue(*cluster.PreferredMaintenanceWindow) + } + + if cluster.PreferredOutpostArn != nil { + commonLabels[elasticacheLabelCacheClusterPreferredOutpostARN] = model.LabelValue(*cluster.PreferredOutpostArn) + } + + if cluster.ReplicationGroupId != nil { + commonLabels[elasticacheLabelCacheClusterReplicationGroupID] = model.LabelValue(*cluster.ReplicationGroupId) + } + + if cluster.ReplicationGroupLogDeliveryEnabled != nil { + commonLabels[elasticacheLabelCacheClusterReplicationGroupLogDeliveryEnabled] = model.LabelValue(strconv.FormatBool(*cluster.ReplicationGroupLogDeliveryEnabled)) + } + + if cluster.SnapshotRetentionLimit != nil { + commonLabels[elasticacheLabelCacheClusterSnapshotRetentionLimit] = model.LabelValue(strconv.Itoa(int(*cluster.SnapshotRetentionLimit))) + } + + if cluster.SnapshotWindow != nil { + commonLabels[elasticacheLabelCacheClusterSnapshotWindow] = model.LabelValue(*cluster.SnapshotWindow) + } + + if cluster.TransitEncryptionEnabled != nil { + commonLabels[elasticacheLabelCacheClusterTransitEncryptionEnabled] = model.LabelValue(strconv.FormatBool(*cluster.TransitEncryptionEnabled)) + } + + if len(cluster.TransitEncryptionMode) > 0 { + commonLabels[elasticacheLabelCacheClusterTransitEncryptionMode] = model.LabelValue(cluster.TransitEncryptionMode) + } + + // Log delivery configurations (slice) + for i, logDelivery := range cluster.LogDeliveryConfigurations { + if len(logDelivery.DestinationType) > 0 { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterLogDeliveryConfigurationDestinationType, i))] = model.LabelValue(logDelivery.DestinationType) + } + if len(logDelivery.LogFormat) > 0 { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterLogDeliveryConfigurationLogFormat, i))] = model.LabelValue(logDelivery.LogFormat) + } + if len(logDelivery.LogType) > 0 { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterLogDeliveryConfigurationLogType, i))] = model.LabelValue(logDelivery.LogType) + } + if len(logDelivery.Status) > 0 { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterLogDeliveryConfigurationStatus, i))] = model.LabelValue(logDelivery.Status) + } + if logDelivery.Message != nil { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterLogDeliveryConfigurationMessage, i))] = model.LabelValue(*logDelivery.Message) + } + if logDelivery.DestinationDetails != nil { + if logDelivery.DestinationDetails.CloudWatchLogsDetails != nil && logDelivery.DestinationDetails.CloudWatchLogsDetails.LogGroup != nil { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterLogDeliveryConfigurationLogGroup, i))] = model.LabelValue(*logDelivery.DestinationDetails.CloudWatchLogsDetails.LogGroup) + } + if logDelivery.DestinationDetails.KinesisFirehoseDetails != nil && logDelivery.DestinationDetails.KinesisFirehoseDetails.DeliveryStream != nil { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterLogDeliveryConfigurationDeliveryStream, i))] = model.LabelValue(*logDelivery.DestinationDetails.KinesisFirehoseDetails.DeliveryStream) + } + } + } + + // Pending modified values + if cluster.PendingModifiedValues != nil { + if len(cluster.PendingModifiedValues.AuthTokenStatus) > 0 { + commonLabels[elasticacheLabelCacheClusterPendingModifiedValuesAuthTokenStatus] = model.LabelValue(cluster.PendingModifiedValues.AuthTokenStatus) + } + if cluster.PendingModifiedValues.CacheNodeType != nil { + commonLabels[elasticacheLabelCacheClusterPendingModifiedValuesCacheNodeType] = model.LabelValue(*cluster.PendingModifiedValues.CacheNodeType) + } + if cluster.PendingModifiedValues.EngineVersion != nil { + commonLabels[elasticacheLabelCacheClusterPendingModifiedValuesEngineVersion] = model.LabelValue(*cluster.PendingModifiedValues.EngineVersion) + } + if cluster.PendingModifiedValues.NumCacheNodes != nil { + commonLabels[elasticacheLabelCacheClusterPendingModifiedValuesNumCacheNodes] = model.LabelValue(strconv.Itoa(int(*cluster.PendingModifiedValues.NumCacheNodes))) + } + if cluster.PendingModifiedValues.TransitEncryptionEnabled != nil { + commonLabels[elasticacheLabelCacheClusterPendingModifiedValuesTransitEncryptionEnabled] = model.LabelValue(strconv.FormatBool(*cluster.PendingModifiedValues.TransitEncryptionEnabled)) + } + if len(cluster.PendingModifiedValues.TransitEncryptionMode) > 0 { + commonLabels[elasticacheLabelCacheClusterPendingModifiedValuesTransitEncryptionMode] = model.LabelValue(cluster.PendingModifiedValues.TransitEncryptionMode) + } + if len(cluster.PendingModifiedValues.CacheNodeIdsToRemove) > 0 { + commonLabels[elasticacheLabelCacheClusterPendingModifiedValuesCacheNodeIDsToRemove] = model.LabelValue(strings.Join(cluster.PendingModifiedValues.CacheNodeIdsToRemove, ",")) + } + } + + // Security group membership (slice) + for i, sg := range cluster.SecurityGroups { + if sg.SecurityGroupId != nil { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterSecurityGroupMembershipID, i))] = model.LabelValue(*sg.SecurityGroupId) + } + if sg.Status != nil { + commonLabels[model.LabelName(fmt.Sprintf("%s_%d", elasticacheLabelCacheClusterSecurityGroupMembershipStatus, i))] = model.LabelValue(*sg.Status) + } + } + + // Tags + for _, tag := range tags { + if tag.Key != nil && tag.Value != nil { + commonLabels[model.LabelName(elasticacheLabelCacheClusterTag+strutil.SanitizeLabelName(*tag.Key))] = model.LabelValue(*tag.Value) + } + } + + // Create one target per cache node + for _, node := range cluster.CacheNodes { + // Clone common labels for this node + labels := make(model.LabelSet, len(commonLabels)) + maps.Copy(labels, commonLabels) + + // Add node-specific labels + if node.CacheNodeId != nil { + labels[elasticacheLabelCacheClusterNodeID] = model.LabelValue(*node.CacheNodeId) + } + if node.CacheNodeStatus != nil { + labels[elasticacheLabelCacheClusterNodeStatus] = model.LabelValue(*node.CacheNodeStatus) + } + if node.CacheNodeCreateTime != nil { + labels[elasticacheLabelCacheClusterNodeCreateTime] = model.LabelValue(node.CacheNodeCreateTime.Format(time.RFC3339)) + } + if node.CustomerAvailabilityZone != nil { + labels[elasticacheLabelCacheClusterNodeAZ] = model.LabelValue(*node.CustomerAvailabilityZone) + } + if node.CustomerOutpostArn != nil { + labels[elasticacheLabelCacheClusterNodeCustomerOutpostARN] = model.LabelValue(*node.CustomerOutpostArn) + } + if node.SourceCacheNodeId != nil { + labels[elasticacheLabelCacheClusterNodeSourceCacheNodeID] = model.LabelValue(*node.SourceCacheNodeId) + } + if node.ParameterGroupStatus != nil { + labels[elasticacheLabelCacheClusterNodeParameterGroupStatus] = model.LabelValue(*node.ParameterGroupStatus) + } + if node.Endpoint != nil { + if node.Endpoint.Address != nil { + labels[elasticacheLabelCacheClusterNodeEndpointAddress] = model.LabelValue(*node.Endpoint.Address) + } + if node.Endpoint.Port != nil { + labels[elasticacheLabelCacheClusterNodeEndpointPort] = model.LabelValue(strconv.Itoa(int(*node.Endpoint.Port))) + } + + // Set the address label to this node's endpoint + if node.Endpoint.Address != nil && node.Endpoint.Port != nil { + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(*node.Endpoint.Address, strconv.Itoa(int(*node.Endpoint.Port)))) + } + } + + tg.Targets = append(tg.Targets, labels) + } +} diff --git a/discovery/aws/elasticache_test.go b/discovery/aws/elasticache_test.go new file mode 100644 index 00000000000..646c5d2a81d --- /dev/null +++ b/discovery/aws/elasticache_test.go @@ -0,0 +1,620 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/elasticache" + "github.com/aws/aws-sdk-go-v2/service/elasticache/types" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +// Struct for test data. +type elasticacheDataStore struct { + region string + serverlessCaches []types.ServerlessCache + cacheClusters []types.CacheCluster + tags map[string][]types.Tag // keyed by cache ARN +} + +func TestElasticacheDiscoveryDescribeServerlessCaches(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + ecData *elasticacheDataStore + cacheNames []string + expectedCount int + }{ + { + name: "MultipleCaches", + ecData: &elasticacheDataStore{ + region: "us-west-2", + serverlessCaches: []types.ServerlessCache{ + { + ServerlessCacheName: strptr("test-cache"), + ARN: strptr("arn:aws:elasticache:us-west-2:123456789012:serverlesscache:test-cache"), + Status: strptr("available"), + Engine: strptr("redis"), + FullEngineVersion: strptr("7.1"), + CreateTime: aws.Time(time.Now()), + Endpoint: &types.Endpoint{ + Address: strptr("test-cache.serverless.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + }, + { + ServerlessCacheName: strptr("prod-cache"), + ARN: strptr("arn:aws:elasticache:us-west-2:123456789012:serverlesscache:prod-cache"), + Status: strptr("available"), + Engine: strptr("valkey"), + FullEngineVersion: strptr("7.2"), + CreateTime: aws.Time(time.Now()), + Endpoint: &types.Endpoint{ + Address: strptr("prod-cache.serverless.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + }, + }, + }, + cacheNames: []string{}, + expectedCount: 2, + }, + { + name: "SingleCache", + ecData: &elasticacheDataStore{ + region: "us-east-1", + serverlessCaches: []types.ServerlessCache{ + { + ServerlessCacheName: strptr("single-cache"), + ARN: strptr("arn:aws:elasticache:us-east-1:123456789012:serverlesscache:single-cache"), + Status: strptr("available"), + Engine: strptr("redis"), + FullEngineVersion: strptr("7.1"), + CreateTime: aws.Time(time.Now()), + }, + }, + }, + cacheNames: []string{"single-cache"}, + expectedCount: 1, + }, + { + name: "NoCaches", + ecData: &elasticacheDataStore{ + region: "us-east-1", + serverlessCaches: []types.ServerlessCache{}, + }, + cacheNames: []string{}, + expectedCount: 0, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockElasticacheClient(tt.ecData) + + d := &ElasticacheDiscovery{ + elasticacheClient: client, + cfg: &ElasticacheSDConfig{ + Region: tt.ecData.region, + RequestConcurrency: 10, + }, + } + + caches, err := d.describeServerlessCaches(ctx, tt.cacheNames) + require.NoError(t, err) + require.Len(t, caches, tt.expectedCount) + }) + } +} + +func TestElasticacheDiscoveryDescribeCacheClusters(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + ecData *elasticacheDataStore + clusterIDs []string + expectedCount int + skipTest bool + }{ + { + name: "MockValidation", + ecData: &elasticacheDataStore{ + region: "us-west-2", + cacheClusters: []types.CacheCluster{ + { + CacheClusterId: strptr("test-cluster-001"), + ARN: strptr("arn:aws:elasticache:us-west-2:123456789012:cluster:test-cluster-001"), + CacheClusterStatus: strptr("available"), + Engine: strptr("redis"), + EngineVersion: strptr("7.1"), + CacheNodeType: strptr("cache.t3.micro"), + NumCacheNodes: aws.Int32(1), + ConfigurationEndpoint: &types.Endpoint{ + Address: strptr("test-cluster.abc123.cfg.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + }, + }, + }, + clusterIDs: []string{}, + expectedCount: 1, + skipTest: false, + }, + { + name: "NoClusters", + ecData: &elasticacheDataStore{ + region: "us-east-1", + cacheClusters: []types.CacheCluster{}, + }, + clusterIDs: []string{}, + expectedCount: 0, + skipTest: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if tt.skipTest { + t.Skip("Skipping complex test with concurrency") + } + client := newMockElasticacheClient(tt.ecData) + + // Verify mock returns expected data + output, err := client.DescribeCacheClusters(ctx, &elasticache.DescribeCacheClustersInput{}) + require.NoError(t, err) + require.Len(t, output.CacheClusters, tt.expectedCount) + }) + } +} + +func TestAddServerlessCacheTargets(t *testing.T) { + t.Parallel() + testTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + tests := []struct { + name string + cache *types.ServerlessCache + tags []types.Tag + expectedLabels model.LabelSet + }{ + { + name: "ServerlessCacheWithEndpoint", + cache: &types.ServerlessCache{ + ServerlessCacheName: strptr("my-cache"), + ARN: strptr("arn:aws:elasticache:us-east-1:123456789012:serverlesscache:my-cache"), + Status: strptr("available"), + Engine: strptr("redis"), + FullEngineVersion: strptr("7.1"), + MajorEngineVersion: strptr("7"), + CreateTime: aws.Time(testTime), + Endpoint: &types.Endpoint{ + Address: strptr("my-cache.serverless.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + ReaderEndpoint: &types.Endpoint{ + Address: strptr("my-cache-ro.serverless.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + SecurityGroupIds: []string{"sg-12345"}, + SubnetIds: []string{"subnet-abcdef"}, + CacheUsageLimits: &types.CacheUsageLimits{ + DataStorage: &types.DataStorage{ + Maximum: aws.Int32(10), + Minimum: aws.Int32(1), + Unit: types.DataStorageUnitGb, + }, + ECPUPerSecond: &types.ECPUPerSecond{ + Maximum: aws.Int32(5000), + Minimum: aws.Int32(1000), + }, + }, + }, + tags: []types.Tag{ + {Key: strptr("Environment"), Value: strptr("test")}, + }, + expectedLabels: model.LabelSet{ + "__meta_elasticache_deployment_option": "serverless", + "__meta_elasticache_serverless_cache_arn": "arn:aws:elasticache:us-east-1:123456789012:serverlesscache:my-cache", + "__meta_elasticache_serverless_cache_name": "my-cache", + "__meta_elasticache_serverless_cache_status": "available", + "__meta_elasticache_serverless_cache_engine": "redis", + "__meta_elasticache_serverless_cache_full_engine_version": "7.1", + "__meta_elasticache_serverless_cache_major_engine_version": "7", + "__meta_elasticache_serverless_cache_create_time": "2024-01-01T00:00:00Z", + "__meta_elasticache_serverless_cache_endpoint_address": "my-cache.serverless.use1.cache.amazonaws.com", + "__meta_elasticache_serverless_cache_endpoint_port": "6379", + + "__meta_elasticache_serverless_cache_security_group_id_0": "sg-12345", + "__meta_elasticache_serverless_cache_subnet_id_0": "subnet-abcdef", + + "__address__": "my-cache.serverless.use1.cache.amazonaws.com:6379", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tg := &targetgroup.Group{ + Source: "test", + } + + addServerlessCacheTargets(tg, tt.cache, tt.tags) + + require.Len(t, tg.Targets, 1) + labels := tg.Targets[0] + + // Check that all expected labels are present with correct values + for k, v := range tt.expectedLabels { + actualValue, exists := labels[k] + require.True(t, exists, "label %s should exist", k) + require.Equal(t, v, actualValue, "label %s mismatch", k) + } + }) + } +} + +func TestAddCacheClusterTargets(t *testing.T) { + t.Parallel() + testTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + tests := []struct { + name string + cluster *types.CacheCluster + tags []types.Tag + expectedTargetCount int + expectedLabels []model.LabelSet // One per node + }{ + { + name: "CacheClusterWithMultipleNodes", + cluster: &types.CacheCluster{ + CacheClusterId: strptr("my-cluster-001"), + ARN: strptr("arn:aws:elasticache:us-east-1:123456789012:cluster:my-cluster-001"), + CacheClusterStatus: strptr("available"), + Engine: strptr("redis"), + EngineVersion: strptr("7.1"), + CacheNodeType: strptr("cache.t3.micro"), + NumCacheNodes: aws.Int32(2), + CacheClusterCreateTime: aws.Time(testTime), + ConfigurationEndpoint: &types.Endpoint{ + Address: strptr("my-cluster.abc123.cfg.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + AtRestEncryptionEnabled: aws.Bool(true), + TransitEncryptionEnabled: aws.Bool(true), + AuthTokenEnabled: aws.Bool(true), + AutoMinorVersionUpgrade: aws.Bool(true), + CacheSubnetGroupName: strptr("my-subnet-group"), + PreferredAvailabilityZone: strptr("us-east-1a"), + SecurityGroups: []types.SecurityGroupMembership{ + { + SecurityGroupId: strptr("sg-12345"), + Status: strptr("active"), + }, + }, + CacheNodes: []types.CacheNode{ + { + CacheNodeId: strptr("0001"), + CacheNodeStatus: strptr("available"), + CacheNodeCreateTime: aws.Time(testTime), + CustomerAvailabilityZone: strptr("us-east-1a"), + Endpoint: &types.Endpoint{ + Address: strptr("my-cluster-001.abc123.0001.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + }, + { + CacheNodeId: strptr("0002"), + CacheNodeStatus: strptr("available"), + CacheNodeCreateTime: aws.Time(testTime), + CustomerAvailabilityZone: strptr("us-east-1b"), + Endpoint: &types.Endpoint{ + Address: strptr("my-cluster-001.abc123.0002.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + }, + }, + }, + tags: []types.Tag{ + {Key: strptr("Environment"), Value: strptr("production")}, + {Key: strptr("Application"), Value: strptr("web-app")}, + }, + expectedTargetCount: 2, + expectedLabels: []model.LabelSet{ + { + "__meta_elasticache_deployment_option": "node", + "__meta_elasticache_cache_cluster_arn": "arn:aws:elasticache:us-east-1:123456789012:cluster:my-cluster-001", + "__meta_elasticache_cache_cluster_cache_cluster_id": "my-cluster-001", + "__meta_elasticache_cache_cluster_cache_cluster_status": "available", + "__meta_elasticache_cache_cluster_engine": "redis", + "__meta_elasticache_cache_cluster_engine_version": "7.1", + "__meta_elasticache_cache_cluster_cache_node_type": "cache.t3.micro", + "__meta_elasticache_cache_cluster_num_cache_nodes": "2", + "__meta_elasticache_cache_cluster_cache_cluster_create_time": "2024-01-01T00:00:00Z", + "__meta_elasticache_cache_cluster_configuration_endpoint_address": "my-cluster.abc123.cfg.use1.cache.amazonaws.com", + "__meta_elasticache_cache_cluster_configuration_endpoint_port": "6379", + "__meta_elasticache_cache_cluster_at_rest_encryption_enabled": "true", + "__meta_elasticache_cache_cluster_transit_encryption_enabled": "true", + "__meta_elasticache_cache_cluster_auth_token_enabled": "true", + "__meta_elasticache_cache_cluster_auto_minor_version_upgrade": "true", + "__meta_elasticache_cache_cluster_cache_subnet_group_name": "my-subnet-group", + "__meta_elasticache_cache_cluster_preferred_availability_zone": "us-east-1a", + "__meta_elasticache_cache_cluster_security_group_membership_id_0": "sg-12345", + "__meta_elasticache_cache_cluster_security_group_membership_status_0": "active", + "__meta_elasticache_cache_cluster_tag_Environment": "production", + "__meta_elasticache_cache_cluster_tag_Application": "web-app", + "__meta_elasticache_cache_cluster_node_id": "0001", + "__meta_elasticache_cache_cluster_node_status": "available", + "__meta_elasticache_cache_cluster_node_create_time": "2024-01-01T00:00:00Z", + "__meta_elasticache_cache_cluster_node_availability_zone": "us-east-1a", + "__meta_elasticache_cache_cluster_node_endpoint_address": "my-cluster-001.abc123.0001.use1.cache.amazonaws.com", + "__meta_elasticache_cache_cluster_node_endpoint_port": "6379", + "__address__": "my-cluster-001.abc123.0001.use1.cache.amazonaws.com:6379", + }, + { + "__meta_elasticache_deployment_option": "node", + "__meta_elasticache_cache_cluster_arn": "arn:aws:elasticache:us-east-1:123456789012:cluster:my-cluster-001", + "__meta_elasticache_cache_cluster_cache_cluster_id": "my-cluster-001", + "__meta_elasticache_cache_cluster_cache_cluster_status": "available", + "__meta_elasticache_cache_cluster_engine": "redis", + "__meta_elasticache_cache_cluster_engine_version": "7.1", + "__meta_elasticache_cache_cluster_cache_node_type": "cache.t3.micro", + "__meta_elasticache_cache_cluster_num_cache_nodes": "2", + "__meta_elasticache_cache_cluster_cache_cluster_create_time": "2024-01-01T00:00:00Z", + "__meta_elasticache_cache_cluster_configuration_endpoint_address": "my-cluster.abc123.cfg.use1.cache.amazonaws.com", + "__meta_elasticache_cache_cluster_configuration_endpoint_port": "6379", + "__meta_elasticache_cache_cluster_at_rest_encryption_enabled": "true", + "__meta_elasticache_cache_cluster_transit_encryption_enabled": "true", + "__meta_elasticache_cache_cluster_auth_token_enabled": "true", + "__meta_elasticache_cache_cluster_auto_minor_version_upgrade": "true", + "__meta_elasticache_cache_cluster_cache_subnet_group_name": "my-subnet-group", + "__meta_elasticache_cache_cluster_preferred_availability_zone": "us-east-1a", + "__meta_elasticache_cache_cluster_security_group_membership_id_0": "sg-12345", + "__meta_elasticache_cache_cluster_security_group_membership_status_0": "active", + "__meta_elasticache_cache_cluster_tag_Environment": "production", + "__meta_elasticache_cache_cluster_tag_Application": "web-app", + "__meta_elasticache_cache_cluster_node_id": "0002", + "__meta_elasticache_cache_cluster_node_status": "available", + "__meta_elasticache_cache_cluster_node_create_time": "2024-01-01T00:00:00Z", + "__meta_elasticache_cache_cluster_node_availability_zone": "us-east-1b", + "__meta_elasticache_cache_cluster_node_endpoint_address": "my-cluster-001.abc123.0002.use1.cache.amazonaws.com", + "__meta_elasticache_cache_cluster_node_endpoint_port": "6379", + "__address__": "my-cluster-001.abc123.0002.use1.cache.amazonaws.com:6379", + }, + }, + }, + { + name: "CacheClusterWithSingleNode", + cluster: &types.CacheCluster{ + CacheClusterId: strptr("node-cluster-001"), + ARN: strptr("arn:aws:elasticache:us-east-1:123456789012:cluster:node-cluster-001"), + CacheClusterStatus: strptr("available"), + Engine: strptr("redis"), + EngineVersion: strptr("6.2"), + CacheNodeType: strptr("cache.r6g.large"), + NumCacheNodes: aws.Int32(1), + CacheNodes: []types.CacheNode{ + { + CacheNodeId: strptr("0001"), + CacheNodeStatus: strptr("available"), + CacheNodeCreateTime: aws.Time(testTime), + CustomerAvailabilityZone: strptr("us-east-1a"), + Endpoint: &types.Endpoint{ + Address: strptr("node-cluster-001.abc123.0001.use1.cache.amazonaws.com"), + Port: aws.Int32(6379), + }, + }, + }, + }, + tags: []types.Tag{}, + expectedTargetCount: 1, + expectedLabels: []model.LabelSet{ + { + "__meta_elasticache_deployment_option": "node", + "__meta_elasticache_cache_cluster_arn": "arn:aws:elasticache:us-east-1:123456789012:cluster:node-cluster-001", + "__meta_elasticache_cache_cluster_cache_cluster_id": "node-cluster-001", + "__meta_elasticache_cache_cluster_cache_cluster_status": "available", + "__meta_elasticache_cache_cluster_engine": "redis", + "__meta_elasticache_cache_cluster_engine_version": "6.2", + "__meta_elasticache_cache_cluster_cache_node_type": "cache.r6g.large", + "__meta_elasticache_cache_cluster_num_cache_nodes": "1", + "__meta_elasticache_cache_cluster_node_id": "0001", + "__meta_elasticache_cache_cluster_node_status": "available", + "__meta_elasticache_cache_cluster_node_create_time": "2024-01-01T00:00:00Z", + "__meta_elasticache_cache_cluster_node_availability_zone": "us-east-1a", + "__meta_elasticache_cache_cluster_node_endpoint_address": "node-cluster-001.abc123.0001.use1.cache.amazonaws.com", + "__meta_elasticache_cache_cluster_node_endpoint_port": "6379", + "__address__": "node-cluster-001.abc123.0001.use1.cache.amazonaws.com:6379", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tg := &targetgroup.Group{ + Source: "test", + } + + addCacheClusterTargets(tg, tt.cluster, tt.tags) + + require.Len(t, tg.Targets, tt.expectedTargetCount) + + // Check each target + for i, expectedLabels := range tt.expectedLabels { + labels := tg.Targets[i] + + // Check that all expected labels are present with correct values + for k, v := range expectedLabels { + actualValue, exists := labels[k] + require.True(t, exists, "label %s should exist in target %d", k, i) + require.Equal(t, v, actualValue, "label %s mismatch in target %d", k, i) + } + } + }) + } +} + +// Mock Elasticache client. +type mockElasticacheClient struct { + data *elasticacheDataStore +} + +func newMockElasticacheClient(data *elasticacheDataStore) *mockElasticacheClient { + return &mockElasticacheClient{data: data} +} + +func (m *mockElasticacheClient) DescribeServerlessCaches(_ context.Context, input *elasticache.DescribeServerlessCachesInput, _ ...func(*elasticache.Options)) (*elasticache.DescribeServerlessCachesOutput, error) { + if input.ServerlessCacheName != nil { + // Filter by name + for _, cache := range m.data.serverlessCaches { + if cache.ServerlessCacheName != nil && *cache.ServerlessCacheName == *input.ServerlessCacheName { + return &elasticache.DescribeServerlessCachesOutput{ + ServerlessCaches: []types.ServerlessCache{cache}, + }, nil + } + } + return &elasticache.DescribeServerlessCachesOutput{ + ServerlessCaches: []types.ServerlessCache{}, + }, nil + } + + return &elasticache.DescribeServerlessCachesOutput{ + ServerlessCaches: m.data.serverlessCaches, + }, nil +} + +func (m *mockElasticacheClient) DescribeCacheClusters(_ context.Context, input *elasticache.DescribeCacheClustersInput, _ ...func(*elasticache.Options)) (*elasticache.DescribeCacheClustersOutput, error) { + if input.CacheClusterId != nil { + // Single cluster lookup + for _, cluster := range m.data.cacheClusters { + if cluster.CacheClusterId != nil && *cluster.CacheClusterId == *input.CacheClusterId { + return &elasticache.DescribeCacheClustersOutput{ + CacheClusters: []types.CacheCluster{cluster}, + }, nil + } + } + return &elasticache.DescribeCacheClustersOutput{ + CacheClusters: []types.CacheCluster{}, + }, nil + } + + return &elasticache.DescribeCacheClustersOutput{ + CacheClusters: m.data.cacheClusters, + }, nil +} + +func (m *mockElasticacheClient) ListTagsForResource(_ context.Context, input *elasticache.ListTagsForResourceInput, _ ...func(*elasticache.Options)) (*elasticache.ListTagsForResourceOutput, error) { + if input.ResourceName != nil { + if tags, ok := m.data.tags[*input.ResourceName]; ok { + return &elasticache.ListTagsForResourceOutput{ + TagList: tags, + }, nil + } + } + + return &elasticache.ListTagsForResourceOutput{ + TagList: []types.Tag{}, + }, nil +} + +func TestSplitCacheDeploymentOptions(t *testing.T) { + t.Parallel() + tests := []struct { + name string + caches []string + expectedServerlessCacheIDs []string + expectedCacheClusterIDs []string + }{ + { + name: "MixedARNs", + caches: []string{ + "arn:aws:elasticache:us-east-1:123456789012:serverlesscache:my-serverless-cache", + "arn:aws:elasticache:us-east-1:123456789012:replicationgroup:my-replication-group", + "arn:aws:elasticache:us-west-2:123456789012:serverlesscache:prod-cache", + }, + expectedServerlessCacheIDs: []string{"my-serverless-cache", "prod-cache"}, + expectedCacheClusterIDs: []string{"my-replication-group"}, + }, + { + name: "OnlyServerlessCaches", + caches: []string{ + "arn:aws:elasticache:us-east-1:123456789012:serverlesscache:cache-1", + "arn:aws:elasticache:us-east-1:123456789012:serverlesscache:cache-2", + }, + expectedServerlessCacheIDs: []string{"cache-1", "cache-2"}, + expectedCacheClusterIDs: nil, + }, + { + name: "OnlyReplicationGroups", + caches: []string{ + "arn:aws:elasticache:us-east-1:123456789012:replicationgroup:cluster-1", + "arn:aws:elasticache:us-east-1:123456789012:replicationgroup:cluster-2", + }, + expectedServerlessCacheIDs: nil, + expectedCacheClusterIDs: []string{"cluster-1", "cluster-2"}, + }, + { + name: "EmptyInput", + caches: []string{}, + expectedServerlessCacheIDs: nil, + expectedCacheClusterIDs: nil, + }, + { + name: "InvalidARNs", + caches: []string{ + "not-an-arn", + "arn:aws:elasticache:us-east-1", + "", + }, + expectedServerlessCacheIDs: nil, + expectedCacheClusterIDs: nil, + }, + { + name: "UnknownResourceType", + caches: []string{ + "arn:aws:elasticache:us-east-1:123456789012:unknown:resource-id", + }, + expectedServerlessCacheIDs: nil, + expectedCacheClusterIDs: nil, + }, + { + name: "MixedWithInvalidARNs", + caches: []string{ + "arn:aws:elasticache:us-east-1:123456789012:serverlesscache:valid-cache", + "invalid-arn", + "arn:aws:elasticache:us-east-1:123456789012:replicationgroup:valid-cluster", + "", + "arn:aws:elasticache:us-east-1:123456789012:unknown:ignored", + }, + expectedServerlessCacheIDs: []string{"valid-cache"}, + expectedCacheClusterIDs: []string{"valid-cluster"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + serverlessCacheIDs, cacheClusterIDs := splitCacheDeploymentOptions(tt.caches) + + require.Equal(t, tt.expectedServerlessCacheIDs, serverlessCacheIDs, "serverless cache IDs mismatch") + require.Equal(t, tt.expectedCacheClusterIDs, cacheClusterIDs, "cache cluster IDs mismatch") + }) + } +} diff --git a/discovery/aws/lightsail.go b/discovery/aws/lightsail.go index 0ad7f2d5416..c56fe759480 100644 --- a/discovery/aws/lightsail.go +++ b/discovery/aws/lightsail.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -22,17 +22,17 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/lightsail" - "github.com/go-kit/log" + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/lightsail" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/refresh" @@ -75,6 +75,7 @@ type LightsailSDConfig struct { SecretKey config.Secret `yaml:"secret_key,omitempty"` Profile string `yaml:"profile,omitempty"` RoleARN string `yaml:"role_arn,omitempty"` + ExternalID string `yaml:"external_id,omitempty"` RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` Port int `yaml:"port"` @@ -82,7 +83,7 @@ type LightsailSDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*LightsailSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*LightsailSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &lightsailMetrics{ refreshMetrics: rmi, } @@ -93,51 +94,65 @@ func (*LightsailSDConfig) Name() string { return "lightsail" } // NewDiscoverer returns a Discoverer for the Lightsail Config. func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewLightsailDiscovery(c, opts.Logger, opts.Metrics) + return NewLightsailDiscovery(c, opts) +} + +// SetDirectory joins any relative file paths with dir. +func (c *LightsailSDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) } // UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config. -func (c *LightsailSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *LightsailSDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultLightsailSDConfig type plain LightsailSDConfig err := unmarshal((*plain)(c)) if err != nil { return err } - if c.Region == "" { - sess, err := session.NewSession() - if err != nil { - return err - } - - metadata := ec2metadata.New(sess) - region, err := metadata.Region() - if err != nil { - return errors.New("Lightsail SD configuration requires a region") - } - c.Region = region + c.Region, err = loadRegion(context.Background(), c.Region) + if err != nil { + return fmt.Errorf("could not determine AWS region: %w", err) } + return c.HTTPClientConfig.Validate() } +// lightsailClientAdapter captures only the Lightsail API calls AWS discovery +// uses as method-value closures, keeping the concrete *lightsail.Client out of +// any interface-boxed struct field. See ec2ClientAdapter for the full +// rationale: this stops the linker from retaining the entire Lightsail API +// surface (~3.4 MB). +type lightsailClientAdapter struct { + getInstances func(ctx context.Context, params *lightsail.GetInstancesInput, optFns ...func(*lightsail.Options)) (*lightsail.GetInstancesOutput, error) +} + +func newLightsailClientAdapter(c *lightsail.Client) *lightsailClientAdapter { + return &lightsailClientAdapter{getInstances: c.GetInstances} +} + +func (a *lightsailClientAdapter) GetInstances(ctx context.Context, params *lightsail.GetInstancesInput, optFns ...func(*lightsail.Options)) (*lightsail.GetInstancesOutput, error) { + return a.getInstances(ctx, params, optFns...) +} + // LightsailDiscovery periodically performs Lightsail-SD requests. It implements // the Discoverer interface. type LightsailDiscovery struct { *refresh.Discovery cfg *LightsailSDConfig - lightsail *lightsail.Lightsail + lightsail *lightsailClientAdapter } // NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets. -func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*LightsailDiscovery, error) { - m, ok := metrics.(*lightsailMetrics) +func NewLightsailDiscovery(conf *LightsailSDConfig, opts discovery.DiscovererOptions) (*LightsailDiscovery, error) { + m, ok := opts.Metrics.(*lightsailMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - if logger == nil { - logger = log.NewNopLogger() + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() } d := &LightsailDiscovery{ @@ -145,8 +160,9 @@ func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, metrics d } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "lightsail", + SetName: opts.SetName, Interval: time.Duration(d.cfg.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -155,46 +171,62 @@ func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, metrics d return d, nil } -func (d *LightsailDiscovery) lightsailClient() (*lightsail.Lightsail, error) { +func (d *LightsailDiscovery) lightsailClient(ctx context.Context) (*lightsailClientAdapter, error) { if d.lightsail != nil { return d.lightsail, nil } - creds := credentials.NewStaticCredentials(d.cfg.AccessKey, string(d.cfg.SecretKey), "") - if d.cfg.AccessKey == "" && d.cfg.SecretKey == "" { - creds = nil - } - - client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "lightsail_sd") + // Build the HTTP client from the provided HTTPClientConfig. + httpClient, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "lightsail_sd") if err != nil { return nil, err } - sess, err := session.NewSessionWithOptions(session.Options{ - Config: aws.Config{ - Endpoint: &d.cfg.Endpoint, - Region: &d.cfg.Region, - Credentials: creds, - HTTPClient: client, - }, - Profile: d.cfg.Profile, - }) + // Build the AWS config with the provided region. + configOptions := []func(*awsConfig.LoadOptions) error{ + awsConfig.WithRegion(d.cfg.Region), + awsConfig.WithHTTPClient(httpClient), + } + + // Only set static credentials if both access key and secret key are provided. + // Otherwise, let the AWS SDK use its default credential chain (environment variables, IAM role, etc.). + if d.cfg.AccessKey != "" && d.cfg.SecretKey != "" { + credProvider := credentials.NewStaticCredentialsProvider(d.cfg.AccessKey, string(d.cfg.SecretKey), "") + configOptions = append(configOptions, awsConfig.WithCredentialsProvider(credProvider)) + } + + // Set the profile if provided. + if d.cfg.Profile != "" { + configOptions = append(configOptions, awsConfig.WithSharedConfigProfile(d.cfg.Profile)) + } + + cfg, err := awsConfig.LoadDefaultConfig(ctx, configOptions...) if err != nil { - return nil, fmt.Errorf("could not create aws session: %w", err) + return nil, fmt.Errorf("could not create aws config: %w", err) } + // If the role ARN is set, assume the role to get credentials and set the credentials provider in the config. if d.cfg.RoleARN != "" { - creds := stscreds.NewCredentials(sess, d.cfg.RoleARN) - d.lightsail = lightsail.New(sess, &aws.Config{Credentials: creds}) - } else { - d.lightsail = lightsail.New(sess) + assumeProvider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), d.cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { + if d.cfg.ExternalID != "" { + o.ExternalID = aws.String(d.cfg.ExternalID) + } + }) + cfg.Credentials = aws.NewCredentialsCache(assumeProvider) } + d.lightsail = newLightsailClientAdapter(lightsail.NewFromConfig(cfg, func(options *lightsail.Options) { + if d.cfg.Endpoint != "" { + options.BaseEndpoint = &d.cfg.Endpoint + } + options.HTTPClient = httpClient + })) + return d.lightsail, nil } func (d *LightsailDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - lightsailClient, err := d.lightsailClient() + lightsailClient, err := d.lightsailClient(ctx) if err != nil { return nil, err } @@ -205,10 +237,10 @@ func (d *LightsailDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, input := &lightsail.GetInstancesInput{} - output, err := lightsailClient.GetInstancesWithContext(ctx, input) + output, err := lightsailClient.GetInstances(ctx, input) if err != nil { - var awsErr awserr.Error - if errors.As(err, &awsErr) && (awsErr.Code() == "AuthFailure" || awsErr.Code() == "UnauthorizedOperation") { + var awsErr smithy.APIError + if errors.As(err, &awsErr) && (awsErr.ErrorCode() == "AuthFailure" || awsErr.ErrorCode() == "UnauthorizedOperation") { d.lightsail = nil } return nil, fmt.Errorf("could not get instances: %w", err) @@ -239,9 +271,7 @@ func (d *LightsailDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, if len(inst.Ipv6Addresses) > 0 { var ipv6addrs []string - for _, ipv6addr := range inst.Ipv6Addresses { - ipv6addrs = append(ipv6addrs, *ipv6addr) - } + ipv6addrs = append(ipv6addrs, inst.Ipv6Addresses...) labels[lightsailLabelIPv6Addresses] = model.LabelValue( lightsailLabelSeparator + strings.Join(ipv6addrs, lightsailLabelSeparator) + @@ -249,7 +279,7 @@ func (d *LightsailDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, } for _, t := range inst.Tags { - if t == nil || t.Key == nil || t.Value == nil { + if t.Key == nil || t.Value == nil { continue } name := strutil.SanitizeLabelName(*t.Key) diff --git a/util/logging/ratelimit.go b/discovery/aws/metrics_aws.go similarity index 54% rename from util/logging/ratelimit.go rename to discovery/aws/metrics_aws.go index 32d1e249e68..4cb9b25041c 100644 --- a/util/logging/ratelimit.go +++ b/discovery/aws/metrics_aws.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -11,29 +11,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package logging +package aws import ( - "github.com/go-kit/log" - "golang.org/x/time/rate" + "github.com/prometheus/prometheus/discovery" ) -type ratelimiter struct { - limiter *rate.Limiter - next log.Logger +type awsMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator } -// RateLimit write to a logger. -func RateLimit(next log.Logger, limit rate.Limit) log.Logger { - return &ratelimiter{ - limiter: rate.NewLimiter(limit, int(limit)), - next: next, - } -} +var _ discovery.DiscovererMetrics = (*awsMetrics)(nil) -func (r *ratelimiter) Log(keyvals ...interface{}) error { - if r.limiter.Allow() { - return r.next.Log(keyvals...) - } +// Register implements discovery.DiscovererMetrics. +func (*awsMetrics) Register() error { return nil } + +// Unregister implements discovery.DiscovererMetrics. +func (*awsMetrics) Unregister() {} diff --git a/discovery/aws/metrics_ec2.go b/discovery/aws/metrics_ec2.go index 73c061c3d88..1a37347b40b 100644 --- a/discovery/aws/metrics_ec2.go +++ b/discovery/aws/metrics_ec2.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type ec2Metrics struct { var _ discovery.DiscovererMetrics = (*ec2Metrics)(nil) // Register implements discovery.DiscovererMetrics. -func (m *ec2Metrics) Register() error { +func (*ec2Metrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *ec2Metrics) Unregister() {} +func (*ec2Metrics) Unregister() {} diff --git a/discovery/aws/metrics_ecs.go b/discovery/aws/metrics_ecs.go new file mode 100644 index 00000000000..dde3483c068 --- /dev/null +++ b/discovery/aws/metrics_ecs.go @@ -0,0 +1,32 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "github.com/prometheus/prometheus/discovery" +) + +type ecsMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +var _ discovery.DiscovererMetrics = (*ecsMetrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (*ecsMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (*ecsMetrics) Unregister() {} diff --git a/discovery/aws/metrics_elasticache.go b/discovery/aws/metrics_elasticache.go new file mode 100644 index 00000000000..7ecfcb4b72c --- /dev/null +++ b/discovery/aws/metrics_elasticache.go @@ -0,0 +1,32 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "github.com/prometheus/prometheus/discovery" +) + +type elasticacheMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +var _ discovery.DiscovererMetrics = (*elasticacheMetrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (*elasticacheMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (*elasticacheMetrics) Unregister() {} diff --git a/discovery/aws/metrics_lightsail.go b/discovery/aws/metrics_lightsail.go index 69593e701d5..40f76394591 100644 --- a/discovery/aws/metrics_lightsail.go +++ b/discovery/aws/metrics_lightsail.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type lightsailMetrics struct { var _ discovery.DiscovererMetrics = (*lightsailMetrics)(nil) // Register implements discovery.DiscovererMetrics. -func (m *lightsailMetrics) Register() error { +func (*lightsailMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *lightsailMetrics) Unregister() {} +func (*lightsailMetrics) Unregister() {} diff --git a/discovery/aws/metrics_msk.go b/discovery/aws/metrics_msk.go new file mode 100644 index 00000000000..fc69f57aa10 --- /dev/null +++ b/discovery/aws/metrics_msk.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "github.com/prometheus/prometheus/discovery" +) + +type mskMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +var _ discovery.DiscovererMetrics = (*mskMetrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (*mskMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (*mskMetrics) Unregister() {} diff --git a/discovery/aws/metrics_rds.go b/discovery/aws/metrics_rds.go new file mode 100644 index 00000000000..b8e29cfcecd --- /dev/null +++ b/discovery/aws/metrics_rds.go @@ -0,0 +1,32 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "github.com/prometheus/prometheus/discovery" +) + +type rdsMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +var _ discovery.DiscovererMetrics = (*rdsMetrics)(nil) + +// Register implements discovery.DiscovererMetrics. +func (*rdsMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (*rdsMetrics) Unregister() {} diff --git a/discovery/aws/msk.go b/discovery/aws/msk.go new file mode 100644 index 00000000000..475d2db1454 --- /dev/null +++ b/discovery/aws/msk.go @@ -0,0 +1,491 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "strconv" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/kafka" + "github.com/aws/aws-sdk-go-v2/service/kafka/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "golang.org/x/sync/errgroup" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +type NodeType string + +const ( + NodeTypeBroker NodeType = "BROKER" + NodeTypeController NodeType = "CONTROLLER" +) + +const ( + mskLabel = model.MetaLabelPrefix + "msk_" + + // Cluster labels. + mskLabelCluster = mskLabel + "cluster_" + mskLabelClusterName = mskLabelCluster + "name" + mskLabelClusterARN = mskLabelCluster + "arn" + mskLabelClusterState = mskLabelCluster + "state" + mskLabelClusterType = mskLabelCluster + "type" + mskLabelClusterVersion = mskLabelCluster + "version" + mskLabelClusterJmxExporterEnabled = mskLabelCluster + "jmx_exporter_enabled" + mskLabelClusterConfigurationARN = mskLabelCluster + "configuration_arn" + mskLabelClusterConfigurationRevision = mskLabelCluster + "configuration_revision" + mskLabelClusterKafkaVersion = mskLabelCluster + "kafka_version" + mskLabelClusterTags = mskLabelCluster + "tag_" + + // Node labels. + mskLabelNode = mskLabel + "node_" + mskLabelNodeType = mskLabelNode + "type" + mskLabelNodeARN = mskLabelNode + "arn" + mskLabelNodeAddedTime = mskLabelNode + "added_time" + mskLabelNodeInstanceType = mskLabelNode + "instance_type" + mskLabelNodeAttachedENI = mskLabelNode + "attached_eni" + + // Broker labels. + mskLabelBroker = mskLabel + "broker_" + mskLabelBrokerEndpointIndex = mskLabelBroker + "endpoint_index" + mskLabelBrokerID = mskLabelBroker + "id" + mskLabelBrokerClientSubnet = mskLabelBroker + "client_subnet" + mskLabelBrokerClientVPCIP = mskLabelBroker + "client_vpc_ip" + mskLabelBrokerNodeExporterEnabled = mskLabelBroker + "node_exporter_enabled" + + // Controller labels. + mskLabelController = mskLabel + "controller_" + mskLabelControllerEndpointIndex = mskLabelController + "endpoint_index" +) + +// DefaultMSKSDConfig is the default MSK SD configuration. +var DefaultMSKSDConfig = MSKSDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + RequestConcurrency: 10, + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +func init() { + discovery.RegisterConfig(&MSKSDConfig{}) +} + +// MSKSDConfig is the configuration for MSK based service discovery. +type MSKSDConfig struct { + Region string `yaml:"region"` + Endpoint string `yaml:"endpoint"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + ExternalID string `yaml:"external_id,omitempty"` + Clusters []string `yaml:"clusters,omitempty"` + Port int `yaml:"port"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + + RequestConcurrency int `yaml:"request_concurrency,omitempty"` + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` +} + +// NewDiscovererMetrics implements discovery.Config. +func (*MSKSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &mskMetrics{ + refreshMetrics: rmi, + } +} + +// Name returns the name of the MSK Config. +func (*MSKSDConfig) Name() string { return "msk" } + +// NewDiscoverer returns a Discoverer for the MSK Config. +func (c *MSKSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewMSKDiscovery(c, opts) +} + +// SetDirectory joins any relative file paths with dir. +func (c *MSKSDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for the MSK Config. +func (c *MSKSDConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultMSKSDConfig + type plain MSKSDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + + c.Region, err = loadRegion(context.Background(), c.Region) + if err != nil { + return fmt.Errorf("could not determine AWS region: %w", err) + } + + return c.HTTPClientConfig.Validate() +} + +type mskClient interface { + DescribeClusterV2(context.Context, *kafka.DescribeClusterV2Input, ...func(*kafka.Options)) (*kafka.DescribeClusterV2Output, error) + ListClustersV2(context.Context, *kafka.ListClustersV2Input, ...func(*kafka.Options)) (*kafka.ListClustersV2Output, error) + ListNodes(context.Context, *kafka.ListNodesInput, ...func(*kafka.Options)) (*kafka.ListNodesOutput, error) +} + +// mskClientAdapter captures only the MSK (Kafka) API calls AWS discovery uses +// as method-value closures, keeping the concrete *kafka.Client out of any +// interface-boxed struct field. See ec2ClientAdapter for the full rationale: +// this stops the linker from retaining the entire MSK API surface (~1.4 MB). +type mskClientAdapter struct { + describeClusterV2 func(context.Context, *kafka.DescribeClusterV2Input, ...func(*kafka.Options)) (*kafka.DescribeClusterV2Output, error) + listClustersV2 func(context.Context, *kafka.ListClustersV2Input, ...func(*kafka.Options)) (*kafka.ListClustersV2Output, error) + listNodes func(context.Context, *kafka.ListNodesInput, ...func(*kafka.Options)) (*kafka.ListNodesOutput, error) +} + +func newMSKClientAdapter(c *kafka.Client) mskClientAdapter { + return mskClientAdapter{ + describeClusterV2: c.DescribeClusterV2, + listClustersV2: c.ListClustersV2, + listNodes: c.ListNodes, + } +} + +func (a mskClientAdapter) DescribeClusterV2(ctx context.Context, params *kafka.DescribeClusterV2Input, optFns ...func(*kafka.Options)) (*kafka.DescribeClusterV2Output, error) { + return a.describeClusterV2(ctx, params, optFns...) +} + +func (a mskClientAdapter) ListClustersV2(ctx context.Context, params *kafka.ListClustersV2Input, optFns ...func(*kafka.Options)) (*kafka.ListClustersV2Output, error) { + return a.listClustersV2(ctx, params, optFns...) +} + +func (a mskClientAdapter) ListNodes(ctx context.Context, params *kafka.ListNodesInput, optFns ...func(*kafka.Options)) (*kafka.ListNodesOutput, error) { + return a.listNodes(ctx, params, optFns...) +} + +// MSKDiscovery periodically performs MSK-SD requests. It implements +// the Discoverer interface. +type MSKDiscovery struct { + *refresh.Discovery + logger *slog.Logger + cfg *MSKSDConfig + msk mskClient +} + +// NewMSKDiscovery returns a new MSKDiscovery which periodically refreshes its targets. +func NewMSKDiscovery(conf *MSKSDConfig, opts discovery.DiscovererOptions) (*MSKDiscovery, error) { + m, ok := opts.Metrics.(*mskMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type") + } + + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() + } + d := &MSKDiscovery{ + logger: opts.Logger, + cfg: conf, + } + d.Discovery = refresh.NewDiscovery( + refresh.Options{ + Logger: opts.Logger, + Mech: "msk", + Interval: time.Duration(d.cfg.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ) + return d, nil +} + +func (d *MSKDiscovery) initMskClient(ctx context.Context) error { + if d.msk != nil { + return nil + } + + if d.cfg.Region == "" { + return errors.New("region must be set for MSK service discovery") + } + + // Build the HTTP client from the provided HTTPClientConfig. + client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "msk_sd") + if err != nil { + return err + } + + // Build the AWS config with the provided region. + var configOptions []func(*awsConfig.LoadOptions) error + configOptions = append(configOptions, awsConfig.WithRegion(d.cfg.Region)) + configOptions = append(configOptions, awsConfig.WithHTTPClient(client)) + + // Only set static credentials if both access key and secret key are provided + // Otherwise, let AWS SDK use its default credential chain + if d.cfg.AccessKey != "" && d.cfg.SecretKey != "" { + credProvider := credentials.NewStaticCredentialsProvider(d.cfg.AccessKey, string(d.cfg.SecretKey), "") + configOptions = append(configOptions, awsConfig.WithCredentialsProvider(credProvider)) + } + + if d.cfg.Profile != "" { + configOptions = append(configOptions, awsConfig.WithSharedConfigProfile(d.cfg.Profile)) + } + + cfg, err := awsConfig.LoadDefaultConfig(ctx, configOptions...) + if err != nil { + d.logger.Error("Failed to create AWS config", "error", err) + return fmt.Errorf("could not create aws config: %w", err) + } + + // If the role ARN is set, assume the role to get credentials and set the credentials provider in the config. + if d.cfg.RoleARN != "" { + assumeProvider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), d.cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { + if d.cfg.ExternalID != "" { + o.ExternalID = aws.String(d.cfg.ExternalID) + } + }) + cfg.Credentials = aws.NewCredentialsCache(assumeProvider) + } + + d.msk = newMSKClientAdapter(kafka.NewFromConfig(cfg, func(options *kafka.Options) { + if d.cfg.Endpoint != "" { + options.BaseEndpoint = &d.cfg.Endpoint + } + options.HTTPClient = client + })) + + // Test credentials by making a simple API call + testCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + _, err = d.msk.ListClustersV2(testCtx, &kafka.ListClustersV2Input{}) + if err != nil { + d.logger.Error("Failed to test MSK credentials", "error", err) + return fmt.Errorf("MSK credential test failed: %w", err) + } + + return nil +} + +// describeClusters describes the clusters with the given ARNs and returns their details. +func (d *MSKDiscovery) describeClusters(ctx context.Context, clusterARNs []string) ([]types.Cluster, error) { + var ( + clusters []types.Cluster + mu sync.Mutex + ) + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for _, clusterARN := range clusterARNs { + errg.Go(func() error { + cluster, err := d.msk.DescribeClusterV2(ectx, &kafka.DescribeClusterV2Input{ + ClusterArn: aws.String(clusterARN), + }) + if err != nil { + return fmt.Errorf("could not describe cluster %v: %w", clusterARN, err) + } + mu.Lock() + clusters = append(clusters, *cluster.ClusterInfo) + mu.Unlock() + return nil + }) + } + + return clusters, errg.Wait() +} + +// listClusters lists all MSK clusters in the configured region and returns their details. +func (d *MSKDiscovery) listClusters(ctx context.Context) ([]types.Cluster, error) { + var ( + clusters []types.Cluster + nextToken *string + ) + for { + listClustersInput := kafka.ListClustersV2Input{ + ClusterTypeFilter: aws.String("PROVISIONED"), + MaxResults: aws.Int32(100), + NextToken: nextToken, + } + + resp, err := d.msk.ListClustersV2(ctx, &listClustersInput) + if err != nil { + return nil, fmt.Errorf("could not list clusters: %w", err) + } + + clusters = append(clusters, resp.ClusterInfoList...) + if resp.NextToken == nil { + break + } + nextToken = resp.NextToken + } + + return clusters, nil +} + +// listNodes lists all nodes for the given clusters and returns a map of cluster ARN to its nodes. +func (d *MSKDiscovery) listNodes(ctx context.Context, clusters []types.Cluster) (map[string][]types.NodeInfo, error) { + clusterNodeMap := make(map[string][]types.NodeInfo) + mu := sync.Mutex{} + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + for _, cluster := range clusters { + clusterARN := aws.ToString(cluster.ClusterArn) + errg.Go(func() error { + var clusterNodes []types.NodeInfo + var nextToken *string + for { + resp, err := d.msk.ListNodes(ectx, &kafka.ListNodesInput{ + ClusterArn: aws.String(clusterARN), + MaxResults: aws.Int32(100), + NextToken: nextToken, + }) + if err != nil { + return fmt.Errorf("could not list nodes for cluster %v: %w", clusterARN, err) + } + + clusterNodes = append(clusterNodes, resp.NodeInfoList...) + if resp.NextToken == nil { + break + } + nextToken = resp.NextToken + } + + mu.Lock() + clusterNodeMap[clusterARN] = clusterNodes + mu.Unlock() + return nil + }) + } + + return clusterNodeMap, errg.Wait() +} + +func (d *MSKDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + err := d.initMskClient(ctx) + if err != nil { + return nil, err + } + + tg := &targetgroup.Group{ + Source: d.cfg.Region, + } + + var clusters []types.Cluster + if len(d.cfg.Clusters) > 0 { + clusters, err = d.describeClusters(ctx, d.cfg.Clusters) + if err != nil { + return nil, err + } + } else { + clusters, err = d.listClusters(ctx) + if err != nil { + return nil, err + } + } + + clusterNodeMap, err := d.listNodes(ctx, clusters) + if err != nil { + return nil, err + } + + var ( + targetsMu sync.Mutex + wg sync.WaitGroup + ) + for _, cluster := range clusters { + wg.Add(1) + + go func(cluster types.Cluster, nodes []types.NodeInfo) { + defer wg.Done() + for _, node := range nodes { + labels := model.LabelSet{ + mskLabelClusterName: model.LabelValue(aws.ToString(cluster.ClusterName)), + mskLabelClusterARN: model.LabelValue(aws.ToString(cluster.ClusterArn)), + mskLabelClusterState: model.LabelValue(string(cluster.State)), + mskLabelClusterType: model.LabelValue(string(cluster.ClusterType)), + mskLabelClusterVersion: model.LabelValue(aws.ToString(cluster.CurrentVersion)), + mskLabelNodeARN: model.LabelValue(aws.ToString(node.NodeARN)), + mskLabelNodeAddedTime: model.LabelValue(aws.ToString(node.AddedToClusterTime)), + mskLabelNodeInstanceType: model.LabelValue(aws.ToString(node.InstanceType)), + mskLabelClusterJmxExporterEnabled: model.LabelValue(strconv.FormatBool(*cluster.Provisioned.OpenMonitoring.Prometheus.JmxExporter.EnabledInBroker)), + mskLabelClusterConfigurationARN: model.LabelValue(aws.ToString(cluster.Provisioned.CurrentBrokerSoftwareInfo.ConfigurationArn)), + mskLabelClusterConfigurationRevision: model.LabelValue(strconv.FormatInt(*cluster.Provisioned.CurrentBrokerSoftwareInfo.ConfigurationRevision, 10)), + mskLabelClusterKafkaVersion: model.LabelValue(aws.ToString(cluster.Provisioned.CurrentBrokerSoftwareInfo.KafkaVersion)), + } + + for key, value := range cluster.Tags { + labels[model.LabelName(mskLabelClusterTags+strutil.SanitizeLabelName(key))] = model.LabelValue(value) + } + + switch nodeType(node) { + case NodeTypeBroker: + labels[mskLabelNodeType] = model.LabelValue(NodeTypeBroker) + labels[mskLabelNodeAttachedENI] = model.LabelValue(aws.ToString(node.BrokerNodeInfo.AttachedENIId)) + labels[mskLabelBrokerID] = model.LabelValue(fmt.Sprintf("%.0f", aws.ToFloat64(node.BrokerNodeInfo.BrokerId))) + labels[mskLabelBrokerClientSubnet] = model.LabelValue(aws.ToString(node.BrokerNodeInfo.ClientSubnet)) + labels[mskLabelBrokerClientVPCIP] = model.LabelValue(aws.ToString(node.BrokerNodeInfo.ClientVpcIpAddress)) + labels[mskLabelBrokerNodeExporterEnabled] = model.LabelValue(strconv.FormatBool(*cluster.Provisioned.OpenMonitoring.Prometheus.NodeExporter.EnabledInBroker)) + + for idx, endpoint := range node.BrokerNodeInfo.Endpoints { + endpointLabels := labels.Clone() + endpointLabels[mskLabelBrokerEndpointIndex] = model.LabelValue(strconv.Itoa(idx)) + endpointLabels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(endpoint, strconv.Itoa(d.cfg.Port))) + + targetsMu.Lock() + tg.Targets = append(tg.Targets, endpointLabels) + targetsMu.Unlock() + } + + case NodeTypeController: + labels[mskLabelNodeType] = model.LabelValue(NodeTypeController) + + for idx, endpoint := range node.ControllerNodeInfo.Endpoints { + endpointLabels := labels.Clone() + endpointLabels[mskLabelControllerEndpointIndex] = model.LabelValue(strconv.Itoa(idx)) + endpointLabels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(endpoint, strconv.Itoa(d.cfg.Port))) + + targetsMu.Lock() + tg.Targets = append(tg.Targets, endpointLabels) + targetsMu.Unlock() + } + default: + continue + } + } + }(cluster, clusterNodeMap[aws.ToString(cluster.ClusterArn)]) + } + wg.Wait() + + return []*targetgroup.Group{tg}, nil +} + +func nodeType(node types.NodeInfo) NodeType { + if node.BrokerNodeInfo != nil { + return NodeTypeBroker + } else if node.ControllerNodeInfo != nil { + return NodeTypeController + } + return "" +} diff --git a/discovery/aws/msk_test.go b/discovery/aws/msk_test.go new file mode 100644 index 00000000000..b8c1dabec72 --- /dev/null +++ b/discovery/aws/msk_test.go @@ -0,0 +1,1136 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "fmt" + "sort" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/kafka" + "github.com/aws/aws-sdk-go-v2/service/kafka/types" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +// Struct for test data. +type mskDataStore struct { + region string + clusters []types.Cluster + nodes map[string][]types.NodeInfo // keyed by cluster ARN +} + +func TestMSKDiscoveryListClusters(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + mskData *mskDataStore + expected []types.Cluster + }{ + { + name: "MultipleClusters", + mskData: &mskDataStore{ + region: "us-west-2", + clusters: []types.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + { + ClusterName: strptr("prod-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/prod-cluster/def-456"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + }, + }, + expected: []types.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + { + ClusterName: strptr("prod-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/prod-cluster/def-456"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + }, + }, + { + name: "SingleCluster", + mskData: &mskDataStore{ + region: "us-east-1", + clusters: []types.Cluster{ + { + ClusterName: strptr("single-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-east-1:123456789012:cluster/single-cluster/xyz-789"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + }, + }, + expected: []types.Cluster{ + { + ClusterName: strptr("single-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-east-1:123456789012:cluster/single-cluster/xyz-789"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + }, + }, + { + name: "NoClusters", + mskData: &mskDataStore{ + region: "us-east-1", + clusters: []types.Cluster{}, + }, + expected: nil, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockMSKClient(tt.mskData) + + d := &MSKDiscovery{ + msk: client, + cfg: &MSKSDConfig{ + Region: tt.mskData.region, + }, + } + + clusters, err := d.listClusters(ctx) + require.NoError(t, err) + require.Equal(t, tt.expected, clusters) + }) + } +} + +func TestMSKDiscoveryDescribeClusters(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + mskData *mskDataStore + clusterARNs []string + expected []types.Cluster + }{ + { + name: "SingleCluster", + mskData: &mskDataStore{ + region: "us-west-2", + clusters: []types.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + CurrentVersion: strptr("1.2.3"), + Tags: map[string]string{ + "Environment": "production", + "Team": "platform", + }, + }, + }, + }, + clusterARNs: []string{"arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"}, + expected: []types.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + CurrentVersion: strptr("1.2.3"), + Tags: map[string]string{ + "Environment": "production", + "Team": "platform", + }, + }, + }, + }, + { + name: "MultipleClusters", + mskData: &mskDataStore{ + region: "us-east-1", + clusters: []types.Cluster{ + { + ClusterName: strptr("cluster-1"), + ClusterArn: strptr("arn:aws:kafka:us-east-1:123456789012:cluster/cluster-1/xyz-789"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + { + ClusterName: strptr("cluster-2"), + ClusterArn: strptr("arn:aws:kafka:us-east-1:123456789012:cluster/cluster-2/def-456"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + Tags: map[string]string{ + "Stage": "prod", + }, + }, + }, + }, + clusterARNs: []string{ + "arn:aws:kafka:us-east-1:123456789012:cluster/cluster-1/xyz-789", + "arn:aws:kafka:us-east-1:123456789012:cluster/cluster-2/def-456", + }, + expected: []types.Cluster{ + { + ClusterName: strptr("cluster-1"), + ClusterArn: strptr("arn:aws:kafka:us-east-1:123456789012:cluster/cluster-1/xyz-789"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + }, + { + ClusterName: strptr("cluster-2"), + ClusterArn: strptr("arn:aws:kafka:us-east-1:123456789012:cluster/cluster-2/def-456"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + Tags: map[string]string{ + "Stage": "prod", + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockMSKClient(tt.mskData) + + d := &MSKDiscovery{ + msk: client, + cfg: &MSKSDConfig{ + Region: tt.mskData.region, + RequestConcurrency: 10, + }, + } + + clusters, err := d.describeClusters(ctx, tt.clusterARNs) + require.NoError(t, err) + + // Sort clusters by ARN to handle non-deterministic ordering from goroutines + sort.Slice(clusters, func(i, j int) bool { + return aws.ToString(clusters[i].ClusterArn) < aws.ToString(clusters[j].ClusterArn) + }) + sort.Slice(tt.expected, func(i, j int) bool { + return aws.ToString(tt.expected[i].ClusterArn) < aws.ToString(tt.expected[j].ClusterArn) + }) + + require.Equal(t, tt.expected, clusters) + }) + } +} + +func TestMSKDiscoveryListNodes(t *testing.T) { + t.Parallel() + ctx := context.Background() + + for _, tt := range []struct { + name string + mskData *mskDataStore + clusters []types.Cluster + expected map[string][]types.NodeInfo + }{ + { + name: "ClusterWithBrokers", + mskData: &mskDataStore{ + region: "us-west-2", + nodes: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + AddedToClusterTime: strptr("2023-01-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(1), + ClientSubnet: strptr("subnet-12345"), + ClientVpcIpAddress: strptr("10.0.1.100"), + Endpoints: []string{"b-1.test-cluster.abc123.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-12345"), + }, + }, + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-2"), + AddedToClusterTime: strptr("2023-01-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(2), + ClientSubnet: strptr("subnet-67890"), + ClientVpcIpAddress: strptr("10.0.1.101"), + Endpoints: []string{"b-2.test-cluster.abc123.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-67890"), + }, + }, + }, + }, + }, + clusters: []types.Cluster{ + { + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"), + }, + }, + expected: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + AddedToClusterTime: strptr("2023-01-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(1), + ClientSubnet: strptr("subnet-12345"), + ClientVpcIpAddress: strptr("10.0.1.100"), + Endpoints: []string{"b-1.test-cluster.abc123.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-12345"), + }, + }, + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-2"), + AddedToClusterTime: strptr("2023-01-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(2), + ClientSubnet: strptr("subnet-67890"), + ClientVpcIpAddress: strptr("10.0.1.101"), + Endpoints: []string{"b-2.test-cluster.abc123.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-67890"), + }, + }, + }, + }, + }, + { + name: "ClusterWithNoNodes", + mskData: &mskDataStore{ + region: "us-west-2", + nodes: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/empty-cluster/xyz-789": {}, + }, + }, + clusters: []types.Cluster{ + { + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/empty-cluster/xyz-789"), + }, + }, + expected: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/empty-cluster/xyz-789": nil, + }, + }, + { + name: "MultipleClusters", + mskData: &mskDataStore{ + region: "us-west-2", + nodes: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/cluster-1/abc-123": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(1), + }, + }, + }, + "arn:aws:kafka:us-west-2:123456789012:cluster/cluster-2/def-456": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-2"), + InstanceType: strptr("kafka.m5.xlarge"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(2), + }, + }, + }, + }, + }, + clusters: []types.Cluster{ + { + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/cluster-1/abc-123"), + }, + { + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/cluster-2/def-456"), + }, + }, + expected: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/cluster-1/abc-123": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(1), + }, + }, + }, + "arn:aws:kafka:us-west-2:123456789012:cluster/cluster-2/def-456": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-2"), + InstanceType: strptr("kafka.m5.xlarge"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(2), + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + client := newMockMSKClient(tt.mskData) + + d := &MSKDiscovery{ + msk: client, + cfg: &MSKSDConfig{ + Region: tt.mskData.region, + RequestConcurrency: 10, + }, + } + + nodes, err := d.listNodes(ctx, tt.clusters) + require.NoError(t, err) + require.Equal(t, tt.expected, nodes) + }) + } +} + +func TestMSKDiscoveryRefresh(t *testing.T) { + t.Parallel() + ctx := context.Background() + + tests := []struct { + name string + mskData *mskDataStore + config *MSKSDConfig + expected []*targetgroup.Group + }{ + { + name: "ClusterWithBrokersUsingClustersConfig", + mskData: &mskDataStore{ + region: "us-west-2", + clusters: []types.Cluster{ + { + ClusterName: strptr("test-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + CurrentVersion: strptr("1.2.3"), + Tags: map[string]string{ + "Environment": "production", + "Team": "platform", + }, + Provisioned: &types.Provisioned{ + CurrentBrokerSoftwareInfo: &types.BrokerSoftwareInfo{ + ConfigurationArn: strptr("arn:aws:kafka:us-west-2:123456789012:configuration/my-config/abc-123"), + ConfigurationRevision: aws.Int64(1), + KafkaVersion: strptr("2.8.1"), + }, + OpenMonitoring: &types.OpenMonitoringInfo{ + Prometheus: &types.PrometheusInfo{ + JmxExporter: &types.JmxExporterInfo{ + EnabledInBroker: aws.Bool(true), + }, + NodeExporter: &types.NodeExporterInfo{ + EnabledInBroker: aws.Bool(true), + }, + }, + }, + }, + }, + }, + nodes: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + AddedToClusterTime: strptr("2023-01-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(1), + ClientSubnet: strptr("subnet-12345"), + ClientVpcIpAddress: strptr("10.0.1.100"), + Endpoints: []string{"b-1.test-cluster.abc123.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-12345"), + }, + }, + }, + }, + }, + config: &MSKSDConfig{ + Region: "us-west-2", + Port: 80, + RequestConcurrency: 10, + Clusters: []string{"arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"}, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("b-1.test-cluster.abc123.kafka.us-west-2.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("test-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:cluster/test-cluster/abc-123"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("1.2.3"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:configuration/my-config/abc-123"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("2.8.1"), + "__meta_msk_cluster_tag_Environment": model.LabelValue("production"), + "__meta_msk_cluster_tag_Team": model.LabelValue("platform"), + "__meta_msk_node_type": model.LabelValue("BROKER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + "__meta_msk_node_added_time": model.LabelValue("2023-01-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_node_attached_eni": model.LabelValue("eni-12345"), + "__meta_msk_broker_id": model.LabelValue("1"), + "__meta_msk_broker_client_subnet": model.LabelValue("subnet-12345"), + "__meta_msk_broker_client_vpc_ip": model.LabelValue("10.0.1.100"), + "__meta_msk_broker_node_exporter_enabled": model.LabelValue("true"), + "__meta_msk_broker_endpoint_index": model.LabelValue("0"), + }, + }, + }, + }, + }, + { + name: "NoClustersWithEmptyClustersConfig", + mskData: &mskDataStore{ + region: "us-east-1", + clusters: []types.Cluster{}, + }, + config: &MSKSDConfig{ + Region: "us-east-1", + Port: 80, + RequestConcurrency: 10, + Clusters: []string{}, // Empty clusters list uses listClusters + }, + expected: []*targetgroup.Group{ + { + Source: "us-east-1", + }, + }, + }, + { + name: "ClusterWithBrokersUsingListClusters", + mskData: &mskDataStore{ + region: "us-west-2", + clusters: []types.Cluster{ + { + ClusterName: strptr("auto-discovered-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/auto-discovered-cluster/xyz-123"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + CurrentVersion: strptr("1.0.0"), + Provisioned: &types.Provisioned{ + CurrentBrokerSoftwareInfo: &types.BrokerSoftwareInfo{ + ConfigurationArn: strptr("arn:aws:kafka:us-west-2:123456789012:configuration/config/xyz"), + ConfigurationRevision: aws.Int64(1), + KafkaVersion: strptr("3.3.1"), + }, + OpenMonitoring: &types.OpenMonitoringInfo{ + Prometheus: &types.PrometheusInfo{ + JmxExporter: &types.JmxExporterInfo{ + EnabledInBroker: aws.Bool(true), + }, + NodeExporter: &types.NodeExporterInfo{ + EnabledInBroker: aws.Bool(true), + }, + }, + }, + }, + }, + }, + nodes: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/auto-discovered-cluster/xyz-123": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-auto"), + AddedToClusterTime: strptr("2023-01-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(1), + ClientSubnet: strptr("subnet-auto"), + ClientVpcIpAddress: strptr("10.0.1.200"), + Endpoints: []string{"b-auto.cluster.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-auto"), + }, + }, + }, + }, + }, + config: &MSKSDConfig{ + Region: "us-west-2", + Port: 80, + RequestConcurrency: 10, + Clusters: nil, // nil clusters list uses listClusters (backward compatibility) + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("b-auto.cluster.kafka.us-west-2.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("auto-discovered-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:cluster/auto-discovered-cluster/xyz-123"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("1.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:configuration/config/xyz"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.3.1"), + "__meta_msk_node_type": model.LabelValue("BROKER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:node/broker-auto"), + "__meta_msk_node_added_time": model.LabelValue("2023-01-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_node_attached_eni": model.LabelValue("eni-auto"), + "__meta_msk_broker_id": model.LabelValue("1"), + "__meta_msk_broker_client_subnet": model.LabelValue("subnet-auto"), + "__meta_msk_broker_client_vpc_ip": model.LabelValue("10.0.1.200"), + "__meta_msk_broker_node_exporter_enabled": model.LabelValue("true"), + "__meta_msk_broker_endpoint_index": model.LabelValue("0"), + }, + }, + }, + }, + }, + { + name: "ClusterWithBrokersAndControllersUsingClustersConfig", + mskData: &mskDataStore{ + region: "us-west-2", + clusters: []types.Cluster{ + { + ClusterName: strptr("kraft-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-west-2:123456789012:cluster/kraft-cluster/xyz-789"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + CurrentVersion: strptr("1.0.0"), + Tags: map[string]string{ + "Type": "kraft", + }, + Provisioned: &types.Provisioned{ + CurrentBrokerSoftwareInfo: &types.BrokerSoftwareInfo{ + ConfigurationArn: strptr("arn:aws:kafka:us-west-2:123456789012:configuration/config/xyz"), + ConfigurationRevision: aws.Int64(2), + KafkaVersion: strptr("3.3.1"), + }, + OpenMonitoring: &types.OpenMonitoringInfo{ + Prometheus: &types.PrometheusInfo{ + JmxExporter: &types.JmxExporterInfo{ + EnabledInBroker: aws.Bool(true), + }, + NodeExporter: &types.NodeExporterInfo{ + EnabledInBroker: aws.Bool(false), + }, + }, + }, + }, + }, + }, + nodes: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-west-2:123456789012:cluster/kraft-cluster/xyz-789": { + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + AddedToClusterTime: strptr("2023-06-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(1), + ClientSubnet: strptr("subnet-abc123"), + ClientVpcIpAddress: strptr("10.0.2.100"), + Endpoints: []string{"b-1.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-broker-1"), + }, + }, + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/broker-2"), + AddedToClusterTime: strptr("2023-06-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(2), + ClientSubnet: strptr("subnet-abc124"), + ClientVpcIpAddress: strptr("10.0.2.101"), + Endpoints: []string{"b-2.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com"}, + AttachedENIId: strptr("eni-broker-2"), + }, + }, + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/controller-1"), + AddedToClusterTime: strptr("2023-06-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + ControllerNodeInfo: &types.ControllerNodeInfo{ + Endpoints: []string{"c-1.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com"}, + }, + }, + { + NodeARN: strptr("arn:aws:kafka:us-west-2:123456789012:node/controller-2"), + AddedToClusterTime: strptr("2023-06-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + ControllerNodeInfo: &types.ControllerNodeInfo{ + Endpoints: []string{"c-2.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com"}, + }, + }, + }, + }, + }, + config: &MSKSDConfig{ + Region: "us-west-2", + Port: 80, + RequestConcurrency: 10, + Clusters: []string{"arn:aws:kafka:us-west-2:123456789012:cluster/kraft-cluster/xyz-789"}, + }, + expected: []*targetgroup.Group{ + { + Source: "us-west-2", + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("b-1.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("kraft-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:cluster/kraft-cluster/xyz-789"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("1.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:configuration/config/xyz"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("2"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.3.1"), + "__meta_msk_cluster_tag_Type": model.LabelValue("kraft"), + "__meta_msk_node_type": model.LabelValue("BROKER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:node/broker-1"), + "__meta_msk_node_added_time": model.LabelValue("2023-06-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_node_attached_eni": model.LabelValue("eni-broker-1"), + "__meta_msk_broker_id": model.LabelValue("1"), + "__meta_msk_broker_client_subnet": model.LabelValue("subnet-abc123"), + "__meta_msk_broker_client_vpc_ip": model.LabelValue("10.0.2.100"), + "__meta_msk_broker_node_exporter_enabled": model.LabelValue("false"), + "__meta_msk_broker_endpoint_index": model.LabelValue("0"), + }, + { + model.AddressLabel: model.LabelValue("b-2.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("kraft-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:cluster/kraft-cluster/xyz-789"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("1.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:configuration/config/xyz"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("2"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.3.1"), + "__meta_msk_cluster_tag_Type": model.LabelValue("kraft"), + "__meta_msk_node_type": model.LabelValue("BROKER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:node/broker-2"), + "__meta_msk_node_added_time": model.LabelValue("2023-06-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_node_attached_eni": model.LabelValue("eni-broker-2"), + "__meta_msk_broker_id": model.LabelValue("2"), + "__meta_msk_broker_client_subnet": model.LabelValue("subnet-abc124"), + "__meta_msk_broker_client_vpc_ip": model.LabelValue("10.0.2.101"), + "__meta_msk_broker_node_exporter_enabled": model.LabelValue("false"), + "__meta_msk_broker_endpoint_index": model.LabelValue("0"), + }, + { + model.AddressLabel: model.LabelValue("c-1.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("kraft-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:cluster/kraft-cluster/xyz-789"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("1.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:configuration/config/xyz"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("2"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.3.1"), + "__meta_msk_cluster_tag_Type": model.LabelValue("kraft"), + "__meta_msk_node_type": model.LabelValue("CONTROLLER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:node/controller-1"), + "__meta_msk_node_added_time": model.LabelValue("2023-06-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_controller_endpoint_index": model.LabelValue("0"), + }, + { + model.AddressLabel: model.LabelValue("c-2.kraft-cluster.xyz789.kafka.us-west-2.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("kraft-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:cluster/kraft-cluster/xyz-789"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("1.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:configuration/config/xyz"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("2"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.3.1"), + "__meta_msk_cluster_tag_Type": model.LabelValue("kraft"), + "__meta_msk_node_type": model.LabelValue("CONTROLLER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-west-2:123456789012:node/controller-2"), + "__meta_msk_node_added_time": model.LabelValue("2023-06-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_controller_endpoint_index": model.LabelValue("0"), + }, + }, + }, + }, + }, + { + name: "NodesWithMultipleEndpointsUsingClustersConfig", + mskData: &mskDataStore{ + region: "us-east-1", + clusters: []types.Cluster{ + { + ClusterName: strptr("multi-endpoint-cluster"), + ClusterArn: strptr("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + State: types.ClusterStateActive, + ClusterType: types.ClusterTypeProvisioned, + CurrentVersion: strptr("2.0.0"), + Provisioned: &types.Provisioned{ + CurrentBrokerSoftwareInfo: &types.BrokerSoftwareInfo{ + ConfigurationArn: strptr("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + ConfigurationRevision: aws.Int64(1), + KafkaVersion: strptr("3.4.0"), + }, + OpenMonitoring: &types.OpenMonitoringInfo{ + Prometheus: &types.PrometheusInfo{ + JmxExporter: &types.JmxExporterInfo{ + EnabledInBroker: aws.Bool(true), + }, + NodeExporter: &types.NodeExporterInfo{ + EnabledInBroker: aws.Bool(true), + }, + }, + }, + }, + }, + }, + nodes: map[string][]types.NodeInfo{ + "arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999": { + { + NodeARN: strptr("arn:aws:kafka:us-east-1:123456789012:node/broker-multi"), + AddedToClusterTime: strptr("2023-08-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.xlarge"), + BrokerNodeInfo: &types.BrokerNodeInfo{ + BrokerId: aws.Float64(3), + ClientSubnet: strptr("subnet-multi-1"), + ClientVpcIpAddress: strptr("10.0.3.50"), + // Multiple endpoints for this broker + Endpoints: []string{"b-3-1.cluster.kafka.us-east-1.amazonaws.com", "b-3-2.cluster.kafka.us-east-1.amazonaws.com", "b-3-3.cluster.kafka.us-east-1.amazonaws.com"}, + AttachedENIId: strptr("eni-multi-broker"), + }, + }, + { + NodeARN: strptr("arn:aws:kafka:us-east-1:123456789012:node/controller-multi"), + AddedToClusterTime: strptr("2023-08-01T00:00:00Z"), + InstanceType: strptr("kafka.m5.large"), + ControllerNodeInfo: &types.ControllerNodeInfo{ + // Multiple endpoints for this controller + Endpoints: []string{"c-1-1.cluster.kafka.us-east-1.amazonaws.com", "c-1-2.cluster.kafka.us-east-1.amazonaws.com", "c-1-3.cluster.kafka.us-east-1.amazonaws.com", "c-1-4.cluster.kafka.us-east-1.amazonaws.com"}, + }, + }, + }, + }, + }, + config: &MSKSDConfig{ + Region: "us-east-1", + Port: 80, + RequestConcurrency: 10, + Clusters: []string{"arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"}, + }, + expected: []*targetgroup.Group{ + { + Source: "us-east-1", + Targets: []model.LabelSet{ + // Broker with 3 endpoints - creates 3 targets with different endpoint indices + { + model.AddressLabel: model.LabelValue("b-3-1.cluster.kafka.us-east-1.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("multi-endpoint-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("2.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.4.0"), + "__meta_msk_node_type": model.LabelValue("BROKER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:node/broker-multi"), + "__meta_msk_node_added_time": model.LabelValue("2023-08-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.xlarge"), + "__meta_msk_node_attached_eni": model.LabelValue("eni-multi-broker"), + "__meta_msk_broker_id": model.LabelValue("3"), + "__meta_msk_broker_client_subnet": model.LabelValue("subnet-multi-1"), + "__meta_msk_broker_client_vpc_ip": model.LabelValue("10.0.3.50"), + "__meta_msk_broker_node_exporter_enabled": model.LabelValue("true"), + "__meta_msk_broker_endpoint_index": model.LabelValue("0"), + }, + { + model.AddressLabel: model.LabelValue("b-3-2.cluster.kafka.us-east-1.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("multi-endpoint-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("2.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.4.0"), + "__meta_msk_node_type": model.LabelValue("BROKER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:node/broker-multi"), + "__meta_msk_node_added_time": model.LabelValue("2023-08-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.xlarge"), + "__meta_msk_node_attached_eni": model.LabelValue("eni-multi-broker"), + "__meta_msk_broker_id": model.LabelValue("3"), + "__meta_msk_broker_client_subnet": model.LabelValue("subnet-multi-1"), + "__meta_msk_broker_client_vpc_ip": model.LabelValue("10.0.3.50"), + "__meta_msk_broker_node_exporter_enabled": model.LabelValue("true"), + "__meta_msk_broker_endpoint_index": model.LabelValue("1"), + }, + { + model.AddressLabel: model.LabelValue("b-3-3.cluster.kafka.us-east-1.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("multi-endpoint-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("2.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.4.0"), + "__meta_msk_node_type": model.LabelValue("BROKER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:node/broker-multi"), + "__meta_msk_node_added_time": model.LabelValue("2023-08-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.xlarge"), + "__meta_msk_node_attached_eni": model.LabelValue("eni-multi-broker"), + "__meta_msk_broker_id": model.LabelValue("3"), + "__meta_msk_broker_client_subnet": model.LabelValue("subnet-multi-1"), + "__meta_msk_broker_client_vpc_ip": model.LabelValue("10.0.3.50"), + "__meta_msk_broker_node_exporter_enabled": model.LabelValue("true"), + "__meta_msk_broker_endpoint_index": model.LabelValue("2"), + }, + // Controller with 4 endpoints - creates 4 targets with different endpoint indices + { + model.AddressLabel: model.LabelValue("c-1-1.cluster.kafka.us-east-1.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("multi-endpoint-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("2.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.4.0"), + "__meta_msk_node_type": model.LabelValue("CONTROLLER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:node/controller-multi"), + "__meta_msk_node_added_time": model.LabelValue("2023-08-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_controller_endpoint_index": model.LabelValue("0"), + }, + { + model.AddressLabel: model.LabelValue("c-1-2.cluster.kafka.us-east-1.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("multi-endpoint-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("2.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.4.0"), + "__meta_msk_node_type": model.LabelValue("CONTROLLER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:node/controller-multi"), + "__meta_msk_node_added_time": model.LabelValue("2023-08-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_controller_endpoint_index": model.LabelValue("1"), + }, + { + model.AddressLabel: model.LabelValue("c-1-3.cluster.kafka.us-east-1.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("multi-endpoint-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("2.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.4.0"), + "__meta_msk_node_type": model.LabelValue("CONTROLLER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:node/controller-multi"), + "__meta_msk_node_added_time": model.LabelValue("2023-08-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_controller_endpoint_index": model.LabelValue("2"), + }, + { + model.AddressLabel: model.LabelValue("c-1-4.cluster.kafka.us-east-1.amazonaws.com:80"), + "__meta_msk_cluster_name": model.LabelValue("multi-endpoint-cluster"), + "__meta_msk_cluster_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:cluster/multi-endpoint-cluster/abc-999"), + "__meta_msk_cluster_state": model.LabelValue("ACTIVE"), + "__meta_msk_cluster_type": model.LabelValue("PROVISIONED"), + "__meta_msk_cluster_version": model.LabelValue("2.0.0"), + "__meta_msk_cluster_jmx_exporter_enabled": model.LabelValue("true"), + "__meta_msk_cluster_configuration_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:configuration/config/abc"), + "__meta_msk_cluster_configuration_revision": model.LabelValue("1"), + "__meta_msk_cluster_kafka_version": model.LabelValue("3.4.0"), + "__meta_msk_node_type": model.LabelValue("CONTROLLER"), + "__meta_msk_node_arn": model.LabelValue("arn:aws:kafka:us-east-1:123456789012:node/controller-multi"), + "__meta_msk_node_added_time": model.LabelValue("2023-08-01T00:00:00Z"), + "__meta_msk_node_instance_type": model.LabelValue("kafka.m5.large"), + "__meta_msk_controller_endpoint_index": model.LabelValue("3"), + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := newMockMSKClient(tt.mskData) + + config := tt.config + if config == nil { + // Default config for backward compatibility + config = &MSKSDConfig{ + Region: tt.mskData.region, + Port: 80, + RequestConcurrency: 10, + } + } + + d := &MSKDiscovery{ + msk: client, + cfg: config, + } + + groups, err := d.refresh(ctx) + require.NoError(t, err) + + // Sort targets within each group by address to handle non-deterministic ordering from goroutines + for _, group := range groups { + if len(group.Targets) > 0 { + sort.Slice(group.Targets, func(i, j int) bool { + return string(group.Targets[i][model.AddressLabel]) < string(group.Targets[j][model.AddressLabel]) + }) + } + } + for _, group := range tt.expected { + if len(group.Targets) > 0 { + sort.Slice(group.Targets, func(i, j int) bool { + return string(group.Targets[i][model.AddressLabel]) < string(group.Targets[j][model.AddressLabel]) + }) + } + } + + require.Equal(t, tt.expected, groups) + }) + } +} + +func TestNodeType(t *testing.T) { + t.Parallel() + tests := []struct { + name string + node types.NodeInfo + expected NodeType + }{ + { + name: "BrokerNode", + node: types.NodeInfo{ + BrokerNodeInfo: &types.BrokerNodeInfo{}, + }, + expected: NodeTypeBroker, + }, + { + name: "ControllerNode", + node: types.NodeInfo{ + ControllerNodeInfo: &types.ControllerNodeInfo{}, + }, + expected: NodeTypeController, + }, + { + name: "UnknownNode", + node: types.NodeInfo{}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := nodeType(tt.node) + require.Equal(t, tt.expected, result) + }) + } +} + +// MSK client mock. +type mockMSKClient struct { + mskData mskDataStore +} + +func newMockMSKClient(mskData *mskDataStore) *mockMSKClient { + return &mockMSKClient{ + mskData: *mskData, + } +} + +func (m *mockMSKClient) DescribeClusterV2(_ context.Context, input *kafka.DescribeClusterV2Input, _ ...func(*kafka.Options)) (*kafka.DescribeClusterV2Output, error) { + inputARN := aws.ToString(input.ClusterArn) + for i := range m.mskData.clusters { + cluster := &m.mskData.clusters[i] + if aws.ToString(cluster.ClusterArn) == inputARN { + return &kafka.DescribeClusterV2Output{ + ClusterInfo: cluster, + }, nil + } + } + + return nil, fmt.Errorf("cluster not found: %s", inputARN) +} + +func (m *mockMSKClient) ListClustersV2(_ context.Context, input *kafka.ListClustersV2Input, _ ...func(*kafka.Options)) (*kafka.ListClustersV2Output, error) { + var clusters []types.Cluster + + for _, cluster := range m.mskData.clusters { + // Apply cluster name filter if specified + if input.ClusterNameFilter != nil && *input.ClusterNameFilter != "" { + if cluster.ClusterName != nil && *cluster.ClusterName != *input.ClusterNameFilter { + continue + } + } + + // Apply cluster type filter if specified + if input.ClusterTypeFilter != nil && *input.ClusterTypeFilter != "" { + if string(cluster.ClusterType) != *input.ClusterTypeFilter { + continue + } + } + + clusters = append(clusters, cluster) + } + + return &kafka.ListClustersV2Output{ + ClusterInfoList: clusters, + }, nil +} + +func (m *mockMSKClient) ListNodes(_ context.Context, input *kafka.ListNodesInput, _ ...func(*kafka.Options)) (*kafka.ListNodesOutput, error) { + clusterARN := aws.ToString(input.ClusterArn) + nodes, exists := m.mskData.nodes[clusterARN] + if !exists { + return &kafka.ListNodesOutput{ + NodeInfoList: nil, + }, nil + } + + return &kafka.ListNodesOutput{ + NodeInfoList: nodes, + }, nil +} diff --git a/discovery/aws/rds.go b/discovery/aws/rds.go new file mode 100644 index 00000000000..cb16d36f409 --- /dev/null +++ b/discovery/aws/rds.go @@ -0,0 +1,1035 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "strconv" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "golang.org/x/sync/errgroup" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + rdsLabel = model.MetaLabelPrefix + "rds_" + + // DB cluster labels. + rdsLabelCluster = rdsLabel + "cluster_" + rdsLabelClusterActivityStreamKinesisStreamName = rdsLabelCluster + "activity_stream_kinesis_stream_name" + rdsLabelClusterActivityStreamKMSKeyID = rdsLabelCluster + "activity_stream_kms_key_id" + rdsLabelClusterActivityStreamMode = rdsLabelCluster + "activity_stream_mode" + rdsLabelClusterActivityStreamStatus = rdsLabelCluster + "activity_stream_status" + rdsLabelClusterAllocatedStorage = rdsLabelCluster + "allocated_storage" + rdsLabelClusterAutoMinorVersionUpgrade = rdsLabelCluster + "auto_minor_version_upgrade" + rdsLabelClusterAutomaticRestartTime = rdsLabelCluster + "automatic_restart_time" + rdsLabelClusterAwsBackupRecoveryPointArn = rdsLabelCluster + "aws_backup_recovery_point_arn" + rdsLabelClusterBacktrackConsumedChangeRecords = rdsLabelCluster + "backtrack_consumed_change_records" + rdsLabelClusterBacktrackWindow = rdsLabelCluster + "backtrack_window" + rdsLabelClusterBackupRetentionPeriod = rdsLabelCluster + "backup_retention_period" + rdsLabelClusterCapacity = rdsLabelCluster + "capacity" + rdsLabelClusterCharacterSetName = rdsLabelCluster + "character_set_name" + rdsLabelClusterCloneGroupID = rdsLabelCluster + "clone_group_id" + rdsLabelClusterClusterCreateTime = rdsLabelCluster + "cluster_create_time" + rdsLabelClusterClusterScalabilityType = rdsLabelCluster + "cluster_scalability_type" + rdsLabelClusterCopyTagsToSnapshot = rdsLabelCluster + "copy_tags_to_snapshot" + rdsLabelClusterCrossAccountClone = rdsLabelCluster + "cross_account_clone" + rdsLabelClusterDBClusterArn = rdsLabelCluster + "arn" + rdsLabelClusterDBClusterIdentifier = rdsLabelCluster + "identifier" + rdsLabelClusterDBClusterInstanceClass = rdsLabelCluster + "instance_class" + rdsLabelClusterDBClusterParameterGroup = rdsLabelCluster + "parameter_group" + rdsLabelClusterDBSubnetGroup = rdsLabelCluster + "subnet_group" + rdsLabelClusterDBSystemID = rdsLabelCluster + "db_system_id" + rdsLabelClusterDatabaseInsightsMode = rdsLabelCluster + "database_insights_mode" + rdsLabelClusterDatabaseName = rdsLabelCluster + "database_name" + rdsLabelClusterDBClusterResourceID = rdsLabelCluster + "resource_id" + rdsLabelClusterDeletionProtection = rdsLabelCluster + "deletion_protection" + rdsLabelClusterEarliestBacktrackTime = rdsLabelCluster + "earliest_backtrack_time" + rdsLabelClusterEarliestRestorableTime = rdsLabelCluster + "earliest_restorable_time" + rdsLabelClusterEndpoint = rdsLabelCluster + "endpoint" + rdsLabelClusterEngine = rdsLabelCluster + "engine" + rdsLabelClusterEngineLifecycleSupport = rdsLabelCluster + "engine_lifecycle_support" + rdsLabelClusterEngineMode = rdsLabelCluster + "engine_mode" + rdsLabelClusterEngineVersion = rdsLabelCluster + "engine_version" + rdsLabelClusterGlobalClusterIdentifier = rdsLabelCluster + "global_cluster_identifier" + rdsLabelClusterGlobalWriteForwardingRequested = rdsLabelCluster + "global_write_forwarding_requested" + rdsLabelClusterGlobalWriteForwardingStatus = rdsLabelCluster + "global_write_forwarding_status" + rdsLabelClusterHostedZoneID = rdsLabelCluster + "hosted_zone_id" + rdsLabelClusterHTTPEndpointEnabled = rdsLabelCluster + "http_endpoint_enabled" + rdsLabelClusterIAMDatabaseAuthenticationEnabled = rdsLabelCluster + "iam_database_authentication_enabled" + rdsLabelClusterIOOptimizedNextAllowedModificationTime = rdsLabelCluster + "io_optimized_next_allowed_modification_time" + rdsLabelClusterIops = rdsLabelCluster + "iops" + rdsLabelClusterKMSKeyID = rdsLabelCluster + "kms_key_id" + rdsLabelClusterLatestRestorableTime = rdsLabelCluster + "latest_restorable_time" + rdsLabelClusterLocalWriteForwardingStatus = rdsLabelCluster + "local_write_forwarding_status" + rdsLabelClusterMasterUsername = rdsLabelCluster + "master_username" + rdsLabelClusterMonitoringInterval = rdsLabelCluster + "monitoring_interval" + rdsLabelClusterMonitoringRoleArn = rdsLabelCluster + "monitoring_role_arn" + rdsLabelClusterMultiAZ = rdsLabelCluster + "multi_az" + rdsLabelClusterNetworkType = rdsLabelCluster + "network_type" + rdsLabelClusterPercentProgress = rdsLabelCluster + "percent_progress" + rdsLabelClusterPerformanceInsightsEnabled = rdsLabelCluster + "performance_insights_enabled" + rdsLabelClusterPerformanceInsightsKMSKeyID = rdsLabelCluster + "performance_insights_kms_key_id" + rdsLabelClusterPerformanceInsightsRetentionPeriod = rdsLabelCluster + "performance_insights_retention_period" + rdsLabelClusterPort = rdsLabelCluster + "port" + rdsLabelClusterPreferredBackupWindow = rdsLabelCluster + "preferred_backup_window" + rdsLabelClusterPreferredMaintenanceWindow = rdsLabelCluster + "preferred_maintenance_window" + rdsLabelClusterPubliclyAccessible = rdsLabelCluster + "publicly_accessible" + rdsLabelClusterReaderEndpoint = rdsLabelCluster + "reader_endpoint" + rdsLabelClusterReplicationSourceIdentifier = rdsLabelCluster + "replication_source_identifier" + rdsLabelClusterServerlessV2PlatformVersion = rdsLabelCluster + "serverless_v2_platform_version" + rdsLabelClusterStatus = rdsLabelCluster + "status" + rdsLabelClusterStorageEncrypted = rdsLabelCluster + "storage_encrypted" + rdsLabelClusterStorageEncryptionType = rdsLabelCluster + "storage_encryption_type" + rdsLabelClusterStorageThroughput = rdsLabelCluster + "storage_throughput" + rdsLabelClusterStorageType = rdsLabelCluster + "storage_type" + rdsLabelClusterUpgradeRolloutOrder = rdsLabelCluster + "upgrade_rollout_order" + + // DB cluster tags - create one label per tag key, with the format: rds_cluster_tag_. + rdsLabelClusterTag = rdsLabelCluster + "tag_" + + // DB instance labels. + rdsLabelInstance = rdsLabel + "instance_" + rdsLabelInstanceIsClusterWriter = rdsLabelInstance + "is_cluster_writer" + rdsLabelInstanceActivityStreamEngineNativeAuditFieldsIncluded = rdsLabelInstance + "activity_stream_engine_native_audit_fields_included" + rdsLabelInstanceActivityStreamKinesisStreamName = rdsLabelInstance + "activity_stream_kinesis_stream_name" + rdsLabelInstanceActivityStreamKmsKeyID = rdsLabelInstance + "activity_stream_kms_key_id" + rdsLabelInstanceActivityStreamMode = rdsLabelInstance + "activity_stream_mode" + rdsLabelInstanceActivityStreamPolicyStatus = rdsLabelInstance + "activity_stream_policy_status" + rdsLabelInstanceActivityStreamStatus = rdsLabelInstance + "activity_stream_status" + rdsLabelInstanceAllocatedStorage = rdsLabelInstance + "allocated_storage" + rdsLabelInstanceAutoMinorVersionUpgrade = rdsLabelInstance + "auto_minor_version_upgrade" + rdsLabelInstanceAutomaticRestartTime = rdsLabelInstance + "automatic_restart_time" + rdsLabelInstanceAutomationMode = rdsLabelInstance + "automation_mode" + rdsLabelInstanceAvailabilityZone = rdsLabelInstance + "availability_zone" + rdsLabelInstanceAwsBackupRecoveryPointArn = rdsLabelInstance + "aws_backup_recovery_point_arn" + rdsLabelInstanceBackupRetentionPeriod = rdsLabelInstance + "backup_retention_period" + rdsLabelInstanceBackupTarget = rdsLabelInstance + "backup_target" + rdsLabelInstanceCACertificateIdentifier = rdsLabelInstance + "ca_certificate_identifier" + rdsLabelInstanceCharacterSetName = rdsLabelInstance + "character_set_name" + rdsLabelInstanceCopyTagsToSnapshot = rdsLabelInstance + "copy_tags_to_snapshot" + rdsLabelInstanceCustomIamInstanceProfile = rdsLabelInstance + "custom_iam_instance_profile" + rdsLabelInstanceCustomerOwnedIPEnabled = rdsLabelInstance + "customer_owned_ip_enabled" + rdsLabelInstanceDBClusterIdentifier = rdsLabelInstance + "db_cluster_identifier" + rdsLabelInstanceDBInstanceArn = rdsLabelInstance + "arn" + rdsLabelInstanceDBInstanceClass = rdsLabelInstance + "class" + rdsLabelInstanceDBInstanceIdentifier = rdsLabelInstance + "identifier" + rdsLabelInstanceDBInstanceStatus = rdsLabelInstance + "status" + rdsLabelInstanceDBName = rdsLabelInstance + "db_name" + rdsLabelInstanceDBSubnetGroup = rdsLabelInstance + "subnet_group" + rdsLabelInstanceDBSystemID = rdsLabelInstance + "db_system_id" + rdsLabelInstanceDatabaseInsightsMode = rdsLabelInstance + "database_insights_mode" + rdsLabelInstanceDBInstancePort = rdsLabelInstance + "port" + rdsLabelInstanceDBResourceID = rdsLabelInstance + "resource_id" + rdsLabelInstanceDedicatedLogVolume = rdsLabelInstance + "dedicated_log_volume" + rdsLabelInstanceDeletionProtection = rdsLabelInstance + "deletion_protection" + rdsLabelInstanceEndpointAddress = rdsLabelInstance + "endpoint_address" + rdsLabelInstanceEndpointHostedZoneID = rdsLabelInstance + "endpoint_hosted_zone_id" + rdsLabelInstanceEndpointPort = rdsLabelInstance + "endpoint_port" + rdsLabelInstanceEngine = rdsLabelInstance + "engine" + rdsLabelInstanceEngineLifecycleSupport = rdsLabelInstance + "engine_lifecycle_support" + rdsLabelInstanceEngineVersion = rdsLabelInstance + "engine_version" + rdsLabelInstanceEnhancedMonitoringResourceArn = rdsLabelInstance + "enhanced_monitoring_resource_arn" + rdsLabelInstanceIAMDatabaseAuthenticationEnabled = rdsLabelInstance + "iam_database_authentication_enabled" + rdsLabelInstanceInstanceCreateTime = rdsLabelInstance + "instance_create_time" + rdsLabelInstanceIops = rdsLabelInstance + "iops" + rdsLabelInstanceIsStorageConfigUpgradeAvailable = rdsLabelInstance + "is_storage_config_upgrade_available" + rdsLabelInstanceKMSKeyID = rdsLabelInstance + "kms_key_id" + rdsLabelInstanceLatestRestorableTime = rdsLabelInstance + "latest_restorable_time" + rdsLabelInstanceLicenseModel = rdsLabelInstance + "license_model" + rdsLabelInstanceListenerEndpointAddress = rdsLabelInstance + "listener_endpoint_address" + rdsLabelInstanceListenerEndpointHostedZoneID = rdsLabelInstance + "listener_endpoint_hosted_zone_id" + rdsLabelInstanceListenerEndpointPort = rdsLabelInstance + "listener_endpoint_port" + rdsLabelInstanceMasterUsername = rdsLabelInstance + "master_username" + rdsLabelInstanceMaxAllocatedStorage = rdsLabelInstance + "max_allocated_storage" + rdsLabelInstanceMonitoringInterval = rdsLabelInstance + "monitoring_interval" + rdsLabelInstanceMonitoringRoleArn = rdsLabelInstance + "monitoring_role_arn" + rdsLabelInstanceMultiAZ = rdsLabelInstance + "multi_az" + rdsLabelInstanceMultiTenant = rdsLabelInstance + "multi_tenant" + rdsLabelInstanceNcharCharacterSetName = rdsLabelInstance + "nchar_character_set_name" + rdsLabelInstanceNetworkType = rdsLabelInstance + "network_type" + rdsLabelInstancePercentProgress = rdsLabelInstance + "percent_progress" + rdsLabelInstancePerformanceInsightsEnabled = rdsLabelInstance + "performance_insights_enabled" + rdsLabelInstancePerformanceInsightsKMSKeyID = rdsLabelInstance + "performance_insights_kms_key_id" + rdsLabelInstancePerformanceInsightsRetentionPeriod = rdsLabelInstance + "performance_insights_retention_period" + rdsLabelInstancePreferredBackupWindow = rdsLabelInstance + "preferred_backup_window" + rdsLabelInstancePreferredMaintenanceWindow = rdsLabelInstance + "preferred_maintenance_window" + rdsLabelInstancePromotionTier = rdsLabelInstance + "promotion_tier" + rdsLabelInstancePubliclyAccessible = rdsLabelInstance + "publicly_accessible" + rdsLabelInstanceReadReplicaSourceDBClusterIdentifier = rdsLabelInstance + "read_replica_source_db_cluster_identifier" + rdsLabelInstanceReadReplicaSourceDBInstanceIdentifier = rdsLabelInstance + "read_replica_source_db_instance_identifier" + rdsLabelInstanceReplicaMode = rdsLabelInstance + "replica_mode" + rdsLabelInstanceResumeFullAutomationModeTime = rdsLabelInstance + "resume_full_automation_mode_time" + rdsLabelInstanceSecondaryAvailabilityZone = rdsLabelInstance + "secondary_availability_zone" + rdsLabelInstanceStorageEncrypted = rdsLabelInstance + "storage_encrypted" + rdsLabelInstanceStorageEncryptionType = rdsLabelInstance + "storage_encryption_type" + rdsLabelInstanceStorageThroughput = rdsLabelInstance + "storage_throughput" + rdsLabelInstanceStorageType = rdsLabelInstance + "storage_type" + rdsLabelInstanceStorageVolumeStatus = rdsLabelInstance + "storage_volume_status" + rdsLabelInstanceTdeCredentialArn = rdsLabelInstance + "tde_credential_arn" + rdsLabelInstanceTimezone = rdsLabelInstance + "timezone" + rdsLabelInstanceUpgradeRolloutOrder = rdsLabelInstance + "upgrade_rollout_order" + + // DB instance tags - create one label per tag key, with the format: rds_instance_tag_. + rdsLabelInstanceTag = rdsLabelInstance + "tag_" +) + +// DefaultRDSSDConfig is the default RDS SD configuration. +var DefaultRDSSDConfig = RDSSDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + RequestConcurrency: 10, + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +func init() { + discovery.RegisterConfig(&RDSSDConfig{}) +} + +// RDSSDConfig is the configuration for RDS based service discovery. +type RDSSDConfig struct { + Region string `yaml:"region"` + Endpoint string `yaml:"endpoint"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + ExternalID string `yaml:"external_id,omitempty"` + Clusters []string `yaml:"clusters,omitempty"` + Port int `yaml:"port"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Filters []*Filter `yaml:"filters"` + + RequestConcurrency int `yaml:"request_concurrency,omitempty"` + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` +} + +// NewDiscovererMetrics implements discovery.Config. +func (*RDSSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &rdsMetrics{ + refreshMetrics: rmi, + } +} + +// Name returns the name of the RDS Config. +func (*RDSSDConfig) Name() string { return "rds" } + +// NewDiscoverer returns a Discoverer for the RDS Config. +func (c *RDSSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewRDSDiscovery(c, opts) +} + +// SetDirectory joins any relative file paths with dir. +func (c *RDSSDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for the RDS Config. +func (c *RDSSDConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultRDSSDConfig + type plain RDSSDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + + c.Region, err = loadRegion(context.Background(), c.Region) + if err != nil { + return fmt.Errorf("could not determine AWS region: %w", err) + } + + return c.HTTPClientConfig.Validate() +} + +type rdsClient interface { + DescribeDBClusters(context.Context, *rds.DescribeDBClustersInput, ...func(*rds.Options)) (*rds.DescribeDBClustersOutput, error) + DescribeDBInstances(context.Context, *rds.DescribeDBInstancesInput, ...func(*rds.Options)) (*rds.DescribeDBInstancesOutput, error) +} + +// rdsClientAdapter captures only the RDS API calls AWS discovery uses as +// method-value closures, keeping the concrete *rds.Client out of any +// interface-boxed struct field. See ec2ClientAdapter for the full rationale: +// this stops the linker from retaining the entire RDS API surface (~5 MB). +type rdsClientAdapter struct { + describeDBClusters func(context.Context, *rds.DescribeDBClustersInput, ...func(*rds.Options)) (*rds.DescribeDBClustersOutput, error) + describeDBInstances func(context.Context, *rds.DescribeDBInstancesInput, ...func(*rds.Options)) (*rds.DescribeDBInstancesOutput, error) +} + +func newRDSClientAdapter(c *rds.Client) rdsClientAdapter { + return rdsClientAdapter{ + describeDBClusters: c.DescribeDBClusters, + describeDBInstances: c.DescribeDBInstances, + } +} + +func (a rdsClientAdapter) DescribeDBClusters(ctx context.Context, params *rds.DescribeDBClustersInput, optFns ...func(*rds.Options)) (*rds.DescribeDBClustersOutput, error) { + return a.describeDBClusters(ctx, params, optFns...) +} + +func (a rdsClientAdapter) DescribeDBInstances(ctx context.Context, params *rds.DescribeDBInstancesInput, optFns ...func(*rds.Options)) (*rds.DescribeDBInstancesOutput, error) { + return a.describeDBInstances(ctx, params, optFns...) +} + +// RDSDiscovery periodically performs RDS-SD requests. It implements +// the Discoverer interface. +type RDSDiscovery struct { + *refresh.Discovery + logger *slog.Logger + cfg *RDSSDConfig + rds rdsClient +} + +// NewRDSDiscovery returns a new RDSDiscovery which periodically refreshes its targets. +func NewRDSDiscovery(conf *RDSSDConfig, opts discovery.DiscovererOptions) (*RDSDiscovery, error) { + m, ok := opts.Metrics.(*rdsMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type") + } + + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() + } + d := &RDSDiscovery{ + logger: opts.Logger, + cfg: conf, + } + d.Discovery = refresh.NewDiscovery( + refresh.Options{ + Logger: opts.Logger, + Mech: "rds", + Interval: time.Duration(d.cfg.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ) + return d, nil +} + +func (d *RDSDiscovery) initRdsClient(ctx context.Context) error { + if d.rds != nil { + return nil + } + + if d.cfg.Region == "" { + return errors.New("region must be set for RDS service discovery") + } + + // Build the HTTP client from the provided HTTPClientConfig. + client, err := config.NewClientFromConfig(d.cfg.HTTPClientConfig, "rds_sd") + if err != nil { + return err + } + + // Build the AWS config with the provided region. + var configOptions []func(*awsConfig.LoadOptions) error + configOptions = append(configOptions, awsConfig.WithRegion(d.cfg.Region)) + configOptions = append(configOptions, awsConfig.WithHTTPClient(client)) + + // Only set static credentials if both access key and secret key are provided + // Otherwise, let AWS SDK use its default credential chain + if d.cfg.AccessKey != "" && d.cfg.SecretKey != "" { + credProvider := credentials.NewStaticCredentialsProvider(d.cfg.AccessKey, string(d.cfg.SecretKey), "") + configOptions = append(configOptions, awsConfig.WithCredentialsProvider(credProvider)) + } + + if d.cfg.Profile != "" { + configOptions = append(configOptions, awsConfig.WithSharedConfigProfile(d.cfg.Profile)) + } + + cfg, err := awsConfig.LoadDefaultConfig(ctx, configOptions...) + if err != nil { + d.logger.Error("Failed to create AWS config", "error", err) + return fmt.Errorf("could not create aws config: %w", err) + } + + // If the role ARN is set, assume the role to get credentials and set the credentials provider in the config. + if d.cfg.RoleARN != "" { + assumeProvider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), d.cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { + if d.cfg.ExternalID != "" { + o.ExternalID = aws.String(d.cfg.ExternalID) + } + }) + cfg.Credentials = aws.NewCredentialsCache(assumeProvider) + } + + d.rds = newRDSClientAdapter(rds.NewFromConfig(cfg, func(options *rds.Options) { + if d.cfg.Endpoint != "" { + options.BaseEndpoint = &d.cfg.Endpoint + } + options.HTTPClient = client + })) + + // Test credentials by making a simple API call + testCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + _, err = d.rds.DescribeDBClusters(testCtx, &rds.DescribeDBClustersInput{}) + if err != nil { + d.logger.Error("Failed to test RDS credentials", "error", err) + return fmt.Errorf("RDS credential test failed: %w", err) + } + + return nil +} + +func (d *RDSDiscovery) describeAllDBClusters(ctx context.Context) (map[string]types.DBCluster, error) { + dbClustersByARN := make(map[string]types.DBCluster) + var nextToken *string + + for { + output, err := d.rds.DescribeDBClusters(ctx, &rds.DescribeDBClustersInput{ + Marker: nextToken, + MaxRecords: aws.Int32(100), + }) + if err != nil { + return nil, fmt.Errorf("failed to describe DB clusters: %w", err) + } + for _, dbCluster := range output.DBClusters { + if dbCluster.DBClusterArn != nil { + dbClustersByARN[*dbCluster.DBClusterArn] = dbCluster + } + } + if output.Marker == nil { + break + } + nextToken = output.Marker + } + + return dbClustersByARN, nil +} + +func (d *RDSDiscovery) describeDBClusters(ctx context.Context, dbClusterARNS []string) (map[string]types.DBCluster, error) { + mu := &sync.Mutex{} + errg, ectx := errgroup.WithContext(ctx) + errg.SetLimit(d.cfg.RequestConcurrency) + dbClustersByARN := make(map[string]types.DBCluster) + var nextToken *string + + for _, arn := range dbClusterARNS { + errg.Go(func() error { + for { + output, err := d.rds.DescribeDBClusters(ectx, &rds.DescribeDBClustersInput{ + DBClusterIdentifier: aws.String(arn), + Marker: nextToken, + MaxRecords: aws.Int32(100), + }) + if err != nil { + return fmt.Errorf("failed to describe DB cluster %s: %w", arn, err) + } + if len(output.DBClusters) == 0 { + return fmt.Errorf("no DB cluster found for ARN %s", arn) + } + + for _, dbCluster := range output.DBClusters { + mu.Lock() + dbClustersByARN[arn] = dbCluster + mu.Unlock() + } + if output.Marker == nil { + break + } + nextToken = output.Marker + } + return nil + }) + } + return dbClustersByARN, errg.Wait() +} + +func (d *RDSDiscovery) describeDBInstances(ctx context.Context, dbClusterARN string) ([]types.DBInstance, error) { + dbInstances := []types.DBInstance{} + var nextToken *string + + filters := []types.Filter{ + { + Name: aws.String("db-cluster-id"), + Values: []string{dbClusterARN}, + }, + } + + for _, f := range d.cfg.Filters { + filters = append(filters, types.Filter{ + Name: aws.String(f.Name), + Values: f.Values, + }) + } + + for { + output, err := d.rds.DescribeDBInstances(ctx, &rds.DescribeDBInstancesInput{ + Filters: filters, + Marker: nextToken, + MaxRecords: aws.Int32(100), + }) + if err != nil { + return nil, fmt.Errorf("failed to describe DB instances for cluster ARN %s: %w", dbClusterARN, err) + } + + dbInstances = append(dbInstances, output.DBInstances...) + + if output.Marker == nil { + break + } + nextToken = output.Marker + } + return dbInstances, nil +} + +func (d *RDSDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + err := d.initRdsClient(ctx) + if err != nil { + return nil, err + } + + tg := &targetgroup.Group{ + Source: d.cfg.Region, + } + + var clusters map[string]types.DBCluster + if len(d.cfg.Clusters) == 0 { + clusters, err = d.describeAllDBClusters(ctx) + if err != nil { + return nil, fmt.Errorf("error describing all DB clusters: %w", err) + } + } else { + clusters, err = d.describeDBClusters(ctx, d.cfg.Clusters) + if err != nil { + return nil, fmt.Errorf("error describing DB clusters: %w", err) + } + } + + var ( + mu sync.Mutex + wg sync.WaitGroup + ) + for _, cluster := range clusters { + wg.Add(1) + + instances, err := d.describeDBInstances(ctx, *cluster.DBClusterArn) + if err != nil { + return nil, fmt.Errorf("error describing DB instances: %w", err) + } + + go func(cluster types.DBCluster, instances []types.DBInstance) { + defer wg.Done() + + // Build a map of instance identifiers to their IsClusterWriter status + writerMap := make(map[string]bool) + for _, member := range cluster.DBClusterMembers { + if member.DBInstanceIdentifier != nil && member.IsClusterWriter != nil { + writerMap[*member.DBInstanceIdentifier] = *member.IsClusterWriter + } + } + + for _, instance := range instances { + labels := model.LabelSet{} + + // Cluster labels + if cluster.DBClusterArn != nil { + labels[rdsLabelClusterDBClusterArn] = model.LabelValue(*cluster.DBClusterArn) + } + if cluster.DBClusterIdentifier != nil { + labels[rdsLabelClusterDBClusterIdentifier] = model.LabelValue(*cluster.DBClusterIdentifier) + } + if cluster.ActivityStreamKinesisStreamName != nil { + labels[rdsLabelClusterActivityStreamKinesisStreamName] = model.LabelValue(*cluster.ActivityStreamKinesisStreamName) + } + if cluster.ActivityStreamKmsKeyId != nil { + labels[rdsLabelClusterActivityStreamKMSKeyID] = model.LabelValue(*cluster.ActivityStreamKmsKeyId) + } + if cluster.ActivityStreamMode != "" { + labels[rdsLabelClusterActivityStreamMode] = model.LabelValue(cluster.ActivityStreamMode) + } + if cluster.ActivityStreamStatus != "" { + labels[rdsLabelClusterActivityStreamStatus] = model.LabelValue(cluster.ActivityStreamStatus) + } + if cluster.AllocatedStorage != nil { + labels[rdsLabelClusterAllocatedStorage] = model.LabelValue(strconv.Itoa(int(*cluster.AllocatedStorage))) + } + if cluster.AutoMinorVersionUpgrade != nil { + labels[rdsLabelClusterAutoMinorVersionUpgrade] = model.LabelValue(strconv.FormatBool(*cluster.AutoMinorVersionUpgrade)) + } + if cluster.AutomaticRestartTime != nil { + labels[rdsLabelClusterAutomaticRestartTime] = model.LabelValue(cluster.AutomaticRestartTime.Format(time.RFC3339)) + } + if cluster.AwsBackupRecoveryPointArn != nil { + labels[rdsLabelClusterAwsBackupRecoveryPointArn] = model.LabelValue(*cluster.AwsBackupRecoveryPointArn) + } + if cluster.BacktrackConsumedChangeRecords != nil { + labels[rdsLabelClusterBacktrackConsumedChangeRecords] = model.LabelValue(strconv.FormatInt(*cluster.BacktrackConsumedChangeRecords, 10)) + } + if cluster.BacktrackWindow != nil { + labels[rdsLabelClusterBacktrackWindow] = model.LabelValue(strconv.FormatInt(*cluster.BacktrackWindow, 10)) + } + if cluster.BackupRetentionPeriod != nil { + labels[rdsLabelClusterBackupRetentionPeriod] = model.LabelValue(strconv.Itoa(int(*cluster.BackupRetentionPeriod))) + } + if cluster.Capacity != nil { + labels[rdsLabelClusterCapacity] = model.LabelValue(strconv.Itoa(int(*cluster.Capacity))) + } + if cluster.CharacterSetName != nil { + labels[rdsLabelClusterCharacterSetName] = model.LabelValue(*cluster.CharacterSetName) + } + if cluster.CloneGroupId != nil { + labels[rdsLabelClusterCloneGroupID] = model.LabelValue(*cluster.CloneGroupId) + } + if cluster.ClusterCreateTime != nil { + labels[rdsLabelClusterClusterCreateTime] = model.LabelValue(cluster.ClusterCreateTime.Format(time.RFC3339)) + } + if cluster.ClusterScalabilityType != "" { + labels[rdsLabelClusterClusterScalabilityType] = model.LabelValue(cluster.ClusterScalabilityType) + } + if cluster.CopyTagsToSnapshot != nil { + labels[rdsLabelClusterCopyTagsToSnapshot] = model.LabelValue(strconv.FormatBool(*cluster.CopyTagsToSnapshot)) + } + if cluster.CrossAccountClone != nil { + labels[rdsLabelClusterCrossAccountClone] = model.LabelValue(strconv.FormatBool(*cluster.CrossAccountClone)) + } + if cluster.DBClusterInstanceClass != nil { + labels[rdsLabelClusterDBClusterInstanceClass] = model.LabelValue(*cluster.DBClusterInstanceClass) + } + if cluster.DBClusterParameterGroup != nil { + labels[rdsLabelClusterDBClusterParameterGroup] = model.LabelValue(*cluster.DBClusterParameterGroup) + } + if cluster.DBSubnetGroup != nil { + labels[rdsLabelClusterDBSubnetGroup] = model.LabelValue(*cluster.DBSubnetGroup) + } + if cluster.DBSystemId != nil { + labels[rdsLabelClusterDBSystemID] = model.LabelValue(*cluster.DBSystemId) + } + if cluster.DatabaseInsightsMode != "" { + labels[rdsLabelClusterDatabaseInsightsMode] = model.LabelValue(cluster.DatabaseInsightsMode) + } + if cluster.DatabaseName != nil { + labels[rdsLabelClusterDatabaseName] = model.LabelValue(*cluster.DatabaseName) + } + if cluster.DbClusterResourceId != nil { + labels[rdsLabelClusterDBClusterResourceID] = model.LabelValue(*cluster.DbClusterResourceId) + } + if cluster.DeletionProtection != nil { + labels[rdsLabelClusterDeletionProtection] = model.LabelValue(strconv.FormatBool(*cluster.DeletionProtection)) + } + if cluster.EarliestBacktrackTime != nil { + labels[rdsLabelClusterEarliestBacktrackTime] = model.LabelValue(cluster.EarliestBacktrackTime.Format(time.RFC3339)) + } + if cluster.EarliestRestorableTime != nil { + labels[rdsLabelClusterEarliestRestorableTime] = model.LabelValue(cluster.EarliestRestorableTime.Format(time.RFC3339)) + } + if cluster.Endpoint != nil { + labels[rdsLabelClusterEndpoint] = model.LabelValue(*cluster.Endpoint) + } + if cluster.Engine != nil { + labels[rdsLabelClusterEngine] = model.LabelValue(*cluster.Engine) + } + if cluster.EngineLifecycleSupport != nil { + labels[rdsLabelClusterEngineLifecycleSupport] = model.LabelValue(*cluster.EngineLifecycleSupport) + } + if cluster.EngineMode != nil { + labels[rdsLabelClusterEngineMode] = model.LabelValue(*cluster.EngineMode) + } + if cluster.EngineVersion != nil { + labels[rdsLabelClusterEngineVersion] = model.LabelValue(*cluster.EngineVersion) + } + if cluster.GlobalClusterIdentifier != nil { + labels[rdsLabelClusterGlobalClusterIdentifier] = model.LabelValue(*cluster.GlobalClusterIdentifier) + } + if cluster.GlobalWriteForwardingRequested != nil { + labels[rdsLabelClusterGlobalWriteForwardingRequested] = model.LabelValue(strconv.FormatBool(*cluster.GlobalWriteForwardingRequested)) + } + if cluster.GlobalWriteForwardingStatus != "" { + labels[rdsLabelClusterGlobalWriteForwardingStatus] = model.LabelValue(cluster.GlobalWriteForwardingStatus) + } + if cluster.HostedZoneId != nil { + labels[rdsLabelClusterHostedZoneID] = model.LabelValue(*cluster.HostedZoneId) + } + if cluster.HttpEndpointEnabled != nil { + labels[rdsLabelClusterHTTPEndpointEnabled] = model.LabelValue(strconv.FormatBool(*cluster.HttpEndpointEnabled)) + } + if cluster.IAMDatabaseAuthenticationEnabled != nil { + labels[rdsLabelClusterIAMDatabaseAuthenticationEnabled] = model.LabelValue(strconv.FormatBool(*cluster.IAMDatabaseAuthenticationEnabled)) + } + if cluster.IOOptimizedNextAllowedModificationTime != nil { + labels[rdsLabelClusterIOOptimizedNextAllowedModificationTime] = model.LabelValue(cluster.IOOptimizedNextAllowedModificationTime.Format(time.RFC3339)) + } + if cluster.Iops != nil { + labels[rdsLabelClusterIops] = model.LabelValue(strconv.Itoa(int(*cluster.Iops))) + } + if cluster.KmsKeyId != nil { + labels[rdsLabelClusterKMSKeyID] = model.LabelValue(*cluster.KmsKeyId) + } + if cluster.LatestRestorableTime != nil { + labels[rdsLabelClusterLatestRestorableTime] = model.LabelValue(cluster.LatestRestorableTime.Format(time.RFC3339)) + } + if cluster.LocalWriteForwardingStatus != "" { + labels[rdsLabelClusterLocalWriteForwardingStatus] = model.LabelValue(cluster.LocalWriteForwardingStatus) + } + if cluster.MasterUsername != nil { + labels[rdsLabelClusterMasterUsername] = model.LabelValue(*cluster.MasterUsername) + } + if cluster.MonitoringInterval != nil { + labels[rdsLabelClusterMonitoringInterval] = model.LabelValue(strconv.Itoa(int(*cluster.MonitoringInterval))) + } + if cluster.MonitoringRoleArn != nil { + labels[rdsLabelClusterMonitoringRoleArn] = model.LabelValue(*cluster.MonitoringRoleArn) + } + if cluster.MultiAZ != nil { + labels[rdsLabelClusterMultiAZ] = model.LabelValue(strconv.FormatBool(*cluster.MultiAZ)) + } + if cluster.NetworkType != nil { + labels[rdsLabelClusterNetworkType] = model.LabelValue(*cluster.NetworkType) + } + if cluster.PercentProgress != nil { + labels[rdsLabelClusterPercentProgress] = model.LabelValue(*cluster.PercentProgress) + } + if cluster.PerformanceInsightsEnabled != nil { + labels[rdsLabelClusterPerformanceInsightsEnabled] = model.LabelValue(strconv.FormatBool(*cluster.PerformanceInsightsEnabled)) + } + if cluster.PerformanceInsightsKMSKeyId != nil { + labels[rdsLabelClusterPerformanceInsightsKMSKeyID] = model.LabelValue(*cluster.PerformanceInsightsKMSKeyId) + } + if cluster.PerformanceInsightsRetentionPeriod != nil { + labels[rdsLabelClusterPerformanceInsightsRetentionPeriod] = model.LabelValue(strconv.Itoa(int(*cluster.PerformanceInsightsRetentionPeriod))) + } + if cluster.Port != nil { + labels[rdsLabelClusterPort] = model.LabelValue(strconv.Itoa(int(*cluster.Port))) + } + if cluster.PreferredBackupWindow != nil { + labels[rdsLabelClusterPreferredBackupWindow] = model.LabelValue(*cluster.PreferredBackupWindow) + } + if cluster.PreferredMaintenanceWindow != nil { + labels[rdsLabelClusterPreferredMaintenanceWindow] = model.LabelValue(*cluster.PreferredMaintenanceWindow) + } + if cluster.PubliclyAccessible != nil { + labels[rdsLabelClusterPubliclyAccessible] = model.LabelValue(strconv.FormatBool(*cluster.PubliclyAccessible)) + } + if cluster.ReaderEndpoint != nil { + labels[rdsLabelClusterReaderEndpoint] = model.LabelValue(*cluster.ReaderEndpoint) + } + if cluster.ReplicationSourceIdentifier != nil { + labels[rdsLabelClusterReplicationSourceIdentifier] = model.LabelValue(*cluster.ReplicationSourceIdentifier) + } + if cluster.ServerlessV2PlatformVersion != nil { + labels[rdsLabelClusterServerlessV2PlatformVersion] = model.LabelValue(*cluster.ServerlessV2PlatformVersion) + } + if cluster.Status != nil { + labels[rdsLabelClusterStatus] = model.LabelValue(*cluster.Status) + } + if cluster.StorageEncrypted != nil { + labels[rdsLabelClusterStorageEncrypted] = model.LabelValue(strconv.FormatBool(*cluster.StorageEncrypted)) + } + if cluster.StorageEncryptionType != "" { + labels[rdsLabelClusterStorageEncryptionType] = model.LabelValue(cluster.StorageEncryptionType) + } + if cluster.StorageThroughput != nil { + labels[rdsLabelClusterStorageThroughput] = model.LabelValue(strconv.Itoa(int(*cluster.StorageThroughput))) + } + if cluster.StorageType != nil { + labels[rdsLabelClusterStorageType] = model.LabelValue(*cluster.StorageType) + } + if cluster.UpgradeRolloutOrder != "" { + labels[rdsLabelClusterUpgradeRolloutOrder] = model.LabelValue(cluster.UpgradeRolloutOrder) + } + + // Cluster tags + for _, tag := range cluster.TagList { + if tag.Key != nil && tag.Value != nil { + labels[model.LabelName(rdsLabelClusterTag+strutil.SanitizeLabelName(*tag.Key))] = model.LabelValue(*tag.Value) + } + } + + // Instance labels + if instance.DBInstanceArn != nil { + labels[rdsLabelInstanceDBInstanceArn] = model.LabelValue(*instance.DBInstanceArn) + } + if instance.DBInstanceIdentifier != nil { + labels[rdsLabelInstanceDBInstanceIdentifier] = model.LabelValue(*instance.DBInstanceIdentifier) + // Set IsClusterWriter based on cluster membership information + if isWriter, found := writerMap[*instance.DBInstanceIdentifier]; found { + labels[rdsLabelInstanceIsClusterWriter] = model.LabelValue(strconv.FormatBool(isWriter)) + } + } + if instance.ActivityStreamEngineNativeAuditFieldsIncluded != nil { + labels[rdsLabelInstanceActivityStreamEngineNativeAuditFieldsIncluded] = model.LabelValue(strconv.FormatBool(*instance.ActivityStreamEngineNativeAuditFieldsIncluded)) + } + if instance.ActivityStreamKinesisStreamName != nil { + labels[rdsLabelInstanceActivityStreamKinesisStreamName] = model.LabelValue(*instance.ActivityStreamKinesisStreamName) + } + if instance.ActivityStreamKmsKeyId != nil { + labels[rdsLabelInstanceActivityStreamKmsKeyID] = model.LabelValue(*instance.ActivityStreamKmsKeyId) + } + if instance.ActivityStreamMode != "" { + labels[rdsLabelInstanceActivityStreamMode] = model.LabelValue(instance.ActivityStreamMode) + } + if instance.ActivityStreamPolicyStatus != "" { + labels[rdsLabelInstanceActivityStreamPolicyStatus] = model.LabelValue(instance.ActivityStreamPolicyStatus) + } + if instance.ActivityStreamStatus != "" { + labels[rdsLabelInstanceActivityStreamStatus] = model.LabelValue(instance.ActivityStreamStatus) + } + if instance.AllocatedStorage != nil { + labels[rdsLabelInstanceAllocatedStorage] = model.LabelValue(strconv.Itoa(int(*instance.AllocatedStorage))) + } + if instance.AutoMinorVersionUpgrade != nil { + labels[rdsLabelInstanceAutoMinorVersionUpgrade] = model.LabelValue(strconv.FormatBool(*instance.AutoMinorVersionUpgrade)) + } + if instance.AutomaticRestartTime != nil { + labels[rdsLabelInstanceAutomaticRestartTime] = model.LabelValue(instance.AutomaticRestartTime.Format(time.RFC3339)) + } + if instance.AutomationMode != "" { + labels[rdsLabelInstanceAutomationMode] = model.LabelValue(instance.AutomationMode) + } + if instance.AvailabilityZone != nil { + labels[rdsLabelInstanceAvailabilityZone] = model.LabelValue(*instance.AvailabilityZone) + } + if instance.AwsBackupRecoveryPointArn != nil { + labels[rdsLabelInstanceAwsBackupRecoveryPointArn] = model.LabelValue(*instance.AwsBackupRecoveryPointArn) + } + if instance.BackupRetentionPeriod != nil { + labels[rdsLabelInstanceBackupRetentionPeriod] = model.LabelValue(strconv.Itoa(int(*instance.BackupRetentionPeriod))) + } + if instance.BackupTarget != nil { + labels[rdsLabelInstanceBackupTarget] = model.LabelValue(*instance.BackupTarget) + } + if instance.CACertificateIdentifier != nil { + labels[rdsLabelInstanceCACertificateIdentifier] = model.LabelValue(*instance.CACertificateIdentifier) + } + if instance.CharacterSetName != nil { + labels[rdsLabelInstanceCharacterSetName] = model.LabelValue(*instance.CharacterSetName) + } + if instance.CopyTagsToSnapshot != nil { + labels[rdsLabelInstanceCopyTagsToSnapshot] = model.LabelValue(strconv.FormatBool(*instance.CopyTagsToSnapshot)) + } + if instance.CustomIamInstanceProfile != nil { + labels[rdsLabelInstanceCustomIamInstanceProfile] = model.LabelValue(*instance.CustomIamInstanceProfile) + } + if instance.CustomerOwnedIpEnabled != nil { + labels[rdsLabelInstanceCustomerOwnedIPEnabled] = model.LabelValue(strconv.FormatBool(*instance.CustomerOwnedIpEnabled)) + } + if instance.DBClusterIdentifier != nil { + labels[rdsLabelInstanceDBClusterIdentifier] = model.LabelValue(*instance.DBClusterIdentifier) + } + if instance.DBInstanceClass != nil { + labels[rdsLabelInstanceDBInstanceClass] = model.LabelValue(*instance.DBInstanceClass) + } + if instance.DBInstanceStatus != nil { + labels[rdsLabelInstanceDBInstanceStatus] = model.LabelValue(*instance.DBInstanceStatus) + } + if instance.DBName != nil { + labels[rdsLabelInstanceDBName] = model.LabelValue(*instance.DBName) + } + if instance.DbInstancePort != nil { + labels[rdsLabelInstanceDBInstancePort] = model.LabelValue(strconv.Itoa(int(*instance.DbInstancePort))) + } + if instance.DbiResourceId != nil { + labels[rdsLabelInstanceDBResourceID] = model.LabelValue(*instance.DbiResourceId) + } + if instance.DedicatedLogVolume != nil { + labels[rdsLabelInstanceDedicatedLogVolume] = model.LabelValue(strconv.FormatBool(*instance.DedicatedLogVolume)) + } + if instance.DeletionProtection != nil { + labels[rdsLabelInstanceDeletionProtection] = model.LabelValue(strconv.FormatBool(*instance.DeletionProtection)) + } + if instance.Endpoint != nil { + if instance.Endpoint.Address != nil { + labels[rdsLabelInstanceEndpointAddress] = model.LabelValue(*instance.Endpoint.Address) + } + if instance.Endpoint.HostedZoneId != nil { + labels[rdsLabelInstanceEndpointHostedZoneID] = model.LabelValue(*instance.Endpoint.HostedZoneId) + } + if instance.Endpoint.Port != nil { + labels[rdsLabelInstanceEndpointPort] = model.LabelValue(strconv.Itoa(int(*instance.Endpoint.Port))) + } + } + if instance.Engine != nil { + labels[rdsLabelInstanceEngine] = model.LabelValue(*instance.Engine) + } + if instance.EngineLifecycleSupport != nil { + labels[rdsLabelInstanceEngineLifecycleSupport] = model.LabelValue(*instance.EngineLifecycleSupport) + } + if instance.EngineVersion != nil { + labels[rdsLabelInstanceEngineVersion] = model.LabelValue(*instance.EngineVersion) + } + if instance.EnhancedMonitoringResourceArn != nil { + labels[rdsLabelInstanceEnhancedMonitoringResourceArn] = model.LabelValue(*instance.EnhancedMonitoringResourceArn) + } + if instance.IAMDatabaseAuthenticationEnabled != nil { + labels[rdsLabelInstanceIAMDatabaseAuthenticationEnabled] = model.LabelValue(strconv.FormatBool(*instance.IAMDatabaseAuthenticationEnabled)) + } + if instance.InstanceCreateTime != nil { + labels[rdsLabelInstanceInstanceCreateTime] = model.LabelValue(instance.InstanceCreateTime.Format(time.RFC3339)) + } + if instance.Iops != nil { + labels[rdsLabelInstanceIops] = model.LabelValue(strconv.Itoa(int(*instance.Iops))) + } + if instance.IsStorageConfigUpgradeAvailable != nil { + labels[rdsLabelInstanceIsStorageConfigUpgradeAvailable] = model.LabelValue(strconv.FormatBool(*instance.IsStorageConfigUpgradeAvailable)) + } + if instance.KmsKeyId != nil { + labels[rdsLabelInstanceKMSKeyID] = model.LabelValue(*instance.KmsKeyId) + } + if instance.LatestRestorableTime != nil { + labels[rdsLabelInstanceLatestRestorableTime] = model.LabelValue(instance.LatestRestorableTime.Format(time.RFC3339)) + } + if instance.LicenseModel != nil { + labels[rdsLabelInstanceLicenseModel] = model.LabelValue(*instance.LicenseModel) + } + if instance.ListenerEndpoint != nil { + if instance.ListenerEndpoint.Address != nil { + labels[rdsLabelInstanceListenerEndpointAddress] = model.LabelValue(*instance.ListenerEndpoint.Address) + } + if instance.ListenerEndpoint.HostedZoneId != nil { + labels[rdsLabelInstanceListenerEndpointHostedZoneID] = model.LabelValue(*instance.ListenerEndpoint.HostedZoneId) + } + if instance.ListenerEndpoint.Port != nil { + labels[rdsLabelInstanceListenerEndpointPort] = model.LabelValue(strconv.Itoa(int(*instance.ListenerEndpoint.Port))) + } + } + if instance.MasterUsername != nil { + labels[rdsLabelInstanceMasterUsername] = model.LabelValue(*instance.MasterUsername) + } + if instance.MaxAllocatedStorage != nil { + labels[rdsLabelInstanceMaxAllocatedStorage] = model.LabelValue(strconv.Itoa(int(*instance.MaxAllocatedStorage))) + } + if instance.MonitoringInterval != nil { + labels[rdsLabelInstanceMonitoringInterval] = model.LabelValue(strconv.Itoa(int(*instance.MonitoringInterval))) + } + if instance.MonitoringRoleArn != nil { + labels[rdsLabelInstanceMonitoringRoleArn] = model.LabelValue(*instance.MonitoringRoleArn) + } + if instance.MultiAZ != nil { + labels[rdsLabelInstanceMultiAZ] = model.LabelValue(strconv.FormatBool(*instance.MultiAZ)) + } + if instance.MultiTenant != nil { + labels[rdsLabelInstanceMultiTenant] = model.LabelValue(strconv.FormatBool(*instance.MultiTenant)) + } + if instance.NcharCharacterSetName != nil { + labels[rdsLabelInstanceNcharCharacterSetName] = model.LabelValue(*instance.NcharCharacterSetName) + } + if instance.NetworkType != nil { + labels[rdsLabelInstanceNetworkType] = model.LabelValue(*instance.NetworkType) + } + if instance.PercentProgress != nil { + labels[rdsLabelInstancePercentProgress] = model.LabelValue(*instance.PercentProgress) + } + if instance.PerformanceInsightsEnabled != nil { + labels[rdsLabelInstancePerformanceInsightsEnabled] = model.LabelValue(strconv.FormatBool(*instance.PerformanceInsightsEnabled)) + } + if instance.PerformanceInsightsKMSKeyId != nil { + labels[rdsLabelInstancePerformanceInsightsKMSKeyID] = model.LabelValue(*instance.PerformanceInsightsKMSKeyId) + } + if instance.PerformanceInsightsRetentionPeriod != nil { + labels[rdsLabelInstancePerformanceInsightsRetentionPeriod] = model.LabelValue(strconv.Itoa(int(*instance.PerformanceInsightsRetentionPeriod))) + } + if instance.PreferredBackupWindow != nil { + labels[rdsLabelInstancePreferredBackupWindow] = model.LabelValue(*instance.PreferredBackupWindow) + } + if instance.PreferredMaintenanceWindow != nil { + labels[rdsLabelInstancePreferredMaintenanceWindow] = model.LabelValue(*instance.PreferredMaintenanceWindow) + } + if instance.PromotionTier != nil { + labels[rdsLabelInstancePromotionTier] = model.LabelValue(strconv.Itoa(int(*instance.PromotionTier))) + } + if instance.PubliclyAccessible != nil { + labels[rdsLabelInstancePubliclyAccessible] = model.LabelValue(strconv.FormatBool(*instance.PubliclyAccessible)) + } + if instance.ReadReplicaSourceDBClusterIdentifier != nil { + labels[rdsLabelInstanceReadReplicaSourceDBClusterIdentifier] = model.LabelValue(*instance.ReadReplicaSourceDBClusterIdentifier) + } + if instance.ReadReplicaSourceDBInstanceIdentifier != nil { + labels[rdsLabelInstanceReadReplicaSourceDBInstanceIdentifier] = model.LabelValue(*instance.ReadReplicaSourceDBInstanceIdentifier) + } + if instance.ReplicaMode != "" { + labels[rdsLabelInstanceReplicaMode] = model.LabelValue(instance.ReplicaMode) + } + if instance.ResumeFullAutomationModeTime != nil { + labels[rdsLabelInstanceResumeFullAutomationModeTime] = model.LabelValue(instance.ResumeFullAutomationModeTime.Format(time.RFC3339)) + } + if instance.SecondaryAvailabilityZone != nil { + labels[rdsLabelInstanceSecondaryAvailabilityZone] = model.LabelValue(*instance.SecondaryAvailabilityZone) + } + if instance.StorageEncrypted != nil { + labels[rdsLabelInstanceStorageEncrypted] = model.LabelValue(strconv.FormatBool(*instance.StorageEncrypted)) + } + if instance.StorageEncryptionType != "" { + labels[rdsLabelInstanceStorageEncryptionType] = model.LabelValue(instance.StorageEncryptionType) + } + if instance.StorageThroughput != nil { + labels[rdsLabelInstanceStorageThroughput] = model.LabelValue(strconv.Itoa(int(*instance.StorageThroughput))) + } + if instance.StorageType != nil { + labels[rdsLabelInstanceStorageType] = model.LabelValue(*instance.StorageType) + } + if instance.StorageVolumeStatus != nil { + labels[rdsLabelInstanceStorageVolumeStatus] = model.LabelValue(*instance.StorageVolumeStatus) + } + if instance.TdeCredentialArn != nil { + labels[rdsLabelInstanceTdeCredentialArn] = model.LabelValue(*instance.TdeCredentialArn) + } + if instance.Timezone != nil { + labels[rdsLabelInstanceTimezone] = model.LabelValue(*instance.Timezone) + } + if instance.UpgradeRolloutOrder != "" { + labels[rdsLabelInstanceUpgradeRolloutOrder] = model.LabelValue(instance.UpgradeRolloutOrder) + } + if instance.DBSubnetGroup != nil && instance.DBSubnetGroup.DBSubnetGroupName != nil { + labels[rdsLabelInstanceDBSubnetGroup] = model.LabelValue(*instance.DBSubnetGroup.DBSubnetGroupName) + } + if instance.DBSystemId != nil { + labels[rdsLabelInstanceDBSystemID] = model.LabelValue(*instance.DBSystemId) + } + if instance.DatabaseInsightsMode != "" { + labels[rdsLabelInstanceDatabaseInsightsMode] = model.LabelValue(instance.DatabaseInsightsMode) + } + + // Instance tags + for _, tag := range instance.TagList { + if tag.Key != nil && tag.Value != nil { + labels[model.LabelName(rdsLabelInstanceTag+strutil.SanitizeLabelName(*tag.Key))] = model.LabelValue(*tag.Value) + } + } + + // Set the address label + if instance.Endpoint != nil && instance.Endpoint.Address != nil && instance.Endpoint.Port != nil { + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(*instance.Endpoint.Address, strconv.Itoa(d.cfg.Port))) + } + + mu.Lock() + tg.Targets = append(tg.Targets, labels) + mu.Unlock() + } + }(cluster, instances) + } + + wg.Wait() + return []*targetgroup.Group{tg}, nil +} diff --git a/discovery/aws/rds_test.go b/discovery/aws/rds_test.go new file mode 100644 index 00000000000..fa85a365b6f --- /dev/null +++ b/discovery/aws/rds_test.go @@ -0,0 +1,559 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "net" + "slices" + "strconv" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +// Mock RDS client for testing. +type mockRDSClient struct { + clusters map[string]types.DBCluster + instances map[string][]types.DBInstance +} + +func (m *mockRDSClient) DescribeDBClusters(_ context.Context, input *rds.DescribeDBClustersInput, _ ...func(*rds.Options)) (*rds.DescribeDBClustersOutput, error) { + var clusters []types.DBCluster + + if input.DBClusterIdentifier != nil { + // Specific cluster requested + if cluster, ok := m.clusters[*input.DBClusterIdentifier]; ok { + clusters = append(clusters, cluster) + } + } else { + // All clusters + for _, cluster := range m.clusters { + clusters = append(clusters, cluster) + } + } + + return &rds.DescribeDBClustersOutput{ + DBClusters: clusters, + }, nil +} + +func (m *mockRDSClient) DescribeDBInstances(_ context.Context, input *rds.DescribeDBInstancesInput, _ ...func(*rds.Options)) (*rds.DescribeDBInstancesOutput, error) { + var instances []types.DBInstance + + // Check if filtering by cluster + if input.Filters != nil { + for _, filter := range input.Filters { + if filter.Name != nil && *filter.Name == "db-cluster-id" { + for _, clusterID := range filter.Values { + if clusterInstances, ok := m.instances[clusterID]; ok { + instances = append(instances, clusterInstances...) + } + } + } + } + } else { + // All instances + for _, clusterInstances := range m.instances { + instances = append(instances, clusterInstances...) + } + } + + for _, filter := range input.Filters { + if filter.Name == nil || *filter.Name != "engine" { + continue + } + + var filtered []types.DBInstance + for _, inst := range instances { + if inst.Engine == nil { + continue + } + if slices.Contains(filter.Values, *inst.Engine) { + filtered = append(filtered, inst) + } + } + instances = filtered + } + + return &rds.DescribeDBInstancesOutput{ + DBInstances: instances, + }, nil +} + +func TestRDSDiscoveryRefresh(t *testing.T) { + t.Parallel() + testTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + tests := []struct { + name string + clusters map[string]types.DBCluster + instances map[string][]types.DBInstance + filters []*Filter + expectedLabels []model.LabelSet + }{ + { + name: "SingleClusterWithInstance", + clusters: map[string]types.DBCluster{ + "arn:aws:rds:us-east-1:123456789012:cluster:test-cluster": { + DBClusterArn: aws.String("arn:aws:rds:us-east-1:123456789012:cluster:test-cluster"), + DBClusterIdentifier: aws.String("test-cluster"), + Engine: aws.String("aurora-postgresql"), + EngineVersion: aws.String("15.4"), + Status: aws.String("available"), + Endpoint: aws.String("test-cluster.cluster-xyz.us-east-1.rds.amazonaws.com"), + Port: aws.Int32(5432), + MasterUsername: aws.String("admin"), + MultiAZ: aws.Bool(true), + ClusterCreateTime: aws.Time(testTime), + DBClusterMembers: []types.DBClusterMember{ + { + DBInstanceIdentifier: aws.String("test-instance-1"), + IsClusterWriter: aws.Bool(true), + }, + }, + TagList: []types.Tag{ + {Key: aws.String("Environment"), Value: aws.String("test")}, + }, + }, + }, + instances: map[string][]types.DBInstance{ + "arn:aws:rds:us-east-1:123456789012:cluster:test-cluster": { + { + DBInstanceArn: aws.String("arn:aws:rds:us-east-1:123456789012:db:test-instance-1"), + DBInstanceIdentifier: aws.String("test-instance-1"), + DBInstanceClass: aws.String("db.r5.large"), + DBInstanceStatus: aws.String("available"), + Engine: aws.String("aurora-postgresql"), + EngineVersion: aws.String("15.4"), + AvailabilityZone: aws.String("us-east-1a"), + DBClusterIdentifier: aws.String("test-cluster"), + PubliclyAccessible: aws.Bool(false), + InstanceCreateTime: aws.Time(testTime), + Endpoint: &types.Endpoint{ + Address: aws.String("test-instance-1.xyz.us-east-1.rds.amazonaws.com"), + Port: aws.Int32(5432), + HostedZoneId: aws.String("Z2R2ITUGPM61AM"), + }, + TagList: []types.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-instance")}, + }, + }, + }, + }, + expectedLabels: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("test-instance-1.xyz.us-east-1.rds.amazonaws.com:5432"), + rdsLabelClusterDBClusterArn: model.LabelValue("arn:aws:rds:us-east-1:123456789012:cluster:test-cluster"), + rdsLabelClusterDBClusterIdentifier: model.LabelValue("test-cluster"), + rdsLabelClusterEngine: model.LabelValue("aurora-postgresql"), + rdsLabelClusterEngineVersion: model.LabelValue("15.4"), + rdsLabelClusterStatus: model.LabelValue("available"), + rdsLabelClusterEndpoint: model.LabelValue("test-cluster.cluster-xyz.us-east-1.rds.amazonaws.com"), + rdsLabelClusterPort: model.LabelValue("5432"), + rdsLabelClusterMasterUsername: model.LabelValue("admin"), + rdsLabelClusterMultiAZ: model.LabelValue("true"), + rdsLabelClusterClusterCreateTime: model.LabelValue(testTime.Format(time.RFC3339)), + model.LabelName(rdsLabelClusterTag + "Environment"): model.LabelValue("test"), + rdsLabelInstanceDBInstanceArn: model.LabelValue("arn:aws:rds:us-east-1:123456789012:db:test-instance-1"), + rdsLabelInstanceDBInstanceIdentifier: model.LabelValue("test-instance-1"), + rdsLabelInstanceIsClusterWriter: model.LabelValue("true"), + rdsLabelInstanceDBInstanceClass: model.LabelValue("db.r5.large"), + rdsLabelInstanceDBInstanceStatus: model.LabelValue("available"), + rdsLabelInstanceEngine: model.LabelValue("aurora-postgresql"), + rdsLabelInstanceEngineVersion: model.LabelValue("15.4"), + rdsLabelInstanceAvailabilityZone: model.LabelValue("us-east-1a"), + rdsLabelInstanceDBClusterIdentifier: model.LabelValue("test-cluster"), + rdsLabelInstancePubliclyAccessible: model.LabelValue("false"), + rdsLabelInstanceInstanceCreateTime: model.LabelValue(testTime.Format(time.RFC3339)), + rdsLabelInstanceEndpointAddress: model.LabelValue("test-instance-1.xyz.us-east-1.rds.amazonaws.com"), + rdsLabelInstanceEndpointPort: model.LabelValue("5432"), + rdsLabelInstanceEndpointHostedZoneID: model.LabelValue("Z2R2ITUGPM61AM"), + model.LabelName(rdsLabelInstanceTag + "Name"): model.LabelValue("test-instance"), + }, + }, + }, + { + name: "MultipleInstancesInCluster", + clusters: map[string]types.DBCluster{ + "arn:aws:rds:us-west-2:123456789012:cluster:prod-cluster": { + DBClusterArn: aws.String("arn:aws:rds:us-west-2:123456789012:cluster:prod-cluster"), + DBClusterIdentifier: aws.String("prod-cluster"), + Engine: aws.String("aurora-mysql"), + EngineVersion: aws.String("8.0.mysql_aurora.3.04.0"), + Status: aws.String("available"), + DBClusterMembers: []types.DBClusterMember{ + { + DBInstanceIdentifier: aws.String("prod-instance-1"), + IsClusterWriter: aws.Bool(true), + }, + { + DBInstanceIdentifier: aws.String("prod-instance-2"), + IsClusterWriter: aws.Bool(false), + }, + }, + }, + }, + instances: map[string][]types.DBInstance{ + "arn:aws:rds:us-west-2:123456789012:cluster:prod-cluster": { + { + DBInstanceArn: aws.String("arn:aws:rds:us-west-2:123456789012:db:prod-instance-1"), + DBInstanceIdentifier: aws.String("prod-instance-1"), + DBInstanceClass: aws.String("db.r6g.xlarge"), + DBInstanceStatus: aws.String("available"), + Endpoint: &types.Endpoint{ + Address: aws.String("prod-instance-1.xyz.us-west-2.rds.amazonaws.com"), + Port: aws.Int32(3306), + }, + }, + { + DBInstanceArn: aws.String("arn:aws:rds:us-west-2:123456789012:db:prod-instance-2"), + DBInstanceIdentifier: aws.String("prod-instance-2"), + DBInstanceClass: aws.String("db.r6g.xlarge"), + DBInstanceStatus: aws.String("available"), + Endpoint: &types.Endpoint{ + Address: aws.String("prod-instance-2.xyz.us-west-2.rds.amazonaws.com"), + Port: aws.Int32(3306), + }, + }, + }, + }, + expectedLabels: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("prod-instance-1.xyz.us-west-2.rds.amazonaws.com:3306"), + rdsLabelClusterDBClusterArn: model.LabelValue("arn:aws:rds:us-west-2:123456789012:cluster:prod-cluster"), + rdsLabelClusterDBClusterIdentifier: model.LabelValue("prod-cluster"), + rdsLabelClusterEngine: model.LabelValue("aurora-mysql"), + rdsLabelClusterEngineVersion: model.LabelValue("8.0.mysql_aurora.3.04.0"), + rdsLabelClusterStatus: model.LabelValue("available"), + rdsLabelInstanceDBInstanceArn: model.LabelValue("arn:aws:rds:us-west-2:123456789012:db:prod-instance-1"), + rdsLabelInstanceDBInstanceIdentifier: model.LabelValue("prod-instance-1"), + rdsLabelInstanceIsClusterWriter: model.LabelValue("true"), + rdsLabelInstanceDBInstanceClass: model.LabelValue("db.r6g.xlarge"), + rdsLabelInstanceDBInstanceStatus: model.LabelValue("available"), + rdsLabelInstanceEndpointAddress: model.LabelValue("prod-instance-1.xyz.us-west-2.rds.amazonaws.com"), + rdsLabelInstanceEndpointPort: model.LabelValue("3306"), + }, + { + model.AddressLabel: model.LabelValue("prod-instance-2.xyz.us-west-2.rds.amazonaws.com:3306"), + rdsLabelClusterDBClusterArn: model.LabelValue("arn:aws:rds:us-west-2:123456789012:cluster:prod-cluster"), + rdsLabelClusterDBClusterIdentifier: model.LabelValue("prod-cluster"), + rdsLabelClusterEngine: model.LabelValue("aurora-mysql"), + rdsLabelClusterEngineVersion: model.LabelValue("8.0.mysql_aurora.3.04.0"), + rdsLabelClusterStatus: model.LabelValue("available"), + rdsLabelInstanceDBInstanceArn: model.LabelValue("arn:aws:rds:us-west-2:123456789012:db:prod-instance-2"), + rdsLabelInstanceDBInstanceIdentifier: model.LabelValue("prod-instance-2"), + rdsLabelInstanceIsClusterWriter: model.LabelValue("false"), + rdsLabelInstanceDBInstanceClass: model.LabelValue("db.r6g.xlarge"), + rdsLabelInstanceDBInstanceStatus: model.LabelValue("available"), + rdsLabelInstanceEndpointAddress: model.LabelValue("prod-instance-2.xyz.us-west-2.rds.amazonaws.com"), + rdsLabelInstanceEndpointPort: model.LabelValue("3306"), + }, + }, + }, + { + name: "NoInstancesInCluster", + clusters: map[string]types.DBCluster{ + "arn:aws:rds:us-west-2:123456789012:cluster:prod-cluster": { + DBClusterArn: aws.String("arn:aws:rds:us-west-2:123456789012:cluster:prod-cluster"), + DBClusterIdentifier: aws.String("prod-cluster"), + Engine: aws.String("aurora-mysql"), + EngineVersion: aws.String("8.0.mysql_aurora.3.04.0"), + Status: aws.String("available"), + DBClusterMembers: []types.DBClusterMember{}, + }, + }, + instances: map[string][]types.DBInstance{}, + expectedLabels: []model.LabelSet{}, + }, + { + name: "FiltersMatchSingleInstance", + clusters: map[string]types.DBCluster{ + "arn:aws:rds:us-east-1:123456789012:cluster:filter-cluster": { + DBClusterArn: aws.String("arn:aws:rds:us-east-1:123456789012:cluster:filter-cluster"), + DBClusterIdentifier: aws.String("filter-cluster"), + Engine: aws.String("aurora-postgresql"), + Status: aws.String("available"), + DBClusterMembers: []types.DBClusterMember{ + {DBInstanceIdentifier: aws.String("filter-instance-1"), IsClusterWriter: aws.Bool(true)}, + {DBInstanceIdentifier: aws.String("filter-instance-2"), IsClusterWriter: aws.Bool(false)}, + }, + }, + }, + instances: map[string][]types.DBInstance{ + "arn:aws:rds:us-east-1:123456789012:cluster:filter-cluster": { + { + DBInstanceArn: aws.String("arn:aws:rds:us-east-1:123456789012:db:filter-instance-1"), + DBInstanceIdentifier: aws.String("filter-instance-1"), + DBInstanceClass: aws.String("db.r6g.large"), + DBInstanceStatus: aws.String("available"), + Engine: aws.String("aurora-postgresql"), + Endpoint: &types.Endpoint{Address: aws.String("filter-instance-1.rds.amazonaws.com"), Port: aws.Int32(5432)}, + }, + { + DBInstanceArn: aws.String("arn:aws:rds:us-east-1:123456789012:db:filter-instance-2"), + DBInstanceIdentifier: aws.String("filter-instance-2"), + DBInstanceClass: aws.String("db.r6g.large"), + DBInstanceStatus: aws.String("available"), + Engine: aws.String("mysql"), + Endpoint: &types.Endpoint{Address: aws.String("filter-instance-2.rds.amazonaws.com"), Port: aws.Int32(3306)}, + }, + }, + }, + filters: []*Filter{{Name: "engine", Values: []string{"aurora-postgresql"}}}, + expectedLabels: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("filter-instance-1.rds.amazonaws.com:5432"), + rdsLabelClusterDBClusterArn: model.LabelValue("arn:aws:rds:us-east-1:123456789012:cluster:filter-cluster"), + rdsLabelClusterDBClusterIdentifier: model.LabelValue("filter-cluster"), + rdsLabelClusterEngine: model.LabelValue("aurora-postgresql"), + rdsLabelClusterStatus: model.LabelValue("available"), + rdsLabelInstanceDBInstanceArn: model.LabelValue("arn:aws:rds:us-east-1:123456789012:db:filter-instance-1"), + rdsLabelInstanceDBInstanceIdentifier: model.LabelValue("filter-instance-1"), + rdsLabelInstanceIsClusterWriter: model.LabelValue("true"), + rdsLabelInstanceDBInstanceClass: model.LabelValue("db.r6g.large"), + rdsLabelInstanceDBInstanceStatus: model.LabelValue("available"), + rdsLabelInstanceEngine: model.LabelValue("aurora-postgresql"), + rdsLabelInstanceEndpointAddress: model.LabelValue("filter-instance-1.rds.amazonaws.com"), + rdsLabelInstanceEndpointPort: model.LabelValue("5432"), + }, + }, + }, + { + name: "FiltersMatchNoInstances", + clusters: map[string]types.DBCluster{ + "arn:aws:rds:us-east-1:123456789012:cluster:filter-cluster": { + DBClusterArn: aws.String("arn:aws:rds:us-east-1:123456789012:cluster:filter-cluster"), + DBClusterIdentifier: aws.String("filter-cluster"), + Engine: aws.String("aurora-postgresql"), + Status: aws.String("available"), + }, + }, + instances: map[string][]types.DBInstance{ + "arn:aws:rds:us-east-1:123456789012:cluster:filter-cluster": { + {DBInstanceIdentifier: aws.String("filter-instance-1"), Engine: aws.String("aurora-postgresql")}, + {DBInstanceIdentifier: aws.String("filter-instance-2"), Engine: aws.String("mysql")}, + }, + }, + filters: []*Filter{{Name: "engine", Values: []string{"sqlserver-ee"}}}, + expectedLabels: []model.LabelSet{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := &mockRDSClient{ + clusters: tt.clusters, + instances: tt.instances, + } + + d := &RDSDiscovery{ + rds: mockClient, + cfg: &RDSSDConfig{ + Region: "us-east-1", + RequestConcurrency: 10, + Filters: tt.filters, + }, + } + + tg := &targetgroup.Group{} + + // Get all cluster ARNs + var clusterARNs []string + for arn := range tt.clusters { + clusterARNs = append(clusterARNs, arn) + } + + clusters, err := d.describeAllDBClusters(context.Background()) + require.NoError(t, err) + require.Len(t, clusters, len(tt.clusters)) + + instances := make(map[string][]types.DBInstance) + for _, arn := range clusterARNs { + clusterInstances, err := d.describeDBInstances(context.Background(), arn) + require.NoError(t, err) + instances[arn] = clusterInstances + } + + // Build targets like the refresh function does + for _, cluster := range clusters { + writerMap := make(map[string]bool) + for _, member := range cluster.DBClusterMembers { + if member.DBInstanceIdentifier != nil && member.IsClusterWriter != nil { + writerMap[*member.DBInstanceIdentifier] = *member.IsClusterWriter + } + } + + clusterInstances := instances[*cluster.DBClusterArn] + for _, instance := range clusterInstances { + labels := model.LabelSet{} + + // Add basic cluster labels + if cluster.DBClusterArn != nil { + labels[rdsLabelClusterDBClusterArn] = model.LabelValue(*cluster.DBClusterArn) + } + if cluster.DBClusterIdentifier != nil { + labels[rdsLabelClusterDBClusterIdentifier] = model.LabelValue(*cluster.DBClusterIdentifier) + } + if cluster.Engine != nil { + labels[rdsLabelClusterEngine] = model.LabelValue(*cluster.Engine) + } + if cluster.EngineVersion != nil { + labels[rdsLabelClusterEngineVersion] = model.LabelValue(*cluster.EngineVersion) + } + if cluster.Status != nil { + labels[rdsLabelClusterStatus] = model.LabelValue(*cluster.Status) + } + if cluster.Endpoint != nil { + labels[rdsLabelClusterEndpoint] = model.LabelValue(*cluster.Endpoint) + } + if cluster.Port != nil { + labels[rdsLabelClusterPort] = model.LabelValue(strconv.Itoa(int(*cluster.Port))) + } + if cluster.MasterUsername != nil { + labels[rdsLabelClusterMasterUsername] = model.LabelValue(*cluster.MasterUsername) + } + if cluster.MultiAZ != nil { + labels[rdsLabelClusterMultiAZ] = model.LabelValue(strconv.FormatBool(*cluster.MultiAZ)) + } + if cluster.ClusterCreateTime != nil { + labels[rdsLabelClusterClusterCreateTime] = model.LabelValue(cluster.ClusterCreateTime.Format(time.RFC3339)) + } + + // Cluster tags + for _, tag := range cluster.TagList { + if tag.Key != nil && tag.Value != nil { + labels[model.LabelName(rdsLabelClusterTag+*tag.Key)] = model.LabelValue(*tag.Value) + } + } + + // Add basic instance labels + if instance.DBInstanceArn != nil { + labels[rdsLabelInstanceDBInstanceArn] = model.LabelValue(*instance.DBInstanceArn) + } + if instance.DBInstanceIdentifier != nil { + labels[rdsLabelInstanceDBInstanceIdentifier] = model.LabelValue(*instance.DBInstanceIdentifier) + if isWriter, found := writerMap[*instance.DBInstanceIdentifier]; found { + labels[rdsLabelInstanceIsClusterWriter] = model.LabelValue(strconv.FormatBool(isWriter)) + } + } + if instance.DBInstanceClass != nil { + labels[rdsLabelInstanceDBInstanceClass] = model.LabelValue(*instance.DBInstanceClass) + } + if instance.DBInstanceStatus != nil { + labels[rdsLabelInstanceDBInstanceStatus] = model.LabelValue(*instance.DBInstanceStatus) + } + if instance.Engine != nil { + labels[rdsLabelInstanceEngine] = model.LabelValue(*instance.Engine) + } + if instance.EngineVersion != nil { + labels[rdsLabelInstanceEngineVersion] = model.LabelValue(*instance.EngineVersion) + } + if instance.AvailabilityZone != nil { + labels[rdsLabelInstanceAvailabilityZone] = model.LabelValue(*instance.AvailabilityZone) + } + if instance.DBClusterIdentifier != nil { + labels[rdsLabelInstanceDBClusterIdentifier] = model.LabelValue(*instance.DBClusterIdentifier) + } + if instance.PubliclyAccessible != nil { + labels[rdsLabelInstancePubliclyAccessible] = model.LabelValue(strconv.FormatBool(*instance.PubliclyAccessible)) + } + if instance.InstanceCreateTime != nil { + labels[rdsLabelInstanceInstanceCreateTime] = model.LabelValue(instance.InstanceCreateTime.Format(time.RFC3339)) + } + if instance.Endpoint != nil { + if instance.Endpoint.Address != nil { + labels[rdsLabelInstanceEndpointAddress] = model.LabelValue(*instance.Endpoint.Address) + } + if instance.Endpoint.Port != nil { + labels[rdsLabelInstanceEndpointPort] = model.LabelValue(strconv.Itoa(int(*instance.Endpoint.Port))) + } + if instance.Endpoint.HostedZoneId != nil { + labels[rdsLabelInstanceEndpointHostedZoneID] = model.LabelValue(*instance.Endpoint.HostedZoneId) + } + } + + // Instance tags + for _, tag := range instance.TagList { + if tag.Key != nil && tag.Value != nil { + labels[model.LabelName(rdsLabelInstanceTag+*tag.Key)] = model.LabelValue(*tag.Value) + } + } + + // Set address + if instance.Endpoint != nil && instance.Endpoint.Address != nil && instance.Endpoint.Port != nil { + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(*instance.Endpoint.Address, strconv.Itoa(int(*instance.Endpoint.Port)))) + } + + tg.Targets = append(tg.Targets, labels) + } + } + + require.Len(t, tg.Targets, len(tt.expectedLabels)) + + // Verify each expected label set is present + for _, expectedLabels := range tt.expectedLabels { + found := false + for _, target := range tg.Targets { + if target[model.AddressLabel] == expectedLabels[model.AddressLabel] { + found = true + // Check all expected labels are present with correct values + for key, expectedValue := range expectedLabels { + require.Equal(t, expectedValue, target[key], "Label %s mismatch", key) + } + break + } + } + require.True(t, found, "Expected target with address %s not found", expectedLabels[model.AddressLabel]) + } + }) + } +} + +func TestDescribeAllDBClusters(t *testing.T) { + t.Parallel() + mockClient := &mockRDSClient{ + clusters: map[string]types.DBCluster{ + "arn:aws:rds:us-east-1:123456789012:cluster:cluster-1": { + DBClusterArn: aws.String("arn:aws:rds:us-east-1:123456789012:cluster:cluster-1"), + DBClusterIdentifier: aws.String("cluster-1"), + }, + "arn:aws:rds:us-east-1:123456789012:cluster:cluster-2": { + DBClusterArn: aws.String("arn:aws:rds:us-east-1:123456789012:cluster:cluster-2"), + DBClusterIdentifier: aws.String("cluster-2"), + }, + }, + instances: map[string][]types.DBInstance{}, + } + + d := &RDSDiscovery{ + rds: mockClient, + cfg: &RDSSDConfig{ + RequestConcurrency: 10, + }, + } + + clusters, err := d.describeAllDBClusters(context.Background()) + require.NoError(t, err) + require.Len(t, clusters, 2) + require.Contains(t, clusters, "arn:aws:rds:us-east-1:123456789012:cluster:cluster-1") + require.Contains(t, clusters, "arn:aws:rds:us-east-1:123456789012:cluster:cluster-2") +} diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go index 70d95b9f3a0..30f9d88703f 100644 --- a/discovery/azure/azure.go +++ b/discovery/azure/azure.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,6 +17,7 @@ import ( "context" "errors" "fmt" + "log/slog" "math/rand" "net" "net/http" @@ -29,18 +30,17 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" cache "github.com/Code-Hex/go-generics-cache" "github.com/Code-Hex/go-generics-cache/policy/lru" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" - "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/discovery" @@ -65,23 +65,20 @@ const ( azureLabelMachineScaleSet = azureLabel + "machine_scale_set" azureLabelMachineSize = azureLabel + "machine_size" - authMethodOAuth = "OAuth" - authMethodSDK = "SDK" - authMethodManagedIdentity = "ManagedIdentity" + authMethodOAuth = "OAuth" + authMethodSDK = "SDK" + authMethodManagedIdentity = "ManagedIdentity" + authMethodWorkloadIdentity = "WorkloadIdentity" ) -var ( - userAgent = fmt.Sprintf("Prometheus/%s", version.Version) - - // DefaultSDConfig is the default Azure SD configuration. - DefaultSDConfig = SDConfig{ - Port: 80, - RefreshInterval: model.Duration(5 * time.Minute), - Environment: "AzurePublicCloud", - AuthenticationMethod: authMethodOAuth, - HTTPClientConfig: config_util.DefaultHTTPClientConfig, - } -) +// DefaultSDConfig is the default Azure SD configuration. +var DefaultSDConfig = SDConfig{ + Port: 80, + RefreshInterval: model.Duration(5 * time.Minute), + Environment: "AzurePublicCloud", + AuthenticationMethod: authMethodOAuth, + HTTPClientConfig: config_util.DefaultHTTPClientConfig, +} var environments = map[string]cloud.Configuration{ "AZURECHINACLOUD": cloud.AzureChina, @@ -132,18 +129,18 @@ func (*SDConfig) Name() string { return "azure" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } func validateAuthParam(param, name string) error { - if len(param) == 0 { + if param == "" { return fmt.Errorf("azure SD configuration requires a %s", name) } return nil } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -166,8 +163,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { } } - if c.AuthenticationMethod != authMethodOAuth && c.AuthenticationMethod != authMethodManagedIdentity && c.AuthenticationMethod != authMethodSDK { - return fmt.Errorf("unknown authentication_type %q. Supported types are %q, %q or %q", c.AuthenticationMethod, authMethodOAuth, authMethodManagedIdentity, authMethodSDK) + if c.AuthenticationMethod != authMethodOAuth && c.AuthenticationMethod != authMethodManagedIdentity && c.AuthenticationMethod != authMethodSDK && c.AuthenticationMethod != authMethodWorkloadIdentity { + return fmt.Errorf("unknown authentication_type %q. Supported types are %q, %q, %q or %q", c.AuthenticationMethod, authMethodOAuth, authMethodManagedIdentity, authMethodSDK, authMethodWorkloadIdentity) } return c.HTTPClientConfig.Validate() @@ -175,7 +172,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { type Discovery struct { *refresh.Discovery - logger log.Logger + logger *slog.Logger cfg *SDConfig port int cache *cache.Cache[string, *armnetwork.Interface] @@ -183,28 +180,29 @@ type Discovery struct { } // NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets. -func NewDiscovery(cfg *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*azureMetrics) +func NewDiscovery(cfg *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*azureMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - if logger == nil { - logger = log.NewNopLogger() + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() } l := cache.New(cache.AsLRU[string, *armnetwork.Interface](lru.WithCapacity(5000))) d := &Discovery{ cfg: cfg, port: cfg.Port, - logger: logger, + logger: opts.Logger, cache: l, metrics: m, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "azure", + SetName: opts.SetName, Interval: time.Duration(cfg.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -224,30 +222,120 @@ type client interface { // azureClient represents multiple Azure Resource Manager providers. type azureClient struct { - nic *armnetwork.InterfacesClient - vm *armcompute.VirtualMachinesClient - vmss *armcompute.VirtualMachineScaleSetsClient - vmssvm *armcompute.VirtualMachineScaleSetVMsClient - logger log.Logger + nic interfacesClientAdapter + vm virtualMachinesClientAdapter + vmss virtualMachineScaleSetsClientAdapter + vmssvm virtualMachineScaleSetVMsClientAdapter + logger *slog.Logger } var _ client = &azureClient{} -// createAzureClient is a helper function for creating an Azure compute client to ARM. -func createAzureClient(cfg SDConfig, logger log.Logger) (client, error) { - cloudConfiguration, err := CloudConfigurationFromName(cfg.Environment) +// The *ClientAdapter types below capture only the operations discovery uses as +// closures, hiding the concrete SDK clients from reflection so dead-code +// elimination can drop the unused operations and shrink the binary. + +// virtualMachinesClientAdapter adapts *armcompute.VirtualMachinesClient. +type virtualMachinesClientAdapter struct { + newListAllPager func(options *armcompute.VirtualMachinesClientListAllOptions) *runtime.Pager[armcompute.VirtualMachinesClientListAllResponse] + newListPager func(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) *runtime.Pager[armcompute.VirtualMachinesClientListResponse] +} + +func newVirtualMachinesClientAdapter(c *armcompute.VirtualMachinesClient) virtualMachinesClientAdapter { + return virtualMachinesClientAdapter{ + newListAllPager: c.NewListAllPager, + newListPager: c.NewListPager, + } +} + +// NewListAllPager lists all virtual machines in the subscription. +func (a virtualMachinesClientAdapter) NewListAllPager(options *armcompute.VirtualMachinesClientListAllOptions) *runtime.Pager[armcompute.VirtualMachinesClientListAllResponse] { + return a.newListAllPager(options) +} + +// NewListPager lists the virtual machines in a resource group. +func (a virtualMachinesClientAdapter) NewListPager(resourceGroupName string, options *armcompute.VirtualMachinesClientListOptions) *runtime.Pager[armcompute.VirtualMachinesClientListResponse] { + return a.newListPager(resourceGroupName, options) +} + +// virtualMachineScaleSetsClientAdapter adapts *armcompute.VirtualMachineScaleSetsClient. +type virtualMachineScaleSetsClientAdapter struct { + newListAllPager func(options *armcompute.VirtualMachineScaleSetsClientListAllOptions) *runtime.Pager[armcompute.VirtualMachineScaleSetsClientListAllResponse] + newListPager func(resourceGroupName string, options *armcompute.VirtualMachineScaleSetsClientListOptions) *runtime.Pager[armcompute.VirtualMachineScaleSetsClientListResponse] +} + +func newVirtualMachineScaleSetsClientAdapter(c *armcompute.VirtualMachineScaleSetsClient) virtualMachineScaleSetsClientAdapter { + return virtualMachineScaleSetsClientAdapter{ + newListAllPager: c.NewListAllPager, + newListPager: c.NewListPager, + } +} + +// NewListAllPager lists all scale sets in the subscription. +func (a virtualMachineScaleSetsClientAdapter) NewListAllPager(options *armcompute.VirtualMachineScaleSetsClientListAllOptions) *runtime.Pager[armcompute.VirtualMachineScaleSetsClientListAllResponse] { + return a.newListAllPager(options) +} + +// NewListPager lists the scale sets in a resource group. +func (a virtualMachineScaleSetsClientAdapter) NewListPager(resourceGroupName string, options *armcompute.VirtualMachineScaleSetsClientListOptions) *runtime.Pager[armcompute.VirtualMachineScaleSetsClientListResponse] { + return a.newListPager(resourceGroupName, options) +} + +// virtualMachineScaleSetVMsClientAdapter adapts *armcompute.VirtualMachineScaleSetVMsClient. +type virtualMachineScaleSetVMsClientAdapter struct { + newListPager func(resourceGroupName, virtualMachineScaleSetName string, options *armcompute.VirtualMachineScaleSetVMsClientListOptions) *runtime.Pager[armcompute.VirtualMachineScaleSetVMsClientListResponse] +} + +func newVirtualMachineScaleSetVMsClientAdapter(c *armcompute.VirtualMachineScaleSetVMsClient) virtualMachineScaleSetVMsClientAdapter { + return virtualMachineScaleSetVMsClientAdapter{ + newListPager: c.NewListPager, + } +} + +// NewListPager lists the virtual machines of a scale set. +func (a virtualMachineScaleSetVMsClientAdapter) NewListPager(resourceGroupName, virtualMachineScaleSetName string, options *armcompute.VirtualMachineScaleSetVMsClientListOptions) *runtime.Pager[armcompute.VirtualMachineScaleSetVMsClientListResponse] { + return a.newListPager(resourceGroupName, virtualMachineScaleSetName, options) +} + +// interfacesClientAdapter adapts *armnetwork.InterfacesClient. +type interfacesClientAdapter struct { + get func(ctx context.Context, resourceGroupName, networkInterfaceName string, options *armnetwork.InterfacesClientGetOptions) (armnetwork.InterfacesClientGetResponse, error) + + getVirtualMachineScaleSetNetworkInterface func(ctx context.Context, resourceGroupName, virtualMachineScaleSetName, virtualMachineIndex, networkInterfaceName string, options *armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceOptions) (armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse, error) +} + +func newInterfacesClientAdapter(c *armnetwork.InterfacesClient) interfacesClientAdapter { + return interfacesClientAdapter{ + get: c.Get, + getVirtualMachineScaleSetNetworkInterface: c.GetVirtualMachineScaleSetNetworkInterface, + } +} + +// Get retrieves a network interface. +func (a interfacesClientAdapter) Get(ctx context.Context, resourceGroupName, networkInterfaceName string, options *armnetwork.InterfacesClientGetOptions) (armnetwork.InterfacesClientGetResponse, error) { + return a.get(ctx, resourceGroupName, networkInterfaceName, options) +} + +// GetVirtualMachineScaleSetNetworkInterface retrieves a scale set VM network interface. +func (a interfacesClientAdapter) GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName, virtualMachineScaleSetName, virtualMachineIndex, networkInterfaceName string, options *armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceOptions) (armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse, error) { + return a.getVirtualMachineScaleSetNetworkInterface(ctx, resourceGroupName, virtualMachineScaleSetName, virtualMachineIndex, networkInterfaceName, options) +} + +// createAzureClient is a helper method for creating an Azure compute client to ARM. +func (d *Discovery) createAzureClient() (client, error) { + cloudConfiguration, err := CloudConfigurationFromName(d.cfg.Environment) if err != nil { return &azureClient{}, err } var c azureClient - c.logger = logger + c.logger = d.logger telemetry := policy.TelemetryOptions{ - ApplicationID: userAgent, + ApplicationID: version.PrometheusUserAgent(), } - credential, err := newCredential(cfg, policy.ClientOptions{ + credential, err := newCredential(*d.cfg, policy.ClientOptions{ Cloud: cloudConfiguration, Telemetry: telemetry, }) @@ -255,7 +343,7 @@ func createAzureClient(cfg SDConfig, logger log.Logger) (client, error) { return &azureClient{}, err } - client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "azure_sd") + client, err := config_util.NewClientFromConfig(d.cfg.HTTPClientConfig, "azure_sd") if err != nil { return &azureClient{}, err } @@ -267,25 +355,29 @@ func createAzureClient(cfg SDConfig, logger log.Logger) (client, error) { }, } - c.vm, err = armcompute.NewVirtualMachinesClient(cfg.SubscriptionID, credential, options) + vmClient, err := armcompute.NewVirtualMachinesClient(d.cfg.SubscriptionID, credential, options) if err != nil { return &azureClient{}, err } + c.vm = newVirtualMachinesClientAdapter(vmClient) - c.nic, err = armnetwork.NewInterfacesClient(cfg.SubscriptionID, credential, options) + nicClient, err := armnetwork.NewInterfacesClient(d.cfg.SubscriptionID, credential, options) if err != nil { return &azureClient{}, err } + c.nic = newInterfacesClientAdapter(nicClient) - c.vmss, err = armcompute.NewVirtualMachineScaleSetsClient(cfg.SubscriptionID, credential, options) + vmssClient, err := armcompute.NewVirtualMachineScaleSetsClient(d.cfg.SubscriptionID, credential, options) if err != nil { return &azureClient{}, err } + c.vmss = newVirtualMachineScaleSetsClientAdapter(vmssClient) - c.vmssvm, err = armcompute.NewVirtualMachineScaleSetVMsClient(cfg.SubscriptionID, credential, options) + vmssvmClient, err := armcompute.NewVirtualMachineScaleSetVMsClient(d.cfg.SubscriptionID, credential, options) if err != nil { return &azureClient{}, err } + c.vmssvm = newVirtualMachineScaleSetVMsClientAdapter(vmssvmClient) return &c, nil } @@ -293,8 +385,18 @@ func createAzureClient(cfg SDConfig, logger log.Logger) (client, error) { func newCredential(cfg SDConfig, policyClientOptions policy.ClientOptions) (azcore.TokenCredential, error) { var credential azcore.TokenCredential switch cfg.AuthenticationMethod { + case authMethodWorkloadIdentity: + options := &azidentity.WorkloadIdentityCredentialOptions{ClientOptions: policyClientOptions} + workloadIdentityCredential, err := azidentity.NewWorkloadIdentityCredential(options) + if err != nil { + return nil, err + } + credential = azcore.TokenCredential(workloadIdentityCredential) case authMethodManagedIdentity: - options := &azidentity.ManagedIdentityCredentialOptions{ClientOptions: policyClientOptions, ID: azidentity.ClientID(cfg.ClientID)} + options := &azidentity.ManagedIdentityCredentialOptions{ClientOptions: policyClientOptions} + if cfg.ClientID != "" { + options.ID = azidentity.ClientID(cfg.ClientID) + } managedIdentityCredential, err := azidentity.NewManagedIdentityCredential(options) if err != nil { return nil, err @@ -309,7 +411,7 @@ func newCredential(cfg SDConfig, policyClientOptions policy.ClientOptions) (azco credential = azcore.TokenCredential(secretCredential) case authMethodSDK: options := &azidentity.DefaultAzureCredentialOptions{ClientOptions: policyClientOptions} - if len(cfg.TenantID) != 0 { + if cfg.TenantID != "" { options.TenantID = cfg.TenantID } sdkCredential, err := azidentity.NewDefaultAzureCredential(options) @@ -337,35 +439,27 @@ type virtualMachine struct { } // Create a new azureResource object from an ID string. -func newAzureResourceFromID(id string, logger log.Logger) (*arm.ResourceID, error) { +func newAzureResourceFromID(id string, logger *slog.Logger) (*arm.ResourceID, error) { if logger == nil { - logger = log.NewNopLogger() + logger = promslog.NewNopLogger() } resourceID, err := arm.ParseResourceID(id) if err != nil { err := fmt.Errorf("invalid ID '%s': %w", id, err) - level.Error(logger).Log("err", err) + logger.Error("Failed to parse resource ID", "err", err) return &arm.ResourceID{}, err } return resourceID, nil } -func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - defer level.Debug(d.logger).Log("msg", "Azure discovery completed") - - client, err := createAzureClient(*d.cfg, d.logger) - if err != nil { - d.metrics.failuresCount.Inc() - return nil, fmt.Errorf("could not create Azure client: %w", err) - } - +func (d *Discovery) refreshAzureClient(ctx context.Context, client client) ([]*targetgroup.Group, error) { machines, err := client.getVMs(ctx, d.cfg.ResourceGroup) if err != nil { d.metrics.failuresCount.Inc() return nil, fmt.Errorf("could not get virtual machines: %w", err) } - level.Debug(d.logger).Log("msg", "Found virtual machines during Azure discovery.", "count", len(machines)) + d.logger.Debug("Found virtual machines during Azure discovery.", "count", len(machines)) // Load the vms managed by scale sets. scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup) @@ -418,6 +512,18 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { return []*targetgroup.Group{&tg}, nil } +func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + defer d.logger.Debug("Azure discovery completed") + + client, err := d.createAzureClient() + if err != nil { + d.metrics.failuresCount.Inc() + return nil, fmt.Errorf("could not create Azure client: %w", err) + } + + return d.refreshAzureClient(ctx, client) +} + func (d *Discovery) vmToLabelSet(ctx context.Context, client client, vm virtualMachine) (model.LabelSet, error) { r, err := newAzureResourceFromID(vm.ID, d.logger) if err != nil { @@ -458,11 +564,10 @@ func (d *Discovery) vmToLabelSet(ctx context.Context, client client, vm virtualM networkInterface, err = client.getVMScaleSetVMNetworkInterfaceByID(ctx, nicID, vm.ScaleSet, vm.InstanceID) } if err != nil { - if errors.Is(err, errorNotFound) { - level.Warn(d.logger).Log("msg", "Network interface does not exist", "name", nicID, "err", err) - } else { + if !errors.Is(err, errorNotFound) { return nil, err } + d.logger.Warn("Network interface does not exist", "name", nicID, "err", err) // Get out of this routine because we cannot continue without a network interface. return nil, nil } @@ -480,7 +585,7 @@ func (d *Discovery) vmToLabelSet(ctx context.Context, client client, vm virtualM // yet support this. On deallocated machines, this value happens to be nil so it // is a cheap and easy way to determine if a machine is allocated or not. if networkInterface.Properties.Primary == nil { - level.Debug(d.logger).Log("msg", "Skipping deallocated virtual machine", "machine", vm.Name) + d.logger.Debug("Skipping deallocated virtual machine", "machine", vm.Name) return nil, nil } @@ -509,7 +614,7 @@ func (d *Discovery) vmToLabelSet(ctx context.Context, client client, vm virtualM func (client *azureClient) getVMs(ctx context.Context, resourceGroup string) ([]virtualMachine, error) { var vms []virtualMachine - if len(resourceGroup) == 0 { + if resourceGroup == "" { pager := client.vm.NewListAllPager(nil) for pager.More() { nextResult, err := pager.NextPage(ctx) @@ -537,7 +642,7 @@ func (client *azureClient) getVMs(ctx context.Context, resourceGroup string) ([] func (client *azureClient) getScaleSets(ctx context.Context, resourceGroup string) ([]armcompute.VirtualMachineScaleSet, error) { var scaleSets []armcompute.VirtualMachineScaleSet - if len(resourceGroup) == 0 { + if resourceGroup == "" { pager := client.vmss.NewListAllPager(nil) for pager.More() { nextResult, err := pager.NextPage(ctx) @@ -724,7 +829,7 @@ func (d *Discovery) addToCache(nicID string, netInt *armnetwork.Interface) { rs := time.Duration(random) * time.Second exptime := time.Duration(d.cfg.RefreshInterval*10) + rs d.cache.Set(nicID, netInt, cache.WithExpiration(exptime)) - level.Debug(d.logger).Log("msg", "Adding nic", "nic", nicID, "time", exptime.Seconds()) + d.logger.Debug("Adding nic", "nic", nicID, "time", exptime.Seconds()) } // getFromCache will get the network Interface for the specified nicID diff --git a/discovery/azure/azure_test.go b/discovery/azure/azure_test.go index 32dab66c8c1..4b95055ab84 100644 --- a/discovery/azure/azure_test.go +++ b/discovery/azure/azure_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,19 +15,35 @@ package azure import ( "context" - "fmt" + "log/slog" + "net/http" + "slices" + "strings" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + azfake "github.com/Azure/azure-sdk-for-go/sdk/azcore/fake" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + fake "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5/fake" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + fakenetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4/fake" cache "github.com/Code-Hex/go-generics-cache" "github.com/Code-Hex/go-generics-cache/policy/lru" - "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "go.uber.org/goleak" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/targetgroup" ) +const defaultMockNetworkID string = "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/networkInterfaces/{networkInterfaceName}" + func TestMain(m *testing.M) { goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("github.com/Code-Hex/go-generics-cache.(*janitor).run.func1"), @@ -96,13 +112,12 @@ func TestVMToLabelSet(t *testing.T) { vmType := "type" location := "westeurope" computerName := "computer_name" - networkID := "/subscriptions/00000000-0000-0000-0000-000000000000/network1" ipAddress := "10.20.30.40" primary := true networkProfile := armcompute.NetworkProfile{ NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ { - ID: &networkID, + ID: to.Ptr(defaultMockNetworkID), Properties: &armcompute.NetworkInterfaceReferenceProperties{Primary: &primary}, }, }, @@ -139,7 +154,7 @@ func TestVMToLabelSet(t *testing.T) { Location: location, OsType: "Linux", Tags: map[string]*string{}, - NetworkInterfaces: []string{networkID}, + NetworkInterfaces: []string{defaultMockNetworkID}, Size: size, } @@ -150,11 +165,12 @@ func TestVMToLabelSet(t *testing.T) { cfg := DefaultSDConfig d := &Discovery{ cfg: &cfg, - logger: log.NewNopLogger(), + logger: promslog.NewNopLogger(), cache: cache.New(cache.AsLRU[string, *armnetwork.Interface](lru.WithCapacity(5))), } network := armnetwork.Interface{ - Name: &networkID, + Name: to.Ptr(defaultMockNetworkID), + ID: to.Ptr(defaultMockNetworkID), Properties: &armnetwork.InterfacePropertiesFormat{ Primary: &primary, IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ @@ -164,9 +180,9 @@ func TestVMToLabelSet(t *testing.T) { }, }, } - client := &mockAzureClient{ - networkInterface: &network, - } + + client := createMockAzureClient(t, nil, nil, nil, network, nil) + labelSet, err := d.vmToLabelSet(context.Background(), client, actualVM) require.NoError(t, err) require.Len(t, labelSet, 11) @@ -475,34 +491,397 @@ func TestNewAzureResourceFromID(t *testing.T) { } } +func TestNewCredentialManagedIdentity(t *testing.T) { + // Test that system-assigned managed identity (empty ClientID) creates + // a valid credential. Previously, an empty ClientID was passed as + // azidentity.ClientID("") which is not nil and caused Azure SDK to + // look up a non-existent user-assigned identity instead of falling + // back to system-assigned identity. + cfg := SDConfig{ + AuthenticationMethod: authMethodManagedIdentity, + ClientID: "", + } + cred, err := newCredential(cfg, policy.ClientOptions{}) + require.NoError(t, err) + require.NotNil(t, cred) + + // Test that user-assigned managed identity (non-empty ClientID) also works. + cfg.ClientID = "00000000-0000-0000-0000-000000000000" + cred, err = newCredential(cfg, policy.ClientOptions{}) + require.NoError(t, err) + require.NotNil(t, cred) +} + +func TestAzureRefresh(t *testing.T) { + tests := []struct { + scenario string + vmResp []armcompute.VirtualMachinesClientListAllResponse + vmssResp []armcompute.VirtualMachineScaleSetsClientListAllResponse + vmssvmResp []armcompute.VirtualMachineScaleSetVMsClientListResponse + interfacesResp armnetwork.Interface + expectedTG []*targetgroup.Group + }{ + { + scenario: "VMs, VMSS and VMSSVMs in Multiple Responses", + vmResp: []armcompute.VirtualMachinesClientListAllResponse{ + { + VirtualMachineListResult: armcompute.VirtualMachineListResult{ + Value: []*armcompute.VirtualMachine{ + defaultVMWithIDAndName(to.Ptr("/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm1"), to.Ptr("vm1")), + defaultVMWithIDAndName(to.Ptr("/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm2"), to.Ptr("vm2")), + }, + }, + }, + { + VirtualMachineListResult: armcompute.VirtualMachineListResult{ + Value: []*armcompute.VirtualMachine{ + defaultVMWithIDAndName(to.Ptr("/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm3"), to.Ptr("vm3")), + defaultVMWithIDAndName(to.Ptr("/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm4"), to.Ptr("vm4")), + }, + }, + }, + }, + vmssResp: []armcompute.VirtualMachineScaleSetsClientListAllResponse{ + { + VirtualMachineScaleSetListWithLinkResult: armcompute.VirtualMachineScaleSetListWithLinkResult{ + Value: []*armcompute.VirtualMachineScaleSet{ + { + ID: to.Ptr("/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachineScaleSets/vmScaleSet1"), + Name: to.Ptr("vmScaleSet1"), + Location: to.Ptr("australiaeast"), + Type: to.Ptr("Microsoft.Compute/virtualMachineScaleSets"), + }, + }, + }, + }, + }, + vmssvmResp: []armcompute.VirtualMachineScaleSetVMsClientListResponse{ + { + VirtualMachineScaleSetVMListResult: armcompute.VirtualMachineScaleSetVMListResult{ + Value: []*armcompute.VirtualMachineScaleSetVM{ + defaultVMSSVMWithIDAndName(to.Ptr("/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachineScaleSets/vmScaleSet1/virtualMachines/vmScaleSet1_vm1"), to.Ptr("vmScaleSet1_vm1")), + defaultVMSSVMWithIDAndName(to.Ptr("/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachineScaleSets/vmScaleSet1/virtualMachines/vmScaleSet1_vm2"), to.Ptr("vmScaleSet1_vm2")), + }, + }, + }, + }, + interfacesResp: armnetwork.Interface{ + ID: to.Ptr(defaultMockNetworkID), + Properties: &armnetwork.InterfacePropertiesFormat{ + Primary: to.Ptr(true), + IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ + {Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.Ptr("10.0.0.1"), + }}, + }, + }, + }, + expectedTG: []*targetgroup.Group{ + { + Targets: []model.LabelSet{ + { + "__address__": "10.0.0.1:80", + "__meta_azure_machine_computer_name": "computer_name", + "__meta_azure_machine_id": "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm1", + "__meta_azure_machine_location": "australiaeast", + "__meta_azure_machine_name": "vm1", + "__meta_azure_machine_os_type": "Linux", + "__meta_azure_machine_private_ip": "10.0.0.1", + "__meta_azure_machine_resource_group": "{resourceGroup}", + "__meta_azure_machine_size": "size", + "__meta_azure_machine_tag_prometheus": "", + "__meta_azure_subscription_id": "", + "__meta_azure_tenant_id": "", + }, + { + "__address__": "10.0.0.1:80", + "__meta_azure_machine_computer_name": "computer_name", + "__meta_azure_machine_id": "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm2", + "__meta_azure_machine_location": "australiaeast", + "__meta_azure_machine_name": "vm2", + "__meta_azure_machine_os_type": "Linux", + "__meta_azure_machine_private_ip": "10.0.0.1", + "__meta_azure_machine_resource_group": "{resourceGroup}", + "__meta_azure_machine_size": "size", + "__meta_azure_machine_tag_prometheus": "", + "__meta_azure_subscription_id": "", + "__meta_azure_tenant_id": "", + }, + { + "__address__": "10.0.0.1:80", + "__meta_azure_machine_computer_name": "computer_name", + "__meta_azure_machine_id": "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm3", + "__meta_azure_machine_location": "australiaeast", + "__meta_azure_machine_name": "vm3", + "__meta_azure_machine_os_type": "Linux", + "__meta_azure_machine_private_ip": "10.0.0.1", + "__meta_azure_machine_resource_group": "{resourceGroup}", + "__meta_azure_machine_size": "size", + "__meta_azure_machine_tag_prometheus": "", + "__meta_azure_subscription_id": "", + "__meta_azure_tenant_id": "", + }, + { + "__address__": "10.0.0.1:80", + "__meta_azure_machine_computer_name": "computer_name", + "__meta_azure_machine_id": "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/vm4", + "__meta_azure_machine_location": "australiaeast", + "__meta_azure_machine_name": "vm4", + "__meta_azure_machine_os_type": "Linux", + "__meta_azure_machine_private_ip": "10.0.0.1", + "__meta_azure_machine_resource_group": "{resourceGroup}", + "__meta_azure_machine_size": "size", + "__meta_azure_machine_tag_prometheus": "", + "__meta_azure_subscription_id": "", + "__meta_azure_tenant_id": "", + }, + { + "__address__": "10.0.0.1:80", + "__meta_azure_machine_computer_name": "computer_name", + "__meta_azure_machine_id": "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachineScaleSets/vmScaleSet1/virtualMachines/vmScaleSet1_vm1", + "__meta_azure_machine_location": "australiaeast", + "__meta_azure_machine_name": "vmScaleSet1_vm1", + "__meta_azure_machine_os_type": "Linux", + "__meta_azure_machine_private_ip": "10.0.0.1", + "__meta_azure_machine_resource_group": "{resourceGroup}", + "__meta_azure_machine_scale_set": "vmScaleSet1", + "__meta_azure_machine_size": "size", + "__meta_azure_machine_tag_prometheus": "", + "__meta_azure_subscription_id": "", + "__meta_azure_tenant_id": "", + }, + { + "__address__": "10.0.0.1:80", + "__meta_azure_machine_computer_name": "computer_name", + "__meta_azure_machine_id": "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachineScaleSets/vmScaleSet1/virtualMachines/vmScaleSet1_vm2", + "__meta_azure_machine_location": "australiaeast", + "__meta_azure_machine_name": "vmScaleSet1_vm2", + "__meta_azure_machine_os_type": "Linux", + "__meta_azure_machine_private_ip": "10.0.0.1", + "__meta_azure_machine_resource_group": "{resourceGroup}", + "__meta_azure_machine_scale_set": "vmScaleSet1", + "__meta_azure_machine_size": "size", + "__meta_azure_machine_tag_prometheus": "", + "__meta_azure_subscription_id": "", + "__meta_azure_tenant_id": "", + }, + }, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.scenario, func(t *testing.T) { + t.Parallel() + azureSDConfig := &DefaultSDConfig + + azureClient := createMockAzureClient(t, tc.vmResp, tc.vmssResp, tc.vmssvmResp, tc.interfacesResp, nil) + + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := azureSDConfig.NewDiscovererMetrics(reg, refreshMetrics) + + sd, err := NewDiscovery(azureSDConfig, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "azure", + }) + require.NoError(t, err) + + tg, err := sd.refreshAzureClient(context.Background(), azureClient) + require.NoError(t, err) + + sortTargetsByID(tg[0].Targets) + require.Equal(t, tc.expectedTG, tg) + }) + } +} + type mockAzureClient struct { - networkInterface *armnetwork.Interface + azureClient +} + +func createMockAzureClient(t *testing.T, vmResp []armcompute.VirtualMachinesClientListAllResponse, vmssResp []armcompute.VirtualMachineScaleSetsClientListAllResponse, vmssvmResp []armcompute.VirtualMachineScaleSetVMsClientListResponse, interfaceResp armnetwork.Interface, logger *slog.Logger) client { + t.Helper() + mockVMServer := defaultMockVMServer(vmResp) + mockVMSSServer := defaultMockVMSSServer(vmssResp) + mockVMScaleSetVMServer := defaultMockVMSSVMServer(vmssvmResp) + mockInterfaceServer := defaultMockInterfaceServer(interfaceResp) + + vmClient, err := armcompute.NewVirtualMachinesClient("fake-subscription-id", &azfake.TokenCredential{}, &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: fake.NewVirtualMachinesServerTransport(&mockVMServer), + }, + }) + require.NoError(t, err) + + vmssClient, err := armcompute.NewVirtualMachineScaleSetsClient("fake-subscription-id", &azfake.TokenCredential{}, &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: fake.NewVirtualMachineScaleSetsServerTransport(&mockVMSSServer), + }, + }) + require.NoError(t, err) + + vmssvmClient, err := armcompute.NewVirtualMachineScaleSetVMsClient("fake-subscription-id", &azfake.TokenCredential{}, &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: fake.NewVirtualMachineScaleSetVMsServerTransport(&mockVMScaleSetVMServer), + }, + }) + require.NoError(t, err) + + interfacesClient, err := armnetwork.NewInterfacesClient("fake-subscription-id", &azfake.TokenCredential{}, &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: fakenetwork.NewInterfacesServerTransport(&mockInterfaceServer), + }, + }) + require.NoError(t, err) + + return &mockAzureClient{ + azureClient: azureClient{ + vm: newVirtualMachinesClientAdapter(vmClient), + vmss: newVirtualMachineScaleSetsClientAdapter(vmssClient), + vmssvm: newVirtualMachineScaleSetVMsClientAdapter(vmssvmClient), + nic: newInterfacesClientAdapter(interfacesClient), + logger: logger, + }, + } } -var _ client = &mockAzureClient{} +func defaultMockInterfaceServer(interfaceResp armnetwork.Interface) fakenetwork.InterfacesServer { + return fakenetwork.InterfacesServer{ + Get: func(context.Context, string, string, *armnetwork.InterfacesClientGetOptions) (resp azfake.Responder[armnetwork.InterfacesClientGetResponse], errResp azfake.ErrorResponder) { + resp.SetResponse(http.StatusOK, armnetwork.InterfacesClientGetResponse{Interface: interfaceResp}, nil) + return resp, errResp + }, + GetVirtualMachineScaleSetNetworkInterface: func(context.Context, string, string, string, string, *armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceOptions) (resp azfake.Responder[armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse], errResp azfake.ErrorResponder) { + resp.SetResponse(http.StatusOK, armnetwork.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse{Interface: interfaceResp}, nil) + return resp, errResp + }, + } +} -func (*mockAzureClient) getVMs(ctx context.Context, resourceGroup string) ([]virtualMachine, error) { - return nil, nil +func defaultMockVMServer(vmResp []armcompute.VirtualMachinesClientListAllResponse) fake.VirtualMachinesServer { + return fake.VirtualMachinesServer{ + NewListAllPager: func(*armcompute.VirtualMachinesClientListAllOptions) (resp azfake.PagerResponder[armcompute.VirtualMachinesClientListAllResponse]) { + for _, page := range vmResp { + resp.AddPage(http.StatusOK, page, nil) + } + return resp + }, + } } -func (*mockAzureClient) getScaleSets(ctx context.Context, resourceGroup string) ([]armcompute.VirtualMachineScaleSet, error) { - return nil, nil +func defaultMockVMSSServer(vmssResp []armcompute.VirtualMachineScaleSetsClientListAllResponse) fake.VirtualMachineScaleSetsServer { + return fake.VirtualMachineScaleSetsServer{ + NewListAllPager: func(*armcompute.VirtualMachineScaleSetsClientListAllOptions) (resp azfake.PagerResponder[armcompute.VirtualMachineScaleSetsClientListAllResponse]) { + for _, page := range vmssResp { + resp.AddPage(http.StatusOK, page, nil) + } + return resp + }, + } } -func (*mockAzureClient) getScaleSetVMs(ctx context.Context, scaleSet armcompute.VirtualMachineScaleSet) ([]virtualMachine, error) { - return nil, nil +func defaultMockVMSSVMServer(vmssvmResp []armcompute.VirtualMachineScaleSetVMsClientListResponse) fake.VirtualMachineScaleSetVMsServer { + return fake.VirtualMachineScaleSetVMsServer{ + NewListPager: func(_, _ string, _ *armcompute.VirtualMachineScaleSetVMsClientListOptions) (resp azfake.PagerResponder[armcompute.VirtualMachineScaleSetVMsClientListResponse]) { + for _, page := range vmssvmResp { + resp.AddPage(http.StatusOK, page, nil) + } + return resp + }, + } } -func (m *mockAzureClient) getVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID string) (*armnetwork.Interface, error) { - if networkInterfaceID == "" { - return nil, fmt.Errorf("parameter networkInterfaceID cannot be empty") +func defaultVMWithIDAndName(id, name *string) *armcompute.VirtualMachine { + vmSize := armcompute.VirtualMachineSizeTypes("size") + osType := armcompute.OperatingSystemTypesLinux + defaultID := "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachine/testVM" + defaultName := "testVM" + + if id == nil { + id = &defaultID + } + if name == nil { + name = &defaultName + } + + return &armcompute.VirtualMachine{ + ID: id, + Name: name, + Type: to.Ptr("Microsoft.Compute/virtualMachines"), + Location: to.Ptr("australiaeast"), + Properties: &armcompute.VirtualMachineProperties{ + OSProfile: &armcompute.OSProfile{ + ComputerName: to.Ptr("computer_name"), + }, + StorageProfile: &armcompute.StorageProfile{ + OSDisk: &armcompute.OSDisk{ + OSType: &osType, + }, + }, + NetworkProfile: &armcompute.NetworkProfile{ + NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ + { + ID: to.Ptr(defaultMockNetworkID), + }, + }, + }, + HardwareProfile: &armcompute.HardwareProfile{ + VMSize: &vmSize, + }, + }, + Tags: map[string]*string{ + "prometheus": new(string), + }, } - return m.networkInterface, nil } -func (m *mockAzureClient) getVMScaleSetVMNetworkInterfaceByID(ctx context.Context, networkInterfaceID, scaleSetName, instanceID string) (*armnetwork.Interface, error) { - if scaleSetName == "" { - return nil, fmt.Errorf("parameter virtualMachineScaleSetName cannot be empty") +func defaultVMSSVMWithIDAndName(id, name *string) *armcompute.VirtualMachineScaleSetVM { + vmSize := armcompute.VirtualMachineSizeTypes("size") + osType := armcompute.OperatingSystemTypesLinux + defaultID := "/subscriptions/00000000-0000-0000-0000-00000000000/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/virtualMachineScaleSets/testVMScaleSet/virtualMachines/testVM" + defaultName := "testVM" + + if id == nil { + id = &defaultID } - return m.networkInterface, nil + if name == nil { + name = &defaultName + } + + return &armcompute.VirtualMachineScaleSetVM{ + ID: id, + Name: name, + Type: to.Ptr("Microsoft.Compute/virtualMachines"), + InstanceID: to.Ptr("123"), + Location: to.Ptr("australiaeast"), + Properties: &armcompute.VirtualMachineScaleSetVMProperties{ + OSProfile: &armcompute.OSProfile{ + ComputerName: to.Ptr("computer_name"), + }, + StorageProfile: &armcompute.StorageProfile{ + OSDisk: &armcompute.OSDisk{ + OSType: &osType, + }, + }, + NetworkProfile: &armcompute.NetworkProfile{ + NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ + {ID: to.Ptr(defaultMockNetworkID)}, + }, + }, + HardwareProfile: &armcompute.HardwareProfile{ + VMSize: &vmSize, + }, + }, + Tags: map[string]*string{ + "prometheus": new(string), + }, + } +} + +func sortTargetsByID(targets []model.LabelSet) { + slices.SortFunc(targets, func(a, b model.LabelSet) int { + return strings.Compare(string(a["__meta_azure_machine_id"]), string(b["__meta_azure_machine_id"])) + }) } diff --git a/discovery/azure/metrics.go b/discovery/azure/metrics.go index 3e3dbdbfbbc..dc0291cdb8a 100644 --- a/discovery/azure/metrics.go +++ b/discovery/azure/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/consul/consul.go b/discovery/consul/consul.go index bdc1fc8dce4..a8ffe5e8d3a 100644 --- a/discovery/consul/consul.go +++ b/discovery/consul/consul.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,17 +17,18 @@ import ( "context" "errors" "fmt" + "log/slog" "net" + "slices" "strconv" "strings" "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" consul "github.com/hashicorp/consul/api" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -113,8 +114,14 @@ type SDConfig struct { Services []string `yaml:"services,omitempty"` // A list of tags used to filter instances inside a service. Services must contain all tags in the list. ServiceTags []string `yaml:"tags,omitempty"` - // Desired node metadata. + // Desired node metadata. As of Consul 1.14, consider `filter` instead. NodeMeta map[string]string `yaml:"node_meta,omitempty"` + // Filter expression for the Catalog API. + // See https://developer.hashicorp.com/consul/api-docs/catalog#filtering for syntax. + Filter string `yaml:"filter,omitempty"` + // Filter expression for the Health API. + // See https://developer.hashicorp.com/consul/api-docs/health#filtering for syntax. + HealthFilter string `yaml:"health_filter,omitempty"` HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` } @@ -138,7 +145,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -166,30 +173,32 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // Discovery retrieves target information from a Consul server // and updates them via watches. type Discovery struct { - client *consul.Client - clientDatacenter string - clientNamespace string - clientPartition string - tagSeparator string - watchedServices []string // Set of services which will be discovered. - watchedTags []string // Tags used to filter instances of a service. - watchedNodeMeta map[string]string - allowStale bool - refreshInterval time.Duration - finalizer func() - logger log.Logger - metrics *consulMetrics + client *consul.Client + clientDatacenter string + clientNamespace string + clientPartition string + tagSeparator string + watchedServices []string // Set of services which will be discovered. + watchedTags []string // Tags used to filter instances of a service. + watchedNodeMeta map[string]string + watchedFilter string + watchedHealthFilter string + allowStale bool + refreshInterval time.Duration + finalizer func() + logger *slog.Logger + metrics *consulMetrics } // NewDiscovery returns a new Discovery for the given config. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger *slog.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { m, ok := metrics.(*consulMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } if logger == nil { - logger = log.NewNopLogger() + logger = promslog.NewNopLogger() } wrapper, err := config.NewClientFromConfig(conf.HTTPClientConfig, "consul_sd", config.WithIdleConnTimeout(2*watchTimeout)) @@ -213,19 +222,21 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere return nil, err } cd := &Discovery{ - client: client, - tagSeparator: conf.TagSeparator, - watchedServices: conf.Services, - watchedTags: conf.ServiceTags, - watchedNodeMeta: conf.NodeMeta, - allowStale: conf.AllowStale, - refreshInterval: time.Duration(conf.RefreshInterval), - clientDatacenter: conf.Datacenter, - clientNamespace: conf.Namespace, - clientPartition: conf.Partition, - finalizer: wrapper.CloseIdleConnections, - logger: logger, - metrics: m, + client: client, + tagSeparator: conf.TagSeparator, + watchedServices: conf.Services, + watchedTags: conf.ServiceTags, + watchedNodeMeta: conf.NodeMeta, + watchedFilter: conf.Filter, + watchedHealthFilter: conf.HealthFilter, + allowStale: conf.AllowStale, + refreshInterval: time.Duration(conf.RefreshInterval), + clientDatacenter: conf.Datacenter, + clientNamespace: conf.Namespace, + clientPartition: conf.Partition, + finalizer: wrapper.CloseIdleConnections, + logger: logger, + metrics: m, } return cd, nil @@ -236,22 +247,17 @@ func (d *Discovery) shouldWatch(name string, tags []string) bool { return d.shouldWatchFromName(name) && d.shouldWatchFromTags(tags) } -// shouldWatch returns whether the service of the given name should be watched based on its name. +// shouldWatchFromName returns whether the service of the given name should be watched based on its name. func (d *Discovery) shouldWatchFromName(name string) bool { // If there's no fixed set of watched services, we watch everything. if len(d.watchedServices) == 0 { return true } - for _, sn := range d.watchedServices { - if sn == name { - return true - } - } - return false + return slices.Contains(d.watchedServices, name) } -// shouldWatch returns whether the service of the given name should be watched based on its tags. +// shouldWatchFromTags returns whether the service of the given name should be watched based on its tags. // This gets called when the user doesn't specify a list of services in order to avoid watching // *all* services. Details in https://github.com/prometheus/prometheus/pull/3814 func (d *Discovery) shouldWatchFromTags(tags []string) bool { @@ -282,7 +288,7 @@ func (d *Discovery) getDatacenter() error { info, err := d.client.Agent().Self() if err != nil { - level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err) + d.logger.Error("Error retrieving datacenter name", "err", err) d.metrics.rpcFailuresCount.Inc() return err } @@ -290,12 +296,12 @@ func (d *Discovery) getDatacenter() error { dc, ok := info["Config"]["Datacenter"].(string) if !ok { err := fmt.Errorf("invalid value '%v' for Config.Datacenter", info["Config"]["Datacenter"]) - level.Error(d.logger).Log("msg", "Error retrieving datacenter name", "err", err) + d.logger.Error("Error retrieving datacenter name", "err", err) return err } d.clientDatacenter = dc - d.logger = log.With(d.logger, "datacenter", dc) + d.logger = d.logger.With("datacenter", dc) return nil } @@ -329,7 +335,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } d.initialize(ctx) - if len(d.watchedServices) == 0 || len(d.watchedTags) != 0 { + if len(d.watchedServices) == 0 || len(d.watchedTags) != 0 || d.watchedFilter != "" { // We need to watch the catalog. ticker := time.NewTicker(d.refreshInterval) @@ -361,13 +367,14 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { // entire list of services. func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.Group, lastIndex *uint64, services map[string]func()) { catalog := d.client.Catalog() - level.Debug(d.logger).Log("msg", "Watching services", "tags", strings.Join(d.watchedTags, ",")) + d.logger.Debug("Watching services", "tags", strings.Join(d.watchedTags, ","), "filter", d.watchedFilter) opts := &consul.QueryOptions{ WaitIndex: *lastIndex, WaitTime: watchTimeout, AllowStale: d.allowStale, NodeMeta: d.watchedNodeMeta, + Filter: d.watchedFilter, } t0 := time.Now() srvs, meta, err := catalog.Services(opts.WithContext(ctx)) @@ -382,7 +389,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup. } if err != nil { - level.Error(d.logger).Log("msg", "Error refreshing service list", "err", err) + d.logger.Error("Error refreshing service list", "err", err) d.metrics.rpcFailuresCount.Inc() time.Sleep(retryInterval) return @@ -445,7 +452,7 @@ type consulService struct { discovery *Discovery client *consul.Client tagSeparator string - logger log.Logger + logger *slog.Logger rpcFailuresCount prometheus.Counter serviceRPCDuration prometheus.Observer } @@ -490,13 +497,14 @@ func (d *Discovery) watchService(ctx context.Context, ch chan<- []*targetgroup.G // Get updates for a service. func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Group, health *consul.Health, lastIndex *uint64) { - level.Debug(srv.logger).Log("msg", "Watching service", "service", srv.name, "tags", strings.Join(srv.tags, ",")) + srv.logger.Debug("Watching service", "service", srv.name, "tags", strings.Join(srv.tags, ",")) opts := &consul.QueryOptions{ WaitIndex: *lastIndex, WaitTime: watchTimeout, AllowStale: srv.discovery.allowStale, NodeMeta: srv.discovery.watchedNodeMeta, + Filter: srv.discovery.watchedHealthFilter, } t0 := time.Now() @@ -513,7 +521,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*targetgroup.Gr } if err != nil { - level.Error(srv.logger).Log("msg", "Error refreshing service", "service", srv.name, "tags", strings.Join(srv.tags, ","), "err", err) + srv.logger.Error("Error refreshing service", "service", srv.name, "tags", strings.Join(srv.tags, ","), "err", err) srv.rpcFailuresCount.Inc() time.Sleep(retryInterval) return diff --git a/discovery/consul/consul_test.go b/discovery/consul/consul_test.go index e3bc7938f55..3992ee9fa85 100644 --- a/discovery/consul/consul_test.go +++ b/discovery/consul/consul_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,13 +21,13 @@ import ( "testing" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "go.uber.org/goleak" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -49,6 +49,7 @@ func NewTestMetrics(t *testing.T, conf discovery.Config, reg prometheus.Register } func TestConfiguredService(t *testing.T) { + t.Parallel() conf := &SDConfig{ Services: []string{"configuredServiceName"}, } @@ -64,6 +65,7 @@ func TestConfiguredService(t *testing.T) { } func TestConfiguredServiceWithTag(t *testing.T) { + t.Parallel() conf := &SDConfig{ Services: []string{"configuredServiceName"}, ServiceTags: []string{"http"}, @@ -87,6 +89,7 @@ func TestConfiguredServiceWithTag(t *testing.T) { } func TestConfiguredServiceWithTags(t *testing.T) { + t.Parallel() type testcase struct { // What we've configured to watch. conf *SDConfig @@ -174,6 +177,7 @@ func TestConfiguredServiceWithTags(t *testing.T) { } func TestNonConfiguredService(t *testing.T) { + t.Parallel() conf := &SDConfig{} metrics := NewTestMetrics(t, conf, prometheus.NewRegistry()) @@ -252,6 +256,8 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) { case "/v1/catalog/services?index=1&wait=120000ms": time.Sleep(5 * time.Second) response = ServicesTestAnswer + case "/v1/catalog/services?filter=NodeMeta.rack_name+%3D%3D+%222304%22&index=1&wait=120000ms": + response = ServicesTestAnswer default: t.Errorf("Unhandled consul call: %s", r.URL) } @@ -270,7 +276,7 @@ func newServer(t *testing.T) (*httptest.Server, *SDConfig) { } func newDiscovery(t *testing.T, config *SDConfig) *Discovery { - logger := log.NewNopLogger() + logger := promslog.NewNopLogger() metrics := NewTestMetrics(t, config, prometheus.NewRegistry()) @@ -292,8 +298,9 @@ func checkOneTarget(t *testing.T, tg []*targetgroup.Group) { // Watch all the services in the catalog. func TestAllServices(t *testing.T) { + t.Parallel() stub, config := newServer(t) - defer stub.Close() + t.Cleanup(stub.Close) d := newDiscovery(t, config) @@ -311,8 +318,9 @@ func TestAllServices(t *testing.T) { // targetgroup with no targets is emitted if no services were discovered. func TestNoTargets(t *testing.T) { + t.Parallel() stub, config := newServer(t) - defer stub.Close() + t.Cleanup(stub.Close) config.ServiceTags = []string{"missing"} d := newDiscovery(t, config) @@ -332,8 +340,9 @@ func TestNoTargets(t *testing.T) { // Watch only the test service. func TestOneService(t *testing.T) { + t.Parallel() stub, config := newServer(t) - defer stub.Close() + t.Cleanup(stub.Close) config.Services = []string{"test"} d := newDiscovery(t, config) @@ -347,8 +356,9 @@ func TestOneService(t *testing.T) { // Watch the test service with a specific tag and node-meta. func TestAllOptions(t *testing.T) { + t.Parallel() stub, config := newServer(t) - defer stub.Close() + t.Cleanup(stub.Close) config.Services = []string{"test"} config.NodeMeta = map[string]string{"rack_name": "2304"} @@ -369,21 +379,201 @@ func TestAllOptions(t *testing.T) { <-ch } +// TestFilterOption verifies that when services and filter are both configured, the Catalog API +// is still called and receives the filter parameter, while the Health API does not. +func TestFilterOption(t *testing.T) { + t.Parallel() + var ( + catalogCalled bool + catalogFilter string + healthCalled bool + healthFilter string + ) + + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Consul-Index", "1") + switch r.URL.Path { + case "/v1/agent/self": + w.Write([]byte(AgentAnswer)) + case "/v1/catalog/services": + catalogCalled = true + catalogFilter = r.URL.Query().Get("filter") + w.Write([]byte(`{"test": []}`)) + case "/v1/health/service/test": + healthCalled = true + healthFilter = r.URL.Query().Get("filter") + w.Write([]byte(ServiceTestAnswer)) + default: + t.Errorf("Unhandled consul call: %s", r.URL) + } + })) + t.Cleanup(stub.Close) + + stuburl, err := url.Parse(stub.URL) + require.NoError(t, err) + + cfg := &SDConfig{ + Server: stuburl.Host, + Services: []string{"test"}, + Filter: `NodeMeta.rack_name == "2304"`, + RefreshInterval: model.Duration(1 * time.Second), + } + + d := newDiscovery(t, cfg) + + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan []*targetgroup.Group) + go func() { + d.Run(ctx, ch) + close(ch) + }() + checkOneTarget(t, <-ch) + // All handler writes happened-before the channel receive above. + require.True(t, catalogCalled, "Catalog endpoint should be called when filter is set alongside services.") + require.Equal(t, `NodeMeta.rack_name == "2304"`, catalogFilter, "Catalog should receive the filter parameter.") + require.True(t, healthCalled, "Health endpoint should be called.") + require.Empty(t, healthFilter, "Health endpoint should not receive the catalog filter.") + cancel() + for range ch { + } +} + +// TestHealthFilterOption verifies that health_filter is passed to the Health API and not to +// the Catalog API. +func TestHealthFilterOption(t *testing.T) { + t.Parallel() + var ( + catalogCalled bool + catalogFilter string + healthCalled bool + healthFilter string + ) + + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Consul-Index", "1") + switch r.URL.Path { + case "/v1/agent/self": + w.Write([]byte(AgentAnswer)) + case "/v1/catalog/services": + catalogCalled = true + catalogFilter = r.URL.Query().Get("filter") + w.Write([]byte(`{"test": []}`)) + case "/v1/health/service/test": + healthCalled = true + healthFilter = r.URL.Query().Get("filter") + w.Write([]byte(ServiceTestAnswer)) + default: + t.Errorf("Unhandled consul call: %s", r.URL) + } + })) + t.Cleanup(stub.Close) + + stuburl, err := url.Parse(stub.URL) + require.NoError(t, err) + + // No services configured: catalog path is always used, allowing us to assert + // that health_filter is not forwarded to the Catalog API. + cfg := &SDConfig{ + Server: stuburl.Host, + HealthFilter: `Service.Tags contains "canary"`, + RefreshInterval: model.Duration(1 * time.Second), + } + + d := newDiscovery(t, cfg) + + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan []*targetgroup.Group) + go func() { + d.Run(ctx, ch) + close(ch) + }() + checkOneTarget(t, <-ch) + // All handler writes happened-before the channel receive above. + require.True(t, catalogCalled, "Catalog endpoint should be called.") + require.Empty(t, catalogFilter, "Catalog should not receive the health_filter parameter.") + require.True(t, healthCalled, "Health endpoint should be called.") + require.Equal(t, `Service.Tags contains "canary"`, healthFilter, "Health endpoint should receive the health_filter parameter.") + cancel() + for range ch { + } +} + +// TestBothFiltersOption verifies that when both filter and health_filter are configured, +// each filter is sent exclusively to its respective API endpoint. +func TestBothFiltersOption(t *testing.T) { + t.Parallel() + var ( + catalogCalled bool + catalogFilter string + healthCalled bool + healthFilter string + ) + + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Consul-Index", "1") + switch r.URL.Path { + case "/v1/agent/self": + w.Write([]byte(AgentAnswer)) + case "/v1/catalog/services": + catalogCalled = true + catalogFilter = r.URL.Query().Get("filter") + w.Write([]byte(`{"test": []}`)) + case "/v1/health/service/test": + healthCalled = true + healthFilter = r.URL.Query().Get("filter") + w.Write([]byte(ServiceTestAnswer)) + default: + t.Errorf("Unhandled consul call: %s", r.URL) + } + })) + t.Cleanup(stub.Close) + + stuburl, err := url.Parse(stub.URL) + require.NoError(t, err) + + cfg := &SDConfig{ + Server: stuburl.Host, + Services: []string{"test"}, + Filter: `NodeMeta.rack_name == "2304"`, + HealthFilter: `Service.Tags contains "canary"`, + RefreshInterval: model.Duration(1 * time.Second), + } + + d := newDiscovery(t, cfg) + + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan []*targetgroup.Group) + go func() { + d.Run(ctx, ch) + close(ch) + }() + checkOneTarget(t, <-ch) + // All handler writes happened-before the channel receive above. + require.True(t, catalogCalled, "Catalog endpoint should be called when filter is set.") + require.Equal(t, `NodeMeta.rack_name == "2304"`, catalogFilter, "Catalog should receive only the catalog filter.") + require.True(t, healthCalled, "Health endpoint should be called.") + require.Equal(t, `Service.Tags contains "canary"`, healthFilter, "Health endpoint should receive only the health_filter.") + cancel() + for range ch { + } +} + func TestGetDatacenterShouldReturnError(t *testing.T) { + t.Parallel() for _, tc := range []struct { handler func(http.ResponseWriter, *http.Request) errMessage string }{ { // Define a handler that will return status 500. - handler: func(w http.ResponseWriter, r *http.Request) { + handler: func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, errMessage: "Unexpected response code: 500 ()", }, { // Define a handler that will return incorrect response. - handler: func(w http.ResponseWriter, r *http.Request) { + handler: func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte(`{"Config": {"Not-Datacenter": "test-dc"}}`)) }, errMessage: "invalid value '' for Config.Datacenter", @@ -398,24 +588,25 @@ func TestGetDatacenterShouldReturnError(t *testing.T) { Token: "fake-token", RefreshInterval: model.Duration(1 * time.Second), } - defer stub.Close() + t.Cleanup(stub.Close) d := newDiscovery(t, config) // Should be empty if not initialized. - require.Equal(t, "", d.clientDatacenter) + require.Empty(t, d.clientDatacenter) err = d.getDatacenter() // An error should be returned. - require.Equal(t, tc.errMessage, err.Error()) + require.EqualError(t, err, tc.errMessage) // Should still be empty. - require.Equal(t, "", d.clientDatacenter) + require.Empty(t, d.clientDatacenter) } } func TestUnmarshalConfig(t *testing.T) { - unmarshal := func(d []byte) func(interface{}) error { - return func(o interface{}) error { + t.Parallel() + unmarshal := func(d []byte) func(any) error { + return func(o any) error { return yaml.Unmarshal(d, o) } } @@ -485,6 +676,7 @@ oauth2: for _, test := range cases { t.Run(test.name, func(t *testing.T) { + t.Parallel() var config SDConfig err := config.UnmarshalYAML(unmarshal([]byte(test.config))) if err != nil { diff --git a/discovery/consul/metrics.go b/discovery/consul/metrics.go index 8266e7cc609..903fba5cefd 100644 --- a/discovery/consul/metrics.go +++ b/discovery/consul/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -31,7 +31,7 @@ type consulMetrics struct { metricRegisterer discovery.MetricRegisterer } -func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func newDiscovererMetrics(reg prometheus.Registerer, _ discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { m := &consulMetrics{ rpcFailuresCount: prometheus.NewCounter( prometheus.CounterOpts{ diff --git a/discovery/digitalocean/digitalocean.go b/discovery/digitalocean/digitalocean.go index ecee60cb1f0..f80eb29901a 100644 --- a/discovery/digitalocean/digitalocean.go +++ b/discovery/digitalocean/digitalocean.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,6 +15,7 @@ package digitalocean import ( "context" + "errors" "fmt" "net" "net/http" @@ -23,7 +24,6 @@ import ( "time" "github.com/digitalocean/godo" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -34,29 +34,58 @@ import ( "github.com/prometheus/prometheus/discovery/targetgroup" ) +// metaLabelPrefix is the meta prefix used for all meta labels. const ( - doLabel = model.MetaLabelPrefix + "digitalocean_" - doLabelID = doLabel + "droplet_id" - doLabelName = doLabel + "droplet_name" - doLabelImage = doLabel + "image" - doLabelImageName = doLabel + "image_name" - doLabelPrivateIPv4 = doLabel + "private_ipv4" - doLabelPublicIPv4 = doLabel + "public_ipv4" - doLabelPublicIPv6 = doLabel + "public_ipv6" - doLabelRegion = doLabel + "region" - doLabelSize = doLabel + "size" - doLabelStatus = doLabel + "status" - doLabelFeatures = doLabel + "features" - doLabelTags = doLabel + "tags" - doLabelVPC = doLabel + "vpc" - separator = "," + metaLabelPrefix = model.MetaLabelPrefix + "digitalocean_" + separator = "," ) +const ( + doLabelID = metaLabelPrefix + "droplet_id" + doLabelName = metaLabelPrefix + "droplet_name" + doLabelImage = metaLabelPrefix + "image" + doLabelImageName = metaLabelPrefix + "image_name" + doLabelPrivateIPv4 = metaLabelPrefix + "private_ipv4" + doLabelPublicIPv4 = metaLabelPrefix + "public_ipv4" + doLabelPublicIPv6 = metaLabelPrefix + "public_ipv6" + doLabelRegion = metaLabelPrefix + "region" + doLabelSize = metaLabelPrefix + "size" + doLabelStatus = metaLabelPrefix + "status" + doLabelFeatures = metaLabelPrefix + "features" + doLabelTags = metaLabelPrefix + "tags" + doLabelVPC = metaLabelPrefix + "vpc" +) + +// Role is the role of the target within the DigitalOcean ecosystem. +type Role string + +const ( + // DropletsRole discovers targets from DigitalOcean Droplets. + DropletsRole Role = "droplets" + + // DatabasesRole discovers targets from DigitalOcean Managed Databases. + DatabasesRole Role = "databases" +) + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *Role) UnmarshalYAML(unmarshal func(any) error) error { + if err := unmarshal((*string)(c)); err != nil { + return err + } + switch *c { + case DropletsRole, DatabasesRole: + return nil + default: + return fmt.Errorf("unknown DigitalOcean SD role %q", *c) + } +} + // DefaultSDConfig is the default DigitalOcean SD configuration. var DefaultSDConfig = SDConfig{ Port: 80, RefreshInterval: model.Duration(60 * time.Second), HTTPClientConfig: config.DefaultHTTPClientConfig, + Role: DropletsRole, } func init() { @@ -64,7 +93,7 @@ func init() { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &digitaloceanMetrics{ refreshMetrics: rmi, } @@ -76,6 +105,10 @@ type SDConfig struct { RefreshInterval model.Duration `yaml:"refresh_interval"` Port int `yaml:"port"` + Role Role `yaml:"role"` + + // Internal field for testing. + HTTPClient *http.Client } // Name returns the name of the Config. @@ -83,7 +116,7 @@ func (*SDConfig) Name() string { return "digitalocean" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -92,64 +125,85 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) if err != nil { return err } - return c.HTTPClientConfig.Validate() -} -// Discovery periodically performs DigitalOcean requests. It implements -// the Discoverer interface. -type Discovery struct { - *refresh.Discovery - client *godo.Client - port int + if c.Role == "" { + return errors.New("role missing (one of: droplets, databases)") + } + + return c.HTTPClientConfig.Validate() } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*digitaloceanMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + m, ok := opts.Metrics.(*digitaloceanMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") - } - - d := &Discovery{ - port: conf.Port, + return nil, errors.New("invalid discovery metrics type") } - rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd") + r, err := newRefresher(conf) if err != nil { return nil, err } - d.client, err = godo.New( - &http.Client{ + return refresh.NewDiscovery( + refresh.Options{ + Logger: opts.Logger, + Mech: "digitalocean", + SetName: opts.SetName, + Interval: time.Duration(conf.RefreshInterval), + RefreshF: r.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ), nil +} + +type refresher interface { + refresh(context.Context) ([]*targetgroup.Group, error) +} + +func newRefresher(conf *SDConfig) (refresher, error) { + httpClient := conf.HTTPClient + if httpClient == nil { + rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd") + if err != nil { + return nil, err + } + httpClient = &http.Client{ Transport: rt, Timeout: time.Duration(conf.RefreshInterval), - }, - godo.SetUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)), + } + } + + client, err := godo.New( + httpClient, + godo.SetUserAgent(version.PrometheusUserAgent()), ) if err != nil { return nil, fmt.Errorf("error setting up digital ocean agent: %w", err) } - d.Discovery = refresh.NewDiscovery( - refresh.Options{ - Logger: logger, - Mech: "digitalocean", - Interval: time.Duration(conf.RefreshInterval), - RefreshF: d.refresh, - MetricsInstantiator: m.refreshMetrics, - }, - ) - return d, nil + switch conf.Role { + case DropletsRole: + return &dropletsDiscovery{client: client, port: conf.Port}, nil + case DatabasesRole: + return &databasesDiscovery{client: client, port: conf.Port}, nil + } + return nil, fmt.Errorf("unknown DigitalOcean SD role %q", conf.Role) +} + +type dropletsDiscovery struct { + client *godo.Client + port int } -func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { +func (d *dropletsDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { tg := &targetgroup.Group{ Source: "DigitalOcean", } @@ -212,7 +266,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { return []*targetgroup.Group{tg}, nil } -func (d *Discovery) listDroplets(ctx context.Context) ([]godo.Droplet, error) { +func (d *dropletsDiscovery) listDroplets(ctx context.Context) ([]godo.Droplet, error) { var ( droplets []godo.Droplet opts = &godo.ListOptions{} diff --git a/discovery/digitalocean/digitalocean_db.go b/discovery/digitalocean/digitalocean_db.go new file mode 100644 index 00000000000..9be5b65fb1d --- /dev/null +++ b/discovery/digitalocean/digitalocean_db.go @@ -0,0 +1,124 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package digitalocean + +import ( + "context" + "fmt" + "net" + "strconv" + + "github.com/digitalocean/godo" + "github.com/prometheus/common/model" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + dbLabelID = metaLabelPrefix + "db_id" + dbLabelName = metaLabelPrefix + "db_name" + dbLabelEngine = metaLabelPrefix + "db_engine" + dbLabelVersion = metaLabelPrefix + "db_version" + dbLabelStatus = metaLabelPrefix + "db_status" + dbLabelRegion = metaLabelPrefix + "db_region" + dbLabelSize = metaLabelPrefix + "db_size" + dbLabelNumNodes = metaLabelPrefix + "db_num_nodes" + dbLabelHost = metaLabelPrefix + "db_host" + dbLabelPrivateHost = metaLabelPrefix + "db_private_host" + dbLabelTagPrefix = metaLabelPrefix + "db_tag_" +) + +type databasesDiscovery struct { + client *godo.Client + port int +} + +func (d *databasesDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + tg := &targetgroup.Group{ + Source: "DigitalOcean Databases", + } + + clusters, err := d.listClusters(ctx) + if err != nil { + return nil, err + } + for _, cluster := range clusters { + labels := model.LabelSet{ + dbLabelID: model.LabelValue(cluster.ID), + dbLabelName: model.LabelValue(cluster.Name), + dbLabelEngine: model.LabelValue(cluster.EngineSlug), + dbLabelVersion: model.LabelValue(cluster.VersionSlug), + dbLabelStatus: model.LabelValue(cluster.Status), + dbLabelRegion: model.LabelValue(cluster.RegionSlug), + dbLabelSize: model.LabelValue(cluster.SizeSlug), + dbLabelNumNodes: model.LabelValue(strconv.Itoa(cluster.NumNodes)), + } + + host := "" + if cluster.PrivateConnection != nil { + host = cluster.PrivateConnection.Host + labels[dbLabelPrivateHost] = model.LabelValue(host) + } + + if cluster.Connection != nil { + labels[dbLabelHost] = model.LabelValue(cluster.Connection.Host) + if host == "" { + host = cluster.Connection.Host + } + } + + if host != "" { + addr := net.JoinHostPort(host, strconv.FormatUint(uint64(d.port), 10)) + labels[model.AddressLabel] = model.LabelValue(addr) + } + + for _, tag := range cluster.Tags { + labels[dbLabelTagPrefix+model.LabelName(tag)] = "true" + } + + tg.Targets = append(tg.Targets, labels) + } + return []*targetgroup.Group{tg}, nil +} + +func (d *databasesDiscovery) listClusters(ctx context.Context) ([]godo.Database, error) { + var ( + clusters []godo.Database + opts = &godo.ListOptions{ + Page: 1, + PerPage: 100, + } + ) + for { + paginatedClusters, resp, err := d.client.Databases.List(ctx, opts) + if err != nil { + return nil, fmt.Errorf("error while listing database clusters page %d: %w", opts.Page, err) + } + if len(paginatedClusters) == 0 { + break + } + clusters = append(clusters, paginatedClusters...) + + if resp.Links != nil && !resp.Links.IsLastPage() { + page, err := resp.Links.CurrentPage() + if err == nil { + opts.Page = page + 1 + continue + } + } + + opts.Page++ + } + return clusters, nil +} diff --git a/discovery/digitalocean/digitalocean_db_test.go b/discovery/digitalocean/digitalocean_db_test.go new file mode 100644 index 00000000000..c385ad2cc0c --- /dev/null +++ b/discovery/digitalocean/digitalocean_db_test.go @@ -0,0 +1,141 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package digitalocean + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" +) + +func TestDigitalOceanDBRefresh(t *testing.T) { + mock := NewSDMock(t) + mock.Setup() + defer mock.ShutdownServer() + + mock.HandleDatabasesList() + + cfg := DefaultSDConfig + cfg.Role = DatabasesRole + cfg.Port = 9273 + cfg.HTTPClient = mock.Server.Client() + + d, err := newRefresher(&cfg) + require.NoError(t, err) + + // Inject the mock URL into the godo client since we can't easily change it via HTTPClient. + endpoint, _ := url.Parse(mock.Endpoint()) + d.(*databasesDiscovery).client.BaseURL = endpoint + + tgs, err := d.refresh(context.Background()) + require.NoError(t, err) + require.Len(t, tgs, 1) + + tg := tgs[0] + require.NotNil(t, tg) + require.Len(t, tg.Targets, 2) + + expectedTargets := []struct { + labels model.LabelSet + }{ + { + labels: model.LabelSet{ + "__address__": "do-fra1-pg-trading-001-do-user-XXXXX-0.f.db.ondigitalocean.com:9273", + "__meta_digitalocean_db_id": "9cc10173-e9ea-4176-9dbc-a4cee4c4ff30", + "__meta_digitalocean_db_name": "do-fra1-pg-trading-001", + "__meta_digitalocean_db_engine": "pg", + "__meta_digitalocean_db_version": "16", + "__meta_digitalocean_db_status": "online", + "__meta_digitalocean_db_region": "fra1", + "__meta_digitalocean_db_size": "db-s-1vcpu-2gb", + "__meta_digitalocean_db_num_nodes": "1", + "__meta_digitalocean_db_host": "do-fra1-pg-trading-001-do-user-XXXXX-0.f.db.ondigitalocean.com", + "__meta_digitalocean_db_tag_prod": "true", + "__meta_digitalocean_db_tag_market": "true", + }, + }, + { + labels: model.LabelSet{ + "__address__": "private-db-host:9273", + "__meta_digitalocean_db_id": "a0b1c2d3-e4f5-4677-8899-001122334455", + "__meta_digitalocean_db_name": "private-db", + "__meta_digitalocean_db_engine": "mysql", + "__meta_digitalocean_db_version": "8", + "__meta_digitalocean_db_status": "online", + "__meta_digitalocean_db_region": "nyc1", + "__meta_digitalocean_db_size": "db-s-2vcpu-4gb", + "__meta_digitalocean_db_num_nodes": "2", + "__meta_digitalocean_db_host": "public-db-host", + "__meta_digitalocean_db_private_host": "private-db-host", + }, + }, + } + + for i, expected := range expectedTargets { + require.Equal(t, expected.labels, tg.Targets[i]) + } +} + +func TestDigitalOceanDBRefreshPagination(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := r.URL.Query().Get("page") + switch page { + case "1", "": + fmt.Fprint(w, ` +{ + "databases": [ + {"id": "db-1", "name": "db-1", "engine": "pg", "version": "16", "status": "online", "region": "fra1", "size": "db-s-1vcpu-2gb", "num_nodes": 1} + ], + "links": { + "pages": { + "next": "http://example.com/v2/databases?page=2" + } + } +} +`) + case "2": + fmt.Fprint(w, ` +{ + "databases": [ + {"id": "db-2", "name": "db-2", "engine": "pg", "version": "16", "status": "online", "region": "fra1", "size": "db-s-1vcpu-2gb", "num_nodes": 1} + ] +} +`) + default: + fmt.Fprint(w, `{"databases": []}`) + } + })) + defer ts.Close() + + cfg := DefaultSDConfig + cfg.Role = DatabasesRole + cfg.HTTPClient = ts.Client() + + d, err := newRefresher(&cfg) + require.NoError(t, err) + + endpoint, _ := url.Parse(ts.URL) + d.(*databasesDiscovery).client.BaseURL = endpoint + + tgs, err := d.refresh(context.Background()) + require.NoError(t, err) + require.Len(t, tgs, 1) + require.Len(t, tgs[0].Targets, 2) +} diff --git a/discovery/digitalocean/digitalocean_test.go b/discovery/digitalocean/digitalocean_test.go index 841b5ef977a..733c863498a 100644 --- a/discovery/digitalocean/digitalocean_test.go +++ b/discovery/digitalocean/digitalocean_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,7 +19,6 @@ import ( "net/url" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" @@ -57,11 +56,11 @@ func TestDigitalOceanSDRefresh(t *testing.T) { defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := newRefresher(&cfg) require.NoError(t, err) endpoint, err := url.Parse(sdmock.Mock.Endpoint()) require.NoError(t, err) - d.client.BaseURL = endpoint + d.(*dropletsDiscovery).client.BaseURL = endpoint ctx := context.Background() tgs, err := d.refresh(ctx) diff --git a/discovery/digitalocean/metrics.go b/discovery/digitalocean/metrics.go index 91cfd9bf30e..4b11b825e5f 100644 --- a/discovery/digitalocean/metrics.go +++ b/discovery/digitalocean/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type digitaloceanMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *digitaloceanMetrics) Register() error { +func (*digitaloceanMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *digitaloceanMetrics) Unregister() {} +func (*digitaloceanMetrics) Unregister() {} diff --git a/discovery/digitalocean/mock_test.go b/discovery/digitalocean/mock_test.go index 62d963c3b35..607cd063438 100644 --- a/discovery/digitalocean/mock_test.go +++ b/discovery/digitalocean/mock_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -645,3 +645,68 @@ func (m *SDMock) HandleDropletsList() { ) }) } + +// HandleDatabasesList mocks database clusters list. +func (m *SDMock) HandleDatabasesList() { + m.Mux.HandleFunc("/v2/databases", func(w http.ResponseWriter, r *http.Request) { + page := r.URL.Query().Get("page") + switch page { + case "1", "": + fmt.Fprint(w, ` +{ + "databases": [ + { + "id": "9cc10173-e9ea-4176-9dbc-a4cee4c4ff30", + "name": "do-fra1-pg-trading-001", + "engine": "pg", + "version": "16", + "status": "online", + "region": "fra1", + "size": "db-s-1vcpu-2gb", + "num_nodes": 1, + "connection": { + "host": "do-fra1-pg-trading-001-do-user-XXXXX-0.f.db.ondigitalocean.com", + "port": 25060, + "user": "doadmin", + "password": "XXX", + "database": "defaultdb", + "ssl": true + }, + "tags": ["prod", "market"] + } + ], + "links": { + "pages": { + "next": "http://example.com/v2/databases?page=2" + } + } +} +`) + case "2": + fmt.Fprint(w, ` +{ + "databases": [ + { + "id": "a0b1c2d3-e4f5-4677-8899-001122334455", + "name": "private-db", + "engine": "mysql", + "version": "8", + "status": "online", + "region": "nyc1", + "size": "db-s-2vcpu-4gb", + "num_nodes": 2, + "connection": { + "host": "public-db-host" + }, + "private_connection": { + "host": "private-db-host" + } + } + ] +} +`) + default: + fmt.Fprint(w, `{"databases": []}`) + } + }) +} diff --git a/discovery/discoverer_metrics_noop.go b/discovery/discoverer_metrics_noop.go index 4321204b6c1..b75474dfec1 100644 --- a/discovery/discoverer_metrics_noop.go +++ b/discovery/discoverer_metrics_noop.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/discovery.go b/discovery/discovery.go index a91faf6c864..5ad51a394ce 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,9 +15,9 @@ package discovery import ( "context" + "log/slog" "reflect" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -47,26 +47,30 @@ type DiscovererMetrics interface { // DiscovererOptions provides options for a Discoverer. type DiscovererOptions struct { - Logger log.Logger + Logger *slog.Logger Metrics DiscovererMetrics // Extra HTTP client options to expose to Discoverers. This field may be // ignored; Discoverer implementations must opt-in to reading it. HTTPClientOptions []config.HTTPClientOption + + // SetName identifies this discoverer set. + SetName string } // RefreshMetrics are used by the "refresh" package. // We define them here in the "discovery" package in order to avoid a cyclic dependency between // "discovery" and "refresh". type RefreshMetrics struct { - Failures prometheus.Counter - Duration prometheus.Observer + Failures prometheus.Counter + Duration prometheus.Observer + DurationHistogram prometheus.Observer } // RefreshMetricsInstantiator instantiates the metrics used by the "refresh" package. type RefreshMetricsInstantiator interface { - Instantiate(mech string) *RefreshMetrics + Instantiate(mech, setName string) *RefreshMetrics } // RefreshMetricsManager is an interface for registering, unregistering, and @@ -79,6 +83,13 @@ type RefreshMetricsInstantiator interface { type RefreshMetricsManager interface { DiscovererMetrics RefreshMetricsInstantiator + DeleteLabelValues(mech, config string) +} + +// SDMetrics holds all metrics for service discovery mechanisms. +type SDMetrics struct { + MechanismMetrics map[string]DiscovererMetrics + RefreshManager RefreshMetricsManager } // A Config provides the configuration and constructor for a Discoverer. @@ -108,8 +119,8 @@ func (c *Configs) SetDirectory(dir string) { } // UnmarshalYAML implements yaml.Unmarshaler. -func (c *Configs) UnmarshalYAML(unmarshal func(interface{}) error) error { - cfgTyp := getConfigType(configsType) +func (c *Configs) UnmarshalYAML(unmarshal func(any) error) error { + cfgTyp := reflect.StructOf(configFields) cfgPtr := reflect.New(cfgTyp) cfgVal := cfgPtr.Elem() @@ -123,8 +134,8 @@ func (c *Configs) UnmarshalYAML(unmarshal func(interface{}) error) error { } // MarshalYAML implements yaml.Marshaler. -func (c Configs) MarshalYAML() (interface{}, error) { - cfgTyp := getConfigType(configsType) +func (c Configs) MarshalYAML() (any, error) { + cfgTyp := reflect.StructOf(configFields) cfgPtr := reflect.New(cfgTyp) cfgVal := cfgPtr.Elem() @@ -148,7 +159,7 @@ func (c StaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { // NewDiscovererMetrics returns NoopDiscovererMetrics because no metrics are // needed for this service discovery mechanism. -func (c StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics { +func (StaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetricsInstantiator) DiscovererMetrics { return &NoopDiscovererMetrics{} } diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go new file mode 100644 index 00000000000..53539b6d40f --- /dev/null +++ b/discovery/discovery_test.go @@ -0,0 +1,36 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v2" +) + +func TestConfigsCustomUnMarshalMarshal(t *testing.T) { + input := `static_configs: +- targets: + - foo:1234 + - bar:4321 +` + cfg := &Configs{} + err := yaml.UnmarshalStrict([]byte(input), cfg) + require.NoError(t, err) + + output, err := yaml.Marshal(cfg) + require.NoError(t, err) + require.Equal(t, input, string(output)) +} diff --git a/discovery/dns/dns.go b/discovery/dns/dns.go index 314c3d38cd5..4d9200d7346 100644 --- a/discovery/dns/dns.go +++ b/discovery/dns/dns.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,17 +17,17 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" "strings" "sync" "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/miekg/dns" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/refresh" @@ -78,11 +78,11 @@ func (*SDConfig) Name() string { return "dns" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(*c, opts.Logger, opts.Metrics) + return NewDiscovery(*c, opts) } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -111,21 +111,21 @@ type Discovery struct { names []string port int qtype uint16 - logger log.Logger + logger *slog.Logger metrics *dnsMetrics - lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) + lookupFn func(name string, qtype uint16, logger *slog.Logger) (*dns.Msg, error) } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*dnsMetrics) +func NewDiscovery(conf SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*dnsMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - if logger == nil { - logger = log.NewNopLogger() + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() } qtype := dns.TypeSRV @@ -145,15 +145,16 @@ func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.Discoverer names: conf.Names, qtype: qtype, port: conf.Port, - logger: logger, + logger: opts.Logger, lookupFn: lookupWithSearchPath, metrics: m, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "dns", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -174,7 +175,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { for _, name := range d.names { go func(n string) { if err := d.refreshOne(ctx, n, ch); err != nil && !errors.Is(err, context.Canceled) { - level.Error(d.logger).Log("msg", "Error refreshing DNS targets", "err", err) + d.logger.Error("Error refreshing DNS targets", "err", err) } wg.Done() }(name) @@ -238,7 +239,7 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ // CNAME responses can occur with "Type: A" dns_sd_config requests. continue default: - level.Warn(d.logger).Log("msg", "Invalid record", "record", record) + d.logger.Warn("Invalid record", "record", record) continue } tg.Targets = append(tg.Targets, model.LabelSet{ @@ -288,7 +289,7 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ // error will be generic-looking, because trying to return all the errors // returned by the combination of all name permutations and servers is a // nightmare. -func lookupWithSearchPath(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { +func lookupWithSearchPath(name string, qtype uint16, logger *slog.Logger) (*dns.Msg, error) { conf, err := dns.ClientConfigFromFile(resolvConf) if err != nil { return nil, fmt.Errorf("could not load resolv.conf: %w", err) @@ -337,14 +338,14 @@ func lookupWithSearchPath(name string, qtype uint16, logger log.Logger) (*dns.Ms // A non-viable answer is "anything else", which encompasses both various // system-level problems (like network timeouts) and also // valid-but-unexpected DNS responses (SERVFAIL, REFUSED, etc). -func lookupFromAnyServer(name string, qtype uint16, conf *dns.ClientConfig, logger log.Logger) (*dns.Msg, error) { +func lookupFromAnyServer(name string, qtype uint16, conf *dns.ClientConfig, logger *slog.Logger) (*dns.Msg, error) { client := &dns.Client{} for _, server := range conf.Servers { servAddr := net.JoinHostPort(server, conf.Port) msg, err := askServerForName(name, qtype, client, servAddr, true) if err != nil { - level.Warn(logger).Log("msg", "DNS resolution failed", "server", server, "name", name, "err", err) + logger.Warn("DNS resolution failed", "server", server, "name", name, "err", err) continue } diff --git a/discovery/dns/dns_test.go b/discovery/dns/dns_test.go index 33a976827d3..eeb11378787 100644 --- a/discovery/dns/dns_test.go +++ b/discovery/dns/dns_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,18 +15,18 @@ package dns import ( "context" - "fmt" + "errors" + "log/slog" "net" "testing" "time" - "github.com/go-kit/log" "github.com/miekg/dns" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "go.uber.org/goleak" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -40,7 +40,7 @@ func TestDNS(t *testing.T) { testCases := []struct { name string config SDConfig - lookup func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) + lookup func(name string, qtype uint16, logger *slog.Logger) (*dns.Msg, error) expected []*targetgroup.Group }{ @@ -52,8 +52,8 @@ func TestDNS(t *testing.T) { Port: 80, Type: "A", }, - lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { - return nil, fmt.Errorf("some error") + lookup: func(string, uint16, *slog.Logger) (*dns.Msg, error) { + return nil, errors.New("some error") }, expected: []*targetgroup.Group{}, }, @@ -65,7 +65,7 @@ func TestDNS(t *testing.T) { Port: 80, Type: "A", }, - lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { + lookup: func(string, uint16, *slog.Logger) (*dns.Msg, error) { return &dns.Msg{ Answer: []dns.RR{ &dns.A{A: net.IPv4(192, 0, 2, 2)}, @@ -97,7 +97,7 @@ func TestDNS(t *testing.T) { Port: 80, Type: "AAAA", }, - lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { + lookup: func(string, uint16, *slog.Logger) (*dns.Msg, error) { return &dns.Msg{ Answer: []dns.RR{ &dns.AAAA{AAAA: net.IPv6loopback}, @@ -128,7 +128,7 @@ func TestDNS(t *testing.T) { Type: "SRV", RefreshInterval: model.Duration(time.Minute), }, - lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { + lookup: func(string, uint16, *slog.Logger) (*dns.Msg, error) { return &dns.Msg{ Answer: []dns.RR{ &dns.SRV{Port: 3306, Target: "db1.example.com."}, @@ -167,7 +167,7 @@ func TestDNS(t *testing.T) { Names: []string{"_mysql._tcp.db.example.com."}, RefreshInterval: model.Duration(time.Minute), }, - lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { + lookup: func(string, uint16, *slog.Logger) (*dns.Msg, error) { return &dns.Msg{ Answer: []dns.RR{ &dns.SRV{Port: 3306, Target: "db1.example.com."}, @@ -198,7 +198,7 @@ func TestDNS(t *testing.T) { Names: []string{"_mysql._tcp.db.example.com."}, RefreshInterval: model.Duration(time.Minute), }, - lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { + lookup: func(string, uint16, *slog.Logger) (*dns.Msg, error) { return &dns.Msg{}, nil }, expected: []*targetgroup.Group{ @@ -215,7 +215,7 @@ func TestDNS(t *testing.T) { Port: 25, RefreshInterval: model.Duration(time.Minute), }, - lookup: func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) { + lookup: func(string, uint16, *slog.Logger) (*dns.Msg, error) { return &dns.Msg{ Answer: []dns.RR{ &dns.MX{Preference: 0, Mx: "smtp1.example.com."}, @@ -251,7 +251,6 @@ func TestDNS(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -260,7 +259,11 @@ func TestDNS(t *testing.T) { metrics := tc.config.NewDiscovererMetrics(reg, refreshMetrics) require.NoError(t, metrics.Register()) - sd, err := NewDiscovery(tc.config, nil, metrics) + sd, err := NewDiscovery(tc.config, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "dns", + }) require.NoError(t, err) sd.lookupFn = tc.lookup @@ -282,8 +285,8 @@ func TestSDConfigUnmarshalYAML(t *testing.T) { return d } - unmarshal := func(d []byte) func(interface{}) error { - return func(o interface{}) error { + unmarshal := func(d []byte) func(any) error { + return func(o any) error { return yaml.Unmarshal(d, o) } } diff --git a/discovery/dns/metrics.go b/discovery/dns/metrics.go index 27c96b53e00..b65db5e6c0a 100644 --- a/discovery/dns/metrics.go +++ b/discovery/dns/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/eureka/client.go b/discovery/eureka/client.go index 52e8ce7b489..50168e1d11c 100644 --- a/discovery/eureka/client.go +++ b/discovery/eureka/client.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,7 +23,7 @@ import ( "github.com/prometheus/common/version" ) -var userAgent = fmt.Sprintf("Prometheus/%s", version.Version) +var userAgent = version.PrometheusUserAgent() type Applications struct { VersionsDelta int `xml:"versions__delta"` @@ -81,7 +81,7 @@ const appListPath string = "/apps" func fetchApps(ctx context.Context, server string, client *http.Client) (*Applications, error) { url := fmt.Sprintf("%s%s", server, appListPath) - request, err := http.NewRequest(http.MethodGet, url, nil) + request, err := http.NewRequest(http.MethodGet, url, http.NoBody) if err != nil { return nil, err } diff --git a/discovery/eureka/client_test.go b/discovery/eureka/client_test.go index 83f6fd5ff1f..19812b1f5df 100644 --- a/discovery/eureka/client_test.go +++ b/discovery/eureka/client_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -172,7 +172,7 @@ func TestFetchApps(t *testing.T) { ` // Simulate apps with a valid XML response. - respHandler := func(w http.ResponseWriter, r *http.Request) { + respHandler := func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, appsXML) @@ -199,7 +199,7 @@ func TestFetchApps(t *testing.T) { func Test500ErrorHttpResponse(t *testing.T) { // Simulate 500 error. - respHandler := func(w http.ResponseWriter, r *http.Request) { + respHandler := func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, ``) diff --git a/discovery/eureka/eureka.go b/discovery/eureka/eureka.go index 779c081aee3..e7059107703 100644 --- a/discovery/eureka/eureka.go +++ b/discovery/eureka/eureka.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,14 +16,12 @@ package eureka import ( "context" "errors" - "fmt" "net" "net/http" "net/url" "strconv" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -78,7 +76,7 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &eurekaMetrics{ refreshMetrics: rmi, } @@ -89,7 +87,7 @@ func (*SDConfig) Name() string { return "eureka" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -98,21 +96,21 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) if err != nil { return err } - if len(c.Server) == 0 { + if c.Server == "" { return errors.New("eureka_sd: empty or null eureka server") } url, err := url.Parse(c.Server) if err != nil { return err } - if len(url.Scheme) == 0 || len(url.Host) == 0 { + if url.Scheme == "" || url.Host == "" { return errors.New("eureka_sd: invalid eureka server URL") } return c.HTTPClientConfig.Validate() @@ -126,10 +124,10 @@ type Discovery struct { } // NewDiscovery creates a new Eureka discovery for the given role. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*eurekaMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*eurekaMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "eureka_sd") @@ -143,8 +141,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "eureka", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, diff --git a/discovery/eureka/eureka_test.go b/discovery/eureka/eureka_test.go index b499410bfcc..69612fedb79 100644 --- a/discovery/eureka/eureka_test.go +++ b/discovery/eureka/eureka_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -47,7 +47,11 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err defer metrics.Unregister() defer refreshMetrics.Unregister() - md, err := NewDiscovery(&conf, nil, metrics) + md, err := NewDiscovery(&conf, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "eureka", + }) if err != nil { return nil, err } @@ -58,7 +62,7 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err func TestEurekaSDHandleError(t *testing.T) { var ( errTesting = "non 2xx status '500' response during eureka service discovery" - respHandler = func(w http.ResponseWriter, r *http.Request) { + respHandler = func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, ``) @@ -76,7 +80,7 @@ func TestEurekaSDEmptyList(t *testing.T) { 1 ` - respHandler = func(w http.ResponseWriter, r *http.Request) { + respHandler = func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, appsXML) @@ -235,7 +239,7 @@ func TestEurekaSDSendGroup(t *testing.T) { ` - respHandler = func(w http.ResponseWriter, r *http.Request) { + respHandler = func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, appsXML) diff --git a/discovery/eureka/metrics.go b/discovery/eureka/metrics.go index 6d6463ccd38..5a0720a8d53 100644 --- a/discovery/eureka/metrics.go +++ b/discovery/eureka/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type eurekaMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *eurekaMetrics) Register() error { +func (*eurekaMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *eurekaMetrics) Unregister() {} +func (*eurekaMetrics) Unregister() {} diff --git a/discovery/file/file.go b/discovery/file/file.go index e7e9d0870fc..762705e2ef8 100644 --- a/discovery/file/file.go +++ b/discovery/file/file.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,6 +19,8 @@ import ( "errors" "fmt" "io" + "log/slog" + "maps" "os" "path/filepath" "strings" @@ -26,13 +28,12 @@ import ( "time" "github.com/fsnotify/fsnotify" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/grafana/regexp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" - "gopkg.in/yaml.v2" + "github.com/prometheus/common/promslog" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -78,7 +79,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -120,9 +121,7 @@ func (t *TimestampCollector) Collect(ch chan<- prometheus.Metric) { t.lock.RLock() for fileSD := range t.discoverers { fileSD.lock.RLock() - for filename, timestamp := range fileSD.timestamps { - uniqueFiles[filename] = timestamp - } + maps.Copy(uniqueFiles, fileSD.timestamps) fileSD.lock.RUnlock() } t.lock.RUnlock() @@ -175,20 +174,20 @@ type Discovery struct { // and how many target groups they contained. // This is used to detect deleted target groups. lastRefresh map[string]int - logger log.Logger + logger *slog.Logger metrics *fileMetrics } // NewDiscovery returns a new file discovery for the given paths. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { +func NewDiscovery(conf *SDConfig, logger *slog.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { fm, ok := metrics.(*fileMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } if logger == nil { - logger = log.NewNopLogger() + logger = promslog.NewNopLogger() } disc := &Discovery{ @@ -210,7 +209,7 @@ func (d *Discovery) listFiles() []string { for _, p := range d.paths { files, err := filepath.Glob(p) if err != nil { - level.Error(d.logger).Log("msg", "Error expanding glob", "glob", p, "err", err) + d.logger.Error("Error expanding glob", "glob", p, "err", err) continue } paths = append(paths, files...) @@ -231,7 +230,7 @@ func (d *Discovery) watchFiles() { p = "./" } if err := d.watcher.Add(p); err != nil { - level.Error(d.logger).Log("msg", "Error adding file watch", "path", p, "err", err) + d.logger.Error("Error adding file watch", "path", p, "err", err) } } } @@ -240,7 +239,7 @@ func (d *Discovery) watchFiles() { func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { watcher, err := fsnotify.NewWatcher() if err != nil { - level.Error(d.logger).Log("msg", "Error adding file watcher", "err", err) + d.logger.Error("Error adding file watcher", "err", err) d.metrics.fileWatcherErrorsCount.Inc() return } @@ -260,7 +259,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { case event := <-d.watcher.Events: // fsnotify sometimes sends a bunch of events without name or operation. // It's unclear what they are and why they are sent - filter them out. - if len(event.Name) == 0 { + if event.Name == "" { break } // Everything but a chmod requires rereading. @@ -280,7 +279,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { case err := <-d.watcher.Errors: if err != nil { - level.Error(d.logger).Log("msg", "Error watching file", "err", err) + d.logger.Error("Error watching file", "err", err) } } } @@ -300,7 +299,7 @@ func (d *Discovery) deleteTimestamp(filename string) { // stop shuts down the file watcher. func (d *Discovery) stop() { - level.Debug(d.logger).Log("msg", "Stopping file discovery...", "paths", fmt.Sprintf("%v", d.paths)) + d.logger.Debug("Stopping file discovery...", "paths", fmt.Sprintf("%v", d.paths)) done := make(chan struct{}) defer close(done) @@ -320,10 +319,10 @@ func (d *Discovery) stop() { } }() if err := d.watcher.Close(); err != nil { - level.Error(d.logger).Log("msg", "Error closing file watcher", "paths", fmt.Sprintf("%v", d.paths), "err", err) + d.logger.Error("Error closing file watcher", "paths", fmt.Sprintf("%v", d.paths), "err", err) } - level.Debug(d.logger).Log("msg", "File discovery stopped") + d.logger.Debug("File discovery stopped") } // refresh reads all files matching the discovery's patterns and sends the respective @@ -339,7 +338,7 @@ func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) if err != nil { d.metrics.fileSDReadErrorsCount.Inc() - level.Error(d.logger).Log("msg", "Error reading file", "path", p, "err", err) + d.logger.Error("Error reading file", "path", p, "err", err) // Prevent deletion down below. ref[p] = d.lastRefresh[p] continue @@ -356,7 +355,7 @@ func (d *Discovery) refresh(ctx context.Context, ch chan<- []*targetgroup.Group) for f, n := range d.lastRefresh { m, ok := ref[f] if !ok || n > m { - level.Debug(d.logger).Log("msg", "file_sd refresh found file that should be removed", "file", f) + d.logger.Debug("file_sd refresh found file that should be removed", "file", f) d.deleteTimestamp(f) for i := m; i < n; i++ { select { diff --git a/discovery/file/file_test.go b/discovery/file/file_test.go index 179ac5cd1c2..80239875ce1 100644 --- a/discovery/file/file_test.go +++ b/discovery/file/file_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package file import ( "context" "encoding/json" - "io" "os" "path/filepath" "sort" @@ -64,7 +63,7 @@ func newTestRunner(t *testing.T) *testRunner { } } -// copyFile atomically copies a file to the runner's directory. +// copyFile copies a file to the runner's directory. func (t *testRunner) copyFile(src string) string { t.Helper() return t.copyFileTo(src, filepath.Base(src)) @@ -74,36 +73,50 @@ func (t *testRunner) copyFile(src string) string { func (t *testRunner) copyFileTo(src, name string) string { t.Helper() - newf, err := os.CreateTemp(t.dir, "") + content, err := os.ReadFile(src) require.NoError(t, err) - f, err := os.Open(src) - require.NoError(t, err) - - _, err = io.Copy(newf, f) - require.NoError(t, err) - require.NoError(t, f.Close()) - require.NoError(t, newf.Close()) - dst := filepath.Join(t.dir, name) - err = os.Rename(newf.Name(), dst) - require.NoError(t, err) + t.atomicWrite(dst, content) return dst } -// writeString writes atomically a string to a file. +// writeString atomically writes a string to a file. func (t *testRunner) writeString(file, data string) { t.Helper() + t.atomicWrite(file, []byte(data)) +} - newf, err := os.CreateTemp(t.dir, "") - require.NoError(t, err) +// atomicWrite writes data to dst atomically by writing to a temporary file +// and renaming it. The temp file is created outside the watched directory to +// avoid triggering spurious fsnotify events that could cause readFile to hold +// an open handle on dst (which would make os.Rename fail on Windows). +func (t *testRunner) atomicWrite(dst string, data []byte) { + t.Helper() - _, err = newf.WriteString(data) + // Create the temp file via t.TempDir() rather than in t.dir (the watched + // directory). t.TempDir() returns a fresh directory on the same filesystem + // as t.dir, so os.Rename works, and cleanup is handled by the test framework. + tmp, err := os.CreateTemp(t.TempDir(), ".sd-test-*") require.NoError(t, err) - require.NoError(t, newf.Close()) - err = os.Rename(newf.Name(), file) + _, err = tmp.Write(data) + require.NoError(t, err) + require.NoError(t, tmp.Close()) + + // On Windows, os.Rename fails if another process holds an open handle + // on dst. This can happen if a previous refresh cycle's readFile call + // hasn't released the file yet. Retry a few times to handle this; + // on Linux/macOS os.Rename always succeeds regardless, so the retry + // never triggers. + for retries := 0; ; retries++ { + err = os.Rename(tmp.Name(), dst) + if err == nil || retries >= 5 { + break + } + time.Sleep(50 * time.Millisecond) + } require.NoError(t, err) } @@ -323,7 +336,6 @@ func TestInitialUpdate(t *testing.T) { "fixtures/valid.yml", "fixtures/valid.json", } { - tc := tc t.Run(tc, func(t *testing.T) { t.Parallel() @@ -344,7 +356,6 @@ func TestInvalidFile(t *testing.T) { "fixtures/invalid_nil.yml", "fixtures/invalid_nil.json", } { - tc := tc t.Run(tc, func(t *testing.T) { t.Parallel() diff --git a/discovery/file/metrics.go b/discovery/file/metrics.go index c01501e4efd..0371338d468 100644 --- a/discovery/file/metrics.go +++ b/discovery/file/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -30,7 +30,7 @@ type fileMetrics struct { metricRegisterer discovery.MetricRegisterer } -func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func newDiscovererMetrics(reg prometheus.Registerer, _ discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { fm := &fileMetrics{ fileSDReadErrorsCount: prometheus.NewCounter( prometheus.CounterOpts{ diff --git a/discovery/gce/gce.go b/discovery/gce/gce.go index 15f32dd2473..507a3238a33 100644 --- a/discovery/gce/gce.go +++ b/discovery/gce/gce.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,12 +17,10 @@ import ( "context" "errors" "fmt" - "net/http" "strconv" "strings" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "golang.org/x/oauth2/google" @@ -83,7 +81,7 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &gceMetrics{ refreshMetrics: rmi, } @@ -94,11 +92,11 @@ func (*SDConfig) Name() string { return "gce" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(*c, opts.Logger, opts.Metrics) + return NewDiscovery(*c, opts) } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -114,49 +112,64 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// instancesLister lists the instances of a project/zone, paging through the +// results and invoking f for each page. +type instancesLister func(ctx context.Context, f func(*compute.InstanceList) error) error + +// newInstancesLister captures *compute.Service in a closure instead of storing +// it on the interface-boxed Discovery struct. This keeps the binary smaller: +// otherwise reflection keeps every Compute method live, defeating dead-code +// elimination. +func newInstancesLister(svc *compute.Service, project, zone, filter string) instancesLister { + isvc := compute.NewInstancesService(svc) + return func(ctx context.Context, f func(*compute.InstanceList) error) error { + ilc := isvc.List(project, zone) + if filter != "" { + ilc = ilc.Filter(filter) + } + return ilc.Pages(ctx, f) + } +} + // Discovery periodically performs GCE-SD requests. It implements // the Discoverer interface. type Discovery struct { *refresh.Discovery - project string - zone string - filter string - client *http.Client - svc *compute.Service - isvc *compute.InstancesService - port int - tagSeparator string + project string + zone string + listInstances instancesLister + port int + tagSeparator string } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*gceMetrics) +func NewDiscovery(conf SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*gceMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } d := &Discovery{ project: conf.Project, zone: conf.Zone, - filter: conf.Filter, port: conf.Port, tagSeparator: conf.TagSeparator, } - var err error - d.client, err = google.DefaultClient(context.Background(), compute.ComputeReadonlyScope) + client, err := google.DefaultClient(context.Background(), compute.ComputeReadonlyScope) if err != nil { return nil, fmt.Errorf("error setting up communication with GCE service: %w", err) } - d.svc, err = compute.NewService(context.Background(), option.WithHTTPClient(d.client)) + svc, err := compute.NewService(context.Background(), option.WithHTTPClient(client)) if err != nil { return nil, fmt.Errorf("error setting up communication with GCE service: %w", err) } - d.isvc = compute.NewInstancesService(d.svc) + d.listInstances = newInstancesLister(svc, conf.Project, conf.Zone, conf.Filter) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "gce", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -170,11 +183,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { Source: fmt.Sprintf("GCE_%s_%s", d.project, d.zone), } - ilc := d.isvc.List(d.project, d.zone) - if len(d.filter) > 0 { - ilc = ilc.Filter(d.filter) - } - err := ilc.Pages(ctx, func(l *compute.InstanceList) error { + err := d.listInstances(ctx, func(l *compute.InstanceList) error { for _, inst := range l.Items { if len(inst.NetworkInterfaces) == 0 { continue diff --git a/discovery/gce/metrics.go b/discovery/gce/metrics.go index f53ea5a8c6f..c4020f0a53b 100644 --- a/discovery/gce/metrics.go +++ b/discovery/gce/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type gceMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *gceMetrics) Register() error { +func (*gceMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *gceMetrics) Unregister() {} +func (*gceMetrics) Unregister() {} diff --git a/discovery/hetzner/hcloud.go b/discovery/hetzner/hcloud.go index df56f94c5fb..c28bfd2a1fd 100644 --- a/discovery/hetzner/hcloud.go +++ b/discovery/hetzner/hcloud.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,12 +15,12 @@ package hetzner import ( "context" + "log/slog" "net" "net/http" "strconv" "time" - "github.com/go-kit/log" "github.com/hetznercloud/hcloud-go/v2/hcloud" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -38,8 +38,10 @@ const ( hetznerLabelHcloudImageOSVersion = hetznerHcloudLabelPrefix + "image_os_version" hetznerLabelHcloudImageOSFlavor = hetznerHcloudLabelPrefix + "image_os_flavor" hetznerLabelHcloudPrivateIPv4 = hetznerHcloudLabelPrefix + "private_ipv4_" - hetznerLabelHcloudDatacenterLocation = hetznerHcloudLabelPrefix + "datacenter_location" - hetznerLabelHcloudDatacenterLocationNetworkZone = hetznerHcloudLabelPrefix + "datacenter_location_network_zone" + hetznerLabelHcloudLocation = hetznerHcloudLabelPrefix + "location" + hetznerLabelHcloudLocationNetworkZone = hetznerHcloudLabelPrefix + "location_network_zone" + hetznerLabelHcloudDatacenterLocation = hetznerHcloudLabelPrefix + "datacenter_location" // Label name kept for backward compatibility + hetznerLabelHcloudDatacenterLocationNetworkZone = hetznerHcloudLabelPrefix + "datacenter_location_network_zone" // Label name kept for backward compatibility hetznerLabelHcloudCPUCores = hetznerHcloudLabelPrefix + "cpu_cores" hetznerLabelHcloudCPUType = hetznerHcloudLabelPrefix + "cpu_type" hetznerLabelHcloudMemoryGB = hetznerHcloudLabelPrefix + "memory_size_gb" @@ -53,14 +55,16 @@ const ( // the Discoverer interface. type hcloudDiscovery struct { *refresh.Discovery - client *hcloud.Client - port int + client *hcloud.Client + port int + labelSelector string } // newHcloudDiscovery returns a new hcloudDiscovery which periodically refreshes its targets. -func newHcloudDiscovery(conf *SDConfig, _ log.Logger) (*hcloudDiscovery, error) { +func newHcloudDiscovery(conf *SDConfig, _ *slog.Logger) (*hcloudDiscovery, error) { d := &hcloudDiscovery{ - port: conf.Port, + port: conf.Port, + labelSelector: conf.LabelSelector, } rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "hetzner_sd") @@ -79,7 +83,10 @@ func newHcloudDiscovery(conf *SDConfig, _ log.Logger) (*hcloudDiscovery, error) } func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - servers, err := d.client.Server.All(ctx) + servers, err := d.client.Server.AllWithOpts(ctx, hcloud.ServerListOpts{ListOpts: hcloud.ListOpts{ + PerPage: 50, + LabelSelector: d.labelSelector, + }}) if err != nil { return nil, err } @@ -93,13 +100,14 @@ func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er hetznerLabelRole: model.LabelValue(HetznerRoleHcloud), hetznerLabelServerID: model.LabelValue(strconv.FormatInt(server.ID, 10)), hetznerLabelServerName: model.LabelValue(server.Name), - hetznerLabelDatacenter: model.LabelValue(server.Datacenter.Name), hetznerLabelPublicIPv4: model.LabelValue(server.PublicNet.IPv4.IP.String()), hetznerLabelPublicIPv6Network: model.LabelValue(server.PublicNet.IPv6.Network.String()), hetznerLabelServerStatus: model.LabelValue(server.Status), - hetznerLabelHcloudDatacenterLocation: model.LabelValue(server.Datacenter.Location.Name), - hetznerLabelHcloudDatacenterLocationNetworkZone: model.LabelValue(server.Datacenter.Location.NetworkZone), + hetznerLabelHcloudLocation: model.LabelValue(server.Location.Name), + hetznerLabelHcloudLocationNetworkZone: model.LabelValue(server.Location.NetworkZone), + hetznerLabelHcloudDatacenterLocation: model.LabelValue(server.Location.Name), // Label name kept for backward compatibility + hetznerLabelHcloudDatacenterLocationNetworkZone: model.LabelValue(server.Location.NetworkZone), // Label name kept for backward compatibility hetznerLabelHcloudType: model.LabelValue(server.ServerType.Name), hetznerLabelHcloudCPUCores: model.LabelValue(strconv.Itoa(server.ServerType.Cores)), hetznerLabelHcloudCPUType: model.LabelValue(server.ServerType.CPUType), @@ -109,6 +117,12 @@ func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er model.AddressLabel: model.LabelValue(net.JoinHostPort(server.PublicNet.IPv4.IP.String(), strconv.FormatUint(uint64(d.port), 10))), } + // [hcloud.Server.Datacenter] is deprecated and will be removed after 1 July 2026. + // See https://docs.hetzner.cloud/changelog#2025-12-16-phasing-out-datacenters + if server.Datacenter != nil { // nolint: staticcheck + labels[hetznerLabelDatacenter] = model.LabelValue(server.Datacenter.Name) // nolint: staticcheck + } + if server.Image != nil { labels[hetznerLabelHcloudImageName] = model.LabelValue(server.Image.Name) labels[hetznerLabelHcloudImageDescription] = model.LabelValue(server.Image.Description) diff --git a/discovery/hetzner/hcloud_test.go b/discovery/hetzner/hcloud_test.go index 10b799037ad..e7a11608c5a 100644 --- a/discovery/hetzner/hcloud_test.go +++ b/discovery/hetzner/hcloud_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,8 +18,8 @@ import ( "fmt" "testing" - "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" ) @@ -43,7 +43,7 @@ func TestHCloudSDRefresh(t *testing.T) { cfg.HTTPClientConfig.BearerToken = hcloudTestToken cfg.hcloudEndpoint = suite.Mock.Endpoint() - d, err := newHcloudDiscovery(&cfg, log.NewNopLogger()) + d, err := newHcloudDiscovery(&cfg, promslog.NewNopLogger()) require.NoError(t, err) targetGroups, err := d.refresh(context.Background()) @@ -69,6 +69,8 @@ func TestHCloudSDRefresh(t *testing.T) { "__meta_hetzner_hcloud_image_description": model.LabelValue("Ubuntu 20.04 Standard 64 bit"), "__meta_hetzner_hcloud_image_os_flavor": model.LabelValue("ubuntu"), "__meta_hetzner_hcloud_image_os_version": model.LabelValue("20.04"), + "__meta_hetzner_hcloud_location": model.LabelValue("fsn1"), + "__meta_hetzner_hcloud_location_network_zone": model.LabelValue("eu-central"), "__meta_hetzner_hcloud_datacenter_location": model.LabelValue("fsn1"), "__meta_hetzner_hcloud_datacenter_location_network_zone": model.LabelValue("eu-central"), "__meta_hetzner_hcloud_cpu_cores": model.LabelValue("1"), @@ -93,6 +95,8 @@ func TestHCloudSDRefresh(t *testing.T) { "__meta_hetzner_hcloud_image_description": model.LabelValue("Ubuntu 20.04 Standard 64 bit"), "__meta_hetzner_hcloud_image_os_flavor": model.LabelValue("ubuntu"), "__meta_hetzner_hcloud_image_os_version": model.LabelValue("20.04"), + "__meta_hetzner_hcloud_location": model.LabelValue("fsn1"), + "__meta_hetzner_hcloud_location_network_zone": model.LabelValue("eu-central"), "__meta_hetzner_hcloud_datacenter_location": model.LabelValue("fsn1"), "__meta_hetzner_hcloud_datacenter_location_network_zone": model.LabelValue("eu-central"), "__meta_hetzner_hcloud_cpu_cores": model.LabelValue("2"), @@ -114,6 +118,8 @@ func TestHCloudSDRefresh(t *testing.T) { "__meta_hetzner_datacenter": model.LabelValue("fsn1-dc14"), "__meta_hetzner_public_ipv4": model.LabelValue("1.2.3.6"), "__meta_hetzner_public_ipv6_network": model.LabelValue("2001:db7::/64"), + "__meta_hetzner_hcloud_location": model.LabelValue("fsn1"), + "__meta_hetzner_hcloud_location_network_zone": model.LabelValue("eu-central"), "__meta_hetzner_hcloud_datacenter_location": model.LabelValue("fsn1"), "__meta_hetzner_hcloud_datacenter_location_network_zone": model.LabelValue("eu-central"), "__meta_hetzner_hcloud_cpu_cores": model.LabelValue("2"), diff --git a/discovery/hetzner/hetzner.go b/discovery/hetzner/hetzner.go index 69c823d3829..3b7349e896f 100644 --- a/discovery/hetzner/hetzner.go +++ b/discovery/hetzner/hetzner.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,9 +17,9 @@ import ( "context" "errors" "fmt" + "log/slog" "time" - "github.com/go-kit/log" "github.com/hetznercloud/hcloud-go/v2/hcloud" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -36,7 +36,7 @@ const ( hetznerLabelServerID = hetznerLabelPrefix + "server_id" hetznerLabelServerName = hetznerLabelPrefix + "server_name" hetznerLabelServerStatus = hetznerLabelPrefix + "server_status" - hetznerLabelDatacenter = hetznerLabelPrefix + "datacenter" + hetznerLabelDatacenter = hetznerLabelPrefix + "datacenter" // Label name kept for backward compatibility hetznerLabelPublicIPv4 = hetznerLabelPrefix + "public_ipv4" hetznerLabelPublicIPv6Network = hetznerLabelPrefix + "public_ipv6_network" ) @@ -59,12 +59,15 @@ type SDConfig struct { RefreshInterval model.Duration `yaml:"refresh_interval"` Port int `yaml:"port"` Role Role `yaml:"role"` - hcloudEndpoint string // For tests only. - robotEndpoint string // For tests only. + + LabelSelector string `yaml:"label_selector,omitempty"` + + hcloudEndpoint string // For tests only. + robotEndpoint string // For tests only. } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &hetznerMetrics{ refreshMetrics: rmi, } @@ -75,7 +78,7 @@ func (*SDConfig) Name() string { return "hetzner" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } type refresher interface { @@ -96,7 +99,7 @@ const ( ) // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *Role) UnmarshalYAML(unmarshal func(any) error) error { if err := unmarshal((*string)(c)); err != nil { return err } @@ -109,7 +112,7 @@ func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -135,21 +138,22 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { - m, ok := metrics.(*hetznerMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*refresh.Discovery, error) { + m, ok := opts.Metrics.(*hetznerMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - r, err := newRefresher(conf, logger) + r, err := newRefresher(conf, opts.Logger) if err != nil { return nil, err } return refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "hetzner", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: r.refresh, MetricsInstantiator: m.refreshMetrics, @@ -157,7 +161,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere ), nil } -func newRefresher(conf *SDConfig, l log.Logger) (refresher, error) { +func newRefresher(conf *SDConfig, l *slog.Logger) (refresher, error) { switch conf.Role { case HetznerRoleHcloud: if conf.hcloudEndpoint == "" { diff --git a/discovery/hetzner/metrics.go b/discovery/hetzner/metrics.go index 27361ee755c..cab1d66a3ed 100644 --- a/discovery/hetzner/metrics.go +++ b/discovery/hetzner/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type hetznerMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *hetznerMetrics) Register() error { +func (*hetznerMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *hetznerMetrics) Unregister() {} +func (*hetznerMetrics) Unregister() {} diff --git a/discovery/hetzner/mock_test.go b/discovery/hetzner/mock_test.go index d192a4eae94..fb69a76b04a 100644 --- a/discovery/hetzner/mock_test.go +++ b/discovery/hetzner/mock_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -124,6 +124,16 @@ func (m *SDMock) HandleHcloudServers() { "storage_type": "local", "cpu_type": "shared" }, + "location": { + "id": 1, + "name": "fsn1", + "description": "Falkenstein DC Park 1", + "country": "DE", + "city": "Falkenstein", + "latitude": 50.47612, + "longitude": 12.370071, + "network_zone": "eu-central" + }, "datacenter": { "id": 1, "name": "fsn1-dc8", @@ -244,6 +254,16 @@ func (m *SDMock) HandleHcloudServers() { "storage_type": "local", "cpu_type": "shared" }, + "location": { + "id": 1, + "name": "fsn1", + "description": "Falkenstein DC Park 1", + "country": "DE", + "city": "Falkenstein", + "latitude": 50.47612, + "longitude": 12.370071, + "network_zone": "eu-central" + }, "datacenter": { "id": 2, "name": "fsn1-dc14", @@ -365,6 +385,16 @@ func (m *SDMock) HandleHcloudServers() { "storage_type": "local", "cpu_type": "shared" }, + "location": { + "id": 1, + "name": "fsn1", + "description": "Falkenstein DC Park 1", + "country": "DE", + "city": "Falkenstein", + "latitude": 50.47612, + "longitude": 12.370071, + "network_zone": "eu-central" + }, "datacenter": { "id": 2, "name": "fsn1-dc14", diff --git a/discovery/hetzner/robot.go b/discovery/hetzner/robot.go index 516470b05ad..5b1c149ccb6 100644 --- a/discovery/hetzner/robot.go +++ b/discovery/hetzner/robot.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,13 +18,13 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net" "net/http" "strconv" "strings" "time" - "github.com/go-kit/log" "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" @@ -34,12 +34,13 @@ import ( ) const ( - hetznerRobotLabelPrefix = hetznerLabelPrefix + "robot_" - hetznerLabelRobotProduct = hetznerRobotLabelPrefix + "product" - hetznerLabelRobotCancelled = hetznerRobotLabelPrefix + "cancelled" + hetznerRobotLabelPrefix = hetznerLabelPrefix + "robot_" + hetznerLabelRobotDatacenter = hetznerRobotLabelPrefix + "datacenter" + hetznerLabelRobotProduct = hetznerRobotLabelPrefix + "product" + hetznerLabelRobotCancelled = hetznerRobotLabelPrefix + "cancelled" ) -var userAgent = fmt.Sprintf("Prometheus/%s", version.Version) +var userAgent = version.PrometheusUserAgent() // Discovery periodically performs Hetzner Robot requests. It implements // the Discoverer interface. @@ -51,7 +52,7 @@ type robotDiscovery struct { } // newRobotDiscovery returns a new robotDiscovery which periodically refreshes its targets. -func newRobotDiscovery(conf *SDConfig, _ log.Logger) (*robotDiscovery, error) { +func newRobotDiscovery(conf *SDConfig, _ *slog.Logger) (*robotDiscovery, error) { d := &robotDiscovery{ port: conf.Port, endpoint: conf.robotEndpoint, @@ -70,7 +71,7 @@ func newRobotDiscovery(conf *SDConfig, _ log.Logger) (*robotDiscovery, error) { } func (d *robotDiscovery) refresh(context.Context) ([]*targetgroup.Group, error) { - req, err := http.NewRequest(http.MethodGet, d.endpoint+"/server", nil) + req, err := http.NewRequest(http.MethodGet, d.endpoint+"/server", http.NoBody) if err != nil { return nil, err } @@ -105,14 +106,15 @@ func (d *robotDiscovery) refresh(context.Context) ([]*targetgroup.Group, error) targets := make([]model.LabelSet, len(servers)) for i, server := range servers { labels := model.LabelSet{ - hetznerLabelRole: model.LabelValue(HetznerRoleRobot), - hetznerLabelServerID: model.LabelValue(strconv.Itoa(server.Server.ServerNumber)), - hetznerLabelServerName: model.LabelValue(server.Server.ServerName), - hetznerLabelDatacenter: model.LabelValue(strings.ToLower(server.Server.Dc)), - hetznerLabelPublicIPv4: model.LabelValue(server.Server.ServerIP), - hetznerLabelServerStatus: model.LabelValue(server.Server.Status), - hetznerLabelRobotProduct: model.LabelValue(server.Server.Product), - hetznerLabelRobotCancelled: model.LabelValue(strconv.FormatBool(server.Server.Canceled)), + hetznerLabelRole: model.LabelValue(HetznerRoleRobot), + hetznerLabelServerID: model.LabelValue(strconv.Itoa(server.Server.ServerNumber)), + hetznerLabelServerName: model.LabelValue(server.Server.ServerName), + hetznerLabelDatacenter: model.LabelValue(strings.ToLower(server.Server.Dc)), // Label name kept for backward compatibility + hetznerLabelPublicIPv4: model.LabelValue(server.Server.ServerIP), + hetznerLabelServerStatus: model.LabelValue(server.Server.Status), + hetznerLabelRobotDatacenter: model.LabelValue(strings.ToLower(server.Server.Dc)), + hetznerLabelRobotProduct: model.LabelValue(server.Server.Product), + hetznerLabelRobotCancelled: model.LabelValue(strconv.FormatBool(server.Server.Canceled)), model.AddressLabel: model.LabelValue(net.JoinHostPort(server.Server.ServerIP, strconv.FormatUint(uint64(d.port), 10))), } diff --git a/discovery/hetzner/robot_test.go b/discovery/hetzner/robot_test.go index abee5fea900..56f9978858f 100644 --- a/discovery/hetzner/robot_test.go +++ b/discovery/hetzner/robot_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,9 +18,9 @@ import ( "fmt" "testing" - "github.com/go-kit/log" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" ) @@ -42,7 +42,7 @@ func TestRobotSDRefresh(t *testing.T) { cfg.HTTPClientConfig.BasicAuth = &config.BasicAuth{Username: robotTestUsername, Password: robotTestPassword} cfg.robotEndpoint = suite.Mock.Endpoint() - d, err := newRobotDiscovery(&cfg, log.NewNopLogger()) + d, err := newRobotDiscovery(&cfg, promslog.NewNopLogger()) require.NoError(t, err) targetGroups, err := d.refresh(context.Background()) @@ -64,19 +64,21 @@ func TestRobotSDRefresh(t *testing.T) { "__meta_hetzner_public_ipv4": model.LabelValue("123.123.123.123"), "__meta_hetzner_public_ipv6_network": model.LabelValue("2a01:4f8:111:4221::/64"), "__meta_hetzner_datacenter": model.LabelValue("nbg1-dc1"), + "__meta_hetzner_robot_datacenter": model.LabelValue("nbg1-dc1"), "__meta_hetzner_robot_product": model.LabelValue("DS 3000"), "__meta_hetzner_robot_cancelled": model.LabelValue("false"), }, { - "__address__": model.LabelValue("123.123.123.124:80"), - "__meta_hetzner_role": model.LabelValue("robot"), - "__meta_hetzner_server_id": model.LabelValue("421"), - "__meta_hetzner_server_name": model.LabelValue("server2"), - "__meta_hetzner_server_status": model.LabelValue("in process"), - "__meta_hetzner_public_ipv4": model.LabelValue("123.123.123.124"), - "__meta_hetzner_datacenter": model.LabelValue("fsn1-dc10"), - "__meta_hetzner_robot_product": model.LabelValue("X5"), - "__meta_hetzner_robot_cancelled": model.LabelValue("true"), + "__address__": model.LabelValue("123.123.123.124:80"), + "__meta_hetzner_role": model.LabelValue("robot"), + "__meta_hetzner_server_id": model.LabelValue("421"), + "__meta_hetzner_server_name": model.LabelValue("server2"), + "__meta_hetzner_server_status": model.LabelValue("in process"), + "__meta_hetzner_public_ipv4": model.LabelValue("123.123.123.124"), + "__meta_hetzner_datacenter": model.LabelValue("fsn1-dc10"), + "__meta_hetzner_robot_datacenter": model.LabelValue("fsn1-dc10"), + "__meta_hetzner_robot_product": model.LabelValue("X5"), + "__meta_hetzner_robot_cancelled": model.LabelValue("true"), }, } { t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) { @@ -91,12 +93,11 @@ func TestRobotSDRefreshHandleError(t *testing.T) { cfg := DefaultSDConfig cfg.robotEndpoint = suite.Mock.Endpoint() - d, err := newRobotDiscovery(&cfg, log.NewNopLogger()) + d, err := newRobotDiscovery(&cfg, promslog.NewNopLogger()) require.NoError(t, err) targetGroups, err := d.refresh(context.Background()) - require.Error(t, err) - require.Equal(t, "non 2xx status '401' response during hetzner service discovery with role robot", err.Error()) + require.EqualError(t, err, "non 2xx status '401' response during hetzner service discovery with role robot") require.Empty(t, targetGroups) } diff --git a/discovery/http/http.go b/discovery/http/http.go index ff76fd76274..ce8cfa86767 100644 --- a/discovery/http/http.go +++ b/discovery/http/http.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -25,11 +25,11 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/grafana/regexp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/discovery" @@ -40,10 +40,10 @@ import ( var ( // DefaultSDConfig is the default HTTP SD configuration. DefaultSDConfig = SDConfig{ - RefreshInterval: model.Duration(60 * time.Second), HTTPClientConfig: config.DefaultHTTPClientConfig, + RefreshInterval: model.Duration(60 * time.Second), } - userAgent = fmt.Sprintf("Prometheus/%s", version.Version) + userAgent = version.PrometheusUserAgent() matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`) ) @@ -68,7 +68,7 @@ func (*SDConfig) Name() string { return "http" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.HTTPClientOptions, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -77,7 +77,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -85,17 +85,17 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } if c.URL == "" { - return fmt.Errorf("URL is missing") + return errors.New("URL is missing") } parsedURL, err := url.Parse(c.URL) if err != nil { return err } if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { - return fmt.Errorf("URL scheme must be 'http' or 'https'") + return errors.New("URL scheme must be 'http' or 'https'") } if parsedURL.Host == "" { - return fmt.Errorf("host is missing in URL") + return errors.New("host is missing in URL") } return c.HTTPClientConfig.Validate() } @@ -114,17 +114,17 @@ type Discovery struct { } // NewDiscovery returns a new HTTP discovery for the given config. -func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPClientOption, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*httpMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*httpMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - if logger == nil { - logger = log.NewNopLogger() + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() } - client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http", clientOpts...) + client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http", opts.HTTPClientOptions...) if err != nil { return nil, err } @@ -139,8 +139,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "http", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.Refresh, MetricsInstantiator: m.refreshMetrics, @@ -150,7 +151,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, clientOpts []config.HTTPCli } func (d *Discovery) Refresh(ctx context.Context) ([]*targetgroup.Group, error) { - req, err := http.NewRequest(http.MethodGet, d.url, nil) + req, err := http.NewRequest(http.MethodGet, d.url, http.NoBody) if err != nil { return nil, err } diff --git a/discovery/http/http_test.go b/discovery/http/http_test.go index 0cafe035dc3..50a5800fc67 100644 --- a/discovery/http/http_test.go +++ b/discovery/http/http_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,11 +21,11 @@ import ( "testing" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/discovery" @@ -49,7 +49,12 @@ func TestHTTPValidRefresh(t *testing.T) { require.NoError(t, metrics.Register()) defer metrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + HTTPClientOptions: nil, + Metrics: metrics, + SetName: "http", + }) require.NoError(t, err) ctx := context.Background() @@ -75,7 +80,7 @@ func TestHTTPValidRefresh(t *testing.T) { } func TestHTTPInvalidCode(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadRequest) })) @@ -94,7 +99,12 @@ func TestHTTPInvalidCode(t *testing.T) { require.NoError(t, metrics.Register()) defer metrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + HTTPClientOptions: nil, + Metrics: metrics, + SetName: "http", + }) require.NoError(t, err) ctx := context.Background() @@ -104,7 +114,7 @@ func TestHTTPInvalidCode(t *testing.T) { } func TestHTTPInvalidFormat(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintln(w, "{}") })) @@ -123,7 +133,12 @@ func TestHTTPInvalidFormat(t *testing.T) { require.NoError(t, metrics.Register()) defer metrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + HTTPClientOptions: nil, + Metrics: metrics, + SetName: "http", + }) require.NoError(t, err) ctx := context.Background() @@ -212,7 +227,7 @@ func TestContentTypeRegex(t *testing.T) { func TestSourceDisappeared(t *testing.T) { var stubResponse string - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") fmt.Fprintln(w, stubResponse) })) @@ -442,7 +457,12 @@ func TestSourceDisappeared(t *testing.T) { require.NoError(t, metrics.Register()) defer metrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), nil, metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + HTTPClientOptions: nil, + Metrics: metrics, + SetName: "http", + }) require.NoError(t, err) for _, test := range cases { ctx := context.Background() diff --git a/discovery/http/metrics.go b/discovery/http/metrics.go index b1f8b844339..57fbcac15a5 100644 --- a/discovery/http/metrics.go +++ b/discovery/http/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/install/install.go b/discovery/install/install.go index f090076b7ff..2cd0c838015 100644 --- a/discovery/install/install.go +++ b/discovery/install/install.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -33,9 +33,11 @@ import ( _ "github.com/prometheus/prometheus/discovery/moby" // register moby _ "github.com/prometheus/prometheus/discovery/nomad" // register nomad _ "github.com/prometheus/prometheus/discovery/openstack" // register openstack + _ "github.com/prometheus/prometheus/discovery/outscale" // register outscale _ "github.com/prometheus/prometheus/discovery/ovhcloud" // register ovhcloud _ "github.com/prometheus/prometheus/discovery/puppetdb" // register puppetdb _ "github.com/prometheus/prometheus/discovery/scaleway" // register scaleway + _ "github.com/prometheus/prometheus/discovery/stackit" // register stackit _ "github.com/prometheus/prometheus/discovery/triton" // register triton _ "github.com/prometheus/prometheus/discovery/uyuni" // register uyuni _ "github.com/prometheus/prometheus/discovery/vultr" // register vultr diff --git a/discovery/ionos/ionos.go b/discovery/ionos/ionos.go index c8b4f7f8e54..93d57654e85 100644 --- a/discovery/ionos/ionos.go +++ b/discovery/ionos/ionos.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,10 +15,8 @@ package ionos import ( "errors" - "fmt" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -43,25 +41,26 @@ func init() { type Discovery struct{} // NewDiscovery returns a new refresh.Discovery for IONOS Cloud. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { - m, ok := metrics.(*ionosMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*refresh.Discovery, error) { + m, ok := opts.Metrics.(*ionosMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } if conf.ionosEndpoint == "" { conf.ionosEndpoint = "https://api.ionos.com" } - d, err := newServerDiscovery(conf, logger) + d, err := newServerDiscovery(conf, opts.Logger) if err != nil { return nil, err } return refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "ionos", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -90,24 +89,24 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &ionosMetrics{ refreshMetrics: rmi, } } // Name returns the name of the IONOS Cloud service discovery. -func (c SDConfig) Name() string { +func (SDConfig) Name() string { return "ionos" } // NewDiscoverer returns a new discovery.Discoverer for IONOS Cloud. -func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(&c, options.Logger, options.Metrics) +func (c SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(&c, opts) } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) diff --git a/discovery/ionos/metrics.go b/discovery/ionos/metrics.go index 88e465acf3b..7fc78fdfa50 100644 --- a/discovery/ionos/metrics.go +++ b/discovery/ionos/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type ionosMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *ionosMetrics) Register() error { +func (*ionosMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *ionosMetrics) Unregister() {} +func (*ionosMetrics) Unregister() {} diff --git a/discovery/ionos/server.go b/discovery/ionos/server.go index a850fbbfb42..bd351625db2 100644 --- a/discovery/ionos/server.go +++ b/discovery/ionos/server.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,14 +15,13 @@ package ionos import ( "context" - "fmt" + "log/slog" "net" "net/http" "strconv" "strings" "time" - "github.com/go-kit/log" ionoscloud "github.com/ionos-cloud/sdk-go/v6" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -60,7 +59,7 @@ type serverDiscovery struct { datacenterID string } -func newServerDiscovery(conf *SDConfig, _ log.Logger) (*serverDiscovery, error) { +func newServerDiscovery(conf *SDConfig, _ *slog.Logger) (*serverDiscovery, error) { d := &serverDiscovery{ port: conf.Port, datacenterID: conf.DatacenterID, @@ -77,7 +76,7 @@ func newServerDiscovery(conf *SDConfig, _ log.Logger) (*serverDiscovery, error) Transport: rt, Timeout: time.Duration(conf.RefreshInterval), } - cfg.UserAgent = fmt.Sprintf("Prometheus/%s", version.Version) + cfg.UserAgent = version.PrometheusUserAgent() d.client = ionoscloud.NewAPIClient(cfg) diff --git a/discovery/ionos/server_test.go b/discovery/ionos/server_test.go index 30f358e3253..28fd285f67b 100644 --- a/discovery/ionos/server_test.go +++ b/discovery/ionos/server_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/kubernetes/client.go b/discovery/kubernetes/client.go new file mode 100644 index 00000000000..21757a8b742 --- /dev/null +++ b/discovery/kubernetes/client.go @@ -0,0 +1,76 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "k8s.io/client-go/kubernetes" + appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1" + batchv1client "k8s.io/client-go/kubernetes/typed/batch/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + discoveryv1client "k8s.io/client-go/kubernetes/typed/discovery/v1" + networkingv1client "k8s.io/client-go/kubernetes/typed/networking/v1" +) + +// k8sClient exposes only the API-group accessors discovery uses. It is +// satisfied by both clientAdapter and the fake clientset used in tests. +type k8sClient interface { + CoreV1() corev1client.CoreV1Interface + AppsV1() appsv1client.AppsV1Interface + BatchV1() batchv1client.BatchV1Interface + DiscoveryV1() discoveryv1client.DiscoveryV1Interface + NetworkingV1() networkingv1client.NetworkingV1Interface +} + +// clientAdapter captures the used API-group accessors as method-value closures. +// Keeping the concrete clientset out of an interface-typed field avoids forcing +// the linker, via reflection, to retain every API group and resource client, +// which reduces the binary size. +type clientAdapter struct { + coreV1 func() corev1client.CoreV1Interface + appsV1 func() appsv1client.AppsV1Interface + batchV1 func() batchv1client.BatchV1Interface + discoveryV1 func() discoveryv1client.DiscoveryV1Interface + networkingV1 func() networkingv1client.NetworkingV1Interface +} + +// newClientAdapter wraps a concrete clientset, exposing only the API groups +// discovery uses. +func newClientAdapter(c *kubernetes.Clientset) *clientAdapter { + return &clientAdapter{ + coreV1: c.CoreV1, + appsV1: c.AppsV1, + batchV1: c.BatchV1, + discoveryV1: c.DiscoveryV1, + networkingV1: c.NetworkingV1, + } +} + +// CoreV1 returns the core/v1 API-group client. +func (a *clientAdapter) CoreV1() corev1client.CoreV1Interface { return a.coreV1() } + +// AppsV1 returns the apps/v1 API-group client. +func (a *clientAdapter) AppsV1() appsv1client.AppsV1Interface { return a.appsV1() } + +// BatchV1 returns the batch/v1 API-group client. +func (a *clientAdapter) BatchV1() batchv1client.BatchV1Interface { return a.batchV1() } + +// DiscoveryV1 returns the discovery/v1 API-group client. +func (a *clientAdapter) DiscoveryV1() discoveryv1client.DiscoveryV1Interface { + return a.discoveryV1() +} + +// NetworkingV1 returns the networking/v1 API-group client. +func (a *clientAdapter) NetworkingV1() networkingv1client.NetworkingV1Interface { + return a.networkingV1() +} diff --git a/discovery/kubernetes/endpoints.go b/discovery/kubernetes/endpoints.go index c7a60ae6d3f..9bfcea40249 100644 --- a/discovery/kubernetes/endpoints.go +++ b/discovery/kubernetes/endpoints.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,13 +17,13 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" apiv1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -32,26 +32,37 @@ import ( ) // Endpoints discovers new endpoint targets. +// +// Deprecated: The Endpoints API is deprecated starting in K8s v1.33+. Use EndpointSlice. type Endpoints struct { - logger log.Logger - - endpointsInf cache.SharedIndexInformer - serviceInf cache.SharedInformer - podInf cache.SharedInformer - nodeInf cache.SharedInformer - withNodeMetadata bool + logger *slog.Logger + + endpointsInf cache.SharedIndexInformer + serviceInf cache.SharedInformer + podInf cache.SharedInformer + nodeInf cache.SharedInformer + withNodeMetadata bool + namespaceInf cache.SharedInformer + withNamespaceMetadata bool + replicaSetInf cache.SharedInformer + withDeploymentMetadata bool + jobInf cache.SharedInformer + withJobMetadata bool + withCronJobMetadata bool podStore cache.Store endpointsStore cache.Store serviceStore cache.Store - queue *workqueue.Type + queue *workqueue.Typed[string] } // NewEndpoints returns a new endpoints discovery. -func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *Endpoints { +// +// Deprecated: The Endpoints API is deprecated starting in K8s v1.33+. Use NewEndpointSlice. +func NewEndpoints(l *slog.Logger, eps cache.SharedIndexInformer, svc, pod, node, namespace, rs, job cache.SharedInformer, withDeploymentMetadata, withJobMetadata, withCronJobMetadata bool, eventCount *prometheus.CounterVec) *Endpoints { if l == nil { - l = log.NewNopLogger() + l = promslog.NewNopLogger() } epAddCount := eventCount.WithLabelValues(RoleEndpoint.String(), MetricLabelRoleAdd) @@ -65,76 +76,82 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca podUpdateCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleUpdate) e := &Endpoints{ - logger: l, - endpointsInf: eps, - endpointsStore: eps.GetStore(), - serviceInf: svc, - serviceStore: svc.GetStore(), - podInf: pod, - podStore: pod.GetStore(), - nodeInf: node, - withNodeMetadata: node != nil, - queue: workqueue.NewNamed(RoleEndpoint.String()), + logger: l, + endpointsInf: eps, + endpointsStore: eps.GetStore(), + serviceInf: svc, + serviceStore: svc.GetStore(), + podInf: pod, + podStore: pod.GetStore(), + nodeInf: node, + withNodeMetadata: node != nil, + namespaceInf: namespace, + withNamespaceMetadata: namespace != nil, + replicaSetInf: rs, + withDeploymentMetadata: withDeploymentMetadata, + jobInf: job, + withJobMetadata: withJobMetadata, + withCronJobMetadata: withCronJobMetadata, + queue: workqueue.NewTypedWithConfig(workqueue.TypedQueueConfig[string]{ + Name: RoleEndpoint.String(), + }), } _, err := e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { epAddCount.Inc() e.enqueue(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { epUpdateCount.Inc() e.enqueue(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { epDeleteCount.Inc() e.enqueue(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding endpoints event handler.", "err", err) + l.Error("Error adding endpoints event handler.", "err", err) } - serviceUpdate := func(o interface{}) { + serviceUpdate := func(o any) { svc, err := convertToService(o) if err != nil { - level.Error(e.logger).Log("msg", "converting to Service object failed", "err", err) + e.logger.Error("converting to Service object failed", "err", err) return } - ep := &apiv1.Endpoints{} - ep.Namespace = svc.Namespace - ep.Name = svc.Name - obj, exists, err := e.endpointsStore.Get(ep) + obj, exists, err := e.endpointsStore.GetByKey(namespacedName(svc.Namespace, svc.Name)) if exists && err == nil { e.enqueue(obj.(*apiv1.Endpoints)) } if err != nil { - level.Error(e.logger).Log("msg", "retrieving endpoints failed", "err", err) + e.logger.Error("retrieving endpoints failed", "err", err) } } _, err = e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ // TODO(fabxc): potentially remove add and delete event handlers. Those should // be triggered via the endpoint handlers already. - AddFunc: func(o interface{}) { + AddFunc: func(o any) { svcAddCount.Inc() serviceUpdate(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { svcUpdateCount.Inc() serviceUpdate(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { svcDeleteCount.Inc() serviceUpdate(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding services event handler.", "err", err) + l.Error("Error adding services event handler.", "err", err) } _, err = e.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - UpdateFunc: func(old, cur interface{}) { + UpdateFunc: func(old, cur any) { podUpdateCount.Inc() oldPod, ok := old.(*apiv1.Pod) if !ok { @@ -154,25 +171,42 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca }, }) if err != nil { - level.Error(l).Log("msg", "Error adding pods event handler.", "err", err) + l.Error("Error adding pods event handler.", "err", err) } if e.withNodeMetadata { _, err = e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { node := o.(*apiv1.Node) e.enqueueNode(node.Name) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { node := o.(*apiv1.Node) e.enqueueNode(node.Name) }, - DeleteFunc: func(o interface{}) { - node := o.(*apiv1.Node) - e.enqueueNode(node.Name) + DeleteFunc: func(o any) { + nodeName, err := nodeName(o) + if err != nil { + l.Error("Error getting Node name", "err", err) + } + e.enqueueNode(nodeName) + }, + }) + if err != nil { + l.Error("Error adding nodes event handler.", "err", err) + } + } + + if e.withNamespaceMetadata { + _, err = e.namespaceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, o any) { + namespace := o.(*apiv1.Namespace) + e.enqueueNamespace(namespace.Name) }, + // Creation and deletion will trigger events for the change handlers of the resources within the namespace. + // No need to have additional handlers for them here. }) if err != nil { - level.Error(l).Log("msg", "Error adding nodes event handler.", "err", err) + l.Error("Error adding namespaces event handler.", "err", err) } } @@ -182,7 +216,19 @@ func NewEndpoints(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node ca func (e *Endpoints) enqueueNode(nodeName string) { endpoints, err := e.endpointsInf.GetIndexer().ByIndex(nodeIndex, nodeName) if err != nil { - level.Error(e.logger).Log("msg", "Error getting endpoints for node", "node", nodeName, "err", err) + e.logger.Error("Error getting endpoints for node", "node", nodeName, "err", err) + return + } + + for _, endpoint := range endpoints { + e.enqueue(endpoint) + } +} + +func (e *Endpoints) enqueueNamespace(namespace string) { + endpoints, err := e.endpointsInf.GetIndexer().ByIndex(cache.NamespaceIndex, namespace) + if err != nil { + e.logger.Error("Error getting endpoints in namespace", "namespace", namespace, "err", err) return } @@ -194,7 +240,7 @@ func (e *Endpoints) enqueueNode(nodeName string) { func (e *Endpoints) enqueuePod(podNamespacedName string) { endpoints, err := e.endpointsInf.GetIndexer().ByIndex(podIndex, podNamespacedName) if err != nil { - level.Error(e.logger).Log("msg", "Error getting endpoints for pod", "pod", podNamespacedName, "err", err) + e.logger.Error("Error getting endpoints for pod", "pod", podNamespacedName, "err", err) return } @@ -203,7 +249,7 @@ func (e *Endpoints) enqueuePod(podNamespacedName string) { } } -func (e *Endpoints) enqueue(obj interface{}) { +func (e *Endpoints) enqueue(obj any) { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { return @@ -220,10 +266,13 @@ func (e *Endpoints) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { if e.withNodeMetadata { cacheSyncs = append(cacheSyncs, e.nodeInf.HasSynced) } + if e.withNamespaceMetadata { + cacheSyncs = append(cacheSyncs, e.namespaceInf.HasSynced) + } if !cache.WaitForCacheSync(ctx.Done(), cacheSyncs...) { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(e.logger).Log("msg", "endpoints informer unable to sync cache") + e.logger.Error("endpoints informer unable to sync cache") } return } @@ -238,22 +287,21 @@ func (e *Endpoints) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } func (e *Endpoints) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool { - keyObj, quit := e.queue.Get() + key, quit := e.queue.Get() if quit { return false } - defer e.queue.Done(keyObj) - key := keyObj.(string) + defer e.queue.Done(key) namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { - level.Error(e.logger).Log("msg", "splitting key failed", "key", key) + e.logger.Error("splitting key failed", "key", key) return true } o, exists, err := e.endpointsStore.GetByKey(key) if err != nil { - level.Error(e.logger).Log("msg", "getting object from store failed", "key", key) + e.logger.Error("getting object from store failed", "key", key) return true } if !exists { @@ -262,14 +310,14 @@ func (e *Endpoints) process(ctx context.Context, ch chan<- []*targetgroup.Group) } eps, err := convertToEndpoints(o) if err != nil { - level.Error(e.logger).Log("msg", "converting to Endpoints object failed", "err", err) + e.logger.Error("converting to Endpoints object failed", "err", err) return true } send(ctx, ch, e.buildEndpoints(eps)) return true } -func convertToEndpoints(o interface{}) (*apiv1.Endpoints, error) { +func convertToEndpoints(o any) (*apiv1.Endpoints, error) { endpoints, ok := o.(*apiv1.Endpoints) if ok { return endpoints, nil @@ -307,6 +355,10 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { // Add endpoints labels metadata. addObjectMetaLabels(tg.Labels, eps.ObjectMeta, RoleEndpoint) + if e.withNamespaceMetadata { + tg.Labels = addNamespaceLabels(tg.Labels, e.namespaceInf, e.logger, eps.Namespace) + } + type podEntry struct { pod *apiv1.Pod servicePorts []apiv1.EndpointPort @@ -358,19 +410,22 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { } // Attach standard pod labels. - target = target.Merge(podLabels(pod)) + target = target.Merge(podLabels(pod, e.replicaSetInf, e.jobInf, e.withDeploymentMetadata, e.withJobMetadata, e.withCronJobMetadata)) // Attach potential container port labels matching the endpoint port. - for _, c := range pod.Spec.Containers { + containers := append(pod.Spec.Containers, pod.Spec.InitContainers...) + for i, c := range containers { for _, cport := range c.Ports { if port.Port == cport.ContainerPort { ports := strconv.FormatUint(uint64(port.Port), 10) + isInit := i >= len(pod.Spec.Containers) target[podContainerNameLabel] = lv(c.Name) target[podContainerImageLabel] = lv(c.Image) target[podContainerPortNameLabel] = lv(cport.Name) target[podContainerPortNumberLabel] = lv(ports) target[podContainerPortProtocolLabel] = lv(string(port.Protocol)) + target[podContainerIsInit] = lv(strconv.FormatBool(isInit)) break } } @@ -397,21 +452,22 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { v := eps.Labels[apiv1.EndpointsOverCapacity] if v == "truncated" { - level.Warn(e.logger).Log("msg", "Number of endpoints in one Endpoints object exceeds 1000 and has been truncated, please use \"role: endpointslice\" instead", "endpoint", eps.Name) + e.logger.Warn("Number of endpoints in one Endpoints object exceeds 1000 and has been truncated, please use \"role: endpointslice\" instead", "endpoint", eps.Name) } if v == "warning" { - level.Warn(e.logger).Log("msg", "Number of endpoints in one Endpoints object exceeds 1000, please use \"role: endpointslice\" instead", "endpoint", eps.Name) + e.logger.Warn("Number of endpoints in one Endpoints object exceeds 1000, please use \"role: endpointslice\" instead", "endpoint", eps.Name) } // For all seen pods, check all container ports. If they were not covered // by one of the service endpoints, generate targets for them. for _, pe := range seenPods { // PodIP can be empty when a pod is starting or has been evicted. - if len(pe.pod.Status.PodIP) == 0 { + if pe.pod.Status.PodIP == "" { continue } - for _, c := range pe.pod.Spec.Containers { + containers := append(pe.pod.Spec.Containers, pe.pod.Spec.InitContainers...) + for i, c := range containers { for _, cport := range c.Ports { hasSeenPort := func() bool { for _, eport := range pe.servicePorts { @@ -428,6 +484,7 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10)) ports := strconv.FormatUint(uint64(cport.ContainerPort), 10) + isInit := i >= len(pe.pod.Spec.Containers) target := model.LabelSet{ model.AddressLabel: lv(a), podContainerNameLabel: lv(c.Name), @@ -435,8 +492,9 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { podContainerPortNameLabel: lv(cport.Name), podContainerPortNumberLabel: lv(ports), podContainerPortProtocolLabel: lv(string(cport.Protocol)), + podContainerIsInit: lv(strconv.FormatBool(isInit)), } - tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod))) + tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod, e.replicaSetInf, e.jobInf, e.withDeploymentMetadata, e.withJobMetadata, e.withCronJobMetadata))) } } } @@ -448,13 +506,10 @@ func (e *Endpoints) resolvePodRef(ref *apiv1.ObjectReference) *apiv1.Pod { if ref == nil || ref.Kind != "Pod" { return nil } - p := &apiv1.Pod{} - p.Namespace = ref.Namespace - p.Name = ref.Name - obj, exists, err := e.podStore.Get(p) + obj, exists, err := e.podStore.GetByKey(namespacedName(ref.Namespace, ref.Name)) if err != nil { - level.Error(e.logger).Log("msg", "resolving pod ref failed", "err", err) + e.logger.Error("resolving pod ref failed", "err", err) return nil } if !exists { @@ -464,31 +519,27 @@ func (e *Endpoints) resolvePodRef(ref *apiv1.ObjectReference) *apiv1.Pod { } func (e *Endpoints) addServiceLabels(ns, name string, tg *targetgroup.Group) { - svc := &apiv1.Service{} - svc.Namespace = ns - svc.Name = name - - obj, exists, err := e.serviceStore.Get(svc) + obj, exists, err := e.serviceStore.GetByKey(namespacedName(ns, name)) if err != nil { - level.Error(e.logger).Log("msg", "retrieving service failed", "err", err) + e.logger.Error("retrieving service failed", "err", err) return } if !exists { return } - svc = obj.(*apiv1.Service) + svc := obj.(*apiv1.Service) tg.Labels = tg.Labels.Merge(serviceLabels(svc)) } -func addNodeLabels(tg model.LabelSet, nodeInf cache.SharedInformer, logger log.Logger, nodeName *string) model.LabelSet { +func addNodeLabels(tg model.LabelSet, nodeInf cache.SharedInformer, logger *slog.Logger, nodeName *string) model.LabelSet { if nodeName == nil { return tg } obj, exists, err := nodeInf.GetStore().GetByKey(*nodeName) if err != nil { - level.Error(logger).Log("msg", "Error getting node", "node", *nodeName, "err", err) + logger.Error("Error getting node", "node", *nodeName, "err", err) return tg } @@ -502,3 +553,20 @@ func addNodeLabels(tg model.LabelSet, nodeInf cache.SharedInformer, logger log.L addObjectMetaLabels(nodeLabelset, node.ObjectMeta, RoleNode) return tg.Merge(nodeLabelset) } + +func addNamespaceLabels(tg model.LabelSet, namespaceInf cache.SharedInformer, logger *slog.Logger, namespace string) model.LabelSet { + obj, exists, err := namespaceInf.GetStore().GetByKey(namespace) + if err != nil { + logger.Error("Error getting namespace", "namespace", namespace, "err", err) + return tg + } + + if !exists { + return tg + } + + n := obj.(*apiv1.Namespace) + namespaceLabelset := make(model.LabelSet) + addNamespaceMetaLabels(namespaceLabelset, n.ObjectMeta) + return tg.Merge(namespaceLabelset) +} diff --git a/discovery/kubernetes/endpoints_test.go b/discovery/kubernetes/endpoints_test.go index 3ea98c5db97..0427ad43c1b 100644 --- a/discovery/kubernetes/endpoints_test.go +++ b/discovery/kubernetes/endpoints_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,23 +15,26 @@ package kubernetes import ( "context" + "fmt" "testing" "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" "github.com/prometheus/prometheus/discovery/targetgroup" ) -func makeEndpoints() *v1.Endpoints { +func makeEndpoints(namespace string) *v1.Endpoints { nodeName := "foobar" return &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "testendpoints", - Namespace: "default", + Namespace: namespace, Annotations: map[string]string{ "test.annotation": "test", }, @@ -95,12 +98,13 @@ func makeEndpoints() *v1.Endpoints { } func TestEndpointsDiscoveryBeforeRun(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}) k8sDiscoveryTest{ discovery: n, beforeRun: func() { - obj := makeEndpoints() + obj := makeEndpoints("default") c.CoreV1().Endpoints(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) }, expectedMaxItems: 1, @@ -149,6 +153,7 @@ func TestEndpointsDiscoveryBeforeRun(t *testing.T) { } func TestEndpointsDiscoveryAdd(t *testing.T) { + t.Parallel() obj := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "testpod", @@ -244,6 +249,7 @@ func TestEndpointsDiscoveryAdd(t *testing.T) { "__meta_kubernetes_pod_container_port_number": "9000", "__meta_kubernetes_pod_container_port_protocol": "TCP", "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", }, { "__address__": "1.2.3.4:9001", @@ -259,6 +265,7 @@ func TestEndpointsDiscoveryAdd(t *testing.T) { "__meta_kubernetes_pod_container_port_number": "9001", "__meta_kubernetes_pod_container_port_protocol": "TCP", "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", }, }, Labels: model.LabelSet{ @@ -272,12 +279,13 @@ func TestEndpointsDiscoveryAdd(t *testing.T) { } func TestEndpointsDiscoveryDelete(t *testing.T) { - n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints()) + t.Parallel() + n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints("default")) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makeEndpoints() + obj := makeEndpoints("default") c.CoreV1().Endpoints(obj.Namespace).Delete(context.Background(), obj.Name, metav1.DeleteOptions{}) }, expectedMaxItems: 2, @@ -290,7 +298,8 @@ func TestEndpointsDiscoveryDelete(t *testing.T) { } func TestEndpointsDiscoveryUpdate(t *testing.T) { - n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints()) + t.Parallel() + n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints("default")) k8sDiscoveryTest{ discovery: n, @@ -361,7 +370,8 @@ func TestEndpointsDiscoveryUpdate(t *testing.T) { } func TestEndpointsDiscoveryEmptySubsets(t *testing.T) { - n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints()) + t.Parallel() + n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints("default")) k8sDiscoveryTest{ discovery: n, @@ -389,7 +399,8 @@ func TestEndpointsDiscoveryEmptySubsets(t *testing.T) { } func TestEndpointsDiscoveryWithService(t *testing.T) { - n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints()) + t.Parallel() + n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints("default")) k8sDiscoveryTest{ discovery: n, @@ -454,7 +465,8 @@ func TestEndpointsDiscoveryWithService(t *testing.T) { } func TestEndpointsDiscoveryWithServiceUpdate(t *testing.T) { - n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints()) + t.Parallel() + n, c := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, makeEndpoints("default")) k8sDiscoveryTest{ discovery: n, @@ -534,11 +546,12 @@ func TestEndpointsDiscoveryWithServiceUpdate(t *testing.T) { } func TestEndpointsDiscoveryWithNodeMetadata(t *testing.T) { + t.Parallel() metadataConfig := AttachMetadataConfig{Node: true} nodeLabels1 := map[string]string{"az": "us-east1"} nodeLabels2 := map[string]string{"az": "us-west2"} - node1 := makeNode("foobar", "", "", nodeLabels1, nil) - node2 := makeNode("barbaz", "", "", nodeLabels2, nil) + node1 := makeNode("foobar", "", "", nodeLabels1, nil, nil) + node2 := makeNode("barbaz", "", "", nodeLabels2, nil, nil) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "testendpoints", @@ -548,7 +561,7 @@ func TestEndpointsDiscoveryWithNodeMetadata(t *testing.T) { }, }, } - n, _ := makeDiscoveryWithMetadata(RoleEndpoint, NamespaceDiscovery{}, metadataConfig, makeEndpoints(), svc, node1, node2) + n, _ := makeDiscoveryWithMetadata(RoleEndpoint, NamespaceDiscovery{}, metadataConfig, makeEndpoints("default"), svc, node1, node2) k8sDiscoveryTest{ discovery: n, @@ -607,10 +620,11 @@ func TestEndpointsDiscoveryWithNodeMetadata(t *testing.T) { } func TestEndpointsDiscoveryWithUpdatedNodeMetadata(t *testing.T) { + t.Parallel() nodeLabels1 := map[string]string{"az": "us-east1"} nodeLabels2 := map[string]string{"az": "us-west2"} - node1 := makeNode("foobar", "", "", nodeLabels1, nil) - node2 := makeNode("barbaz", "", "", nodeLabels2, nil) + node1 := makeNode("foobar", "", "", nodeLabels1, nil, nil) + node2 := makeNode("barbaz", "", "", nodeLabels2, nil, nil) metadataConfig := AttachMetadataConfig{Node: true} svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -621,7 +635,7 @@ func TestEndpointsDiscoveryWithUpdatedNodeMetadata(t *testing.T) { }, }, } - n, c := makeDiscoveryWithMetadata(RoleEndpoint, NamespaceDiscovery{}, metadataConfig, makeEndpoints(), node1, node2, svc) + n, c := makeDiscoveryWithMetadata(RoleEndpoint, NamespaceDiscovery{}, metadataConfig, makeEndpoints("default"), node1, node2, svc) k8sDiscoveryTest{ discovery: n, @@ -684,7 +698,8 @@ func TestEndpointsDiscoveryWithUpdatedNodeMetadata(t *testing.T) { } func TestEndpointsDiscoveryNamespaces(t *testing.T) { - epOne := makeEndpoints() + t.Parallel() + epOne := makeEndpoints("default") epOne.Namespace = "ns1" objs := []runtime.Object{ epOne, @@ -821,6 +836,7 @@ func TestEndpointsDiscoveryNamespaces(t *testing.T) { "__meta_kubernetes_pod_container_port_number": "9000", "__meta_kubernetes_pod_container_port_protocol": "TCP", "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", }, }, Labels: model.LabelSet{ @@ -834,10 +850,11 @@ func TestEndpointsDiscoveryNamespaces(t *testing.T) { } func TestEndpointsDiscoveryOwnNamespace(t *testing.T) { - epOne := makeEndpoints() + t.Parallel() + epOne := makeEndpoints("default") epOne.Namespace = "own-ns" - epTwo := makeEndpoints() + epTwo := makeEndpoints("default") epTwo.Namespace = "non-own-ns" podOne := &v1.Pod{ @@ -928,7 +945,8 @@ func TestEndpointsDiscoveryOwnNamespace(t *testing.T) { } func TestEndpointsDiscoveryEmptyPodStatus(t *testing.T) { - ep := makeEndpoints() + t.Parallel() + ep := makeEndpoints("default") ep.Namespace = "ns" pod := &v1.Pod{ @@ -973,6 +991,7 @@ func TestEndpointsDiscoveryEmptyPodStatus(t *testing.T) { // TestEndpointsDiscoveryUpdatePod makes sure that Endpoints discovery detects underlying Pods changes. // See https://github.com/prometheus/prometheus/issues/11305 for more details. func TestEndpointsDiscoveryUpdatePod(t *testing.T) { + t.Parallel() pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "testpod", @@ -1078,6 +1097,7 @@ func TestEndpointsDiscoveryUpdatePod(t *testing.T) { "__meta_kubernetes_pod_container_port_number": "9000", "__meta_kubernetes_pod_container_port_protocol": "TCP", "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", }, }, Labels: model.LabelSet{ @@ -1089,3 +1109,325 @@ func TestEndpointsDiscoveryUpdatePod(t *testing.T) { }, }.Run(t) } + +func TestEndpointsDiscoverySidecarContainer(t *testing.T) { + t.Parallel() + objs := []runtime.Object{ + &v1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testsidecar", + Namespace: "default", + }, + Subsets: []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{ + { + IP: "4.3.2.1", + TargetRef: &v1.ObjectReference{ + Kind: "Pod", + Name: "testpod", + Namespace: "default", + }, + }, + }, + Ports: []v1.EndpointPort{ + { + Name: "testport", + Port: 9000, + Protocol: v1.ProtocolTCP, + }, + { + Name: "initport", + Port: 9111, + Protocol: v1.ProtocolTCP, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + Namespace: "default", + UID: types.UID("deadbeef"), + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + InitContainers: []v1.Container{ + { + Name: "ic1", + Image: "ic1:latest", + Ports: []v1.ContainerPort{ + { + Name: "initport", + ContainerPort: 1111, + Protocol: v1.ProtocolTCP, + }, + }, + }, + { + Name: "ic2", + Image: "ic2:latest", + Ports: []v1.ContainerPort{ + { + Name: "initport", + ContainerPort: 9111, + Protocol: v1.ProtocolTCP, + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: "c1", + Image: "c1:latest", + Ports: []v1.ContainerPort{ + { + Name: "mainport", + ContainerPort: 9000, + Protocol: v1.ProtocolTCP, + }, + }, + }, + }, + }, + Status: v1.PodStatus{ + HostIP: "2.3.4.5", + PodIP: "4.3.2.1", + }, + }, + } + + n, _ := makeDiscovery(RoleEndpoint, NamespaceDiscovery{}, objs...) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "endpoints/default/testsidecar": { + Targets: []model.LabelSet{ + { + "__address__": "4.3.2.1:9000", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "testpod", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_pod_container_image": "c1:latest", + "__meta_kubernetes_pod_container_name": "c1", + "__meta_kubernetes_pod_container_port_name": "mainport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ip": "4.3.2.1", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", + }, + { + "__address__": "4.3.2.1:9111", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "testpod", + "__meta_kubernetes_endpoint_port_name": "initport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_pod_container_image": "ic2:latest", + "__meta_kubernetes_pod_container_name": "ic2", + "__meta_kubernetes_pod_container_port_name": "initport", + "__meta_kubernetes_pod_container_port_number": "9111", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ip": "4.3.2.1", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "true", + }, + { + "__address__": "4.3.2.1:1111", + "__meta_kubernetes_pod_container_image": "ic1:latest", + "__meta_kubernetes_pod_container_name": "ic1", + "__meta_kubernetes_pod_container_port_name": "initport", + "__meta_kubernetes_pod_container_port_number": "1111", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ip": "4.3.2.1", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "true", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_endpoints_name": "testsidecar", + "__meta_kubernetes_namespace": "default", + }, + Source: "endpoints/default/testsidecar", + }, + }, + }.Run(t) +} + +func TestEndpointsDiscoveryWithNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"environment": "production", "team": "backend"} + nsAnnotations := map[string]string{"owner": "platform", "version": "v1.2.3"} + + n, _ := makeDiscoveryWithMetadata(RoleEndpoint, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, makeNamespace(ns, nsLabels, nsAnnotations), makeEndpoints(ns)) + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("endpoints/%s/testendpoints", ns): { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_endpoint_hostname": "testendpoint1", + "__meta_kubernetes_endpoint_node_name": "foobar", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + }, + { + "__address__": "2.3.4.5:9001", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + }, + { + "__address__": "2.3.4.5:9001", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "false", + }, + { + "__address__": "6.7.8.9:9002", + "__meta_kubernetes_endpoint_address_target_kind": "Node", + "__meta_kubernetes_endpoint_address_target_name": "barbaz", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_owner": "platform", + "__meta_kubernetes_namespace_annotationpresent_owner": "true", + "__meta_kubernetes_namespace_annotation_version": "v1.2.3", + "__meta_kubernetes_namespace_annotationpresent_version": "true", + "__meta_kubernetes_namespace_label_environment": "production", + "__meta_kubernetes_namespace_labelpresent_environment": "true", + "__meta_kubernetes_namespace_label_team": "backend", + "__meta_kubernetes_namespace_labelpresent_team": "true", + "__meta_kubernetes_endpoints_name": "testendpoints", + "__meta_kubernetes_endpoints_annotation_test_annotation": "test", + "__meta_kubernetes_endpoints_annotationpresent_test_annotation": "true", + }, + Source: fmt.Sprintf("endpoints/%s/testendpoints", ns), + }, + }, + }.Run(t) +} + +func TestEndpointsDiscoveryWithUpdatedNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"environment": "development", "team": "frontend"} + nsAnnotations := map[string]string{"owner": "devops", "version": "v2.1.0"} + + namespace := makeNamespace(ns, nsLabels, nsAnnotations) + n, c := makeDiscoveryWithMetadata(RoleEndpoint, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, namespace, makeEndpoints(ns)) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 2, + afterStart: func() { + namespace.Labels["environment"] = "staging" + namespace.Labels["region"] = "us-west" + namespace.Annotations["owner"] = "sre" + namespace.Annotations["cost-center"] = "engineering" + c.CoreV1().Namespaces().Update(context.Background(), namespace, metav1.UpdateOptions{}) + }, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("endpoints/%s/testendpoints", ns): { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_endpoint_hostname": "testendpoint1", + "__meta_kubernetes_endpoint_node_name": "foobar", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + }, + { + "__address__": "2.3.4.5:9001", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + }, + { + "__address__": "2.3.4.5:9001", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "false", + }, + { + "__address__": "6.7.8.9:9002", + "__meta_kubernetes_endpoint_address_target_kind": "Node", + "__meta_kubernetes_endpoint_address_target_name": "barbaz", + "__meta_kubernetes_endpoint_port_name": "testport", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_owner": "sre", + "__meta_kubernetes_namespace_annotationpresent_owner": "true", + "__meta_kubernetes_namespace_annotation_version": "v2.1.0", + "__meta_kubernetes_namespace_annotationpresent_version": "true", + "__meta_kubernetes_namespace_annotation_cost_center": "engineering", + "__meta_kubernetes_namespace_annotationpresent_cost_center": "true", + "__meta_kubernetes_namespace_label_environment": "staging", + "__meta_kubernetes_namespace_labelpresent_environment": "true", + "__meta_kubernetes_namespace_label_team": "frontend", + "__meta_kubernetes_namespace_labelpresent_team": "true", + "__meta_kubernetes_namespace_label_region": "us-west", + "__meta_kubernetes_namespace_labelpresent_region": "true", + "__meta_kubernetes_endpoints_name": "testendpoints", + "__meta_kubernetes_endpoints_annotation_test_annotation": "test", + "__meta_kubernetes_endpoints_annotationpresent_test_annotation": "true", + }, + Source: fmt.Sprintf("endpoints/%s/testendpoints", ns), + }, + }, + }.Run(t) +} + +func BenchmarkResolvePodRef(b *testing.B) { + indexer := cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, nil) + e := &Endpoints{ + podStore: indexer, + } + + b.ReportAllocs() + + for b.Loop() { + p := e.resolvePodRef(&v1.ObjectReference{ + Kind: "Pod", + Name: "testpod", + Namespace: "foo", + }) + require.Nil(b, p) + } +} diff --git a/discovery/kubernetes/endpointslice.go b/discovery/kubernetes/endpointslice.go index 7a70255c122..708a41f3eef 100644 --- a/discovery/kubernetes/endpointslice.go +++ b/discovery/kubernetes/endpointslice.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,17 +16,15 @@ package kubernetes import ( "context" "errors" - "fmt" + "log/slog" "net" "strconv" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" apiv1 "k8s.io/api/core/v1" v1 "k8s.io/api/discovery/v1" - "k8s.io/api/discovery/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -34,27 +32,36 @@ import ( "github.com/prometheus/prometheus/util/strutil" ) +const serviceIndex = "service" + // EndpointSlice discovers new endpoint targets. type EndpointSlice struct { - logger log.Logger - - endpointSliceInf cache.SharedIndexInformer - serviceInf cache.SharedInformer - podInf cache.SharedInformer - nodeInf cache.SharedInformer - withNodeMetadata bool + logger *slog.Logger + + endpointSliceInf cache.SharedIndexInformer + serviceInf cache.SharedInformer + podInf cache.SharedInformer + nodeInf cache.SharedInformer + withNodeMetadata bool + namespaceInf cache.SharedInformer + withNamespaceMetadata bool + replicaSetInf cache.SharedInformer + withDeploymentMetadata bool + jobInf cache.SharedInformer + withJobMetadata bool + withCronJobMetadata bool podStore cache.Store endpointSliceStore cache.Store serviceStore cache.Store - queue *workqueue.Type + queue *workqueue.Typed[string] } // NewEndpointSlice returns a new endpointslice discovery. -func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, node cache.SharedInformer, eventCount *prometheus.CounterVec) *EndpointSlice { +func NewEndpointSlice(l *slog.Logger, eps cache.SharedIndexInformer, svc, pod, node, namespace, rs, job cache.SharedInformer, withDeploymentMetadata, withJobMetadata, withCronJobMetadata bool, eventCount *prometheus.CounterVec) *EndpointSlice { if l == nil { - l = log.NewNopLogger() + l = promslog.NewNopLogger() } epslAddCount := eventCount.WithLabelValues(RoleEndpointSlice.String(), MetricLabelRoleAdd) @@ -66,92 +73,114 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod svcDeleteCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleDelete) e := &EndpointSlice{ - logger: l, - endpointSliceInf: eps, - endpointSliceStore: eps.GetStore(), - serviceInf: svc, - serviceStore: svc.GetStore(), - podInf: pod, - podStore: pod.GetStore(), - nodeInf: node, - withNodeMetadata: node != nil, - queue: workqueue.NewNamed(RoleEndpointSlice.String()), + logger: l, + endpointSliceInf: eps, + endpointSliceStore: eps.GetStore(), + serviceInf: svc, + serviceStore: svc.GetStore(), + podInf: pod, + podStore: pod.GetStore(), + nodeInf: node, + withNodeMetadata: node != nil, + namespaceInf: namespace, + withNamespaceMetadata: namespace != nil, + replicaSetInf: rs, + withDeploymentMetadata: withDeploymentMetadata, + jobInf: job, + withJobMetadata: withJobMetadata, + withCronJobMetadata: withCronJobMetadata, + queue: workqueue.NewTypedWithConfig(workqueue.TypedQueueConfig[string]{ + Name: RoleEndpointSlice.String(), + }), } _, err := e.endpointSliceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { epslAddCount.Inc() e.enqueue(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { epslUpdateCount.Inc() e.enqueue(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { epslDeleteCount.Inc() e.enqueue(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding endpoint slices event handler.", "err", err) + l.Error("Error adding endpoint slices event handler.", "err", err) } - serviceUpdate := func(o interface{}) { + serviceUpdate := func(o any) { svc, err := convertToService(o) if err != nil { - level.Error(e.logger).Log("msg", "converting to Service object failed", "err", err) + e.logger.Error("converting to Service object failed", "err", err) return } - // TODO(brancz): use cache.Indexer to index endpoints by - // disv1beta1.LabelServiceName so this operation doesn't have to - // iterate over all endpoint objects. - for _, obj := range e.endpointSliceStore.List() { - esa, err := e.getEndpointSliceAdaptor(obj) - if err != nil { - level.Error(e.logger).Log("msg", "converting to EndpointSlice object failed", "err", err) - continue - } - if lv, exists := esa.labels()[esa.labelServiceName()]; exists && lv == svc.Name { - e.enqueue(esa.get()) - } + endpointSlices, err := e.endpointSliceInf.GetIndexer().ByIndex(serviceIndex, namespacedName(svc.Namespace, svc.Name)) + if err != nil { + e.logger.Error("getting endpoint slices by service name failed", "err", err) + return + } + + for _, endpointSlice := range endpointSlices { + e.enqueue(endpointSlice) } } _, err = e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { svcAddCount.Inc() serviceUpdate(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { svcUpdateCount.Inc() serviceUpdate(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { svcDeleteCount.Inc() serviceUpdate(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding services event handler.", "err", err) + l.Error("Error adding services event handler.", "err", err) } if e.withNodeMetadata { _, err = e.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { node := o.(*apiv1.Node) e.enqueueNode(node.Name) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { node := o.(*apiv1.Node) e.enqueueNode(node.Name) }, - DeleteFunc: func(o interface{}) { - node := o.(*apiv1.Node) - e.enqueueNode(node.Name) + DeleteFunc: func(o any) { + nodeName, err := nodeName(o) + if err != nil { + l.Error("Error getting Node name", "err", err) + } + e.enqueueNode(nodeName) + }, + }) + if err != nil { + l.Error("Error adding nodes event handler.", "err", err) + } + } + + if e.withNamespaceMetadata { + _, err = e.namespaceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, o any) { + namespace := o.(*apiv1.Namespace) + e.enqueueNamespace(namespace.Name) }, + // Creation and deletion will trigger events for the change handlers of the resources within the namespace. + // No need to have additional handlers for them here. }) if err != nil { - level.Error(l).Log("msg", "Error adding nodes event handler.", "err", err) + l.Error("Error adding namespaces event handler.", "err", err) } } @@ -161,7 +190,19 @@ func NewEndpointSlice(l log.Logger, eps cache.SharedIndexInformer, svc, pod, nod func (e *EndpointSlice) enqueueNode(nodeName string) { endpoints, err := e.endpointSliceInf.GetIndexer().ByIndex(nodeIndex, nodeName) if err != nil { - level.Error(e.logger).Log("msg", "Error getting endpoints for node", "node", nodeName, "err", err) + e.logger.Error("Error getting endpoints for node", "node", nodeName, "err", err) + return + } + + for _, endpoint := range endpoints { + e.enqueue(endpoint) + } +} + +func (e *EndpointSlice) enqueueNamespace(namespace string) { + endpoints, err := e.endpointSliceInf.GetIndexer().ByIndex(cache.NamespaceIndex, namespace) + if err != nil { + e.logger.Error("Error getting endpoints in namespace", "namespace", namespace, "err", err) return } @@ -170,7 +211,7 @@ func (e *EndpointSlice) enqueueNode(nodeName string) { } } -func (e *EndpointSlice) enqueue(obj interface{}) { +func (e *EndpointSlice) enqueue(obj any) { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { return @@ -187,9 +228,12 @@ func (e *EndpointSlice) Run(ctx context.Context, ch chan<- []*targetgroup.Group) if e.withNodeMetadata { cacheSyncs = append(cacheSyncs, e.nodeInf.HasSynced) } + if e.withNamespaceMetadata { + cacheSyncs = append(cacheSyncs, e.namespaceInf.HasSynced) + } if !cache.WaitForCacheSync(ctx.Done(), cacheSyncs...) { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(e.logger).Log("msg", "endpointslice informer unable to sync cache") + e.logger.Error("endpointslice informer unable to sync cache") } return } @@ -204,22 +248,21 @@ func (e *EndpointSlice) Run(ctx context.Context, ch chan<- []*targetgroup.Group) } func (e *EndpointSlice) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool { - keyObj, quit := e.queue.Get() + key, quit := e.queue.Get() if quit { return false } - defer e.queue.Done(keyObj) - key := keyObj.(string) + defer e.queue.Done(key) namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { - level.Error(e.logger).Log("msg", "splitting key failed", "key", key) + e.logger.Error("splitting key failed", "key", key) return true } o, exists, err := e.endpointSliceStore.GetByKey(key) if err != nil { - level.Error(e.logger).Log("msg", "getting object from store failed", "key", key) + e.logger.Error("getting object from store failed", "key", key) return true } if !exists { @@ -227,29 +270,17 @@ func (e *EndpointSlice) process(ctx context.Context, ch chan<- []*targetgroup.Gr return true } - esa, err := e.getEndpointSliceAdaptor(o) - if err != nil { - level.Error(e.logger).Log("msg", "converting to EndpointSlice object failed", "err", err) - return true + if es, ok := o.(*v1.EndpointSlice); ok { + send(ctx, ch, e.buildEndpointSlice(*es)) + } else { + e.logger.Error("received unexpected object", "object", o) + return false } - - send(ctx, ch, e.buildEndpointSlice(esa)) return true } -func (e *EndpointSlice) getEndpointSliceAdaptor(o interface{}) (endpointSliceAdaptor, error) { - switch endpointSlice := o.(type) { - case *v1.EndpointSlice: - return newEndpointSliceAdaptorFromV1(endpointSlice), nil - case *v1beta1.EndpointSlice: - return newEndpointSliceAdaptorFromV1beta1(endpointSlice), nil - default: - return nil, fmt.Errorf("received unexpected object: %v", o) - } -} - -func endpointSliceSource(ep endpointSliceAdaptor) string { - return endpointSliceSourceFromNamespaceAndName(ep.namespace(), ep.name()) +func endpointSliceSource(ep v1.EndpointSlice) string { + return endpointSliceSourceFromNamespaceAndName(ep.Namespace, ep.Name) } func endpointSliceSourceFromNamespaceAndName(namespace, name string) string { @@ -274,95 +305,103 @@ const ( endpointSliceEndpointTopologyLabelPresentPrefix = metaLabelPrefix + "endpointslice_endpoint_topology_present_" ) -func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgroup.Group { +func (e *EndpointSlice) buildEndpointSlice(eps v1.EndpointSlice) *targetgroup.Group { tg := &targetgroup.Group{ Source: endpointSliceSource(eps), } tg.Labels = model.LabelSet{ - namespaceLabel: lv(eps.namespace()), - endpointSliceAddressTypeLabel: lv(eps.addressType()), + namespaceLabel: lv(eps.Namespace), + endpointSliceAddressTypeLabel: lv(string(eps.AddressType)), } - addObjectMetaLabels(tg.Labels, eps.getObjectMeta(), RoleEndpointSlice) + addObjectMetaLabels(tg.Labels, eps.ObjectMeta, RoleEndpointSlice) + + svc := e.addServiceLabels(eps, tg) - e.addServiceLabels(eps, tg) + if e.withNamespaceMetadata { + tg.Labels = addNamespaceLabels(tg.Labels, e.namespaceInf, e.logger, eps.Namespace) + } + + if nonPrimaryIPFamilySlice(svc, eps) { + return tg + } type podEntry struct { pod *apiv1.Pod - servicePorts []endpointSlicePortAdaptor + servicePorts []v1.EndpointPort } seenPods := map[string]*podEntry{} - add := func(addr string, ep endpointSliceEndpointAdaptor, port endpointSlicePortAdaptor) { + add := func(addr string, ep v1.Endpoint, port v1.EndpointPort) { a := addr - if port.port() != nil { - a = net.JoinHostPort(addr, strconv.FormatUint(uint64(*port.port()), 10)) + if port.Port != nil { + a = net.JoinHostPort(addr, strconv.FormatUint(uint64(*port.Port), 10)) } target := model.LabelSet{ model.AddressLabel: lv(a), } - if port.name() != nil { - target[endpointSlicePortNameLabel] = lv(*port.name()) + if port.Name != nil { + target[endpointSlicePortNameLabel] = lv(*port.Name) } - if port.protocol() != nil { - target[endpointSlicePortProtocolLabel] = lv(*port.protocol()) + if port.Protocol != nil { + target[endpointSlicePortProtocolLabel] = lv(string(*port.Protocol)) } - if port.port() != nil { - target[endpointSlicePortLabel] = lv(strconv.FormatUint(uint64(*port.port()), 10)) + if port.Port != nil { + target[endpointSlicePortLabel] = lv(strconv.FormatUint(uint64(*port.Port), 10)) } - if port.appProtocol() != nil { - target[endpointSlicePortAppProtocol] = lv(*port.appProtocol()) + if port.AppProtocol != nil { + target[endpointSlicePortAppProtocol] = lv(*port.AppProtocol) } - if ep.conditions().ready() != nil { - target[endpointSliceEndpointConditionsReadyLabel] = lv(strconv.FormatBool(*ep.conditions().ready())) + if ep.Conditions.Ready != nil { + target[endpointSliceEndpointConditionsReadyLabel] = lv(strconv.FormatBool(*ep.Conditions.Ready)) } - if ep.conditions().serving() != nil { - target[endpointSliceEndpointConditionsServingLabel] = lv(strconv.FormatBool(*ep.conditions().serving())) + if ep.Conditions.Serving != nil { + target[endpointSliceEndpointConditionsServingLabel] = lv(strconv.FormatBool(*ep.Conditions.Serving)) } - if ep.conditions().terminating() != nil { - target[endpointSliceEndpointConditionsTerminatingLabel] = lv(strconv.FormatBool(*ep.conditions().terminating())) + if ep.Conditions.Terminating != nil { + target[endpointSliceEndpointConditionsTerminatingLabel] = lv(strconv.FormatBool(*ep.Conditions.Terminating)) } - if ep.hostname() != nil { - target[endpointSliceEndpointHostnameLabel] = lv(*ep.hostname()) + if ep.Hostname != nil { + target[endpointSliceEndpointHostnameLabel] = lv(*ep.Hostname) } - if ep.targetRef() != nil { - target[model.LabelName(endpointSliceAddressTargetKindLabel)] = lv(ep.targetRef().Kind) - target[model.LabelName(endpointSliceAddressTargetNameLabel)] = lv(ep.targetRef().Name) + if ep.TargetRef != nil { + target[model.LabelName(endpointSliceAddressTargetKindLabel)] = lv(ep.TargetRef.Kind) + target[model.LabelName(endpointSliceAddressTargetNameLabel)] = lv(ep.TargetRef.Name) } - if ep.nodename() != nil { - target[endpointSliceEndpointNodenameLabel] = lv(*ep.nodename()) + if ep.NodeName != nil { + target[endpointSliceEndpointNodenameLabel] = lv(*ep.NodeName) } - if ep.zone() != nil { - target[model.LabelName(endpointSliceEndpointZoneLabel)] = lv(*ep.zone()) + if ep.Zone != nil { + target[model.LabelName(endpointSliceEndpointZoneLabel)] = lv(*ep.Zone) } - for k, v := range ep.topology() { + for k, v := range ep.DeprecatedTopology { ln := strutil.SanitizeLabelName(k) target[model.LabelName(endpointSliceEndpointTopologyLabelPrefix+ln)] = lv(v) target[model.LabelName(endpointSliceEndpointTopologyLabelPresentPrefix+ln)] = presentValue } if e.withNodeMetadata { - if ep.targetRef() != nil && ep.targetRef().Kind == "Node" { - target = addNodeLabels(target, e.nodeInf, e.logger, &ep.targetRef().Name) + if ep.TargetRef != nil && ep.TargetRef.Kind == "Node" { + target = addNodeLabels(target, e.nodeInf, e.logger, &ep.TargetRef.Name) } else { - target = addNodeLabels(target, e.nodeInf, e.logger, ep.nodename()) + target = addNodeLabels(target, e.nodeInf, e.logger, ep.NodeName) } } - pod := e.resolvePodRef(ep.targetRef()) + pod := e.resolvePodRef(ep.TargetRef) if pod == nil { // This target is not a Pod, so don't continue with Pod specific logic. tg.Targets = append(tg.Targets, target) @@ -377,22 +416,26 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou } // Attach standard pod labels. - target = target.Merge(podLabels(pod)) + target = target.Merge(podLabels(pod, e.replicaSetInf, e.jobInf, e.withDeploymentMetadata, e.withJobMetadata, e.withCronJobMetadata)) // Attach potential container port labels matching the endpoint port. - for _, c := range pod.Spec.Containers { + containers := append(pod.Spec.Containers, pod.Spec.InitContainers...) + for i, c := range containers { for _, cport := range c.Ports { - if port.port() == nil { + if port.Port == nil { continue } - if *port.port() == cport.ContainerPort { - ports := strconv.FormatUint(uint64(*port.port()), 10) + + if *port.Port == cport.ContainerPort { + ports := strconv.FormatUint(uint64(*port.Port), 10) + isInit := i >= len(pod.Spec.Containers) target[podContainerNameLabel] = lv(c.Name) target[podContainerImageLabel] = lv(c.Image) target[podContainerPortNameLabel] = lv(cport.Name) target[podContainerPortNumberLabel] = lv(ports) target[podContainerPortProtocolLabel] = lv(string(cport.Protocol)) + target[podContainerIsInit] = lv(strconv.FormatBool(isInit)) break } } @@ -404,9 +447,9 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou tg.Targets = append(tg.Targets, target) } - for _, ep := range eps.endpoints() { - for _, port := range eps.ports() { - for _, addr := range ep.addresses() { + for _, ep := range eps.Endpoints { + for _, port := range eps.Ports { + for _, addr := range ep.Addresses { add(addr, ep, port) } } @@ -416,18 +459,19 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou // by one of the service endpoints, generate targets for them. for _, pe := range seenPods { // PodIP can be empty when a pod is starting or has been evicted. - if len(pe.pod.Status.PodIP) == 0 { + if pe.pod.Status.PodIP == "" { continue } - for _, c := range pe.pod.Spec.Containers { + containers := append(pe.pod.Spec.Containers, pe.pod.Spec.InitContainers...) + for i, c := range containers { for _, cport := range c.Ports { hasSeenPort := func() bool { for _, eport := range pe.servicePorts { - if eport.port() == nil { + if eport.Port == nil { continue } - if cport.ContainerPort == *eport.port() { + if cport.ContainerPort == *eport.Port { return true } } @@ -440,6 +484,7 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou a := net.JoinHostPort(pe.pod.Status.PodIP, strconv.FormatUint(uint64(cport.ContainerPort), 10)) ports := strconv.FormatUint(uint64(cport.ContainerPort), 10) + isInit := i >= len(pe.pod.Spec.Containers) target := model.LabelSet{ model.AddressLabel: lv(a), podContainerNameLabel: lv(c.Name), @@ -447,8 +492,9 @@ func (e *EndpointSlice) buildEndpointSlice(eps endpointSliceAdaptor) *targetgrou podContainerPortNameLabel: lv(cport.Name), podContainerPortNumberLabel: lv(ports), podContainerPortProtocolLabel: lv(string(cport.Protocol)), + podContainerIsInit: lv(strconv.FormatBool(isInit)), } - tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod))) + tg.Targets = append(tg.Targets, target.Merge(podLabels(pe.pod, e.replicaSetInf, e.jobInf, e.withDeploymentMetadata, e.withJobMetadata, e.withCronJobMetadata))) } } } @@ -460,13 +506,10 @@ func (e *EndpointSlice) resolvePodRef(ref *apiv1.ObjectReference) *apiv1.Pod { if ref == nil || ref.Kind != "Pod" { return nil } - p := &apiv1.Pod{} - p.Namespace = ref.Namespace - p.Name = ref.Name - obj, exists, err := e.podStore.Get(p) + obj, exists, err := e.podStore.GetByKey(namespacedName(ref.Namespace, ref.Name)) if err != nil { - level.Error(e.logger).Log("msg", "resolving pod ref failed", "err", err) + e.logger.Error("resolving pod ref failed", "err", err) return nil } if !exists { @@ -475,29 +518,57 @@ func (e *EndpointSlice) resolvePodRef(ref *apiv1.ObjectReference) *apiv1.Pod { return obj.(*apiv1.Pod) } -func (e *EndpointSlice) addServiceLabels(esa endpointSliceAdaptor, tg *targetgroup.Group) { +// nonPrimaryIPFamilySlice reports whether eps is the secondary slice of a +// dual-stack service, i.e. its address type does not match the service's +// primary IP family. Targets from such slices would duplicate those of the +// primary slice. +func nonPrimaryIPFamilySlice(svc *apiv1.Service, eps v1.EndpointSlice) bool { + if svc == nil { + return false + } + policy := svc.Spec.IPFamilyPolicy + if policy == nil { + return false + } + if *policy != apiv1.IPFamilyPolicyPreferDualStack && *policy != apiv1.IPFamilyPolicyRequireDualStack { + return false + } + if len(svc.Spec.IPFamilies) == 0 { + return false + } + + // NOTE: For ServiceSpec.IPFamilies: + // * A maximum of two values (dual-stack IPFamilies) are allowed. + // * The field is conditionally mutable: it allows for adding or + // removing a secondary IPFamily, but it does not allow changing the primary + // IPFamily of the service. + return string(eps.AddressType) != string(svc.Spec.IPFamilies[0]) +} + +func (e *EndpointSlice) addServiceLabels(esa v1.EndpointSlice, tg *targetgroup.Group) *apiv1.Service { var ( - svc = &apiv1.Service{} found bool + name string ) - svc.Namespace = esa.namespace() + ns := esa.Namespace // Every EndpointSlice object has the Service they belong to in the // kubernetes.io/service-name label. - svc.Name, found = esa.labels()[esa.labelServiceName()] + name, found = esa.Labels[v1.LabelServiceName] if !found { - return + return nil } - obj, exists, err := e.serviceStore.Get(svc) + obj, exists, err := e.serviceStore.GetByKey(namespacedName(ns, name)) if err != nil { - level.Error(e.logger).Log("msg", "retrieving service failed", "err", err) - return + e.logger.Error("retrieving service failed", "err", err) + return nil } if !exists { - return + return nil } - svc = obj.(*apiv1.Service) + svc := obj.(*apiv1.Service) tg.Labels = tg.Labels.Merge(serviceLabels(svc)) + return svc } diff --git a/discovery/kubernetes/endpointslice_adaptor.go b/discovery/kubernetes/endpointslice_adaptor.go deleted file mode 100644 index edd64fcb327..00000000000 --- a/discovery/kubernetes/endpointslice_adaptor.go +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2020 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubernetes - -import ( - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/discovery/v1" - "k8s.io/api/discovery/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// endpointSliceAdaptor is an adaptor for the different EndpointSlice versions. -type endpointSliceAdaptor interface { - get() interface{} - getObjectMeta() metav1.ObjectMeta - name() string - namespace() string - addressType() string - endpoints() []endpointSliceEndpointAdaptor - ports() []endpointSlicePortAdaptor - labels() map[string]string - labelServiceName() string -} - -type endpointSlicePortAdaptor interface { - name() *string - port() *int32 - protocol() *string - appProtocol() *string -} - -type endpointSliceEndpointAdaptor interface { - addresses() []string - hostname() *string - nodename() *string - zone() *string - conditions() endpointSliceEndpointConditionsAdaptor - targetRef() *corev1.ObjectReference - topology() map[string]string -} - -type endpointSliceEndpointConditionsAdaptor interface { - ready() *bool - serving() *bool - terminating() *bool -} - -// Adaptor for k8s.io/api/discovery/v1. -type endpointSliceAdaptorV1 struct { - endpointSlice *v1.EndpointSlice -} - -func newEndpointSliceAdaptorFromV1(endpointSlice *v1.EndpointSlice) endpointSliceAdaptor { - return &endpointSliceAdaptorV1{endpointSlice: endpointSlice} -} - -func (e *endpointSliceAdaptorV1) get() interface{} { - return e.endpointSlice -} - -func (e *endpointSliceAdaptorV1) getObjectMeta() metav1.ObjectMeta { - return e.endpointSlice.ObjectMeta -} - -func (e *endpointSliceAdaptorV1) name() string { - return e.endpointSlice.ObjectMeta.Name -} - -func (e *endpointSliceAdaptorV1) namespace() string { - return e.endpointSlice.ObjectMeta.Namespace -} - -func (e *endpointSliceAdaptorV1) addressType() string { - return string(e.endpointSlice.AddressType) -} - -func (e *endpointSliceAdaptorV1) endpoints() []endpointSliceEndpointAdaptor { - eps := make([]endpointSliceEndpointAdaptor, 0, len(e.endpointSlice.Endpoints)) - for i := 0; i < len(e.endpointSlice.Endpoints); i++ { - eps = append(eps, newEndpointSliceEndpointAdaptorFromV1(e.endpointSlice.Endpoints[i])) - } - return eps -} - -func (e *endpointSliceAdaptorV1) ports() []endpointSlicePortAdaptor { - ports := make([]endpointSlicePortAdaptor, 0, len(e.endpointSlice.Ports)) - for i := 0; i < len(e.endpointSlice.Ports); i++ { - ports = append(ports, newEndpointSlicePortAdaptorFromV1(e.endpointSlice.Ports[i])) - } - return ports -} - -func (e *endpointSliceAdaptorV1) labels() map[string]string { - return e.endpointSlice.Labels -} - -func (e *endpointSliceAdaptorV1) labelServiceName() string { - return v1.LabelServiceName -} - -// Adaptor for k8s.io/api/discovery/v1beta1. -type endpointSliceAdaptorV1Beta1 struct { - endpointSlice *v1beta1.EndpointSlice -} - -func newEndpointSliceAdaptorFromV1beta1(endpointSlice *v1beta1.EndpointSlice) endpointSliceAdaptor { - return &endpointSliceAdaptorV1Beta1{endpointSlice: endpointSlice} -} - -func (e *endpointSliceAdaptorV1Beta1) get() interface{} { - return e.endpointSlice -} - -func (e *endpointSliceAdaptorV1Beta1) getObjectMeta() metav1.ObjectMeta { - return e.endpointSlice.ObjectMeta -} - -func (e *endpointSliceAdaptorV1Beta1) name() string { - return e.endpointSlice.Name -} - -func (e *endpointSliceAdaptorV1Beta1) namespace() string { - return e.endpointSlice.Namespace -} - -func (e *endpointSliceAdaptorV1Beta1) addressType() string { - return string(e.endpointSlice.AddressType) -} - -func (e *endpointSliceAdaptorV1Beta1) endpoints() []endpointSliceEndpointAdaptor { - eps := make([]endpointSliceEndpointAdaptor, 0, len(e.endpointSlice.Endpoints)) - for i := 0; i < len(e.endpointSlice.Endpoints); i++ { - eps = append(eps, newEndpointSliceEndpointAdaptorFromV1beta1(e.endpointSlice.Endpoints[i])) - } - return eps -} - -func (e *endpointSliceAdaptorV1Beta1) ports() []endpointSlicePortAdaptor { - ports := make([]endpointSlicePortAdaptor, 0, len(e.endpointSlice.Ports)) - for i := 0; i < len(e.endpointSlice.Ports); i++ { - ports = append(ports, newEndpointSlicePortAdaptorFromV1beta1(e.endpointSlice.Ports[i])) - } - return ports -} - -func (e *endpointSliceAdaptorV1Beta1) labels() map[string]string { - return e.endpointSlice.Labels -} - -func (e *endpointSliceAdaptorV1Beta1) labelServiceName() string { - return v1beta1.LabelServiceName -} - -type endpointSliceEndpointAdaptorV1 struct { - endpoint v1.Endpoint -} - -func newEndpointSliceEndpointAdaptorFromV1(endpoint v1.Endpoint) endpointSliceEndpointAdaptor { - return &endpointSliceEndpointAdaptorV1{endpoint: endpoint} -} - -func (e *endpointSliceEndpointAdaptorV1) addresses() []string { - return e.endpoint.Addresses -} - -func (e *endpointSliceEndpointAdaptorV1) hostname() *string { - return e.endpoint.Hostname -} - -func (e *endpointSliceEndpointAdaptorV1) nodename() *string { - return e.endpoint.NodeName -} - -func (e *endpointSliceEndpointAdaptorV1) zone() *string { - return e.endpoint.Zone -} - -func (e *endpointSliceEndpointAdaptorV1) conditions() endpointSliceEndpointConditionsAdaptor { - return newEndpointSliceEndpointConditionsAdaptorFromV1(e.endpoint.Conditions) -} - -func (e *endpointSliceEndpointAdaptorV1) targetRef() *corev1.ObjectReference { - return e.endpoint.TargetRef -} - -func (e *endpointSliceEndpointAdaptorV1) topology() map[string]string { - return e.endpoint.DeprecatedTopology -} - -type endpointSliceEndpointConditionsAdaptorV1 struct { - endpointConditions v1.EndpointConditions -} - -func newEndpointSliceEndpointConditionsAdaptorFromV1(endpointConditions v1.EndpointConditions) endpointSliceEndpointConditionsAdaptor { - return &endpointSliceEndpointConditionsAdaptorV1{endpointConditions: endpointConditions} -} - -func (e *endpointSliceEndpointConditionsAdaptorV1) ready() *bool { - return e.endpointConditions.Ready -} - -func (e *endpointSliceEndpointConditionsAdaptorV1) serving() *bool { - return e.endpointConditions.Serving -} - -func (e *endpointSliceEndpointConditionsAdaptorV1) terminating() *bool { - return e.endpointConditions.Terminating -} - -type endpointSliceEndpointAdaptorV1beta1 struct { - endpoint v1beta1.Endpoint -} - -func newEndpointSliceEndpointAdaptorFromV1beta1(endpoint v1beta1.Endpoint) endpointSliceEndpointAdaptor { - return &endpointSliceEndpointAdaptorV1beta1{endpoint: endpoint} -} - -func (e *endpointSliceEndpointAdaptorV1beta1) addresses() []string { - return e.endpoint.Addresses -} - -func (e *endpointSliceEndpointAdaptorV1beta1) hostname() *string { - return e.endpoint.Hostname -} - -func (e *endpointSliceEndpointAdaptorV1beta1) nodename() *string { - return e.endpoint.NodeName -} - -func (e *endpointSliceEndpointAdaptorV1beta1) zone() *string { - return nil -} - -func (e *endpointSliceEndpointAdaptorV1beta1) conditions() endpointSliceEndpointConditionsAdaptor { - return newEndpointSliceEndpointConditionsAdaptorFromV1beta1(e.endpoint.Conditions) -} - -func (e *endpointSliceEndpointAdaptorV1beta1) targetRef() *corev1.ObjectReference { - return e.endpoint.TargetRef -} - -func (e *endpointSliceEndpointAdaptorV1beta1) topology() map[string]string { - return e.endpoint.Topology -} - -type endpointSliceEndpointConditionsAdaptorV1beta1 struct { - endpointConditions v1beta1.EndpointConditions -} - -func newEndpointSliceEndpointConditionsAdaptorFromV1beta1(endpointConditions v1beta1.EndpointConditions) endpointSliceEndpointConditionsAdaptor { - return &endpointSliceEndpointConditionsAdaptorV1beta1{endpointConditions: endpointConditions} -} - -func (e *endpointSliceEndpointConditionsAdaptorV1beta1) ready() *bool { - return e.endpointConditions.Ready -} - -func (e *endpointSliceEndpointConditionsAdaptorV1beta1) serving() *bool { - return e.endpointConditions.Serving -} - -func (e *endpointSliceEndpointConditionsAdaptorV1beta1) terminating() *bool { - return e.endpointConditions.Terminating -} - -type endpointSlicePortAdaptorV1 struct { - endpointPort v1.EndpointPort -} - -func newEndpointSlicePortAdaptorFromV1(port v1.EndpointPort) endpointSlicePortAdaptor { - return &endpointSlicePortAdaptorV1{endpointPort: port} -} - -func (e *endpointSlicePortAdaptorV1) name() *string { - return e.endpointPort.Name -} - -func (e *endpointSlicePortAdaptorV1) port() *int32 { - return e.endpointPort.Port -} - -func (e *endpointSlicePortAdaptorV1) protocol() *string { - val := string(*e.endpointPort.Protocol) - return &val -} - -func (e *endpointSlicePortAdaptorV1) appProtocol() *string { - return e.endpointPort.AppProtocol -} - -type endpointSlicePortAdaptorV1beta1 struct { - endpointPort v1beta1.EndpointPort -} - -func newEndpointSlicePortAdaptorFromV1beta1(port v1beta1.EndpointPort) endpointSlicePortAdaptor { - return &endpointSlicePortAdaptorV1beta1{endpointPort: port} -} - -func (e *endpointSlicePortAdaptorV1beta1) name() *string { - return e.endpointPort.Name -} - -func (e *endpointSlicePortAdaptorV1beta1) port() *int32 { - return e.endpointPort.Port -} - -func (e *endpointSlicePortAdaptorV1beta1) protocol() *string { - val := string(*e.endpointPort.Protocol) - return &val -} - -func (e *endpointSlicePortAdaptorV1beta1) appProtocol() *string { - return e.endpointPort.AppProtocol -} diff --git a/discovery/kubernetes/endpointslice_adaptor_test.go b/discovery/kubernetes/endpointslice_adaptor_test.go deleted file mode 100644 index 1ee33371938..00000000000 --- a/discovery/kubernetes/endpointslice_adaptor_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2020 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubernetes - -import ( - "testing" - - "github.com/stretchr/testify/require" - v1 "k8s.io/api/discovery/v1" - "k8s.io/api/discovery/v1beta1" -) - -func Test_EndpointSliceAdaptor_v1(t *testing.T) { - endpointSlice := makeEndpointSliceV1() - adaptor := newEndpointSliceAdaptorFromV1(endpointSlice) - - require.Equal(t, endpointSlice.ObjectMeta.Name, adaptor.name()) - require.Equal(t, endpointSlice.ObjectMeta.Namespace, adaptor.namespace()) - require.Equal(t, endpointSlice.AddressType, v1.AddressType(adaptor.addressType())) - require.Equal(t, endpointSlice.Labels, adaptor.labels()) - require.Equal(t, "testendpoints", endpointSlice.Labels[v1.LabelServiceName]) - - for i, endpointAdaptor := range adaptor.endpoints() { - require.Equal(t, endpointSlice.Endpoints[i].Addresses, endpointAdaptor.addresses()) - require.Equal(t, endpointSlice.Endpoints[i].Hostname, endpointAdaptor.hostname()) - require.Equal(t, endpointSlice.Endpoints[i].Conditions.Ready, endpointAdaptor.conditions().ready()) - require.Equal(t, endpointSlice.Endpoints[i].Conditions.Serving, endpointAdaptor.conditions().serving()) - require.Equal(t, endpointSlice.Endpoints[i].Conditions.Terminating, endpointAdaptor.conditions().terminating()) - require.Equal(t, endpointSlice.Endpoints[i].TargetRef, endpointAdaptor.targetRef()) - require.Equal(t, endpointSlice.Endpoints[i].DeprecatedTopology, endpointAdaptor.topology()) - } - - for i, portAdaptor := range adaptor.ports() { - require.Equal(t, endpointSlice.Ports[i].Name, portAdaptor.name()) - require.Equal(t, endpointSlice.Ports[i].Port, portAdaptor.port()) - require.EqualValues(t, endpointSlice.Ports[i].Protocol, portAdaptor.protocol()) - require.Equal(t, endpointSlice.Ports[i].AppProtocol, portAdaptor.appProtocol()) - } -} - -func Test_EndpointSliceAdaptor_v1beta1(t *testing.T) { - endpointSlice := makeEndpointSliceV1beta1() - adaptor := newEndpointSliceAdaptorFromV1beta1(endpointSlice) - - require.Equal(t, endpointSlice.ObjectMeta.Name, adaptor.name()) - require.Equal(t, endpointSlice.ObjectMeta.Namespace, adaptor.namespace()) - require.Equal(t, endpointSlice.AddressType, v1beta1.AddressType(adaptor.addressType())) - require.Equal(t, endpointSlice.Labels, adaptor.labels()) - require.Equal(t, "testendpoints", endpointSlice.Labels[v1beta1.LabelServiceName]) - - for i, endpointAdaptor := range adaptor.endpoints() { - require.Equal(t, endpointSlice.Endpoints[i].Addresses, endpointAdaptor.addresses()) - require.Equal(t, endpointSlice.Endpoints[i].Hostname, endpointAdaptor.hostname()) - require.Equal(t, endpointSlice.Endpoints[i].Conditions.Ready, endpointAdaptor.conditions().ready()) - require.Equal(t, endpointSlice.Endpoints[i].Conditions.Serving, endpointAdaptor.conditions().serving()) - require.Equal(t, endpointSlice.Endpoints[i].Conditions.Terminating, endpointAdaptor.conditions().terminating()) - require.Equal(t, endpointSlice.Endpoints[i].TargetRef, endpointAdaptor.targetRef()) - require.Equal(t, endpointSlice.Endpoints[i].Topology, endpointAdaptor.topology()) - } - - for i, portAdaptor := range adaptor.ports() { - require.Equal(t, endpointSlice.Ports[i].Name, portAdaptor.name()) - require.Equal(t, endpointSlice.Ports[i].Port, portAdaptor.port()) - require.EqualValues(t, endpointSlice.Ports[i].Protocol, portAdaptor.protocol()) - require.Equal(t, endpointSlice.Ports[i].AppProtocol, portAdaptor.appProtocol()) - } -} diff --git a/discovery/kubernetes/endpointslice_test.go b/discovery/kubernetes/endpointslice_test.go index 6ef83081be2..fe3f277e63d 100644 --- a/discovery/kubernetes/endpointslice_test.go +++ b/discovery/kubernetes/endpointslice_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,13 +15,13 @@ package kubernetes import ( "context" + "fmt" "testing" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/discovery/v1" - "k8s.io/api/discovery/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -45,11 +45,11 @@ func protocolptr(p corev1.Protocol) *corev1.Protocol { return &p } -func makeEndpointSliceV1() *v1.EndpointSlice { +func makeEndpointSliceV1(namespace string) *v1.EndpointSlice { return &v1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "testendpoints", - Namespace: "default", + Namespace: namespace, Labels: map[string]string{ v1.LabelServiceName: "testendpoints", }, @@ -114,67 +114,24 @@ func makeEndpointSliceV1() *v1.EndpointSlice { } } -func makeEndpointSliceV1beta1() *v1beta1.EndpointSlice { - return &v1beta1.EndpointSlice{ +func makeNamespace(name string, labels, annotations map[string]string) *corev1.Namespace { + return &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: "testendpoints", - Namespace: "default", - Labels: map[string]string{ - v1beta1.LabelServiceName: "testendpoints", - }, - Annotations: map[string]string{ - "test.annotation": "test", - }, - }, - AddressType: v1beta1.AddressTypeIPv4, - Ports: []v1beta1.EndpointPort{ - { - Name: strptr("testport"), - Port: int32ptr(9000), - Protocol: protocolptr(corev1.ProtocolTCP), - }, - }, - Endpoints: []v1beta1.Endpoint{ - { - Addresses: []string{"1.2.3.4"}, - Hostname: strptr("testendpoint1"), - }, { - Addresses: []string{"2.3.4.5"}, - Conditions: v1beta1.EndpointConditions{ - Ready: boolptr(true), - Serving: boolptr(true), - Terminating: boolptr(false), - }, - }, { - Addresses: []string{"3.4.5.6"}, - Conditions: v1beta1.EndpointConditions{ - Ready: boolptr(false), - Serving: boolptr(true), - Terminating: boolptr(true), - }, - }, { - Addresses: []string{"4.5.6.7"}, - Conditions: v1beta1.EndpointConditions{ - Ready: boolptr(true), - Serving: boolptr(true), - Terminating: boolptr(false), - }, - TargetRef: &corev1.ObjectReference{ - Kind: "Node", - Name: "barbaz", - }, - }, + Name: name, + Labels: labels, + Annotations: annotations, }, } } func TestEndpointSliceDiscoveryBeforeRun(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "v1.25.0") + t.Parallel() + n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}) k8sDiscoveryTest{ discovery: n, beforeRun: func() { - obj := makeEndpointSliceV1() + obj := makeEndpointSliceV1("default") c.DiscoveryV1().EndpointSlices(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) }, expectedMaxItems: 1, @@ -249,72 +206,8 @@ func TestEndpointSliceDiscoveryBeforeRun(t *testing.T) { }.Run(t) } -func TestEndpointSliceDiscoveryBeforeRunV1beta1(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "1.20.0") - - k8sDiscoveryTest{ - discovery: n, - beforeRun: func() { - obj := makeEndpointSliceV1beta1() - c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) - }, - expectedMaxItems: 1, - expectedRes: map[string]*targetgroup.Group{ - "endpointslice/default/testendpoints": { - Targets: []model.LabelSet{ - { - "__address__": "1.2.3.4:9000", - "__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - { - "__address__": "2.3.4.5:9000", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - { - "__address__": "3.4.5.6:9000", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - { - "__address__": "4.5.6.7:9000", - "__meta_kubernetes_endpointslice_address_target_kind": "Node", - "__meta_kubernetes_endpointslice_address_target_name": "barbaz", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - }, - Labels: model.LabelSet{ - "__meta_kubernetes_endpointslice_address_type": "IPv4", - "__meta_kubernetes_namespace": "default", - "__meta_kubernetes_endpointslice_name": "testendpoints", - "__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "testendpoints", - "__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true", - "__meta_kubernetes_endpointslice_annotation_test_annotation": "test", - "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", - }, - Source: "endpointslice/default/testendpoints", - }, - }, - }.Run(t) -} - func TestEndpointSliceDiscoveryAdd(t *testing.T) { + t.Parallel() obj := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "testpod", @@ -353,25 +246,25 @@ func TestEndpointSliceDiscoveryAdd(t *testing.T) { PodIP: "1.2.3.4", }, } - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "v1.20.0", obj) + n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, obj) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := &v1beta1.EndpointSlice{ + obj := &v1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "testendpoints", Namespace: "default", }, - AddressType: v1beta1.AddressTypeIPv4, - Ports: []v1beta1.EndpointPort{ + AddressType: v1.AddressTypeIPv4, + Ports: []v1.EndpointPort{ { Name: strptr("testport"), Port: int32ptr(9000), Protocol: protocolptr(corev1.ProtocolTCP), }, }, - Endpoints: []v1beta1.Endpoint{ + Endpoints: []v1.Endpoint{ { Addresses: []string{"4.3.2.1"}, TargetRef: &corev1.ObjectReference{ @@ -379,13 +272,13 @@ func TestEndpointSliceDiscoveryAdd(t *testing.T) { Name: "testpod", Namespace: "default", }, - Conditions: v1beta1.EndpointConditions{ + Conditions: v1.EndpointConditions{ Ready: boolptr(false), }, }, }, } - c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) + c.DiscoveryV1().EndpointSlices(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) }, expectedMaxItems: 1, expectedRes: map[string]*targetgroup.Group{ @@ -411,6 +304,7 @@ func TestEndpointSliceDiscoveryAdd(t *testing.T) { "__meta_kubernetes_pod_phase": "", "__meta_kubernetes_pod_ready": "unknown", "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", }, { "__address__": "1.2.3.4:9001", @@ -426,6 +320,7 @@ func TestEndpointSliceDiscoveryAdd(t *testing.T) { "__meta_kubernetes_pod_phase": "", "__meta_kubernetes_pod_ready": "unknown", "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", }, }, Labels: model.LabelSet{ @@ -440,13 +335,36 @@ func TestEndpointSliceDiscoveryAdd(t *testing.T) { } func TestEndpointSliceDiscoveryDelete(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "v1.21.0", makeEndpointSliceV1()) + t.Parallel() + n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, makeEndpointSliceV1("default")) + + k8sDiscoveryTest{ + discovery: n, + afterStart: func() { + obj := makeEndpointSliceV1("default") + c.DiscoveryV1().EndpointSlices(obj.Namespace).Delete(context.Background(), obj.Name, metav1.DeleteOptions{}) + }, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testendpoints": { + Source: "endpointslice/default/testendpoints", + }, + }, + }.Run(t) +} + +func TestEndpointSliceDiscoveryUpdate(t *testing.T) { + t.Parallel() + n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, makeEndpointSliceV1("default")) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makeEndpointSliceV1() - c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Delete(context.Background(), obj.Name, metav1.DeleteOptions{}) + obj := makeEndpointSliceV1("default") + obj.ObjectMeta.Labels = nil + obj.ObjectMeta.Annotations = nil + obj.Endpoints = obj.Endpoints[0:2] + c.DiscoveryV1().EndpointSlices(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{}) }, expectedMaxItems: 2, expectedRes: map[string]*targetgroup.Group{ @@ -481,32 +399,32 @@ func TestEndpointSliceDiscoveryDelete(t *testing.T) { "__meta_kubernetes_endpointslice_port_name": "testport", "__meta_kubernetes_endpointslice_port_protocol": "TCP", }, - { - "__address__": "3.4.5.6:9000", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1c", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - { - "__address__": "4.5.6.7:9000", - "__meta_kubernetes_endpointslice_address_target_kind": "Node", - "__meta_kubernetes_endpointslice_address_target_name": "barbaz", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, }, - Labels: map[model.LabelName]model.LabelValue{ + Labels: model.LabelSet{ + "__meta_kubernetes_endpointslice_address_type": "IPv4", + "__meta_kubernetes_endpointslice_name": "testendpoints", + "__meta_kubernetes_namespace": "default", + }, + }, + }, + }.Run(t) +} + +func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) { + t.Parallel() + n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, makeEndpointSliceV1("default")) + + k8sDiscoveryTest{ + discovery: n, + afterStart: func() { + obj := makeEndpointSliceV1("default") + obj.Endpoints = []v1.Endpoint{} + c.DiscoveryV1().EndpointSlices(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{}) + }, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testendpoints": { + Labels: model.LabelSet{ "__meta_kubernetes_endpointslice_address_type": "IPv4", "__meta_kubernetes_endpointslice_name": "testendpoints", "__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "testendpoints", @@ -515,48 +433,33 @@ func TestEndpointSliceDiscoveryDelete(t *testing.T) { "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", "__meta_kubernetes_namespace": "default", }, + Source: "endpointslice/default/testendpoints", }, }, }.Run(t) } -func TestEndpointSliceDiscoveryUpdate(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "v1.21.0", makeEndpointSliceV1()) +func TestEndpointSliceDiscoveryWithService(t *testing.T) { + t.Parallel() + n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, makeEndpointSliceV1("default")) k8sDiscoveryTest{ discovery: n, - afterStart: func() { - obj := &v1beta1.EndpointSlice{ + beforeRun: func() { + obj := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "testendpoints", Namespace: "default", - }, - AddressType: v1beta1.AddressTypeIPv4, - Ports: []v1beta1.EndpointPort{ - { - Name: strptr("testport"), - Port: int32ptr(9000), - Protocol: protocolptr(corev1.ProtocolTCP), - }, - }, - Endpoints: []v1beta1.Endpoint{ - { - Addresses: []string{"1.2.3.4"}, - Hostname: strptr("testendpoint1"), - }, { - Addresses: []string{"2.3.4.5"}, - Conditions: v1beta1.EndpointConditions{ - Ready: boolptr(true), - }, + Labels: map[string]string{ + "app/name": "test", }, }, } - c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{}) + c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) }, - expectedMaxItems: 2, + expectedMaxItems: 1, expectedRes: map[string]*targetgroup.Group{ "endpointslice/default/testendpoints": { - Source: "endpointslice/default/testendpoints", Targets: []model.LabelSet{ { "__address__": "1.2.3.4:9000", @@ -619,34 +522,46 @@ func TestEndpointSliceDiscoveryUpdate(t *testing.T) { "__meta_kubernetes_endpointslice_annotation_test_annotation": "test", "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_service_label_app_name": "test", + "__meta_kubernetes_service_labelpresent_app_name": "true", + "__meta_kubernetes_service_name": "testendpoints", }, + Source: "endpointslice/default/testendpoints", }, }, }.Run(t) } -func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "v1.21.0", makeEndpointSliceV1()) +func TestEndpointSliceDiscoveryWithServiceUpdate(t *testing.T) { + t.Parallel() + n, c := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, makeEndpointSliceV1("default")) k8sDiscoveryTest{ discovery: n, - afterStart: func() { - obj := &v1beta1.EndpointSlice{ + beforeRun: func() { + obj := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "testendpoints", Namespace: "default", + Labels: map[string]string{ + "app/name": "test", + }, }, - AddressType: v1beta1.AddressTypeIPv4, - Ports: []v1beta1.EndpointPort{ - { - Name: strptr("testport"), - Port: int32ptr(9000), - Protocol: protocolptr(corev1.ProtocolTCP), + } + c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) + }, + afterStart: func() { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testendpoints", + Namespace: "default", + Labels: map[string]string{ + "app/name": "svc", + "component": "testing", }, }, - Endpoints: []v1beta1.Endpoint{}, } - c.DiscoveryV1beta1().EndpointSlices(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{}) + c.CoreV1().Services(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{}) }, expectedMaxItems: 2, expectedRes: map[string]*targetgroup.Group{ @@ -676,9 +591,9 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) { "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1b", "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", "__meta_kubernetes_endpointslice_port_name": "testport", "__meta_kubernetes_endpointslice_port_protocol": "TCP", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", }, { "__address__": "3.4.5.6:9000", @@ -687,9 +602,9 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) { "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true", "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1c", "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", "__meta_kubernetes_endpointslice_port_name": "testport", "__meta_kubernetes_endpointslice_port_protocol": "TCP", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", }, { "__address__": "4.5.6.7:9000", @@ -713,6 +628,11 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) { "__meta_kubernetes_endpointslice_annotation_test_annotation": "test", "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_service_label_app_name": "svc", + "__meta_kubernetes_service_label_component": "testing", + "__meta_kubernetes_service_labelpresent_app_name": "true", + "__meta_kubernetes_service_labelpresent_component": "true", + "__meta_kubernetes_service_name": "testendpoints", }, Source: "endpointslice/default/testendpoints", }, @@ -720,23 +640,25 @@ func TestEndpointSliceDiscoveryEmptyEndpoints(t *testing.T) { }.Run(t) } -func TestEndpointSliceDiscoveryWithService(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "v1.21.0", makeEndpointSliceV1()) +func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) { + t.Parallel() + metadataConfig := AttachMetadataConfig{Node: true} + nodeLabels1 := map[string]string{"az": "us-east1"} + nodeLabels2 := map[string]string{"az": "us-west2"} + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testendpoints", + Namespace: "default", + Labels: map[string]string{ + "app/name": "test", + }, + }, + } + objs := []runtime.Object{makeEndpointSliceV1("default"), makeNode("foobar", "", "", nodeLabels1, nil, nil), makeNode("barbaz", "", "", nodeLabels2, nil, nil), svc} + n, _ := makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, metadataConfig, objs...) k8sDiscoveryTest{ - discovery: n, - beforeRun: func() { - obj := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testendpoints", - Namespace: "default", - Labels: map[string]string{ - "app/name": "test", - }, - }, - } - c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) - }, + discovery: n, expectedMaxItems: 1, expectedRes: map[string]*targetgroup.Group{ "endpointslice/default/testendpoints": { @@ -757,6 +679,9 @@ func TestEndpointSliceDiscoveryWithService(t *testing.T) { "__meta_kubernetes_endpointslice_port_app_protocol": "http", "__meta_kubernetes_endpointslice_port_name": "testport", "__meta_kubernetes_endpointslice_port_protocol": "TCP", + "__meta_kubernetes_node_label_az": "us-east1", + "__meta_kubernetes_node_labelpresent_az": "true", + "__meta_kubernetes_node_name": "foobar", }, { "__address__": "2.3.4.5:9000", @@ -792,212 +717,9 @@ func TestEndpointSliceDiscoveryWithService(t *testing.T) { "__meta_kubernetes_endpointslice_port_app_protocol": "http", "__meta_kubernetes_endpointslice_port_name": "testport", "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - }, - Labels: model.LabelSet{ - "__meta_kubernetes_endpointslice_address_type": "IPv4", - "__meta_kubernetes_endpointslice_name": "testendpoints", - "__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "testendpoints", - "__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true", - "__meta_kubernetes_endpointslice_annotation_test_annotation": "test", - "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", - "__meta_kubernetes_namespace": "default", - "__meta_kubernetes_service_label_app_name": "test", - "__meta_kubernetes_service_labelpresent_app_name": "true", - "__meta_kubernetes_service_name": "testendpoints", - }, - Source: "endpointslice/default/testendpoints", - }, - }, - }.Run(t) -} - -func TestEndpointSliceDiscoveryWithServiceUpdate(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, "v1.21.0", makeEndpointSliceV1()) - - k8sDiscoveryTest{ - discovery: n, - beforeRun: func() { - obj := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testendpoints", - Namespace: "default", - Labels: map[string]string{ - "app/name": "test", - }, - }, - } - c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) - }, - afterStart: func() { - obj := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testendpoints", - Namespace: "default", - Labels: map[string]string{ - "app/name": "svc", - "component": "testing", - }, - }, - } - c.CoreV1().Services(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{}) - }, - expectedMaxItems: 2, - expectedRes: map[string]*targetgroup.Group{ - "endpointslice/default/testendpoints": { - Targets: []model.LabelSet{ - { - "__address__": "1.2.3.4:9000", - "__meta_kubernetes_endpointslice_address_target_kind": "", - "__meta_kubernetes_endpointslice_address_target_name": "", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1", - "__meta_kubernetes_endpointslice_endpoint_node_name": "foobar", - "__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true", - "__meta_kubernetes_endpointslice_endpoint_topology_topology": "value", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - { - "__address__": "2.3.4.5:9000", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1b", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - }, - { - "__address__": "3.4.5.6:9000", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1c", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - }, - { - "__address__": "4.5.6.7:9000", - "__meta_kubernetes_endpointslice_address_target_kind": "Node", - "__meta_kubernetes_endpointslice_address_target_name": "barbaz", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - }, - Labels: model.LabelSet{ - "__meta_kubernetes_endpointslice_address_type": "IPv4", - "__meta_kubernetes_endpointslice_name": "testendpoints", - "__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "testendpoints", - "__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true", - "__meta_kubernetes_endpointslice_annotation_test_annotation": "test", - "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", - "__meta_kubernetes_namespace": "default", - "__meta_kubernetes_service_label_app_name": "svc", - "__meta_kubernetes_service_label_component": "testing", - "__meta_kubernetes_service_labelpresent_app_name": "true", - "__meta_kubernetes_service_labelpresent_component": "true", - "__meta_kubernetes_service_name": "testendpoints", - }, - Source: "endpointslice/default/testendpoints", - }, - }, - }.Run(t) -} - -func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) { - metadataConfig := AttachMetadataConfig{Node: true} - nodeLabels1 := map[string]string{"az": "us-east1"} - nodeLabels2 := map[string]string{"az": "us-west2"} - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testendpoints", - Namespace: "default", - Labels: map[string]string{ - "app/name": "test", - }, - }, - } - objs := []runtime.Object{makeEndpointSliceV1(), makeNode("foobar", "", "", nodeLabels1, nil), makeNode("barbaz", "", "", nodeLabels2, nil), svc} - n, _ := makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, metadataConfig, objs...) - - k8sDiscoveryTest{ - discovery: n, - expectedMaxItems: 1, - expectedRes: map[string]*targetgroup.Group{ - "endpointslice/default/testendpoints": { - Targets: []model.LabelSet{ - { - "__address__": "1.2.3.4:9000", - "__meta_kubernetes_endpointslice_address_target_kind": "", - "__meta_kubernetes_endpointslice_address_target_name": "", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1", - "__meta_kubernetes_endpointslice_endpoint_node_name": "foobar", - "__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true", - "__meta_kubernetes_endpointslice_endpoint_topology_topology": "value", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - "__meta_kubernetes_node_label_az": "us-east1", - "__meta_kubernetes_node_labelpresent_az": "true", - "__meta_kubernetes_node_name": "foobar", - }, - { - "__address__": "2.3.4.5:9000", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1b", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - { - "__address__": "3.4.5.6:9000", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1c", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - }, - { - "__address__": "4.5.6.7:9000", - "__meta_kubernetes_endpointslice_address_target_kind": "Node", - "__meta_kubernetes_endpointslice_address_target_name": "barbaz", - "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", - "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", - "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", - "__meta_kubernetes_endpointslice_port": "9000", - "__meta_kubernetes_endpointslice_port_app_protocol": "http", - "__meta_kubernetes_endpointslice_port_name": "testport", - "__meta_kubernetes_endpointslice_port_protocol": "TCP", - "__meta_kubernetes_node_label_az": "us-west2", - "__meta_kubernetes_node_labelpresent_az": "true", - "__meta_kubernetes_node_name": "barbaz", + "__meta_kubernetes_node_label_az": "us-west2", + "__meta_kubernetes_node_labelpresent_az": "true", + "__meta_kubernetes_node_name": "barbaz", }, }, Labels: model.LabelSet{ @@ -1019,6 +741,7 @@ func TestEndpointsSlicesDiscoveryWithNodeMetadata(t *testing.T) { } func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) { + t.Parallel() metadataConfig := AttachMetadataConfig{Node: true} nodeLabels1 := map[string]string{"az": "us-east1"} nodeLabels2 := map[string]string{"az": "us-west2"} @@ -1031,9 +754,9 @@ func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) { }, }, } - node1 := makeNode("foobar", "", "", nodeLabels1, nil) - node2 := makeNode("barbaz", "", "", nodeLabels2, nil) - objs := []runtime.Object{makeEndpointSliceV1(), node1, node2, svc} + node1 := makeNode("foobar", "", "", nodeLabels1, nil, nil) + node2 := makeNode("barbaz", "", "", nodeLabels2, nil, nil) + objs := []runtime.Object{makeEndpointSliceV1("default"), node1, node2, svc} n, c := makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, metadataConfig, objs...) k8sDiscoveryTest{ @@ -1124,7 +847,8 @@ func TestEndpointsSlicesDiscoveryWithUpdatedNodeMetadata(t *testing.T) { } func TestEndpointSliceDiscoveryNamespaces(t *testing.T) { - epOne := makeEndpointSliceV1() + t.Parallel() + epOne := makeEndpointSliceV1("default") epOne.Namespace = "ns1" objs := []runtime.Object{ epOne, @@ -1285,6 +1009,7 @@ func TestEndpointSliceDiscoveryNamespaces(t *testing.T) { "__meta_kubernetes_pod_phase": "", "__meta_kubernetes_pod_ready": "unknown", "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", }, }, Labels: model.LabelSet{ @@ -1299,10 +1024,11 @@ func TestEndpointSliceDiscoveryNamespaces(t *testing.T) { } func TestEndpointSliceDiscoveryOwnNamespace(t *testing.T) { - epOne := makeEndpointSliceV1() + t.Parallel() + epOne := makeEndpointSliceV1("default") epOne.Namespace = "own-ns" - epTwo := makeEndpointSliceV1() + epTwo := makeEndpointSliceV1("default") epTwo.Namespace = "non-own-ns" podOne := &corev1.Pod{ @@ -1419,7 +1145,8 @@ func TestEndpointSliceDiscoveryOwnNamespace(t *testing.T) { } func TestEndpointSliceDiscoveryEmptyPodStatus(t *testing.T) { - ep := makeEndpointSliceV1() + t.Parallel() + ep := makeEndpointSliceV1("default") ep.Namespace = "ns" pod := &corev1.Pod{ @@ -1465,6 +1192,7 @@ func TestEndpointSliceDiscoveryEmptyPodStatus(t *testing.T) { // sets up indexing for the main Kube informer only when needed. // See: https://github.com/prometheus/prometheus/pull/13554#discussion_r1490965817 func TestEndpointSliceInfIndexersCount(t *testing.T) { + t.Parallel() tests := []struct { name string withNodeMetadata bool @@ -1475,12 +1203,14 @@ func TestEndpointSliceInfIndexersCount(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + t.Parallel() var ( - n *Discovery - mainInfIndexersCount int + n *Discovery + // service indexer is enabled by default + mainInfIndexersCount = 1 ) if tc.withNodeMetadata { - mainInfIndexersCount = 1 + mainInfIndexersCount++ n, _ = makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, AttachMetadataConfig{Node: true}) } else { n, _ = makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}) @@ -1498,3 +1228,580 @@ func TestEndpointSliceInfIndexersCount(t *testing.T) { }) } } + +func TestEndpointSliceDiscoverySidecarContainer(t *testing.T) { + t.Parallel() + objs := []runtime.Object{ + &v1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testsidecar", + Namespace: "default", + }, + AddressType: v1.AddressTypeIPv4, + Ports: []v1.EndpointPort{ + { + Name: strptr("testport"), + Port: int32ptr(9000), + Protocol: protocolptr(corev1.ProtocolTCP), + }, + { + Name: strptr("initport"), + Port: int32ptr(9111), + Protocol: protocolptr(corev1.ProtocolTCP), + }, + }, + Endpoints: []v1.Endpoint{ + { + Addresses: []string{"4.3.2.1"}, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "testpod", + Namespace: "default", + }, + }, + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + Namespace: "default", + UID: types.UID("deadbeef"), + }, + Spec: corev1.PodSpec{ + NodeName: "testnode", + InitContainers: []corev1.Container{ + { + Name: "ic1", + Image: "ic1:latest", + Ports: []corev1.ContainerPort{ + { + Name: "initport", + ContainerPort: 1111, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + { + Name: "ic2", + Image: "ic2:latest", + Ports: []corev1.ContainerPort{ + { + Name: "initport", + ContainerPort: 9111, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "c1", + Image: "c1:latest", + Ports: []corev1.ContainerPort{ + { + Name: "mainport", + ContainerPort: 9000, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + Status: corev1.PodStatus{ + HostIP: "2.3.4.5", + PodIP: "4.3.2.1", + }, + }, + } + + n, _ := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{}, objs...) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testsidecar": { + Targets: []model.LabelSet{ + { + "__address__": "4.3.2.1:9000", + "__meta_kubernetes_endpointslice_address_target_kind": "Pod", + "__meta_kubernetes_endpointslice_address_target_name": "testpod", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + "__meta_kubernetes_pod_container_image": "c1:latest", + "__meta_kubernetes_pod_container_name": "c1", + "__meta_kubernetes_pod_container_port_name": "mainport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ip": "4.3.2.1", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "false", + }, + { + "__address__": "4.3.2.1:9111", + "__meta_kubernetes_endpointslice_address_target_kind": "Pod", + "__meta_kubernetes_endpointslice_address_target_name": "testpod", + "__meta_kubernetes_endpointslice_port": "9111", + "__meta_kubernetes_endpointslice_port_name": "initport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + "__meta_kubernetes_pod_container_image": "ic2:latest", + "__meta_kubernetes_pod_container_name": "ic2", + "__meta_kubernetes_pod_container_port_name": "initport", + "__meta_kubernetes_pod_container_port_number": "9111", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ip": "4.3.2.1", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "true", + }, + { + "__address__": "4.3.2.1:1111", + "__meta_kubernetes_pod_container_image": "ic1:latest", + "__meta_kubernetes_pod_container_name": "ic1", + "__meta_kubernetes_pod_container_port_name": "initport", + "__meta_kubernetes_pod_container_port_number": "1111", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ip": "4.3.2.1", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "deadbeef", + "__meta_kubernetes_pod_container_init": "true", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_endpointslice_address_type": "IPv4", + "__meta_kubernetes_endpointslice_name": "testsidecar", + "__meta_kubernetes_namespace": "default", + }, + Source: "endpointslice/default/testsidecar", + }, + }, + }.Run(t) +} + +func TestEndpointsSlicesDiscoveryWithNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"service": "web", "layer": "frontend"} + nsAnnotations := map[string]string{"contact": "platform", "release": "v5.6.7"} + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testendpoints", + Namespace: ns, + Labels: map[string]string{ + "app/name": "test", + }, + }, + } + objs := []runtime.Object{makeNamespace(ns, nsLabels, nsAnnotations), svc, makeEndpointSliceV1(ns)} + n, _ := makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, objs...) + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("endpointslice/%s/testendpoints", ns): { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_endpointslice_address_target_kind": "", + "__meta_kubernetes_endpointslice_address_target_name": "", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", + "__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1", + "__meta_kubernetes_endpointslice_endpoint_node_name": "foobar", + "__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true", + "__meta_kubernetes_endpointslice_endpoint_topology_topology": "value", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + { + "__address__": "2.3.4.5:9000", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1b", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + { + "__address__": "3.4.5.6:9000", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1c", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + { + "__address__": "4.5.6.7:9000", + "__meta_kubernetes_endpointslice_address_target_kind": "Node", + "__meta_kubernetes_endpointslice_address_target_name": "barbaz", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_endpointslice_address_type": "IPv4", + "__meta_kubernetes_endpointslice_name": "testendpoints", + "__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "testendpoints", + "__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true", + "__meta_kubernetes_endpointslice_annotation_test_annotation": "test", + "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_contact": "platform", + "__meta_kubernetes_namespace_annotationpresent_contact": "true", + "__meta_kubernetes_namespace_annotation_release": "v5.6.7", + "__meta_kubernetes_namespace_annotationpresent_release": "true", + "__meta_kubernetes_namespace_label_service": "web", + "__meta_kubernetes_namespace_labelpresent_service": "true", + "__meta_kubernetes_namespace_label_layer": "frontend", + "__meta_kubernetes_namespace_labelpresent_layer": "true", + "__meta_kubernetes_service_label_app_name": "test", + "__meta_kubernetes_service_labelpresent_app_name": "true", + "__meta_kubernetes_service_name": "testendpoints", + }, + Source: fmt.Sprintf("endpointslice/%s/testendpoints", ns), + }, + }, + }.Run(t) +} + +func TestEndpointsSlicesDiscoveryWithUpdatedNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"component": "database", "layer": "backend"} + nsAnnotations := map[string]string{"contact": "dba", "release": "v6.7.8"} + metadataConfig := AttachMetadataConfig{Namespace: true} + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testendpoints", + Namespace: ns, + Labels: map[string]string{ + "app/name": "test", + }, + }, + } + + namespace := makeNamespace(ns, nsLabels, nsAnnotations) + n, c := makeDiscoveryWithMetadata(RoleEndpointSlice, NamespaceDiscovery{}, metadataConfig, namespace, svc, makeEndpointSliceV1(ns)) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 2, + afterStart: func() { + namespace.Labels["component"] = "cache" + namespace.Labels["region"] = "us-central" + namespace.Annotations["contact"] = "sre" + namespace.Annotations["monitoring"] = "enabled" + c.CoreV1().Namespaces().Update(context.Background(), namespace, metav1.UpdateOptions{}) + }, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("endpointslice/%s/testendpoints", ns): { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_endpointslice_address_target_kind": "", + "__meta_kubernetes_endpointslice_address_target_name": "", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", + "__meta_kubernetes_endpointslice_endpoint_hostname": "testendpoint1", + "__meta_kubernetes_endpointslice_endpoint_node_name": "foobar", + "__meta_kubernetes_endpointslice_endpoint_topology_present_topology": "true", + "__meta_kubernetes_endpointslice_endpoint_topology_topology": "value", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + { + "__address__": "2.3.4.5:9000", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1b", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + { + "__address__": "3.4.5.6:9000", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "false", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "true", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1c", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + { + "__address__": "4.5.6.7:9000", + "__meta_kubernetes_endpointslice_address_target_kind": "Node", + "__meta_kubernetes_endpointslice_address_target_name": "barbaz", + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true", + "__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false", + "__meta_kubernetes_endpointslice_endpoint_zone": "us-east-1a", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_app_protocol": "http", + "__meta_kubernetes_endpointslice_port_name": "testport", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_endpointslice_address_type": "IPv4", + "__meta_kubernetes_endpointslice_name": "testendpoints", + "__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "testendpoints", + "__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true", + "__meta_kubernetes_endpointslice_annotation_test_annotation": "test", + "__meta_kubernetes_endpointslice_annotationpresent_test_annotation": "true", + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_contact": "sre", + "__meta_kubernetes_namespace_annotationpresent_contact": "true", + "__meta_kubernetes_namespace_annotation_release": "v6.7.8", + "__meta_kubernetes_namespace_annotationpresent_release": "true", + "__meta_kubernetes_namespace_annotation_monitoring": "enabled", + "__meta_kubernetes_namespace_annotationpresent_monitoring": "true", + "__meta_kubernetes_namespace_label_component": "cache", + "__meta_kubernetes_namespace_labelpresent_component": "true", + "__meta_kubernetes_namespace_label_layer": "backend", + "__meta_kubernetes_namespace_labelpresent_layer": "true", + "__meta_kubernetes_namespace_label_region": "us-central", + "__meta_kubernetes_namespace_labelpresent_region": "true", + "__meta_kubernetes_service_label_app_name": "test", + "__meta_kubernetes_service_labelpresent_app_name": "true", + "__meta_kubernetes_service_name": "testendpoints", + }, + Source: fmt.Sprintf("endpointslice/%s/testendpoints", ns), + }, + }, + }.Run(t) +} + +func makeDualStackService(name, namespace string, policy corev1.IPFamilyPolicy, families []corev1.IPFamily) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + IPFamilyPolicy: &policy, + IPFamilies: families, + }, + } +} + +func makeDualStackEndpointSlice(name, namespace, svcName string, addrType v1.AddressType, addr string) *v1.EndpointSlice { + return &v1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{v1.LabelServiceName: svcName}, + }, + AddressType: addrType, + Ports: []v1.EndpointPort{ + { + Name: strptr("http"), + Port: int32ptr(9000), + Protocol: protocolptr(corev1.ProtocolTCP), + }, + }, + Endpoints: []v1.Endpoint{ + { + Addresses: []string{addr}, + Conditions: v1.EndpointConditions{Ready: boolptr(true)}, + }, + }, + } +} + +func dualStackGroupLabels(namespace, addrType, sliceName, svcName string) model.LabelSet { + return model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(namespace), + "__meta_kubernetes_endpointslice_address_type": model.LabelValue(addrType), + "__meta_kubernetes_endpointslice_name": model.LabelValue(sliceName), + "__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": model.LabelValue(svcName), + "__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true", + "__meta_kubernetes_service_name": model.LabelValue(svcName), + } +} + +func dualStackTarget(addr string) model.LabelSet { + return model.LabelSet{ + "__address__": model.LabelValue(addr), + "__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true", + "__meta_kubernetes_endpointslice_port": "9000", + "__meta_kubernetes_endpointslice_port_name": "http", + "__meta_kubernetes_endpointslice_port_protocol": "TCP", + } +} + +// TestEndpointSliceDiscoveryDualStackPreferredSkipsSecondarySlice verifies +// that when a service has ipFamilyPolicy=PreferDualStack with IPv4 as the +// primary family, only the IPv4 EndpointSlice generates targets. The IPv6 +// (secondary) EndpointSlice produces an empty target group. +func TestEndpointSliceDiscoveryDualStackPreferredSkipsSecondarySlice(t *testing.T) { + t.Parallel() + svc := makeDualStackService("testsvc", "default", corev1.IPFamilyPolicyPreferDualStack, + []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}) + epsIPv4 := makeDualStackEndpointSlice("testsvc-ipv4", "default", "testsvc", v1.AddressTypeIPv4, "1.2.3.4") + epsIPv6 := makeDualStackEndpointSlice("testsvc-ipv6", "default", "testsvc", v1.AddressTypeIPv6, "fd00::1") + + n, _ := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, svc, epsIPv4, epsIPv6) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testsvc-ipv4": { + Targets: []model.LabelSet{dualStackTarget("1.2.3.4:9000")}, + Labels: dualStackGroupLabels("default", "IPv4", "testsvc-ipv4", "testsvc"), + Source: "endpointslice/default/testsvc-ipv4", + }, + "endpointslice/default/testsvc-ipv6": { + Targets: nil, + Labels: dualStackGroupLabels("default", "IPv6", "testsvc-ipv6", "testsvc"), + Source: "endpointslice/default/testsvc-ipv6", + }, + }, + }.Run(t) +} + +// TestEndpointSliceDiscoveryDualStackRequiredSkipsSecondarySlice verifies the +// same deduplication behaviour for ipFamilyPolicy=RequireDualStack. +func TestEndpointSliceDiscoveryDualStackRequiredSkipsSecondarySlice(t *testing.T) { + t.Parallel() + svc := makeDualStackService("testsvc", "default", corev1.IPFamilyPolicyRequireDualStack, + []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}) + epsIPv4 := makeDualStackEndpointSlice("testsvc-ipv4", "default", "testsvc", v1.AddressTypeIPv4, "1.2.3.4") + epsIPv6 := makeDualStackEndpointSlice("testsvc-ipv6", "default", "testsvc", v1.AddressTypeIPv6, "fd00::1") + + n, _ := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, svc, epsIPv4, epsIPv6) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testsvc-ipv4": { + Targets: []model.LabelSet{dualStackTarget("1.2.3.4:9000")}, + Labels: dualStackGroupLabels("default", "IPv4", "testsvc-ipv4", "testsvc"), + Source: "endpointslice/default/testsvc-ipv4", + }, + "endpointslice/default/testsvc-ipv6": { + Targets: nil, + Labels: dualStackGroupLabels("default", "IPv6", "testsvc-ipv6", "testsvc"), + Source: "endpointslice/default/testsvc-ipv6", + }, + }, + }.Run(t) +} + +func TestEndpointSliceDiscoverySingleStackIPv4Unaffected(t *testing.T) { + t.Parallel() + svc := makeDualStackService("testsvc", "default", corev1.IPFamilyPolicySingleStack, + []corev1.IPFamily{corev1.IPv4Protocol}) + epsIPv4 := makeDualStackEndpointSlice("testsvc-ipv4", "default", "testsvc", v1.AddressTypeIPv4, "1.2.3.4") + + n, _ := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, svc, epsIPv4) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testsvc-ipv4": { + Targets: []model.LabelSet{dualStackTarget("1.2.3.4:9000")}, + Labels: dualStackGroupLabels("default", "IPv4", "testsvc-ipv4", "testsvc"), + Source: "endpointslice/default/testsvc-ipv4", + }, + }, + }.Run(t) +} + +// TestEndpointSliceDiscoverySingleStackIPv6Unaffected verifies that a +// single-stack IPv6 service is not affected by the dual-stack deduplication +// logic and still generates targets. +func TestEndpointSliceDiscoverySingleStackIPv6Unaffected(t *testing.T) { + t.Parallel() + svc := makeDualStackService("testsvc", "default", corev1.IPFamilyPolicySingleStack, + []corev1.IPFamily{corev1.IPv6Protocol}) + epsIPv6 := makeDualStackEndpointSlice("testsvc-ipv6", "default", "testsvc", v1.AddressTypeIPv6, "fd00::1") + + n, _ := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, svc, epsIPv6) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testsvc-ipv6": { + Targets: []model.LabelSet{dualStackTarget("[fd00::1]:9000")}, + Labels: dualStackGroupLabels("default", "IPv6", "testsvc-ipv6", "testsvc"), + Source: "endpointslice/default/testsvc-ipv6", + }, + }, + }.Run(t) +} + +// TestEndpointSliceDiscoveryDualStackPreferredIPv6Primary verifies that when +// IPv6 is the primary family of a dual-stack service, the IPv6 EndpointSlice +// generates targets and the IPv4 (secondary) EndpointSlice produces an empty +// target group. +func TestEndpointSliceDiscoveryDualStackPreferredIPv6Primary(t *testing.T) { + t.Parallel() + svc := makeDualStackService("testsvc", "default", corev1.IPFamilyPolicyPreferDualStack, + []corev1.IPFamily{corev1.IPv6Protocol, corev1.IPv4Protocol}) + epsIPv6 := makeDualStackEndpointSlice("testsvc-ipv6", "default", "testsvc", v1.AddressTypeIPv6, "fd00::1") + epsIPv4 := makeDualStackEndpointSlice("testsvc-ipv4", "default", "testsvc", v1.AddressTypeIPv4, "1.2.3.4") + + n, _ := makeDiscovery(RoleEndpointSlice, NamespaceDiscovery{Names: []string{"default"}}, svc, epsIPv6, epsIPv4) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "endpointslice/default/testsvc-ipv6": { + Targets: []model.LabelSet{dualStackTarget("[fd00::1]:9000")}, + Labels: dualStackGroupLabels("default", "IPv6", "testsvc-ipv6", "testsvc"), + Source: "endpointslice/default/testsvc-ipv6", + }, + "endpointslice/default/testsvc-ipv4": { + Targets: nil, + Labels: dualStackGroupLabels("default", "IPv4", "testsvc-ipv4", "testsvc"), + Source: "endpointslice/default/testsvc-ipv4", + }, + }, + }.Run(t) +} diff --git a/discovery/kubernetes/ingress.go b/discovery/kubernetes/ingress.go index 7b6366b257b..985cc8f138b 100644 --- a/discovery/kubernetes/ingress.go +++ b/discovery/kubernetes/ingress.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,14 +17,13 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + apiv1 "k8s.io/api/core/v1" v1 "k8s.io/api/networking/v1" - "k8s.io/api/networking/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -33,14 +32,16 @@ import ( // Ingress implements discovery of Kubernetes ingress. type Ingress struct { - logger log.Logger - informer cache.SharedInformer - store cache.Store - queue *workqueue.Type + logger *slog.Logger + informer cache.SharedIndexInformer + store cache.Store + queue *workqueue.Typed[string] + namespaceInf cache.SharedInformer + withNamespaceMetadata bool } // NewIngress returns a new ingress discovery. -func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress { +func NewIngress(l *slog.Logger, inf cache.SharedIndexInformer, namespace cache.SharedInformer, eventCount *prometheus.CounterVec) *Ingress { ingressAddCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleAdd) ingressUpdateCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleUpdate) ingressDeleteCount := eventCount.WithLabelValues(RoleIngress.String(), MetricLabelRoleDelete) @@ -49,30 +50,49 @@ func NewIngress(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.C logger: l, informer: inf, store: inf.GetStore(), - queue: workqueue.NewNamed(RoleIngress.String()), + queue: workqueue.NewTypedWithConfig(workqueue.TypedQueueConfig[string]{ + Name: RoleIngress.String(), + }), + namespaceInf: namespace, + withNamespaceMetadata: namespace != nil, } _, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { ingressAddCount.Inc() s.enqueue(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { ingressDeleteCount.Inc() s.enqueue(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { ingressUpdateCount.Inc() s.enqueue(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding ingresses event handler.", "err", err) + l.Error("Error adding ingresses event handler.", "err", err) } + + if s.withNamespaceMetadata { + _, err = s.namespaceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, o any) { + namespace := o.(*apiv1.Namespace) + s.enqueueNamespace(namespace.Name) + }, + // Creation and deletion will trigger events for the change handlers of the resources within the namespace. + // No need to have additional handlers for them here. + }) + if err != nil { + l.Error("Error adding namespaces event handler.", "err", err) + } + } + return s } -func (i *Ingress) enqueue(obj interface{}) { +func (i *Ingress) enqueue(obj any) { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { return @@ -81,13 +101,30 @@ func (i *Ingress) enqueue(obj interface{}) { i.queue.Add(key) } +func (i *Ingress) enqueueNamespace(namespace string) { + ingresses, err := i.informer.GetIndexer().ByIndex(cache.NamespaceIndex, namespace) + if err != nil { + i.logger.Error("Error getting ingresses in namespace", "namespace", namespace, "err", err) + return + } + + for _, ingress := range ingresses { + i.enqueue(ingress) + } +} + // Run implements the Discoverer interface. func (i *Ingress) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { defer i.queue.ShutDown() - if !cache.WaitForCacheSync(ctx.Done(), i.informer.HasSynced) { + cacheSyncs := []cache.InformerSynced{i.informer.HasSynced} + if i.withNamespaceMetadata { + cacheSyncs = append(cacheSyncs, i.namespaceInf.HasSynced) + } + + if !cache.WaitForCacheSync(ctx.Done(), cacheSyncs...) { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(i.logger).Log("msg", "ingress informer unable to sync cache") + i.logger.Error("ingress informer unable to sync cache") } return } @@ -102,12 +139,11 @@ func (i *Ingress) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } func (i *Ingress) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool { - keyObj, quit := i.queue.Get() + key, quit := i.queue.Get() if quit { return false } - defer i.queue.Done(keyObj) - key := keyObj.(string) + defer i.queue.Done(key) namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -123,23 +159,18 @@ func (i *Ingress) process(ctx context.Context, ch chan<- []*targetgroup.Group) b return true } - var ia ingressAdaptor - switch ingress := o.(type) { - case *v1.Ingress: - ia = newIngressAdaptorFromV1(ingress) - case *v1beta1.Ingress: - ia = newIngressAdaptorFromV1beta1(ingress) - default: - level.Error(i.logger).Log("msg", "converting to Ingress object failed", "err", + if ingress, ok := o.(*v1.Ingress); ok { + send(ctx, ch, i.buildIngress(*ingress)) + } else { + i.logger.Error("converting to Ingress object failed", "err", fmt.Errorf("received unexpected object: %v", o)) return true } - send(ctx, ch, i.buildIngress(ia)) return true } -func ingressSource(s ingressAdaptor) string { - return ingressSourceFromNamespaceAndName(s.namespace(), s.name()) +func ingressSource(s v1.Ingress) string { + return ingressSourceFromNamespaceAndName(s.Namespace, s.Name) } func ingressSourceFromNamespaceAndName(namespace, name string) string { @@ -153,15 +184,15 @@ const ( ingressClassNameLabel = metaLabelPrefix + "ingress_class_name" ) -func ingressLabels(ingress ingressAdaptor) model.LabelSet { +func ingressLabels(ingress v1.Ingress) model.LabelSet { // Each label and annotation will create two key-value pairs in the map. ls := make(model.LabelSet) - ls[namespaceLabel] = lv(ingress.namespace()) - if cls := ingress.ingressClassName(); cls != nil { + ls[namespaceLabel] = lv(ingress.Namespace) + if cls := ingress.Spec.IngressClassName; cls != nil { ls[ingressClassNameLabel] = lv(*cls) } - addObjectMetaLabels(ls, ingress.getObjectMeta(), RoleIngress) + addObjectMetaLabels(ls, ingress.ObjectMeta, RoleIngress) return ls } @@ -181,19 +212,43 @@ func pathsFromIngressPaths(ingressPaths []string) []string { return paths } -func (i *Ingress) buildIngress(ingress ingressAdaptor) *targetgroup.Group { +func rulePaths(rule v1.IngressRule) []string { + rv := rule.IngressRuleValue + if rv.HTTP == nil { + return nil + } + paths := make([]string, len(rv.HTTP.Paths)) + for n, p := range rv.HTTP.Paths { + paths[n] = p.Path + } + return paths +} + +func tlsHosts(ingressTLS []v1.IngressTLS) []string { + var hosts []string + for _, tls := range ingressTLS { + hosts = append(hosts, tls.Hosts...) + } + return hosts +} + +func (i *Ingress) buildIngress(ingress v1.Ingress) *targetgroup.Group { tg := &targetgroup.Group{ Source: ingressSource(ingress), } tg.Labels = ingressLabels(ingress) - for _, rule := range ingress.rules() { + if i.withNamespaceMetadata { + tg.Labels = addNamespaceLabels(tg.Labels, i.namespaceInf, i.logger, ingress.Namespace) + } + + for _, rule := range ingress.Spec.Rules { scheme := "http" - paths := pathsFromIngressPaths(rule.paths()) + paths := pathsFromIngressPaths(rulePaths(rule)) out: - for _, pattern := range ingress.tlsHosts() { - if matchesHostnamePattern(pattern, rule.host()) { + for _, pattern := range tlsHosts(ingress.Spec.TLS) { + if matchesHostnamePattern(pattern, rule.Host) { scheme = "https" break out } @@ -201,9 +256,9 @@ func (i *Ingress) buildIngress(ingress ingressAdaptor) *targetgroup.Group { for _, path := range paths { tg.Targets = append(tg.Targets, model.LabelSet{ - model.AddressLabel: lv(rule.host()), + model.AddressLabel: lv(rule.Host), ingressSchemeLabel: lv(scheme), - ingressHostLabel: lv(rule.host()), + ingressHostLabel: lv(rule.Host), ingressPathLabel: lv(path), }) } diff --git a/discovery/kubernetes/ingress_adaptor.go b/discovery/kubernetes/ingress_adaptor.go deleted file mode 100644 index d1a7b7f2a2f..00000000000 --- a/discovery/kubernetes/ingress_adaptor.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubernetes - -import ( - v1 "k8s.io/api/networking/v1" - "k8s.io/api/networking/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// ingressAdaptor is an adaptor for the different Ingress versions. -type ingressAdaptor interface { - getObjectMeta() metav1.ObjectMeta - name() string - namespace() string - labels() map[string]string - annotations() map[string]string - tlsHosts() []string - ingressClassName() *string - rules() []ingressRuleAdaptor -} - -type ingressRuleAdaptor interface { - paths() []string - host() string -} - -// Adaptor for networking.k8s.io/v1. -type ingressAdaptorV1 struct { - ingress *v1.Ingress -} - -func newIngressAdaptorFromV1(ingress *v1.Ingress) ingressAdaptor { - return &ingressAdaptorV1{ingress: ingress} -} - -func (i *ingressAdaptorV1) getObjectMeta() metav1.ObjectMeta { return i.ingress.ObjectMeta } -func (i *ingressAdaptorV1) name() string { return i.ingress.Name } -func (i *ingressAdaptorV1) namespace() string { return i.ingress.Namespace } -func (i *ingressAdaptorV1) labels() map[string]string { return i.ingress.Labels } -func (i *ingressAdaptorV1) annotations() map[string]string { return i.ingress.Annotations } -func (i *ingressAdaptorV1) ingressClassName() *string { return i.ingress.Spec.IngressClassName } - -func (i *ingressAdaptorV1) tlsHosts() []string { - var hosts []string - for _, tls := range i.ingress.Spec.TLS { - hosts = append(hosts, tls.Hosts...) - } - return hosts -} - -func (i *ingressAdaptorV1) rules() []ingressRuleAdaptor { - var rules []ingressRuleAdaptor - for _, rule := range i.ingress.Spec.Rules { - rules = append(rules, newIngressRuleAdaptorFromV1(rule)) - } - return rules -} - -type ingressRuleAdaptorV1 struct { - rule v1.IngressRule -} - -func newIngressRuleAdaptorFromV1(rule v1.IngressRule) ingressRuleAdaptor { - return &ingressRuleAdaptorV1{rule: rule} -} - -func (i *ingressRuleAdaptorV1) paths() []string { - rv := i.rule.IngressRuleValue - if rv.HTTP == nil { - return nil - } - paths := make([]string, len(rv.HTTP.Paths)) - for n, p := range rv.HTTP.Paths { - paths[n] = p.Path - } - return paths -} - -func (i *ingressRuleAdaptorV1) host() string { return i.rule.Host } - -// Adaptor for networking.k8s.io/v1beta1. -type ingressAdaptorV1Beta1 struct { - ingress *v1beta1.Ingress -} - -func newIngressAdaptorFromV1beta1(ingress *v1beta1.Ingress) ingressAdaptor { - return &ingressAdaptorV1Beta1{ingress: ingress} -} -func (i *ingressAdaptorV1Beta1) getObjectMeta() metav1.ObjectMeta { return i.ingress.ObjectMeta } -func (i *ingressAdaptorV1Beta1) name() string { return i.ingress.Name } -func (i *ingressAdaptorV1Beta1) namespace() string { return i.ingress.Namespace } -func (i *ingressAdaptorV1Beta1) labels() map[string]string { return i.ingress.Labels } -func (i *ingressAdaptorV1Beta1) annotations() map[string]string { return i.ingress.Annotations } -func (i *ingressAdaptorV1Beta1) ingressClassName() *string { return i.ingress.Spec.IngressClassName } - -func (i *ingressAdaptorV1Beta1) tlsHosts() []string { - var hosts []string - for _, tls := range i.ingress.Spec.TLS { - hosts = append(hosts, tls.Hosts...) - } - return hosts -} - -func (i *ingressAdaptorV1Beta1) rules() []ingressRuleAdaptor { - var rules []ingressRuleAdaptor - for _, rule := range i.ingress.Spec.Rules { - rules = append(rules, newIngressRuleAdaptorFromV1Beta1(rule)) - } - return rules -} - -type ingressRuleAdaptorV1Beta1 struct { - rule v1beta1.IngressRule -} - -func newIngressRuleAdaptorFromV1Beta1(rule v1beta1.IngressRule) ingressRuleAdaptor { - return &ingressRuleAdaptorV1Beta1{rule: rule} -} - -func (i *ingressRuleAdaptorV1Beta1) paths() []string { - rv := i.rule.IngressRuleValue - if rv.HTTP == nil { - return nil - } - paths := make([]string, len(rv.HTTP.Paths)) - for n, p := range rv.HTTP.Paths { - paths[n] = p.Path - } - return paths -} - -func (i *ingressRuleAdaptorV1Beta1) host() string { return i.rule.Host } diff --git a/discovery/kubernetes/ingress_test.go b/discovery/kubernetes/ingress_test.go index 8e6654c2ccb..15fa28002ab 100644 --- a/discovery/kubernetes/ingress_test.go +++ b/discovery/kubernetes/ingress_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,11 +16,11 @@ package kubernetes import ( "context" "fmt" + "maps" "testing" "github.com/prometheus/common/model" v1 "k8s.io/api/networking/v1" - "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -35,11 +35,11 @@ const ( TLSWildcard ) -func makeIngress(tls TLSMode) *v1.Ingress { +func makeIngress(namespace string, tls TLSMode) *v1.Ingress { ret := &v1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "testingress", - Namespace: "default", + Namespace: namespace, Labels: map[string]string{"test/label": "testvalue"}, Annotations: map[string]string{"test/annotation": "testannotationvalue"}, }, @@ -89,60 +89,6 @@ func makeIngress(tls TLSMode) *v1.Ingress { return ret } -func makeIngressV1beta1(tls TLSMode) *v1beta1.Ingress { - ret := &v1beta1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testingress", - Namespace: "default", - Labels: map[string]string{"test/label": "testvalue"}, - Annotations: map[string]string{"test/annotation": "testannotationvalue"}, - }, - Spec: v1beta1.IngressSpec{ - IngressClassName: classString("testclass"), - TLS: nil, - Rules: []v1beta1.IngressRule{ - { - Host: "example.com", - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - {Path: "/"}, - {Path: "/foo"}, - }, - }, - }, - }, - { - // No backend config, ignored - Host: "nobackend.example.com", - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{}, - }, - }, - { - Host: "test.example.com", - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{{}}, - }, - }, - }, - }, - }, - } - - switch tls { - case TLSYes: - ret.Spec.TLS = []v1beta1.IngressTLS{{Hosts: []string{"example.com", "test.example.com"}}} - case TLSMixed: - ret.Spec.TLS = []v1beta1.IngressTLS{{Hosts: []string{"example.com"}}} - case TLSWildcard: - ret.Spec.TLS = []v1beta1.IngressTLS{{Hosts: []string{"*.example.com"}}} - } - - return ret -} - func classString(v string) *string { return &v } @@ -199,12 +145,13 @@ func expectedTargetGroups(ns string, tls TLSMode) map[string]*targetgroup.Group } func TestIngressDiscoveryAdd(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleIngress, NamespaceDiscovery{Names: []string{"default"}}) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makeIngress(TLSNo) + obj := makeIngress("default", TLSNo) c.NetworkingV1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{}) }, expectedMaxItems: 1, @@ -212,27 +159,14 @@ func TestIngressDiscoveryAdd(t *testing.T) { }.Run(t) } -func TestIngressDiscoveryAddV1beta1(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleIngress, NamespaceDiscovery{Names: []string{"default"}}, "v1.18.0") - - k8sDiscoveryTest{ - discovery: n, - afterStart: func() { - obj := makeIngressV1beta1(TLSNo) - c.NetworkingV1beta1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{}) - }, - expectedMaxItems: 1, - expectedRes: expectedTargetGroups("default", TLSNo), - }.Run(t) -} - func TestIngressDiscoveryAddTLS(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleIngress, NamespaceDiscovery{Names: []string{"default"}}) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makeIngress(TLSYes) + obj := makeIngress("default", TLSYes) c.NetworkingV1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{}) }, expectedMaxItems: 1, @@ -240,27 +174,14 @@ func TestIngressDiscoveryAddTLS(t *testing.T) { }.Run(t) } -func TestIngressDiscoveryAddTLSV1beta1(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleIngress, NamespaceDiscovery{Names: []string{"default"}}, "v1.18.0") - - k8sDiscoveryTest{ - discovery: n, - afterStart: func() { - obj := makeIngressV1beta1(TLSYes) - c.NetworkingV1beta1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{}) - }, - expectedMaxItems: 1, - expectedRes: expectedTargetGroups("default", TLSYes), - }.Run(t) -} - func TestIngressDiscoveryAddMixed(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleIngress, NamespaceDiscovery{Names: []string{"default"}}) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makeIngress(TLSMixed) + obj := makeIngress("default", TLSMixed) c.NetworkingV1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{}) }, expectedMaxItems: 1, @@ -268,33 +189,17 @@ func TestIngressDiscoveryAddMixed(t *testing.T) { }.Run(t) } -func TestIngressDiscoveryAddMixedV1beta1(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleIngress, NamespaceDiscovery{Names: []string{"default"}}, "v1.18.0") - - k8sDiscoveryTest{ - discovery: n, - afterStart: func() { - obj := makeIngressV1beta1(TLSMixed) - c.NetworkingV1beta1().Ingresses("default").Create(context.Background(), obj, metav1.CreateOptions{}) - }, - expectedMaxItems: 1, - expectedRes: expectedTargetGroups("default", TLSMixed), - }.Run(t) -} - func TestIngressDiscoveryNamespaces(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleIngress, NamespaceDiscovery{Names: []string{"ns1", "ns2"}}) expected := expectedTargetGroups("ns1", TLSNo) - for k, v := range expectedTargetGroups("ns2", TLSNo) { - expected[k] = v - } + maps.Copy(expected, expectedTargetGroups("ns2", TLSNo)) k8sDiscoveryTest{ discovery: n, afterStart: func() { for _, ns := range []string{"ns1", "ns2"} { - obj := makeIngress(TLSNo) - obj.Namespace = ns + obj := makeIngress(ns, TLSNo) c.NetworkingV1().Ingresses(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) } }, @@ -303,41 +208,145 @@ func TestIngressDiscoveryNamespaces(t *testing.T) { }.Run(t) } -func TestIngressDiscoveryNamespacesV1beta1(t *testing.T) { - n, c := makeDiscoveryWithVersion(RoleIngress, NamespaceDiscovery{Names: []string{"ns1", "ns2"}}, "v1.18.0") +func TestIngressDiscoveryOwnNamespace(t *testing.T) { + t.Parallel() + n, c := makeDiscovery(RoleIngress, NamespaceDiscovery{IncludeOwnNamespace: true}) - expected := expectedTargetGroups("ns1", TLSNo) - for k, v := range expectedTargetGroups("ns2", TLSNo) { - expected[k] = v - } + expected := expectedTargetGroups("own-ns", TLSNo) k8sDiscoveryTest{ discovery: n, afterStart: func() { - for _, ns := range []string{"ns1", "ns2"} { - obj := makeIngressV1beta1(TLSNo) - obj.Namespace = ns - c.NetworkingV1beta1().Ingresses(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) + for _, ns := range []string{"own-ns", "non-own-ns"} { + obj := makeIngress(ns, TLSNo) + c.NetworkingV1().Ingresses(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) } }, - expectedMaxItems: 2, + expectedMaxItems: 1, expectedRes: expected, }.Run(t) } -func TestIngressDiscoveryOwnNamespace(t *testing.T) { - n, c := makeDiscovery(RoleIngress, NamespaceDiscovery{IncludeOwnNamespace: true}) +func TestIngressDiscoveryWithNamespaceMetadata(t *testing.T) { + t.Parallel() - expected := expectedTargetGroups("own-ns", TLSNo) + ns := "test-ns" + nsLabels := map[string]string{"service": "web", "layer": "frontend"} + nsAnnotations := map[string]string{"contact": "platform", "release": "v5.6.7"} + + n, _ := makeDiscoveryWithMetadata(RoleIngress, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, makeNamespace(ns, nsLabels, nsAnnotations), makeIngress(ns, TLSNo)) k8sDiscoveryTest{ - discovery: n, + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("ingress/%s/testingress", ns): { + Targets: []model.LabelSet{ + { + "__address__": "example.com", + "__meta_kubernetes_ingress_host": "example.com", + "__meta_kubernetes_ingress_path": "/", + "__meta_kubernetes_ingress_scheme": "http", + }, + { + "__address__": "example.com", + "__meta_kubernetes_ingress_host": "example.com", + "__meta_kubernetes_ingress_path": "/foo", + "__meta_kubernetes_ingress_scheme": "http", + }, + { + "__address__": "test.example.com", + "__meta_kubernetes_ingress_host": "test.example.com", + "__meta_kubernetes_ingress_path": "/", + "__meta_kubernetes_ingress_scheme": "http", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_contact": "platform", + "__meta_kubernetes_namespace_annotationpresent_contact": "true", + "__meta_kubernetes_namespace_annotation_release": "v5.6.7", + "__meta_kubernetes_namespace_annotationpresent_release": "true", + "__meta_kubernetes_namespace_label_service": "web", + "__meta_kubernetes_namespace_labelpresent_service": "true", + "__meta_kubernetes_namespace_label_layer": "frontend", + "__meta_kubernetes_namespace_labelpresent_layer": "true", + "__meta_kubernetes_ingress_name": "testingress", + "__meta_kubernetes_ingress_label_test_label": "testvalue", + "__meta_kubernetes_ingress_labelpresent_test_label": "true", + "__meta_kubernetes_ingress_annotation_test_annotation": "testannotationvalue", + "__meta_kubernetes_ingress_annotationpresent_test_annotation": "true", + "__meta_kubernetes_ingress_class_name": "testclass", + }, + Source: fmt.Sprintf("ingress/%s/testingress", ns), + }, + }, + }.Run(t) +} + +func TestIngressDiscoveryWithUpdatedNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"component": "database", "layer": "backend"} + nsAnnotations := map[string]string{"contact": "dba", "release": "v6.7.8"} + + namespace := makeNamespace(ns, nsLabels, nsAnnotations) + n, c := makeDiscoveryWithMetadata(RoleIngress, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, namespace, makeIngress(ns, TLSNo)) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 2, afterStart: func() { - for _, ns := range []string{"own-ns", "non-own-ns"} { - obj := makeIngress(TLSNo) - obj.Namespace = ns - c.NetworkingV1().Ingresses(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) - } + namespace.Labels["component"] = "cache" + namespace.Labels["region"] = "us-central" + namespace.Annotations["contact"] = "sre" + namespace.Annotations["monitoring"] = "enabled" + c.CoreV1().Namespaces().Update(context.Background(), namespace, metav1.UpdateOptions{}) + }, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("ingress/%s/testingress", ns): { + Targets: []model.LabelSet{ + { + "__address__": "example.com", + "__meta_kubernetes_ingress_host": "example.com", + "__meta_kubernetes_ingress_path": "/", + "__meta_kubernetes_ingress_scheme": "http", + }, + { + "__address__": "example.com", + "__meta_kubernetes_ingress_host": "example.com", + "__meta_kubernetes_ingress_path": "/foo", + "__meta_kubernetes_ingress_scheme": "http", + }, + { + "__address__": "test.example.com", + "__meta_kubernetes_ingress_host": "test.example.com", + "__meta_kubernetes_ingress_path": "/", + "__meta_kubernetes_ingress_scheme": "http", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_contact": "sre", + "__meta_kubernetes_namespace_annotationpresent_contact": "true", + "__meta_kubernetes_namespace_annotation_release": "v6.7.8", + "__meta_kubernetes_namespace_annotationpresent_release": "true", + "__meta_kubernetes_namespace_annotation_monitoring": "enabled", + "__meta_kubernetes_namespace_annotationpresent_monitoring": "true", + "__meta_kubernetes_namespace_label_component": "cache", + "__meta_kubernetes_namespace_labelpresent_component": "true", + "__meta_kubernetes_namespace_label_layer": "backend", + "__meta_kubernetes_namespace_labelpresent_layer": "true", + "__meta_kubernetes_namespace_label_region": "us-central", + "__meta_kubernetes_namespace_labelpresent_region": "true", + "__meta_kubernetes_ingress_name": "testingress", + "__meta_kubernetes_ingress_label_test_label": "testvalue", + "__meta_kubernetes_ingress_labelpresent_test_label": "true", + "__meta_kubernetes_ingress_annotation_test_annotation": "testannotationvalue", + "__meta_kubernetes_ingress_annotationpresent_test_annotation": "true", + "__meta_kubernetes_ingress_class_name": "testclass", + }, + Source: fmt.Sprintf("ingress/%s/testingress", ns), + }, }, - expectedMaxItems: 1, - expectedRes: expected, }.Run(t) } diff --git a/discovery/kubernetes/kubernetes.go b/discovery/kubernetes/kubernetes.go index a8b6f85899d..67cc2336e39 100644 --- a/discovery/kubernetes/kubernetes.go +++ b/discovery/kubernetes/kubernetes.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,42 +17,38 @@ import ( "context" "errors" "fmt" + "log/slog" "os" "reflect" + "slices" "strings" "sync" "time" - "github.com/prometheus/prometheus/util/strutil" - - disv1beta1 "k8s.io/api/discovery/v1beta1" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/common/version" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" apiv1 "k8s.io/api/core/v1" disv1 "k8s.io/api/discovery/v1" networkv1 "k8s.io/api/networking/v1" - "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // Required to get the GCP auth provider working. "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" - // Required to get the GCP auth provider working. - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" ) const ( @@ -63,14 +59,10 @@ const ( presentValue = model.LabelValue("true") ) -var ( - // Http header. - userAgent = fmt.Sprintf("Prometheus/%s", version.Version) - // DefaultSDConfig is the default Kubernetes SD configuration. - DefaultSDConfig = SDConfig{ - HTTPClientConfig: config.DefaultHTTPClientConfig, - } -) +// DefaultSDConfig is the default Kubernetes SD configuration. +var DefaultSDConfig = SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, +} func init() { discovery.RegisterConfig(&SDConfig{}) @@ -90,7 +82,7 @@ const ( ) // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *Role) UnmarshalYAML(unmarshal func(any) error) error { if err := unmarshal((*string)(c)); err != nil { return err } @@ -163,13 +155,22 @@ type resourceSelector struct { } // AttachMetadataConfig is the configuration for attaching additional metadata -// coming from nodes on which the targets are scheduled. +// coming from namespaces or nodes on which the targets are scheduled. type AttachMetadataConfig struct { - Node bool `yaml:"node"` + Node bool `yaml:"node"` + Namespace bool `yaml:"namespace"` + PodMetadataConfig `yaml:",inline"` +} + +// PodMetadataConfig allows configuring which pod-related metadata to attach. +type PodMetadataConfig struct { + Deployment bool `yaml:"deployment"` + Job bool `yaml:"job"` + CronJob bool `yaml:"cronjob"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -177,7 +178,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } if c.Role == "" { - return fmt.Errorf("role missing (one of: pod, service, endpoints, endpointslice, node, ingress)") + return errors.New("role missing (one of: pod, service, endpoints, endpointslice, node, ingress)") } err = c.HTTPClientConfig.Validate() if err != nil { @@ -185,25 +186,25 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { } if c.APIServer.URL != nil && c.KubeConfig != "" { // Api-server and kubeconfig_file are mutually exclusive - return fmt.Errorf("cannot use 'kubeconfig_file' and 'api_server' simultaneously") + return errors.New("cannot use 'kubeconfig_file' and 'api_server' simultaneously") } if c.KubeConfig != "" && !reflect.DeepEqual(c.HTTPClientConfig, config.DefaultHTTPClientConfig) { // Kubeconfig_file and custom http config are mutually exclusive - return fmt.Errorf("cannot use a custom HTTP client configuration together with 'kubeconfig_file'") + return errors.New("cannot use a custom HTTP client configuration together with 'kubeconfig_file'") } if c.APIServer.URL == nil && !reflect.DeepEqual(c.HTTPClientConfig, config.DefaultHTTPClientConfig) { - return fmt.Errorf("to use custom HTTP client configuration please provide the 'api_server' URL explicitly") + return errors.New("to use custom HTTP client configuration please provide the 'api_server' URL explicitly") } if c.APIServer.URL != nil && c.NamespaceDiscovery.IncludeOwnNamespace { - return fmt.Errorf("cannot use 'api_server' and 'namespaces.own_namespace' simultaneously") + return errors.New("cannot use 'api_server' and 'namespaces.own_namespace' simultaneously") } if c.KubeConfig != "" && c.NamespaceDiscovery.IncludeOwnNamespace { - return fmt.Errorf("cannot use 'kubeconfig_file' and 'namespaces.own_namespace' simultaneously") + return errors.New("cannot use 'kubeconfig_file' and 'namespaces.own_namespace' simultaneously") } foundSelectorRoles := make(map[Role]struct{}) allowedSelectors := map[Role][]string{ - RolePod: {string(RolePod)}, + RolePod: {string(RolePod), string(RoleNode)}, RoleService: {string(RoleService)}, RoleEndpointSlice: {string(RolePod), string(RoleService), string(RoleEndpointSlice)}, RoleEndpoint: {string(RolePod), string(RoleService), string(RoleEndpoint)}, @@ -220,18 +221,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if _, ok := allowedSelectors[c.Role]; !ok { return fmt.Errorf("invalid role: %q, expecting one of: pod, service, endpoints, endpointslice, node or ingress", c.Role) } - var allowed bool - for _, role := range allowedSelectors[c.Role] { - if role == string(selector.Role) { - allowed = true - break - } - } - - if !allowed { + if !slices.Contains(allowedSelectors[c.Role], string(selector.Role)) { return fmt.Errorf("%s role supports only %s selectors", c.Role, strings.Join(allowedSelectors[c.Role], ", ")) } - _, err := fields.ParseSelector(selector.Field) if err != nil { return err @@ -252,7 +244,7 @@ type NamespaceDiscovery struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *NamespaceDiscovery) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *NamespaceDiscovery) UnmarshalYAML(unmarshal func(any) error) error { *c = NamespaceDiscovery{} type plain NamespaceDiscovery return unmarshal((*plain)(c)) @@ -262,9 +254,9 @@ func (c *NamespaceDiscovery) UnmarshalYAML(unmarshal func(interface{}) error) er // targets from Kubernetes. type Discovery struct { sync.RWMutex - client kubernetes.Interface + client k8sClient role Role - logger log.Logger + logger *slog.Logger namespaceDiscovery *NamespaceDiscovery discoverers []discovery.Discoverer selectors roleSelector @@ -289,14 +281,14 @@ func (d *Discovery) getNamespaces() []string { } // New creates a new Kubernetes discovery for the given role. -func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Discovery, error) { +func New(l *slog.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Discovery, error) { m, ok := metrics.(*kubernetesMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } if l == nil { - l = log.NewNopLogger() + l = promslog.NewNopLogger() } var ( kcfg *rest.Config @@ -328,7 +320,7 @@ func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Di ownNamespace = string(ownNamespaceContents) } - level.Info(l).Log("msg", "Using pod service account via in-cluster config") + l.Info("Using pod service account via in-cluster config") default: rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "kubernetes_sd") if err != nil { @@ -340,7 +332,7 @@ func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Di } } - kcfg.UserAgent = userAgent + kcfg.UserAgent = version.PrometheusUserAgent() kcfg.ContentType = "application/vnd.kubernetes.protobuf" c, err := kubernetes.NewForConfig(kcfg) @@ -349,7 +341,7 @@ func New(l log.Logger, metrics discovery.DiscovererMetrics, conf *SDConfig) (*Di } d := &Discovery{ - client: c, + client: newClientAdapter(c), logger: l, role: conf.Role, namespaceDiscovery: &conf.NamespaceDiscovery, @@ -401,64 +393,31 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { switch d.role { case RoleEndpointSlice: - // Check "networking.k8s.io/v1" availability with retries. - // If "v1" is not available, use "networking.k8s.io/v1beta1" for backward compatibility - var v1Supported bool - if retryOnError(ctx, 10*time.Second, - func() (err error) { - v1Supported, err = checkDiscoveryV1Supported(d.client) - if err != nil { - level.Error(d.logger).Log("msg", "Failed to check networking.k8s.io/v1 availability", "err", err) - } - return err - }, - ) { - d.Unlock() - return - } - for _, namespace := range namespaces { var informer cache.SharedIndexInformer - if v1Supported { - e := d.client.DiscoveryV1().EndpointSlices(namespace) - elw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - options.FieldSelector = d.selectors.endpointslice.field - options.LabelSelector = d.selectors.endpointslice.label - return e.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - options.FieldSelector = d.selectors.endpointslice.field - options.LabelSelector = d.selectors.endpointslice.label - return e.Watch(ctx, options) - }, - } - informer = d.newEndpointSlicesByNodeInformer(elw, &disv1.EndpointSlice{}) - } else { - e := d.client.DiscoveryV1beta1().EndpointSlices(namespace) - elw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - options.FieldSelector = d.selectors.endpointslice.field - options.LabelSelector = d.selectors.endpointslice.label - return e.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - options.FieldSelector = d.selectors.endpointslice.field - options.LabelSelector = d.selectors.endpointslice.label - return e.Watch(ctx, options) - }, - } - informer = d.newEndpointSlicesByNodeInformer(elw, &disv1beta1.EndpointSlice{}) + e := d.client.DiscoveryV1().EndpointSlices(namespace) + elw := &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = d.selectors.endpointslice.field + options.LabelSelector = d.selectors.endpointslice.label + return e.List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = d.selectors.endpointslice.field + options.LabelSelector = d.selectors.endpointslice.label + return e.Watch(ctx, options) + }, } + informer = d.newIndexedEndpointSlicesInformer(elw, &disv1.EndpointSlice{}) s := d.client.CoreV1().Services(namespace) slw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.service.field options.LabelSelector = d.selectors.service.label return s.List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.service.field options.LabelSelector = d.selectors.service.label return s.Watch(ctx, options) @@ -466,12 +425,12 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } p := d.client.CoreV1().Pods(namespace) plw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.pod.field options.LabelSelector = d.selectors.pod.label return p.List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.pod.field options.LabelSelector = d.selectors.pod.label return p.Watch(ctx, options) @@ -482,12 +441,32 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { nodeInf = d.newNodeInformer(context.Background()) go nodeInf.Run(ctx.Done()) } + var namespaceInf cache.SharedInformer + if d.attachMetadata.Namespace { + namespaceInf = d.newNamespaceInformer(context.Background()) + go namespaceInf.Run(ctx.Done()) + } + var replicaSetInf, jobInformer cache.SharedInformer + if d.attachMetadata.Deployment { + replicaSetInf = d.newReplicaSetInformer(ctx, namespace) + go replicaSetInf.Run(ctx.Done()) + } + if d.attachMetadata.Job || d.attachMetadata.CronJob { + jobInformer = d.newJobInformer(ctx, namespace) + go jobInformer.Run(ctx.Done()) + } eps := NewEndpointSlice( - log.With(d.logger, "role", "endpointslice"), + d.logger.With("role", "endpointslice"), informer, d.mustNewSharedInformer(slw, &apiv1.Service{}, resyncDisabled), d.mustNewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled), nodeInf, + namespaceInf, + replicaSetInf, + jobInformer, + d.attachMetadata.Deployment, + d.attachMetadata.Job, + d.attachMetadata.CronJob, d.metrics.eventCount, ) d.discoverers = append(d.discoverers, eps) @@ -499,12 +478,12 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { for _, namespace := range namespaces { e := d.client.CoreV1().Endpoints(namespace) elw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.endpoints.field options.LabelSelector = d.selectors.endpoints.label return e.List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.endpoints.field options.LabelSelector = d.selectors.endpoints.label return e.Watch(ctx, options) @@ -512,12 +491,12 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } s := d.client.CoreV1().Services(namespace) slw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.service.field options.LabelSelector = d.selectors.service.label return s.List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.service.field options.LabelSelector = d.selectors.service.label return s.Watch(ctx, options) @@ -525,12 +504,12 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } p := d.client.CoreV1().Pods(namespace) plw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.pod.field options.LabelSelector = d.selectors.pod.label return p.List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.pod.field options.LabelSelector = d.selectors.pod.label return p.Watch(ctx, options) @@ -541,13 +520,34 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { nodeInf = d.newNodeInformer(ctx) go nodeInf.Run(ctx.Done()) } + var namespaceInf cache.SharedInformer + if d.attachMetadata.Namespace { + namespaceInf = d.newNamespaceInformer(ctx) + go namespaceInf.Run(ctx.Done()) + } + + var replicaSetInf, jobInformer cache.SharedInformer + if d.attachMetadata.Deployment { + replicaSetInf = d.newReplicaSetInformer(ctx, namespace) + go replicaSetInf.Run(ctx.Done()) + } + if d.attachMetadata.Job || d.attachMetadata.CronJob { + jobInformer = d.newJobInformer(ctx, namespace) + go jobInformer.Run(ctx.Done()) + } eps := NewEndpoints( - log.With(d.logger, "role", "endpoint"), - d.newEndpointsByNodeInformer(elw), + d.logger.With("role", "endpoint"), + d.newIndexedEndpointsInformer(elw), d.mustNewSharedInformer(slw, &apiv1.Service{}, resyncDisabled), d.mustNewSharedInformer(plw, &apiv1.Pod{}, resyncDisabled), nodeInf, + namespaceInf, + replicaSetInf, + jobInformer, + d.attachMetadata.Deployment, + d.attachMetadata.Job, + d.attachMetadata.CronJob, d.metrics.eventCount, ) d.discoverers = append(d.discoverers, eps) @@ -561,106 +561,105 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { nodeInformer = d.newNodeInformer(ctx) go nodeInformer.Run(ctx.Done()) } + var namespaceInformer cache.SharedInformer + if d.attachMetadata.Namespace { + namespaceInformer = d.newNamespaceInformer(ctx) + go namespaceInformer.Run(ctx.Done()) + } + var replicaSetInformer, jobInformer cache.SharedInformer for _, namespace := range namespaces { p := d.client.CoreV1().Pods(namespace) plw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.pod.field options.LabelSelector = d.selectors.pod.label return p.List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.pod.field options.LabelSelector = d.selectors.pod.label return p.Watch(ctx, options) }, } + if d.attachMetadata.Deployment { + replicaSetInformer = d.newReplicaSetInformer(ctx, namespace) + go replicaSetInformer.Run(ctx.Done()) + } + if d.attachMetadata.Job || d.attachMetadata.CronJob { + jobInformer = d.newJobInformer(ctx, namespace) + go jobInformer.Run(ctx.Done()) + } pod := NewPod( - log.With(d.logger, "role", "pod"), - d.newPodsByNodeInformer(plw), + d.logger.With("role", "pod"), + d.newIndexedPodsInformer(plw), nodeInformer, + namespaceInformer, + replicaSetInformer, + jobInformer, + d.attachMetadata.Deployment, + d.attachMetadata.Job, + d.attachMetadata.CronJob, d.metrics.eventCount, ) d.discoverers = append(d.discoverers, pod) go pod.podInf.Run(ctx.Done()) } case RoleService: + var namespaceInformer cache.SharedInformer + if d.attachMetadata.Namespace { + namespaceInformer = d.newNamespaceInformer(ctx) + go namespaceInformer.Run(ctx.Done()) + } + for _, namespace := range namespaces { s := d.client.CoreV1().Services(namespace) slw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.service.field options.LabelSelector = d.selectors.service.label return s.List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.service.field options.LabelSelector = d.selectors.service.label return s.Watch(ctx, options) }, } svc := NewService( - log.With(d.logger, "role", "service"), - d.mustNewSharedInformer(slw, &apiv1.Service{}, resyncDisabled), + d.logger.With("role", "service"), + d.newIndexedServicesInformer(slw), + namespaceInformer, d.metrics.eventCount, ) d.discoverers = append(d.discoverers, svc) go svc.informer.Run(ctx.Done()) } case RoleIngress: - // Check "networking.k8s.io/v1" availability with retries. - // If "v1" is not available, use "networking.k8s.io/v1beta1" for backward compatibility - var v1Supported bool - if retryOnError(ctx, 10*time.Second, - func() (err error) { - v1Supported, err = checkNetworkingV1Supported(d.client) - if err != nil { - level.Error(d.logger).Log("msg", "Failed to check networking.k8s.io/v1 availability", "err", err) - } - return err - }, - ) { - d.Unlock() - return + var namespaceInformer cache.SharedInformer + if d.attachMetadata.Namespace { + namespaceInformer = d.newNamespaceInformer(ctx) + go namespaceInformer.Run(ctx.Done()) } for _, namespace := range namespaces { - var informer cache.SharedInformer - if v1Supported { - i := d.client.NetworkingV1().Ingresses(namespace) - ilw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - options.FieldSelector = d.selectors.ingress.field - options.LabelSelector = d.selectors.ingress.label - return i.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - options.FieldSelector = d.selectors.ingress.field - options.LabelSelector = d.selectors.ingress.label - return i.Watch(ctx, options) - }, - } - informer = d.mustNewSharedInformer(ilw, &networkv1.Ingress{}, resyncDisabled) - } else { - i := d.client.NetworkingV1beta1().Ingresses(namespace) - ilw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - options.FieldSelector = d.selectors.ingress.field - options.LabelSelector = d.selectors.ingress.label - return i.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - options.FieldSelector = d.selectors.ingress.field - options.LabelSelector = d.selectors.ingress.label - return i.Watch(ctx, options) - }, - } - informer = d.mustNewSharedInformer(ilw, &v1beta1.Ingress{}, resyncDisabled) + i := d.client.NetworkingV1().Ingresses(namespace) + ilw := &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = d.selectors.ingress.field + options.LabelSelector = d.selectors.ingress.label + return i.List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = d.selectors.ingress.field + options.LabelSelector = d.selectors.ingress.label + return i.Watch(ctx, options) + }, } ingress := NewIngress( - log.With(d.logger, "role", "ingress"), - informer, + d.logger.With("role", "ingress"), + d.newIndexedIngressesInformer(ilw), + namespaceInformer, d.metrics.eventCount, ) d.discoverers = append(d.discoverers, ingress) @@ -668,11 +667,11 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } case RoleNode: nodeInformer := d.newNodeInformer(ctx) - node := NewNode(log.With(d.logger, "role", "node"), nodeInformer, d.metrics.eventCount) + node := NewNode(d.logger.With("role", "node"), nodeInformer, d.metrics.eventCount) d.discoverers = append(d.discoverers, node) go node.informer.Run(ctx.Done()) default: - level.Error(d.logger).Log("msg", "unknown Kubernetes discovery kind", "role", d.role) + d.logger.Error("unknown Kubernetes discovery kind", "role", d.role) } var wg sync.WaitGroup @@ -720,28 +719,40 @@ func retryOnError(ctx context.Context, interval time.Duration, f func() error) ( } } -func checkNetworkingV1Supported(client kubernetes.Interface) (bool, error) { - k8sVer, err := client.Discovery().ServerVersion() - if err != nil { - return false, err +func (d *Discovery) newReplicaSetInformer(_ context.Context, namespace string) cache.SharedInformer { + rs := d.client.AppsV1().ReplicaSets(namespace) + rslw := &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + return rs.List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + return rs.Watch(ctx, options) + }, } - semVer, err := utilversion.ParseSemantic(k8sVer.String()) - if err != nil { - return false, err + return d.mustNewSharedInformer(rslw, &appsv1.ReplicaSet{}, resyncDisabled) +} + +func (d *Discovery) newJobInformer(_ context.Context, namespace string) cache.SharedInformer { + j := d.client.BatchV1().Jobs(namespace) + jlw := &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + return j.List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + return j.Watch(ctx, options) + }, } - // networking.k8s.io/v1 is available since Kubernetes v1.19 - // https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.19.md - return semVer.Major() >= 1 && semVer.Minor() >= 19, nil + return d.mustNewSharedInformer(jlw, &batchv1.Job{}, resyncDisabled) } -func (d *Discovery) newNodeInformer(ctx context.Context) cache.SharedInformer { +func (d *Discovery) newNodeInformer(_ context.Context) cache.SharedInformer { nlw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = d.selectors.node.field options.LabelSelector = d.selectors.node.label return d.client.CoreV1().Nodes().List(ctx, options) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { options.FieldSelector = d.selectors.node.field options.LabelSelector = d.selectors.node.label return d.client.CoreV1().Nodes().Watch(ctx, options) @@ -750,27 +761,72 @@ func (d *Discovery) newNodeInformer(ctx context.Context) cache.SharedInformer { return d.mustNewSharedInformer(nlw, &apiv1.Node{}, resyncDisabled) } -func (d *Discovery) newPodsByNodeInformer(plw *cache.ListWatch) cache.SharedIndexInformer { +func (d *Discovery) newNamespaceInformer(_ context.Context) cache.SharedInformer { + // We don't filter on NamespaceDiscovery. + nlw := &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + return d.client.CoreV1().Namespaces().List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + return d.client.CoreV1().Namespaces().Watch(ctx, options) + }, + } + return d.mustNewSharedInformer(nlw, &apiv1.Namespace{}, resyncDisabled) +} + +func (d *Discovery) newIndexedPodsInformer(plw *cache.ListWatch) cache.SharedIndexInformer { indexers := make(map[string]cache.IndexFunc) if d.attachMetadata.Node { - indexers[nodeIndex] = func(obj interface{}) ([]string, error) { + indexers[nodeIndex] = func(obj any) ([]string, error) { pod, ok := obj.(*apiv1.Pod) if !ok { - return nil, fmt.Errorf("object is not a pod") + return nil, errors.New("object is not a pod") } return []string{pod.Spec.NodeName}, nil } } + if d.attachMetadata.Namespace { + indexers[cache.NamespaceIndex] = cache.MetaNamespaceIndexFunc + } + + if d.attachMetadata.Deployment { + indexers[replicaSetIndex] = func(obj any) ([]string, error) { + pod, ok := obj.(*apiv1.Pod) + if !ok { + return nil, errors.New("object is not a pod") + } + owner := GetControllerOf(pod) + if owner == nil || owner.Kind != "ReplicaSet" { + return nil, nil + } + return []string{namespacedName(pod.Namespace, owner.Name)}, nil + } + } + + if d.attachMetadata.Job || d.attachMetadata.CronJob { + indexers[jobIndex] = func(obj any) ([]string, error) { + pod, ok := obj.(*apiv1.Pod) + if !ok { + return nil, errors.New("object is not a pod") + } + owner := GetControllerOf(pod) + if owner == nil || owner.Kind != "Job" { + return nil, nil + } + return []string{namespacedName(pod.Namespace, owner.Name)}, nil + } + } + return d.mustNewSharedIndexInformer(plw, &apiv1.Pod{}, resyncDisabled, indexers) } -func (d *Discovery) newEndpointsByNodeInformer(plw *cache.ListWatch) cache.SharedIndexInformer { +func (d *Discovery) newIndexedEndpointsInformer(plw *cache.ListWatch) cache.SharedIndexInformer { indexers := make(map[string]cache.IndexFunc) - indexers[podIndex] = func(obj interface{}) ([]string, error) { + indexers[podIndex] = func(obj any) ([]string, error) { e, ok := obj.(*apiv1.Endpoints) if !ok { - return nil, fmt.Errorf("object is not endpoints") + return nil, errors.New("object is not endpoints") } var pods []string for _, target := range e.Subsets { @@ -782,59 +838,63 @@ func (d *Discovery) newEndpointsByNodeInformer(plw *cache.ListWatch) cache.Share } return pods, nil } - if !d.attachMetadata.Node { - return d.mustNewSharedIndexInformer(plw, &apiv1.Endpoints{}, resyncDisabled, indexers) - } - indexers[nodeIndex] = func(obj interface{}) ([]string, error) { - e, ok := obj.(*apiv1.Endpoints) - if !ok { - return nil, fmt.Errorf("object is not endpoints") - } - var nodes []string - for _, target := range e.Subsets { - for _, addr := range target.Addresses { - if addr.TargetRef != nil { - switch addr.TargetRef.Kind { - case "Pod": - if addr.NodeName != nil { - nodes = append(nodes, *addr.NodeName) + if d.attachMetadata.Node { + indexers[nodeIndex] = func(obj any) ([]string, error) { + e, ok := obj.(*apiv1.Endpoints) + if !ok { + return nil, errors.New("object is not endpoints") + } + var nodes []string + for _, target := range e.Subsets { + for _, addr := range target.Addresses { + if addr.TargetRef != nil { + switch addr.TargetRef.Kind { + case "Pod": + if addr.NodeName != nil { + nodes = append(nodes, *addr.NodeName) + } + case "Node": + nodes = append(nodes, addr.TargetRef.Name) } - case "Node": - nodes = append(nodes, addr.TargetRef.Name) } } } + return nodes, nil } - return nodes, nil + } + + if d.attachMetadata.Namespace { + indexers[cache.NamespaceIndex] = cache.MetaNamespaceIndexFunc } return d.mustNewSharedIndexInformer(plw, &apiv1.Endpoints{}, resyncDisabled, indexers) } -func (d *Discovery) newEndpointSlicesByNodeInformer(plw *cache.ListWatch, object runtime.Object) cache.SharedIndexInformer { +func (d *Discovery) newIndexedEndpointSlicesInformer(plw *cache.ListWatch, object runtime.Object) cache.SharedIndexInformer { indexers := make(map[string]cache.IndexFunc) - if !d.attachMetadata.Node { - return d.mustNewSharedIndexInformer(plw, object, resyncDisabled, indexers) + indexers[serviceIndex] = func(obj any) ([]string, error) { + e, ok := obj.(*disv1.EndpointSlice) + if !ok { + return nil, errors.New("object is not an endpointslice") + } + + svcName, exists := e.Labels[disv1.LabelServiceName] + if !exists { + return nil, nil + } + + return []string{namespacedName(e.Namespace, svcName)}, nil } - indexers[nodeIndex] = func(obj interface{}) ([]string, error) { - var nodes []string - switch e := obj.(type) { - case *disv1.EndpointSlice: - for _, target := range e.Endpoints { - if target.TargetRef != nil { - switch target.TargetRef.Kind { - case "Pod": - if target.NodeName != nil { - nodes = append(nodes, *target.NodeName) - } - case "Node": - nodes = append(nodes, target.TargetRef.Name) - } - } + if d.attachMetadata.Node { + indexers[nodeIndex] = func(obj any) ([]string, error) { + e, ok := obj.(*disv1.EndpointSlice) + if !ok { + return nil, errors.New("object is not an endpointslice") } - case *disv1beta1.EndpointSlice: + + var nodes []string for _, target := range e.Endpoints { if target.TargetRef != nil { switch target.TargetRef.Kind { @@ -847,26 +907,48 @@ func (d *Discovery) newEndpointSlicesByNodeInformer(plw *cache.ListWatch, object } } } - default: - return nil, fmt.Errorf("object is not an endpointslice") + + return nodes, nil } + } - return nodes, nil + if d.attachMetadata.Namespace { + indexers[cache.NamespaceIndex] = cache.MetaNamespaceIndexFunc } return d.mustNewSharedIndexInformer(plw, object, resyncDisabled, indexers) } -func (d *Discovery) informerWatchErrorHandler(r *cache.Reflector, err error) { +func (d *Discovery) newIndexedServicesInformer(slw *cache.ListWatch) cache.SharedIndexInformer { + indexers := make(map[string]cache.IndexFunc) + + if d.attachMetadata.Namespace { + indexers[cache.NamespaceIndex] = cache.MetaNamespaceIndexFunc + } + + return d.mustNewSharedIndexInformer(slw, &apiv1.Service{}, resyncDisabled, indexers) +} + +func (d *Discovery) newIndexedIngressesInformer(ilw *cache.ListWatch) cache.SharedIndexInformer { + indexers := make(map[string]cache.IndexFunc) + + if d.attachMetadata.Namespace { + indexers[cache.NamespaceIndex] = cache.MetaNamespaceIndexFunc + } + + return d.mustNewSharedIndexInformer(ilw, &networkv1.Ingress{}, resyncDisabled, indexers) +} + +func (d *Discovery) informerWatchErrorHandler(ctx context.Context, r *cache.Reflector, err error) { d.metrics.failuresCount.Inc() - cache.DefaultWatchErrorHandler(r, err) + cache.DefaultWatchErrorHandler(ctx, r, err) } func (d *Discovery) mustNewSharedInformer(lw cache.ListerWatcher, exampleObject runtime.Object, defaultEventHandlerResyncPeriod time.Duration) cache.SharedInformer { informer := cache.NewSharedInformer(lw, exampleObject, defaultEventHandlerResyncPeriod) // Invoking SetWatchErrorHandler should fail only if the informer has been started beforehand. // Such a scenario would suggest an incorrect use of the API, thus the panic. - if err := informer.SetWatchErrorHandler(d.informerWatchErrorHandler); err != nil { + if err := informer.SetWatchErrorHandlerWithContext(d.informerWatchErrorHandler); err != nil { panic(err) } return informer @@ -876,43 +958,45 @@ func (d *Discovery) mustNewSharedIndexInformer(lw cache.ListerWatcher, exampleOb informer := cache.NewSharedIndexInformer(lw, exampleObject, defaultEventHandlerResyncPeriod, indexers) // Invoking SetWatchErrorHandler should fail only if the informer has been started beforehand. // Such a scenario would suggest an incorrect use of the API, thus the panic. - if err := informer.SetWatchErrorHandler(d.informerWatchErrorHandler); err != nil { + if err := informer.SetWatchErrorHandlerWithContext(d.informerWatchErrorHandler); err != nil { panic(err) } return informer } -func checkDiscoveryV1Supported(client kubernetes.Interface) (bool, error) { - k8sVer, err := client.Discovery().ServerVersion() - if err != nil { - return false, err +func addObjectAnnotationsAndLabels(labelSet model.LabelSet, objectMeta metav1.ObjectMeta, resource string) { + for k, v := range objectMeta.Labels { + ln := strutil.SanitizeLabelName(k) + labelSet[model.LabelName(metaLabelPrefix+resource+"_label_"+ln)] = lv(v) + labelSet[model.LabelName(metaLabelPrefix+resource+"_labelpresent_"+ln)] = presentValue } - semVer, err := utilversion.ParseSemantic(k8sVer.String()) - if err != nil { - return false, err + for k, v := range objectMeta.Annotations { + ln := strutil.SanitizeLabelName(k) + labelSet[model.LabelName(metaLabelPrefix+resource+"_annotation_"+ln)] = lv(v) + labelSet[model.LabelName(metaLabelPrefix+resource+"_annotationpresent_"+ln)] = presentValue } - // The discovery.k8s.io/v1beta1 API version of EndpointSlice will no longer be served in v1.25. - // discovery.k8s.io/v1 is available since Kubernetes v1.21 - // https://kubernetes.io/docs/reference/using-api/deprecation-guide/#v1-25 - return semVer.Major() >= 1 && semVer.Minor() >= 21, nil } func addObjectMetaLabels(labelSet model.LabelSet, objectMeta metav1.ObjectMeta, role Role) { labelSet[model.LabelName(metaLabelPrefix+string(role)+"_name")] = lv(objectMeta.Name) + addObjectAnnotationsAndLabels(labelSet, objectMeta, string(role)) +} - for k, v := range objectMeta.Labels { - ln := strutil.SanitizeLabelName(k) - labelSet[model.LabelName(metaLabelPrefix+string(role)+"_label_"+ln)] = lv(v) - labelSet[model.LabelName(metaLabelPrefix+string(role)+"_labelpresent_"+ln)] = presentValue - } - - for k, v := range objectMeta.Annotations { - ln := strutil.SanitizeLabelName(k) - labelSet[model.LabelName(metaLabelPrefix+string(role)+"_annotation_"+ln)] = lv(v) - labelSet[model.LabelName(metaLabelPrefix+string(role)+"_annotationpresent_"+ln)] = presentValue - } +func addNamespaceMetaLabels(labelSet model.LabelSet, objectMeta metav1.ObjectMeta) { + // Omitting the namespace name because should be already injected elsewhere. + addObjectAnnotationsAndLabels(labelSet, objectMeta, "namespace") } func namespacedName(namespace, name string) string { return namespace + "/" + name } + +// nodeName knows how to handle the cache.DeletedFinalStateUnknown tombstone. +// It assumes the MetaNamespaceKeyFunc keyFunc is used, which uses the node name as the tombstone key. +func nodeName(o any) (string, error) { + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(o) + if err != nil { + return "", err + } + return key, nil +} diff --git a/discovery/kubernetes/kubernetes_test.go b/discovery/kubernetes/kubernetes_test.go index 50f25a20ab3..b4bba381a4d 100644 --- a/discovery/kubernetes/kubernetes_test.go +++ b/discovery/kubernetes/kubernetes_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,13 +17,17 @@ import ( "context" "encoding/json" "errors" + "os" "testing" "time" - "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" + apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/version" "k8s.io/apimachinery/pkg/watch" @@ -33,25 +37,31 @@ import ( kubetesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/testutil" ) func TestMain(m *testing.M) { + // Disable the WatchListClient feature gate that is enabled by default in + // client-go v0.35.0+. The WatchList flow requires the server to support + // SendInitialEvents and to send a bookmark event with the + // "k8s.io/initial-events-end" annotation. The fake clientset used in tests + // does not support this protocol, causing informers to hang indefinitely + // waiting for the bookmark. Disabling this feature restores the traditional + // List+Watch flow which is compatible with the fake clientset. + os.Setenv("KUBE_FEATURE_WatchListClient", "false") testutil.TolerantVerifyLeak(m) } // makeDiscovery creates a kubernetes.Discovery instance for testing. func makeDiscovery(role Role, nsDiscovery NamespaceDiscovery, objects ...runtime.Object) (*Discovery, kubernetes.Interface) { - return makeDiscoveryWithVersion(role, nsDiscovery, "v1.22.0", objects...) + return makeDiscoveryWithVersion(role, nsDiscovery, "v1.25.0", objects...) } // makeDiscoveryWithVersion creates a kubernetes.Discovery instance with the specified kubernetes version for testing. func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer string, objects ...runtime.Object) (*Discovery, kubernetes.Interface) { - clientset := fake.NewSimpleClientset(objects...) + clientset := fake.NewClientset(objects...) fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery) fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: k8sVer} @@ -71,7 +81,7 @@ func makeDiscoveryWithVersion(role Role, nsDiscovery NamespaceDiscovery, k8sVer d := &Discovery{ client: clientset, - logger: log.NewNopLogger(), + logger: promslog.NewNopLogger(), role: role, namespaceDiscovery: &nsDiscovery, ownNamespace: "own-ns", @@ -197,7 +207,7 @@ func requireTargetGroups(t *testing.T, expected, res map[string]*targetgroup.Gro panic(err) } - require.Equal(t, string(b1), string(b2)) + require.JSONEq(t, string(b1), string(b2)) } // marshalTargetGroups serializes a set of target groups to JSON, ignoring the @@ -271,6 +281,7 @@ func (s *Service) hasSynced() bool { } func TestRetryOnError(t *testing.T) { + t.Parallel() for _, successAt := range []int{1, 2, 3} { var called int f := func() error { @@ -285,41 +296,8 @@ func TestRetryOnError(t *testing.T) { } } -func TestCheckNetworkingV1Supported(t *testing.T) { - tests := []struct { - version string - wantSupported bool - wantErr bool - }{ - {version: "v1.18.0", wantSupported: false, wantErr: false}, - {version: "v1.18.1", wantSupported: false, wantErr: false}, - // networking v1 is supported since Kubernetes v1.19 - {version: "v1.19.0", wantSupported: true, wantErr: false}, - {version: "v1.20.0-beta.2", wantSupported: true, wantErr: false}, - // error patterns - {version: "", wantSupported: false, wantErr: true}, - {version: "<>", wantSupported: false, wantErr: true}, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.version, func(t *testing.T) { - clientset := fake.NewSimpleClientset() - fakeDiscovery, _ := clientset.Discovery().(*fakediscovery.FakeDiscovery) - fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: tc.version} - supported, err := checkNetworkingV1Supported(clientset) - - if tc.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - require.Equal(t, tc.wantSupported, supported) - }) - } -} - func TestFailuresCountMetric(t *testing.T) { + t.Parallel() tests := []struct { role Role minFailedWatches int @@ -333,7 +311,6 @@ func TestFailuresCountMetric(t *testing.T) { } for _, tc := range tests { - tc := tc t.Run(string(tc.role), func(t *testing.T) { t.Parallel() @@ -342,7 +319,7 @@ func TestFailuresCountMetric(t *testing.T) { require.Equal(t, float64(0), prom_testutil.ToFloat64(n.metrics.failuresCount)) // Simulate an error on watch requests. - c.Discovery().(*fakediscovery.FakeDiscovery).PrependWatchReactor("*", func(action kubetesting.Action) (bool, watch.Interface, error) { + c.Discovery().(*fakediscovery.FakeDiscovery).PrependWatchReactor("*", func(kubetesting.Action) (bool, watch.Interface, error) { return true, nil, apierrors.NewUnauthorized("unauthorized") }) @@ -354,3 +331,19 @@ func TestFailuresCountMetric(t *testing.T) { }) } } + +func TestNodeName(t *testing.T) { + t.Parallel() + node := &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + name, err := nodeName(node) + require.NoError(t, err) + require.Equal(t, "foo", name) + + name, err = nodeName(cache.DeletedFinalStateUnknown{Key: "bar"}) + require.NoError(t, err) + require.Equal(t, "bar", name) +} diff --git a/discovery/kubernetes/metrics.go b/discovery/kubernetes/metrics.go index fe419bc7828..cdf158a0327 100644 --- a/discovery/kubernetes/metrics.go +++ b/discovery/kubernetes/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -28,7 +28,7 @@ type kubernetesMetrics struct { metricRegisterer discovery.MetricRegisterer } -func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func newDiscovererMetrics(reg prometheus.Registerer, _ discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { m := &kubernetesMetrics{ eventCount: prometheus.NewCounterVec( prometheus.CounterOpts{ diff --git a/discovery/kubernetes/node.go b/discovery/kubernetes/node.go index 74d87e22c4b..08e2c604cf8 100644 --- a/discovery/kubernetes/node.go +++ b/discovery/kubernetes/node.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,13 +17,14 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" + "strings" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" apiv1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -38,16 +39,16 @@ const ( // Node discovers Kubernetes nodes. type Node struct { - logger log.Logger + logger *slog.Logger informer cache.SharedInformer store cache.Store - queue *workqueue.Type + queue *workqueue.Typed[string] } // NewNode returns a new node discovery. -func NewNode(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Node { +func NewNode(l *slog.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Node { if l == nil { - l = log.NewNopLogger() + l = promslog.NewNopLogger() } nodeAddCount := eventCount.WithLabelValues(RoleNode.String(), MetricLabelRoleAdd) @@ -58,31 +59,33 @@ func NewNode(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.Coun logger: l, informer: inf, store: inf.GetStore(), - queue: workqueue.NewNamed(RoleNode.String()), + queue: workqueue.NewTypedWithConfig(workqueue.TypedQueueConfig[string]{ + Name: RoleNode.String(), + }), } _, err := n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { nodeAddCount.Inc() n.enqueue(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { nodeDeleteCount.Inc() n.enqueue(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { nodeUpdateCount.Inc() n.enqueue(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding nodes event handler.", "err", err) + l.Error("Error adding nodes event handler.", "err", err) } return n } -func (n *Node) enqueue(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) +func (n *Node) enqueue(obj any) { + key, err := nodeName(obj) if err != nil { return } @@ -96,7 +99,7 @@ func (n *Node) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { if !cache.WaitForCacheSync(ctx.Done(), n.informer.HasSynced) { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(n.logger).Log("msg", "node informer unable to sync cache") + n.logger.Error("node informer unable to sync cache") } return } @@ -111,12 +114,11 @@ func (n *Node) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } func (n *Node) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool { - keyObj, quit := n.queue.Get() + key, quit := n.queue.Get() if quit { return false } - defer n.queue.Done(keyObj) - key := keyObj.(string) + defer n.queue.Done(key) _, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -133,14 +135,14 @@ func (n *Node) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool } node, err := convertToNode(o) if err != nil { - level.Error(n.logger).Log("msg", "converting to Node object failed", "err", err) + n.logger.Error("converting to Node object failed", "err", err) return true } send(ctx, ch, n.buildNode(node)) return true } -func convertToNode(o interface{}) (*apiv1.Node, error) { +func convertToNode(o any) (*apiv1.Node, error) { node, ok := o.(*apiv1.Node) if ok { return node, nil @@ -159,6 +161,7 @@ func nodeSourceFromName(name string) string { const ( nodeProviderIDLabel = metaLabelPrefix + "node_provider_id" + nodeConditionPrefix = metaLabelPrefix + "node_condition_" nodeAddressPrefix = metaLabelPrefix + "node_address_" ) @@ -168,6 +171,13 @@ func nodeLabels(n *apiv1.Node) model.LabelSet { ls[nodeProviderIDLabel] = lv(n.Spec.ProviderID) + // Export all node conditions as individual meta labels + for _, condition := range n.Status.Conditions { + conditionType := strings.ToLower(string(condition.Type)) + labelName := nodeConditionPrefix + strutil.SanitizeLabelName(conditionType) + ls[model.LabelName(labelName)] = lv(strings.ToLower(string(condition.Status))) + } + addObjectMetaLabels(ls, n.ObjectMeta, RoleNode) return ls @@ -181,7 +191,7 @@ func (n *Node) buildNode(node *apiv1.Node) *targetgroup.Group { addr, addrMap, err := nodeAddress(node) if err != nil { - level.Warn(n.logger).Log("msg", "No node address found", "err", err) + n.logger.Warn("No node address found", "err", err) return nil } addr = net.JoinHostPort(addr, strconv.FormatInt(int64(node.Status.DaemonEndpoints.KubeletEndpoint.Port), 10)) diff --git a/discovery/kubernetes/node_test.go b/discovery/kubernetes/node_test.go index bbf7a6b27ce..4bee0a2e69e 100644 --- a/discovery/kubernetes/node_test.go +++ b/discovery/kubernetes/node_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -25,7 +25,7 @@ import ( "github.com/prometheus/prometheus/discovery/targetgroup" ) -func makeNode(name, address, providerID string, labels, annotations map[string]string) *v1.Node { +func makeNode(name, address, providerID string, labels, annotations map[string]string, conditions []v1.NodeCondition) *v1.Node { return &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -47,15 +47,17 @@ func makeNode(name, address, providerID string, labels, annotations map[string]s Port: 10250, }, }, + Conditions: conditions, }, } } func makeEnumeratedNode(i int) *v1.Node { - return makeNode(fmt.Sprintf("test%d", i), "1.2.3.4", fmt.Sprintf("aws:///de-west-3a/i-%d", i), map[string]string{}, map[string]string{}) + return makeNode(fmt.Sprintf("test%d", i), "1.2.3.4", fmt.Sprintf("aws:///de-west-3a/i-%d", i), map[string]string{}, map[string]string{}, nil) } func TestNodeDiscoveryBeforeStart(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleNode, NamespaceDiscovery{}) k8sDiscoveryTest{ @@ -67,6 +69,7 @@ func TestNodeDiscoveryBeforeStart(t *testing.T) { "aws:///nl-north-7b/i-03149834983492827", map[string]string{"test-label": "testvalue"}, map[string]string{"test-annotation": "testannotationvalue"}, + nil, ) c.CoreV1().Nodes().Create(context.Background(), obj, metav1.CreateOptions{}) }, @@ -95,6 +98,7 @@ func TestNodeDiscoveryBeforeStart(t *testing.T) { } func TestNodeDiscoveryAdd(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleNode, NamespaceDiscovery{}) k8sDiscoveryTest{ @@ -124,6 +128,7 @@ func TestNodeDiscoveryAdd(t *testing.T) { } func TestNodeDiscoveryDelete(t *testing.T) { + t.Parallel() obj := makeEnumeratedNode(0) n, c := makeDiscovery(RoleNode, NamespaceDiscovery{}, obj) @@ -142,6 +147,7 @@ func TestNodeDiscoveryDelete(t *testing.T) { } func TestNodeDiscoveryUpdate(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleNode, NamespaceDiscovery{}) k8sDiscoveryTest{ @@ -155,6 +161,7 @@ func TestNodeDiscoveryUpdate(t *testing.T) { "aws:///fr-south-1c/i-49508290343823952", map[string]string{"Unschedulable": "true"}, map[string]string{}, + nil, ) c.CoreV1().Nodes().Update(context.Background(), obj2, metav1.UpdateOptions{}) }, @@ -179,3 +186,122 @@ func TestNodeDiscoveryUpdate(t *testing.T) { }, }.Run(t) } + +func TestNodeDiscoveryConditions(t *testing.T) { + t.Parallel() + tests := []struct { + name string + conditions []v1.NodeCondition + expected model.LabelSet + }{ + { + name: "node ready true", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_ready": "true", + }, + }, + { + name: "node ready false", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionFalse, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_ready": "false", + }, + }, + { + name: "node ready unknown", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionUnknown, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_ready": "unknown", + }, + }, + { + name: "node no conditions", + conditions: nil, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + }, + }, + { + name: "node multiple conditions", + conditions: []v1.NodeCondition{ + { + Type: v1.NodeMemoryPressure, + Status: v1.ConditionFalse, + }, + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + }, + { + Type: v1.NodeDiskPressure, + Status: v1.ConditionFalse, + }, + }, + expected: model.LabelSet{ + "__meta_kubernetes_node_name": "test", + "__meta_kubernetes_node_provider_id": "aws:///test-zone/i-test", + "__meta_kubernetes_node_condition_memorypressure": "false", + "__meta_kubernetes_node_condition_ready": "true", + "__meta_kubernetes_node_condition_diskpressure": "false", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n, c := makeDiscovery(RoleNode, NamespaceDiscovery{}) + + k8sDiscoveryTest{ + discovery: n, + beforeRun: func() { + obj := makeNode( + "test", + "1.2.3.4", + "aws:///test-zone/i-test", + map[string]string{}, + map[string]string{}, + tt.conditions, + ) + c.CoreV1().Nodes().Create(context.Background(), obj, metav1.CreateOptions{}) + }, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "node/test": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:10250", + "instance": "test", + "__meta_kubernetes_node_address_InternalIP": "1.2.3.4", + }, + }, + Labels: tt.expected, + Source: "node/test", + }, + }, + }.Run(t) + }) + } +} diff --git a/discovery/kubernetes/pod.go b/discovery/kubernetes/pod.go index 02990e415f9..7d594e0f357 100644 --- a/discovery/kubernetes/pod.go +++ b/discovery/kubernetes/pod.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,14 +17,16 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" "strings" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" @@ -34,24 +36,33 @@ import ( ) const ( - nodeIndex = "node" - podIndex = "pod" + nodeIndex = "node" + podIndex = "pod" + replicaSetIndex = "replicaset" + jobIndex = "job" ) // Pod discovers new pod targets. type Pod struct { - podInf cache.SharedIndexInformer - nodeInf cache.SharedInformer - withNodeMetadata bool - store cache.Store - logger log.Logger - queue *workqueue.Type + podInf cache.SharedIndexInformer + nodeInf cache.SharedInformer + withNodeMetadata bool + namespaceInf cache.SharedInformer + withNamespaceMetadata bool + replicaSetInf cache.SharedInformer + withDeploymentMetadata bool + jobInf cache.SharedInformer + withJobMetadata bool + withCronJobMetadata bool + store cache.Store + logger *slog.Logger + queue *workqueue.Typed[string] } // NewPod creates a new pod discovery. -func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInformer, eventCount *prometheus.CounterVec) *Pod { +func NewPod(l *slog.Logger, pods cache.SharedIndexInformer, nodes, namespace, replicaSets, jobs cache.SharedInformer, withDeploymentMetadata, withJobMetadata, withCronJobMetadata bool, eventCount *prometheus.CounterVec) *Pod { if l == nil { - l = log.NewNopLogger() + l = promslog.NewNopLogger() } podAddCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleAdd) @@ -59,55 +70,113 @@ func NewPod(l log.Logger, pods cache.SharedIndexInformer, nodes cache.SharedInfo podUpdateCount := eventCount.WithLabelValues(RolePod.String(), MetricLabelRoleUpdate) p := &Pod{ - podInf: pods, - nodeInf: nodes, - withNodeMetadata: nodes != nil, - store: pods.GetStore(), - logger: l, - queue: workqueue.NewNamed(RolePod.String()), + podInf: pods, + nodeInf: nodes, + withNodeMetadata: nodes != nil, + namespaceInf: namespace, + withNamespaceMetadata: namespace != nil, + replicaSetInf: replicaSets, + withDeploymentMetadata: withDeploymentMetadata, + jobInf: jobs, + withJobMetadata: withJobMetadata, + withCronJobMetadata: withCronJobMetadata, + store: pods.GetStore(), + logger: l, + queue: workqueue.NewTypedWithConfig(workqueue.TypedQueueConfig[string]{ + Name: RolePod.String(), + }), } _, err := p.podInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { podAddCount.Inc() p.enqueue(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { podDeleteCount.Inc() p.enqueue(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { podUpdateCount.Inc() p.enqueue(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding pods event handler.", "err", err) + l.Error("Error adding pods event handler.", "err", err) } if p.withNodeMetadata { _, err = p.nodeInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { node := o.(*apiv1.Node) p.enqueuePodsForNode(node.Name) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { node := o.(*apiv1.Node) p.enqueuePodsForNode(node.Name) }, - DeleteFunc: func(o interface{}) { - node := o.(*apiv1.Node) - p.enqueuePodsForNode(node.Name) + DeleteFunc: func(o any) { + nodeName, err := nodeName(o) + if err != nil { + l.Error("Error getting Node name", "err", err) + } + p.enqueuePodsForNode(nodeName) + }, + }) + if err != nil { + l.Error("Error adding pods event handler.", "err", err) + } + } + + if p.withNamespaceMetadata { + _, err = p.namespaceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, o any) { + namespace := o.(*apiv1.Namespace) + p.enqueuePodsForNamespace(namespace.Name) }, + // Creation and deletion will trigger events for the change handlers of the resources within the namespace. + // No need to have additional handlers for them here. }) if err != nil { - level.Error(l).Log("msg", "Error adding pods event handler.", "err", err) + l.Error("Error adding namespaces event handler.", "err", err) + } + } + + if p.withDeploymentMetadata && p.replicaSetInf != nil { + fn := func(o any) { + rs := o.(*appsv1.ReplicaSet) + rsName := namespacedName(rs.Namespace, rs.Name) + p.enqueuePodsForReplicaSet(rsName) + } + _, err = p.replicaSetInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(o any) { fn(o) }, + UpdateFunc: func(_, o any) { fn(o) }, + DeleteFunc: func(o any) { fn(o) }, + }) + if err != nil { + l.Error("Error adding replicasets event handler.", "err", err) + } + } + + if (p.withJobMetadata || p.withCronJobMetadata) && p.jobInf != nil { + fn := func(o any) { + job := o.(*batchv1.Job) + jobName := namespacedName(job.Namespace, job.Name) + p.enqueuePodsForJob(jobName) + } + _, err = p.jobInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(o any) { fn(o) }, + UpdateFunc: func(_, o any) { fn(o) }, + DeleteFunc: func(o any) { fn(o) }, + }) + if err != nil { + l.Error("Error adding jobs event handler.", "err", err) } } return p } -func (p *Pod) enqueue(obj interface{}) { +func (p *Pod) enqueue(obj any) { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { return @@ -124,10 +193,19 @@ func (p *Pod) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { if p.withNodeMetadata { cacheSyncs = append(cacheSyncs, p.nodeInf.HasSynced) } + if p.withNamespaceMetadata { + cacheSyncs = append(cacheSyncs, p.namespaceInf.HasSynced) + } + if p.withDeploymentMetadata && p.replicaSetInf != nil { + cacheSyncs = append(cacheSyncs, p.replicaSetInf.HasSynced) + } + if (p.withJobMetadata || p.withCronJobMetadata) && p.jobInf != nil { + cacheSyncs = append(cacheSyncs, p.jobInf.HasSynced) + } if !cache.WaitForCacheSync(ctx.Done(), cacheSyncs...) { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(p.logger).Log("msg", "pod informer unable to sync cache") + p.logger.Error("pod informer unable to sync cache") } return } @@ -142,12 +220,11 @@ func (p *Pod) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } func (p *Pod) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool { - keyObj, quit := p.queue.Get() + key, quit := p.queue.Get() if quit { return false } - defer p.queue.Done(keyObj) - key := keyObj.(string) + defer p.queue.Done(key) namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -164,14 +241,14 @@ func (p *Pod) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool } pod, err := convertToPod(o) if err != nil { - level.Error(p.logger).Log("msg", "converting to Pod object failed", "err", err) + p.logger.Error("converting to Pod object failed", "err", err) return true } send(ctx, ch, p.buildPod(pod)) return true } -func convertToPod(o interface{}) (*apiv1.Pod, error) { +func convertToPod(o any) (*apiv1.Pod, error) { pod, ok := o.(*apiv1.Pod) if ok { return pod, nil @@ -189,6 +266,9 @@ const ( podContainerPortNumberLabel = metaLabelPrefix + "pod_container_port_number" podContainerPortProtocolLabel = metaLabelPrefix + "pod_container_port_protocol" podContainerIsInit = metaLabelPrefix + "pod_container_init" + podCronJobNameLabel = metaLabelPrefix + "pod_cronjob_name" + podDeploymentNameLabel = metaLabelPrefix + "pod_deployment_name" + podJobNameLabel = metaLabelPrefix + "pod_job_name" podReadyLabel = metaLabelPrefix + "pod_ready" podPhaseLabel = metaLabelPrefix + "pod_phase" podNodeNameLabel = metaLabelPrefix + "pod_node_name" @@ -209,14 +289,14 @@ func GetControllerOf(controllee metav1.Object) *metav1.OwnerReference { return nil } -func podLabels(pod *apiv1.Pod) model.LabelSet { +func podLabels(pod *apiv1.Pod, replicaSetInf, jobInf cache.SharedInformer, withDeploymentMetadata, withJobMetadata, withCronJobMetadata bool) model.LabelSet { ls := model.LabelSet{ podIPLabel: lv(pod.Status.PodIP), podReadyLabel: podReady(pod), podPhaseLabel: lv(string(pod.Status.Phase)), podNodeNameLabel: lv(pod.Spec.NodeName), podHostIPLabel: lv(pod.Status.HostIP), - podUID: lv(string(pod.ObjectMeta.UID)), + podUID: lv(string(pod.UID)), } addObjectMetaLabels(ls, pod.ObjectMeta, RolePod) @@ -229,12 +309,43 @@ func podLabels(pod *apiv1.Pod) model.LabelSet { if createdBy.Name != "" { ls[podControllerName] = lv(createdBy.Name) } + switch createdBy.Kind { + case "ReplicaSet": + if replicaSetInf != nil && withDeploymentMetadata { + key := namespacedName(pod.Namespace, createdBy.Name) + obj, exists, err := replicaSetInf.GetStore().GetByKey(key) + if err == nil && exists { + if rs, ok := obj.(*appsv1.ReplicaSet); ok { + rsOwner := GetControllerOf(rs) + if rsOwner != nil && rsOwner.Kind == "Deployment" { + ls[podDeploymentNameLabel] = lv(rsOwner.Name) + } + } + } + } + case "Job": + if withJobMetadata { + ls[podJobNameLabel] = lv(createdBy.Name) + } + if jobInf != nil && withCronJobMetadata { + key := namespacedName(pod.Namespace, createdBy.Name) + obj, exists, err := jobInf.GetStore().GetByKey(key) + if err == nil && exists { + if job, ok := obj.(*batchv1.Job); ok { + jobOwner := GetControllerOf(job) + if jobOwner != nil && jobOwner.Kind == "CronJob" { + ls[podCronJobNameLabel] = lv(jobOwner.Name) + } + } + } + } + } } return ls } -func (p *Pod) findPodContainerStatus(statuses *[]apiv1.ContainerStatus, containerName string) (*apiv1.ContainerStatus, error) { +func (*Pod) findPodContainerStatus(statuses *[]apiv1.ContainerStatus, containerName string) (*apiv1.ContainerStatus, error) { for _, s := range *statuses { if s.Name == containerName { return &s, nil @@ -246,7 +357,7 @@ func (p *Pod) findPodContainerStatus(statuses *[]apiv1.ContainerStatus, containe func (p *Pod) findPodContainerID(statuses *[]apiv1.ContainerStatus, containerName string) string { cStatus, err := p.findPodContainerStatus(statuses, containerName) if err != nil { - level.Debug(p.logger).Log("msg", "cannot find container ID", "err", err) + p.logger.Debug("cannot find container ID", "err", err) return "" } return cStatus.ContainerID @@ -257,15 +368,31 @@ func (p *Pod) buildPod(pod *apiv1.Pod) *targetgroup.Group { Source: podSource(pod), } // PodIP can be empty when a pod is starting or has been evicted. - if len(pod.Status.PodIP) == 0 { + if pod.Status.PodIP == "" { return tg } - tg.Labels = podLabels(pod) + // Filter out pods scheduled on nodes that are not in the node store, as + // these were filtered out by node selectors. + if p.withNodeMetadata { + _, exists, err := p.nodeInf.GetStore().GetByKey(pod.Spec.NodeName) + if err != nil { + p.logger.Error("failed to get node from store", "node", pod.Spec.NodeName, "err", err) + return tg + } + if !exists { + return tg + } + } + + tg.Labels = podLabels(pod, p.replicaSetInf, p.jobInf, p.withDeploymentMetadata, p.withJobMetadata, p.withCronJobMetadata) tg.Labels[namespaceLabel] = lv(pod.Namespace) if p.withNodeMetadata { tg.Labels = addNodeLabels(tg.Labels, p.nodeInf, p.logger, &pod.Spec.NodeName) } + if p.withNamespaceMetadata { + tg.Labels = addNamespaceLabels(tg.Labels, p.namespaceInf, p.logger, pod.Namespace) + } containers := append(pod.Spec.Containers, pod.Spec.InitContainers...) for i, c := range containers { @@ -312,10 +439,46 @@ func (p *Pod) buildPod(pod *apiv1.Pod) *targetgroup.Group { return tg } +func (p *Pod) enqueuePodsForReplicaSet(rsName string) { + pods, err := p.podInf.GetIndexer().ByIndex(replicaSetIndex, rsName) + if err != nil { + p.logger.Error("Error getting pods for replicaset", "replicaset", rsName, "err", err) + return + } + + for _, pod := range pods { + p.enqueue(pod.(*apiv1.Pod)) + } +} + +func (p *Pod) enqueuePodsForJob(jobName string) { + pods, err := p.podInf.GetIndexer().ByIndex(jobIndex, jobName) + if err != nil { + p.logger.Error("Error getting pods for job", "job", jobName, "err", err) + return + } + + for _, pod := range pods { + p.enqueue(pod.(*apiv1.Pod)) + } +} + func (p *Pod) enqueuePodsForNode(nodeName string) { pods, err := p.podInf.GetIndexer().ByIndex(nodeIndex, nodeName) if err != nil { - level.Error(p.logger).Log("msg", "Error getting pods for node", "node", nodeName, "err", err) + p.logger.Error("Error getting pods for node", "node", nodeName, "err", err) + return + } + + for _, pod := range pods { + p.enqueue(pod.(*apiv1.Pod)) + } +} + +func (p *Pod) enqueuePodsForNamespace(namespace string) { + pods, err := p.podInf.GetIndexer().ByIndex(cache.NamespaceIndex, namespace) + if err != nil { + p.logger.Error("Error getting pods in namespace", "namespace", namespace, "err", err) return } diff --git a/discovery/kubernetes/pod_test.go b/discovery/kubernetes/pod_test.go index 286a1a230d7..8cc48345224 100644 --- a/discovery/kubernetes/pod_test.go +++ b/discovery/kubernetes/pod_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,9 +16,12 @@ package kubernetes import ( "context" "fmt" + "maps" "testing" "github.com/prometheus/common/model" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -95,11 +98,11 @@ func makeMultiPortPods() *v1.Pod { } } -func makePods() *v1.Pod { +func makePods(namespace string) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "testpod", - Namespace: "default", + Namespace: namespace, UID: types.UID("abc123"), }, Spec: v1.PodSpec{ @@ -239,6 +242,7 @@ func expectedPodTargetGroupsWithNodeMeta(ns, nodeName string, nodeLabels map[str } func TestPodDiscoveryBeforeRun(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RolePod, NamespaceDiscovery{}) k8sDiscoveryTest{ @@ -302,6 +306,7 @@ func TestPodDiscoveryBeforeRun(t *testing.T) { } func TestPodDiscoveryInitContainer(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RolePod, NamespaceDiscovery{}) ns := "default" @@ -329,12 +334,13 @@ func TestPodDiscoveryInitContainer(t *testing.T) { } func TestPodDiscoveryAdd(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RolePod, NamespaceDiscovery{}) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makePods() + obj := makePods("default") c.CoreV1().Pods(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) }, expectedMaxItems: 1, @@ -343,13 +349,14 @@ func TestPodDiscoveryAdd(t *testing.T) { } func TestPodDiscoveryDelete(t *testing.T) { - obj := makePods() + t.Parallel() + obj := makePods("default") n, c := makeDiscovery(RolePod, NamespaceDiscovery{}, obj) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makePods() + obj := makePods("default") c.CoreV1().Pods(obj.Namespace).Delete(context.Background(), obj.Name, metav1.DeleteOptions{}) }, expectedMaxItems: 2, @@ -362,6 +369,7 @@ func TestPodDiscoveryDelete(t *testing.T) { } func TestPodDiscoveryUpdate(t *testing.T) { + t.Parallel() obj := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "testpod", @@ -394,7 +402,7 @@ func TestPodDiscoveryUpdate(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makePods() + obj := makePods("default") c.CoreV1().Pods(obj.Namespace).Update(context.Background(), obj, metav1.UpdateOptions{}) }, expectedMaxItems: 2, @@ -403,10 +411,11 @@ func TestPodDiscoveryUpdate(t *testing.T) { } func TestPodDiscoveryUpdateEmptyPodIP(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RolePod, NamespaceDiscovery{}) - initialPod := makePods() + initialPod := makePods("default") - updatedPod := makePods() + updatedPod := makePods("default") updatedPod.Status.PodIP = "" k8sDiscoveryTest{ @@ -427,17 +436,16 @@ func TestPodDiscoveryUpdateEmptyPodIP(t *testing.T) { } func TestPodDiscoveryNamespaces(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RolePod, NamespaceDiscovery{Names: []string{"ns1", "ns2"}}) expected := expectedPodTargetGroups("ns1") - for k, v := range expectedPodTargetGroups("ns2") { - expected[k] = v - } + maps.Copy(expected, expectedPodTargetGroups("ns2")) k8sDiscoveryTest{ discovery: n, beforeRun: func() { for _, ns := range []string{"ns1", "ns2"} { - pod := makePods() + pod := makePods("default") pod.Namespace = ns c.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}) } @@ -448,6 +456,7 @@ func TestPodDiscoveryNamespaces(t *testing.T) { } func TestPodDiscoveryOwnNamespace(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RolePod, NamespaceDiscovery{IncludeOwnNamespace: true}) expected := expectedPodTargetGroups("own-ns") @@ -455,7 +464,7 @@ func TestPodDiscoveryOwnNamespace(t *testing.T) { discovery: n, beforeRun: func() { for _, ns := range []string{"own-ns", "non-own-ns"} { - pod := makePods() + pod := makePods("default") pod.Namespace = ns c.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}) } @@ -466,6 +475,7 @@ func TestPodDiscoveryOwnNamespace(t *testing.T) { } func TestPodDiscoveryWithNodeMetadata(t *testing.T) { + t.Parallel() attachMetadata := AttachMetadataConfig{Node: true} n, c := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, attachMetadata) nodeLbls := map[string]string{"l1": "v1"} @@ -473,10 +483,10 @@ func TestPodDiscoveryWithNodeMetadata(t *testing.T) { k8sDiscoveryTest{ discovery: n, afterStart: func() { - nodes := makeNode("testnode", "", "", nodeLbls, nil) + nodes := makeNode("testnode", "", "", nodeLbls, nil, nil) c.CoreV1().Nodes().Create(context.Background(), nodes, metav1.CreateOptions{}) - pods := makePods() + pods := makePods("default") c.CoreV1().Pods(pods.Namespace).Create(context.Background(), pods, metav1.CreateOptions{}) }, expectedMaxItems: 2, @@ -485,6 +495,7 @@ func TestPodDiscoveryWithNodeMetadata(t *testing.T) { } func TestPodDiscoveryWithNodeMetadataUpdateNode(t *testing.T) { + t.Parallel() nodeLbls := map[string]string{"l2": "v2"} attachMetadata := AttachMetadataConfig{Node: true} n, c := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, attachMetadata) @@ -493,17 +504,666 @@ func TestPodDiscoveryWithNodeMetadataUpdateNode(t *testing.T) { discovery: n, beforeRun: func() { oldNodeLbls := map[string]string{"l1": "v1"} - nodes := makeNode("testnode", "", "", oldNodeLbls, nil) + nodes := makeNode("testnode", "", "", oldNodeLbls, nil, nil) c.CoreV1().Nodes().Create(context.Background(), nodes, metav1.CreateOptions{}) }, afterStart: func() { - pods := makePods() + pods := makePods("default") c.CoreV1().Pods(pods.Namespace).Create(context.Background(), pods, metav1.CreateOptions{}) - nodes := makeNode("testnode", "", "", nodeLbls, nil) + nodes := makeNode("testnode", "", "", nodeLbls, nil, nil) c.CoreV1().Nodes().Update(context.Background(), nodes, metav1.UpdateOptions{}) }, expectedMaxItems: 2, expectedRes: expectedPodTargetGroupsWithNodeMeta("default", "testnode", nodeLbls), }.Run(t) } + +func TestPodDiscoveryWithNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"app": "web", "tier": "frontend"} + nsAnnotations := map[string]string{"maintainer": "devops", "build": "v3.4.5"} + + n, _ := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, makeNamespace(ns, nsLabels, nsAnnotations), makePods(ns)) + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("pod/%s/testpod", ns): { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_maintainer": "devops", + "__meta_kubernetes_namespace_annotationpresent_maintainer": "true", + "__meta_kubernetes_namespace_annotation_build": "v3.4.5", + "__meta_kubernetes_namespace_annotationpresent_build": "true", + "__meta_kubernetes_namespace_label_app": "web", + "__meta_kubernetes_namespace_labelpresent_app": "true", + "__meta_kubernetes_namespace_label_tier": "frontend", + "__meta_kubernetes_namespace_labelpresent_tier": "true", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_uid": "abc123", + }, + Source: fmt.Sprintf("pod/%s/testpod", ns), + }, + }, + }.Run(t) +} + +func TestPodDiscoveryWithUpdatedNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"app": "api", "tier": "backend"} + nsAnnotations := map[string]string{"maintainer": "platform", "build": "v4.5.6"} + + namespace := makeNamespace(ns, nsLabels, nsAnnotations) + n, c := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, namespace, makePods(ns)) + + k8sDiscoveryTest{ + discovery: n, + afterStart: func() { + namespace.Labels["app"] = "service" + namespace.Labels["zone"] = "us-east" + namespace.Annotations["maintainer"] = "sre" + namespace.Annotations["deployment"] = "canary" + c.CoreV1().Namespaces().Update(context.Background(), namespace, metav1.UpdateOptions{}) + }, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("pod/%s/testpod", ns): { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_maintainer": "sre", + "__meta_kubernetes_namespace_annotationpresent_maintainer": "true", + "__meta_kubernetes_namespace_annotation_build": "v4.5.6", + "__meta_kubernetes_namespace_annotationpresent_build": "true", + "__meta_kubernetes_namespace_annotation_deployment": "canary", + "__meta_kubernetes_namespace_annotationpresent_deployment": "true", + "__meta_kubernetes_namespace_label_app": "service", + "__meta_kubernetes_namespace_labelpresent_app": "true", + "__meta_kubernetes_namespace_label_tier": "backend", + "__meta_kubernetes_namespace_labelpresent_tier": "true", + "__meta_kubernetes_namespace_label_zone": "us-east", + "__meta_kubernetes_namespace_labelpresent_zone": "true", + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_uid": "abc123", + }, + Source: fmt.Sprintf("pod/%s/testpod", ns), + }, + }, + }.Run(t) +} + +func TestPodDiscoveryWithNodeSelector(t *testing.T) { + t.Parallel() + + workerNode := makeNode("worker-node", "10.0.0.1", "", map[string]string{"node-type": "worker"}, nil, nil) + filteredNode := makeNode("filtered-node", "10.0.0.2", "", map[string]string{"node-type": "master"}, nil, nil) + + attachMetadata := AttachMetadataConfig{ + Node: true, // necessary for node role selectos to work for pod role + } + n, c := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, attachMetadata, workerNode, filteredNode) + n.selectors = roleSelector{ + node: resourceSelector{ + label: "node-type=worker", + }, + } + + podOnWorker := makePods("default") + podOnWorker.Name = "pod-on-worker" + podOnWorker.UID = types.UID("worker-pod-123") + podOnWorker.Spec.NodeName = "worker-node" + podOnWorker.Status.PodIP = "192.168.1.1" + + podOnFilteredNode := makePods("default") + podOnFilteredNode.Name = "pod-on-filtered-node" + podOnFilteredNode.UID = types.UID("filtered-pod-456") + podOnFilteredNode.Spec.NodeName = "filtered-node" + podOnFilteredNode.Status.PodIP = "192.168.1.2" + + k8sDiscoveryTest{ + discovery: n, + beforeRun: func() { + c.CoreV1().Pods("default").Create(context.Background(), podOnWorker, metav1.CreateOptions{}) + c.CoreV1().Pods("default").Create(context.Background(), podOnFilteredNode, metav1.CreateOptions{}) + }, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "pod/default/pod-on-worker": { + Targets: []model.LabelSet{ + { + "__address__": "192.168.1.1:9000", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_name": "pod-on-worker", + "__meta_kubernetes_pod_ip": "192.168.1.1", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_node_name": "worker-node", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_uid": "worker-pod-123", + "__meta_kubernetes_node_name": "worker-node", + "__meta_kubernetes_node_label_node_type": "worker", + "__meta_kubernetes_node_labelpresent_node_type": "true", + }, + Source: "pod/default/pod-on-worker", + }, + "pod/default/pod-on-filtered-node": { + Source: "pod/default/pod-on-filtered-node", + }, + }, + }.Run(t) +} + +func makeReplicaSet(name, namespace, deploymentName string, deploymentUID types.UID) *appsv1.ReplicaSet { + return &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID("rs123"), + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + Name: deploymentName, + UID: deploymentUID, + Controller: makeOptionalBool(true), + }, + }, + }, + } +} + +func makeDeployment(name, namespace string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID("deploy123"), + }, + } +} + +func makeJob(name, namespace, cronJobName string, cronJobUID types.UID) *batchv1.Job { + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID("job123"), + }, + } + + if cronJobName != "" { + job.OwnerReferences = []metav1.OwnerReference{ + { + Kind: "CronJob", + Name: cronJobName, + UID: cronJobUID, + Controller: makeOptionalBool(true), + }, + } + } + + return job +} + +func makeCronJob(name, namespace string) *batchv1.CronJob { + return &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID("cronjob123"), + }, + } +} + +func makeReplicaSetOwnedPod(namespace, replicaSetName string, replicaSetUID types.UID) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + Namespace: namespace, + UID: types.UID("pod123"), + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: replicaSetName, + UID: replicaSetUID, + Controller: makeOptionalBool(true), + }, + }, + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + Containers: []v1.Container{ + { + Name: "testcontainer", + Image: "testcontainer:latest", + Ports: []v1.ContainerPort{ + { + Name: "testport", + Protocol: v1.ProtocolTCP, + ContainerPort: int32(9000), + }, + }, + }, + }, + }, + Status: v1.PodStatus{ + PodIP: "1.2.3.4", + HostIP: "2.3.4.5", + Phase: "Running", + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "testcontainer", + ContainerID: "docker://a1b2c3d4e5f6", + }, + }, + }, + } +} + +func makeJobOwnedPod(namespace, jobName string, jobUID types.UID) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + Namespace: namespace, + UID: types.UID("pod123"), + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Job", + Name: jobName, + UID: jobUID, + Controller: makeOptionalBool(true), + }, + }, + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + Containers: []v1.Container{ + { + Name: "testcontainer", + Image: "testcontainer:latest", + Ports: []v1.ContainerPort{ + { + Name: "testport", + Protocol: v1.ProtocolTCP, + ContainerPort: int32(9000), + }, + }, + }, + }, + }, + Status: v1.PodStatus{ + PodIP: "1.2.3.4", + HostIP: "2.3.4.5", + Phase: "Running", + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "testcontainer", + ContainerID: "docker://a1b2c3d4e5f6", + }, + }, + }, + } +} + +func TestPodDiscoveryWithDeployment(t *testing.T) { + t.Parallel() + + deployment := makeDeployment("testdeployment", "default") + replicaSet := makeReplicaSet("testdeployment-rs-abc", "default", deployment.Name, deployment.UID) + pod := makeReplicaSetOwnedPod("default", replicaSet.Name, replicaSet.UID) + + n, _ := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{ + PodMetadataConfig: PodMetadataConfig{Deployment: true}, + }, pod, replicaSet, deployment) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "pod/default/testpod": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_uid": "pod123", + "__meta_kubernetes_pod_controller_kind": "ReplicaSet", + "__meta_kubernetes_pod_controller_name": "testdeployment-rs-abc", + "__meta_kubernetes_pod_deployment_name": "testdeployment", + }, + Source: "pod/default/testpod", + }, + }, + }.Run(t) +} + +func TestPodDiscoveryWithJob(t *testing.T) { + t.Parallel() + + job := makeJob("testjob", "default", "", "") + pod := makeJobOwnedPod("default", job.Name, job.UID) + + n, _ := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{ + PodMetadataConfig: PodMetadataConfig{Job: true}, + }, pod, job) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "pod/default/testpod": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_uid": "pod123", + "__meta_kubernetes_pod_controller_kind": "Job", + "__meta_kubernetes_pod_controller_name": "testjob", + "__meta_kubernetes_pod_job_name": "testjob", + }, + Source: "pod/default/testpod", + }, + }, + }.Run(t) +} + +func TestPodDiscoveryWithCronJob(t *testing.T) { + t.Parallel() + + cronJob := makeCronJob("testcronjob", "default") + job := makeJob("testcronjob-job-abc", "default", cronJob.Name, cronJob.UID) + pod := makeJobOwnedPod("default", job.Name, job.UID) + + n, _ := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{ + PodMetadataConfig: PodMetadataConfig{Job: true}, + }, pod, job, cronJob) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + "pod/default/testpod": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_uid": "pod123", + "__meta_kubernetes_pod_controller_kind": "Job", + "__meta_kubernetes_pod_controller_name": "testcronjob-job-abc", + "__meta_kubernetes_pod_job_name": "testcronjob-job-abc", + }, + Source: "pod/default/testpod", + }, + }, + }.Run(t) +} + +func TestPodDiscoveryWithDeploymentUpdate(t *testing.T) { + t.Parallel() + + deployment := makeDeployment("testdeployment", "default") + deployment.Labels = map[string]string{"version": "v1"} + + replicaSet := makeReplicaSet("testdeployment-rs-abc", "default", deployment.Name, deployment.UID) + pod := makeReplicaSetOwnedPod("default", replicaSet.Name, replicaSet.UID) + + n, c := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{ + PodMetadataConfig: PodMetadataConfig{Deployment: true}, + }, pod, replicaSet, deployment) + + k8sDiscoveryTest{ + discovery: n, + afterStart: func() { + // Update deployment with new labels + deployment.Labels["version"] = "v2" + deployment.Labels["env"] = "prod" + c.AppsV1().Deployments(deployment.Namespace).Update(context.Background(), deployment, metav1.UpdateOptions{}) + }, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "pod/default/testpod": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_uid": "pod123", + "__meta_kubernetes_pod_controller_kind": "ReplicaSet", + "__meta_kubernetes_pod_controller_name": "testdeployment-rs-abc", + "__meta_kubernetes_pod_deployment_name": "testdeployment", + }, + Source: "pod/default/testpod", + }, + }, + }.Run(t) +} + +func TestPodDiscoveryWithJobUpdate(t *testing.T) { + t.Parallel() + + job := makeJob("testjob", "default", "", "") + job.Labels = map[string]string{"batch": "v1"} + + pod := makeJobOwnedPod("default", job.Name, job.UID) + + n, c := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{ + PodMetadataConfig: PodMetadataConfig{Job: true}, + }, pod, job) + + k8sDiscoveryTest{ + discovery: n, + afterStart: func() { + // Update job with new labels + job.Labels["batch"] = "v2" + job.Labels["priority"] = "high" + c.BatchV1().Jobs(job.Namespace).Update(context.Background(), job, metav1.UpdateOptions{}) + }, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "pod/default/testpod": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_uid": "pod123", + "__meta_kubernetes_pod_controller_kind": "Job", + "__meta_kubernetes_pod_controller_name": "testjob", + "__meta_kubernetes_pod_job_name": "testjob", + }, + Source: "pod/default/testpod", + }, + }, + }.Run(t) +} + +func TestPodDiscoveryWithCronJobUpdate(t *testing.T) { + t.Parallel() + + cronJob := makeCronJob("testcronjob", "default") + cronJob.Labels = map[string]string{"schedule": "hourly"} + + job := makeJob("testcronjob-job-abc", "default", cronJob.Name, cronJob.UID) + pod := makeJobOwnedPod("default", job.Name, job.UID) + + n, c := makeDiscoveryWithMetadata(RolePod, NamespaceDiscovery{}, AttachMetadataConfig{ + PodMetadataConfig: PodMetadataConfig{CronJob: true}, + }, pod, job, cronJob) + + k8sDiscoveryTest{ + discovery: n, + afterStart: func() { + // Update cronjob with new labels + cronJob.Labels["schedule"] = "daily" + cronJob.Labels["retention"] = "7d" + c.BatchV1().CronJobs(cronJob.Namespace).Update(context.Background(), cronJob, metav1.UpdateOptions{}) + }, + expectedMaxItems: 2, + expectedRes: map[string]*targetgroup.Group{ + "pod/default/testpod": { + Targets: []model.LabelSet{ + { + "__address__": "1.2.3.4:9000", + "__meta_kubernetes_pod_container_name": "testcontainer", + "__meta_kubernetes_pod_container_image": "testcontainer:latest", + "__meta_kubernetes_pod_container_port_name": "testport", + "__meta_kubernetes_pod_container_port_number": "9000", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_container_init": "false", + "__meta_kubernetes_pod_container_id": "docker://a1b2c3d4e5f6", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_pod_name": "testpod", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_node_name": "testnode", + "__meta_kubernetes_pod_ip": "1.2.3.4", + "__meta_kubernetes_pod_host_ip": "2.3.4.5", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_uid": "pod123", + "__meta_kubernetes_pod_controller_kind": "Job", + "__meta_kubernetes_pod_controller_name": "testcronjob-job-abc", + "__meta_kubernetes_pod_cronjob_name": "testcronjob", + }, + Source: "pod/default/testpod", + }, + }, + }.Run(t) +} diff --git a/discovery/kubernetes/service.go b/discovery/kubernetes/service.go index 51204a5a1af..ac2d42fc7c8 100644 --- a/discovery/kubernetes/service.go +++ b/discovery/kubernetes/service.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,13 +17,13 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" apiv1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -33,16 +33,18 @@ import ( // Service implements discovery of Kubernetes services. type Service struct { - logger log.Logger - informer cache.SharedInformer - store cache.Store - queue *workqueue.Type + logger *slog.Logger + informer cache.SharedIndexInformer + store cache.Store + queue *workqueue.Typed[string] + namespaceInf cache.SharedInformer + withNamespaceMetadata bool } // NewService returns a new service discovery. -func NewService(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.CounterVec) *Service { +func NewService(l *slog.Logger, inf cache.SharedIndexInformer, namespace cache.SharedInformer, eventCount *prometheus.CounterVec) *Service { if l == nil { - l = log.NewNopLogger() + l = promslog.NewNopLogger() } svcAddCount := eventCount.WithLabelValues(RoleService.String(), MetricLabelRoleAdd) @@ -53,30 +55,49 @@ func NewService(l log.Logger, inf cache.SharedInformer, eventCount *prometheus.C logger: l, informer: inf, store: inf.GetStore(), - queue: workqueue.NewNamed(RoleService.String()), + queue: workqueue.NewTypedWithConfig(workqueue.TypedQueueConfig[string]{ + Name: RoleService.String(), + }), + namespaceInf: namespace, + withNamespaceMetadata: namespace != nil, } _, err := s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { + AddFunc: func(o any) { svcAddCount.Inc() s.enqueue(o) }, - DeleteFunc: func(o interface{}) { + DeleteFunc: func(o any) { svcDeleteCount.Inc() s.enqueue(o) }, - UpdateFunc: func(_, o interface{}) { + UpdateFunc: func(_, o any) { svcUpdateCount.Inc() s.enqueue(o) }, }) if err != nil { - level.Error(l).Log("msg", "Error adding services event handler.", "err", err) + l.Error("Error adding services event handler.", "err", err) } + + if s.withNamespaceMetadata { + _, err = s.namespaceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, o any) { + namespace := o.(*apiv1.Namespace) + s.enqueueNamespace(namespace.Name) + }, + // Creation and deletion will trigger events for the change handlers of the resources within the namespace. + // No need to have additional handlers for them here. + }) + if err != nil { + l.Error("Error adding namespaces event handler.", "err", err) + } + } + return s } -func (s *Service) enqueue(obj interface{}) { +func (s *Service) enqueue(obj any) { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { return @@ -85,13 +106,30 @@ func (s *Service) enqueue(obj interface{}) { s.queue.Add(key) } +func (s *Service) enqueueNamespace(namespace string) { + services, err := s.informer.GetIndexer().ByIndex(cache.NamespaceIndex, namespace) + if err != nil { + s.logger.Error("Error getting services in namespace", "namespace", namespace, "err", err) + return + } + + for _, service := range services { + s.enqueue(service) + } +} + // Run implements the Discoverer interface. func (s *Service) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { defer s.queue.ShutDown() - if !cache.WaitForCacheSync(ctx.Done(), s.informer.HasSynced) { + cacheSyncs := []cache.InformerSynced{s.informer.HasSynced} + if s.withNamespaceMetadata { + cacheSyncs = append(cacheSyncs, s.namespaceInf.HasSynced) + } + + if !cache.WaitForCacheSync(ctx.Done(), cacheSyncs...) { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(s.logger).Log("msg", "service informer unable to sync cache") + s.logger.Error("service informer unable to sync cache") } return } @@ -106,12 +144,11 @@ func (s *Service) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } func (s *Service) process(ctx context.Context, ch chan<- []*targetgroup.Group) bool { - keyObj, quit := s.queue.Get() + key, quit := s.queue.Get() if quit { return false } - defer s.queue.Done(keyObj) - key := keyObj.(string) + defer s.queue.Done(key) namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -128,14 +165,14 @@ func (s *Service) process(ctx context.Context, ch chan<- []*targetgroup.Group) b } eps, err := convertToService(o) if err != nil { - level.Error(s.logger).Log("msg", "converting to Service object failed", "err", err) + s.logger.Error("converting to Service object failed", "err", err) return true } send(ctx, ch, s.buildService(eps)) return true } -func convertToService(o interface{}) (*apiv1.Service, error) { +func convertToService(o any) (*apiv1.Service, error) { service, ok := o.(*apiv1.Service) if ok { return service, nil @@ -175,6 +212,10 @@ func (s *Service) buildService(svc *apiv1.Service) *targetgroup.Group { } tg.Labels = serviceLabels(svc) + if s.withNamespaceMetadata { + tg.Labels = addNamespaceLabels(tg.Labels, s.namespaceInf, s.logger, svc.Namespace) + } + for _, port := range svc.Spec.Ports { addr := net.JoinHostPort(svc.Name+"."+svc.Namespace+".svc", strconv.FormatInt(int64(port.Port), 10)) diff --git a/discovery/kubernetes/service_test.go b/discovery/kubernetes/service_test.go index dde3aaea57d..56a785d9c22 100644 --- a/discovery/kubernetes/service_test.go +++ b/discovery/kubernetes/service_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -52,11 +52,11 @@ func makeMultiPortService() *v1.Service { } } -func makeSuffixedService(suffix string) *v1.Service { +func makeService(namespace string) *v1.Service { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("testservice%s", suffix), - Namespace: "default", + Name: "testservice", + Namespace: namespace, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ @@ -72,10 +72,6 @@ func makeSuffixedService(suffix string) *v1.Service { } } -func makeService() *v1.Service { - return makeSuffixedService("") -} - func makeExternalService() *v1.Service { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -118,12 +114,13 @@ func makeLoadBalancerService() *v1.Service { } func TestServiceDiscoveryAdd(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleService, NamespaceDiscovery{}) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makeService() + obj := makeService("default") c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) obj = makeExternalService() c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) @@ -189,12 +186,13 @@ func TestServiceDiscoveryAdd(t *testing.T) { } func TestServiceDiscoveryDelete(t *testing.T) { - n, c := makeDiscovery(RoleService, NamespaceDiscovery{}, makeService()) + t.Parallel() + n, c := makeDiscovery(RoleService, NamespaceDiscovery{}, makeService("default")) k8sDiscoveryTest{ discovery: n, afterStart: func() { - obj := makeService() + obj := makeService("default") c.CoreV1().Services(obj.Namespace).Delete(context.Background(), obj.Name, metav1.DeleteOptions{}) }, expectedMaxItems: 2, @@ -207,7 +205,8 @@ func TestServiceDiscoveryDelete(t *testing.T) { } func TestServiceDiscoveryUpdate(t *testing.T) { - n, c := makeDiscovery(RoleService, NamespaceDiscovery{}, makeService()) + t.Parallel() + n, c := makeDiscovery(RoleService, NamespaceDiscovery{}, makeService("default")) k8sDiscoveryTest{ discovery: n, @@ -251,14 +250,14 @@ func TestServiceDiscoveryUpdate(t *testing.T) { } func TestServiceDiscoveryNamespaces(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleService, NamespaceDiscovery{Names: []string{"ns1", "ns2"}}) k8sDiscoveryTest{ discovery: n, afterStart: func() { for _, ns := range []string{"ns1", "ns2"} { - obj := makeService() - obj.Namespace = ns + obj := makeService(ns) c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) } }, @@ -303,14 +302,14 @@ func TestServiceDiscoveryNamespaces(t *testing.T) { } func TestServiceDiscoveryOwnNamespace(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleService, NamespaceDiscovery{IncludeOwnNamespace: true}) k8sDiscoveryTest{ discovery: n, afterStart: func() { for _, ns := range []string{"own-ns", "non-own-ns"} { - obj := makeService() - obj.Namespace = ns + obj := makeService(ns) c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) } }, @@ -338,14 +337,14 @@ func TestServiceDiscoveryOwnNamespace(t *testing.T) { } func TestServiceDiscoveryAllNamespaces(t *testing.T) { + t.Parallel() n, c := makeDiscovery(RoleService, NamespaceDiscovery{}) k8sDiscoveryTest{ discovery: n, afterStart: func() { for _, ns := range []string{"own-ns", "non-own-ns"} { - obj := makeService() - obj.Namespace = ns + obj := makeService(ns) c.CoreV1().Services(obj.Namespace).Create(context.Background(), obj, metav1.CreateOptions{}) } }, @@ -388,3 +387,98 @@ func TestServiceDiscoveryAllNamespaces(t *testing.T) { }, }.Run(t) } + +func TestServiceDiscoveryWithNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"environment": "production", "team": "backend"} + nsAnnotations := map[string]string{"owner": "platform", "version": "v1.2.3"} + + n, _ := makeDiscoveryWithMetadata(RoleService, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, makeNamespace(ns, nsLabels, nsAnnotations), makeService(ns)) + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 1, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("svc/%s/testservice", ns): { + Targets: []model.LabelSet{ + { + "__address__": "testservice.test-ns.svc:30900", + "__meta_kubernetes_service_cluster_ip": "10.0.0.1", + "__meta_kubernetes_service_port_name": "testport", + "__meta_kubernetes_service_port_number": "30900", + "__meta_kubernetes_service_port_protocol": "TCP", + "__meta_kubernetes_service_type": "ClusterIP", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_owner": "platform", + "__meta_kubernetes_namespace_annotationpresent_owner": "true", + "__meta_kubernetes_namespace_annotation_version": "v1.2.3", + "__meta_kubernetes_namespace_annotationpresent_version": "true", + "__meta_kubernetes_namespace_label_environment": "production", + "__meta_kubernetes_namespace_labelpresent_environment": "true", + "__meta_kubernetes_namespace_label_team": "backend", + "__meta_kubernetes_namespace_labelpresent_team": "true", + "__meta_kubernetes_service_name": "testservice", + }, + Source: fmt.Sprintf("svc/%s/testservice", ns), + }, + }, + }.Run(t) +} + +func TestServiceDiscoveryWithUpdatedNamespaceMetadata(t *testing.T) { + t.Parallel() + + ns := "test-ns" + nsLabels := map[string]string{"environment": "development", "team": "frontend"} + nsAnnotations := map[string]string{"owner": "devops", "version": "v2.1.0"} + + namespace := makeNamespace(ns, nsLabels, nsAnnotations) + n, c := makeDiscoveryWithMetadata(RoleService, NamespaceDiscovery{}, AttachMetadataConfig{Namespace: true}, namespace, makeService(ns)) + + k8sDiscoveryTest{ + discovery: n, + expectedMaxItems: 2, + afterStart: func() { + namespace.Labels["environment"] = "staging" + namespace.Labels["region"] = "us-west" + namespace.Annotations["owner"] = "sre" + namespace.Annotations["cost-center"] = "engineering" + c.CoreV1().Namespaces().Update(context.Background(), namespace, metav1.UpdateOptions{}) + }, + expectedRes: map[string]*targetgroup.Group{ + fmt.Sprintf("svc/%s/testservice", ns): { + Targets: []model.LabelSet{ + { + "__address__": "testservice.test-ns.svc:30900", + "__meta_kubernetes_service_cluster_ip": "10.0.0.1", + "__meta_kubernetes_service_port_name": "testport", + "__meta_kubernetes_service_port_number": "30900", + "__meta_kubernetes_service_port_protocol": "TCP", + "__meta_kubernetes_service_type": "ClusterIP", + }, + }, + Labels: model.LabelSet{ + "__meta_kubernetes_namespace": model.LabelValue(ns), + "__meta_kubernetes_namespace_annotation_owner": "sre", + "__meta_kubernetes_namespace_annotationpresent_owner": "true", + "__meta_kubernetes_namespace_annotation_version": "v2.1.0", + "__meta_kubernetes_namespace_annotationpresent_version": "true", + "__meta_kubernetes_namespace_annotation_cost_center": "engineering", + "__meta_kubernetes_namespace_annotationpresent_cost_center": "true", + "__meta_kubernetes_namespace_label_environment": "staging", + "__meta_kubernetes_namespace_labelpresent_environment": "true", + "__meta_kubernetes_namespace_label_team": "frontend", + "__meta_kubernetes_namespace_labelpresent_team": "true", + "__meta_kubernetes_namespace_label_region": "us-west", + "__meta_kubernetes_namespace_labelpresent_region": "true", + "__meta_kubernetes_service_name": "testservice", + }, + Source: fmt.Sprintf("svc/%s/testservice", ns), + }, + }, + }.Run(t) +} diff --git a/discovery/legacymanager/manager.go b/discovery/legacymanager/manager.go deleted file mode 100644 index 6fc61485d11..00000000000 --- a/discovery/legacymanager/manager.go +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacymanager - -import ( - "context" - "fmt" - "reflect" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" - - "github.com/prometheus/prometheus/discovery" - "github.com/prometheus/prometheus/discovery/targetgroup" -) - -type poolKey struct { - setName string - provider string -} - -// provider holds a Discoverer instance, its configuration and its subscribers. -type provider struct { - name string - d discovery.Discoverer - subs []string - config interface{} -} - -// NewManager is the Discovery Manager constructor. -func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]discovery.DiscovererMetrics, options ...func(*Manager)) *Manager { - if logger == nil { - logger = log.NewNopLogger() - } - mgr := &Manager{ - logger: logger, - syncCh: make(chan map[string][]*targetgroup.Group), - targets: make(map[poolKey]map[string]*targetgroup.Group), - discoverCancel: []context.CancelFunc{}, - ctx: ctx, - updatert: 5 * time.Second, - triggerSend: make(chan struct{}, 1), - registerer: registerer, - sdMetrics: sdMetrics, - } - for _, option := range options { - option(mgr) - } - - // Register the metrics. - // We have to do this after setting all options, so that the name of the Manager is set. - if metrics, err := discovery.NewManagerMetrics(registerer, mgr.name); err == nil { - mgr.metrics = metrics - } else { - level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err) - return nil - } - - return mgr -} - -// Name sets the name of the manager. -func Name(n string) func(*Manager) { - return func(m *Manager) { - m.mtx.Lock() - defer m.mtx.Unlock() - m.name = n - } -} - -// Manager maintains a set of discovery providers and sends each update to a map channel. -// Targets are grouped by the target set name. -type Manager struct { - logger log.Logger - name string - mtx sync.RWMutex - ctx context.Context - discoverCancel []context.CancelFunc - - // Some Discoverers(eg. k8s) send only the updates for a given target group - // so we use map[tg.Source]*targetgroup.Group to know which group to update. - targets map[poolKey]map[string]*targetgroup.Group - // providers keeps track of SD providers. - providers []*provider - // The sync channel sends the updates as a map where the key is the job value from the scrape config. - syncCh chan map[string][]*targetgroup.Group - - // How long to wait before sending updates to the channel. The variable - // should only be modified in unit tests. - updatert time.Duration - - // The triggerSend channel signals to the manager that new updates have been received from providers. - triggerSend chan struct{} - - // A registerer for all service discovery metrics. - registerer prometheus.Registerer - - metrics *discovery.Metrics - sdMetrics map[string]discovery.DiscovererMetrics -} - -// Run starts the background processing. -func (m *Manager) Run() error { - go m.sender() - <-m.ctx.Done() - m.cancelDiscoverers() - return m.ctx.Err() -} - -// SyncCh returns a read only channel used by all the clients to receive target updates. -func (m *Manager) SyncCh() <-chan map[string][]*targetgroup.Group { - return m.syncCh -} - -// ApplyConfig removes all running discovery providers and starts new ones using the provided config. -func (m *Manager) ApplyConfig(cfg map[string]discovery.Configs) error { - m.mtx.Lock() - defer m.mtx.Unlock() - - for pk := range m.targets { - if _, ok := cfg[pk.setName]; !ok { - m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, pk.setName) - } - } - m.cancelDiscoverers() - m.targets = make(map[poolKey]map[string]*targetgroup.Group) - m.providers = nil - m.discoverCancel = nil - - failedCount := 0 - for name, scfg := range cfg { - failedCount += m.registerProviders(scfg, name) - m.metrics.DiscoveredTargets.WithLabelValues(name).Set(0) - } - m.metrics.FailedConfigs.Set(float64(failedCount)) - - for _, prov := range m.providers { - m.startProvider(m.ctx, prov) - } - - return nil -} - -// StartCustomProvider is used for sdtool. Only use this if you know what you're doing. -func (m *Manager) StartCustomProvider(ctx context.Context, name string, worker discovery.Discoverer) { - p := &provider{ - name: name, - d: worker, - subs: []string{name}, - } - m.providers = append(m.providers, p) - m.startProvider(ctx, p) -} - -func (m *Manager) startProvider(ctx context.Context, p *provider) { - level.Debug(m.logger).Log("msg", "Starting provider", "provider", p.name, "subs", fmt.Sprintf("%v", p.subs)) - ctx, cancel := context.WithCancel(ctx) - updates := make(chan []*targetgroup.Group) - - m.discoverCancel = append(m.discoverCancel, cancel) - - go p.d.Run(ctx, updates) - go m.updater(ctx, p, updates) -} - -func (m *Manager) updater(ctx context.Context, p *provider, updates chan []*targetgroup.Group) { - for { - select { - case <-ctx.Done(): - return - case tgs, ok := <-updates: - m.metrics.ReceivedUpdates.Inc() - if !ok { - level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name) - return - } - - for _, s := range p.subs { - m.updateGroup(poolKey{setName: s, provider: p.name}, tgs) - } - - select { - case m.triggerSend <- struct{}{}: - default: - } - } - } -} - -func (m *Manager) sender() { - ticker := time.NewTicker(m.updatert) - defer ticker.Stop() - - for { - select { - case <-m.ctx.Done(): - return - case <-ticker.C: // Some discoverers send updates too often so we throttle these with the ticker. - select { - case <-m.triggerSend: - m.metrics.SentUpdates.Inc() - select { - case m.syncCh <- m.allGroups(): - default: - m.metrics.DelayedUpdates.Inc() - level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle") - select { - case m.triggerSend <- struct{}{}: - default: - } - } - default: - } - } - } -} - -func (m *Manager) cancelDiscoverers() { - for _, c := range m.discoverCancel { - c() - } -} - -func (m *Manager) updateGroup(poolKey poolKey, tgs []*targetgroup.Group) { - m.mtx.Lock() - defer m.mtx.Unlock() - - if _, ok := m.targets[poolKey]; !ok { - m.targets[poolKey] = make(map[string]*targetgroup.Group) - } - for _, tg := range tgs { - if tg != nil { // Some Discoverers send nil target group so need to check for it to avoid panics. - m.targets[poolKey][tg.Source] = tg - } - } -} - -func (m *Manager) allGroups() map[string][]*targetgroup.Group { - m.mtx.RLock() - defer m.mtx.RUnlock() - - tSets := map[string][]*targetgroup.Group{} - n := map[string]int{} - for pkey, tsets := range m.targets { - for _, tg := range tsets { - // Even if the target group 'tg' is empty we still need to send it to the 'Scrape manager' - // to signal that it needs to stop all scrape loops for this target set. - tSets[pkey.setName] = append(tSets[pkey.setName], tg) - n[pkey.setName] += len(tg.Targets) - } - } - for setName, v := range n { - m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v)) - } - return tSets -} - -// registerProviders returns a number of failed SD config. -func (m *Manager) registerProviders(cfgs discovery.Configs, setName string) int { - var ( - failed int - added bool - ) - add := func(cfg discovery.Config) { - for _, p := range m.providers { - if reflect.DeepEqual(cfg, p.config) { - p.subs = append(p.subs, setName) - added = true - return - } - } - typ := cfg.Name() - d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{ - Logger: log.With(m.logger, "discovery", typ, "config", setName), - Metrics: m.sdMetrics[typ], - }) - if err != nil { - level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName) - failed++ - return - } - m.providers = append(m.providers, &provider{ - name: fmt.Sprintf("%s/%d", typ, len(m.providers)), - d: d, - config: cfg, - subs: []string{setName}, - }) - added = true - } - for _, cfg := range cfgs { - add(cfg) - } - if !added { - // Add an empty target group to force the refresh of the corresponding - // scrape pool and to notify the receiver that this target set has no - // current targets. - // It can happen because the combined set of SD configurations is empty - // or because we fail to instantiate all the SD configurations. - add(discovery.StaticConfig{{}}) - } - return failed -} - -// StaticProvider holds a list of target groups that never change. -type StaticProvider struct { - TargetGroups []*targetgroup.Group -} - -// Run implements the Worker interface. -func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { - // We still have to consider that the consumer exits right away in which case - // the context will be canceled. - select { - case ch <- sd.TargetGroups: - case <-ctx.Done(): - } - close(ch) -} diff --git a/discovery/legacymanager/manager_test.go b/discovery/legacymanager/manager_test.go deleted file mode 100644 index f1be9631136..00000000000 --- a/discovery/legacymanager/manager_test.go +++ /dev/null @@ -1,1185 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacymanager - -import ( - "context" - "fmt" - "sort" - "strconv" - "testing" - "time" - - "github.com/go-kit/log" - "github.com/prometheus/client_golang/prometheus" - client_testutil "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/discovery" - "github.com/prometheus/prometheus/discovery/targetgroup" - "github.com/prometheus/prometheus/util/testutil" -) - -func TestMain(m *testing.M) { - testutil.TolerantVerifyLeak(m) -} - -func newTestMetrics(t *testing.T, reg prometheus.Registerer) (*discovery.RefreshMetricsManager, map[string]discovery.DiscovererMetrics) { - refreshMetrics := discovery.NewRefreshMetrics(reg) - sdMetrics, err := discovery.RegisterSDMetrics(reg, refreshMetrics) - require.NoError(t, err) - return &refreshMetrics, sdMetrics -} - -// TestTargetUpdatesOrder checks that the target updates are received in the expected order. -func TestTargetUpdatesOrder(t *testing.T) { - // The order by which the updates are send is determined by the interval passed to the mock discovery adapter - // Final targets array is ordered alphabetically by the name of the discoverer. - // For example discoverer "A" with targets "t2,t3" and discoverer "B" with targets "t1,t2" will result in "t2,t3,t1,t2" after the merge. - testCases := []struct { - title string - updates map[string][]update - expectedTargets [][]*targetgroup.Group - }{ - { - title: "Single TP no updates", - updates: map[string][]update{ - "tp1": {}, - }, - expectedTargets: nil, - }, - { - title: "Multiple TPs no updates", - updates: map[string][]update{ - "tp1": {}, - "tp2": {}, - "tp3": {}, - }, - expectedTargets: nil, - }, - { - title: "Single TP empty initials", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{}, - interval: 5 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - {}, - }, - }, - { - title: "Multiple TPs empty initials", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{}, - interval: 5 * time.Millisecond, - }, - }, - "tp2": { - { - targetGroups: []targetgroup.Group{}, - interval: 200 * time.Millisecond, - }, - }, - "tp3": { - { - targetGroups: []targetgroup.Group{}, - interval: 100 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - {}, - {}, - {}, - }, - }, - { - title: "Single TP initials only", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - }, - }, - { - title: "Multiple TPs initials only", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - }, - }, - "tp2": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - }, - interval: 10 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - }, - }, - }, - { - title: "Single TP initials followed by empty updates", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - interval: 0, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{}, - }, - }, - interval: 10 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{}, - }, - }, - }, - }, - { - title: "Single TP initials and new groups", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - interval: 0, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - { - Source: "tp1_group3", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - }, - interval: 10 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - { - Source: "tp1_group3", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - }, - }, - }, - { - title: "Multiple TPs initials and new groups", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - interval: 10 * time.Millisecond, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group3", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group4", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - }, - interval: 500 * time.Millisecond, - }, - }, - "tp2": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "5"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "6"}}, - }, - }, - interval: 100 * time.Millisecond, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp2_group3", - Targets: []model.LabelSet{{"__instance__": "7"}}, - }, - { - Source: "tp2_group4", - Targets: []model.LabelSet{{"__instance__": "8"}}, - }, - }, - interval: 10 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "5"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "6"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "5"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "6"}}, - }, - { - Source: "tp2_group3", - Targets: []model.LabelSet{{"__instance__": "7"}}, - }, - { - Source: "tp2_group4", - Targets: []model.LabelSet{{"__instance__": "8"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - { - Source: "tp1_group3", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group4", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "5"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "6"}}, - }, - { - Source: "tp2_group3", - Targets: []model.LabelSet{{"__instance__": "7"}}, - }, - { - Source: "tp2_group4", - Targets: []model.LabelSet{{"__instance__": "8"}}, - }, - }, - }, - }, - { - title: "One TP initials arrive after other TP updates.", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - interval: 10 * time.Millisecond, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - }, - interval: 150 * time.Millisecond, - }, - }, - "tp2": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "5"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "6"}}, - }, - }, - interval: 200 * time.Millisecond, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "7"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "8"}}, - }, - }, - interval: 100 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "5"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "6"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - { - Source: "tp2_group1", - Targets: []model.LabelSet{{"__instance__": "7"}}, - }, - { - Source: "tp2_group2", - Targets: []model.LabelSet{{"__instance__": "8"}}, - }, - }, - }, - }, - - { - title: "Single TP empty update in between", - updates: map[string][]update{ - "tp1": { - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - interval: 30 * time.Millisecond, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{}, - }, - }, - interval: 10 * time.Millisecond, - }, - { - targetGroups: []targetgroup.Group{ - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - }, - interval: 300 * time.Millisecond, - }, - }, - }, - expectedTargets: [][]*targetgroup.Group{ - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{}, - }, - }, - { - { - Source: "tp1_group1", - Targets: []model.LabelSet{{"__instance__": "3"}}, - }, - { - Source: "tp1_group2", - Targets: []model.LabelSet{{"__instance__": "4"}}, - }, - }, - }, - }, - } - - for i, tc := range testCases { - tc := tc - t.Run(tc.title, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) - require.NotNil(t, discoveryManager) - discoveryManager.updatert = 100 * time.Millisecond - - var totalUpdatesCount int - for _, up := range tc.updates { - if len(up) > 0 { - totalUpdatesCount += len(up) - } - } - provUpdates := make(chan []*targetgroup.Group, totalUpdatesCount) - - for _, up := range tc.updates { - go newMockDiscoveryProvider(up...).Run(ctx, provUpdates) - } - - for x := 0; x < totalUpdatesCount; x++ { - select { - case <-ctx.Done(): - t.Fatalf("%d: no update arrived within the timeout limit", x) - case tgs := <-provUpdates: - discoveryManager.updateGroup(poolKey{setName: strconv.Itoa(i), provider: tc.title}, tgs) - for _, got := range discoveryManager.allGroups() { - assertEqualGroups(t, got, tc.expectedTargets[x]) - } - } - } - }) - } -} - -func assertEqualGroups(t *testing.T, got, expected []*targetgroup.Group) { - t.Helper() - - // Need to sort by the groups's source as the received order is not guaranteed. - sort.Sort(byGroupSource(got)) - sort.Sort(byGroupSource(expected)) - - require.Equal(t, expected, got) -} - -func staticConfig(addrs ...string) discovery.StaticConfig { - var cfg discovery.StaticConfig - for i, addr := range addrs { - cfg = append(cfg, &targetgroup.Group{ - Source: strconv.Itoa(i), - Targets: []model.LabelSet{ - {model.AddressLabel: model.LabelValue(addr)}, - }, - }) - } - return cfg -} - -func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) { - t.Helper() - if _, ok := tSets[poolKey]; !ok { - t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets) - } - - match := false - var mergedTargets string - for _, targetGroup := range tSets[poolKey] { - for _, l := range targetGroup.Targets { - mergedTargets = mergedTargets + " " + l.String() - if l.String() == label { - match = true - } - } - } - if match != present { - msg := "" - if !present { - msg = "not" - } - t.Fatalf("%q should %s be present in Targets labels: %q", label, msg, mergedTargets) - } -} - -func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) - require.NotNil(t, discoveryManager) - discoveryManager.updatert = 100 * time.Millisecond - go discoveryManager.Run() - - c := map[string]discovery.Configs{ - "prometheus": { - staticConfig("foo:9090", "bar:9090"), - }, - } - discoveryManager.ApplyConfig(c) - - <-discoveryManager.SyncCh() - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true) - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true) - - c["prometheus"] = discovery.Configs{ - staticConfig("foo:9090"), - } - discoveryManager.ApplyConfig(c) - - <-discoveryManager.SyncCh() - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true) - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", false) -} - -func TestDiscovererConfigs(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) - require.NotNil(t, discoveryManager) - discoveryManager.updatert = 100 * time.Millisecond - go discoveryManager.Run() - - c := map[string]discovery.Configs{ - "prometheus": { - staticConfig("foo:9090", "bar:9090"), - staticConfig("baz:9090"), - }, - } - discoveryManager.ApplyConfig(c) - - <-discoveryManager.SyncCh() - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true) - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true) - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/1"}, "{__address__=\"baz:9090\"}", true) -} - -// TestTargetSetRecreatesEmptyStaticConfigs ensures that reloading a config file after -// removing all targets from the static_configs sends an update with empty targetGroups. -// This is required to signal the receiver that this target set has no current targets. -func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) - require.NotNil(t, discoveryManager) - discoveryManager.updatert = 100 * time.Millisecond - go discoveryManager.Run() - - c := map[string]discovery.Configs{ - "prometheus": { - staticConfig("foo:9090"), - }, - } - discoveryManager.ApplyConfig(c) - - <-discoveryManager.SyncCh() - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true) - - c["prometheus"] = discovery.Configs{ - discovery.StaticConfig{{}}, - } - discoveryManager.ApplyConfig(c) - - <-discoveryManager.SyncCh() - - pkey := poolKey{setName: "prometheus", provider: "static/0"} - targetGroups, ok := discoveryManager.targets[pkey] - if !ok { - t.Fatalf("'%v' should be present in target groups", pkey) - } - group, ok := targetGroups[""] - if !ok { - t.Fatalf("missing '' key in target groups %v", targetGroups) - } - - if len(group.Targets) != 0 { - t.Fatalf("Invalid number of targets: expected 0, got %d", len(group.Targets)) - } -} - -func TestIdenticalConfigurationsAreCoalesced(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - discoveryManager := NewManager(ctx, nil, reg, sdMetrics) - require.NotNil(t, discoveryManager) - discoveryManager.updatert = 100 * time.Millisecond - go discoveryManager.Run() - - c := map[string]discovery.Configs{ - "prometheus": { - staticConfig("foo:9090"), - }, - "prometheus2": { - staticConfig("foo:9090"), - }, - } - discoveryManager.ApplyConfig(c) - - <-discoveryManager.SyncCh() - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true) - verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus2", provider: "static/0"}, "{__address__=\"foo:9090\"}", true) - if len(discoveryManager.providers) != 1 { - t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers)) - } -} - -func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) { - originalConfig := discovery.Configs{ - staticConfig("foo:9090", "bar:9090", "baz:9090"), - } - processedConfig := discovery.Configs{ - staticConfig("foo:9090", "bar:9090", "baz:9090"), - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) - require.NotNil(t, discoveryManager) - discoveryManager.updatert = 100 * time.Millisecond - go discoveryManager.Run() - - cfgs := map[string]discovery.Configs{ - "prometheus": processedConfig, - } - discoveryManager.ApplyConfig(cfgs) - <-discoveryManager.SyncCh() - - for _, cfg := range cfgs { - require.Equal(t, originalConfig, cfg) - } -} - -type errorConfig struct{ err error } - -func (e errorConfig) Name() string { return "error" } -func (e errorConfig) NewDiscoverer(discovery.DiscovererOptions) (discovery.Discoverer, error) { - return nil, e.err -} - -// NewDiscovererMetrics implements discovery.Config. -func (errorConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { - return &discovery.NoopDiscovererMetrics{} -} - -func TestGaugeFailedConfigs(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) - require.NotNil(t, discoveryManager) - discoveryManager.updatert = 100 * time.Millisecond - go discoveryManager.Run() - - c := map[string]discovery.Configs{ - "prometheus": { - errorConfig{fmt.Errorf("tests error 0")}, - errorConfig{fmt.Errorf("tests error 1")}, - errorConfig{fmt.Errorf("tests error 2")}, - }, - } - discoveryManager.ApplyConfig(c) - <-discoveryManager.SyncCh() - - failedCount := client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs) - if failedCount != 3 { - t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount) - } - - c["prometheus"] = discovery.Configs{ - staticConfig("foo:9090"), - } - discoveryManager.ApplyConfig(c) - <-discoveryManager.SyncCh() - - failedCount = client_testutil.ToFloat64(discoveryManager.metrics.FailedConfigs) - if failedCount != 0 { - t.Fatalf("Expected to get no failed config, got: %v", failedCount) - } -} - -func TestCoordinationWithReceiver(t *testing.T) { - updateDelay := 100 * time.Millisecond - - type expect struct { - delay time.Duration - tgs map[string][]*targetgroup.Group - } - - testCases := []struct { - title string - providers map[string]discovery.Discoverer - expected []expect - }{ - { - title: "Receiver should get all updates even when one provider closes its channel", - providers: map[string]discovery.Discoverer{ - "once1": &onceProvider{ - tgs: []*targetgroup.Group{ - { - Source: "tg1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - }, - }, - "mock1": newMockDiscoveryProvider( - update{ - interval: 2 * updateDelay, - targetGroups: []targetgroup.Group{ - { - Source: "tg2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - }, - ), - }, - expected: []expect{ - { - tgs: map[string][]*targetgroup.Group{ - "once1": { - { - Source: "tg1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - }, - }, - }, - { - tgs: map[string][]*targetgroup.Group{ - "once1": { - { - Source: "tg1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - }, - "mock1": { - { - Source: "tg2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - }, - }, - }, - }, - { - title: "Receiver should get all updates even when the channel is blocked", - providers: map[string]discovery.Discoverer{ - "mock1": newMockDiscoveryProvider( - update{ - targetGroups: []targetgroup.Group{ - { - Source: "tg1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - }, - }, - update{ - interval: 4 * updateDelay, - targetGroups: []targetgroup.Group{ - { - Source: "tg2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - }, - ), - }, - expected: []expect{ - { - delay: 2 * updateDelay, - tgs: map[string][]*targetgroup.Group{ - "mock1": { - { - Source: "tg1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - }, - }, - }, - { - delay: 4 * updateDelay, - tgs: map[string][]*targetgroup.Group{ - "mock1": { - { - Source: "tg1", - Targets: []model.LabelSet{{"__instance__": "1"}}, - }, - { - Source: "tg2", - Targets: []model.LabelSet{{"__instance__": "2"}}, - }, - }, - }, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.title, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - reg := prometheus.NewRegistry() - _, sdMetrics := newTestMetrics(t, reg) - - mgr := NewManager(ctx, nil, reg, sdMetrics) - require.NotNil(t, mgr) - mgr.updatert = updateDelay - go mgr.Run() - - for name, p := range tc.providers { - mgr.StartCustomProvider(ctx, name, p) - } - - for i, expected := range tc.expected { - time.Sleep(expected.delay) - select { - case <-ctx.Done(): - t.Fatalf("step %d: no update received in the expected timeframe", i) - case tgs, ok := <-mgr.SyncCh(): - if !ok { - t.Fatalf("step %d: discovery manager channel is closed", i) - } - if len(tgs) != len(expected.tgs) { - t.Fatalf("step %d: target groups mismatch, got: %d, expected: %d\ngot: %#v\nexpected: %#v", - i, len(tgs), len(expected.tgs), tgs, expected.tgs) - } - for k := range expected.tgs { - if _, ok := tgs[k]; !ok { - t.Fatalf("step %d: target group not found: %s\ngot: %#v", i, k, tgs) - } - assertEqualGroups(t, tgs[k], expected.tgs[k]) - } - } - } - }) - } -} - -type update struct { - targetGroups []targetgroup.Group - interval time.Duration -} - -type mockdiscoveryProvider struct { - updates []update -} - -func newMockDiscoveryProvider(updates ...update) mockdiscoveryProvider { - tp := mockdiscoveryProvider{ - updates: updates, - } - return tp -} - -func (tp mockdiscoveryProvider) Run(ctx context.Context, upCh chan<- []*targetgroup.Group) { - for _, u := range tp.updates { - if u.interval > 0 { - select { - case <-ctx.Done(): - return - case <-time.After(u.interval): - } - } - tgs := make([]*targetgroup.Group, len(u.targetGroups)) - for i := range u.targetGroups { - tgs[i] = &u.targetGroups[i] - } - upCh <- tgs - } - <-ctx.Done() -} - -// byGroupSource implements sort.Interface so we can sort by the Source field. -type byGroupSource []*targetgroup.Group - -func (a byGroupSource) Len() int { return len(a) } -func (a byGroupSource) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byGroupSource) Less(i, j int) bool { return a[i].Source < a[j].Source } - -// onceProvider sends updates once (if any) and closes the update channel. -type onceProvider struct { - tgs []*targetgroup.Group -} - -func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) { - if len(o.tgs) > 0 { - ch <- o.tgs - } - close(ch) -} diff --git a/discovery/legacymanager/registry.go b/discovery/legacymanager/registry.go deleted file mode 100644 index 955705394d9..00000000000 --- a/discovery/legacymanager/registry.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2020 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacymanager - -import ( - "errors" - "fmt" - "reflect" - "sort" - "strconv" - "strings" - "sync" - - "gopkg.in/yaml.v2" - - "github.com/prometheus/prometheus/discovery" - "github.com/prometheus/prometheus/discovery/targetgroup" -) - -const ( - configFieldPrefix = "AUTO_DISCOVERY_" - staticConfigsKey = "static_configs" - staticConfigsFieldName = configFieldPrefix + staticConfigsKey -) - -var ( - configNames = make(map[string]discovery.Config) - configFieldNames = make(map[reflect.Type]string) - configFields []reflect.StructField - - configTypesMu sync.Mutex - configTypes = make(map[reflect.Type]reflect.Type) - - emptyStructType = reflect.TypeOf(struct{}{}) - configsType = reflect.TypeOf(discovery.Configs{}) -) - -// RegisterConfig registers the given Config type for YAML marshaling and unmarshaling. -func RegisterConfig(config discovery.Config) { - registerConfig(config.Name()+"_sd_configs", reflect.TypeOf(config), config) -} - -func init() { - // N.B.: static_configs is the only Config type implemented by default. - // All other types are registered at init by their implementing packages. - elemTyp := reflect.TypeOf(&targetgroup.Group{}) - registerConfig(staticConfigsKey, elemTyp, discovery.StaticConfig{}) -} - -func registerConfig(yamlKey string, elemType reflect.Type, config discovery.Config) { - name := config.Name() - if _, ok := configNames[name]; ok { - panic(fmt.Sprintf("discovery: Config named %q is already registered", name)) - } - configNames[name] = config - - fieldName := configFieldPrefix + yamlKey // Field must be exported. - configFieldNames[elemType] = fieldName - - // Insert fields in sorted order. - i := sort.Search(len(configFields), func(k int) bool { - return fieldName < configFields[k].Name - }) - configFields = append(configFields, reflect.StructField{}) // Add empty field at end. - copy(configFields[i+1:], configFields[i:]) // Shift fields to the right. - configFields[i] = reflect.StructField{ // Write new field in place. - Name: fieldName, - Type: reflect.SliceOf(elemType), - Tag: reflect.StructTag(`yaml:"` + yamlKey + `,omitempty"`), - } -} - -func getConfigType(out reflect.Type) reflect.Type { - configTypesMu.Lock() - defer configTypesMu.Unlock() - if typ, ok := configTypes[out]; ok { - return typ - } - // Initial exported fields map one-to-one. - var fields []reflect.StructField - for i, n := 0, out.NumField(); i < n; i++ { - switch field := out.Field(i); { - case field.PkgPath == "" && field.Type != configsType: - fields = append(fields, field) - default: - fields = append(fields, reflect.StructField{ - Name: "_" + field.Name, // Field must be unexported. - PkgPath: out.PkgPath(), - Type: emptyStructType, - }) - } - } - // Append extra config fields on the end. - fields = append(fields, configFields...) - typ := reflect.StructOf(fields) - configTypes[out] = typ - return typ -} - -// UnmarshalYAMLWithInlineConfigs helps implement yaml.Unmarshal for structs -// that have a Configs field that should be inlined. -func UnmarshalYAMLWithInlineConfigs(out interface{}, unmarshal func(interface{}) error) error { - outVal := reflect.ValueOf(out) - if outVal.Kind() != reflect.Ptr { - return fmt.Errorf("discovery: can only unmarshal into a struct pointer: %T", out) - } - outVal = outVal.Elem() - if outVal.Kind() != reflect.Struct { - return fmt.Errorf("discovery: can only unmarshal into a struct pointer: %T", out) - } - outTyp := outVal.Type() - - cfgTyp := getConfigType(outTyp) - cfgPtr := reflect.New(cfgTyp) - cfgVal := cfgPtr.Elem() - - // Copy shared fields (defaults) to dynamic value. - var configs *discovery.Configs - for i, n := 0, outVal.NumField(); i < n; i++ { - if outTyp.Field(i).Type == configsType { - configs = outVal.Field(i).Addr().Interface().(*discovery.Configs) - continue - } - if cfgTyp.Field(i).PkgPath != "" { - continue // Field is unexported: ignore. - } - cfgVal.Field(i).Set(outVal.Field(i)) - } - if configs == nil { - return fmt.Errorf("discovery: Configs field not found in type: %T", out) - } - - // Unmarshal into dynamic value. - if err := unmarshal(cfgPtr.Interface()); err != nil { - return replaceYAMLTypeError(err, cfgTyp, outTyp) - } - - // Copy shared fields from dynamic value. - for i, n := 0, outVal.NumField(); i < n; i++ { - if cfgTyp.Field(i).PkgPath != "" { - continue // Field is unexported: ignore. - } - outVal.Field(i).Set(cfgVal.Field(i)) - } - - var err error - *configs, err = readConfigs(cfgVal, outVal.NumField()) - return err -} - -func readConfigs(structVal reflect.Value, startField int) (discovery.Configs, error) { - var ( - configs discovery.Configs - targets []*targetgroup.Group - ) - for i, n := startField, structVal.NumField(); i < n; i++ { - field := structVal.Field(i) - if field.Kind() != reflect.Slice { - panic("discovery: internal error: field is not a slice") - } - for k := 0; k < field.Len(); k++ { - val := field.Index(k) - if val.IsZero() || (val.Kind() == reflect.Ptr && val.Elem().IsZero()) { - key := configFieldNames[field.Type().Elem()] - key = strings.TrimPrefix(key, configFieldPrefix) - return nil, fmt.Errorf("empty or null section in %s", key) - } - switch c := val.Interface().(type) { - case *targetgroup.Group: - // Add index to the static config target groups for unique identification - // within scrape pool. - c.Source = strconv.Itoa(len(targets)) - // Coalesce multiple static configs into a single static config. - targets = append(targets, c) - case discovery.Config: - configs = append(configs, c) - default: - panic("discovery: internal error: slice element is not a Config") - } - } - } - if len(targets) > 0 { - configs = append(configs, discovery.StaticConfig(targets)) - } - return configs, nil -} - -// MarshalYAMLWithInlineConfigs helps implement yaml.Marshal for structs -// that have a Configs field that should be inlined. -func MarshalYAMLWithInlineConfigs(in interface{}) (interface{}, error) { - inVal := reflect.ValueOf(in) - for inVal.Kind() == reflect.Ptr { - inVal = inVal.Elem() - } - inTyp := inVal.Type() - - cfgTyp := getConfigType(inTyp) - cfgPtr := reflect.New(cfgTyp) - cfgVal := cfgPtr.Elem() - - // Copy shared fields to dynamic value. - var configs *discovery.Configs - for i, n := 0, inTyp.NumField(); i < n; i++ { - if inTyp.Field(i).Type == configsType { - configs = inVal.Field(i).Addr().Interface().(*discovery.Configs) - } - if cfgTyp.Field(i).PkgPath != "" { - continue // Field is unexported: ignore. - } - cfgVal.Field(i).Set(inVal.Field(i)) - } - if configs == nil { - return nil, fmt.Errorf("discovery: Configs field not found in type: %T", in) - } - - if err := writeConfigs(cfgVal, *configs); err != nil { - return nil, err - } - - return cfgPtr.Interface(), nil -} - -func writeConfigs(structVal reflect.Value, configs discovery.Configs) error { - targets := structVal.FieldByName(staticConfigsFieldName).Addr().Interface().(*[]*targetgroup.Group) - for _, c := range configs { - if sc, ok := c.(discovery.StaticConfig); ok { - *targets = append(*targets, sc...) - continue - } - fieldName, ok := configFieldNames[reflect.TypeOf(c)] - if !ok { - return fmt.Errorf("discovery: cannot marshal unregistered Config type: %T", c) - } - field := structVal.FieldByName(fieldName) - field.Set(reflect.Append(field, reflect.ValueOf(c))) - } - return nil -} - -func replaceYAMLTypeError(err error, oldTyp, newTyp reflect.Type) error { - var e *yaml.TypeError - if errors.As(err, &e) { - oldStr := oldTyp.String() - newStr := newTyp.String() - for i, s := range e.Errors { - e.Errors[i] = strings.ReplaceAll(s, oldStr, newStr) - } - } - return err -} diff --git a/discovery/linode/linode.go b/discovery/linode/linode.go index 634a6b1d4bb..a5f05600c17 100644 --- a/discovery/linode/linode.go +++ b/discovery/linode/linode.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,7 +23,6 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/linode/linodego" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -103,7 +102,7 @@ func (*SDConfig) Name() string { return "linode" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -112,7 +111,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -138,10 +137,10 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*linodeMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*linodeMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } d := &Discovery{ @@ -165,13 +164,14 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere Timeout: time.Duration(conf.RefreshInterval), }, ) - client.SetUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)) + client.SetUserAgent(version.PrometheusUserAgent()) d.client = &client d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "linode", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -194,13 +194,12 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { events, err := d.client.ListEvents(ctx, &eventsOpts) if err != nil { var e *linodego.Error - if errors.As(err, &e) && e.Code == http.StatusUnauthorized { - // If we get a 401, the token doesn't have `events:read_only` scope. - // Disable event polling and fallback to doing a full refresh every interval. - d.eventPollingEnabled = false - } else { + if !errors.As(err, &e) || e.Code != http.StatusUnauthorized { return nil, err } + // If we get a 401, the token doesn't have `events:read_only` scope. + // Disable event polling and fallback to doing a full refresh every interval. + d.eventPollingEnabled = false } else { // Event polling tells us changes the Linode API is aware of. Actions issued outside of the Linode API, // such as issuing a `shutdown` at the VM's console instead of using the API to power off an instance, diff --git a/discovery/linode/linode_test.go b/discovery/linode/linode_test.go index 3c106506534..d795d29698a 100644 --- a/discovery/linode/linode_test.go +++ b/discovery/linode/linode_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,10 +19,10 @@ import ( "net/url" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/discovery" @@ -238,7 +238,11 @@ func TestLinodeSDRefresh(t *testing.T) { defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "linode", + }) require.NoError(t, err) endpoint, err := url.Parse(sdmock.Endpoint()) require.NoError(t, err) diff --git a/discovery/linode/metrics.go b/discovery/linode/metrics.go index 8f813892262..5bc805a60e5 100644 --- a/discovery/linode/metrics.go +++ b/discovery/linode/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/linode/mock_test.go b/discovery/linode/mock_test.go index 50f0572ecdc..b8094ec2117 100644 --- a/discovery/linode/mock_test.go +++ b/discovery/linode/mock_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/manager.go b/discovery/manager.go index cefa90a8669..fa52e164f72 100644 --- a/discovery/manager.go +++ b/discovery/manager.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,16 +16,19 @@ package discovery import ( "context" "fmt" + "log/slog" + "maps" "reflect" "sync" "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" + "github.com/cenkalti/backoff/v5" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/features" ) type poolKey struct { @@ -37,7 +40,7 @@ type poolKey struct { type Provider struct { name string d Discoverer - config interface{} + config any cancel context.CancelFunc // done should be called after cleaning up resources associated with cancelled provider. @@ -57,33 +60,41 @@ func (p *Provider) Discoverer() Discoverer { // IsStarted return true if Discoverer is started. func (p *Provider) IsStarted() bool { + p.mu.RLock() + defer p.mu.RUnlock() return p.cancel != nil } -func (p *Provider) Config() interface{} { +func (p *Provider) Config() any { return p.config } // CreateAndRegisterSDMetrics registers the metrics needed for SD mechanisms. // Does not register the metrics for the Discovery Manager. -// TODO(ptodev): Add ability to unregister the metrics? -func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (map[string]DiscovererMetrics, error) { +func CreateAndRegisterSDMetrics(reg prometheus.Registerer) (*SDMetrics, error) { // Some SD mechanisms use the "refresh" package, which has its own metrics. refreshSdMetrics := NewRefreshMetrics(reg) // Register the metrics specific for each SD mechanism, and the ones for the refresh package. - sdMetrics, err := RegisterSDMetrics(reg, refreshSdMetrics) + mechanismMetrics, err := RegisterSDMetrics(reg, refreshSdMetrics) if err != nil { return nil, fmt.Errorf("failed to register service discovery metrics: %w", err) } - return sdMetrics, nil + return &SDMetrics{ + MechanismMetrics: mechanismMetrics, + RefreshManager: refreshSdMetrics, + }, nil } // NewManager is the Discovery Manager constructor. -func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Registerer, sdMetrics map[string]DiscovererMetrics, options ...func(*Manager)) *Manager { +func NewManager(ctx context.Context, logger *slog.Logger, registerer prometheus.Registerer, sdMetrics *SDMetrics, options ...func(*Manager)) *Manager { if logger == nil { - logger = log.NewNopLogger() + logger = promslog.NewNopLogger() + } + if sdMetrics == nil || sdMetrics.RefreshManager == nil { + logger.Error("Failed to create discovery manager: sdMetrics.RefreshManager must be set") + return nil } mgr := &Manager{ logger: logger, @@ -91,7 +102,7 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re targets: make(map[poolKey]map[string]*targetgroup.Group), ctx: ctx, updatert: 5 * time.Second, - triggerSend: make(chan struct{}, 1), + triggerSend: make(chan struct{}, 1), // At least one element to ensure we can do a delayed read. registerer: registerer, sdMetrics: sdMetrics, } @@ -101,12 +112,19 @@ func NewManager(ctx context.Context, logger log.Logger, registerer prometheus.Re // Register the metrics. // We have to do this after setting all options, so that the name of the Manager is set. - if metrics, err := NewManagerMetrics(registerer, mgr.name); err == nil { - mgr.metrics = metrics - } else { - level.Error(logger).Log("msg", "Failed to create discovery manager metrics", "manager", mgr.name, "err", err) + metrics, err := NewManagerMetrics(registerer, mgr.name) + if err != nil { + logger.Error("Failed to create discovery manager metrics", "manager", mgr.name, "err", err) return nil } + mgr.metrics = metrics + + // Register all available service discovery providers with the feature registry. + if mgr.featureRegistry != nil { + for _, sdName := range RegisteredConfigNames() { + mgr.featureRegistry.Enable(features.ServiceDiscoveryProviders, sdName) + } + } return mgr } @@ -138,10 +156,19 @@ func HTTPClientOptions(opts ...config.HTTPClientOption) func(*Manager) { } } +// FeatureRegistry sets the feature registry for the manager. +func FeatureRegistry(fr features.Collector) func(*Manager) { + return func(m *Manager) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.featureRegistry = fr + } +} + // Manager maintains a set of discovery providers and sends each update to a map channel. // Targets are grouped by the target set name. type Manager struct { - logger log.Logger + logger *slog.Logger name string httpOpts []config.HTTPClientOption mtx sync.RWMutex @@ -171,7 +198,10 @@ type Manager struct { registerer prometheus.Registerer metrics *Metrics - sdMetrics map[string]DiscovererMetrics + sdMetrics *SDMetrics + + // featureRegistry is used to track which service discovery providers are configured. + featureRegistry features.Collector } // Providers returns the currently configured SD providers. @@ -216,15 +246,35 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error { newProviders []*Provider ) for _, prov := range m.providers { - // Cancel obsolete providers. - if len(prov.newSubs) == 0 { + // Cancel obsolete providers if it has no new subs and it has a cancel function. + // prov.cancel != nil is the same check as we use in IsStarted() method but we don't call IsStarted + // here because it would take a lock and we need the same lock ourselves for other reads. + prov.mu.RLock() + if len(prov.newSubs) == 0 && prov.cancel != nil { wg.Add(1) prov.done = func() { wg.Done() } + prov.cancel() + prov.mu.RUnlock() + + // Clear up refresh metrics associated with this cancelled provider (sub means scrape job name). + m.targetsMtx.Lock() + for s := range prov.subs { + // Also clean up discovered targets metric. targetsMtx lock needed for safe access to m.targets. + delete(m.targets, poolKey{s, prov.name}) + m.metrics.DiscoveredTargets.DeleteLabelValues(s) + + if cfg, ok := prov.config.(Config); ok { + m.sdMetrics.RefreshManager.DeleteLabelValues(cfg.Name(), s) + } + } + m.targetsMtx.Unlock() continue } + prov.mu.RUnlock() + newProviders = append(newProviders, prov) // refTargets keeps reference targets used to populate new subs' targets as they should be the same. var refTargets map[string]*targetgroup.Group @@ -236,7 +286,13 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error { // Remove obsolete subs' targets. if _, ok := prov.newSubs[s]; !ok { delete(m.targets, poolKey{s, prov.name}) - m.metrics.DiscoveredTargets.DeleteLabelValues(m.name, s) + m.metrics.DiscoveredTargets.DeleteLabelValues(s) + + // Also clean up refresh metrics for subs that are being removed from a provider that is still running. + cfg, ok := prov.config.(Config) + if ok { + m.sdMetrics.RefreshManager.DeleteLabelValues(cfg.Name(), s) + } } } // Set metrics and targets for new subs. @@ -246,9 +302,7 @@ func (m *Manager) ApplyConfig(cfg map[string]Configs) error { } if l := len(refTargets); l > 0 { m.targets[poolKey{s, prov.name}] = make(map[string]*targetgroup.Group, l) - for k, v := range refTargets { - m.targets[poolKey{s, prov.name}][k] = v - } + maps.Copy(m.targets[poolKey{s, prov.name}], refTargets) } } m.targetsMtx.Unlock() @@ -294,11 +348,13 @@ func (m *Manager) StartCustomProvider(ctx context.Context, name string, worker D } func (m *Manager) startProvider(ctx context.Context, p *Provider) { - level.Debug(m.logger).Log("msg", "Starting provider", "provider", p.name, "subs", fmt.Sprintf("%v", p.subs)) + m.logger.Debug("Starting provider", "provider", p.name, "subs", fmt.Sprintf("%v", p.subs)) ctx, cancel := context.WithCancel(ctx) updates := make(chan []*targetgroup.Group) + p.mu.Lock() p.cancel = cancel + p.mu.Unlock() go p.d.Run(ctx, updates) go m.updater(ctx, p, updates) @@ -306,16 +362,20 @@ func (m *Manager) startProvider(ctx context.Context, p *Provider) { // cleaner cleans resources associated with provider. func (m *Manager) cleaner(p *Provider) { + p.mu.Lock() + defer p.mu.Unlock() + m.targetsMtx.Lock() - p.mu.RLock() for s := range p.subs { delete(m.targets, poolKey{s, p.name}) } - p.mu.RUnlock() m.targetsMtx.Unlock() if p.done != nil { p.done() } + + // Provider was cleaned so mark is as down. + p.cancel = nil } func (m *Manager) updater(ctx context.Context, p *Provider, updates chan []*targetgroup.Group) { @@ -328,7 +388,7 @@ func (m *Manager) updater(ctx context.Context, p *Provider, updates chan []*targ case tgs, ok := <-updates: m.metrics.ReceivedUpdates.Inc() if !ok { - level.Debug(m.logger).Log("msg", "Discoverer channel closed", "provider", p.name) + m.logger.Debug("Discoverer channel closed", "provider", p.name) // Wait for provider cancellation to ensure targets are cleaned up when expected. <-ctx.Done() return @@ -349,22 +409,34 @@ func (m *Manager) updater(ctx context.Context, p *Provider, updates chan []*targ } func (m *Manager) sender() { - ticker := time.NewTicker(m.updatert) - defer ticker.Stop() + defer func() { + close(m.syncCh) + }() + // Some discoverers send updates too often, so we throttle these with a backoff interval that + // increases the interval up to m.updatert delay. + lastSent := time.Now().Add(-1 * m.updatert) + b := &backoff.ExponentialBackOff{ + InitialInterval: 100 * time.Millisecond, + RandomizationFactor: backoff.DefaultRandomizationFactor, + Multiplier: backoff.DefaultMultiplier, + MaxInterval: m.updatert, + } for { select { case <-m.ctx.Done(): return - case <-ticker.C: // Some discoverers send updates too often, so we throttle these with the ticker. + case <-time.After(b.NextBackOff()): select { case <-m.triggerSend: m.metrics.SentUpdates.Inc() select { case m.syncCh <- m.allGroups(): + lastSent = time.Now() default: m.metrics.DelayedUpdates.Inc() - level.Debug(m.logger).Log("msg", "Discovery receiver's channel was full so will retry the next cycle") + m.logger.Debug("Discovery receiver's channel was full so will retry the next cycle") + // Ensure we don't miss this update. select { case m.triggerSend <- struct{}{}: default: @@ -372,6 +444,9 @@ func (m *Manager) sender() { } default: } + if time.Since(lastSent) > m.updatert { + b.Reset() // Nothing happened for a while, start again from low interval for prompt updates. + } } } } @@ -380,9 +455,11 @@ func (m *Manager) cancelDiscoverers() { m.mtx.RLock() defer m.mtx.RUnlock() for _, p := range m.providers { + p.mu.RLock() if p.cancel != nil { p.cancel() } + p.mu.RUnlock() } } @@ -413,9 +490,9 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group { n := map[string]int{} m.mtx.RLock() - m.targetsMtx.Lock() for _, p := range m.providers { p.mu.RLock() + m.targetsMtx.Lock() for s := range p.subs { // Send empty lists for subs without any targets to make sure old stale targets are dropped by consumers. // See: https://github.com/prometheus/prometheus/issues/12858 for details. @@ -430,13 +507,14 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group { } } } + m.targetsMtx.Unlock() p.mu.RUnlock() } - m.targetsMtx.Unlock() m.mtx.RUnlock() for setName, v := range n { m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v)) + m.metrics.LastUpdated.WithLabelValues(setName).SetToCurrentTime() } return tSets @@ -458,12 +536,13 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int { } typ := cfg.Name() d, err := cfg.NewDiscoverer(DiscovererOptions{ - Logger: log.With(m.logger, "discovery", typ, "config", setName), + Logger: m.logger.With("discovery", typ, "config", setName), HTTPClientOptions: m.httpOpts, - Metrics: m.sdMetrics[typ], + Metrics: m.sdMetrics.MechanismMetrics[typ], + SetName: setName, }) if err != nil { - level.Error(m.logger).Log("msg", "Cannot create service discovery", "err", err, "type", typ, "config", setName) + m.logger.Error("Cannot create service discovery", "err", err, "type", typ, "config", setName) failed++ return } @@ -491,19 +570,3 @@ func (m *Manager) registerProviders(cfgs Configs, setName string) int { } return failed } - -// StaticProvider holds a list of target groups that never change. -type StaticProvider struct { - TargetGroups []*targetgroup.Group -} - -// Run implements the Worker interface. -func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { - // We still have to consider that the consumer exits right away in which case - // the context will be canceled. - select { - case ch <- sd.TargetGroups: - case <-ctx.Done(): - } - close(ch) -} diff --git a/discovery/manager_test.go b/discovery/manager_test.go index 831cefe514d..b98dc1dd359 100644 --- a/discovery/manager_test.go +++ b/discovery/manager_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,17 +15,19 @@ package discovery import ( "context" + "errors" "fmt" "sort" "strconv" "sync" "testing" + "testing/synctest" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" client_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -36,11 +38,14 @@ func TestMain(m *testing.M) { testutil.TolerantVerifyLeak(m) } -func NewTestMetrics(t *testing.T, reg prometheus.Registerer) (RefreshMetricsManager, map[string]DiscovererMetrics) { +func NewTestMetrics(t *testing.T, reg prometheus.Registerer) *SDMetrics { refreshMetrics := NewRefreshMetrics(reg) - sdMetrics, err := RegisterSDMetrics(reg, refreshMetrics) + mechanismMetrics, err := RegisterSDMetrics(reg, refreshMetrics) require.NoError(t, err) - return refreshMetrics, sdMetrics + return &SDMetrics{ + MechanismMetrics: mechanismMetrics, + RefreshManager: refreshMetrics, + } } // TestTargetUpdatesOrder checks that the target updates are received in the expected order. @@ -667,15 +672,14 @@ func TestTargetUpdatesOrder(t *testing.T) { } for i, tc := range testCases { - tc := tc t.Run(tc.title, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond @@ -694,7 +698,7 @@ func TestTargetUpdatesOrder(t *testing.T) { for x := 0; x < totalUpdatesCount; x++ { select { case <-ctx.Done(): - require.FailNow(t, "%d: no update arrived within the timeout limit", x) + t.Fatalf("%d: no update arrived within the timeout limit", x) case tgs := <-provUpdates: discoveryManager.updateGroup(poolKey{setName: strconv.Itoa(i), provider: tc.title}, tgs) for _, got := range discoveryManager.allGroups() { @@ -768,12 +772,10 @@ func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Grou } } } - if match != present { - msg := "" - if !present { - msg = "not" - } - require.FailNow(t, "%q should %s be present in Targets labels: %q", label, msg, mergedTargets) + if present { + require.Truef(t, match, "%q must be present in Targets labels: %q", label, mergedTargets) + } else { + require.Falsef(t, match, "%q must be absent in Targets labels: %q", label, mergedTargets) } } @@ -784,14 +786,91 @@ func pk(provider, setName string, n int) poolKey { } } +func TestTargetSetTargetGroupsPresentOnStartup(t *testing.T) { + testCases := []struct { + name string + updatert time.Duration + readTimeout time.Duration + expectedTargets int + }{ + { + name: "startup wait with long interval times out", + updatert: 100 * time.Hour, + readTimeout: 10 * time.Millisecond, + }, + { + name: "startup wait with short interval succeeds", + updatert: 10 * time.Millisecond, + readTimeout: 300 * time.Millisecond, + expectedTargets: 1, + }, + { + name: "skip startup wait", + updatert: 100 * time.Hour, + readTimeout: 300 * time.Millisecond, + expectedTargets: 1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx := t.Context() + + reg := prometheus.NewRegistry() + sdMetrics := NewTestMetrics(t, reg) + + opts := make([]func(*Manager), 0) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics, opts...) + require.NotNil(t, discoveryManager) + + discoveryManager.updatert = tc.updatert + go discoveryManager.Run() + + c := map[string]Configs{ + "prometheus": { + staticConfig("foo:9090"), + }, + } + discoveryManager.ApplyConfig(c) + + synctest.Wait() + + timeout := time.After(tc.readTimeout) + var lastSyncedTargets map[string][]*targetgroup.Group + testFor: + for { + select { + case <-timeout: + break testFor + case lastSyncedTargets = <-discoveryManager.SyncCh(): + } + } + + if tc.expectedTargets == 0 { + require.Nil(t, lastSyncedTargets) + return + } + + require.Len(t, lastSyncedTargets, 1) + require.Len(t, lastSyncedTargets["prometheus"], tc.expectedTargets) + verifySyncedPresence(t, lastSyncedTargets, "prometheus", "{__address__=\"foo:9090\"}", true) + + p := pk("static", "prometheus", 0) + verifyPresence(t, discoveryManager.targets, p, "{__address__=\"foo:9090\"}", true) + require.Len(t, discoveryManager.targets, 1) + }) + }) + } +} + func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -822,13 +901,12 @@ func TestTargetSetTargetGroupsPresentOnConfigReload(t *testing.T) { } func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -862,13 +940,12 @@ func TestTargetSetTargetGroupsPresentOnConfigRename(t *testing.T) { } func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -905,13 +982,12 @@ func TestTargetSetTargetGroupsPresentOnConfigDuplicateAndDeleteOriginal(t *testi } func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -973,13 +1049,12 @@ func TestTargetSetTargetGroupsPresentOnConfigChange(t *testing.T) { } func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1017,13 +1092,12 @@ func TestTargetSetRecreatesTargetGroupsOnConfigChange(t *testing.T) { } func TestDiscovererConfigs(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1054,13 +1128,12 @@ func TestDiscovererConfigs(t *testing.T) { // removing all targets from the static_configs cleans the corresponding targetGroups entries to avoid leaks and sends an empty update. // The update is required to signal the consumers that the previous targets should be dropped. func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1090,17 +1163,16 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) { targetGroups, ok := discoveryManager.targets[p] require.True(t, ok, "'%v' should be present in targets", p) // Otherwise the targetGroups will leak, see https://github.com/prometheus/prometheus/issues/12436. - require.Empty(t, targetGroups, 0, "'%v' should no longer have any associated target groups", p) + require.Empty(t, targetGroups, "'%v' should no longer have any associated target groups", p) require.Len(t, syncedTargets, 1, "an update with no targetGroups should still be sent.") - require.Empty(t, syncedTargets["prometheus"], 0) + require.Empty(t, syncedTargets["prometheus"]) } func TestIdenticalConfigurationsAreCoalesced(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) discoveryManager := NewManager(ctx, nil, reg, sdMetrics) require.NotNil(t, discoveryManager) @@ -1135,13 +1207,12 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) { processedConfig := Configs{ staticConfig("foo:9090", "bar:9090", "baz:9090"), } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1159,7 +1230,7 @@ func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) { type errorConfig struct{ err error } -func (e errorConfig) Name() string { return "error" } +func (errorConfig) Name() string { return "error" } func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err } // NewDiscovererMetrics implements discovery.Config. @@ -1177,7 +1248,7 @@ func (lockStaticConfig) NewDiscovererMetrics(prometheus.Registerer, RefreshMetri return &NoopDiscovererMetrics{} } -func (s lockStaticConfig) Name() string { return "lockstatic" } +func (lockStaticConfig) Name() string { return "lockstatic" } func (s lockStaticConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return (lockStaticDiscoverer)(s), nil } @@ -1196,22 +1267,21 @@ func (s lockStaticDiscoverer) Run(ctx context.Context, up chan<- []*targetgroup. } func TestGaugeFailedConfigs(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() c := map[string]Configs{ "prometheus": { - errorConfig{fmt.Errorf("tests error 0")}, - errorConfig{fmt.Errorf("tests error 1")}, - errorConfig{fmt.Errorf("tests error 2")}, + errorConfig{errors.New("tests error 0")}, + errorConfig{errors.New("tests error 1")}, + errorConfig{errors.New("tests error 2")}, }, } discoveryManager.ApplyConfig(c) @@ -1351,13 +1421,12 @@ func TestCoordinationWithReceiver(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.title, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) mgr := NewManager(ctx, nil, reg, sdMetrics) require.NotNil(t, mgr) @@ -1372,10 +1441,10 @@ func TestCoordinationWithReceiver(t *testing.T) { time.Sleep(expected.delay) select { case <-ctx.Done(): - require.FailNow(t, "step %d: no update received in the expected timeframe", i) + t.Fatalf("step %d: no update received in the expected timeframe", i) case tgs, ok := <-mgr.SyncCh(): require.True(t, ok, "step %d: discovery manager channel is closed", i) - require.Equal(t, len(expected.tgs), len(tgs), "step %d: targets mismatch", i) + require.Len(t, tgs, len(expected.tgs), "step %d: targets mismatch", i) for k := range expected.tgs { _, ok := tgs[k] @@ -1448,13 +1517,12 @@ func (o onceProvider) Run(_ context.Context, ch chan<- []*targetgroup.Group) { // TestTargetSetTargetGroupsUpdateDuringApplyConfig is used to detect races when // ApplyConfig happens at the same time as targets update. func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() reg := prometheus.NewRegistry() - _, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) require.NotNil(t, discoveryManager) discoveryManager.updatert = 100 * time.Millisecond go discoveryManager.Run() @@ -1472,7 +1540,7 @@ func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) { wg.Add(2000) start := make(chan struct{}) - for i := 0; i < 1000; i++ { + for range 1000 { go func() { <-start td.update([]*targetgroup.Group{ @@ -1486,7 +1554,7 @@ func TestTargetSetTargetGroupsUpdateDuringApplyConfig(t *testing.T) { }() } - for i := 0; i < 1000; i++ { + for i := range 1000 { go func(i int) { <-start c := map[string]Configs{ @@ -1522,7 +1590,7 @@ func (*testDiscoverer) NewDiscovererMetrics(prometheus.Registerer, RefreshMetric } // Name implements Config. -func (t *testDiscoverer) Name() string { +func (*testDiscoverer) Name() string { return "test" } @@ -1546,20 +1614,143 @@ func (t *testDiscoverer) update(tgs []*targetgroup.Group) { func TestUnregisterMetrics(t *testing.T) { reg := prometheus.NewRegistry() // Check that all metrics can be unregistered, allowing a second manager to be created. - for i := 0; i < 2; i++ { + for range 2 { ctx, cancel := context.WithCancel(context.Background()) - refreshMetrics, sdMetrics := NewTestMetrics(t, reg) + sdMetrics := NewTestMetrics(t, reg) - discoveryManager := NewManager(ctx, log.NewNopLogger(), reg, sdMetrics) + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) // discoveryManager will be nil if there was an error configuring metrics. require.NotNil(t, discoveryManager) // Unregister all metrics. discoveryManager.UnregisterMetrics() - for _, sdMetric := range sdMetrics { + for _, sdMetric := range sdMetrics.MechanismMetrics { sdMetric.Unregister() } - refreshMetrics.Unregister() + sdMetrics.RefreshManager.Unregister() cancel() } } + +// Refresh and discovery metrics should be deleted for providers that are removed. +func TestMetricsCleanupAfterConfigReload(t *testing.T) { + ctx := t.Context() + + reg := prometheus.NewRegistry() + sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) + require.NotNil(t, discoveryManager) + discoveryManager.updatert = 100 * time.Millisecond + go discoveryManager.Run() + + c := map[string]Configs{ + "prometheus": { + staticConfig("foo:9090", "bar:9090"), + }, + "other": { + staticConfig("baz:9090"), + }, + } + discoveryManager.ApplyConfig(c) + <-discoveryManager.SyncCh() + + // Manually instantiate refresh metrics to make them visible + sdMetrics.RefreshManager.Instantiate("static", "prometheus").Failures.Add(0) + sdMetrics.RefreshManager.Instantiate("static", "other").Failures.Add(0) + + count, err := client_testutil.GatherAndCount(reg, "prometheus_sd_discovered_targets") + require.NoError(t, err) + require.Equal(t, 2, count) + + count, err = client_testutil.GatherAndCount(reg, "prometheus_sd_refresh_failures_total") + require.NoError(t, err) + require.Equal(t, 2, count) + + // Simulate a config refresh. + delete(c, "prometheus") + discoveryManager.ApplyConfig(c) + <-discoveryManager.SyncCh() + + // Ensure we still have metrics for the remaining provider. + count, err = client_testutil.GatherAndCount(reg, "prometheus_sd_discovered_targets") + require.NoError(t, err) + require.Equal(t, 1, count) + + count, err = client_testutil.GatherAndCount(reg, "prometheus_sd_refresh_failures_total") + require.NoError(t, err) + require.Equal(t, 1, count) +} + +// Calling ApplyConfig() that removes providers at the same time as shutting down +// the manager should not hang. +func TestConfigReloadAndShutdownRace(t *testing.T) { + reg := prometheus.NewRegistry() + sdMetrics := NewTestMetrics(t, reg) + + mgrCtx, mgrCancel := context.WithCancel(context.Background()) + discoveryManager := NewManager(mgrCtx, promslog.NewNopLogger(), reg, sdMetrics) + require.NotNil(t, discoveryManager) + discoveryManager.updatert = 100 * time.Millisecond + + var wgDiscovery sync.WaitGroup + wgDiscovery.Go(func() { + discoveryManager.Run() + }) + time.Sleep(time.Millisecond * 200) + + var wgBg sync.WaitGroup + updateChan := discoveryManager.SyncCh() + wgBg.Add(1) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + defer wgBg.Done() + select { + case <-ctx.Done(): + return + case <-updateChan: + } + }() + + c := map[string]Configs{ + "prometheus": {staticConfig("bar:9090")}, + } + discoveryManager.ApplyConfig(c) + + delete(c, "prometheus") + wgBg.Go(func() { + discoveryManager.ApplyConfig(c) + }) + mgrCancel() + wgDiscovery.Wait() + + cancel() + wgBg.Wait() +} + +func TestGaugeLastUpdateTimestamp(t *testing.T) { + ctx := t.Context() + + reg := prometheus.NewRegistry() + sdMetrics := NewTestMetrics(t, reg) + + discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics) + require.NotNil(t, discoveryManager) + discoveryManager.updatert = 100 * time.Millisecond + go discoveryManager.Run() + + c := map[string]Configs{ + "prometheus": { + staticConfig("foo:9090"), + }, + } + discoveryManager.ApplyConfig(c) + + before := time.Now() + <-discoveryManager.SyncCh() + after := time.Now() + + ts := client_testutil.ToFloat64(discoveryManager.metrics.LastUpdated.WithLabelValues("prometheus")) + require.GreaterOrEqual(t, ts, float64(before.UnixNano())/1e9, "last update timestamp should be >= time before sync") + require.LessOrEqual(t, ts, float64(after.UnixNano())/1e9, "last update timestamp should be <= time after sync") +} diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go index 38b47accffb..ee88c1b96c2 100644 --- a/discovery/marathon/marathon.go +++ b/discovery/marathon/marathon.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -27,7 +27,6 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -80,7 +79,7 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &marathonMetrics{ refreshMetrics: rmi, } @@ -91,7 +90,7 @@ func (*SDConfig) Name() string { return "marathon" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(*c, opts.Logger, opts.Metrics) + return NewDiscovery(*c, opts) } // SetDirectory joins any relative file paths with dir. @@ -101,7 +100,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -111,15 +110,15 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if len(c.Servers) == 0 { return errors.New("marathon_sd: must contain at least one Marathon server") } - if len(c.AuthToken) > 0 && len(c.AuthTokenFile) > 0 { + if len(c.AuthToken) > 0 && c.AuthTokenFile != "" { return errors.New("marathon_sd: at most one of auth_token & auth_token_file must be configured") } - if len(c.AuthToken) > 0 || len(c.AuthTokenFile) > 0 { + if len(c.AuthToken) > 0 || c.AuthTokenFile != "" { switch { case c.HTTPClientConfig.BasicAuth != nil: return errors.New("marathon_sd: at most one of basic_auth, auth_token & auth_token_file must be configured") - case len(c.HTTPClientConfig.BearerToken) > 0 || len(c.HTTPClientConfig.BearerTokenFile) > 0: + case len(c.HTTPClientConfig.BearerToken) > 0 || c.HTTPClientConfig.BearerTokenFile != "": return errors.New("marathon_sd: at most one of bearer_token, bearer_token_file, auth_token & auth_token_file must be configured") case c.HTTPClientConfig.Authorization != nil: return errors.New("marathon_sd: at most one of auth_token, auth_token_file & authorization must be configured") @@ -140,10 +139,10 @@ type Discovery struct { } // NewDiscovery returns a new Marathon Discovery. -func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*marathonMetrics) +func NewDiscovery(conf SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*marathonMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "marathon_sd") @@ -154,7 +153,7 @@ func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.Discoverer switch { case len(conf.AuthToken) > 0: rt, err = newAuthTokenRoundTripper(conf.AuthToken, rt) - case len(conf.AuthTokenFile) > 0: + case conf.AuthTokenFile != "": rt, err = newAuthTokenFileRoundTripper(conf.AuthTokenFile, rt) } if err != nil { @@ -168,8 +167,9 @@ func NewDiscovery(conf SDConfig, logger log.Logger, metrics discovery.Discoverer } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "marathon", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -339,7 +339,7 @@ type appListClient func(ctx context.Context, client *http.Client, url string) (* // fetchApps requests a list of applications from a marathon server. func fetchApps(ctx context.Context, client *http.Client, url string) (*appList, error) { - request, err := http.NewRequest(http.MethodGet, url, nil) + request, err := http.NewRequest(http.MethodGet, url, http.NoBody) if err != nil { return nil, err } @@ -513,7 +513,7 @@ func extractPortMapping(portMappings []portMapping, containerNet bool) ([]uint32 ports := make([]uint32, len(portMappings)) labels := make([]map[string]string, len(portMappings)) - for i := 0; i < len(portMappings); i++ { + for i := range portMappings { labels[i] = portMappings[i].Labels if containerNet { diff --git a/discovery/marathon/marathon_test.go b/discovery/marathon/marathon_test.go index 659899f1638..4b61c82155a 100644 --- a/discovery/marathon/marathon_test.go +++ b/discovery/marathon/marathon_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -51,7 +51,11 @@ func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) { defer metrics.Unregister() defer refreshMetrics.Unregister() - md, err := NewDiscovery(cfg, nil, metrics) + md, err := NewDiscovery(cfg, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "marathon", + }) if err != nil { return nil, err } @@ -62,9 +66,10 @@ func testUpdateServices(client appListClient) ([]*targetgroup.Group, error) { } func TestMarathonSDHandleError(t *testing.T) { + t.Parallel() var ( errTesting = errors.New("testing failure") - client = func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + client = func(context.Context, *http.Client, string) (*appList, error) { return nil, errTesting } ) @@ -74,7 +79,8 @@ func TestMarathonSDHandleError(t *testing.T) { } func TestMarathonSDEmptyList(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { return &appList{}, nil } + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return &appList{}, nil } tgs, err := testUpdateServices(client) require.NoError(t, err) require.Empty(t, tgs, "Expected no target groups.") @@ -107,7 +113,8 @@ func marathonTestAppList(labels map[string]string, runningTasks int) *appList { } func TestMarathonSDSendGroup(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppList(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -124,6 +131,7 @@ func TestMarathonSDSendGroup(t *testing.T) { } func TestMarathonSDRemoveApp(t *testing.T) { + t.Parallel() cfg := testConfig() reg := prometheus.NewRegistry() refreshMetrics := discovery.NewRefreshMetrics(reg) @@ -132,10 +140,14 @@ func TestMarathonSDRemoveApp(t *testing.T) { defer metrics.Unregister() defer refreshMetrics.Unregister() - md, err := NewDiscovery(cfg, nil, metrics) + md, err := NewDiscovery(cfg, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "marathon", + }) require.NoError(t, err) - md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + md.appsClient = func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppList(marathonValidLabel, 1), nil } tgs, err := md.refresh(context.Background()) @@ -143,7 +155,7 @@ func TestMarathonSDRemoveApp(t *testing.T) { require.Len(t, tgs, 1, "Expected 1 targetgroup.") tg1 := tgs[0] - md.appsClient = func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + md.appsClient = func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppList(marathonValidLabel, 0), nil } tgs, err = md.refresh(context.Background()) @@ -184,7 +196,8 @@ func marathonTestAppListWithMultiplePorts(labels map[string]string, runningTasks } func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -202,7 +215,7 @@ func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) { tgt = tg.Targets[1] require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port: %s", tgt[model.AddressLabel]) } @@ -229,7 +242,8 @@ func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int) } func TestMarathonZeroTaskPorts(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestZeroTaskPortAppList(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -242,8 +256,9 @@ func TestMarathonZeroTaskPorts(t *testing.T) { } func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) { + t.Parallel() // Simulate 500 error with a valid JSON response. - respHandler := func(w http.ResponseWriter, r *http.Request) { + respHandler := func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Header().Set("Content-Type", "application/json") io.WriteString(w, `{}`) @@ -287,7 +302,8 @@ func marathonTestAppListWithPortDefinitions(labels map[string]string, runningTas } func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -300,9 +316,9 @@ func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) { tgt := tg.Targets[0] require.Equal(t, "mesos-slave1:1234", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") tgt = tg.Targets[1] @@ -341,7 +357,8 @@ func marathonTestAppListWithPortDefinitionsRequirePorts(labels map[string]string } func TestMarathonSDSendGroupWithPortDefinitionsRequirePorts(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppListWithPortDefinitionsRequirePorts(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -354,12 +371,12 @@ func TestMarathonSDSendGroupWithPortDefinitionsRequirePorts(t *testing.T) { tgt := tg.Targets[0] require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") tgt = tg.Targets[1] require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") require.Equal(t, "yes", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") } @@ -388,7 +405,8 @@ func marathonTestAppListWithPorts(labels map[string]string, runningTasks int) *a } func TestMarathonSDSendGroupWithPorts(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppListWithPorts(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -401,13 +419,13 @@ func TestMarathonSDSendGroupWithPorts(t *testing.T) { tgt := tg.Targets[0] require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") tgt = tg.Targets[1] require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") } func marathonTestAppListWithContainerPortMappings(labels map[string]string, runningTasks int) *appList { @@ -444,7 +462,8 @@ func marathonTestAppListWithContainerPortMappings(labels map[string]string, runn } func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppListWithContainerPortMappings(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -458,12 +477,12 @@ func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) { tgt := tg.Targets[0] require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.") require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") tgt = tg.Targets[1] require.Equal(t, "mesos-slave1:32000", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") } func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string, runningTasks int) *appList { @@ -500,7 +519,8 @@ func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string } func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -514,12 +534,12 @@ func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) { tgt := tg.Targets[0] require.Equal(t, "mesos-slave1:31000", string(tgt[model.AddressLabel]), "Wrong target address.") require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") tgt = tg.Targets[1] require.Equal(t, "mesos-slave1:12345", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") } func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]string, runningTasks int) *appList { @@ -560,7 +580,8 @@ func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]st } func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) { - client := func(_ context.Context, _ *http.Client, _ string) (*appList, error) { + t.Parallel() + client := func(context.Context, *http.Client, string) (*appList, error) { return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil } tgs, err := testUpdateServices(client) @@ -574,10 +595,10 @@ func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) { tgt := tg.Targets[0] require.Equal(t, "1.2.3.4:8080", string(tgt[model.AddressLabel]), "Wrong target address.") require.Equal(t, "yes", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the first port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the first port.") tgt = tg.Targets[1] require.Equal(t, "1.2.3.4:1234", string(tgt[model.AddressLabel]), "Wrong target address.") - require.Equal(t, "", string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") - require.Equal(t, "", string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portMappingLabelPrefix+"prometheus")]), "Wrong portMappings label from the second port.") + require.Empty(t, string(tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")]), "Wrong portDefinitions label from the second port.") } diff --git a/discovery/marathon/metrics.go b/discovery/marathon/metrics.go index 790c1674716..3d3d57d9ae7 100644 --- a/discovery/marathon/metrics.go +++ b/discovery/marathon/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type marathonMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *marathonMetrics) Register() error { +func (*marathonMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *marathonMetrics) Unregister() {} +func (*marathonMetrics) Unregister() {} diff --git a/discovery/metrics.go b/discovery/metrics.go index 356be1ddcbe..3eefd3d0bb1 100644 --- a/discovery/metrics.go +++ b/discovery/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -26,6 +26,7 @@ type Metrics struct { ReceivedUpdates prometheus.Counter DelayedUpdates prometheus.Counter SentUpdates prometheus.Counter + LastUpdated *prometheus.GaugeVec } func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) { @@ -72,12 +73,22 @@ func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) ( }, ) + m.LastUpdated = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prometheus_sd_last_update_timestamp_seconds", + Help: "Timestamp of the last update sent to the SD consumers.", + ConstLabels: prometheus.Labels{"name": sdManagerName}, + }, + []string{"config"}, + ) + metrics := []prometheus.Collector{ m.FailedConfigs, m.DiscoveredTargets, m.ReceivedUpdates, m.DelayedUpdates, m.SentUpdates, + m.LastUpdated, } for _, collector := range metrics { @@ -97,4 +108,5 @@ func (m *Metrics) Unregister(registerer prometheus.Registerer) { registerer.Unregister(m.ReceivedUpdates) registerer.Unregister(m.DelayedUpdates) registerer.Unregister(m.SentUpdates) + registerer.Unregister(m.LastUpdated) } diff --git a/discovery/metrics_k8s_client.go b/discovery/metrics_k8s_client.go index c13ce533178..3642eac5680 100644 --- a/discovery/metrics_k8s_client.go +++ b/discovery/metrics_k8s_client.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -176,27 +176,27 @@ func (f *clientGoWorkqueueMetricsProvider) RegisterWithK8sGoClient() { workqueue.SetProvider(f) } -func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { +func (*clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { return clientGoWorkqueueDepthMetricVec.WithLabelValues(name) } -func (f *clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { +func (*clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { return clientGoWorkqueueAddsMetricVec.WithLabelValues(name) } -func (f *clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { +func (*clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { return clientGoWorkqueueLatencyMetricVec.WithLabelValues(name) } -func (f *clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { +func (*clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { return clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name) } -func (f *clientGoWorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { +func (*clientGoWorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { return clientGoWorkqueueUnfinishedWorkSecondsMetricVec.WithLabelValues(name) } -func (f *clientGoWorkqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { +func (*clientGoWorkqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { return clientGoWorkqueueLongestRunningProcessorMetricVec.WithLabelValues(name) } diff --git a/discovery/metrics_refresh.go b/discovery/metrics_refresh.go index ef49e591a35..bb334050cd0 100644 --- a/discovery/metrics_refresh.go +++ b/discovery/metrics_refresh.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,6 +14,8 @@ package discovery import ( + "time" + "github.com/prometheus/client_golang/prometheus" ) @@ -21,8 +23,9 @@ import ( // We define them here in the "discovery" package in order to avoid a cyclic dependency between // "discovery" and "refresh". type RefreshMetricsVecs struct { - failuresVec *prometheus.CounterVec - durationVec *prometheus.SummaryVec + failuresVec *prometheus.CounterVec + durationVec *prometheus.SummaryVec + durationHistVec *prometheus.HistogramVec metricRegisterer MetricRegisterer } @@ -36,13 +39,23 @@ func NewRefreshMetrics(reg prometheus.Registerer) RefreshMetricsManager { Name: "prometheus_sd_refresh_failures_total", Help: "Number of refresh failures for the given SD mechanism.", }, - []string{"mechanism"}), + []string{"mechanism", "config"}), durationVec: prometheus.NewSummaryVec( prometheus.SummaryOpts{ Name: "prometheus_sd_refresh_duration_seconds", Help: "The duration of a refresh in seconds for the given SD mechanism.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, + []string{"mechanism", "config"}), + durationHistVec: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "prometheus_sd_refresh_duration_histogram_seconds", + Help: "The duration of a refresh for the given SD mechanism.", + Buckets: []float64{.01, .1, 1, 10}, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"mechanism"}), } @@ -51,16 +64,18 @@ func NewRefreshMetrics(reg prometheus.Registerer) RefreshMetricsManager { m.metricRegisterer = NewMetricRegisterer(reg, []prometheus.Collector{ m.failuresVec, m.durationVec, + m.durationHistVec, }) return m } -// Instantiate returns metrics out of metric vectors. -func (m *RefreshMetricsVecs) Instantiate(mech string) *RefreshMetrics { +// Instantiate returns metrics out of metric vectors for a given mechanism and config. +func (m *RefreshMetricsVecs) Instantiate(mech, config string) *RefreshMetrics { return &RefreshMetrics{ - Failures: m.failuresVec.WithLabelValues(mech), - Duration: m.durationVec.WithLabelValues(mech), + Failures: m.failuresVec.WithLabelValues(mech, config), + Duration: m.durationVec.WithLabelValues(mech, config), + DurationHistogram: m.durationHistVec.WithLabelValues(mech), } } @@ -73,3 +88,9 @@ func (m *RefreshMetricsVecs) Register() error { func (m *RefreshMetricsVecs) Unregister() { m.metricRegisterer.UnregisterMetrics() } + +// DeleteLabelValues deletes refresh metrics for a specific mechanism and config. Smart to use this when a scrape job is removed. +func (m *RefreshMetricsVecs) DeleteLabelValues(mech, config string) { + m.failuresVec.DeleteLabelValues(mech, config) + m.durationVec.DeleteLabelValues(mech, config) +} diff --git a/discovery/moby/docker.go b/discovery/moby/docker.go index 68f6fe3ccc1..46ed842fa9d 100644 --- a/discovery/moby/docker.go +++ b/discovery/moby/docker.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,6 +15,7 @@ package moby import ( "context" + "errors" "fmt" "net" "net/http" @@ -23,15 +24,13 @@ import ( "strconv" "time" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" - "github.com/go-kit/log" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/client" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/version" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/refresh" @@ -93,7 +92,7 @@ func (*DockerSDConfig) Name() string { return "docker" } // NewDiscoverer returns a Discoverer for the Config. func (c *DockerSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDockerDiscovery(c, opts.Logger, opts.Metrics) + return NewDockerDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -102,7 +101,7 @@ func (c *DockerSDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *DockerSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *DockerSDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultDockerSDConfig type plain DockerSDConfig err := unmarshal((*plain)(c)) @@ -110,7 +109,7 @@ func (c *DockerSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error return err } if c.Host == "" { - return fmt.Errorf("host missing") + return errors.New("host missing") } if _, err = url.Parse(c.Host); err != nil { return err @@ -123,15 +122,15 @@ type DockerDiscovery struct { client *client.Client port int hostNetworkingHost string - filters filters.Args + filters client.Filters matchFirstNetwork bool } // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets. -func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*DockerDiscovery, error) { - m, ok := metrics.(*dockerMetrics) +func NewDockerDiscovery(conf *DockerSDConfig, opts discovery.DiscovererOptions) (*DockerDiscovery, error) { + m, ok := opts.Metrics.(*dockerMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } d := &DockerDiscovery{ @@ -145,12 +144,11 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discove return nil, err } - opts := []client.Opt{ + clientOpts := []client.Opt{ client.WithHost(conf.Host), - client.WithAPIVersionNegotiation(), } - d.filters = filters.NewArgs() + d.filters = make(client.Filters) for _, f := range conf.Filters { for _, v := range f.Values { d.filters.Add(f.Name, v) @@ -165,27 +163,28 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discove if err != nil { return nil, err } - opts = append(opts, + clientOpts = append(clientOpts, client.WithHTTPClient(&http.Client{ Transport: rt, Timeout: time.Duration(conf.RefreshInterval), }), client.WithScheme(hostURL.Scheme), client.WithHTTPHeaders(map[string]string{ - "User-Agent": userAgent, + "User-Agent": version.PrometheusUserAgent(), }), ) } - d.client, err = client.NewClientWithOpts(opts...) + d.client, err = client.New(clientOpts...) if err != nil { return nil, fmt.Errorf("error setting up docker client: %w", err) } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "docker", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -199,7 +198,7 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er Source: "Docker", } - containers, err := d.client.ContainerList(ctx, container.ListOptions{Filters: d.filters}) + containers, err := d.client.ContainerList(ctx, client.ContainerListOptions{Filters: d.filters}) if err != nil { return nil, fmt.Errorf("error while listing containers: %w", err) } @@ -209,12 +208,12 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er return nil, fmt.Errorf("error while computing network labels: %w", err) } - allContainers := make(map[string]types.Container) - for _, c := range containers { + allContainers := make(map[string]container.Summary) + for _, c := range containers.Items { allContainers[c.ID] = c } - for _, c := range containers { + for _, c := range containers.Items { if len(c.Names) == 0 { continue } @@ -234,18 +233,14 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er containerNetworkMode := container.NetworkMode(c.HostConfig.NetworkMode) if len(networks) == 0 { // Try to lookup shared networks - for { - if containerNetworkMode.IsContainer() { - tmpContainer, exists := allContainers[containerNetworkMode.ConnectedContainer()] - if !exists { - break - } - networks = tmpContainer.NetworkSettings.Networks - containerNetworkMode = container.NetworkMode(tmpContainer.HostConfig.NetworkMode) - if len(networks) > 0 { - break - } - } else { + for containerNetworkMode.IsContainer() { + tmpContainer, exists := allContainers[containerNetworkMode.ConnectedContainer()] + if !exists { + break + } + networks = tmpContainer.NetworkSettings.Networks + containerNetworkMode = container.NetworkMode(tmpContainer.HostConfig.NetworkMode) + if len(networks) > 0 { break } } @@ -279,14 +274,23 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er continue } + ipAddr := "" + if n.IPAddress.IsValid() { + ipAddr = n.IPAddress.String() + } + labels := model.LabelSet{ - dockerLabelNetworkIP: model.LabelValue(n.IPAddress), + dockerLabelNetworkIP: model.LabelValue(ipAddr), dockerLabelPortPrivate: model.LabelValue(strconv.FormatUint(uint64(p.PrivatePort), 10)), } if p.PublicPort > 0 { labels[dockerLabelPortPublic] = model.LabelValue(strconv.FormatUint(uint64(p.PublicPort), 10)) - labels[dockerLabelPortPublicIP] = model.LabelValue(p.IP) + publicIP := "" + if p.IP.IsValid() { + publicIP = p.IP.String() + } + labels[dockerLabelPortPublicIP] = model.LabelValue(publicIP) } for k, v := range commonLabels { @@ -297,7 +301,7 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er labels[model.LabelName(k)] = model.LabelValue(v) } - addr := net.JoinHostPort(n.IPAddress, strconv.FormatUint(uint64(p.PrivatePort), 10)) + addr := net.JoinHostPort(ipAddr, strconv.FormatUint(uint64(p.PrivatePort), 10)) labels[model.AddressLabel] = model.LabelValue(addr) tg.Targets = append(tg.Targets, labels) added = true @@ -305,8 +309,13 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er if !added { // Use fallback port when no exposed ports are available or if all are non-TCP + ipAddr := "" + if n.IPAddress.IsValid() { + ipAddr = n.IPAddress.String() + } + labels := model.LabelSet{ - dockerLabelNetworkIP: model.LabelValue(n.IPAddress), + dockerLabelNetworkIP: model.LabelValue(ipAddr), } for k, v := range commonLabels { @@ -321,7 +330,7 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er // so they only end up here, not in the previous loop. var addr string if c.HostConfig.NetworkMode != "host" { - addr = net.JoinHostPort(n.IPAddress, strconv.FormatUint(uint64(d.port), 10)) + addr = net.JoinHostPort(ipAddr, strconv.FormatUint(uint64(d.port), 10)) } else { addr = d.hostNetworkingHost } diff --git a/discovery/moby/docker_test.go b/discovery/moby/docker_test.go index 398393a15ae..effdf90b369 100644 --- a/discovery/moby/docker_test.go +++ b/discovery/moby/docker_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,11 +19,11 @@ import ( "sort" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" ) @@ -48,7 +48,11 @@ host: %s defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDockerDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "docker_swarm", + }) require.NoError(t, err) ctx := context.Background() @@ -226,7 +230,11 @@ host: %s require.NoError(t, metrics.Register()) defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDockerDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "docker_swarm", + }) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/moby/dockerswarm.go b/discovery/moby/dockerswarm.go index b0147467d28..3605bc4900d 100644 --- a/discovery/moby/dockerswarm.go +++ b/discovery/moby/dockerswarm.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,14 +15,13 @@ package moby import ( "context" + "errors" "fmt" "net/http" "net/url" "time" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/client" - "github.com/go-kit/log" + "github.com/moby/moby/client" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -37,8 +36,6 @@ const ( swarmLabel = model.MetaLabelPrefix + "dockerswarm_" ) -var userAgent = fmt.Sprintf("Prometheus/%s", version.Version) - // DefaultDockerSwarmSDConfig is the default Docker Swarm SD configuration. var DefaultDockerSwarmSDConfig = DockerSwarmSDConfig{ RefreshInterval: model.Duration(60 * time.Second), @@ -71,7 +68,7 @@ type Filter struct { } // NewDiscovererMetrics implements discovery.Config. -func (*DockerSwarmSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*DockerSwarmSDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &dockerswarmMetrics{ refreshMetrics: rmi, } @@ -82,7 +79,7 @@ func (*DockerSwarmSDConfig) Name() string { return "dockerswarm" } // NewDiscoverer returns a Discoverer for the Config. func (c *DockerSwarmSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -91,7 +88,7 @@ func (c *DockerSwarmSDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *DockerSwarmSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *DockerSwarmSDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultDockerSwarmSDConfig type plain DockerSwarmSDConfig err := unmarshal((*plain)(c)) @@ -99,7 +96,7 @@ func (c *DockerSwarmSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) e return err } if c.Host == "" { - return fmt.Errorf("host missing") + return errors.New("host missing") } if _, err = url.Parse(c.Host); err != nil { return err @@ -107,7 +104,7 @@ func (c *DockerSwarmSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) e switch c.Role { case "services", "nodes", "tasks": case "": - return fmt.Errorf("role missing (one of: tasks, services, nodes)") + return errors.New("role missing (one of: tasks, services, nodes)") default: return fmt.Errorf("invalid role %s, expected tasks, services, or nodes", c.Role) } @@ -121,14 +118,14 @@ type Discovery struct { client *client.Client role string port int - filters filters.Args + filters client.Filters } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*dockerswarmMetrics) +func NewDiscovery(conf *DockerSwarmSDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*dockerswarmMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } d := &Discovery{ @@ -141,12 +138,11 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, metrics discover return nil, err } - opts := []client.Opt{ + clientOpts := []client.Opt{ client.WithHost(conf.Host), - client.WithAPIVersionNegotiation(), } - d.filters = filters.NewArgs() + d.filters = make(client.Filters) for _, f := range conf.Filters { for _, v := range f.Values { d.filters.Add(f.Name, v) @@ -161,27 +157,28 @@ func NewDiscovery(conf *DockerSwarmSDConfig, logger log.Logger, metrics discover if err != nil { return nil, err } - opts = append(opts, + clientOpts = append(clientOpts, client.WithHTTPClient(&http.Client{ Transport: rt, Timeout: time.Duration(conf.RefreshInterval), }), client.WithScheme(hostURL.Scheme), client.WithHTTPHeaders(map[string]string{ - "User-Agent": userAgent, + "User-Agent": version.PrometheusUserAgent(), }), ) } - d.client, err = client.NewClientWithOpts(opts...) + d.client, err = client.New(clientOpts...) if err != nil { return nil, fmt.Errorf("error setting up docker swarm client: %w", err) } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "dockerswarm", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, diff --git a/discovery/moby/metrics_docker.go b/discovery/moby/metrics_docker.go index 457d520a0cb..8c2518a75e0 100644 --- a/discovery/moby/metrics_docker.go +++ b/discovery/moby/metrics_docker.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type dockerMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *dockerMetrics) Register() error { +func (*dockerMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *dockerMetrics) Unregister() {} +func (*dockerMetrics) Unregister() {} diff --git a/discovery/moby/metrics_dockerswarm.go b/discovery/moby/metrics_dockerswarm.go index 227ba6c2b09..e4682b032ae 100644 --- a/discovery/moby/metrics_dockerswarm.go +++ b/discovery/moby/metrics_dockerswarm.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type dockerswarmMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *dockerswarmMetrics) Register() error { +func (*dockerswarmMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *dockerswarmMetrics) Unregister() {} +func (*dockerswarmMetrics) Unregister() {} diff --git a/discovery/moby/mock_test.go b/discovery/moby/mock_test.go index 3f35258c8f4..e43319494db 100644 --- a/discovery/moby/mock_test.go +++ b/discovery/moby/mock_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,7 +24,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/util/strutil" ) @@ -98,7 +98,7 @@ func (m *SDMock) SetupHandlers() { if len(query) == 2 { h := sha1.New() h.Write([]byte(query[1])) - // Avoing long filenames for Windows. + // Avoiding long filenames for Windows. f += "__" + base64.URLEncoding.EncodeToString(h.Sum(nil))[:10] } } diff --git a/discovery/moby/network.go b/discovery/moby/network.go index ea1ca66bc7d..33a83b1eef1 100644 --- a/discovery/moby/network.go +++ b/discovery/moby/network.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,8 +17,7 @@ import ( "context" "strconv" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" + "github.com/moby/moby/client" "github.com/prometheus/prometheus/util/strutil" ) @@ -33,13 +32,13 @@ const ( labelNetworkLabelPrefix = labelNetworkPrefix + "label_" ) -func getNetworksLabels(ctx context.Context, client *client.Client, labelPrefix string) (map[string]map[string]string, error) { - networks, err := client.NetworkList(ctx, network.ListOptions{}) +func getNetworksLabels(ctx context.Context, c *client.Client, labelPrefix string) (map[string]map[string]string, error) { + networks, err := c.NetworkList(ctx, client.NetworkListOptions{}) if err != nil { return nil, err } - labels := make(map[string]map[string]string, len(networks)) - for _, network := range networks { + labels := make(map[string]map[string]string, len(networks.Items)) + for _, network := range networks.Items { labels[network.ID] = map[string]string{ labelPrefix + labelNetworkID: network.ID, labelPrefix + labelNetworkName: network.Name, diff --git a/discovery/moby/nodes.go b/discovery/moby/nodes.go index b5be844eda6..9001e88e026 100644 --- a/discovery/moby/nodes.go +++ b/discovery/moby/nodes.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,7 +19,7 @@ import ( "net" "strconv" - "github.com/docker/docker/api/types" + "github.com/moby/moby/client" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -48,12 +48,12 @@ func (d *Discovery) refreshNodes(ctx context.Context) ([]*targetgroup.Group, err Source: "DockerSwarm", } - nodes, err := d.client.NodeList(ctx, types.NodeListOptions{Filters: d.filters}) + nodes, err := d.client.NodeList(ctx, client.NodeListOptions{Filters: d.filters}) if err != nil { return nil, fmt.Errorf("error while listing swarm nodes: %w", err) } - for _, n := range nodes { + for _, n := range nodes.Items { labels := model.LabelSet{ swarmLabelNodeID: model.LabelValue(n.ID), swarmLabelNodeRole: model.LabelValue(n.Spec.Role), @@ -85,12 +85,12 @@ func (d *Discovery) refreshNodes(ctx context.Context) ([]*targetgroup.Group, err } func (d *Discovery) getNodesLabels(ctx context.Context) (map[string]map[string]string, error) { - nodes, err := d.client.NodeList(ctx, types.NodeListOptions{}) + nodes, err := d.client.NodeList(ctx, client.NodeListOptions{}) if err != nil { return nil, fmt.Errorf("error while listing swarm nodes: %w", err) } - labels := make(map[string]map[string]string, len(nodes)) - for _, n := range nodes { + labels := make(map[string]map[string]string, len(nodes.Items)) + for _, n := range nodes.Items { labels[n.ID] = map[string]string{ swarmLabelNodeID: n.ID, swarmLabelNodeRole: string(n.Spec.Role), diff --git a/discovery/moby/nodes_test.go b/discovery/moby/nodes_test.go index 4ad1088d1ab..1f97016297b 100644 --- a/discovery/moby/nodes_test.go +++ b/discovery/moby/nodes_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,11 +18,11 @@ import ( "fmt" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" ) @@ -48,7 +48,11 @@ host: %s defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "docker_swarm", + }) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/moby/services.go b/discovery/moby/services.go index c61b499259c..3254ffd1726 100644 --- a/discovery/moby/services.go +++ b/discovery/moby/services.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,8 +19,9 @@ import ( "net" "strconv" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/swarm" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/api/types/swarm" + "github.com/moby/moby/client" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -46,7 +47,7 @@ func (d *Discovery) refreshServices(ctx context.Context) ([]*targetgroup.Group, Source: "DockerSwarm", } - services, err := d.client.ServiceList(ctx, types.ServiceListOptions{Filters: d.filters}) + services, err := d.client.ServiceList(ctx, client.ServiceListOptions{Filters: d.filters}) if err != nil { return nil, fmt.Errorf("error while listing swarm services: %w", err) } @@ -56,7 +57,7 @@ func (d *Discovery) refreshServices(ctx context.Context) ([]*targetgroup.Group, return nil, fmt.Errorf("error while computing swarm network labels: %w", err) } - for _, s := range services { + for _, s := range services.Items { commonLabels := map[string]string{ swarmLabelServiceID: s.ID, swarmLabelServiceName: s.Spec.Name, @@ -76,13 +77,13 @@ func (d *Discovery) refreshServices(ctx context.Context) ([]*targetgroup.Group, for _, p := range s.Endpoint.VirtualIPs { var added bool - ip, _, err := net.ParseCIDR(p.Addr) + ip, _, err := net.ParseCIDR(p.Addr.String()) if err != nil { return nil, fmt.Errorf("error while parsing address %s: %w", p.Addr, err) } for _, e := range s.Endpoint.Ports { - if e.Protocol != swarm.PortConfigProtocolTCP { + if e.Protocol != network.TCP { continue } labels := model.LabelSet{ @@ -127,13 +128,13 @@ func (d *Discovery) refreshServices(ctx context.Context) ([]*targetgroup.Group, } func (d *Discovery) getServicesLabelsAndPorts(ctx context.Context) (map[string]map[string]string, map[string][]swarm.PortConfig, error) { - services, err := d.client.ServiceList(ctx, types.ServiceListOptions{}) + services, err := d.client.ServiceList(ctx, client.ServiceListOptions{}) if err != nil { return nil, nil, err } - servicesLabels := make(map[string]map[string]string, len(services)) - servicesPorts := make(map[string][]swarm.PortConfig, len(services)) - for _, s := range services { + servicesLabels := make(map[string]map[string]string, len(services.Items)) + servicesPorts := make(map[string][]swarm.PortConfig, len(services.Items)) + for _, s := range services.Items { servicesLabels[s.ID] = map[string]string{ swarmLabelServiceID: s.ID, swarmLabelServiceName: s.Spec.Name, diff --git a/discovery/moby/services_test.go b/discovery/moby/services_test.go index 47ca69e33a1..eb5c75c71ef 100644 --- a/discovery/moby/services_test.go +++ b/discovery/moby/services_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,11 +18,11 @@ import ( "fmt" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" ) @@ -48,7 +48,11 @@ host: %s defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "moby", + }) require.NoError(t, err) ctx := context.Background() @@ -349,7 +353,11 @@ filters: defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "moby", + }) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/moby/tasks.go b/discovery/moby/tasks.go index 38b9d33de26..ec247d1d302 100644 --- a/discovery/moby/tasks.go +++ b/discovery/moby/tasks.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,11 +16,12 @@ package moby import ( "context" "fmt" + "maps" "net" "strconv" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/swarm" + mobynetwork "github.com/moby/moby/api/types/network" + "github.com/moby/moby/client" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -43,7 +44,7 @@ func (d *Discovery) refreshTasks(ctx context.Context) ([]*targetgroup.Group, err Source: "DockerSwarm", } - tasks, err := d.client.TaskList(ctx, types.TaskListOptions{Filters: d.filters}) + tasks, err := d.client.TaskList(ctx, client.TaskListOptions{Filters: d.filters}) if err != nil { return nil, fmt.Errorf("error while listing swarm services: %w", err) } @@ -63,7 +64,7 @@ func (d *Discovery) refreshTasks(ctx context.Context) ([]*targetgroup.Group, err return nil, fmt.Errorf("error while computing swarm network labels: %w", err) } - for _, s := range tasks { + for _, s := range tasks.Items { commonLabels := map[string]string{ swarmLabelTaskID: s.ID, swarmLabelTaskDesiredState: string(s.DesiredState), @@ -82,16 +83,12 @@ func (d *Discovery) refreshTasks(ctx context.Context) ([]*targetgroup.Group, err } } - for k, v := range serviceLabels[s.ServiceID] { - commonLabels[k] = v - } + maps.Copy(commonLabels, serviceLabels[s.ServiceID]) - for k, v := range nodeLabels[s.NodeID] { - commonLabels[k] = v - } + maps.Copy(commonLabels, nodeLabels[s.NodeID]) for _, p := range s.Status.PortStatus.Ports { - if p.Protocol != swarm.PortConfigProtocolTCP { + if p.Protocol != mobynetwork.TCP { continue } @@ -112,13 +109,13 @@ func (d *Discovery) refreshTasks(ctx context.Context) ([]*targetgroup.Group, err for _, address := range network.Addresses { var added bool - ip, _, err := net.ParseCIDR(address) + ip, _, err := net.ParseCIDR(address.String()) if err != nil { return nil, fmt.Errorf("error while parsing address %s: %w", address, err) } for _, p := range servicePorts[s.ServiceID] { - if p.Protocol != swarm.PortConfigProtocolTCP { + if p.Protocol != mobynetwork.TCP { continue } labels := model.LabelSet{ diff --git a/discovery/moby/tasks_test.go b/discovery/moby/tasks_test.go index ef71bc02f53..60453990c45 100644 --- a/discovery/moby/tasks_test.go +++ b/discovery/moby/tasks_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,11 +18,11 @@ import ( "fmt" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" ) @@ -48,7 +48,11 @@ host: %s defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "docker_swarm", + }) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/moby/testdata/swarmprom/services.json b/discovery/moby/testdata/swarmprom/services.json index 72caa7a7f8d..8f6c0793dd2 100644 --- a/discovery/moby/testdata/swarmprom/services.json +++ b/discovery/moby/testdata/swarmprom/services.json @@ -224,7 +224,7 @@ "Args": [ "--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", - "--storage.tsdb.retention=24h" + "--storage.tsdb.retention.time=24h" ], "Privileges": { "CredentialSpec": null, diff --git a/discovery/moby/testdata/swarmprom/tasks.json b/discovery/moby/testdata/swarmprom/tasks.json index 33d81f25ce1..af5ff9fe283 100644 --- a/discovery/moby/testdata/swarmprom/tasks.json +++ b/discovery/moby/testdata/swarmprom/tasks.json @@ -973,7 +973,7 @@ "Args": [ "--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", - "--storage.tsdb.retention=24h" + "--storage.tsdb.retention.time=24h" ], "Privileges": { "CredentialSpec": null, diff --git a/discovery/nomad/metrics.go b/discovery/nomad/metrics.go index 9707153d91c..0e5dca47235 100644 --- a/discovery/nomad/metrics.go +++ b/discovery/nomad/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/nomad/nomad.go b/discovery/nomad/nomad.go index d9c48120ae8..da558f54d9a 100644 --- a/discovery/nomad/nomad.go +++ b/discovery/nomad/nomad.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -22,7 +22,6 @@ import ( "strings" "time" - "github.com/go-kit/log" nomad "github.com/hashicorp/nomad/api" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -84,7 +83,7 @@ func (*SDConfig) Name() string { return "nomad" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -93,7 +92,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -121,10 +120,10 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*nomadMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*nomadMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } d := &Discovery{ @@ -157,8 +156,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "nomad", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, diff --git a/discovery/nomad/nomad_test.go b/discovery/nomad/nomad_test.go index 357d4a8e9b6..f5b0f7dc7b1 100644 --- a/discovery/nomad/nomad_test.go +++ b/discovery/nomad/nomad_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,9 +21,9 @@ import ( "net/url" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/discovery" @@ -76,7 +76,7 @@ func (s *NomadSDTestSuite) SetupTest(t *testing.T) { } func (m *SDMock) HandleServicesList() { - m.Mux.HandleFunc("/v1/services", func(w http.ResponseWriter, r *http.Request) { + m.Mux.HandleFunc("/v1/services", func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("content-type", "application/json; charset=utf-8") w.WriteHeader(http.StatusOK) @@ -99,7 +99,7 @@ func (m *SDMock) HandleServicesList() { } func (m *SDMock) HandleServiceHashiCupsGet() { - m.Mux.HandleFunc("/v1/service/hashicups", func(w http.ResponseWriter, r *http.Request) { + m.Mux.HandleFunc("/v1/service/hashicups", func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("content-type", "application/json; charset=utf-8") w.WriteHeader(http.StatusOK) @@ -127,22 +127,47 @@ func (m *SDMock) HandleServiceHashiCupsGet() { } func TestConfiguredService(t *testing.T) { - conf := &SDConfig{ - Server: "http://localhost:4646", + t.Parallel() + testCases := []struct { + name string + server string + acceptedURL bool + }{ + {"invalid hostname URL", "http://foo.bar:4646", true}, + {"invalid even though accepted by parsing", "foo.bar:4646", true}, + {"valid address URL", "http://172.30.29.23:4646", true}, + {"invalid URL", "172.30.29.23:4646", false}, } - reg := prometheus.NewRegistry() - refreshMetrics := discovery.NewRefreshMetrics(reg) - metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) - require.NoError(t, metrics.Register()) - - _, err := NewDiscovery(conf, nil, metrics) - require.NoError(t, err) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + conf := &SDConfig{ + Server: tc.server, + } - metrics.Unregister() + reg := prometheus.NewRegistry() + refreshMetrics := discovery.NewRefreshMetrics(reg) + metrics := conf.NewDiscovererMetrics(reg, refreshMetrics) + require.NoError(t, metrics.Register()) + defer metrics.Unregister() + + _, err := NewDiscovery(conf, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "nomad", + }) + if tc.acceptedURL { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } } func TestNomadSDRefresh(t *testing.T) { + t.Parallel() sdmock := &NomadSDTestSuite{} sdmock.SetupTest(t) t.Cleanup(sdmock.TearDownSuite) @@ -160,7 +185,11 @@ func TestNomadSDRefresh(t *testing.T) { defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "nomad", + }) require.NoError(t, err) tgs, err := d.refresh(context.Background()) diff --git a/discovery/openstack/hypervisor.go b/discovery/openstack/hypervisor.go index 8964da9294f..141b77c706c 100644 --- a/discovery/openstack/hypervisor.go +++ b/discovery/openstack/hypervisor.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,14 +16,14 @@ package openstack import ( "context" "fmt" + "log/slog" "net" "strconv" - "github.com/go-kit/log" - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors" - "github.com/gophercloud/gophercloud/pagination" + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors" + "github.com/gophercloud/gophercloud/v2/pagination" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -43,14 +43,14 @@ type HypervisorDiscovery struct { provider *gophercloud.ProviderClient authOpts *gophercloud.AuthOptions region string - logger log.Logger + logger *slog.Logger port int availability gophercloud.Availability } // newHypervisorDiscovery returns a new hypervisor discovery. func newHypervisorDiscovery(provider *gophercloud.ProviderClient, opts *gophercloud.AuthOptions, - port int, region string, availability gophercloud.Availability, l log.Logger, + port int, region string, availability gophercloud.Availability, l *slog.Logger, ) *HypervisorDiscovery { return &HypervisorDiscovery{ provider: provider, authOpts: opts, @@ -59,8 +59,7 @@ func newHypervisorDiscovery(provider *gophercloud.ProviderClient, opts *gophercl } func (h *HypervisorDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - h.provider.Context = ctx - err := openstack.Authenticate(h.provider, *h.authOpts) + err := openstack.Authenticate(ctx, h.provider, *h.authOpts) if err != nil { return nil, fmt.Errorf("could not authenticate to OpenStack: %w", err) } @@ -78,7 +77,7 @@ func (h *HypervisorDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group // OpenStack API reference // https://developer.openstack.org/api-ref/compute/#list-hypervisors-details pagerHypervisors := hypervisors.List(client, nil) - err = pagerHypervisors.EachPage(func(page pagination.Page) (bool, error) { + err = pagerHypervisors.EachPage(ctx, func(_ context.Context, page pagination.Page) (bool, error) { hypervisorList, err := hypervisors.ExtractHypervisors(page) if err != nil { return false, fmt.Errorf("could not extract hypervisors: %w", err) diff --git a/discovery/openstack/hypervisor_test.go b/discovery/openstack/hypervisor_test.go index 45684b4a2ec..afba84af2da 100644 --- a/discovery/openstack/hypervisor_test.go +++ b/discovery/openstack/hypervisor_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -93,6 +93,5 @@ func TestOpenstackSDHypervisorRefreshWithDoneContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() _, err := hypervisor.refresh(ctx) - require.Error(t, err) - require.Contains(t, err.Error(), context.Canceled.Error(), "%q doesn't contain %q", err, context.Canceled) + require.ErrorContains(t, err, context.Canceled.Error(), "%q doesn't contain %q", err, context.Canceled) } diff --git a/discovery/openstack/instance.go b/discovery/openstack/instance.go index 78c669e6f76..2a6a777e9af 100644 --- a/discovery/openstack/instance.go +++ b/discovery/openstack/instance.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,17 +16,19 @@ package openstack import ( "context" "fmt" + "log/slog" + "maps" "net" "strconv" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/floatingips" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/pagination" + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/ports" + "github.com/gophercloud/gophercloud/v2/pagination" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/strutil" @@ -52,7 +54,7 @@ type InstanceDiscovery struct { provider *gophercloud.ProviderClient authOpts *gophercloud.AuthOptions region string - logger log.Logger + logger *slog.Logger port int allTenants bool availability gophercloud.Availability @@ -60,10 +62,10 @@ type InstanceDiscovery struct { // NewInstanceDiscovery returns a new instance discovery. func newInstanceDiscovery(provider *gophercloud.ProviderClient, opts *gophercloud.AuthOptions, - port int, region string, allTenants bool, availability gophercloud.Availability, l log.Logger, + port int, region string, allTenants bool, availability gophercloud.Availability, l *slog.Logger, ) *InstanceDiscovery { if l == nil { - l = log.NewNopLogger() + l = promslog.NewNopLogger() } return &InstanceDiscovery{ provider: provider, authOpts: opts, @@ -72,13 +74,12 @@ func newInstanceDiscovery(provider *gophercloud.ProviderClient, opts *gopherclou } type floatingIPKey struct { - id string - fixed string + deviceID string + fixed string } func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - i.provider.Context = ctx - err := openstack.Authenticate(i.provider, *i.authOpts) + err := openstack.Authenticate(ctx, i.provider, *i.authOpts) if err != nil { return nil, fmt.Errorf("could not authenticate to OpenStack: %w", err) } @@ -90,23 +91,60 @@ func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, return nil, fmt.Errorf("could not create OpenStack compute session: %w", err) } + networkClient, err := openstack.NewNetworkV2(i.provider, gophercloud.EndpointOpts{ + Region: i.region, Availability: i.availability, + }) + if err != nil { + return nil, fmt.Errorf("could not create OpenStack network session: %w", err) + } + // OpenStack API reference - // https://developer.openstack.org/api-ref/compute/#list-floating-ips - pagerFIP := floatingips.List(client) + // https://docs.openstack.org/api-ref/network/v2/index.html#list-ports + portPages, err := ports.List(networkClient, ports.ListOpts{}).AllPages(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list all ports: %w", err) + } + + allPorts, err := ports.ExtractPorts(portPages) + if err != nil { + return nil, fmt.Errorf("failed to extract Ports: %w", err) + } + + portList := make(map[string]string) + for _, port := range allPorts { + portList[port.ID] = port.DeviceID + } + + // OpenStack API reference + // https://docs.openstack.org/api-ref/network/v2/index.html#list-floating-ips + pagerFIP := floatingips.List(networkClient, floatingips.ListOpts{}) floatingIPList := make(map[floatingIPKey]string) floatingIPPresent := make(map[string]struct{}) - err = pagerFIP.EachPage(func(page pagination.Page) (bool, error) { + err = pagerFIP.EachPage(ctx, func(_ context.Context, page pagination.Page) (bool, error) { result, err := floatingips.ExtractFloatingIPs(page) if err != nil { return false, fmt.Errorf("could not extract floatingips: %w", err) } for _, ip := range result { // Skip not associated ips - if ip.InstanceID == "" || ip.FixedIP == "" { + if ip.PortID == "" || ip.FixedIP == "" { + continue + } + + // Fetch deviceID from portList + deviceID, ok := portList[ip.PortID] + if !ok { + i.logger.Warn("Floating IP PortID not found in portList", "PortID", ip.PortID) continue } - floatingIPList[floatingIPKey{id: ip.InstanceID, fixed: ip.FixedIP}] = ip.IP - floatingIPPresent[ip.IP] = struct{}{} + + key := floatingIPKey{ + deviceID: deviceID, + fixed: ip.FixedIP, + } + + floatingIPList[key] = ip.FloatingIP + floatingIPPresent[ip.FloatingIP] = struct{}{} } return true, nil }) @@ -123,7 +161,7 @@ func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, tg := &targetgroup.Group{ Source: "OS_" + i.region, } - err = pager.EachPage(func(page pagination.Page) (bool, error) { + err = pager.EachPage(ctx, func(ctx context.Context, page pagination.Page) (bool, error) { if ctx.Err() != nil { return false, fmt.Errorf("could not extract instances: %w", ctx.Err()) } @@ -134,7 +172,7 @@ func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, for _, s := range instanceList { if len(s.Addresses) == 0 { - level.Info(i.logger).Log("msg", "Got no IP address", "instance", s.ID) + i.logger.Info("Got no IP address", "instance", s.ID) continue } @@ -151,7 +189,7 @@ func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, if !nameOk { flavorID, idOk := s.Flavor["id"].(string) if !idOk { - level.Warn(i.logger).Log("msg", "Invalid type for both flavor original_name and flavor id, expected string") + i.logger.Warn("Invalid type for both flavor original_name and flavor id, expected string") continue } labels[openstackLabelInstanceFlavor] = model.LabelValue(flavorID) @@ -169,36 +207,34 @@ func (i *InstanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, labels[openstackLabelTagPrefix+model.LabelName(name)] = model.LabelValue(v) } for pool, address := range s.Addresses { - md, ok := address.([]interface{}) + md, ok := address.([]any) if !ok { - level.Warn(i.logger).Log("msg", "Invalid type for address, expected array") + i.logger.Warn("Invalid type for address, expected array") continue } if len(md) == 0 { - level.Debug(i.logger).Log("msg", "Got no IP address", "instance", s.ID) + i.logger.Debug("Got no IP address", "instance", s.ID) continue } for _, address := range md { - md1, ok := address.(map[string]interface{}) + md1, ok := address.(map[string]any) if !ok { - level.Warn(i.logger).Log("msg", "Invalid type for address, expected dict") + i.logger.Warn("Invalid type for address, expected dict") continue } addr, ok := md1["addr"].(string) if !ok { - level.Warn(i.logger).Log("msg", "Invalid type for address, expected string") + i.logger.Warn("Invalid type for address, expected string") continue } if _, ok := floatingIPPresent[addr]; ok { continue } lbls := make(model.LabelSet, len(labels)) - for k, v := range labels { - lbls[k] = v - } + maps.Copy(lbls, labels) lbls[openstackLabelAddressPool] = model.LabelValue(pool) lbls[openstackLabelPrivateIP] = model.LabelValue(addr) - if val, ok := floatingIPList[floatingIPKey{id: s.ID, fixed: addr}]; ok { + if val, ok := floatingIPList[floatingIPKey{deviceID: s.ID, fixed: addr}]; ok { lbls[openstackLabelPublicIP] = model.LabelValue(val) } addr = net.JoinHostPort(addr, strconv.Itoa(i.port)) diff --git a/discovery/openstack/instance_test.go b/discovery/openstack/instance_test.go index 2b5ac1b89eb..aa202cddffa 100644 --- a/discovery/openstack/instance_test.go +++ b/discovery/openstack/instance_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -32,6 +32,7 @@ func (s *OpenstackSDInstanceTestSuite) SetupTest(t *testing.T) { s.Mock.HandleServerListSuccessfully() s.Mock.HandleFloatingIPListSuccessfully() + s.Mock.HandlePortsListSuccessfully() s.Mock.HandleVersionsSuccessfully() s.Mock.HandleAuthSuccessfully() @@ -66,7 +67,7 @@ func TestOpenstackSDInstanceRefresh(t *testing.T) { tg := tgs[0] require.NotNil(t, tg) require.NotNil(t, tg.Targets) - require.Len(t, tg.Targets, 4) + require.Len(t, tg.Targets, 6) for i, lbls := range []model.LabelSet{ { @@ -119,6 +120,31 @@ func TestOpenstackSDInstanceRefresh(t *testing.T) { "__meta_openstack_project_id": model.LabelValue("fcad67a6189847c4aecfa3c81a05783b"), "__meta_openstack_user_id": model.LabelValue("9349aff8be7545ac9d2f1d00999a23cd"), }, + { + "__address__": model.LabelValue("10.0.0.33:0"), + "__meta_openstack_instance_flavor": model.LabelValue("m1.small"), + "__meta_openstack_instance_id": model.LabelValue("87caf8ed-d92a-41f6-9dcd-d1399e39899f"), + "__meta_openstack_instance_status": model.LabelValue("ACTIVE"), + "__meta_openstack_instance_name": model.LabelValue("merp-project2"), + "__meta_openstack_private_ip": model.LabelValue("10.0.0.33"), + "__meta_openstack_address_pool": model.LabelValue("private"), + "__meta_openstack_tag_env": model.LabelValue("prod"), + "__meta_openstack_project_id": model.LabelValue("b78fef2305934dbbbeb9a10b4c326f7a"), + "__meta_openstack_user_id": model.LabelValue("9349aff8be7545ac9d2f1d00999a23cd"), + }, + { + "__address__": model.LabelValue("10.0.0.34:0"), + "__meta_openstack_instance_flavor": model.LabelValue("m1.small"), + "__meta_openstack_instance_id": model.LabelValue("87caf8ed-d92a-41f6-9dcd-d1399e39899f"), + "__meta_openstack_instance_status": model.LabelValue("ACTIVE"), + "__meta_openstack_instance_name": model.LabelValue("merp-project2"), + "__meta_openstack_private_ip": model.LabelValue("10.0.0.34"), + "__meta_openstack_address_pool": model.LabelValue("private"), + "__meta_openstack_tag_env": model.LabelValue("prod"), + "__meta_openstack_public_ip": model.LabelValue("10.10.10.24"), + "__meta_openstack_project_id": model.LabelValue("b78fef2305934dbbbeb9a10b4c326f7a"), + "__meta_openstack_user_id": model.LabelValue("9349aff8be7545ac9d2f1d00999a23cd"), + }, } { t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) { require.Equal(t, lbls, tg.Targets[i]) @@ -134,6 +160,5 @@ func TestOpenstackSDInstanceRefreshWithDoneContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() _, err := hypervisor.refresh(ctx) - require.Error(t, err) - require.Contains(t, err.Error(), context.Canceled.Error(), "%q doesn't contain %q", err, context.Canceled) + require.ErrorContains(t, err, context.Canceled.Error(), "%q doesn't contain %q", err, context.Canceled) } diff --git a/discovery/openstack/loadbalancer.go b/discovery/openstack/loadbalancer.go new file mode 100644 index 00000000000..3b2def0d6ac --- /dev/null +++ b/discovery/openstack/loadbalancer.go @@ -0,0 +1,193 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openstack + +import ( + "context" + "fmt" + "log/slog" + "net" + "strconv" + "strings" + + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack" + "github.com/gophercloud/gophercloud/v2/openstack/loadbalancer/v2/listeners" + "github.com/gophercloud/gophercloud/v2/openstack/loadbalancer/v2/loadbalancers" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + openstackLabelLoadBalancerID = openstackLabelPrefix + "loadbalancer_id" + openstackLabelLoadBalancerName = openstackLabelPrefix + "loadbalancer_name" + openstackLabelLoadBalancerOperatingStatus = openstackLabelPrefix + "loadbalancer_operating_status" + openstackLabelLoadBalancerProvisioningStatus = openstackLabelPrefix + "loadbalancer_provisioning_status" + openstackLabelLoadBalancerAvailabilityZone = openstackLabelPrefix + "loadbalancer_availability_zone" + openstackLabelLoadBalancerFloatingIP = openstackLabelPrefix + "loadbalancer_floating_ip" + openstackLabelLoadBalancerVIP = openstackLabelPrefix + "loadbalancer_vip" + openstackLabelLoadBalancerProvider = openstackLabelPrefix + "loadbalancer_provider" + openstackLabelLoadBalancerTags = openstackLabelPrefix + "loadbalancer_tags" +) + +// LoadBalancerDiscovery discovers OpenStack load balancers. +type LoadBalancerDiscovery struct { + provider *gophercloud.ProviderClient + authOpts *gophercloud.AuthOptions + region string + logger *slog.Logger + availability gophercloud.Availability +} + +// NewLoadBalancerDiscovery returns a new loadbalancer discovery. +func newLoadBalancerDiscovery(provider *gophercloud.ProviderClient, opts *gophercloud.AuthOptions, + region string, availability gophercloud.Availability, l *slog.Logger, +) *LoadBalancerDiscovery { + if l == nil { + l = promslog.NewNopLogger() + } + return &LoadBalancerDiscovery{ + provider: provider, authOpts: opts, + region: region, availability: availability, logger: l, + } +} + +func (i *LoadBalancerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + err := openstack.Authenticate(ctx, i.provider, *i.authOpts) + if err != nil { + return nil, fmt.Errorf("could not authenticate to OpenStack: %w", err) + } + + client, err := openstack.NewLoadBalancerV2(i.provider, gophercloud.EndpointOpts{ + Region: i.region, Availability: i.availability, + }) + if err != nil { + return nil, fmt.Errorf("could not create OpenStack load balancer session: %w", err) + } + + networkClient, err := openstack.NewNetworkV2(i.provider, gophercloud.EndpointOpts{ + Region: i.region, Availability: i.availability, + }) + if err != nil { + return nil, fmt.Errorf("could not create OpenStack network session: %w", err) + } + + allPages, err := loadbalancers.List(client, loadbalancers.ListOpts{}).AllPages(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list load balancers: %w", err) + } + + allLBs, err := loadbalancers.ExtractLoadBalancers(allPages) + if err != nil { + return nil, fmt.Errorf("failed to extract load balancers: %w", err) + } + + // Fetch all listeners in one API call + listenerPages, err := listeners.List(client, listeners.ListOpts{}).AllPages(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list all listeners: %w", err) + } + + allListeners, err := listeners.ExtractListeners(listenerPages) + if err != nil { + return nil, fmt.Errorf("failed to extract all listeners: %w", err) + } + + // Create a map to group listeners by Load Balancer ID + listenerMap := make(map[string][]listeners.Listener) + for _, listener := range allListeners { + // Iterate through each associated Load Balancer ID in the Loadbalancers array + for _, lb := range listener.Loadbalancers { + listenerMap[lb.ID] = append(listenerMap[lb.ID], listener) + } + } + + // Fetch all floating IPs + fipPages, err := floatingips.List(networkClient, floatingips.ListOpts{}).AllPages(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list floating IPs: %w", err) + } + + allFIPs, err := floatingips.ExtractFloatingIPs(fipPages) + if err != nil { + return nil, fmt.Errorf("failed to extract floating IPs: %w", err) + } + + // Create a map to associate floating IPs with their resource IDs + fipMap := make(map[string]string) // Key: LoadBalancerID/PortID, Value: Floating IP + for _, fip := range allFIPs { + if fip.PortID != "" { + fipMap[fip.PortID] = fip.FloatingIP + } + } + + tg := &targetgroup.Group{ + Source: "OS_" + i.region, + } + + for _, lb := range allLBs { + // Retrieve listeners for this load balancer from the map + lbListeners, exists := listenerMap[lb.ID] + if !exists || len(lbListeners) == 0 { + i.logger.Debug("Got no listener", "loadbalancer", lb.ID) + continue + } + + // Variable to store the port of the first PROMETHEUS listener + var listenerPort int + hasPrometheusListener := false + + // Check if any listener has the PROMETHEUS protocol + for _, listener := range lbListeners { + if listener.Protocol == "PROMETHEUS" { + hasPrometheusListener = true + listenerPort = listener.ProtocolPort + break + } + } + + // Skip LBs without PROMETHEUS listener protocol + if !hasPrometheusListener { + i.logger.Debug("Got no PROMETHEUS listener", "loadbalancer", lb.ID) + continue + } + + labels := model.LabelSet{} + addr := net.JoinHostPort(lb.VipAddress, strconv.Itoa(listenerPort)) + labels[model.AddressLabel] = model.LabelValue(addr) + labels[openstackLabelLoadBalancerID] = model.LabelValue(lb.ID) + labels[openstackLabelLoadBalancerName] = model.LabelValue(lb.Name) + labels[openstackLabelLoadBalancerOperatingStatus] = model.LabelValue(lb.OperatingStatus) + labels[openstackLabelLoadBalancerProvisioningStatus] = model.LabelValue(lb.ProvisioningStatus) + labels[openstackLabelLoadBalancerAvailabilityZone] = model.LabelValue(lb.AvailabilityZone) + labels[openstackLabelLoadBalancerVIP] = model.LabelValue(lb.VipAddress) + labels[openstackLabelLoadBalancerProvider] = model.LabelValue(lb.Provider) + labels[openstackLabelProjectID] = model.LabelValue(lb.ProjectID) + + if len(lb.Tags) > 0 { + labels[openstackLabelLoadBalancerTags] = model.LabelValue(strings.Join(lb.Tags, ",")) + } + + if floatingIP, exists := fipMap[lb.VipPortID]; exists { + labels[openstackLabelLoadBalancerFloatingIP] = model.LabelValue(floatingIP) + } + + tg.Targets = append(tg.Targets, labels) + } + + return []*targetgroup.Group{tg}, nil +} diff --git a/discovery/openstack/loadbalancer_test.go b/discovery/openstack/loadbalancer_test.go new file mode 100644 index 00000000000..68be323a5ac --- /dev/null +++ b/discovery/openstack/loadbalancer_test.go @@ -0,0 +1,137 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openstack + +import ( + "context" + "fmt" + "testing" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" +) + +type OpenstackSDLoadBalancerTestSuite struct { + Mock *SDMock +} + +func (s *OpenstackSDLoadBalancerTestSuite) SetupTest(t *testing.T) { + s.Mock = NewSDMock(t) + s.Mock.Setup() + + s.Mock.HandleLoadBalancerListSuccessfully() + s.Mock.HandleListenersListSuccessfully() + s.Mock.HandleFloatingIPListSuccessfully() + + s.Mock.HandleVersionsSuccessfully() + s.Mock.HandleAuthSuccessfully() +} + +func (s *OpenstackSDLoadBalancerTestSuite) openstackAuthSuccess() (refresher, error) { + conf := SDConfig{ + IdentityEndpoint: s.Mock.Endpoint(), + Password: "test", + Username: "test", + DomainName: "12345", + Region: "RegionOne", + Role: "loadbalancer", + } + return newRefresher(&conf, nil) +} + +func TestOpenstackSDLoadBalancerRefresh(t *testing.T) { + mock := &OpenstackSDLoadBalancerTestSuite{} + mock.SetupTest(t) + + instance, err := mock.openstackAuthSuccess() + require.NoError(t, err) + + ctx := context.Background() + tgs, err := instance.refresh(ctx) + + require.NoError(t, err) + require.Len(t, tgs, 1) + + tg := tgs[0] + require.NotNil(t, tg) + require.NotNil(t, tg.Targets) + require.Len(t, tg.Targets, 4) + + for i, lbls := range []model.LabelSet{ + { + "__address__": model.LabelValue("10.0.0.32:9273"), + "__meta_openstack_loadbalancer_id": model.LabelValue("ef079b0c-e610-4dfb-b1aa-b49f07ac48e5"), + "__meta_openstack_loadbalancer_name": model.LabelValue("lb1"), + "__meta_openstack_loadbalancer_operating_status": model.LabelValue("ONLINE"), + "__meta_openstack_loadbalancer_provisioning_status": model.LabelValue("ACTIVE"), + "__meta_openstack_loadbalancer_availability_zone": model.LabelValue("az1"), + "__meta_openstack_loadbalancer_floating_ip": model.LabelValue("192.168.1.2"), + "__meta_openstack_loadbalancer_vip": model.LabelValue("10.0.0.32"), + "__meta_openstack_loadbalancer_provider": model.LabelValue("amphora"), + "__meta_openstack_loadbalancer_tags": model.LabelValue("tag1,tag2"), + "__meta_openstack_project_id": model.LabelValue("fcad67a6189847c4aecfa3c81a05783b"), + }, + { + "__address__": model.LabelValue("10.0.2.78:8080"), + "__meta_openstack_loadbalancer_id": model.LabelValue("d92c471e-8d3e-4b9f-b2b5-9c72a9e3ef54"), + "__meta_openstack_loadbalancer_name": model.LabelValue("lb3"), + "__meta_openstack_loadbalancer_operating_status": model.LabelValue("ONLINE"), + "__meta_openstack_loadbalancer_provisioning_status": model.LabelValue("ACTIVE"), + "__meta_openstack_loadbalancer_availability_zone": model.LabelValue("az3"), + "__meta_openstack_loadbalancer_floating_ip": model.LabelValue("192.168.3.4"), + "__meta_openstack_loadbalancer_vip": model.LabelValue("10.0.2.78"), + "__meta_openstack_loadbalancer_provider": model.LabelValue("amphora"), + "__meta_openstack_loadbalancer_tags": model.LabelValue("tag5,tag6"), + "__meta_openstack_project_id": model.LabelValue("ac57f03dba1a4fdebff3e67201bc7a85"), + }, + { + "__address__": model.LabelValue("10.0.3.99:9090"), + "__meta_openstack_loadbalancer_id": model.LabelValue("f5c7e918-df38-4a5a-a7d4-d9c27ab2cf67"), + "__meta_openstack_loadbalancer_name": model.LabelValue("lb4"), + "__meta_openstack_loadbalancer_operating_status": model.LabelValue("ONLINE"), + "__meta_openstack_loadbalancer_provisioning_status": model.LabelValue("ACTIVE"), + "__meta_openstack_loadbalancer_availability_zone": model.LabelValue("az1"), + "__meta_openstack_loadbalancer_floating_ip": model.LabelValue("192.168.4.5"), + "__meta_openstack_loadbalancer_vip": model.LabelValue("10.0.3.99"), + "__meta_openstack_loadbalancer_provider": model.LabelValue("amphora"), + "__meta_openstack_project_id": model.LabelValue("fa8c372dfe4d4c92b0c4e3a2d9b3c9fa"), + }, + { + "__address__": model.LabelValue("10.0.4.88:9876"), + "__meta_openstack_loadbalancer_id": model.LabelValue("e83a6d92-7a3e-4567-94b3-20c83b32a75e"), + "__meta_openstack_loadbalancer_name": model.LabelValue("lb5"), + "__meta_openstack_loadbalancer_operating_status": model.LabelValue("ONLINE"), + "__meta_openstack_loadbalancer_provisioning_status": model.LabelValue("ACTIVE"), + "__meta_openstack_loadbalancer_availability_zone": model.LabelValue("az4"), + "__meta_openstack_loadbalancer_vip": model.LabelValue("10.0.4.88"), + "__meta_openstack_loadbalancer_provider": model.LabelValue("amphora"), + "__meta_openstack_project_id": model.LabelValue("a5d3b2e1e6f34cd9a5f7c2f01a6b8e29"), + }, + } { + t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) { + require.Equal(t, lbls, tg.Targets[i]) + }) + } +} + +func TestOpenstackSDLoadBalancerRefreshWithDoneContext(t *testing.T) { + mock := &OpenstackSDLoadBalancerTestSuite{} + mock.SetupTest(t) + + loadbalancer, _ := mock.openstackAuthSuccess() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := loadbalancer.refresh(ctx) + require.ErrorContains(t, err, context.Canceled.Error(), "%q doesn't contain %q", err, context.Canceled) +} diff --git a/discovery/openstack/metrics.go b/discovery/openstack/metrics.go index a64c1a6732b..01e7ab3add9 100644 --- a/discovery/openstack/metrics.go +++ b/discovery/openstack/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type openstackMetrics struct { var _ discovery.DiscovererMetrics = (*openstackMetrics)(nil) // Register implements discovery.DiscovererMetrics. -func (m *openstackMetrics) Register() error { +func (*openstackMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *openstackMetrics) Unregister() {} +func (*openstackMetrics) Unregister() {} diff --git a/discovery/openstack/mock_test.go b/discovery/openstack/mock_test.go index 4518f41166e..c44dadfbc08 100644 --- a/discovery/openstack/mock_test.go +++ b/discovery/openstack/mock_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -62,7 +62,7 @@ func testHeader(t *testing.T, r *http.Request, header, expected string) { // HandleVersionsSuccessfully mocks version call. func (m *SDMock) HandleVersionsSuccessfully() { - m.Mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + m.Mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, ` { "versions": { @@ -90,7 +90,7 @@ func (m *SDMock) HandleVersionsSuccessfully() { // HandleAuthSuccessfully mocks auth call. func (m *SDMock) HandleAuthSuccessfully() { - m.Mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + m.Mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, _ *http.Request) { w.Header().Add("X-Subject-Token", tokenID) w.WriteHeader(http.StatusCreated) @@ -124,7 +124,7 @@ func (m *SDMock) HandleAuthSuccessfully() { "type": "identity", "name": "keystone" }, - { + { "endpoints": [ { "id": "e2ffee808abc4a60916715b1d4b489dd", @@ -136,8 +136,33 @@ func (m *SDMock) HandleAuthSuccessfully() { ], "id": "b7f2a5b1a019459cb956e43a8cb41e31", "type": "compute" + }, + { + "endpoints": [ + { + "id": "5448e46679564d7d95466c2bef54c296", + "interface": "public", + "region": "RegionOne", + "region_id": "RegionOne", + "url": "%s" + } + ], + "id": "589f3d99a3d94f5f871e9f5cf206d2e8", + "type": "network" + }, + { + "endpoints": [ + { + "id": "39dc322ce86c1234b4f06c2eeae0841b", + "interface": "public", + "region": "RegionOne", + "region_id": "RegionOne", + "url": "%s" + } + ], + "id": "26968f704a68417bbddd29508455ff90", + "type": "load-balancer" } - ], "expires_at": "2013-02-27T18:30:59.999999Z", "is_domain": false, @@ -174,7 +199,7 @@ func (m *SDMock) HandleAuthSuccessfully() { } } } - `, m.Endpoint()) + `, m.Endpoint(), m.Endpoint(), m.Endpoint()) }) } @@ -461,82 +486,159 @@ const serverListBody = ` "metadata": {} }, { - "status": "ACTIVE", - "updated": "2014-09-25T13:04:49Z", - "hostId": "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", - "OS-EXT-SRV-ATTR:host": "devstack", - "addresses": { - "private": [ - { - "version": 4, - "addr": "10.0.0.33", - "OS-EXT-IPS:type": "fixed" - }, + "status": "ACTIVE", + "updated": "2014-09-25T13:04:49Z", + "hostId": "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", + "OS-EXT-SRV-ATTR:host": "devstack", + "addresses": { + "private": [ + { + "version": 4, + "addr": "10.0.0.33", + "OS-EXT-IPS:type": "fixed" + }, + { + "version": 4, + "addr": "10.0.0.34", + "OS-EXT-IPS:type": "fixed" + }, + { + "version": 4, + "addr": "10.10.10.4", + "OS-EXT-IPS:type": "floating" + } + ] + }, + "links": [ { - "version": 4, - "addr": "10.0.0.34", - "OS-EXT-IPS:type": "fixed" + "href": "http://104.130.131.164:8774/v2/fcad67a6189847c4aecfa3c81a05783b/servers/9e5476bd-a4ec-4653-93d6-72c93aa682ba", + "rel": "self" }, { - "version": 4, - "addr": "10.10.10.4", - "OS-EXT-IPS:type": "floating" + "href": "http://104.130.131.164:8774/fcad67a6189847c4aecfa3c81a05783b/servers/9e5476bd-a4ec-4653-93d6-72c93aa682ba", + "rel": "bookmark" + } + ], + "key_name": null, + "image": "", + "OS-EXT-STS:task_state": null, + "OS-EXT-STS:vm_state": "active", + "OS-EXT-SRV-ATTR:instance_name": "instance-0000001d", + "OS-SRV-USG:launched_at": "2014-09-25T13:04:49.000000", + "OS-EXT-SRV-ATTR:hypervisor_hostname": "devstack", + "flavor": { + "vcpus": 2, + "ram": 4096, + "disk": 0, + "ephemeral": 0, + "swap": 0, + "original_name": "m1.small", + "extra_specs": { + "aggregate_instance_extra_specs:general": "true", + "hw:mem_page_size": "large", + "hw:vif_multiqueue_enabled": "true" } - ] - }, - "links": [ - { - "href": "http://104.130.131.164:8774/v2/fcad67a6189847c4aecfa3c81a05783b/servers/9e5476bd-a4ec-4653-93d6-72c93aa682ba", - "rel": "self" }, - { - "href": "http://104.130.131.164:8774/fcad67a6189847c4aecfa3c81a05783b/servers/9e5476bd-a4ec-4653-93d6-72c93aa682ba", - "rel": "bookmark" - } - ], - "key_name": null, - "image": "", - "OS-EXT-STS:task_state": null, - "OS-EXT-STS:vm_state": "active", - "OS-EXT-SRV-ATTR:instance_name": "instance-0000001d", - "OS-SRV-USG:launched_at": "2014-09-25T13:04:49.000000", - "OS-EXT-SRV-ATTR:hypervisor_hostname": "devstack", - "flavor": { - "vcpus": 2, - "ram": 4096, - "disk": 0, - "ephemeral": 0, - "swap": 0, - "original_name": "m1.small", - "extra_specs": { - "aggregate_instance_extra_specs:general": "true", - "hw:mem_page_size": "large", - "hw:vif_multiqueue_enabled": "true" + "id": "9e5476bd-a4ec-4653-93d6-72c93aa682bb", + "security_groups": [ + { + "name": "default" + } + ], + "OS-SRV-USG:terminated_at": null, + "OS-EXT-AZ:availability_zone": "nova", + "user_id": "9349aff8be7545ac9d2f1d00999a23cd", + "name": "merp", + "created": "2014-09-25T13:04:41Z", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "OS-DCF:diskConfig": "MANUAL", + "os-extended-volumes:volumes_attached": [], + "accessIPv4": "", + "accessIPv6": "", + "progress": 0, + "OS-EXT-STS:power_state": 1, + "config_drive": "", + "metadata": { + "env": "prod" } }, - "id": "9e5476bd-a4ec-4653-93d6-72c93aa682bb", - "security_groups": [ - { - "name": "default" + { + "status": "ACTIVE", + "updated": "2014-09-25T13:04:49Z", + "hostId": "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", + "OS-EXT-SRV-ATTR:host": "devstack", + "addresses": { + "private": [ + { + "version": 4, + "addr": "10.0.0.33", + "OS-EXT-IPS:type": "fixed" + }, + { + "version": 4, + "addr": "10.0.0.34", + "OS-EXT-IPS:type": "fixed" + }, + { + "version": 4, + "addr": "10.10.10.24", + "OS-EXT-IPS:type": "floating" + } + ] + }, + "links": [ + { + "href": "http://104.130.131.164:8774/v2/b78fef2305934dbbbeb9a10b4c326f7a/servers/9e5476bd-a4ec-4653-93d6-72c93aa682ba", + "rel": "self" + }, + { + "href": "http://104.130.131.164:8774/b78fef2305934dbbbeb9a10b4c326f7a/servers/9e5476bd-a4ec-4653-93d6-72c93aa682ba", + "rel": "bookmark" + } + ], + "key_name": null, + "image": "", + "OS-EXT-STS:task_state": null, + "OS-EXT-STS:vm_state": "active", + "OS-EXT-SRV-ATTR:instance_name": "instance-0000002d", + "OS-SRV-USG:launched_at": "2014-09-25T13:04:49.000000", + "OS-EXT-SRV-ATTR:hypervisor_hostname": "devstack", + "flavor": { + "vcpus": 2, + "ram": 4096, + "disk": 0, + "ephemeral": 0, + "swap": 0, + "original_name": "m1.small", + "extra_specs": { + "aggregate_instance_extra_specs:general": "true", + "hw:mem_page_size": "large", + "hw:vif_multiqueue_enabled": "true" + } + }, + "id": "87caf8ed-d92a-41f6-9dcd-d1399e39899f", + "security_groups": [ + { + "name": "default" + } + ], + "OS-SRV-USG:terminated_at": null, + "OS-EXT-AZ:availability_zone": "nova", + "user_id": "9349aff8be7545ac9d2f1d00999a23cd", + "name": "merp-project2", + "created": "2014-09-25T13:04:41Z", + "tenant_id": "b78fef2305934dbbbeb9a10b4c326f7a", + "OS-DCF:diskConfig": "MANUAL", + "os-extended-volumes:volumes_attached": [], + "accessIPv4": "", + "accessIPv6": "", + "progress": 0, + "OS-EXT-STS:power_state": 1, + "config_drive": "", + "metadata": { + "env": "prod" } - ], - "OS-SRV-USG:terminated_at": null, - "OS-EXT-AZ:availability_zone": "nova", - "user_id": "9349aff8be7545ac9d2f1d00999a23cd", - "name": "merp", - "created": "2014-09-25T13:04:41Z", - "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", - "OS-DCF:diskConfig": "MANUAL", - "os-extended-volumes:volumes_attached": [], - "accessIPv4": "", - "accessIPv6": "", - "progress": 0, - "OS-EXT-STS:power_state": 1, - "config_drive": "", - "metadata": { - "env": "prod" } - } ] } ` @@ -554,35 +656,139 @@ func (m *SDMock) HandleServerListSuccessfully() { const listOutput = ` { - "floating_ips": [ - { - "fixed_ip": null, - "id": "1", - "instance_id": null, - "ip": "10.10.10.1", - "pool": "nova" - }, - { - "fixed_ip": "10.0.0.32", - "id": "2", - "instance_id": "ef079b0c-e610-4dfb-b1aa-b49f07ac48e5", - "ip": "10.10.10.2", - "pool": "nova" - }, - { - "fixed_ip": "10.0.0.34", - "id": "3", - "instance_id": "9e5476bd-a4ec-4653-93d6-72c93aa682bb", - "ip": "10.10.10.4", - "pool": "nova" - } - ] + "floatingips": [ + { + "id": "03a77860-ae03-46c4-b502-caea11467a79", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "floating_ip_address": "10.10.10.1", + "floating_network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "router_id": "f03af93b-4e8f-4f55-adcf-a0317782ede2", + "port_id": "d5597901-48c8-4a69-a041-cfc5be158a04", + "fixed_ip_address": null, + "status": "ACTIVE", + "description": "", + "dns_domain": "", + "dns_name": "", + "port_forwardings": [], + "tags": [], + "created_at": "2023-08-30T16:30:27Z", + "updated_at": "2023-08-30T16:30:28Z" + }, + { + "id": "03e28c79-5a4c-491e-a4fe-3ff6bba830c6", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "floating_ip_address": "10.10.10.2", + "floating_network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "router_id": "f03af93b-4e8f-4f55-adcf-a0317782ede2", + "port_id": "4a45b012-0478-484d-8cf3-c8abdb194d08", + "fixed_ip_address": "10.0.0.32", + "status": "ACTIVE", + "description": "", + "dns_domain": "", + "dns_name": "", + "port_forwardings": [], + "tags": [], + "created_at": "2023-09-06T15:45:36Z", + "updated_at": "2023-09-06T15:45:36Z" + }, + { + "id": "087fcdd2-1d13-4f72-9c0e-c759e796d558", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "floating_ip_address": "10.10.10.4", + "floating_network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "router_id": "f03af93b-4e8f-4f55-adcf-a0317782ede2", + "port_id": "a0e244e8-7910-4427-b8d1-20470cad4f8a", + "fixed_ip_address": "10.0.0.34", + "status": "ACTIVE", + "description": "", + "dns_domain": "", + "dns_name": "", + "port_forwardings": [], + "tags": [], + "created_at": "2024-01-24T13:30:50Z", + "updated_at": "2024-01-24T13:30:51Z" + }, + { + "id": "b23df91a-a74a-4f75-b252-750aff4a5a0c", + "tenant_id": "b78fef2305934dbbbeb9a10b4c326f7a", + "floating_ip_address": "10.10.10.24", + "floating_network_id": "b19ff5bc-a49a-46cc-8d14-ca5f1e94791f", + "router_id": "65a5e5af-17f0-4124-9a81-c08b44f5b8a7", + "port_id": "b926ab68-ec54-46d8-8c50-1c07aafd5ae9", + "fixed_ip_address": "10.0.0.34", + "status": "ACTIVE", + "description": "", + "dns_domain": "", + "dns_name": "", + "port_forwardings": [], + "tags": [], + "created_at": "2024-01-24T13:30:50Z", + "updated_at": "2024-01-24T13:30:51Z" + }, + { + "id": "fea7332d-9027-4cf9-bf62-c3c4c6ebaf84", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "floating_ip_address": "192.168.1.2", + "floating_network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "router_id": "f03af93b-4e8f-4f55-adcf-a0317782ede2", + "port_id": "b47c39f5-238d-4b17-ae87-9b5d19af8a2e", + "fixed_ip_address": "10.0.0.32", + "status": "ACTIVE", + "description": "", + "dns_domain": "", + "dns_name": "", + "port_forwardings": [], + "tags": [], + "created_at": "2023-08-30T15:11:37Z", + "updated_at": "2023-08-30T15:11:38Z", + "revision_number": 1, + "project_id": "fcad67a6189847c4aecfa3c81a05783b" + }, + { + "id": "febb9554-cf83-4f9b-94d9-1b3c34be357f", + "tenant_id": "ac57f03dba1a4fdebff3e67201bc7a85", + "floating_ip_address": "192.168.3.4", + "floating_network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "router_id": "f03af93b-4e8f-4f55-adcf-a0317782ede2", + "port_id": "c83b6e12-4e5d-4673-a4b3-5bc72a7f3ef9", + "fixed_ip_address": "10.0.2.78", + "status": "ACTIVE", + "description": "", + "dns_domain": "", + "dns_name": "", + "port_forwardings": [], + "tags": [], + "created_at": "2023-08-30T15:11:37Z", + "updated_at": "2023-08-30T15:11:38Z", + "revision_number": 1, + "project_id": "ac57f03dba1a4fdebff3e67201bc7a85" + }, + { + "id": "febb9554-cf83-4f9b-94d9-1b3c34be357f", + "tenant_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa", + "floating_ip_address": "192.168.4.5", + "floating_network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "router_id": "f03af93b-4e8f-4f55-adcf-a0317782ede2", + "port_id": "f9e8b6e12-7e4d-4963-a5b3-6cd82a7f3ff6", + "fixed_ip_address": "10.0.3.99", + "status": "ACTIVE", + "description": "", + "dns_domain": "", + "dns_name": "", + "port_forwardings": [], + "tags": [], + "created_at": "2023-08-30T15:11:37Z", + "updated_at": "2023-08-30T15:11:38Z", + "revision_number": 1, + "project_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa" + } + ] } ` // HandleFloatingIPListSuccessfully mocks floating ips call. func (m *SDMock) HandleFloatingIPListSuccessfully() { - m.Mux.HandleFunc("/os-floating-ips", func(w http.ResponseWriter, r *http.Request) { + m.Mux.HandleFunc("/v2.0/floatingips", func(w http.ResponseWriter, r *http.Request) { testMethod(m.t, r, http.MethodGet) testHeader(m.t, r, "X-Auth-Token", tokenID) @@ -590,3 +796,608 @@ func (m *SDMock) HandleFloatingIPListSuccessfully() { fmt.Fprint(w, listOutput) }) } + +const portsListBody = ` +{ + "ports": [ + { + "id": "d5597901-48c8-4a69-a041-cfc5be158a04", + "name": "", + "network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "mac_address": "", + "admin_state_up": true, + "status": "DOWN", + "device_id": "", + "device_owner": "", + "fixed_ips": [], + "allowed_address_pairs": [], + "extra_dhcp_opts": [], + "security_groups": [], + "description": "", + "binding:vnic_type": "normal", + "port_security_enabled": true, + "dns_name": "", + "dns_assignment": [], + "dns_domain": "", + "tags": [], + "created_at": "2023-08-30T16:30:27Z", + "updated_at": "2023-08-30T16:30:28Z", + "revision_number": 0, + "project_id": "fcad67a6189847c4aecfa3c81a05783b" + }, + { + "id": "4a45b012-0478-484d-8cf3-c8abdb194d08", + "name": "ovn-lb-vip-0980c8de-58c3-481d-89e3-ed81f44286c0", + "network_id": "03200a39-b399-44f3-a778-6dbb93343a31", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "mac_address": "fa:16:3e:23:12:a3", + "admin_state_up": true, + "status": "ACTIVE", + "device_id": "ef079b0c-e610-4dfb-b1aa-b49f07ac48e5", + "device_owner": "", + "fixed_ips": [ + { + "subnet_id": "", + "ip_address": "10.10.10.2" + } + ], + "allowed_address_pairs": [], + "extra_dhcp_opts": [], + "security_groups": [], + "description": "", + "binding:vnic_type": "normal", + "port_security_enabled": true, + "dns_name": "", + "dns_assignment": [], + "dns_domain": "", + "tags": [], + "created_at": "2023-09-06T15:45:36Z", + "updated_at": "2023-09-06T15:45:36Z", + "revision_number": 0, + "project_id": "fcad67a6189847c4aecfa3c81a05783b" + }, + { + "id": "a0e244e8-7910-4427-b8d1-20470cad4f8a", + "name": "ovn-lb-vip-26c0ccb1-3036-4345-99e8-d8f34a8ba6b2", + "network_id": "03200a39-b399-44f3-a778-6dbb93343a31", + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b", + "mac_address": "fa:16:3e:5f:43:10", + "admin_state_up": true, + "status": "ACTIVE", + "device_id": "9e5476bd-a4ec-4653-93d6-72c93aa682bb", + "device_owner": "", + "fixed_ips": [ + { + "subnet_id": "", + "ip_address": "10.10.10.4" + } + ], + "allowed_address_pairs": [], + "extra_dhcp_opts": [], + "security_groups": [], + "description": "", + "binding:vnic_type": "normal", + "port_security_enabled": true, + "dns_name": "", + "dns_assignment": [], + "dns_domain": "", + "tags": [], + "created_at": "2024-01-24T13:30:50Z", + "updated_at": "2024-01-24T13:30:51Z", + "revision_number": 0, + "project_id": "fcad67a6189847c4aecfa3c81a05783b" + }, + { + "id": "b926ab68-ec54-46d8-8c50-1c07aafd5ae9", + "name": "dummy-port", + "network_id": "03200a39-b399-44f3-a778-6dbb93343a31", + "tenant_id": "b78fef2305934dbbbeb9a10b4c326f7a", + "mac_address": "fa:16:3e:5f:12:10", + "admin_state_up": true, + "status": "ACTIVE", + "device_id": "87caf8ed-d92a-41f6-9dcd-d1399e39899f", + "device_owner": "", + "fixed_ips": [ + { + "subnet_id": "", + "ip_address": "10.10.10.24" + } + ], + "allowed_address_pairs": [], + "extra_dhcp_opts": [], + "security_groups": [], + "description": "", + "binding:vnic_type": "normal", + "port_security_enabled": true, + "dns_name": "", + "dns_assignment": [], + "dns_domain": "", + "tags": [], + "created_at": "2024-01-24T13:30:50Z", + "updated_at": "2024-01-24T13:30:51Z", + "revision_number": 0, + "project_id": "b78fef2305934dbbbeb9a10b4c326f7a" + } + ] +} +` + +// HandlePortsListSuccessfully mocks the ports list API. +func (m *SDMock) HandlePortsListSuccessfully() { + m.Mux.HandleFunc("/v2.0/ports", func(w http.ResponseWriter, r *http.Request) { + testMethod(m.t, r, http.MethodGet) + testHeader(m.t, r, "X-Auth-Token", tokenID) + + w.Header().Add("Content-Type", "application/json") + fmt.Fprint(w, portsListBody) + }) +} + +const lbListBody = ` +{ + "loadbalancers": [ + { + "id": "ef079b0c-e610-4dfb-b1aa-b49f07ac48e5", + "name": "lb1", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "project_id": "fcad67a6189847c4aecfa3c81a05783b", + "created_at": "2024-12-01T10:00:00", + "updated_at": "2024-12-01T10:30:00", + "vip_address": "10.0.0.32", + "vip_port_id": "b47c39f5-238d-4b17-ae87-9b5d19af8a2e", + "vip_subnet_id": "14a4c6a5-fe71-4a94-9071-4cd12fb8337f", + "vip_network_id": "d02c4f18-d606-4864-b12a-1c9b39a46be2", + "tags": ["tag1", "tag2"], + "availability_zone": "az1", + "vip_vnic_type": "normal", + "provider": "amphora", + "listeners": [ + { + "id": "c4146b54-febc-4caf-a53f-ed1cab6faba5" + }, + { + "id": "a058d20e-82de-4eff-bb65-5c76a8554435" + } + ], + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b" + }, + { + "id": "d92c471e-8d3e-4b9f-b2b5-9c72a9e3ef54", + "name": "lb3", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "project_id": "ac57f03dba1a4fdebff3e67201bc7a85", + "created_at": "2024-12-01T12:00:00", + "updated_at": "2024-12-01T12:45:00", + "vip_address": "10.0.2.78", + "vip_port_id": "c83b6e12-4e5d-4673-a4b3-5bc72a7f3ef9", + "vip_subnet_id": "36c5e9f6-e7a2-4975-a8c6-3b8e4f93cf45", + "vip_network_id": "g03c6f27-e617-4975-c8f7-4c9f3f94cf68", + "tags": ["tag5", "tag6"], + "availability_zone": "az3", + "vip_vnic_type": "normal", + "provider": "amphora", + "listeners": [ + { + "id": "5b9529a4-6cbf-48f8-a006-d99cbc717da0" + }, + { + "id": "5d26333b-74d1-4b2a-90ab-2b2c0f5a8048" + } + ], + "tenant_id": "ac57f03dba1a4fdebff3e67201bc7a85" + }, + { + "id": "f5c7e918-df38-4a5a-a7d4-d9c27ab2cf67", + "name": "lb4", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "project_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa", + "created_at": "2024-12-01T13:00:00", + "updated_at": "2024-12-01T13:20:00", + "vip_address": "10.0.3.99", + "vip_port_id": "f9e8b6e12-7e4d-4963-a5b3-6cd82a7f3ff6", + "vip_subnet_id": "47d6f8f9-f7b2-4876-a9d8-4e8f4g95df79", + "vip_network_id": "h04d7f38-f718-4876-d9g8-5d8g5h95df89", + "tags": [], + "availability_zone": "az1", + "vip_vnic_type": "normal", + "provider": "amphora", + "listeners": [ + { + "id": "84c87596-1ff0-4f6d-b151-0a78e1f407a3" + }, + { + "id": "fe460a7c-16a9-4984-9fe6-f6e5153ebab1" + } + ], + "tenant_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa" + }, + { + "id": "e83a6d92-7a3e-4567-94b3-20c83b32a75e", + "name": "lb5", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "project_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29", + "created_at": "2024-12-01T11:00:00", + "updated_at": "2024-12-01T11:15:00", + "vip_address": "10.0.4.88", + "vip_port_id": "d83a6d92-7a3e-4567-94b3-20c83b32a75e", + "vip_subnet_id": "25b4d8e5-fe81-4a87-9071-4cc12fb8337f", + "vip_network_id": "f02c5e19-c507-4864-b16e-2b7a39e56be3", + "tags": [], + "availability_zone": "az4", + "vip_vnic_type": "normal", + "provider": "amphora", + "listeners": [ + { + "id": "50902e62-34b8-46b2-9ed4-9053e7ad46dc" + }, + { + "id": "98a867ad-ff07-4880-b05f-32088866a68a" + } + ], + "tenant_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29" + } + ] +} +` + +// HandleLoadBalancerListSuccessfully mocks the load balancer list API. +func (m *SDMock) HandleLoadBalancerListSuccessfully() { + m.Mux.HandleFunc("/v2.0/lbaas/loadbalancers", func(w http.ResponseWriter, r *http.Request) { + testMethod(m.t, r, http.MethodGet) + testHeader(m.t, r, "X-Auth-Token", tokenID) + + w.Header().Add("Content-Type", "application/json") + fmt.Fprint(w, lbListBody) + }) +} + +const listenerListBody = ` +{ + "listeners": [ + { + "id": "c4146b54-febc-4caf-a53f-ed1cab6faba5", + "name": "stats-listener", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "PROMETHEUS", + "protocol_port": 9273, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "fcad67a6189847c4aecfa3c81a05783b", + "default_pool_id": null, + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-08-29T18:05:24", + "updated_at": "2024-12-04T21:21:10", + "loadbalancers": [ + { + "id": "ef079b0c-e610-4dfb-b1aa-b49f07ac48e5" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "fcad67a6189847c4aecfa3c81a05783b" + }, + { + "id": "5b9529a4-6cbf-48f8-a006-d99cbc717da0", + "name": "stats-listener2", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "PROMETHEUS", + "protocol_port": 8080, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "ac57f03dba1a4fdebff3e67201bc7a85", + "default_pool_id": null, + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-08-29T18:05:24", + "updated_at": "2024-12-04T21:21:10", + "loadbalancers": [ + { + "id": "d92c471e-8d3e-4b9f-b2b5-9c72a9e3ef54" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "ac57f03dba1a4fdebff3e67201bc7a85" + }, + { + "id": "84c87596-1ff0-4f6d-b151-0a78e1f407a3", + "name": "stats-listener3", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "PROMETHEUS", + "protocol_port": 9090, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa", + "default_pool_id": null, + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-08-29T18:05:24", + "updated_at": "2024-12-04T21:21:10", + "loadbalancers": [ + { + "id": "f5c7e918-df38-4a5a-a7d4-d9c27ab2cf67" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa" + }, + { + "id": "50902e62-34b8-46b2-9ed4-9053e7ad46dc", + "name": "stats-listener4", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "PROMETHEUS", + "protocol_port": 9876, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29", + "default_pool_id": null, + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-08-29T18:05:24", + "updated_at": "2024-12-04T21:21:10", + "loadbalancers": [ + { + "id": "e83a6d92-7a3e-4567-94b3-20c83b32a75e" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29" + }, + { + "id": "a058d20e-82de-4eff-bb65-5c76a8554435", + "name": "port6443", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "TCP", + "protocol_port": 6443, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29", + "default_pool_id": "5643208b-b691-4b1f-a6b8-356f14903e56", + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-10-02T19:32:48", + "updated_at": "2024-12-04T21:44:34", + "loadbalancers": [ + { + "id": "ef079b0c-e610-4dfb-b1aa-b49f07ac48e5" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29" + }, + { + "id": "5d26333b-74d1-4b2a-90ab-2b2c0f5a8048", + "name": "port6444", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "TCP", + "protocol_port": 6444, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "ac57f03dba1a4fdebff3e67201bc7a85", + "default_pool_id": "5643208b-b691-4b1f-a6b8-356f14903e56", + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-10-02T19:32:48", + "updated_at": "2024-12-04T21:44:34", + "loadbalancers": [ + { + "id": "d92c471e-8d3e-4b9f-b2b5-9c72a9e3ef54" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "ac57f03dba1a4fdebff3e67201bc7a85" + }, + { + "id": "fe460a7c-16a9-4984-9fe6-f6e5153ebab1", + "name": "port6445", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "TCP", + "protocol_port": 6445, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa", + "default_pool_id": "5643208b-b691-4b1f-a6b8-356f14903e56", + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-10-02T19:32:48", + "updated_at": "2024-12-04T21:44:34", + "loadbalancers": [ + { + "id": "f5c7e918-df38-4a5a-a7d4-d9c27ab2cf67" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "fa8c372dfe4d4c92b0c4e3a2d9b3c9fa" + }, + { + "id": "98a867ad-ff07-4880-b05f-32088866a68a", + "name": "port6446", + "description": "", + "provisioning_status": "ACTIVE", + "operating_status": "ONLINE", + "admin_state_up": true, + "protocol": "TCP", + "protocol_port": 6446, + "connection_limit": -1, + "default_tls_container_ref": null, + "sni_container_refs": [], + "project_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29", + "default_pool_id": "5643208b-b691-4b1f-a6b8-356f14903e56", + "l7policies": [], + "insert_headers": {}, + "created_at": "2024-10-02T19:32:48", + "updated_at": "2024-12-04T21:44:34", + "loadbalancers": [ + { + "id": "e83a6d92-7a3e-4567-94b3-20c83b32a75e" + } + ], + "timeout_client_data": 50000, + "timeout_member_connect": 5000, + "timeout_member_data": 50000, + "timeout_tcp_inspect": 0, + "tags": [], + "client_ca_tls_container_ref": null, + "client_authentication": "NONE", + "client_crl_container_ref": null, + "allowed_cidrs": null, + "tls_ciphers": null, + "tls_versions": null, + "alpn_protocols": null, + "hsts_max_age": null, + "hsts_include_subdomains": null, + "hsts_preload": null, + "tenant_id": "a5d3b2e1e6f34cd9a5f7c2f01a6b8e29" + } + ] +} +` + +// HandleListenersListSuccessfully mocks the listeners endpoint. +func (m *SDMock) HandleListenersListSuccessfully() { + m.Mux.HandleFunc("/v2.0/lbaas/listeners", func(w http.ResponseWriter, r *http.Request) { + testMethod(m.t, r, http.MethodGet) + testHeader(m.t, r, "X-Auth-Token", tokenID) + + w.Header().Add("Content-Type", "application/json") + fmt.Fprint(w, listenerListBody) + }) +} diff --git a/discovery/openstack/openstack.go b/discovery/openstack/openstack.go index c98f78788d4..ce365e6cd01 100644 --- a/discovery/openstack/openstack.go +++ b/discovery/openstack/openstack.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,12 +17,12 @@ import ( "context" "errors" "fmt" + "log/slog" "net/http" "time" - "github.com/go-kit/log" - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack" "github.com/mwitkow/go-conntrack" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -67,7 +67,7 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &openstackMetrics{ refreshMetrics: rmi, } @@ -78,7 +78,7 @@ func (*SDConfig) Name() string { return "openstack" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -97,15 +97,18 @@ const ( // OpenStack document reference // https://docs.openstack.org/horizon/pike/user/launch-instances.html OpenStackRoleInstance Role = "instance" + // Openstack document reference + // https://docs.openstack.org/openstacksdk/rocky/user/resources/load_balancer/index.html + OpenStackRoleLoadBalancer Role = "loadbalancer" ) // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *Role) UnmarshalYAML(unmarshal func(any) error) error { if err := unmarshal((*string)(c)); err != nil { return err } switch *c { - case OpenStackRoleHypervisor, OpenStackRoleInstance: + case OpenStackRoleHypervisor, OpenStackRoleInstance, OpenStackRoleLoadBalancer: return nil default: return fmt.Errorf("unknown OpenStack SD role %q", *c) @@ -113,7 +116,7 @@ func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -128,7 +131,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { } if c.Role == "" { - return errors.New("role missing (one of: instance, hypervisor)") + return errors.New("role missing (one of: instance, hypervisor, loadbalancer)") } if c.Region == "" { return errors.New("openstack SD configuration requires a region") @@ -142,20 +145,21 @@ type refresher interface { } // NewDiscovery returns a new OpenStack Discoverer which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, l log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { - m, ok := metrics.(*openstackMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*refresh.Discovery, error) { + m, ok := opts.Metrics.(*openstackMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - r, err := newRefresher(conf, l) + r, err := newRefresher(conf, opts.Logger) if err != nil { return nil, err } return refresh.NewDiscovery( refresh.Options{ - Logger: l, + Logger: opts.Logger, Mech: "openstack", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: r.refresh, MetricsInstantiator: m.refreshMetrics, @@ -163,7 +167,7 @@ func NewDiscovery(conf *SDConfig, l log.Logger, metrics discovery.DiscovererMetr ), nil } -func newRefresher(conf *SDConfig, l log.Logger) (refresher, error) { +func newRefresher(conf *SDConfig, l *slog.Logger) (refresher, error) { var opts gophercloud.AuthOptions if conf.IdentityEndpoint == "" { var err error @@ -211,6 +215,8 @@ func newRefresher(conf *SDConfig, l log.Logger) (refresher, error) { return newHypervisorDiscovery(client, &opts, conf.Port, conf.Region, availability, l), nil case OpenStackRoleInstance: return newInstanceDiscovery(client, &opts, conf.Port, conf.Region, conf.AllTenants, availability, l), nil + case OpenStackRoleLoadBalancer: + return newLoadBalancerDiscovery(client, &opts, conf.Region, availability, l), nil } return nil, errors.New("unknown OpenStack discovery role") } diff --git a/discovery/outscale/metrics.go b/discovery/outscale/metrics.go new file mode 100644 index 00000000000..6a432215f31 --- /dev/null +++ b/discovery/outscale/metrics.go @@ -0,0 +1,32 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package outscale + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*outscaleMetrics)(nil) + +type outscaleMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (*outscaleMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (*outscaleMetrics) Unregister() {} diff --git a/discovery/outscale/outscale.go b/discovery/outscale/outscale.go new file mode 100644 index 00000000000..61b05ae29f8 --- /dev/null +++ b/discovery/outscale/outscale.go @@ -0,0 +1,171 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package outscale + +import ( + "errors" + "fmt" + "net/http" + "os" + "strings" + "time" + + osc "github.com/outscale/osc-sdk-go/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" +) + +const ( + metaLabelPrefix = model.MetaLabelPrefix + "outscale_" +) + +// DefaultSDConfig is the default Outscale Service Discovery configuration. +var DefaultSDConfig = SDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, + Region: "eu-west-2", +} + +// SDConfig is the configuration for Outscale VM-based service discovery +// using the OUTSCALE API (OAPI). +type SDConfig struct { + Endpoint string `yaml:"endpoint,omitempty"` + Region string `yaml:"region"` + AccessKey string `yaml:"access_key"` + SecretKey config.Secret `yaml:"secret_key"` + SecretKeyFile string `yaml:"secret_key_file"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` +} + +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &outscaleMetrics{ + refreshMetrics: rmi, + } +} + +// Name returns the name of the discovery mechanism. +func (SDConfig) Name() string { + return "outscale" +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultSDConfig + type plain SDConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + if c.Region == "" { + return errors.New("outscale SD configuration requires a region") + } + if c.AccessKey == "" { + return errors.New("outscale SD configuration requires access_key") + } + if c.SecretKey == "" && c.SecretKeyFile == "" { + return errors.New("one of secret_key & secret_key_file must be configured") + } + if c.SecretKey != "" && c.SecretKeyFile != "" { + return errors.New("at most one of secret_key & secret_key_file must be configured") + } + if c.Endpoint == "" { + c.Endpoint = "https://api." + c.Region + ".outscale.com/api/v1" + } + return c.HTTPClientConfig.Validate() +} + +// SetDirectory joins any relative file paths with dir. +func (c *SDConfig) SetDirectory(dir string) { + c.SecretKeyFile = config.JoinDir(dir, c.SecretKeyFile) + c.HTTPClientConfig.SetDirectory(dir) +} + +func (c *SDConfig) SecretKeyValue() (config.Secret, error) { + if c.SecretKeyFile == "" { + return c.SecretKey, nil + } + secret, err := os.ReadFile(c.SecretKeyFile) + if err != nil { + return "", fmt.Errorf("unable to read outscale secret key file %s: %w", c.SecretKeyFile, err) + } + return config.Secret(strings.TrimSpace(string(secret))), nil +} + +// NewDiscoverer returns a Discoverer for the Config. +func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(c, opts) +} + +func init() { + discovery.RegisterConfig(&SDConfig{}) +} + +// Discovery periodically performs Outscale API requests. +type Discovery struct{} + +// NewDiscovery returns a new Discovery which periodically refreshes its targets. +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*refresh.Discovery, error) { + m, ok := opts.Metrics.(*outscaleMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type") + } + + d, err := newVMDiscovery(conf) + if err != nil { + return nil, err + } + + return refresh.NewDiscovery( + refresh.Options{ + Logger: opts.Logger, + Mech: "outscale", + SetName: opts.SetName, + Interval: time.Duration(conf.RefreshInterval), + RefreshF: d.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ), nil +} + +// loadClient builds an Outscale API client from SDConfig using the official osc-sdk-go (OAPI). +func loadClient(conf *SDConfig) (*osc.APIClient, error) { + rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "outscale_sd") + if err != nil { + return nil, fmt.Errorf("outscale SD client config: %w", err) + } + + httpClient := &http.Client{ + Transport: rt, + Timeout: time.Duration(conf.RefreshInterval), + } + + cfg := osc.NewConfiguration() + cfg.HTTPClient = httpClient + cfg.UserAgent = version.PrometheusUserAgent() + // Use a single server URL (custom endpoint or default per-region). + cfg.Servers = osc.ServerConfigurations{ + osc.ServerConfiguration{URL: conf.Endpoint, Description: "Outscale API"}, + } + + client := osc.NewAPIClient(cfg) + return client, nil +} diff --git a/discovery/outscale/vm.go b/discovery/outscale/vm.go new file mode 100644 index 00000000000..420265dc451 --- /dev/null +++ b/discovery/outscale/vm.go @@ -0,0 +1,144 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package outscale + +import ( + "context" + "fmt" + "net" + "strconv" + + osc "github.com/outscale/osc-sdk-go/v2" + "github.com/prometheus/common/model" + + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + vmLabelPrefix = metaLabelPrefix + "vm_" + vmLabelInstanceID = vmLabelPrefix + "instance_id" + vmLabelRegion = vmLabelPrefix + "region" + vmLabelSubregion = vmLabelPrefix + "subregion" + vmLabelState = vmLabelPrefix + "state" + vmLabelPrivateIP = vmLabelPrefix + "private_ip" + vmLabelPublicIP = vmLabelPrefix + "public_ip" + vmLabelTag = vmLabelPrefix + "tag_" +) + +type vmDiscovery struct { + client *osc.APIClient + cfg *SDConfig +} + +func newVMDiscovery(conf *SDConfig) (*vmDiscovery, error) { + client, err := loadClient(conf) + if err != nil { + return nil, err + } + return &vmDiscovery{ + client: client, + cfg: conf, + }, nil +} + +func (d *vmDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + secretKey, err := d.cfg.SecretKeyValue() + if err != nil { + return nil, err + } + + // Merge request context with auth context so the SDK signs requests. + ctx = context.WithValue(ctx, osc.ContextAWSv4, osc.AWSv4{ + AccessKey: d.cfg.AccessKey, + SecretKey: string(secretKey), + }) + + tg := &targetgroup.Group{Source: d.cfg.Region} + + var allVms []osc.Vm + req := d.client.VmApi.ReadVms(ctx).ReadVmsRequest(osc.ReadVmsRequest{}) + + for { + resp, _, err := req.Execute() + if err != nil { + return nil, fmt.Errorf("outscale ReadVms: %w", err) + } + if resp.Vms != nil { + allVms = append(allVms, *resp.Vms...) + } + if resp.NextPageToken == nil || *resp.NextPageToken == "" { + break + } + req = d.client.VmApi.ReadVms(ctx).ReadVmsRequest(osc.ReadVmsRequest{NextPageToken: resp.NextPageToken}) + } + + tg.Targets = vmsToLabelSets(allVms, d.cfg) + return []*targetgroup.Group{tg}, nil +} + +// vmsToLabelSets converts Outscale VMs into target label sets. +func vmsToLabelSets(vms []osc.Vm, cfg *SDConfig) []model.LabelSet { + var out []model.LabelSet + for _, vm := range vms { + var addr string + switch { + case vm.PrivateIp != nil && *vm.PrivateIp != "": + addr = net.JoinHostPort(*vm.PrivateIp, strconv.Itoa(cfg.Port)) + case vm.PublicIp != nil && *vm.PublicIp != "": + addr = net.JoinHostPort(*vm.PublicIp, strconv.Itoa(cfg.Port)) + default: + continue + } + + vmID := "" + if vm.VmId != nil { + vmID = *vm.VmId + } + state := "" + if vm.State != nil { + state = *vm.State + } + + labels := model.LabelSet{ + model.AddressLabel: model.LabelValue(addr), + vmLabelInstanceID: model.LabelValue(vmID), + vmLabelRegion: model.LabelValue(cfg.Region), + vmLabelState: model.LabelValue(state), + } + + if vm.Placement != nil && vm.Placement.SubregionName != nil { + labels[vmLabelSubregion] = model.LabelValue(*vm.Placement.SubregionName) + } + if vm.PrivateIp != nil { + labels[vmLabelPrivateIP] = model.LabelValue(*vm.PrivateIp) + } + if vm.PublicIp != nil { + labels[vmLabelPublicIP] = model.LabelValue(*vm.PublicIp) + } + + if vm.Tags != nil { + for _, t := range *vm.Tags { + if t.Key == "" || t.Value == "" { + continue + } + name := strutil.SanitizeLabelName(t.Key) + labels[vmLabelTag+model.LabelName(name)] = model.LabelValue(t.Value) + } + } + + out = append(out, labels) + } + return out +} diff --git a/discovery/outscale/vm_test.go b/discovery/outscale/vm_test.go new file mode 100644 index 00000000000..26d74262a97 --- /dev/null +++ b/discovery/outscale/vm_test.go @@ -0,0 +1,123 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package outscale + +import ( + "testing" + + osc "github.com/outscale/osc-sdk-go/v2" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" +) + +func strptr(s string) *string { return &s } + +func TestVmsToLabelSets(t *testing.T) { + cfg := &SDConfig{ + Region: "eu-west-2", + Port: 9090, + } + + t.Run("empty", func(t *testing.T) { + out := vmsToLabelSets(nil, cfg) + require.Empty(t, out) + out = vmsToLabelSets([]osc.Vm{}, cfg) + require.Empty(t, out) + }) + + t.Run("skip_no_ip", func(t *testing.T) { + vms := []osc.Vm{ + {VmId: strptr("i-1"), State: strptr("running")}, // no PrivateIp nor PublicIp + } + out := vmsToLabelSets(vms, cfg) + require.Empty(t, out) + }) + + t.Run("private_ip_preferred", func(t *testing.T) { + vms := []osc.Vm{ + { + VmId: strptr("i-abc"), + State: strptr("running"), + PrivateIp: strptr("10.1.2.3"), + PublicIp: strptr("1.2.3.4"), + }, + } + out := vmsToLabelSets(vms, cfg) + require.Len(t, out, 1) + require.Equal(t, model.LabelValue("10.1.2.3:9090"), out[0][model.AddressLabel]) + require.Equal(t, model.LabelValue("i-abc"), out[0][vmLabelInstanceID]) + require.Equal(t, model.LabelValue("eu-west-2"), out[0][vmLabelRegion]) + require.Equal(t, model.LabelValue("running"), out[0][vmLabelState]) + require.Equal(t, model.LabelValue("10.1.2.3"), out[0][vmLabelPrivateIP]) + require.Equal(t, model.LabelValue("1.2.3.4"), out[0][vmLabelPublicIP]) + }) + + t.Run("public_ip_fallback", func(t *testing.T) { + vms := []osc.Vm{ + { + VmId: strptr("i-pub"), + State: strptr("running"), + PublicIp: strptr("203.0.113.10"), + }, + } + out := vmsToLabelSets(vms, cfg) + require.Len(t, out, 1) + require.Equal(t, model.LabelValue("203.0.113.10:9090"), out[0][model.AddressLabel]) + require.Equal(t, model.LabelValue("203.0.113.10"), out[0][vmLabelPublicIP]) + _, hasPrivate := out[0][vmLabelPrivateIP] + require.False(t, hasPrivate) + }) + + t.Run("subregion_and_tags", func(t *testing.T) { + subregion := "eu-west-2a" + vms := []osc.Vm{ + { + VmId: strptr("i-tagged"), + State: strptr("stopped"), + PrivateIp: strptr("10.0.0.5"), + Placement: &osc.Placement{SubregionName: &subregion}, + Tags: &[]osc.ResourceTag{ + {Key: "Name", Value: "my-vm"}, + {Key: "env", Value: "prod"}, + }, + }, + } + out := vmsToLabelSets(vms, cfg) + require.Len(t, out, 1) + require.Equal(t, model.LabelValue("eu-west-2a"), out[0][vmLabelSubregion]) + require.Equal(t, model.LabelValue("my-vm"), out[0][vmLabelTag+model.LabelName("Name")]) + require.Equal(t, model.LabelValue("prod"), out[0][vmLabelTag+model.LabelName("env")]) + }) + + t.Run("skips_empty_tag_key_or_value", func(t *testing.T) { + vms := []osc.Vm{ + { + VmId: strptr("i-tags"), + PrivateIp: strptr("10.0.0.1"), + Tags: &[]osc.ResourceTag{ + {Key: "Good", Value: "yes"}, + {Key: "", Value: "skip"}, + {Key: "EmptyVal", Value: ""}, + }, + }, + } + out := vmsToLabelSets(vms, cfg) + require.Len(t, out, 1) + require.Equal(t, model.LabelValue("yes"), out[0][vmLabelTag+model.LabelName("Good")]) + _, hasBad := out[0][vmLabelTag+model.LabelName("")] + require.False(t, hasBad) + _, hasEmptyVal := out[0][vmLabelTag+model.LabelName("EmptyVal")] + require.False(t, hasEmptyVal) + }) +} diff --git a/discovery/ovhcloud/dedicated_server.go b/discovery/ovhcloud/dedicated_server.go index a70857a08b2..e892607c342 100644 --- a/discovery/ovhcloud/dedicated_server.go +++ b/discovery/ovhcloud/dedicated_server.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,13 +16,12 @@ package ovhcloud import ( "context" "fmt" + "log/slog" "net/netip" "net/url" "path" "strconv" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/ovh/go-ovh/ovh" "github.com/prometheus/common/model" @@ -55,10 +54,10 @@ type dedicatedServer struct { type dedicatedServerDiscovery struct { *refresh.Discovery config *SDConfig - logger log.Logger + logger *slog.Logger } -func newDedicatedServerDiscovery(conf *SDConfig, logger log.Logger) *dedicatedServerDiscovery { +func newDedicatedServerDiscovery(conf *SDConfig, logger *slog.Logger) *dedicatedServerDiscovery { return &dedicatedServerDiscovery{config: conf, logger: logger} } @@ -94,7 +93,7 @@ func getDedicatedServerDetails(client *ovh.Client, serverName string) (*dedicate return &dedicatedServerDetails, nil } -func (d *dedicatedServerDiscovery) getService() string { +func (*dedicatedServerDiscovery) getService() string { return "dedicated_server" } @@ -115,10 +114,7 @@ func (d *dedicatedServerDiscovery) refresh(context.Context) ([]*targetgroup.Grou for _, dedicatedServerName := range dedicatedServerList { dedicatedServer, err := getDedicatedServerDetails(client, dedicatedServerName) if err != nil { - err := level.Warn(d.logger).Log("msg", fmt.Sprintf("%s: Could not get details of %s", d.getSource(), dedicatedServerName), "err", err.Error()) - if err != nil { - return nil, err - } + d.logger.Warn(fmt.Sprintf("%s: Could not get details of %s", d.getSource(), dedicatedServerName), "err", err.Error()) continue } dedicatedServerDetailedList = append(dedicatedServerDetailedList, *dedicatedServer) diff --git a/discovery/ovhcloud/dedicated_server_test.go b/discovery/ovhcloud/dedicated_server_test.go index 52311bcc876..84fa2c4c12f 100644 --- a/discovery/ovhcloud/dedicated_server_test.go +++ b/discovery/ovhcloud/dedicated_server_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,10 +21,10 @@ import ( "os" "testing" - "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" ) func TestOvhcloudDedicatedServerRefresh(t *testing.T) { @@ -41,7 +41,7 @@ application_secret: %s consumer_key: %s`, mock.URL, ovhcloudApplicationKeyTest, ovhcloudApplicationSecretTest, ovhcloudConsumerKeyTest) require.NoError(t, yaml.UnmarshalStrict([]byte(cfgString), &cfg)) - d, err := newRefresher(&cfg, log.NewNopLogger()) + d, err := newRefresher(&cfg, promslog.NewNopLogger()) require.NoError(t, err) ctx := context.Background() targetGroups, err := d.refresh(ctx) diff --git a/discovery/ovhcloud/metrics.go b/discovery/ovhcloud/metrics.go index 1c517094694..dbcfe130e93 100644 --- a/discovery/ovhcloud/metrics.go +++ b/discovery/ovhcloud/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type ovhcloudMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *ovhcloudMetrics) Register() error { +func (*ovhcloudMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *ovhcloudMetrics) Unregister() {} +func (*ovhcloudMetrics) Unregister() {} diff --git a/discovery/ovhcloud/ovhcloud.go b/discovery/ovhcloud/ovhcloud.go index 988b4482f22..863fcfeaf95 100644 --- a/discovery/ovhcloud/ovhcloud.go +++ b/discovery/ovhcloud/ovhcloud.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,10 +17,10 @@ import ( "context" "errors" "fmt" + "log/slog" "net/netip" "time" - "github.com/go-kit/log" "github.com/ovh/go-ovh/ovh" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -54,19 +54,19 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &ovhcloudMetrics{ refreshMetrics: rmi, } } // Name implements the Discoverer interface. -func (c SDConfig) Name() string { +func (SDConfig) Name() string { return "ovhcloud" } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -100,8 +100,8 @@ func createClient(config *SDConfig) (*ovh.Client, error) { } // NewDiscoverer returns a Discoverer for the Config. -func (c *SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, options.Logger, options.Metrics) +func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(c, opts) } func init() { @@ -137,7 +137,7 @@ func parseIPList(ipList []string) ([]netip.Addr, error) { return ipAddresses, nil } -func newRefresher(conf *SDConfig, logger log.Logger) (refresher, error) { +func newRefresher(conf *SDConfig, logger *slog.Logger) (refresher, error) { switch conf.Service { case "vps": return newVpsDiscovery(conf, logger), nil @@ -148,21 +148,22 @@ func newRefresher(conf *SDConfig, logger log.Logger) (refresher, error) { } // NewDiscovery returns a new OVHcloud Discoverer which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { - m, ok := metrics.(*ovhcloudMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*refresh.Discovery, error) { + m, ok := opts.Metrics.(*ovhcloudMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - r, err := newRefresher(conf, logger) + r, err := newRefresher(conf, opts.Logger) if err != nil { return nil, err } return refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "ovhcloud", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: r.refresh, MetricsInstantiator: m.refreshMetrics, diff --git a/discovery/ovhcloud/ovhcloud_test.go b/discovery/ovhcloud/ovhcloud_test.go index 9c95bf90e67..acb1c43fad0 100644 --- a/discovery/ovhcloud/ovhcloud_test.go +++ b/discovery/ovhcloud/ovhcloud_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -20,11 +20,11 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery" - "github.com/prometheus/prometheus/util/testutil" ) var ( @@ -121,7 +121,7 @@ func TestParseIPs(t *testing.T) { func TestDiscoverer(t *testing.T) { conf, _ := getMockConf("vps") - logger := testutil.NewLogger(t) + logger := promslog.NewNopLogger() reg := prometheus.NewRegistry() refreshMetrics := discovery.NewRefreshMetrics(reg) diff --git a/discovery/ovhcloud/vps.go b/discovery/ovhcloud/vps.go index 58ceeabd87a..4023c4ff496 100644 --- a/discovery/ovhcloud/vps.go +++ b/discovery/ovhcloud/vps.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,13 +16,12 @@ package ovhcloud import ( "context" "fmt" + "log/slog" "net/netip" "net/url" "path" "strconv" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/ovh/go-ovh/ovh" "github.com/prometheus/common/model" @@ -68,10 +67,10 @@ type virtualPrivateServer struct { type vpsDiscovery struct { *refresh.Discovery config *SDConfig - logger log.Logger + logger *slog.Logger } -func newVpsDiscovery(conf *SDConfig, logger log.Logger) *vpsDiscovery { +func newVpsDiscovery(conf *SDConfig, logger *slog.Logger) *vpsDiscovery { return &vpsDiscovery{config: conf, logger: logger} } @@ -110,7 +109,7 @@ func getVpsList(client *ovh.Client) ([]string, error) { return vpsListName, nil } -func (d *vpsDiscovery) getService() string { +func (*vpsDiscovery) getService() string { return "vps" } @@ -133,10 +132,7 @@ func (d *vpsDiscovery) refresh(context.Context) ([]*targetgroup.Group, error) { for _, vpsName := range vpsList { vpsDetailed, err := getVpsDetails(client, vpsName) if err != nil { - err := level.Warn(d.logger).Log("msg", fmt.Sprintf("%s: Could not get details of %s", d.getSource(), vpsName), "err", err.Error()) - if err != nil { - return nil, err - } + d.logger.Warn(fmt.Sprintf("%s: Could not get details of %s", d.getSource(), vpsName), "err", err.Error()) continue } vpsDetailedList = append(vpsDetailedList, *vpsDetailed) diff --git a/discovery/ovhcloud/vps_test.go b/discovery/ovhcloud/vps_test.go index 2d2d6dcd219..d997f2bb0e6 100644 --- a/discovery/ovhcloud/vps_test.go +++ b/discovery/ovhcloud/vps_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,11 +21,10 @@ import ( "os" "testing" - yaml "gopkg.in/yaml.v2" - - "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" + yaml "go.yaml.in/yaml/v2" ) func TestOvhCloudVpsRefresh(t *testing.T) { @@ -43,7 +42,7 @@ consumer_key: %s`, mock.URL, ovhcloudApplicationKeyTest, ovhcloudApplicationSecr require.NoError(t, yaml.UnmarshalStrict([]byte(cfgString), &cfg)) - d, err := newRefresher(&cfg, log.NewNopLogger()) + d, err := newRefresher(&cfg, promslog.NewNopLogger()) require.NoError(t, err) ctx := context.Background() targetGroups, err := d.refresh(ctx) diff --git a/discovery/puppetdb/metrics.go b/discovery/puppetdb/metrics.go index 2f5faa57cef..5a8e9736c2d 100644 --- a/discovery/puppetdb/metrics.go +++ b/discovery/puppetdb/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type puppetdbMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *puppetdbMetrics) Register() error { +func (*puppetdbMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *puppetdbMetrics) Unregister() {} +func (*puppetdbMetrics) Unregister() {} diff --git a/discovery/puppetdb/puppetdb.go b/discovery/puppetdb/puppetdb.go index 8f89acbf936..52a1cf73c69 100644 --- a/discovery/puppetdb/puppetdb.go +++ b/discovery/puppetdb/puppetdb.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,6 +17,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net" @@ -27,11 +28,11 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/grafana/regexp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/discovery" @@ -62,7 +63,7 @@ var ( HTTPClientConfig: config.DefaultHTTPClientConfig, } matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`) - userAgent = fmt.Sprintf("Prometheus/%s", version.Version) + userAgent = version.PrometheusUserAgent() ) func init() { @@ -80,7 +81,7 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &puppetdbMetrics{ refreshMetrics: rmi, } @@ -91,7 +92,7 @@ func (*SDConfig) Name() string { return "puppetdb" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -100,7 +101,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -108,20 +109,20 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } if c.URL == "" { - return fmt.Errorf("URL is missing") + return errors.New("URL is missing") } parsedURL, err := url.Parse(c.URL) if err != nil { return err } if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { - return fmt.Errorf("URL scheme must be 'http' or 'https'") + return errors.New("URL scheme must be 'http' or 'https'") } if parsedURL.Host == "" { - return fmt.Errorf("host is missing in URL") + return errors.New("host is missing in URL") } if c.Query == "" { - return fmt.Errorf("query missing") + return errors.New("query missing") } return c.HTTPClientConfig.Validate() } @@ -138,14 +139,14 @@ type Discovery struct { } // NewDiscovery returns a new PuppetDB discovery for the given config. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*puppetdbMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*puppetdbMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } - if logger == nil { - logger = log.NewNopLogger() + if opts.Logger == nil { + opts.Logger = promslog.NewNopLogger() } client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http") @@ -170,8 +171,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "puppetdb", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, diff --git a/discovery/puppetdb/puppetdb_test.go b/discovery/puppetdb/puppetdb_test.go index bf9c7b215e5..b12835b47c6 100644 --- a/discovery/puppetdb/puppetdb_test.go +++ b/discovery/puppetdb/puppetdb_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -22,10 +22,10 @@ import ( "testing" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/discovery" @@ -70,7 +70,11 @@ func TestPuppetSlashInURL(t *testing.T) { metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) require.NoError(t, metrics.Register()) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "puppetdb", + }) require.NoError(t, err) require.Equal(t, apiURL, d.url) @@ -94,7 +98,11 @@ func TestPuppetDBRefresh(t *testing.T) { metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) require.NoError(t, metrics.Register()) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "puppetdb", + }) require.NoError(t, err) ctx := context.Background() @@ -142,7 +150,11 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) { metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) require.NoError(t, metrics.Register()) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "puppetdb", + }) require.NoError(t, err) ctx := context.Background() @@ -184,7 +196,7 @@ func TestPuppetDBRefreshWithParameters(t *testing.T) { } func TestPuppetDBInvalidCode(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadRequest) })) @@ -201,7 +213,11 @@ func TestPuppetDBInvalidCode(t *testing.T) { metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) require.NoError(t, metrics.Register()) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "puppetdb", + }) require.NoError(t, err) ctx := context.Background() @@ -212,7 +228,7 @@ func TestPuppetDBInvalidCode(t *testing.T) { } func TestPuppetDBInvalidFormat(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintln(w, "{}") })) @@ -229,7 +245,11 @@ func TestPuppetDBInvalidFormat(t *testing.T) { metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics) require.NoError(t, metrics.Register()) - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "puppetdb", + }) require.NoError(t, err) ctx := context.Background() diff --git a/discovery/puppetdb/resources.go b/discovery/puppetdb/resources.go index d6387fe1b8b..09aa43a776f 100644 --- a/discovery/puppetdb/resources.go +++ b/discovery/puppetdb/resources.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -34,7 +34,7 @@ type Resource struct { Parameters Parameters `json:"parameters"` } -type Parameters map[string]interface{} +type Parameters map[string]any func (p *Parameters) toLabels() model.LabelSet { labels := model.LabelSet{} @@ -52,7 +52,7 @@ func (p *Parameters) toLabels() model.LabelSet { labelValue = strconv.FormatFloat(value, 'g', -1, 64) case []string: labelValue = separator + strings.Join(value, separator) + separator - case []interface{}: + case []any: if len(value) == 0 { continue } @@ -72,7 +72,7 @@ func (p *Parameters) toLabels() model.LabelSet { } } labelValue = strings.Join(values, separator) - case map[string]interface{}: + case map[string]any: subParameter := Parameters(value) prefix := strutil.SanitizeLabelName(k + "_") for subk, subv := range subParameter.toLabels() { diff --git a/discovery/refresh/refresh.go b/discovery/refresh/refresh.go index f037a90cff0..3e766d1c848 100644 --- a/discovery/refresh/refresh.go +++ b/discovery/refresh/refresh.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,18 +16,19 @@ package refresh import ( "context" "errors" + "log/slog" "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" ) type Options struct { - Logger log.Logger + Logger *slog.Logger Mech string + SetName string Interval time.Duration RefreshF func(ctx context.Context) ([]*targetgroup.Group, error) MetricsInstantiator discovery.RefreshMetricsInstantiator @@ -35,7 +36,7 @@ type Options struct { // Discovery implements the Discoverer interface. type Discovery struct { - logger log.Logger + logger *slog.Logger interval time.Duration refreshf func(ctx context.Context) ([]*targetgroup.Group, error) metrics *discovery.RefreshMetrics @@ -43,11 +44,11 @@ type Discovery struct { // NewDiscovery returns a Discoverer function that calls a refresh() function at every interval. func NewDiscovery(opts Options) *Discovery { - m := opts.MetricsInstantiator.Instantiate(opts.Mech) + m := opts.MetricsInstantiator.Instantiate(opts.Mech, opts.SetName) - var logger log.Logger + var logger *slog.Logger if opts.Logger == nil { - logger = log.NewNopLogger() + logger = promslog.NewNopLogger() } else { logger = opts.Logger } @@ -68,7 +69,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { tgs, err := d.refresh(ctx) if err != nil { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(d.logger).Log("msg", "Unable to refresh target groups", "err", err.Error()) + d.logger.Error("Unable to refresh target groups", "err", err.Error()) } } else { select { @@ -87,7 +88,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { tgs, err := d.refresh(ctx) if err != nil { if !errors.Is(ctx.Err(), context.Canceled) { - level.Error(d.logger).Log("msg", "Unable to refresh target groups", "err", err.Error()) + d.logger.Error("Unable to refresh target groups", "err", err.Error()) } continue } @@ -107,6 +108,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { now := time.Now() defer func() { d.metrics.Duration.Observe(time.Since(now).Seconds()) + d.metrics.DurationHistogram.Observe(time.Since(now).Seconds()) }() tgs, err := d.refreshf(ctx) diff --git a/discovery/refresh/refresh_test.go b/discovery/refresh/refresh_test.go index b70a3263554..e227d0abc9b 100644 --- a/discovery/refresh/refresh_test.go +++ b/discovery/refresh/refresh_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,7 +15,7 @@ package refresh import ( "context" - "fmt" + "errors" "testing" "time" @@ -56,7 +56,7 @@ func TestRefresh(t *testing.T) { } var i int - refresh := func(ctx context.Context) ([]*targetgroup.Group, error) { + refresh := func(context.Context) ([]*targetgroup.Group, error) { i++ switch i { case 1: @@ -64,7 +64,7 @@ func TestRefresh(t *testing.T) { case 2: return tg2, nil } - return nil, fmt.Errorf("some error") + return nil, errors.New("some error") } interval := time.Millisecond @@ -76,6 +76,7 @@ func TestRefresh(t *testing.T) { Options{ Logger: nil, Mech: "test", + SetName: "test-refresh", Interval: interval, RefreshF: refresh, MetricsInstantiator: metrics, @@ -83,8 +84,7 @@ func TestRefresh(t *testing.T) { ) ch := make(chan []*targetgroup.Group) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() go d.Run(ctx, ch) tg := <-ch diff --git a/discovery/registry.go b/discovery/registry.go index 1f491d4ca9f..04145e72e46 100644 --- a/discovery/registry.go +++ b/discovery/registry.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -22,9 +22,8 @@ import ( "strings" "sync" - "gopkg.in/yaml.v2" - "github.com/prometheus/client_golang/prometheus" + "go.yaml.in/yaml/v2" "github.com/prometheus/prometheus/discovery/targetgroup" ) @@ -43,8 +42,8 @@ var ( configTypesMu sync.Mutex configTypes = make(map[reflect.Type]reflect.Type) - emptyStructType = reflect.TypeOf(struct{}{}) - configsType = reflect.TypeOf(Configs{}) + emptyStructType = reflect.TypeFor[struct{}]() + configsType = reflect.TypeFor[Configs]() ) // RegisterConfig registers the given Config type for YAML marshaling and unmarshaling. @@ -55,7 +54,7 @@ func RegisterConfig(config Config) { func init() { // N.B.: static_configs is the only Config type implemented by default. // All other types are registered at init by their implementing packages. - elemTyp := reflect.TypeOf(&targetgroup.Group{}) + elemTyp := reflect.TypeFor[*targetgroup.Group]() registerConfig(staticConfigsKey, elemTyp, StaticConfig{}) } @@ -111,7 +110,7 @@ func getConfigType(out reflect.Type) reflect.Type { // UnmarshalYAMLWithInlineConfigs helps implement yaml.Unmarshal for structs // that have a Configs field that should be inlined. -func UnmarshalYAMLWithInlineConfigs(out interface{}, unmarshal func(interface{}) error) error { +func UnmarshalYAMLWithInlineConfigs(out any, unmarshal func(any) error) error { outVal := reflect.ValueOf(out) if outVal.Kind() != reflect.Ptr { return fmt.Errorf("discovery: can only unmarshal into a struct pointer: %T", out) @@ -199,7 +198,7 @@ func readConfigs(structVal reflect.Value, startField int) (Configs, error) { // MarshalYAMLWithInlineConfigs helps implement yaml.Marshal for structs // that have a Configs field that should be inlined. -func MarshalYAMLWithInlineConfigs(in interface{}) (interface{}, error) { +func MarshalYAMLWithInlineConfigs(in any) (any, error) { inVal := reflect.ValueOf(in) for inVal.Kind() == reflect.Ptr { inVal = inVal.Elem() @@ -267,7 +266,7 @@ func replaceYAMLTypeError(err error, oldTyp, newTyp reflect.Type) error { func RegisterSDMetrics(registerer prometheus.Registerer, rmm RefreshMetricsManager) (map[string]DiscovererMetrics, error) { err := rmm.Register() if err != nil { - return nil, fmt.Errorf("failed to create service discovery refresh metrics") + return nil, fmt.Errorf("failed to create service discovery refresh metrics: %w", err) } metrics := make(map[string]DiscovererMetrics) @@ -275,9 +274,19 @@ func RegisterSDMetrics(registerer prometheus.Registerer, rmm RefreshMetricsManag currentSdMetrics := conf.NewDiscovererMetrics(registerer, rmm) err = currentSdMetrics.Register() if err != nil { - return nil, fmt.Errorf("failed to create service discovery metrics") + return nil, fmt.Errorf("failed to create service discovery metrics: %w", err) } metrics[conf.Name()] = currentSdMetrics } return metrics, nil } + +// RegisteredConfigNames returns the names of all registered service discovery providers. +func RegisteredConfigNames() []string { + names := make([]string, 0, len(configNames)) + for name := range configNames { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/discovery/scaleway/baremetal.go b/discovery/scaleway/baremetal.go index c313e6695dc..ecdf3cd7a23 100644 --- a/discovery/scaleway/baremetal.go +++ b/discovery/scaleway/baremetal.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,12 +17,9 @@ import ( "context" "fmt" "net" - "net/http" "strconv" "strings" - "time" - "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" "github.com/scaleway/scaleway-sdk-go/api/baremetal/v1" @@ -71,29 +68,19 @@ func newBaremetalDiscovery(conf *SDConfig) (*baremetalDiscovery, error) { tagsFilter: conf.TagsFilter, } - rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "scaleway_sd") + client, err := newScalewayHTTPClient(conf) if err != nil { return nil, err } - if conf.SecretKeyFile != "" { - rt, err = newAuthTokenFileRoundTripper(conf.SecretKeyFile, rt) - if err != nil { - return nil, err - } - } - profile, err := loadProfile(conf) if err != nil { return nil, err } d.client, err = scw.NewClient( - scw.WithHTTPClient(&http.Client{ - Transport: rt, - Timeout: time.Duration(conf.RefreshInterval), - }), - scw.WithUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)), + scw.WithHTTPClient(client), + scw.WithUserAgent(version.PrometheusUserAgent()), scw.WithProfile(profile), ) if err != nil { diff --git a/discovery/scaleway/instance.go b/discovery/scaleway/instance.go index 2542c632533..e6747db3ec6 100644 --- a/discovery/scaleway/instance.go +++ b/discovery/scaleway/instance.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,15 +17,13 @@ import ( "context" "fmt" "net" - "net/http" "strconv" "strings" - "time" - "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1" "github.com/scaleway/scaleway-sdk-go/scw" "github.com/prometheus/prometheus/discovery/refresh" @@ -35,28 +33,30 @@ import ( const ( instanceLabelPrefix = metaLabelPrefix + "instance_" - instanceBootTypeLabel = instanceLabelPrefix + "boot_type" - instanceHostnameLabel = instanceLabelPrefix + "hostname" - instanceIDLabel = instanceLabelPrefix + "id" - instanceImageArchLabel = instanceLabelPrefix + "image_arch" - instanceImageIDLabel = instanceLabelPrefix + "image_id" - instanceImageNameLabel = instanceLabelPrefix + "image_name" - instanceLocationClusterID = instanceLabelPrefix + "location_cluster_id" - instanceLocationHypervisorID = instanceLabelPrefix + "location_hypervisor_id" - instanceLocationNodeID = instanceLabelPrefix + "location_node_id" - instanceNameLabel = instanceLabelPrefix + "name" - instanceOrganizationLabel = instanceLabelPrefix + "organization_id" - instancePrivateIPv4Label = instanceLabelPrefix + "private_ipv4" - instanceProjectLabel = instanceLabelPrefix + "project_id" - instancePublicIPv4Label = instanceLabelPrefix + "public_ipv4" - instancePublicIPv6Label = instanceLabelPrefix + "public_ipv6" - instanceSecurityGroupIDLabel = instanceLabelPrefix + "security_group_id" - instanceSecurityGroupNameLabel = instanceLabelPrefix + "security_group_name" - instanceStateLabel = instanceLabelPrefix + "status" - instanceTagsLabel = instanceLabelPrefix + "tags" - instanceTypeLabel = instanceLabelPrefix + "type" - instanceZoneLabel = instanceLabelPrefix + "zone" - instanceRegionLabel = instanceLabelPrefix + "region" + instanceBootTypeLabel = instanceLabelPrefix + "boot_type" + instanceHostnameLabel = instanceLabelPrefix + "hostname" + instanceIDLabel = instanceLabelPrefix + "id" + instanceImageArchLabel = instanceLabelPrefix + "image_arch" + instanceImageIDLabel = instanceLabelPrefix + "image_id" + instanceImageNameLabel = instanceLabelPrefix + "image_name" + instanceLocationClusterID = instanceLabelPrefix + "location_cluster_id" + instanceLocationHypervisorID = instanceLabelPrefix + "location_hypervisor_id" + instanceLocationNodeID = instanceLabelPrefix + "location_node_id" + instanceNameLabel = instanceLabelPrefix + "name" + instanceOrganizationLabel = instanceLabelPrefix + "organization_id" + instancePrivateIPv4Label = instanceLabelPrefix + "private_ipv4" + instanceProjectLabel = instanceLabelPrefix + "project_id" + instancePublicIPv4Label = instanceLabelPrefix + "public_ipv4" + instancePublicIPv6Label = instanceLabelPrefix + "public_ipv6" + instancePublicIPv4AddressesLabel = instanceLabelPrefix + "public_ipv4_addresses" + instancePublicIPv6AddressesLabel = instanceLabelPrefix + "public_ipv6_addresses" + instanceSecurityGroupIDLabel = instanceLabelPrefix + "security_group_id" + instanceSecurityGroupNameLabel = instanceLabelPrefix + "security_group_name" + instanceStateLabel = instanceLabelPrefix + "status" + instanceTagsLabel = instanceLabelPrefix + "tags" + instanceTypeLabel = instanceLabelPrefix + "type" + instanceZoneLabel = instanceLabelPrefix + "zone" + instanceRegionLabel = instanceLabelPrefix + "region" ) type instanceDiscovery struct { @@ -82,29 +82,19 @@ func newInstanceDiscovery(conf *SDConfig) (*instanceDiscovery, error) { tagsFilter: conf.TagsFilter, } - rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "scaleway_sd") + client, err := newScalewayHTTPClient(conf) if err != nil { return nil, err } - if conf.SecretKeyFile != "" { - rt, err = newAuthTokenFileRoundTripper(conf.SecretKeyFile, rt) - if err != nil { - return nil, err - } - } - profile, err := loadProfile(conf) if err != nil { return nil, err } d.client, err = scw.NewClient( - scw.WithHTTPClient(&http.Client{ - Transport: rt, - Timeout: time.Duration(conf.RefreshInterval), - }), - scw.WithUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)), + scw.WithHTTPClient(client), + scw.WithUserAgent(version.PrometheusUserAgent()), scw.WithProfile(profile), ) if err != nil { @@ -115,7 +105,8 @@ func newInstanceDiscovery(conf *SDConfig) (*instanceDiscovery, error) { } func (d *instanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - api := instance.NewAPI(d.client) + instanceAPI := instance.NewAPI(d.client) + ipamAPI := ipam.NewAPI(d.client) req := &instance.ListServersRequest{} @@ -127,7 +118,12 @@ func (d *instanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, req.Tags = d.tagsFilter } - servers, err := api.ListServers(req, scw.WithAllPages(), scw.WithContext(ctx)) + servers, err := instanceAPI.ListServers(req, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return nil, err + } + + privateNICIPByID, err := privateNICIPs(ctx, ipamAPI, servers.Servers) if err != nil { return nil, err } @@ -175,14 +171,43 @@ func (d *instanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, } addr := "" + if len(server.PublicIPs) > 0 { + var ipv4Addresses []string + var ipv6Addresses []string + + for _, ip := range server.PublicIPs { + switch ip.Family { + case instance.ServerIPIPFamilyInet: + ipv4Addresses = append(ipv4Addresses, ip.Address.String()) + case instance.ServerIPIPFamilyInet6: + ipv6Addresses = append(ipv6Addresses, ip.Address.String()) + } + } + + if len(ipv6Addresses) > 0 { + labels[instancePublicIPv6AddressesLabel] = model.LabelValue( + separator + + strings.Join(ipv6Addresses, separator) + + separator) + } + if len(ipv4Addresses) > 0 { + labels[instancePublicIPv4AddressesLabel] = model.LabelValue( + separator + + strings.Join(ipv4Addresses, separator) + + separator) + } + } + if server.IPv6 != nil { //nolint:staticcheck labels[instancePublicIPv6Label] = model.LabelValue(server.IPv6.Address.String()) //nolint:staticcheck addr = server.IPv6.Address.String() //nolint:staticcheck } if server.PublicIP != nil { //nolint:staticcheck - labels[instancePublicIPv4Label] = model.LabelValue(server.PublicIP.Address.String()) //nolint:staticcheck - addr = server.PublicIP.Address.String() //nolint:staticcheck + if server.PublicIP.Family != instance.ServerIPIPFamilyInet6 { //nolint:staticcheck + labels[instancePublicIPv4Label] = model.LabelValue(server.PublicIP.Address.String()) //nolint:staticcheck + } + addr = server.PublicIP.Address.String() //nolint:staticcheck } if server.PrivateIP != nil { @@ -190,6 +215,19 @@ func (d *instanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, addr = *server.PrivateIP } + if addr == "" { + for _, privateNIC := range server.PrivateNics { + if privateNIC == nil { + continue + } + if privateIP := privateNICIPByID[privateNIC.ID]; privateIP != "" { + labels[instancePrivateIPv4Label] = model.LabelValue(privateIP) + addr = privateIP + break + } + } + } + if addr != "" { addr := net.JoinHostPort(addr, strconv.FormatUint(uint64(d.port), 10)) labels[model.AddressLabel] = model.LabelValue(addr) @@ -199,3 +237,43 @@ func (d *instanceDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, return []*targetgroup.Group{{Source: "scaleway", Targets: targets}}, nil } + +func privateNICIPs(ctx context.Context, api *ipam.API, servers []*instance.Server) (map[string]string, error) { + privateNICIDsByRegion := map[scw.Region][]string{} + for _, server := range servers { + if server.PrivateIP != nil || server.PublicIP != nil || server.IPv6 != nil { //nolint:staticcheck + continue + } + + region, err := server.Zone.Region() + if err != nil { + return nil, err + } + + for _, privateNIC := range server.PrivateNics { + if privateNIC != nil && privateNIC.ID != "" { + privateNICIDsByRegion[region] = append(privateNICIDsByRegion[region], privateNIC.ID) + } + } + } + + privateNICIPs := map[string]string{} + for region, privateNICIDs := range privateNICIDsByRegion { + ips, err := api.ListIPs(&ipam.ListIPsRequest{ + Region: region, + ResourceIDs: privateNICIDs, + ResourceTypes: []ipam.ResourceType{ipam.ResourceTypeInstancePrivateNic}, + }, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return nil, err + } + + for _, ip := range ips.IPs { + if ip != nil && ip.Resource != nil && !ip.IsIPv6 && ip.Address.IP != nil { + privateNICIPs[ip.Resource.ID] = ip.Address.IP.String() + } + } + } + + return privateNICIPs, nil +} diff --git a/discovery/scaleway/instance_test.go b/discovery/scaleway/instance_test.go index ae70a9ed253..3a6aad1f1cb 100644 --- a/discovery/scaleway/instance_test.go +++ b/discovery/scaleway/instance_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,7 +23,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" ) var ( @@ -60,7 +60,7 @@ api_url: %s tg := tgs[0] require.NotNil(t, tg) require.NotNil(t, tg.Targets) - require.Len(t, tg.Targets, 3) + require.Len(t, tg.Targets, 4) for i, lbls := range []model.LabelSet{ { @@ -125,6 +125,8 @@ api_url: %s "__meta_scaleway_instance_organization_id": "20b3d507-96ac-454c-a795-bc731b46b12f", "__meta_scaleway_instance_project_id": "20b3d507-96ac-454c-a795-bc731b46b12f", "__meta_scaleway_instance_public_ipv4": "51.158.183.115", + "__meta_scaleway_instance_public_ipv4_addresses": ",51.158.183.115,", + "__meta_scaleway_instance_public_ipv6_addresses": ",2001:bc8:1640:1568:dc00:ff:fe21:91b,", "__meta_scaleway_instance_region": "nl-ams", "__meta_scaleway_instance_security_group_id": "984414da-9fc2-49c0-a925-fed6266fe092", "__meta_scaleway_instance_security_group_name": "Default security group", @@ -132,6 +134,30 @@ api_url: %s "__meta_scaleway_instance_type": "DEV1-S", "__meta_scaleway_instance_zone": "nl-ams-1", }, + { + "__address__": "163.172.136.10:80", + "__meta_scaleway_instance_boot_type": "local", + "__meta_scaleway_instance_hostname": "multiple-ips", + "__meta_scaleway_instance_id": "658abbf4-e6c6-4239-a483-3307763cf6e0", + "__meta_scaleway_instance_image_arch": "x86_64", + "__meta_scaleway_instance_image_id": "f583f58c-1ea5-44ab-a1e6-2b2e7df32a86", + "__meta_scaleway_instance_image_name": "Ubuntu 24.04 Noble Numbat", + "__meta_scaleway_instance_location_cluster_id": "7", + "__meta_scaleway_instance_location_hypervisor_id": "801", + "__meta_scaleway_instance_location_node_id": "95", + "__meta_scaleway_instance_name": "multiple-ips", + "__meta_scaleway_instance_organization_id": "ee7bd9e1-9cbd-4724-b2f4-19e50f3cf38b", + "__meta_scaleway_instance_project_id": "ee7bd9e1-9cbd-4724-b2f4-19e50f3cf38b", + "__meta_scaleway_instance_public_ipv4": "163.172.136.10", + "__meta_scaleway_instance_public_ipv4_addresses": ",163.172.136.10,212.47.248.223,51.15.231.134,", + "__meta_scaleway_instance_public_ipv6_addresses": ",2001:bc8:710:4a69:dc00:ff:fe58:40c1,2001:bc8:710:d::,2001:bc8:710:5417::,", + "__meta_scaleway_instance_region": "fr-par", + "__meta_scaleway_instance_security_group_id": "0fe819c3-274d-472a-b3f5-ddb258d2d8bb", + "__meta_scaleway_instance_security_group_name": "Default security group", + "__meta_scaleway_instance_status": "running", + "__meta_scaleway_instance_type": "PLAY2-PICO", + "__meta_scaleway_instance_zone": "fr-par-1", + }, } { t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) { require.Equal(t, lbls, tg.Targets[i]) @@ -161,6 +187,95 @@ func mockScalewayInstance(w http.ResponseWriter, r *http.Request) { } } +func TestScalewayInstanceRefreshIPAMPrivateNIC(t *testing.T) { + mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Auth-Token") != testSecretKey { + http.Error(w, "bad token id", http.StatusUnauthorized) + return + } + + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/instance/v1/zones/fr-par-1/servers": + _, err := w.Write([]byte(`{ + "servers": [ + { + "id": "78d6aa69-6a5f-47bd-9f1a-3cdde6973d96", + "name": "ipam-only", + "organization": "cb334986-b054-4725-9d3a-40850fdc6015", + "project": "cb334986-b054-4725-9d5a-30850fdc6015", + "commercial_type": "DEV1-S", + "hostname": "ipam-only", + "state": "running", + "boot_type": "local", + "routed_ip_enabled": true, + "public_ip": null, + "public_ips": [], + "private_ip": null, + "private_nics": [ + { + "id": "2d17a5a0-f8e5-4e7f-a029-f7c8d182af53", + "server_id": "78d6aa69-6a5f-47bd-9f1a-3cdde6973d96", + "private_network_id": "18ba4ed4-d194-4b5d-9083-447325234a71", + "mac_address": "02:00:00:00:31:90", + "state": "available", + "zone": "fr-par-1" + } + ], + "zone": "fr-par-1" + } + ] + }`)) + require.NoError(t, err) + case "/ipam/v1/regions/fr-par/ips": + require.Equal(t, "2d17a5a0-f8e5-4e7f-a029-f7c8d182af53", r.URL.Query().Get("resource_ids")) + require.Equal(t, "instance_private_nic", r.URL.Query().Get("resource_types")) + _, err := w.Write([]byte(`{ + "ips": [ + { + "id": "b50244c4-b0b1-4cc9-a78c-26034b4967d3", + "address": "10.0.0.7/32", + "project_id": "cb334986-b054-4725-9d5a-30850fdc6015", + "is_ipv6": false, + "resource": { + "type": "instance_private_nic", + "id": "2d17a5a0-f8e5-4e7f-a029-f7c8d182af53" + }, + "region": "fr-par" + } + ], + "total_count": 1 + }`)) + require.NoError(t, err) + default: + http.Error(w, "bad url", http.StatusNotFound) + } + })) + defer mock.Close() + + cfgString := fmt.Sprintf(` +--- +role: instance +project_id: %s +secret_key: %s +access_key: %s +api_url: %s +`, testProjectID, testSecretKey, testAccessKey, mock.URL) + var cfg SDConfig + require.NoError(t, yaml.UnmarshalStrict([]byte(cfgString), &cfg)) + + d, err := newRefresher(&cfg) + require.NoError(t, err) + + tgs, err := d.refresh(context.Background()) + require.NoError(t, err) + + require.Len(t, tgs, 1) + require.Len(t, tgs[0].Targets, 1) + require.Equal(t, model.LabelValue("10.0.0.7:80"), tgs[0].Targets[0][model.AddressLabel]) + require.Equal(t, model.LabelValue("10.0.0.7"), tgs[0].Targets[0][instancePrivateIPv4Label]) +} + func TestScalewayInstanceAuthToken(t *testing.T) { mock := httptest.NewServer(http.HandlerFunc(mockScalewayInstance)) defer mock.Close() diff --git a/discovery/scaleway/metrics.go b/discovery/scaleway/metrics.go index a8277d0fb38..5871f7e31b6 100644 --- a/discovery/scaleway/metrics.go +++ b/discovery/scaleway/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type scalewayMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *scalewayMetrics) Register() error { +func (*scalewayMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *scalewayMetrics) Unregister() {} +func (*scalewayMetrics) Unregister() {} diff --git a/discovery/scaleway/scaleway.go b/discovery/scaleway/scaleway.go index f8e1a83f5ee..a804fbde37c 100644 --- a/discovery/scaleway/scaleway.go +++ b/discovery/scaleway/scaleway.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -22,7 +22,6 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -55,7 +54,7 @@ const ( ) // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *role) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *role) UnmarshalYAML(unmarshal func(any) error) error { if err := unmarshal((*string)(c)); err != nil { return err } @@ -105,13 +104,13 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &scalewayMetrics{ refreshMetrics: rmi, } } -func (c SDConfig) Name() string { +func (SDConfig) Name() string { return "scaleway" } @@ -125,7 +124,7 @@ func (c SDConfig) secretKeyForConfig() string { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -167,8 +166,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return c.HTTPClientConfig.Validate() } -func (c SDConfig) NewDiscoverer(options discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(&c, options.Logger, options.Metrics) +func (c SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(&c, opts) } // SetDirectory joins any relative file paths with dir. @@ -185,10 +184,10 @@ func init() { // the Discoverer interface. type Discovery struct{} -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { - m, ok := metrics.(*scalewayMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*refresh.Discovery, error) { + m, ok := opts.Metrics.(*scalewayMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } r, err := newRefresher(conf) @@ -198,8 +197,9 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere return refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "scaleway", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: r.refresh, MetricsInstantiator: m.refreshMetrics, @@ -221,6 +221,27 @@ func newRefresher(conf *SDConfig) (refresher, error) { return nil, errors.New("unknown Scaleway discovery role") } +// newScalewayHTTPClient creates an HTTP client from the SD config, optionally +// wrapping the transport with token-file authentication. +func newScalewayHTTPClient(conf *SDConfig) (*http.Client, error) { + client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "scaleway_sd") + if err != nil { + return nil, err + } + + client.Timeout = time.Duration(conf.RefreshInterval) + + if conf.SecretKeyFile != "" { + rt, err := newAuthTokenFileRoundTripper(conf.SecretKeyFile, client.Transport) + if err != nil { + return nil, err + } + client.Transport = rt + } + + return client, nil +} + func loadProfile(sdConfig *SDConfig) (*scw.Profile, error) { // Profile coming from Prometheus Configuration file prometheusConfigProfile := &scw.Profile{ diff --git a/discovery/scaleway/testdata/instance.json b/discovery/scaleway/testdata/instance.json index b433f7598e7..19a6106daa4 100644 --- a/discovery/scaleway/testdata/instance.json +++ b/discovery/scaleway/testdata/instance.json @@ -356,6 +356,222 @@ "placement_group": null, "private_nics": [], "zone": "nl-ams-1" + }, + { + "id": "658abbf4-e6c6-4239-a483-3307763cf6e0", + "name": "multiple-ips", + "arch": "x86_64", + "commercial_type": "PLAY2-PICO", + "boot_type": "local", + "organization": "ee7bd9e1-9cbd-4724-b2f4-19e50f3cf38b", + "project": "ee7bd9e1-9cbd-4724-b2f4-19e50f3cf38b", + "hostname": "multiple-ips", + "image": { + "id": "f583f58c-1ea5-44ab-a1e6-2b2e7df32a86", + "name": "Ubuntu 24.04 Noble Numbat", + "organization": "51b656e3-4865-41e8-adbc-0c45bdd780db", + "project": "51b656e3-4865-41e8-adbc-0c45bdd780db", + "root_volume": { + "id": "cfab1e2e-fa24-480a-a372-61b19e8e2fda", + "name": "Ubuntu 24.04 Noble Numbat", + "volume_type": "unified", + "size": 10000000000 + }, + "extra_volumes": { + + }, + "public": true, + "arch": "x86_64", + "creation_date": "2024-04-26T09:24:38.624912+00:00", + "modification_date": "2024-04-26T09:24:38.624912+00:00", + "default_bootscript": null, + "from_server": "", + "state": "available", + "tags": [ + + ], + "zone": "fr-par-1" + }, + "volumes": { + "0": { + "boot": false, + "id": "7d4dc5ae-3f3c-4f9c-91cf-4a47a2193e94", + "name": "Ubuntu 24.04 Noble Numbat", + "volume_type": "b_ssd", + "export_uri": null, + "organization": "ee7bd9e1-9cbd-4724-b2f4-19e50f3cf38b", + "project": "ee7bd9e1-9cbd-4724-b2f4-19e50f3cf38b", + "server": { + "id": "658abbf4-e6c6-4239-a483-3307763cf6e0", + "name": "multiple-ips" + }, + "size": 10000000000, + "state": "available", + "creation_date": "2024-06-07T13:33:17.697162+00:00", + "modification_date": "2024-06-07T13:33:17.697162+00:00", + "tags": [ + + ], + "zone": "fr-par-1" + } + }, + "tags": [ + + ], + "state": "running", + "protected": false, + "state_detail": "booted", + "public_ip": { + "id": "63fd9ede-58d7-482c-b21d-1d79d81a90dc", + "address": "163.172.136.10", + "dynamic": false, + "gateway": "62.210.0.1", + "netmask": "32", + "family": "inet", + "provisioning_mode": "dhcp", + "tags": [ + + ], + "state": "attached", + "ipam_id": "700644ed-f6a2-4c64-8508-bb867bc07673" + }, + "public_ips": [ + { + "id": "63fd9ede-58d7-482c-b21d-1d79d81a90dc", + "address": "163.172.136.10", + "dynamic": false, + "gateway": "62.210.0.1", + "netmask": "32", + "family": "inet", + "provisioning_mode": "dhcp", + "tags": [ + + ], + "state": "attached", + "ipam_id": "700644ed-f6a2-4c64-8508-bb867bc07673" + }, + { + "id": "eed4575b-90e5-4102-b956-df874c911e2b", + "address": "212.47.248.223", + "dynamic": false, + "gateway": "62.210.0.1", + "netmask": "32", + "family": "inet", + "provisioning_mode": "manual", + "tags": [ + + ], + "state": "attached", + "ipam_id": "e2bdef64-828b-4f4a-a56b-954a85759adf" + }, + { + "id": "fca8b329-6c0e-4f9c-aa88-d5c197b5919a", + "address": "51.15.231.134", + "dynamic": false, + "gateway": "62.210.0.1", + "netmask": "32", + "family": "inet", + "provisioning_mode": "manual", + "tags": [ + + ], + "state": "attached", + "ipam_id": "e56808db-b348-4b7e-ad23-995ae08dc1a1" + }, + { + "id": "3ffa6774-124c-4e64-8afb-148c15304b25", + "address": "2001:bc8:710:4a69:dc00:ff:fe58:40c1", + "dynamic": false, + "gateway": "fe80::dc00:ff:fe58:40c2", + "netmask": "64", + "family": "inet6", + "provisioning_mode": "slaac", + "tags": [ + + ], + "state": "attached", + "ipam_id": "d97773d9-fd2c-4085-92bb-5c471abd132e" + }, + { + "id": "28fcf539-8492-4603-b627-de0501ce8489", + "address": "2001:bc8:710:d::", + "dynamic": false, + "gateway": "fe80::dc00:ff:fe58:40c2", + "netmask": "64", + "family": "inet6", + "provisioning_mode": "manual", + "tags": [ + + ], + "state": "attached", + "ipam_id": "005d19ac-203c-4034-ab16-9ff4caadbdd5" + }, + { + "id": "db6fafda-3a12-403d-8c9c-1a1cb1c315ba", + "address": "2001:bc8:710:5417::", + "dynamic": false, + "gateway": "fe80::dc00:ff:fe58:40c2", + "netmask": "64", + "family": "inet6", + "provisioning_mode": "manual", + "tags": [ + + ], + "state": "attached", + "ipam_id": "8c62fac8-4134-462c-ba48-71ce4f8ac939" + } + ], + "mac_address": "de:00:00:58:40:c1", + "routed_ip_enabled": true, + "ipv6": null, + "extra_networks": [ + + ], + "dynamic_ip_required": false, + "enable_ipv6": false, + "private_ip": null, + "creation_date": "2024-06-07T13:33:17.697162+00:00", + "modification_date": "2024-06-07T13:33:26.021167+00:00", + "bootscript": { + "id": "fdfe150f-a870-4ce4-b432-9f56b5b995c1", + "public": true, + "title": "x86_64 mainline 4.4.230 rev1", + "architecture": "x86_64", + "organization": "11111111-1111-4111-8111-111111111111", + "project": "11111111-1111-4111-8111-111111111111", + "kernel": "http://10.194.3.9/kernel/x86_64-mainline-lts-4.4-4.4.230-rev1/vmlinuz-4.4.230", + "dtb": "", + "initrd": "http://10.194.3.9/initrd/initrd-Linux-x86_64-v3.14.6.gz", + "bootcmdargs": "LINUX_COMMON scaleway boot=local nbd.max_part=16", + "default": true, + "zone": "fr-par-1" + }, + "security_group": { + "id": "0fe819c3-274d-472a-b3f5-ddb258d2d8bb", + "name": "Default security group" + }, + "location": { + "zone_id": "par1", + "platform_id": "14", + "cluster_id": "7", + "hypervisor_id": "801", + "node_id": "95" + }, + "maintenances": [ + + ], + "allowed_actions": [ + "poweroff", + "terminate", + "reboot", + "stop_in_place", + "backup" + ], + "placement_group": null, + "private_nics": [ + + ], + "zone": "fr-par-1" } ] } \ No newline at end of file diff --git a/discovery/stackit/metrics.go b/discovery/stackit/metrics.go new file mode 100644 index 00000000000..a44d0728e35 --- /dev/null +++ b/discovery/stackit/metrics.go @@ -0,0 +1,32 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*stackitMetrics)(nil) + +type stackitMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (*stackitMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (*stackitMetrics) Unregister() {} diff --git a/discovery/stackit/mock_test.go b/discovery/stackit/mock_test.go new file mode 100644 index 00000000000..d1366508a36 --- /dev/null +++ b/discovery/stackit/mock_test.go @@ -0,0 +1,162 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// SDMock is the interface for the STACKIT IAAS API mock. +type SDMock struct { + t *testing.T + Server *httptest.Server + Mux *http.ServeMux +} + +// NewSDMock returns a new SDMock. +func NewSDMock(t *testing.T) *SDMock { + return &SDMock{ + t: t, + } +} + +// Endpoint returns the URI to the mock server. +func (m *SDMock) Endpoint() string { + return m.Server.URL + "/" +} + +// Setup creates the mock server. +func (m *SDMock) Setup() { + m.Mux = http.NewServeMux() + m.Server = httptest.NewServer(m.Mux) + m.t.Cleanup(m.Server.Close) +} + +// ShutdownServer creates the mock server. +func (m *SDMock) ShutdownServer() { + m.Server.Close() +} + +const ( + testToken = "LRK9DAWQ1ZAEFSrCNEEzLCUwhYX1U3g7wMg4dTlkkDC96fyDuyJ39nVbVjCKSDfj" + testProjectID = "00000000-0000-0000-0000-000000000000" +) + +// HandleServers mocks the STACKIT IAAS API. +func (m *SDMock) HandleServers() { + // /token endpoint mocks the token endpoint for service account authentication. + // It checks if the request body starts with "assertion=ey" to simulate a valid assertion + // as defined in RFC 7523. + m.Mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) { + reqBody, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = fmt.Fprint(w, err) + return + } + + // Expecting HTTP form encoded body with the field assertion. + // JWT always start with "ey" (base64url encoded). + if !bytes.HasPrefix(reqBody, []byte("assertion=ey")) { + w.WriteHeader(http.StatusUnauthorized) + return + } + + w.Header().Add("content-type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + + _, _ = fmt.Fprintf(w, `{"access_token": "%s"}`, testToken) + }) + + m.Mux.HandleFunc(fmt.Sprintf("/v1/projects/%s/servers", testProjectID), func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", testToken) { + w.WriteHeader(http.StatusUnauthorized) + return + } + + w.Header().Add("content-type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + + _, _ = fmt.Fprint(w, ` +{ + "items": [ + { + "availabilityZone": "eu01-3", + "bootVolume": { + "deleteOnTermination": false, + "id": "1c15e4cc-8474-46be-b875-b473ea9fe80c" + }, + "createdAt": "2025-03-12T14:48:17Z", + "id": "b4176700-596a-4f80-9fc8-5f9c58a606e1", + "labels": { + "provisionSTACKITServerAgent": "true", + "stackit_project_id": "00000000-0000-0000-0000-000000000000" + }, + "launchedAt": "2025-03-12T14:48:52Z", + "machineType": "g1.1", + "name": "runcommandtest", + "nics": [ + { + "ipv4": "10.0.0.153", + "mac": "fa:16:4f:42:1c:d3", + "networkId": "3173494f-2f6c-490d-8c12-4b3c86b4338b", + "networkName": "test", + "publicIp": "192.0.2.1", + "nicId": "b36097c5-e1c5-4e12-ae97-c03e144db127", + "nicSecurity": true, + "securityGroups": [ + "6e60809f-bed3-46c6-a39c-adddd6455674" + ] + } + ], + "powerStatus": "STOPPED", + "serviceAccountMails": [], + "status": "INACTIVE", + "updatedAt": "2025-03-13T07:08:29Z", + "userData": null, + "volumes": [ + "1c15e4cc-8474-46be-b875-b473ea9fe80c" + ] + }, + { + "availabilityZone": "eu01-m", + "bootVolume": { + "deleteOnTermination": false, + "id": "1e3ffe2b-878f-46e5-b39e-372e13a09551" + }, + "createdAt": "2025-04-10T16:45:25Z", + "id": "ee337436-1f15-4647-a03e-154009966179", + "labels": {}, + "launchedAt": "2025-04-10T16:46:00Z", + "machineType": "t1.1", + "name": "server1", + "nics": [], + "powerStatus": "RUNNING", + "serviceAccountMails": [], + "status": "ACTIVE", + "updatedAt": "2025-04-10T16:46:00Z", + "volumes": [ + "1e3ffe2b-878f-46e5-b39e-372e13a09551" + ] + } + ] +}`, + ) + }) +} diff --git a/discovery/stackit/server.go b/discovery/stackit/server.go new file mode 100644 index 00000000000..c5d76393a50 --- /dev/null +++ b/discovery/stackit/server.go @@ -0,0 +1,222 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/stackitcloud/stackit-sdk-go/core/auth" + stackitconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + stackitAPIEndpoint = "https://iaas.api.%s.stackit.cloud" + + stackitLabelPrivateIPv4 = stackitLabelPrefix + "private_ipv4_" + stackitLabelType = stackitLabelPrefix + "type" + stackitLabelLabel = stackitLabelPrefix + "label_" + stackitLabelLabelPresent = stackitLabelPrefix + "labelpresent_" +) + +// Discovery periodically performs STACKIT Cloud requests. +// It implements the Discoverer interface. +type iaasDiscovery struct { + *refresh.Discovery + httpClient *http.Client + logger *slog.Logger + apiEndpoint string + project string + port int +} + +// newServerDiscovery returns a new iaasDiscovery, which periodically refreshes its targets. +func newServerDiscovery(conf *SDConfig, logger *slog.Logger) (*iaasDiscovery, error) { + d := &iaasDiscovery{ + project: conf.Project, + port: conf.Port, + apiEndpoint: conf.Endpoint, + logger: logger, + } + + rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "stackit_sd") + if err != nil { + return nil, err + } + + d.apiEndpoint = conf.Endpoint + if d.apiEndpoint == "" { + d.apiEndpoint = fmt.Sprintf(stackitAPIEndpoint, conf.Region) + } + + servers := stackitconfig.ServerConfigurations{stackitconfig.ServerConfiguration{ + URL: d.apiEndpoint, + Description: "STACKIT IAAS API", + }} + + d.httpClient = &http.Client{ + Timeout: time.Duration(conf.RefreshInterval), + Transport: rt, + } + + stackitConfiguration := &stackitconfig.Configuration{ + UserAgent: userAgent, + HTTPClient: d.httpClient, + Servers: servers, + NoAuth: conf.ServiceAccountKey == "" && conf.ServiceAccountKeyPath == "", + + ServiceAccountKey: string(conf.ServiceAccountKey), + PrivateKey: string(conf.PrivateKey), + ServiceAccountKeyPath: conf.ServiceAccountKeyPath, + PrivateKeyPath: conf.PrivateKeyPath, + CredentialsFilePath: conf.CredentialsFilePath, + } + + if conf.tokenURL != "" { + stackitConfiguration.TokenCustomUrl = conf.tokenURL + } + + authRoundTripper, err := auth.SetupAuth(stackitConfiguration) + if err != nil { + return nil, fmt.Errorf("setting up authentication: %w", err) + } + + d.httpClient.Transport = authRoundTripper + + return d, nil +} + +func (i *iaasDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + apiURL, err := url.Parse(i.apiEndpoint) + if err != nil { + return nil, fmt.Errorf("invalid API endpoint URL %s: %w", i.apiEndpoint, err) + } + + apiURL.Path, err = url.JoinPath(apiURL.Path, "v1", "projects", i.project, "servers") + if err != nil { + return nil, fmt.Errorf("joining URL path: %w", err) + } + + q := apiURL.Query() + q.Set("details", "true") + apiURL.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL.String(), http.NoBody) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Accept", "application/json") + + res, err := i.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("sending request: %w", err) + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errorMessage, _ := io.ReadAll(res.Body) + + return nil, fmt.Errorf("unexpected status code %d: %s", res.StatusCode, string(errorMessage)) + } + + var serversResponse *ServerListResponse + + if err := json.NewDecoder(res.Body).Decode(&serversResponse); err != nil { + return nil, fmt.Errorf("decoding response: %w", err) + } + + if serversResponse == nil || serversResponse.Items == nil || len(*serversResponse.Items) == 0 { + return []*targetgroup.Group{{Source: "stackit", Targets: []model.LabelSet{}}}, nil + } + + targets := make([]model.LabelSet, 0, len(*serversResponse.Items)) + for _, server := range *serversResponse.Items { + if server.Nics == nil { + i.logger.Debug("server has no network interfaces. Skipping", slog.String("server_id", server.ID)) + continue + } + + labels := model.LabelSet{ + stackitLabelProject: model.LabelValue(i.project), + stackitLabelID: model.LabelValue(server.ID), + stackitLabelName: model.LabelValue(server.Name), + stackitLabelAvailabilityZone: model.LabelValue(server.AvailabilityZone), + stackitLabelStatus: model.LabelValue(server.Status), + stackitLabelPowerStatus: model.LabelValue(server.PowerStatus), + stackitLabelType: model.LabelValue(server.MachineType), + } + + var ( + addressLabel string + serverPublicIP string + ) + + for _, nic := range server.Nics { + if nic.PublicIP != nil && *nic.PublicIP != "" && serverPublicIP == "" { + serverPublicIP = *nic.PublicIP + addressLabel = serverPublicIP + } + + if nic.IPv4 != nil && *nic.IPv4 != "" { + networkLabel := model.LabelName(stackitLabelPrivateIPv4 + strutil.SanitizeLabelName(nic.NetworkName)) + labels[networkLabel] = model.LabelValue(*nic.IPv4) + if addressLabel == "" { + addressLabel = *nic.IPv4 + } + } + } + + if addressLabel == "" { + // Skip servers without IPs. + continue + } + + // Public IPs for servers are optional. + if serverPublicIP != "" { + labels[stackitLabelPublicIPv4] = model.LabelValue(serverPublicIP) + } + + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(addressLabel, strconv.FormatUint(uint64(i.port), 10))) + + for labelKey, labelValue := range server.Labels { + if labelStringValue, ok := labelValue.(string); ok { + presentLabel := model.LabelName(stackitLabelLabelPresent + strutil.SanitizeLabelName(labelKey)) + labels[presentLabel] = "true" + + label := model.LabelName(stackitLabelLabel + strutil.SanitizeLabelName(labelKey)) + labels[label] = model.LabelValue(labelStringValue) + } + } + + targets = append(targets, labels) + } + + return []*targetgroup.Group{{Source: "stackit", Targets: targets}}, nil +} diff --git a/discovery/stackit/server_test.go b/discovery/stackit/server_test.go new file mode 100644 index 00000000000..3859c542de4 --- /dev/null +++ b/discovery/stackit/server_test.go @@ -0,0 +1,132 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "testing" + + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "github.com/stretchr/testify/require" +) + +type serverSDTestSuite struct { + Mock *SDMock +} + +func (s *serverSDTestSuite) SetupTest(t *testing.T) { + s.Mock = NewSDMock(t) + s.Mock.Setup() + + s.Mock.HandleServers() +} + +func TestServerSDRefresh(t *testing.T) { + for _, tc := range []struct { + name string + cfg SDConfig + }{ + { + name: "default with token", + cfg: func() SDConfig { + cfg := DefaultSDConfig + cfg.HTTPClientConfig.BearerToken = testToken + + return cfg + }(), + }, + { + name: "default with service account key", + cfg: func() SDConfig { + // Generate a new RSA key pair with a size of 2048 bits + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + cfg := DefaultSDConfig + cfg.PrivateKey = config.Secret(pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + })) + + cfg.ServiceAccountKey = config.Secret(`{ + "Active": true, + "CreatedAt": "2025-04-05T12:34:56Z", + "Credentials": { + "Aud": "https://stackit-service-account-prod.apps.01.cf.eu01.stackit.cloud", + "Iss": "stackit@sa.stackit.cloud", + "Kid": "123e4567-e89b-12d3-a456-426614174000", + "Sub": "123e4567-e89b-12d3-a456-426614174001" + }, + "ID": "123e4567-e89b-12d3-a456-426614174002", + "KeyAlgorithm": "RSA_2048", + "KeyOrigin": "USER_PROVIDED", + "KeyType": "USER_MANAGED", + "PublicKey": "...", + "ValidUntil": "2025-04-05T13:34:56Z" +}`) + + return cfg + }(), + }, + } { + t.Run(tc.name, func(t *testing.T) { + suite := &serverSDTestSuite{} + suite.SetupTest(t) + defer suite.Mock.ShutdownServer() + + tc.cfg.Endpoint = suite.Mock.Endpoint() + tc.cfg.tokenURL = suite.Mock.Endpoint() + "token" + tc.cfg.Project = testProjectID + + d, err := newServerDiscovery(&tc.cfg, promslog.NewNopLogger()) + require.NoError(t, err) + + targetGroups, err := d.refresh(context.Background()) + require.NoError(t, err) + require.Len(t, targetGroups, 1) + + targetGroup := targetGroups[0] + require.NotNil(t, targetGroup, "targetGroup should not be nil") + require.NotNil(t, targetGroup.Targets, "targetGroup.targets should not be nil") + require.Len(t, targetGroup.Targets, 1) + + for i, labelSet := range []model.LabelSet{ + { + "__address__": model.LabelValue("192.0.2.1:80"), + "__meta_stackit_project": model.LabelValue("00000000-0000-0000-0000-000000000000"), + "__meta_stackit_id": model.LabelValue("b4176700-596a-4f80-9fc8-5f9c58a606e1"), + "__meta_stackit_type": model.LabelValue("g1.1"), + "__meta_stackit_private_ipv4_test": model.LabelValue("10.0.0.153"), + "__meta_stackit_public_ipv4": model.LabelValue("192.0.2.1"), + "__meta_stackit_labelpresent_provisionSTACKITServerAgent": model.LabelValue("true"), + "__meta_stackit_label_provisionSTACKITServerAgent": model.LabelValue("true"), + "__meta_stackit_labelpresent_stackit_project_id": model.LabelValue("true"), + "__meta_stackit_name": model.LabelValue("runcommandtest"), + "__meta_stackit_availability_zone": model.LabelValue("eu01-3"), + "__meta_stackit_status": model.LabelValue("INACTIVE"), + "__meta_stackit_power_status": model.LabelValue("STOPPED"), + "__meta_stackit_label_stackit_project_id": model.LabelValue("00000000-0000-0000-0000-000000000000"), + }, + } { + require.Equal(t, labelSet, targetGroup.Targets[i]) + } + }) + } +} diff --git a/discovery/stackit/stackit.go b/discovery/stackit/stackit.go new file mode 100644 index 00000000000..fe5139429f7 --- /dev/null +++ b/discovery/stackit/stackit.go @@ -0,0 +1,154 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + stackitLabelPrefix = model.MetaLabelPrefix + "stackit_" + stackitLabelProject = stackitLabelPrefix + "project" + stackitLabelID = stackitLabelPrefix + "id" + stackitLabelName = stackitLabelPrefix + "name" + stackitLabelStatus = stackitLabelPrefix + "status" + stackitLabelPowerStatus = stackitLabelPrefix + "power_status" + stackitLabelAvailabilityZone = stackitLabelPrefix + "availability_zone" + stackitLabelPublicIPv4 = stackitLabelPrefix + "public_ipv4" +) + +var userAgent = version.PrometheusUserAgent() + +// DefaultSDConfig is the default STACKIT SD configuration. +var DefaultSDConfig = SDConfig{ + Region: "eu01", + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +func init() { + discovery.RegisterConfig(&SDConfig{}) +} + +// SDConfig is the configuration for STACKIT based service discovery. +type SDConfig struct { + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` + + Project string `yaml:"project"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port,omitempty"` + Region string `yaml:"region,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` + ServiceAccountKey config.Secret `yaml:"service_account_key,omitempty"` + PrivateKey config.Secret `yaml:"private_key,omitempty"` + ServiceAccountKeyPath string `yaml:"service_account_key_path,omitempty"` + PrivateKeyPath string `yaml:"private_key_path,omitempty"` + CredentialsFilePath string `yaml:"credentials_file_path,omitempty"` + + // For testing only + tokenURL string +} + +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &stackitMetrics{ + refreshMetrics: rmi, + } +} + +// Name returns the name of the Config. +func (*SDConfig) Name() string { return "stackit" } + +// NewDiscoverer returns a Discoverer for the Config. +func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(c, opts) +} + +type refresher interface { + refresh(context.Context) ([]*targetgroup.Group, error) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + + if c.Endpoint == "" && c.Region == "" { + return errors.New("stackit_sd: endpoint and region missing") + } + + if _, err = url.Parse(c.Endpoint); err != nil { + return fmt.Errorf("stackit_sd: invalid endpoint %q: %w", c.Endpoint, err) + } + + return c.HTTPClientConfig.Validate() +} + +// SetDirectory joins any relative file paths with dir. +func (c *SDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// Discovery periodically performs STACKIT API requests. It implements +// the Discoverer interface. +type Discovery struct { + *refresh.Discovery +} + +// NewDiscovery returns a new Discovery which periodically refreshes its targets. +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*refresh.Discovery, error) { + m, ok := opts.Metrics.(*stackitMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type") + } + + r, err := newRefresher(conf, opts.Logger) + if err != nil { + return nil, err + } + + return refresh.NewDiscovery( + refresh.Options{ + Logger: opts.Logger, + Mech: "stackit", + SetName: opts.SetName, + Interval: time.Duration(conf.RefreshInterval), + RefreshF: r.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ), nil +} + +func newRefresher(conf *SDConfig, l *slog.Logger) (refresher, error) { + return newServerDiscovery(conf, l) +} diff --git a/discovery/stackit/types.go b/discovery/stackit/types.go new file mode 100644 index 00000000000..575acbbe561 --- /dev/null +++ b/discovery/stackit/types.go @@ -0,0 +1,38 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +// ServerListResponse Response object for server list request. +// https://docs.api.eu01.stackit.cloud/documentation/iaas/version/v1#tag/Servers/operation/v1ListServersInProject +type ServerListResponse struct { + Items *[]Server `json:"items"` +} + +type Server struct { + AvailabilityZone string `json:"availabilityZone"` + ID string `json:"id"` + Labels map[string]any `json:"labels"` + MachineType string `json:"machineType"` + Name string `json:"name"` + Nics []ServerNetwork `json:"nics"` + PowerStatus string `json:"powerStatus"` + Status string `json:"status"` +} + +// ServerNetwork Describes the object that matches servers to its networks. +type ServerNetwork struct { + NetworkName string `json:"networkName"` + IPv4 *string `json:"ipv4,omitempty"` + PublicIP *string `json:"publicIp,omitempty"` +} diff --git a/discovery/targetgroup/targetgroup.go b/discovery/targetgroup/targetgroup.go index e74870f0462..4b1670ae1bf 100644 --- a/discovery/targetgroup/targetgroup.go +++ b/discovery/targetgroup/targetgroup.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -37,7 +37,7 @@ func (tg Group) String() string { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (tg *Group) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (tg *Group) UnmarshalYAML(unmarshal func(any) error) error { g := struct { Targets []string `yaml:"targets"` Labels model.LabelSet `yaml:"labels"` @@ -56,7 +56,7 @@ func (tg *Group) UnmarshalYAML(unmarshal func(interface{}) error) error { } // MarshalYAML implements the yaml.Marshaler interface. -func (tg Group) MarshalYAML() (interface{}, error) { +func (tg Group) MarshalYAML() (any, error) { g := &struct { Targets []string `yaml:"targets"` Labels model.LabelSet `yaml:"labels,omitempty"` diff --git a/discovery/targetgroup/targetgroup_test.go b/discovery/targetgroup/targetgroup_test.go index 1e79859c6ef..1c1583d33d7 100644 --- a/discovery/targetgroup/targetgroup_test.go +++ b/discovery/targetgroup/targetgroup_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,7 +19,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" ) func TestTargetGroupStrictJSONUnmarshal(t *testing.T) { @@ -93,7 +93,7 @@ func TestTargetGroupJSONMarshal(t *testing.T) { } func TestTargetGroupYamlMarshal(t *testing.T) { - marshal := func(g interface{}) []byte { + marshal := func(g any) []byte { d, err := yaml.Marshal(g) if err != nil { panic(err) @@ -134,8 +134,8 @@ func TestTargetGroupYamlMarshal(t *testing.T) { } func TestTargetGroupYamlUnmarshal(t *testing.T) { - unmarshal := func(d []byte) func(interface{}) error { - return func(o interface{}) error { + unmarshal := func(d []byte) func(any) error { + return func(o any) error { return yaml.Unmarshal(d, o) } } diff --git a/discovery/triton/metrics.go b/discovery/triton/metrics.go index 3e34a840af1..2d4193ee1fc 100644 --- a/discovery/triton/metrics.go +++ b/discovery/triton/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type tritonMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *tritonMetrics) Register() error { +func (*tritonMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *tritonMetrics) Unregister() {} +func (*tritonMetrics) Unregister() {} diff --git a/discovery/triton/triton.go b/discovery/triton/triton.go index 675149f2a30..b234545d692 100644 --- a/discovery/triton/triton.go +++ b/discovery/triton/triton.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,7 +24,6 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/mwitkow/go-conntrack" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -71,7 +70,7 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &tritonMetrics{ refreshMetrics: rmi, } @@ -82,7 +81,7 @@ func (*SDConfig) Name() string { return "triton" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return New(opts.Logger, c, opts.Metrics) + return New(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -91,7 +90,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -146,10 +145,10 @@ type Discovery struct { } // New returns a new Discovery which periodically refreshes its targets. -func New(logger log.Logger, conf *SDConfig, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*tritonMetrics) +func New(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*tritonMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } tls, err := config.NewTLSConfig(&conf.TLSConfig) @@ -173,8 +172,9 @@ func New(logger log.Logger, conf *SDConfig, metrics discovery.DiscovererMetrics) } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "triton", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -211,7 +211,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { endpoint = fmt.Sprintf("%s?groups=%s", endpoint, groups) } - req, err := http.NewRequest(http.MethodGet, endpoint, nil) + req, err := http.NewRequest(http.MethodGet, endpoint, http.NoBody) if err != nil { return nil, err } diff --git a/discovery/triton/triton_test.go b/discovery/triton/triton_test.go index e37693e6bfc..f2b6398bc80 100644 --- a/discovery/triton/triton_test.go +++ b/discovery/triton/triton_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,7 +21,6 @@ import ( "net/http/httptest" "net/url" "strconv" - "strings" "testing" "github.com/prometheus/client_golang/prometheus" @@ -84,14 +83,17 @@ var ( func newTritonDiscovery(c SDConfig) (*Discovery, discovery.DiscovererMetrics, error) { reg := prometheus.NewRegistry() refreshMetrics := discovery.NewRefreshMetrics(reg) - // TODO(ptodev): Add the ability to unregister refresh metrics. metrics := c.NewDiscovererMetrics(reg, refreshMetrics) err := metrics.Register() if err != nil { return nil, nil, err } - d, err := New(nil, &c, metrics) + d, err := New(&c, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "triton", + }) if err != nil { return nil, nil, err } @@ -182,8 +184,7 @@ func TestTritonSDRefreshNoServer(t *testing.T) { td, m, _ := newTritonDiscovery(conf) _, err := td.refresh(context.Background()) - require.Error(t, err) - require.True(t, strings.Contains(err.Error(), "an error occurred when requesting targets from the discovery endpoint")) + require.ErrorContains(t, err, "an error occurred when requesting targets from the discovery endpoint") m.Unregister() } @@ -193,8 +194,7 @@ func TestTritonSDRefreshCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() _, err := td.refresh(ctx) - require.Error(t, err) - require.True(t, strings.Contains(err.Error(), context.Canceled.Error())) + require.ErrorContains(t, err, context.Canceled.Error()) m.Unregister() } @@ -233,7 +233,7 @@ func TestTritonSDRefreshCNsWithHostname(t *testing.T) { func testTritonSDRefresh(t *testing.T, c SDConfig, dstr string) []model.LabelSet { var ( td, m, _ = newTritonDiscovery(c) - s = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintln(w, dstr) })) ) diff --git a/discovery/util.go b/discovery/util.go index 4e2a0885186..064a5312a7a 100644 --- a/discovery/util.go +++ b/discovery/util.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/uyuni/metrics.go b/discovery/uyuni/metrics.go index 6793b592069..e1a9fd4db05 100644 --- a/discovery/uyuni/metrics.go +++ b/discovery/uyuni/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type uyuniMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *uyuniMetrics) Register() error { +func (*uyuniMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *uyuniMetrics) Unregister() {} +func (*uyuniMetrics) Unregister() {} diff --git a/discovery/uyuni/uyuni.go b/discovery/uyuni/uyuni.go index c8af2f15878..6f29fa130c8 100644 --- a/discovery/uyuni/uyuni.go +++ b/discovery/uyuni/uyuni.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,6 +17,7 @@ import ( "context" "errors" "fmt" + "log/slog" "net/http" "net/url" "path" @@ -24,7 +25,6 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/kolo/xmlrpc" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" @@ -41,10 +41,10 @@ const ( uyuniMetaLabelPrefix = model.MetaLabelPrefix + "uyuni_" uyuniLabelMinionHostname = uyuniMetaLabelPrefix + "minion_hostname" uyuniLabelPrimaryFQDN = uyuniMetaLabelPrefix + "primary_fqdn" - uyuniLablelSystemID = uyuniMetaLabelPrefix + "system_id" - uyuniLablelGroups = uyuniMetaLabelPrefix + "groups" - uyuniLablelEndpointName = uyuniMetaLabelPrefix + "endpoint_name" - uyuniLablelExporter = uyuniMetaLabelPrefix + "exporter" + uyuniLabelSystemID = uyuniMetaLabelPrefix + "system_id" + uyuniLabelGroups = uyuniMetaLabelPrefix + "groups" + uyuniLabelEndpointName = uyuniMetaLabelPrefix + "endpoint_name" + uyuniLabelExporter = uyuniMetaLabelPrefix + "exporter" uyuniLabelProxyModule = uyuniMetaLabelPrefix + "proxy_module" uyuniLabelMetricsPath = uyuniMetaLabelPrefix + "metrics_path" uyuniLabelScheme = uyuniMetaLabelPrefix + "scheme" @@ -109,11 +109,11 @@ type Discovery struct { entitlement string separator string interval time.Duration - logger log.Logger + logger *slog.Logger } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &uyuniMetrics{ refreshMetrics: rmi, } @@ -124,7 +124,7 @@ func (*SDConfig) Name() string { return "uyuni" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -133,7 +133,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig err := unmarshal((*plain)(c)) @@ -141,18 +141,22 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } if c.Server == "" { + //nolint:staticcheck // Capitalized first word. return errors.New("Uyuni SD configuration requires server host") } _, err = url.Parse(c.Server) if err != nil { + //nolint:staticcheck // Capitalized first word. return fmt.Errorf("Uyuni Server URL is not valid: %w", err) } if c.Username == "" { + //nolint:staticcheck // Capitalized first word. return errors.New("Uyuni SD configuration requires a username") } if c.Password == "" { + //nolint:staticcheck // Capitalized first word. return errors.New("Uyuni SD configuration requires a password") } return c.HTTPClientConfig.Validate() @@ -160,7 +164,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { func login(rpcclient *xmlrpc.Client, user, pass string, duration int) (string, error) { var result string - err := rpcclient.Call("auth.login", []interface{}{user, pass, duration}, &result) + err := rpcclient.Call("auth.login", []any{user, pass, duration}, &result) return result, err } @@ -170,7 +174,7 @@ func getSystemGroupsInfoOfMonitoredClients(rpcclient *xmlrpc.Client, token, enti SystemGroups []systemGroupID `xmlrpc:"system_groups"` } - err := rpcclient.Call("system.listSystemGroupsForSystemsWithEntitlement", []interface{}{token, entitlement}, &systemGroupsInfos) + err := rpcclient.Call("system.listSystemGroupsForSystemsWithEntitlement", []any{token, entitlement}, &systemGroupsInfos) if err != nil { return nil, err } @@ -184,7 +188,7 @@ func getSystemGroupsInfoOfMonitoredClients(rpcclient *xmlrpc.Client, token, enti func getNetworkInformationForSystems(rpcclient *xmlrpc.Client, token string, systemIDs []int) (map[int]networkInfo, error) { var networkInfos []networkInfo - err := rpcclient.Call("system.getNetworkForSystems", []interface{}{token, systemIDs}, &networkInfos) + err := rpcclient.Call("system.getNetworkForSystems", []any{token, systemIDs}, &networkInfos) if err != nil { return nil, err } @@ -204,18 +208,15 @@ func getEndpointInfoForSystems( var endpointInfos []endpointInfo err := rpcclient.Call( "system.monitoring.listEndpoints", - []interface{}{token, systemIDs}, &endpointInfos) - if err != nil { - return nil, err - } + []any{token, systemIDs}, &endpointInfos) return endpointInfos, err } // NewDiscovery returns a uyuni discovery for the given configuration. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*uyuniMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*uyuniMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } apiURL, err := url.Parse(conf.Server) @@ -237,13 +238,14 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere entitlement: conf.Entitlement, separator: conf.Separator, interval: time.Duration(conf.RefreshInterval), - logger: logger, + logger: opts.Logger, } d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "uyuni", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -270,10 +272,10 @@ func (d *Discovery) getEndpointLabels( model.AddressLabel: model.LabelValue(addr), uyuniLabelMinionHostname: model.LabelValue(networkInfo.Hostname), uyuniLabelPrimaryFQDN: model.LabelValue(networkInfo.PrimaryFQDN), - uyuniLablelSystemID: model.LabelValue(strconv.Itoa(endpoint.SystemID)), - uyuniLablelGroups: model.LabelValue(strings.Join(managedGroupNames, d.separator)), - uyuniLablelEndpointName: model.LabelValue(endpoint.EndpointName), - uyuniLablelExporter: model.LabelValue(endpoint.ExporterName), + uyuniLabelSystemID: model.LabelValue(strconv.Itoa(endpoint.SystemID)), + uyuniLabelGroups: model.LabelValue(strings.Join(managedGroupNames, d.separator)), + uyuniLabelEndpointName: model.LabelValue(endpoint.EndpointName), + uyuniLabelExporter: model.LabelValue(endpoint.ExporterName), uyuniLabelProxyModule: model.LabelValue(endpoint.Module), uyuniLabelMetricsPath: model.LabelValue(endpoint.Path), uyuniLabelScheme: model.LabelValue(scheme), @@ -329,7 +331,7 @@ func (d *Discovery) getTargetsForSystems( return result, nil } -func (d *Discovery) refresh(_ context.Context) ([]*targetgroup.Group, error) { +func (d *Discovery) refresh(context.Context) ([]*targetgroup.Group, error) { rpcClient, err := xmlrpc.NewClient(d.apiURL.String(), d.roundTripper) if err != nil { return nil, err diff --git a/discovery/uyuni/uyuni_test.go b/discovery/uyuni/uyuni_test.go index 09be23e2b4f..8f5330f0f73 100644 --- a/discovery/uyuni/uyuni_test.go +++ b/discovery/uyuni/uyuni_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -21,9 +21,8 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -48,7 +47,11 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err defer metrics.Unregister() defer refreshMetrics.Unregister() - md, err := NewDiscovery(&conf, nil, metrics) + md, err := NewDiscovery(&conf, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "uyuni", + }) if err != nil { return nil, err } @@ -57,9 +60,10 @@ func testUpdateServices(respHandler http.HandlerFunc) ([]*targetgroup.Group, err } func TestUyuniSDHandleError(t *testing.T) { + t.Parallel() var ( errTesting = "unable to login to Uyuni API: request error: bad status code - 500" - respHandler = func(w http.ResponseWriter, r *http.Request) { + respHandler = func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, ``) @@ -72,10 +76,11 @@ func TestUyuniSDHandleError(t *testing.T) { } func TestUyuniSDLogin(t *testing.T) { + t.Parallel() var ( errTesting = "unable to get the managed system groups information of monitored clients: request error: bad status code - 500" call = 0 - respHandler = func(w http.ResponseWriter, r *http.Request) { + respHandler = func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/xml") switch call { case 0: @@ -104,9 +109,10 @@ func TestUyuniSDLogin(t *testing.T) { } func TestUyuniSDSkipLogin(t *testing.T) { + t.Parallel() var ( errTesting = "unable to get the managed system groups information of monitored clients: request error: bad status code - 500" - respHandler = func(w http.ResponseWriter, r *http.Request) { + respHandler = func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, ``) @@ -128,7 +134,11 @@ func TestUyuniSDSkipLogin(t *testing.T) { defer metrics.Unregister() defer refreshMetrics.Unregister() - md, err := NewDiscovery(&conf, nil, metrics) + md, err := NewDiscovery(&conf, discovery.DiscovererOptions{ + Logger: nil, + Metrics: metrics, + SetName: "uyuni", + }) if err != nil { t.Error(err) } diff --git a/discovery/vultr/metrics.go b/discovery/vultr/metrics.go index 193436aad2e..823fe4bdc0c 100644 --- a/discovery/vultr/metrics.go +++ b/discovery/vultr/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ type vultrMetrics struct { } // Register implements discovery.DiscovererMetrics. -func (m *vultrMetrics) Register() error { +func (*vultrMetrics) Register() error { return nil } // Unregister implements discovery.DiscovererMetrics. -func (m *vultrMetrics) Unregister() {} +func (*vultrMetrics) Unregister() {} diff --git a/discovery/vultr/mock_test.go b/discovery/vultr/mock_test.go index bfc24d06fbc..03e5952dd02 100644 --- a/discovery/vultr/mock_test.go +++ b/discovery/vultr/mock_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/discovery/vultr/vultr.go b/discovery/vultr/vultr.go index aaa9c64e47a..56d8862a203 100644 --- a/discovery/vultr/vultr.go +++ b/discovery/vultr/vultr.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,6 +15,7 @@ package vultr import ( "context" + "errors" "fmt" "net" "net/http" @@ -22,12 +23,11 @@ import ( "strings" "time" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" - "github.com/vultr/govultr/v2" + "github.com/vultr/govultr/v3" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/refresh" @@ -75,7 +75,7 @@ type SDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*SDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &vultrMetrics{ refreshMetrics: rmi, } @@ -86,7 +86,7 @@ func (*SDConfig) Name() string { return "vultr" } // NewDiscoverer returns a Discoverer for the Config. func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { - return NewDiscovery(c, opts.Logger, opts.Metrics) + return NewDiscovery(c, opts) } // SetDirectory joins any relative file paths with dir. @@ -95,7 +95,7 @@ func (c *SDConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultSDConfig type plain SDConfig if err := unmarshal((*plain)(c)); err != nil { @@ -114,10 +114,10 @@ type Discovery struct { } // NewDiscovery returns a new Discovery which periodically refreshes its targets. -func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (*Discovery, error) { - m, ok := metrics.(*vultrMetrics) +func NewDiscovery(conf *SDConfig, opts discovery.DiscovererOptions) (*Discovery, error) { + m, ok := opts.Metrics.(*vultrMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } d := &Discovery{ @@ -134,16 +134,13 @@ func NewDiscovery(conf *SDConfig, logger log.Logger, metrics discovery.Discovere Timeout: time.Duration(conf.RefreshInterval), }) - d.client.SetUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)) - - if err != nil { - return nil, fmt.Errorf("error setting up vultr agent: %w", err) - } + d.client.SetUserAgent(version.PrometheusUserAgent()) d.Discovery = refresh.NewDiscovery( refresh.Options{ - Logger: logger, + Logger: opts.Logger, Mech: "vultr", + SetName: opts.SetName, Interval: time.Duration(conf.RefreshInterval), RefreshF: d.refresh, MetricsInstantiator: m.refreshMetrics, @@ -210,10 +207,13 @@ func (d *Discovery) listInstances(ctx context.Context) ([]govultr.Instance, erro } for { - pagedInstances, meta, err := d.client.Instance.List(ctx, listOptions) + pagedInstances, meta, resp, err := d.client.Instance.List(ctx, listOptions) if err != nil { return nil, err } + if resp != nil && resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("vultr API returned unexpected status %d", resp.StatusCode) + } instances = append(instances, pagedInstances...) if meta.Links.Next == "" { diff --git a/discovery/vultr/vultr_test.go b/discovery/vultr/vultr_test.go index 2f12a35529f..d116c419b7c 100644 --- a/discovery/vultr/vultr_test.go +++ b/discovery/vultr/vultr_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,9 +19,9 @@ import ( "net/url" "testing" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/discovery" @@ -57,7 +57,11 @@ func TestVultrSDRefresh(t *testing.T) { defer metrics.Unregister() defer refreshMetrics.Unregister() - d, err := NewDiscovery(&cfg, log.NewNopLogger(), metrics) + d, err := NewDiscovery(&cfg, discovery.DiscovererOptions{ + Logger: promslog.NewNopLogger(), + Metrics: metrics, + SetName: "vultr", + }) require.NoError(t, err) endpoint, err := url.Parse(sdMock.Mock.Endpoint()) require.NoError(t, err) diff --git a/discovery/xds/client.go b/discovery/xds/client.go index 027ceb27155..b9d1772d3dc 100644 --- a/discovery/xds/client.go +++ b/discovery/xds/client.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -30,7 +30,7 @@ import ( "github.com/prometheus/common/version" ) -var userAgent = fmt.Sprintf("Prometheus/%s", version.Version) +var userAgent = version.PrometheusUserAgent() // ResourceClient exposes the xDS protocol for a single resource type. // See https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#rest-json-polling-subscriptions . @@ -93,7 +93,7 @@ func NewHTTPResourceClient(conf *HTTPResourceClientConfig, protocolVersion Proto return nil, errors.New("only the v3 protocol is supported") } - if len(conf.Server) == 0 { + if conf.Server == "" { return nil, errors.New("empty xDS server") } @@ -132,7 +132,7 @@ func makeXDSResourceHTTPEndpointURL(protocolVersion ProtocolVersion, serverURL * return nil, errors.New("empty xDS server URL") } - if len(serverURL.Scheme) == 0 || len(serverURL.Host) == 0 { + if serverURL.Scheme == "" || serverURL.Host == "" { return nil, errors.New("invalid xDS server URL") } diff --git a/discovery/xds/client_test.go b/discovery/xds/client_test.go index b699995fb7d..e6639021613 100644 --- a/discovery/xds/client_test.go +++ b/discovery/xds/client_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -49,22 +49,26 @@ func urlMustParse(str string) *url.URL { } func TestMakeXDSResourceHttpEndpointEmptyServerURLScheme(t *testing.T) { + t.Parallel() + endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("127.0.0.1"), "monitoring") require.Empty(t, endpointURL) - require.Error(t, err) - require.Equal(t, "invalid xDS server URL", err.Error()) + require.EqualError(t, err, "invalid xDS server URL") } func TestMakeXDSResourceHttpEndpointEmptyServerURLHost(t *testing.T) { + t.Parallel() + endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("grpc://127.0.0.1"), "monitoring") require.Empty(t, endpointURL) - require.Error(t, err) - require.Contains(t, err.Error(), "must be either 'http' or 'https'") + require.ErrorContains(t, err, "must be either 'http' or 'https'") } func TestMakeXDSResourceHttpEndpoint(t *testing.T) { + t.Parallel() + endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("http://127.0.0.1:5000"), "monitoring") require.NoError(t, err) @@ -72,6 +76,8 @@ func TestMakeXDSResourceHttpEndpoint(t *testing.T) { } func TestCreateNewHTTPResourceClient(t *testing.T) { + t.Parallel() + c := &HTTPResourceClientConfig{ HTTPClientConfig: sdConf.HTTPClientConfig, Name: "test", @@ -108,7 +114,9 @@ func createTestHTTPResourceClient(t *testing.T, conf *HTTPResourceClientConfig, } func TestHTTPResourceClientFetchEmptyResponse(t *testing.T) { - client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { + t.Parallel() + + client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(*v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { return nil, nil }) defer cleanup() @@ -119,6 +127,8 @@ func TestHTTPResourceClientFetchEmptyResponse(t *testing.T) { } func TestHTTPResourceClientFetchFullResponse(t *testing.T) { + t.Parallel() + client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { if request.VersionInfo == "1" { return nil, nil @@ -148,7 +158,9 @@ func TestHTTPResourceClientFetchFullResponse(t *testing.T) { } func TestHTTPResourceClientServerError(t *testing.T) { - client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { + t.Parallel() + + client, cleanup := createTestHTTPResourceClient(t, testHTTPResourceConfig(), ProtocolV3, func(*v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { return nil, errors.New("server error") }) defer cleanup() diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go index d1d540aaf4d..6ebfad7de1b 100644 --- a/discovery/xds/kuma.go +++ b/discovery/xds/kuma.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,15 +14,16 @@ package xds import ( + "errors" "fmt" + "log/slog" "net/url" "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "google.golang.org/protobuf/types/known/anypb" "github.com/prometheus/prometheus/discovery" @@ -64,7 +65,7 @@ func (*KumaSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discove } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultKumaSDConfig type plainKumaConf KumaSDConfig err := unmarshal((*plainKumaConf)(c)) @@ -72,7 +73,7 @@ func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } - if len(c.Server) == 0 { + if c.Server == "" { return fmt.Errorf("kuma SD server must not be empty: %s", c.Server) } parsedURL, err := url.Parse(c.Server) @@ -80,14 +81,14 @@ func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } - if len(parsedURL.Scheme) == 0 || len(parsedURL.Host) == 0 { + if parsedURL.Scheme == "" || parsedURL.Host == "" { return fmt.Errorf("kuma SD server must not be empty and have a scheme: %s", c.Server) } return c.HTTPClientConfig.Validate() } -func (c *KumaSDConfig) Name() string { +func (*KumaSDConfig) Name() string { return "kuma" } @@ -99,7 +100,7 @@ func (c *KumaSDConfig) SetDirectory(dir string) { func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { logger := opts.Logger if logger == nil { - logger = log.NewNopLogger() + logger = promslog.NewNopLogger() } return NewKumaHTTPDiscovery(c, logger, opts.Metrics) @@ -158,10 +159,10 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.L return targets, nil } -func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, metrics discovery.DiscovererMetrics) (discovery.Discoverer, error) { +func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger *slog.Logger, metrics discovery.DiscovererMetrics) (discovery.Discoverer, error) { m, ok := metrics.(*xdsMetrics) if !ok { - return nil, fmt.Errorf("invalid discovery metrics type") + return nil, errors.New("invalid discovery metrics type") } // Default to "prometheus" if hostname is unavailable. @@ -170,7 +171,7 @@ func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger, metrics discove var err error clientID, err = osutil.GetFQDN() if err != nil { - level.Debug(logger).Log("msg", "error getting FQDN", "err", err) + logger.Debug("error getting FQDN", "err", err) clientID = "prometheus" } } diff --git a/discovery/xds/kuma_mads.pb.go b/discovery/xds/kuma_mads.pb.go index b1079bf23f7..d2342414531 100644 --- a/discovery/xds/kuma_mads.pb.go +++ b/discovery/xds/kuma_mads.pb.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,13 +23,14 @@ package xds import ( context "context" + reflect "reflect" + sync "sync" + v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" _ "github.com/envoyproxy/protoc-gen-validate/validate" _ "google.golang.org/genproto/googleapis/api/annotations" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" ) const ( diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go index cfb9cbac501..6620f9fac6d 100644 --- a/discovery/xds/kuma_test.go +++ b/discovery/xds/kuma_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,9 +24,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v2" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "gopkg.in/yaml.v2" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -111,7 +111,6 @@ func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.Dis func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) { reg := prometheus.NewRegistry() refreshMetrics := discovery.NewRefreshMetrics(reg) - // TODO(ptodev): Add the ability to unregister refresh metrics. metrics := c.NewDiscovererMetrics(reg, refreshMetrics) err := metrics.Register() if err != nil { @@ -131,6 +130,8 @@ func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) { } func TestKumaMadsV1ResourceParserInvalidTypeURL(t *testing.T) { + t.Parallel() + resources := make([]*anypb.Any, 0) groups, err := kumaMadsV1ResourceParser(resources, "type.googleapis.com/some.api.v1.Monitoring") require.Nil(t, groups) @@ -138,6 +139,8 @@ func TestKumaMadsV1ResourceParserInvalidTypeURL(t *testing.T) { } func TestKumaMadsV1ResourceParserEmptySlice(t *testing.T) { + t.Parallel() + resources := make([]*anypb.Any, 0) groups, err := kumaMadsV1ResourceParser(resources, KumaMadsV1ResourceTypeURL) require.Empty(t, groups) @@ -145,6 +148,8 @@ func TestKumaMadsV1ResourceParserEmptySlice(t *testing.T) { } func TestKumaMadsV1ResourceParserValidResources(t *testing.T) { + t.Parallel() + res, err := getKumaMadsV1DiscoveryResponse(testKumaMadsV1Resources...) require.NoError(t, err) @@ -192,6 +197,8 @@ func TestKumaMadsV1ResourceParserValidResources(t *testing.T) { } func TestKumaMadsV1ResourceParserInvalidResources(t *testing.T) { + t.Parallel() + data, err := protoJSONMarshalOptions.Marshal(&MonitoringAssignment_Target{}) require.NoError(t, err) @@ -201,12 +208,13 @@ func TestKumaMadsV1ResourceParserInvalidResources(t *testing.T) { }} groups, err := kumaMadsV1ResourceParser(resources, KumaMadsV1ResourceTypeURL) require.Nil(t, groups) - require.Error(t, err) - require.Contains(t, err.Error(), "cannot parse") + require.ErrorContains(t, err, "cannot parse") } func TestNewKumaHTTPDiscovery(t *testing.T) { + t.Parallel() + kd, err := newKumaTestHTTPDiscovery(kumaConf) require.NoError(t, err) require.NotNil(t, kd) @@ -222,6 +230,8 @@ func TestNewKumaHTTPDiscovery(t *testing.T) { } func TestKumaHTTPDiscoveryRefresh(t *testing.T) { + t.Parallel() + s := createTestHTTPServer(t, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { if request.VersionInfo == "1" { return nil, nil diff --git a/discovery/xds/metrics.go b/discovery/xds/metrics.go index 597d516566a..7e5be89bd38 100644 --- a/discovery/xds/metrics.go +++ b/discovery/xds/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -29,7 +29,7 @@ type xdsMetrics struct { metricRegisterer discovery.MetricRegisterer } -func newDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func newDiscovererMetrics(reg prometheus.Registerer, _ discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { m := &xdsMetrics{ fetchFailuresCount: prometheus.NewCounter( prometheus.CounterOpts{ diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go index 8191d6be1ae..29da7b7c89b 100644 --- a/discovery/xds/xds.go +++ b/discovery/xds/xds.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,11 +15,10 @@ package xds import ( "context" + "log/slog" "time" v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/common/config" "github.com/prometheus/common/model" "google.golang.org/protobuf/encoding/protojson" @@ -104,7 +103,7 @@ type fetchDiscovery struct { refreshInterval time.Duration parseResources resourceParser - logger log.Logger + logger *slog.Logger metrics *xdsMetrics } @@ -140,7 +139,7 @@ func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Grou } if err != nil { - level.Error(d.logger).Log("msg", "error parsing resources", "err", err) + d.logger.Error("error parsing resources", "err", err) d.metrics.fetchFailuresCount.Inc() return } @@ -153,12 +152,12 @@ func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Grou parsedTargets, err := d.parseResources(response.Resources, response.TypeUrl) if err != nil { - level.Error(d.logger).Log("msg", "error parsing resources", "err", err) + d.logger.Error("error parsing resources", "err", err) d.metrics.fetchFailuresCount.Inc() return } - level.Debug(d.logger).Log("msg", "Updated to version", "version", response.VersionInfo, "targets", len(parsedTargets)) + d.logger.Debug("Updated to version", "version", response.VersionInfo, "targets", len(parsedTargets)) select { case <-ctx.Done(): diff --git a/discovery/xds/xds_test.go b/discovery/xds/xds_test.go index 7cce021c5f0..c11cdd2c054 100644 --- a/discovery/xds/xds_test.go +++ b/discovery/xds/xds_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -22,9 +22,9 @@ import ( "time" v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" "go.uber.org/goleak" "google.golang.org/protobuf/types/known/anypb" @@ -85,12 +85,12 @@ func createTestHTTPServer(t *testing.T, responder discoveryResponder) *httptest. } func constantResourceParser(targets []model.LabelSet, err error) resourceParser { - return func(resources []*anypb.Any, typeUrl string) ([]model.LabelSet, error) { + return func([]*anypb.Any, string) ([]model.LabelSet, error) { return targets, err } } -var nopLogger = log.NewNopLogger() +var nopLogger = promslog.NewNopLogger() type testResourceClient struct { resourceTypeURL string @@ -111,16 +111,18 @@ func (rc testResourceClient) Fetch(ctx context.Context) (*v3.DiscoveryResponse, return rc.fetch(ctx) } -func (rc testResourceClient) ID() string { +func (testResourceClient) ID() string { return "test-client" } -func (rc testResourceClient) Close() { +func (testResourceClient) Close() { } func TestPollingRefreshSkipUpdate(t *testing.T) { + t.Parallel() + rc := &testResourceClient{ - fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) { + fetch: func(context.Context) (*v3.DiscoveryResponse, error) { return nil, nil }, } @@ -162,12 +164,14 @@ func TestPollingRefreshSkipUpdate(t *testing.T) { } func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { + t.Parallel() + server := "http://198.161.2.0" source := "test" rc := &testResourceClient{ server: server, protocolVersion: ProtocolV3, - fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) { + fetch: func(context.Context) (*v3.DiscoveryResponse, error) { return &v3.DiscoveryResponse{}, nil }, } @@ -218,19 +222,21 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { } func TestPollingDisappearingTargets(t *testing.T) { + t.Parallel() + server := "http://198.161.2.0" source := "test" rc := &testResourceClient{ server: server, protocolVersion: ProtocolV3, - fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) { + fetch: func(context.Context) (*v3.DiscoveryResponse, error) { return &v3.DiscoveryResponse{}, nil }, } // On the first poll, send back two targets. On the next, send just one. counter := 0 - parser := func(resources []*anypb.Any, typeUrl string) ([]model.LabelSet, error) { + parser := func([]*anypb.Any, string) ([]model.LabelSet, error) { counter++ if counter == 1 { return []model.LabelSet{ diff --git a/discovery/zookeeper/zookeeper.go b/discovery/zookeeper/zookeeper.go index 92904dd71c8..6ac9b25cd6f 100644 --- a/discovery/zookeeper/zookeeper.go +++ b/discovery/zookeeper/zookeeper.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,15 +18,16 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "net" "strconv" "strings" "time" - "github.com/go-kit/log" "github.com/go-zookeeper/zk" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -58,7 +59,7 @@ type ServersetSDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*ServersetSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*ServersetSDConfig) NewDiscovererMetrics(prometheus.Registerer, discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &discovery.NoopDiscovererMetrics{} } @@ -71,7 +72,7 @@ func (c *ServersetSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (dis } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *ServersetSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *ServersetSDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultServersetSDConfig type plain ServersetSDConfig err := unmarshal((*plain)(c)) @@ -100,7 +101,7 @@ type NerveSDConfig struct { } // NewDiscovererMetrics implements discovery.Config. -func (*NerveSDConfig) NewDiscovererMetrics(reg prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { +func (*NerveSDConfig) NewDiscovererMetrics(prometheus.Registerer, discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { return &discovery.NoopDiscovererMetrics{} } @@ -113,7 +114,7 @@ func (c *NerveSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discove } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *NerveSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *NerveSDConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = DefaultNerveSDConfig type plain NerveSDConfig err := unmarshal((*plain)(c)) @@ -146,16 +147,16 @@ type Discovery struct { treeCaches []*treecache.ZookeeperTreeCache parse func(data []byte, path string) (model.LabelSet, error) - logger log.Logger + logger *slog.Logger } // NewNerveDiscovery returns a new Discovery for the given Nerve config. -func NewNerveDiscovery(conf *NerveSDConfig, logger log.Logger) (*Discovery, error) { +func NewNerveDiscovery(conf *NerveSDConfig, logger *slog.Logger) (*Discovery, error) { return NewDiscovery(conf.Servers, time.Duration(conf.Timeout), conf.Paths, logger, parseNerveMember) } // NewServersetDiscovery returns a new Discovery for the given serverset config. -func NewServersetDiscovery(conf *ServersetSDConfig, logger log.Logger) (*Discovery, error) { +func NewServersetDiscovery(conf *ServersetSDConfig, logger *slog.Logger) (*Discovery, error) { return NewDiscovery(conf.Servers, time.Duration(conf.Timeout), conf.Paths, logger, parseServersetMember) } @@ -165,11 +166,11 @@ func NewDiscovery( srvs []string, timeout time.Duration, paths []string, - logger log.Logger, + logger *slog.Logger, pf func(data []byte, path string) (model.LabelSet, error), ) (*Discovery, error) { if logger == nil { - logger = log.NewNopLogger() + logger = promslog.NewNopLogger() } conn, _, err := zk.Connect( diff --git a/discovery/zookeeper/zookeeper_test.go b/discovery/zookeeper/zookeeper_test.go index c2b41ce7a3c..ae2d23e6075 100644 --- a/discovery/zookeeper/zookeeper_test.go +++ b/discovery/zookeeper/zookeeper_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -26,11 +26,13 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m) } +// TestNewDiscoveryError can fail if the DNS resolver mistakenly resolves the domain below. +// See https://github.com/prometheus/prometheus/issues/16191 for a precedent. func TestNewDiscoveryError(t *testing.T) { _, err := NewDiscovery( - []string{"unreachable.test"}, + []string{"unreachable.invalid"}, time.Second, []string{"/"}, nil, - func(data []byte, path string) (model.LabelSet, error) { return nil, nil }) + func([]byte, string) (model.LabelSet, error) { return nil, nil }) require.Error(t, err) } diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md index 7d9e5a3c809..5ec738e768c 100644 --- a/docs/command-line/prometheus.md +++ b/docs/command-line/prometheus.md @@ -2,12 +2,9 @@ title: prometheus --- -# prometheus - The Prometheus monitoring server - ## Flags | Flag | Description | Default | @@ -15,11 +12,16 @@ The Prometheus monitoring server | -h, --help | Show context-sensitive help (also try --help-long and --help-man). | | | --version | Show application version. | | | --config.file | Prometheus configuration file path. | `prometheus.yml` | -| --web.listen-address ... | Address to listen on for UI, API, and telemetry. Can be repeated. | `0.0.0.0:9090` | +| --config.auto-reload | Enable automatic configuration file reloading. See also --config.auto-reload-interval. | `false` | +| --config.auto-reload-interval | Specifies the interval for checking and automatically reloading the Prometheus configuration file upon detecting changes. Only used when --config.auto-reload is set. | `30s` | +| --web.listen-address ... | Address to listen on for UI, API, and telemetry. Can be repeated. | `0.0.0.0:9090` | +| --auto-gomaxprocs | Automatically set GOMAXPROCS to match Linux container CPU quota | `true` | +| --auto-gomemlimit | Automatically set GOMEMLIMIT to match Linux container or system memory limit | `true` | | --auto-gomemlimit.ratio | The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory | `0.9` | | --web.config.file | [EXPERIMENTAL] Path to configuration file that can enable TLS or authentication. | | | --web.read-timeout | Maximum duration before timing out read of the request, and closing idle connections. | `5m` | | --web.max-connections | Maximum number of simultaneous connections across all listeners. | `512` | +| --web.max-notifications-subscribers | Limits the maximum number of subscribers that can concurrently receive live notifications. If the limit is reached, new subscription requests will be denied until existing connections close. | `16` | | --web.external-url | The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for generating relative and absolute links back to Prometheus itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived automatically. | | | --web.route-prefix | Prefix for the internal routes of web endpoints. Defaults to path of --web.external-url. | | | --web.user-assets | Path to static asset directory, available at /user. | | @@ -27,37 +29,42 @@ The Prometheus monitoring server | --web.enable-admin-api | Enable API endpoints for admin control actions. | `false` | | --web.enable-remote-write-receiver | Enable API endpoint accepting remote write requests. | `false` | | --web.remote-write-receiver.accepted-protobuf-messages | List of the remote write protobuf messages to accept when receiving the remote writes. Supported values: prometheus.WriteRequest, io.prometheus.write.v2.Request | `prometheus.WriteRequest` | +| --web.enable-otlp-receiver | Enable API endpoint accepting OTLP write requests. | `false` | | --web.console.templates | Path to the console template directory, available at /consoles. | `consoles` | | --web.console.libraries | Path to the console library directory. | `console_libraries` | | --web.page-title | Document title of Prometheus instance. | `Prometheus Time Series Collection and Processing Server` | | --web.cors.origin | Regex for CORS origin. It is fully anchored. Example: 'https?://(domain1\|domain2)\.com' | `.*` | | --storage.tsdb.path | Base path for metrics storage. Use with server mode only. | `data/` | -| --storage.tsdb.retention | [DEPRECATED] How long to retain samples in storage. This flag has been deprecated, use "storage.tsdb.retention.time" instead. Use with server mode only. | | -| --storage.tsdb.retention.time | How long to retain samples in storage. When this flag is set it overrides "storage.tsdb.retention". If neither this flag nor "storage.tsdb.retention" nor "storage.tsdb.retention.size" is set, the retention time defaults to 15d. Units Supported: y, w, d, h, m, s, ms. Use with server mode only. | | -| --storage.tsdb.retention.size | Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. Use with server mode only. | | +| --storage.tsdb.retention.time | [DEPRECATED] How long to retain samples in storage. If neither this flag nor "storage.tsdb.retention.size" is set, the retention time defaults to 15d. Units Supported: y, w, d, h, m, s, ms. This flag has been deprecated, use the storage.tsdb.retention.time field in the config file instead. Use with server mode only. | | +| --storage.tsdb.retention.size | [DEPRECATED] Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. This flag has been deprecated, use the storage.tsdb.retention.size field in the config file instead. Use with server mode only. | | | --storage.tsdb.no-lockfile | Do not create lockfile in data directory. Use with server mode only. | `false` | | --storage.tsdb.head-chunks-write-queue-size | Size of the queue through which head chunks are written to the disk to be m-mapped, 0 disables the queue completely. Experimental. Use with server mode only. | `0` | +| --storage.tsdb.delay-compact-file.path | Path to a JSON file with uploaded TSDB blocks e.g. Thanos shipper meta file. If set TSDB will only compact 1 level blocks that are marked as uploaded in that file, improving external storage integrations e.g. with Thanos sidecar. 1+ level compactions won't be delayed. Use with server mode only. | | | --storage.agent.path | Base path for metrics storage. Use with agent mode only. | `data-agent/` | -| --storage.agent.wal-compression | Compress the agent WAL. Use with agent mode only. | `true` | -| --storage.agent.retention.min-time | Minimum age samples may be before being considered for deletion when the WAL is truncated Use with agent mode only. | | -| --storage.agent.retention.max-time | Maximum age samples may be before being forcibly deleted when the WAL is truncated Use with agent mode only. | | +| --storage.agent.wal-compression | Compress the agent WAL. If false, the --storage.agent.wal-compression-type flag is ignored. Use with agent mode only. | `true` | +| --storage.agent.retention.min-time | Minimum age samples may be before being considered for deletion when the WAL is truncated Use with agent mode only. | `5m` | +| --storage.agent.retention.max-time | Maximum age samples may be before being forcibly deleted when the WAL is truncated Use with agent mode only. | `4h` | +| --storage.agent.checkpoint-from-in-memory-series | Use only in-memory series data when building a checkpoint. Use with agent mode only. | `false` | +| --storage.agent.checkpoint-batch-size | Size of a single WAL log entry chunk to be flushed. Has no effect without --storage.agent.checkpoint-from-in-memory-series flag. Use with agent mode only. | `1000` | | --storage.agent.no-lockfile | Do not create lockfile in data directory. Use with agent mode only. | `false` | | --storage.remote.flush-deadline | How long to wait flushing sample on shutdown or config reload. | `1m` | | --storage.remote.read-sample-limit | Maximum overall number of samples to return via the remote read interface, in a single query. 0 means no limit. This limit is ignored for streamed response types. Use with server mode only. | `5e7` | | --storage.remote.read-concurrent-limit | Maximum number of concurrent remote read calls. 0 means no limit. Use with server mode only. | `10` | | --storage.remote.read-max-bytes-in-frame | Maximum number of bytes in a single frame for streaming remote read response types before marshalling. Note that client might have limit on frame size as well. 1MB as recommended by protobuf by default. Use with server mode only. | `1048576` | +| --web.search.max-limit | Hard upper bound on the "limit" query parameter accepted by the experimental search API (--enable-feature=search-api). Requests with a higher limit are rejected with HTTP 400. 0 disables the cap. Use with server mode only. | `10000` | | --rules.alert.for-outage-tolerance | Max time to tolerate prometheus outage for restoring "for" state of alert. Use with server mode only. | `1h` | | --rules.alert.for-grace-period | Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. Use with server mode only. | `10m` | | --rules.alert.resend-delay | Minimum amount of time to wait before resending an alert to Alertmanager. Use with server mode only. | `1m` | | --rules.max-concurrent-evals | Global concurrency limit for independent rules that can run concurrently. When set, "query.max-concurrency" may need to be adjusted accordingly. Use with server mode only. | `4` | | --alertmanager.notification-queue-capacity | The capacity of the queue for pending Alertmanager notifications. Use with server mode only. | `10000` | +| --alertmanager.notification-batch-size | The maximum number of notifications per batch to send to the Alertmanager. Use with server mode only. | `256` | | --alertmanager.drain-notification-queue-on-shutdown | Send any outstanding Alertmanager notifications when shutting down. If false, any outstanding Alertmanager notifications will be dropped when shutting down. Use with server mode only. | `true` | | --query.lookback-delta | The maximum lookback duration for retrieving metrics during expression evaluations and federation. Use with server mode only. | `5m` | | --query.timeout | Maximum time a query may take before being aborted. Use with server mode only. | `2m` | | --query.max-concurrency | Maximum number of queries executed concurrently. Use with server mode only. | `20` | | --query.max-samples | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` | -| --scrape.name-escaping-scheme | Method for escaping legacy invalid names when sending to Prometheus that does not support UTF-8. Can be one of "values", "underscores", or "dots". | `values` | -| --enable-feature ... | Comma separated feature names to enable. Valid options: agent, auto-gomaxprocs, auto-gomemlimit, concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, expand-external-labels, extra-scrape-metrics, memory-snapshot-on-shutdown, native-histograms, new-service-discovery-manager, no-default-scrape-port, otlp-write-receiver, promql-experimental-functions, promql-delayed-name-removal, promql-per-step-stats, remote-write-receiver (DEPRECATED), utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | +| --enable-feature ... | Comma separated feature names to enable. Valid options: concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, extra-scrape-metrics, memory-snapshot-on-shutdown, metadata-wal-records, old-ui, otlp-deltatocumulative, otlp-native-delta-ingestion, promql-binop-fill-modifiers, promql-delayed-name-removal, promql-duration-expr, promql-experimental-functions, promql-extended-range-selectors, promql-per-step-stats, search-api, st-storage, st-synthesis, type-and-unit-labels, use-start-timestamps, use-uncached-io, xor2-encoding. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | +| --agent | Run Prometheus in 'Agent mode'. | | | --log.level | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` | | --log.format | Output format of log messages. One of: [logfmt, json] | `logfmt` | diff --git a/docs/command-line/promtool.md b/docs/command-line/promtool.md index e48cede79c1..06c6f878749 100644 --- a/docs/command-line/promtool.md +++ b/docs/command-line/promtool.md @@ -2,12 +2,9 @@ title: promtool --- -# promtool - Tooling for the Prometheus monitoring system. - ## Flags | Flag | Description | @@ -15,7 +12,7 @@ Tooling for the Prometheus monitoring system. | -h, --help | Show context-sensitive help (also try --help-long and --help-man). | | --version | Show application version. | | --experimental | Enable experimental commands. | -| --enable-feature ... | Comma separated feature names to enable (only PromQL related and no-default-scrape-port). See https://prometheus.io/docs/prometheus/latest/feature_flags/ for the options and more details. | +| --enable-feature ... | Comma separated feature names to enable. Valid options: promql-experimental-functions, promql-delayed-name-removal, promql-duration-expr, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details | @@ -59,9 +56,9 @@ Check the resources for validity. #### Flags -| Flag | Description | -| --- | --- | -| --extended | Print extended information related to the cardinality of the metrics. | +| Flag | Description | Default | +| --- | --- | --- | +| --query.lookback-delta | The server's maximum query lookback duration. | `5m` | @@ -102,8 +99,9 @@ Check if the config files are valid or not. | Flag | Description | Default | | --- | --- | --- | | --syntax-only | Only check the config file syntax, ignoring file and content validation referenced in the config | | -| --lint | Linting checks to apply to the rules specified in the config. Available options are: all, duplicate-rules, none. Use --lint=none to disable linting | `duplicate-rules` | +| --lint | Linting checks to apply to the rules/scrape configs specified in the config. Available options are: all, duplicate-rules, none, too-long-scrape-interval. Use --lint=none to disable linting | `duplicate-rules` | | --lint-fatal | Make lint errors exit with exit code 3. | `false` | +| --ignore-unknown-fields | Ignore unknown fields in the rule groups read by the config files. This is useful when you want to extend rule files with custom metadata. Ensure that those fields are removed before loading them into the Prometheus server as it performs strict checks by default. | `false` | | --agent | Check config file for Prometheus in Agent mode. | | @@ -143,7 +141,7 @@ Check if the Prometheus server is healthy. | Flag | Description | Default | | --- | --- | --- | -| --http.config.file | HTTP client configuration file for promtool to connect to Prometheus. | | +| --http.config.file | HTTP client configuration file, see details at https://prometheus.io/docs/prometheus/latest/configuration/promtool | | | --url | The URL for the Prometheus server. | `http://localhost:9090` | @@ -159,7 +157,7 @@ Check if the Prometheus server is ready. | Flag | Description | Default | | --- | --- | --- | -| --http.config.file | HTTP client configuration file for promtool to connect to Prometheus. | | +| --http.config.file | HTTP client configuration file, see details at https://prometheus.io/docs/prometheus/latest/configuration/promtool | | | --url | The URL for the Prometheus server. | `http://localhost:9090` | @@ -177,6 +175,7 @@ Check if the rule files are valid or not. | --- | --- | --- | | --lint | Linting checks to apply. Available options are: all, duplicate-rules, none. Use --lint=none to disable linting | `duplicate-rules` | | --lint-fatal | Make lint errors exit with exit code 3. | `false` | +| --ignore-unknown-fields | Ignore unknown fields in the rule files. This is useful when you want to extend rule files with custom metadata. Ensure that those fields are removed before loading them into the Prometheus server as it performs strict checks by default. | `false` | @@ -192,13 +191,25 @@ Check if the rule files are valid or not. ##### `promtool check metrics` -Pass Prometheus metrics over stdin to lint them for consistency and correctness. +Pass Prometheus metrics over stdin to lint them for consistency and correctness, and optionally perform cardinality analysis. examples: $ cat metrics.prom | promtool check metrics -$ curl -s http://localhost:9090/metrics | promtool check metrics +$ curl -s http://localhost:9090/metrics | promtool check metrics `--extended` + +$ curl -s http://localhost:9100/metrics | promtool check metrics `--extended` `--lint`=none + + + +###### Flags + +| Flag | Description | Default | +| --- | --- | --- | +| --extended | Print extended information related to the cardinality of the metrics. | | +| --lint | Linting checks to apply for metrics. Available options are: all, none. Use --lint=none to disable metrics linting. | `all` | + @@ -213,7 +224,7 @@ Run query against a Prometheus server. | Flag | Description | Default | | --- | --- | --- | | -o, --format | Output format of the query. | `promql` | -| --http.config.file | HTTP client configuration file for promtool to connect to Prometheus. | | +| --http.config.file | HTTP client configuration file, see details at https://prometheus.io/docs/prometheus/latest/configuration/promtool | | @@ -229,6 +240,7 @@ Run instant query. | Flag | Description | | --- | --- | | --time | Query evaluation time (RFC3339 or Unix timestamp). | +| --header | Extra headers to send to server. | @@ -281,7 +293,7 @@ Run series query. | Flag | Description | | --- | --- | -| --match ... | Series selector. Can be specified multiple times. | +| --match ... | Series selector. Can be specified multiple times. | | --start | Start time (RFC3339 or Unix timestamp). | | --end | End time (RFC3339 or Unix timestamp). | @@ -309,7 +321,7 @@ Run labels query. | --- | --- | | --start | Start time (RFC3339 or Unix timestamp). | | --end | End time (RFC3339 or Unix timestamp). | -| --match ... | Series selector. Can be specified multiple times. | +| --match ... | Series selector. Can be specified multiple times. | @@ -338,7 +350,7 @@ Run queries against your Prometheus to analyze the usage pattern of certain metr | --type | Type of metric: histogram. | | | --duration | Time frame to analyze. | `1h` | | --time | Query time (RFC3339 or Unix timestamp), defaults to now. | | -| --match ... | Series selector. Can be specified multiple times. | | +| --match ... | Series selector. Can be specified multiple times. | | @@ -404,7 +416,7 @@ Push to a Prometheus server. | Flag | Description | | --- | --- | -| --http.config.file | HTTP client configuration file for promtool to connect to Prometheus. | +| --http.config.file | HTTP client configuration file, see details at https://prometheus.io/docs/prometheus/latest/configuration/promtool | @@ -422,6 +434,7 @@ Push metrics to a prometheus remote write (for testing purpose only). | --label | Label to attach to metrics. Can be specified multiple times. | `job=promtool` | | --timeout | The time to wait for pushing metrics. | `30s` | | --header | Prometheus remote write header. | | +| --protobuf_message | Protobuf message to use when writing (prometheus.WriteRequest or io.prometheus.write.v2.Request). | `prometheus.WriteRequest` | @@ -461,8 +474,10 @@ Unit tests for rules. | Flag | Description | Default | | --- | --- | --- | -| --run ... | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. | | +| --run ... | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. | | +| --debug | Enable unit test debugging. | `false` | | --diff | [Experimental] Print colored differential output between expected & received output. | `false` | +| --ignore-unknown-fields | Ignore unknown fields in the test files. This is useful when you want to extend rule files with custom metadata. Ensure that those fields are removed before loading them into the Prometheus server as it performs strict checks by default. | `false` | @@ -567,7 +582,7 @@ List tsdb blocks. ##### `promtool tsdb dump` -Dump samples from a TSDB. +Dump data (series+samples or optionally just series) from a TSDB. @@ -576,9 +591,10 @@ Dump samples from a TSDB. | Flag | Description | Default | | --- | --- | --- | | --sandbox-dir-root | Root directory where a sandbox directory will be created, this sandbox is used in case WAL replay generates chunks (default is the database path). The sandbox is cleaned up at the end. | | -| --min-time | Minimum timestamp to dump. | `-9223372036854775808` | -| --max-time | Maximum timestamp to dump. | `9223372036854775807` | -| --match ... | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` | +| --min-time | Minimum timestamp to dump, in milliseconds since the Unix epoch. | `-9223372036854775808` | +| --max-time | Maximum timestamp to dump, in milliseconds since the Unix epoch. | `9223372036854775807` | +| --match ... | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` | +| --format | Output format of the dump (prom (default) or seriesjson). | `prom` | @@ -603,9 +619,9 @@ Dump samples from a TSDB. | Flag | Description | Default | | --- | --- | --- | | --sandbox-dir-root | Root directory where a sandbox directory will be created, this sandbox is used in case WAL replay generates chunks (default is the database path). The sandbox is cleaned up at the end. | | -| --min-time | Minimum timestamp to dump. | `-9223372036854775808` | -| --max-time | Maximum timestamp to dump. | `9223372036854775807` | -| --match ... | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` | +| --min-time | Minimum timestamp to dump, in milliseconds since the Unix epoch. | `-9223372036854775808` | +| --max-time | Maximum timestamp to dump, in milliseconds since the Unix epoch. | `9223372036854775807` | +| --match ... | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` | @@ -670,7 +686,7 @@ Create blocks of data for new recording rules. | Flag | Description | Default | | --- | --- | --- | -| --http.config.file | HTTP client configuration file for promtool to connect to Prometheus. | | +| --http.config.file | HTTP client configuration file, see details at https://prometheus.io/docs/prometheus/latest/configuration/promtool | | | --url | The URL for the Prometheus API with the data where the rule will be backfilled from. | `http://localhost:9090` | | --start | The time to start backfilling the new rule from. Must be a RFC3339 formatted date or Unix timestamp. Required. | | | --end | If an end time is provided, all recording rules in the rule files provided will be backfilled to the end time. Default will backfill up to 3 hours ago. Must be a RFC3339 formatted date or Unix timestamp. | | diff --git a/docs/configuration/alerting_rules.md b/docs/configuration/alerting_rules.md index 3c1ec84f0f6..faffad56f2f 100644 --- a/docs/configuration/alerting_rules.md +++ b/docs/configuration/alerting_rules.md @@ -3,15 +3,13 @@ title: Alerting rules sort_rank: 3 --- -# Alerting rules - Alerting rules allow you to define alert conditions based on Prometheus expression language expressions and to send notifications about firing alerts to an external service. Whenever the alert expression results in one or more vector elements at a given point in time, the alert counts as active for these elements' label sets. -### Defining alerting rules +## Defining alerting rules Alerting rules are configured in Prometheus in the same way as [recording rules](recording_rules.md). @@ -21,10 +19,13 @@ An example rules file with an alert would be: ```yaml groups: - name: example + labels: + team: myteam rules: - alert: HighRequestLatency expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5 for: 10m + keep_firing_for: 5m labels: severity: page annotations: @@ -38,13 +39,20 @@ the alert continues to be active during each evaluation for 10 minutes before firing the alert. Elements that are active, but not firing yet, are in the pending state. Alerting rules without the `for` clause will become active on the first evaluation. +There is also an optional `keep_firing_for` clause that tells Prometheus to keep +this alert firing for the specified duration after the firing condition was last met. +This can be used to prevent situations such as flapping alerts, false resolutions +due to lack of data loss, etc. Alerting rules without the `keep_firing_for` clause +will deactivate on the first evaluation where the condition is not met (assuming +any optional `for` duration described above has been satisfied). + The `labels` clause allows specifying a set of additional labels to be attached to the alert. Any existing conflicting labels will be overwritten. The label values can be templated. The `annotations` clause specifies a set of informational labels that can be used to store longer additional information such as alert descriptions or runbook links. The annotation values can be templated. -#### Templating +### Templating Label and annotation values can be templated using [console templates](https://prometheus.io/docs/visualization/consoles). The `$labels` @@ -83,7 +91,7 @@ groups: description: "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)" ``` -### Inspecting alerts during runtime +## Inspecting alerts during runtime To manually inspect which alerts are active (pending or firing), navigate to the "Alerts" tab of your Prometheus instance. This will show you the exact @@ -95,7 +103,7 @@ The sample value is set to `1` as long as the alert is in the indicated active (pending or firing) state, and the series is marked stale when this is no longer the case. -### Sending alert notifications +## Sending alert notifications Prometheus's alerting rules are good at figuring what is broken *right now*, but they are not a fully-fledged notification solution. Another layer is needed to @@ -104,6 +112,6 @@ on top of the simple alert definitions. In Prometheus's ecosystem, the [Alertmanager](https://prometheus.io/docs/alerting/alertmanager/) takes on this role. Thus, Prometheus may be configured to periodically send information about alert states to an Alertmanager instance, which then takes care of dispatching -the right notifications. +the right notifications. Prometheus can be [configured](configuration.md) to automatically discover available Alertmanager instances through its service discovery integrations. diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 1f2db6ee82e..19a1bdbca53 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -3,8 +3,6 @@ title: Configuration sort_rank: 1 --- -# Configuration - Prometheus is configured via command-line flags and a configuration file. While the command-line flags configure immutable system parameters (such as storage locations, amount of data to keep on disk and in memory, etc.), the @@ -59,24 +57,37 @@ global: [ scrape_interval: | default = 1m ] # How long until a scrape request times out. + # It cannot be greater than the scrape interval. [ scrape_timeout: | default = 10s ] # The protocols to negotiate during a scrape with the client. # Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, - # OpenMetricsText1.0.0, PrometheusText0.0.4. - # The default value changes to [ PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4 ] - # when native_histogram feature flag is set. - [ scrape_protocols: [, ...] | default = [ OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4 ] ] + # OpenMetricsText1.0.0, PrometheusText0.0.4, PrometheusText1.0.0. + # If left unset both here and in an individual scrape config, the + # negotiation order used in that scrape config depends on the effective + # value of scrape_native_histograms for that scrape config. + # If scrape_native_histograms is false, the order is + # [ OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText1.0.0, PrometheusText0.0.4 ]. + # If scrape_native_histograms is true, the order is + # [ PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText1.0.0, PrometheusText0.0.4 ]. + [ scrape_protocols: [, ...] ] # How frequently to evaluate rules. [ evaluation_interval: | default = 1m ] - # Offset the rule evaluation timestamp of this particular group by the specified duration into the past to ensure the underlying metrics have been received. - # Metric availability delays are more likely to occur when Prometheus is running as a remote write target, but can also occur when there's anomalies with scraping. + # Offset the rule evaluation timestamp of this particular group by the + # specified duration into the past to ensure the underlying metrics have + # been received. Metric availability delays are more likely to occur when + # Prometheus is running as a remote write target, but can also occur when + # there's anomalies with scraping. [ rule_query_offset: | default = 0s ] # The labels to add to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). + # Environment variable references `${var}` or `$var` are replaced according + # to the values of the current environment variables. + # References to undefined variables are replaced by the empty string. + # The `$` character can be escaped by using `$$`. external_labels: [ : ... ] @@ -94,27 +105,29 @@ global: # change or be removed in the future. [ body_size_limit: | default = 0 ] - # Per-scrape limit on number of scraped samples that will be accepted. + # Per-scrape limit on the number of scraped samples that will be accepted. # If more than this number of samples are present after metric relabeling # the entire scrape will be treated as failed. 0 means no limit. [ sample_limit: | default = 0 ] - # Per-scrape limit on number of labels that will be accepted for a sample. If - # more than this number of labels are present post metric-relabeling, the - # entire scrape will be treated as failed. 0 means no limit. + # Limit on the number of labels that will be accepted per sample. If more + # than this number of labels are present on any sample post metric-relabeling, + # the entire scrape will be treated as failed. 0 means no limit. [ label_limit: | default = 0 ] - # Per-scrape limit on length of labels name that will be accepted for a sample. - # If a label name is longer than this number post metric-relabeling, the entire - # scrape will be treated as failed. 0 means no limit. + # Limit on the length (in bytes) of each individual label name. If any label + # name in a scrape is longer than this number post metric-relabeling, the + # entire scrape will be treated as failed. Note that label names are UTF-8 + # encoded, and characters can take up to 4 bytes. 0 means no limit. [ label_name_length_limit: | default = 0 ] - # Per-scrape limit on length of labels value that will be accepted for a sample. - # If a label value is longer than this number post metric-relabeling, the - # entire scrape will be treated as failed. 0 means no limit. + # Limit on the length (in bytes) of each individual label value. If any label + # value in a scrape is longer than this number post metric-relabeling, the + # entire scrape will be treated as failed. Note that label values are UTF-8 + # encoded, and characters can take up to 4 bytes. 0 means no limit. [ label_value_length_limit: | default = 0 ] - # Per-scrape config limit on number of unique targets that will be + # Limit per scrape config on number of unique targets that will be # accepted. If more than this number of targets are present after target # relabeling, Prometheus will mark the targets as failed without scraping them. # 0 means no limit. This is an experimental feature, this behaviour could @@ -126,9 +139,67 @@ global: [ keep_dropped_targets: | default = 0 ] # Specifies the validation scheme for metric and label names. Either blank or - # "legacy" for letters, numbers, colons, and underscores; or "utf8" for full - # UTF-8 support. - [ metric_name_validation_scheme | default "legacy" ] + # "utf8" for full UTF-8 support, or "legacy" for letters, numbers, colons, + # and underscores. + [ metric_name_validation_scheme: | default "utf8" ] + + # If true, native histograms exposed by a target are recognized during + # scraping and ingested as such. If false, any native parts of histograms + # are ignored and only the classic parts are recognized (possibly as + # a classic histogram with only the +Inf buckets if no explicit classic + # buckets are part of the histogram). + [ scrape_native_histograms: | default = false ] + + # Specifies whether to convert scraped classic histograms into native + # histograms with custom buckets. + [ convert_classic_histograms_to_nhcb: | default = false ] + + # Specifies whether to additionally scrape the classic parts of a histogram, + # even if it is also exposed with native parts or it is converted into a + # native histogram with custom buckets. + [ always_scrape_classic_histograms: | default = false ] + + # When enabled, Prometheus stores additional time series for each scrape: + # scrape_timeout_seconds, scrape_sample_limit, and scrape_body_size_bytes. + # These metrics help monitor how close targets are to their configured limits. + # This option can be overridden per scrape config. + [ extra_scrape_metrics: | default = false ] + + # The following explains the various combinations of the last three options + # in various exposition cases. + # + # CASE 1: A histogram is solely exposed as a classic histogram. (Note that + # this also applies if the used scrape protocol (also see the + # scrape_protocols setting) does not support native histograms.) In this + # case, the scrape_native_histograms setting has no effect. If + # convert_classic_histograms_to_nhcb is false, the histogram is ingested as + # a classic histograms. If convert_classic_histograms_to_nhcb is true, the + # histograms is converted to an NHCB. In this case, + # always_scrape_classic_histograms determines whether it is also ingested + # as a classic histograms or not. + # + # CASE 2: A histogram is solely exposed as a native histogram, i.e. it has + # no classic buckets except the optional +Inf bucket but it is marked as a + # native histogram (by some "native parts", at the very least by a no-op + # span). If scrape_native_histograms is false, this case is handled like case + # 1, but the resulting classic histogram or NHCB only has a sole bucket, the + # +Inf bucket. If scrape_native_histograms is true, however, the histogram is + # recognized as a pure native histogram and ingested as such. There will be + # no classic histogram ingested, no matter what + # always_scrape_classic_histograms is set to, and there will be no + # conversion to an NHCB, no matter what convert_classic_histograms_to_nhcb + # is set to. + # + # CASE 3: A histogram is exposed as both a native and a classic histogram, + # i.e. it has "native parts" (at the very least a no-op span) and it has at + # least one classic bucket that is not the +Inf bucket. If + # scrape_native_histograms is false, this case is handled like case 1. The + # native parts are ignored, and there will be either a classic histogram, an + # NHCB, or both. If scrape_native_histograms is true, the histogram is + # ingested as a native histogram. There will be no NHCB, no matter what + # convert_classic_histograms_to_nhcb is set to (it would collide with the + # actual native histogram). However, there will be a classic histogram if (and + # only if) always_scrape_classic_histograms is set to true. runtime: # Configure the Go garbage collector GOGC parameter @@ -162,8 +233,58 @@ remote_write: [ - ... ] # Settings related to the OTLP receiver feature. +# See https://prometheus.io/docs/guides/opentelemetry/ for best practices. otlp: + # Promote specific list of resource attributes to labels. + # It cannot be configured simultaneously with 'promote_all_resource_attributes: true'. [ promote_resource_attributes: [, ...] | default = [ ] ] + # Promoting all resource attributes to labels, except for the ones configured with 'ignore_resource_attributes'. + # Be aware that changes in attributes received by the OTLP endpoint may result in time series churn and lead to high memory usage by the Prometheus server. + # It cannot be set to 'true' simultaneously with 'promote_resource_attributes'. + [ promote_all_resource_attributes: | default = false ] + # Which resource attributes to ignore, can only be set when 'promote_all_resource_attributes' is true. + [ ignore_resource_attributes: [, ...] | default = [] ] + # Configures translation of OTLP metrics when received through the OTLP metrics + # endpoint. Available values: + # - "UnderscoreEscapingWithSuffixes" refers to commonly agreed normalization used + # by OpenTelemetry in https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/prometheus + # - "NoUTF8EscapingWithSuffixes" is a mode that relies on UTF-8 support in Prometheus. + # It preserves all special characters like dots, but still adds required metric name suffixes + # for units and _total, as UnderscoreEscapingWithSuffixes does. + # - "UnderscoreEscapingWithoutSuffixes" translates metric name characters that + # are not alphanumerics/underscores/colons to underscores, and label name + # characters that are not alphanumerics/underscores to underscores, but + # unlike UnderscoreEscapingWithSuffixes it does not append any suffixes to + # the names. + # - (EXPERIMENTAL) "NoTranslation" is a mode that relies on UTF-8 support in Prometheus. + # It preserves all special character like dots and won't append special suffixes for metric + # unit and type. + # + # WARNING: The "NoTranslation" setting has significant known risks and limitations (see https://prometheus.io/docs/practices/naming/ + # for details): + # * Impaired UX when using PromQL in plain YAML (e.g. alerts, rules, dashboard, autoscaling configuration). + # * Series collisions which in the best case may result in OOO errors, in the worst case a silently malformed + # time series. For instance, you may end up in situation of ingesting `foo.bar` series with unit + # `seconds` and a separate series `foo.bar` with unit `milliseconds`. + [ translation_strategy: | default = "UnderscoreEscapingWithSuffixes" ] + # Enables adding "service.name", "service.namespace" and "service.instance.id" + # resource attributes to the "target_info" metric, on top of converting + # them into the "instance" and "job" labels. + [ keep_identifying_resource_attributes: | default = false ] + # Configures optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. + [ convert_histograms_to_nhcb: | default = false ] + # Enables promotion of OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels. + # This is disabled by default for backwards compatibility, but according to OTel spec, scope metadata _should_ be identifying, i.e. translated to metric labels. + [ promote_scope_metadata: | default = false ] + # Controls whether to enable prepending of 'key_' to labels starting with '_'. + # Reserved labels starting with '__' are not modified. + # This is only relevant when translation_strategy uses underscore escaping + # (e.g., "UnderscoreEscapingWithSuffixes" or "UnderscoreEscapingWithoutSuffixes"). + [ label_name_underscore_sanitization: | default = true ] + # Enables preserving of multiple consecutive underscores in label names when + # translation_strategy uses underscore escaping. When true (default), multiple + # consecutive underscores are preserved during label name sanitization. + [ label_name_preserve_multiple_underscores: | default = true ] # Settings related to the remote read feature. remote_read: @@ -199,16 +320,24 @@ job_name: [ scrape_interval: | default = ] # Per-scrape timeout when scraping this job. +# It cannot be greater than the scrape interval. [ scrape_timeout: | default = ] # The protocols to negotiate during a scrape with the client. # Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, -# OpenMetricsText1.0.0, PrometheusText0.0.4. -[ scrape_protocols: [, ...] | default = ] - -# Whether to scrape a classic histogram that is also exposed as a native -# histogram (has no effect without --enable-feature=native-histograms). -[ scrape_classic_histograms: | default = false ] +# OpenMetricsText1.0.0, PrometheusText0.0.4, PrometheusText1.0.0. +# If not set in the global config, the default value depends on the +# setting of scrape_native_histograms. If false, it is +# [ OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText1.0.0, PrometheusText0.0.4 ]. +# If true, it is +# [ PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText1.0.0, PrometheusText0.0.4 ]. +[ scrape_protocols: [, ...] | default = ] + +# Fallback protocol to use if a scrape returns blank, unparsable, or otherwise +# invalid Content-Type. +# Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, +# OpenMetricsText1.0.0, PrometheusText0.0.4, PrometheusText1.0.0. +[ fallback_scrape_protocol: ] # The HTTP resource path on which to fetch metrics from targets. [ metrics_path: | default = /metrics ] @@ -264,69 +393,18 @@ params: # response from the scraped target. [ enable_compression: | default = true ] -# Sets the `Authorization` header on every scrape request with the -# configured username and password. -# password and password_file are mutually exclusive. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Sets the `Authorization` header on every scrape request with -# the configured credentials. -authorization: - # Sets the authentication type of the request. - [ type: | default: Bearer ] - # Sets the credentials of the request. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials of the request with the credentials read from the - # configured file. It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configure whether scrape requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# Configures the scrape request's TLS settings. -tls_config: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - # File to which scrape failures are logged. # Reloading the configuration will reopen the file. [ scrape_failure_log_file: ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] + +# List of AWS service discovery configurations. +aws_sd_configs: + [ - ... ] + # List of Azure service discovery configurations. azure_sd_configs: [ - ... ] @@ -412,6 +490,10 @@ nomad_sd_configs: openstack_sd_configs: [ - ... ] +# List of Outscale service discovery configurations. +outscale_sd_configs: + [ - ... ] + # List of OVHcloud service discovery configurations. ovhcloud_sd_configs: [ - ... ] @@ -428,6 +510,10 @@ scaleway_sd_configs: serverset_sd_configs: [ - ... ] +# List of STACKIT service discovery configurations. +stackit_sd_configs: + [ - ... ] + # List of Triton service discovery configurations. triton_sd_configs: [ - ... ] @@ -454,41 +540,59 @@ metric_relabel_configs: # change or be removed in the future. [ body_size_limit: | default = 0 ] -# Per-scrape limit on number of scraped samples that will be accepted. +# Per-scrape limit on the number of scraped samples that will be accepted. # If more than this number of samples are present after metric relabeling # the entire scrape will be treated as failed. 0 means no limit. [ sample_limit: | default = 0 ] -# Per-scrape limit on number of labels that will be accepted for a sample. If -# more than this number of labels are present post metric-relabeling, the -# entire scrape will be treated as failed. 0 means no limit. +# Limit on the number of labels that will be accepted per sample. If more +# than this number of labels are present on any sample post metric-relabeling, +# the entire scrape will be treated as failed. 0 means no limit. [ label_limit: | default = 0 ] -# Per-scrape limit on length of labels name that will be accepted for a sample. -# If a label name is longer than this number post metric-relabeling, the entire -# scrape will be treated as failed. 0 means no limit. +# Limit on the length (in bytes) of each individual label name. If any label +# name in a scrape is longer than this number post metric-relabeling, the +# entire scrape will be treated as failed. Note that label names are UTF-8 +# encoded, and characters can take up to 4 bytes. 0 means no limit. [ label_name_length_limit: | default = 0 ] -# Per-scrape limit on length of labels value that will be accepted for a sample. -# If a label value is longer than this number post metric-relabeling, the -# entire scrape will be treated as failed. 0 means no limit. +# Limit on the length (in bytes) of each individual label value. If any label +# value in a scrape is longer than this number post metric-relabeling, the +# entire scrape will be treated as failed. Note that label values are UTF-8 +# encoded, and characters can take up to 4 bytes. 0 means no limit. [ label_value_length_limit: | default = 0 ] -# Per-scrape config limit on number of unique targets that will be +# Limit per scrape config on number of unique targets that will be # accepted. If more than this number of targets are present after target # relabeling, Prometheus will mark the targets as failed without scraping them. # 0 means no limit. This is an experimental feature, this behaviour could # change in the future. [ target_limit: | default = 0 ] -# Per-job limit on the number of targets dropped by relabeling +# Limit per scrape config on the number of targets dropped by relabeling # that will be kept in memory. 0 means no limit. [ keep_dropped_targets: | default = 0 ] # Specifies the validation scheme for metric and label names. Either blank or -# "legacy" for letters, numbers, colons, and underscores; or "utf8" for full -# UTF-8 support. -[ metric_name_validation_scheme | default "legacy" ] +# "utf8" for full UTF-8 support, or "legacy" for letters, numbers, colons, and +# underscores. +[ metric_name_validation_scheme: | default "utf8" ] + +# Specifies the character escaping scheme that will be requested when scraping +# for metric and label names that do not conform to the legacy Prometheus +# character set. Available options are: +# * `allow-utf-8`: Full UTF-8 support, no escaping needed. +# * `underscores`: Escape all legacy-invalid characters to underscores. +# * `dots`: Escapes dots to `_dot_`, underscores to `__`, and all other +# legacy-invalid characters to underscores. +# * `values`: Prepend the name with `U__` and replace all invalid +# characters with their unicode value, surrounded by underscores. Single +# underscores are replaced with double underscores. +# e.g. "U__my_2e_dotted_2e_name". +# If this value is left blank, Prometheus will default to `allow-utf-8` if the +# validation scheme for the current scrape config is set to utf8, or +# `underscores` if the validation scheme is set to `legacy`. +[ metric_name_escaping_scheme: | default "allow-utf-8" ] # Limit on total number of positive and negative buckets allowed in a single # native histogram. The resolution of a histogram with more buckets will be @@ -502,7 +606,7 @@ metric_relabel_configs: # reduced as much as possible until it is within the limit. # To set an upper limit for the schema (equivalent to "scale" in OTel's # exponential histograms), use the following factor limits: -# +# # +----------------------------+----------------------------+ # | growth factor | resulting schema AKA scale | # +----------------------------+----------------------------+ @@ -532,14 +636,107 @@ metric_relabel_configs: # +----------------------------+----------------------------+ # | 1.002 | 8 | # +----------------------------+----------------------------+ -# +# # 0 results in the smallest supported factor (which is currently ~1.0027 or # schema 8, but might change in the future). [ native_histogram_min_bucket_factor: | default = 0 ] + +# If true, native histograms exposed by a target are recognized during +# scraping and ingested as such. If false, any native parts of histograms +# are ignored and only the classic parts are recognized (possibly as +# a classic histogram with only the +Inf buckets if no explicit classic +# buckets are part of the histogram). +[ scrape_native_histograms: | default = ] + +# Specifies whether to convert classic histograms into native histograms with +# custom buckets. +[ convert_classic_histograms_to_nhcb: | default = ] + +# Specifies whether to additionally scrape the classic parts of a histogram, +# even if it is also exposed with native parts or it is converted into a +# native histogram with custom buckets. +[ always_scrape_classic_histograms: | default = ] + +# When enabled, Prometheus stores additional time series for this scrape job: +# scrape_timeout_seconds, scrape_sample_limit, and scrape_body_size_bytes. +# These metrics help monitor how close targets are to their configured limits. +# If not set, inherits the value from the global configuration. +[ extra_scrape_metrics: | default = ] + +# See global configuration above for further explanations of how the last three +# options combine their effects. + ``` Where `` must be unique across all scrape configurations. +### `` + +A `http_config` allows configuring HTTP requests. + +```yaml +# Sets the `Authorization` header on every request with the +# configured username and password. +# username and username_file are mutually exclusive. +# password and password_file are mutually exclusive. +basic_auth: + [ username: ] + [ username_file: ] + [ password: ] + [ password_file: ] + +# Sets the `Authorization` header on every request with +# the configured credentials. +authorization: + # Sets the authentication type of the request. + [ type: | default: Bearer ] + # Sets the credentials of the request. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials of the request with the credentials read from the + # configured file. It is mutually exclusive with `credentials`. + [ credentials_file: ] + +# Optional OAuth 2.0 configuration. +# Cannot be used at the same time as basic_auth or authorization. +oauth2: + [ ] + +# Configure whether requests follow HTTP 3xx redirects. +[ follow_redirects: | default = true ] + +# Whether to enable HTTP2. +[ enable_http2: | default: true ] + +# Configures the request's TLS settings. +tls_config: + [ ] + +# Optional proxy URL. +[ proxy_url: ] +# Comma-separated string that can contain IPs, CIDR notation, domain names +# that should be excluded from proxying. IP and domain names can +# contain port numbers. +[ no_proxy: ] +# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) +[ proxy_from_environment: | default: false ] +# Specifies headers to send to proxies during CONNECT requests. +[ proxy_connect_header: + [ : [, ...] ] ] + +# Custom HTTP headers to be sent along with each request. +# Headers that are set by Prometheus itself can't be overwritten. +http_headers: + # Header name. + [ : + # Header values. + [ values: [, ...] ] + # Headers values. Hidden in configuration page. + [ secrets: [, ...] ] + # Files to read header values from. + [ files: [, ...] ] ] +``` + ### `` A `tls_config` allows configuring TLS connections. @@ -578,18 +775,58 @@ A `tls_config` allows configuring TLS connections. ### `` -OAuth 2.0 authentication using the client credentials grant type. +OAuth 2.0 authentication using the client credentials or password grant type. Prometheus fetches an access token from the specified endpoint with -the given client access and secret keys. +the given client access and credentials. ```yaml client_id: + +# OAuth2 grant type to use. It can be one of +# "client_credentials" or "urn:ietf:params:oauth:grant-type:jwt-bearer" (RFC 7523). +# Default value is "client_credentials" +[ grant_type: ] + +# Client secret to provide to authorization server. Only used if +# GrantType is set empty or set to "client_credentials". [ client_secret: ] # Read the client secret from a file. # It is mutually exclusive with `client_secret`. [ client_secret_file: ] +# Secret key to sign JWT with. Only used if +# GrantType is set to "urn:ietf:params:oauth:grant-type:jwt-bearer". +[ client_certificate_key: ] + +# Read the secret key from a file. +# It is mutually exclusive with `client_certificate_key`. +[ client_certificate_key_file: ] + +# JWT kid value to include in the JWT header. Only used if +# GrantType is set to "urn:ietf:params:oauth:grant-type:jwt-bearer". +[ client_certificate_key_id: ] + +# Signature algorithm used to sign JWT token. Only used if +# GrantType is set to "urn:ietf:params:oauth:grant-type:jwt-bearer". +# Default value is RS256 and valid values RS256, RS384, RS512 +[ signature_algorithm: ] + +# OAuth client identifier used when communicating with +# the configured OAuth provider. Default value is client_id. Only used if +# GrantType is set to "urn:ietf:params:oauth:grant-type:jwt-bearer". +[ iss: ] + +# Intended audience of the request. If empty, the value +# of TokenURL is used as the intended audience. Only used if +# GrantType is set to "urn:ietf:params:oauth:grant-type:jwt-bearer". +[ audience: ] + +# Map of claims to be added to the JWT token. Only used if +# GrantType is set to "urn:ietf:params:oauth:grant-type:jwt-bearer". +claims: + [ : ... ] + # Scopes for the token request. scopes: [ - ... ] @@ -598,6 +835,11 @@ scopes: token_url: # Optional parameters to append to the token URL. +# To set 'password' grant type, add it to params: +# endpoint_params: +# grant_type: 'password' +# username: 'username@example.com' +# password: 'strongpassword' endpoint_params: [ : ... ] @@ -630,10 +872,508 @@ http_headers: [ files: [, ...] ] ] ``` +### `` + +AWS SD configurations allow retrieving scrape targets from AWS services. +This is a unified service discovery that supports multiple AWS service types through the `role` parameter. + +One of the following `role` types can be configured to discover targets: + +#### `ec2` + +The `ec2` role discovers targets from AWS EC2 instances. The private IP address is used by default, but may be changed to +the public IP address with relabeling. + +The IAM credentials used must have the `ec2:DescribeInstances` permission to +discover scrape targets, and may optionally have the +`ec2:DescribeAvailabilityZones` permission if you want the availability zone ID +available as a label (see below). + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_ec2_ami`: the EC2 Amazon Machine Image +* `__meta_ec2_architecture`: the architecture of the instance +* `__meta_ec2_availability_zone`: the availability zone in which the instance is running +* `__meta_ec2_availability_zone_id`: the [availability zone ID](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) in which the instance is running (requires `ec2:DescribeAvailabilityZones`) +* `__meta_ec2_instance_id`: the EC2 instance ID +* `__meta_ec2_instance_lifecycle`: the lifecycle of the EC2 instance, set only for 'spot' or 'scheduled' instances, absent otherwise +* `__meta_ec2_instance_state`: the state of the EC2 instance +* `__meta_ec2_instance_type`: the type of the EC2 instance +* `__meta_ec2_ipv6_addresses`: comma separated list of IPv6 addresses assigned to the instance's network interfaces, if present +* `__meta_ec2_owner_id`: the ID of the AWS account that owns the EC2 instance +* `__meta_ec2_platform`: the Operating System platform, set to 'windows' on Windows servers, absent otherwise +* `__meta_ec2_default_ipv6_address`: the first primary IPv6 address found if present, otherwise first non-primary IPv6 address, if present +* `__meta_ec2_primary_ipv6_addresses`: comma separated list of the Primary IPv6 addresses of the instance, if present. The list is ordered based on the position of each corresponding network interface in the attachment order. +* `__meta_ec2_primary_subnet_id`: the subnet ID of the primary network interface, if available +* `__meta_ec2_private_dns_name`: the private DNS name of the instance, if available +* `__meta_ec2_private_ip`: the private IP address of the instance, if present +* `__meta_ec2_public_dns_name`: the public DNS name of the instance, if available +* `__meta_ec2_public_ip`: the public IP address of the instance, if available +* `__meta_ec2_region`: the region of the instance +* `__meta_ec2_subnet_id`: comma separated list of subnets IDs in which the instance is running, if available +* `__meta_ec2_tag_`: each tag value of the instance +* `__meta_ec2_vpc_id`: the ID of the VPC in which the instance is running, if available + +#### `lightsail` + +The `lightsail` role discovers targets from [AWS Lightsail](https://aws.amazon.com/lightsail/) +instances. The private IP address is used by default, but may be changed to +the public IP address with relabeling. + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_lightsail_availability_zone`: the availability zone in which the instance is running +* `__meta_lightsail_blueprint_id`: the Lightsail blueprint ID +* `__meta_lightsail_bundle_id`: the Lightsail bundle ID +* `__meta_lightsail_instance_name`: the name of the Lightsail instance +* `__meta_lightsail_instance_state`: the state of the Lightsail instance +* `__meta_lightsail_instance_support_code`: the support code of the Lightsail instance +* `__meta_lightsail_ipv6_addresses`: comma separated list of IPv6 addresses assigned to the instance's network interfaces, if present +* `__meta_lightsail_private_ip`: the private IP address of the instance +* `__meta_lightsail_public_ip`: the public IP address of the instance, if available +* `__meta_lightsail_region`: the region of the instance +* `__meta_lightsail_tag_`: each tag value of the instance + +#### `ecs` + +The `ecs` role discovers targets from AWS ECS containers. + +ECS service discovery supports all ECS networking modes: +- **awsvpc mode** (Fargate and EC2 with ENI): Uses the task's private IP address from its elastic network interface +- **bridge mode** (EC2): Uses the EC2 host instance's private IP address +- **host mode** (EC2): Uses the EC2 host instance's private IP address + +The private IP address is used by default, but may be changed to the public IP address with relabeling. + +The IAM credentials used must have the following permissions to discover scrape targets: + +- `ecs:ListClusters` +- `ecs:DescribeClusters` +- `ecs:ListServices` +- `ecs:DescribeServices` +- `ecs:ListTasks` +- `ecs:DescribeTasks` +- `ecs:DescribeContainerInstances` (required for EC2 launch type tasks) +- `ec2:DescribeInstances` (required for EC2 launch type tasks) +- `ec2:DescribeNetworkInterfaces` (required to get public IP for awsvpc mode tasks) + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_ecs_cluster`: the name of the ECS cluster +* `__meta_ecs_cluster_arn`: the ARN of the ECS cluster +* `__meta_ecs_service`: the name of the ECS service +* `__meta_ecs_service_arn`: the ARN of the ECS service +* `__meta_ecs_service_status`: the status of the ECS service +* `__meta_ecs_task_group`: the ECS task group (typically service:service-name) +* `__meta_ecs_task_arn`: the ARN of the ECS task +* `__meta_ecs_task_definition`: the ARN of the ECS task definition +* `__meta_ecs_ip_address`: the private IP address of the task +* `__meta_ecs_launch_type`: the launch type of the task (EC2 or Fargate) +* `__meta_ecs_desired_status`: the desired status of the task +* `__meta_ecs_last_status`: the last known status of the task +* `__meta_ecs_health_status`: the health status of the task +* `__meta_ecs_platform_family`: the platform family (e.g., Linux, Windows) +* `__meta_ecs_platform_version`: the platform version +* `__meta_ecs_subnet_id`: the subnet ID where the task is running +* `__meta_ecs_availability_zone`: the availability zone where the task is running +* `__meta_ecs_region`: the AWS region +* `__meta_ecs_public_ip`: the public IP address (from ENI for awsvpc mode, from EC2 instance for bridge/host mode), if available +* `__meta_ecs_network_mode`: the network mode of the task (awsvpc or bridge) +* `__meta_ecs_container_instance_arn`: the ARN of the container instance (EC2 launch type only) +* `__meta_ecs_ec2_instance_id`: the EC2 instance ID (EC2 launch type only) +* `__meta_ecs_ec2_instance_type`: the EC2 instance type (EC2 launch type only) +* `__meta_ecs_ec2_instance_private_ip`: the private IP address of the EC2 instance (EC2 launch type only) +* `__meta_ecs_ec2_instance_public_ip`: the public IP address of the EC2 instance, if available (EC2 launch type only) +* `__meta_ecs_tag_cluster_`: each cluster tag value, keyed by tag name +* `__meta_ecs_tag_service_`: each service tag value, keyed by tag name +* `__meta_ecs_tag_task_`: each task tag value, keyed by tag name +* `__meta_ecs_tag_ec2_`: each EC2 instance tag value, keyed by tag name (EC2 launch type only) + +#### `msk` + +The `msk` role discovers targets from AWS MSK (Managed Streaming for Apache Kafka) provisioned clusters. + +**Important**: This service discovery only works with **provisioned clusters**. Serverless clusters are not supported as they do not expose individual broker nodes. + +Discovery includes: +- **Broker nodes**: Kafka broker instances (supports both ZooKeeper-based and KRaft-based clusters) +- **KRaft Controller nodes**: Controller instances (KRaft-based clusters only) + +Note: ZooKeeper nodes are not discoverable via the MSK API. For monitoring, MSK provides: +- **JMX Exporter**: Available on both broker and KRaft controller nodes (when enabled) +- **Node Exporter**: Available on broker nodes only (when enabled) + +The IAM credentials used must have the following permissions to discover +scrape targets: + +- `kafka:DescribeClusterV2` +- `kafka:ListClustersV2` +- `kafka:ListNodes` + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_msk_cluster_name`: the name of the MSK cluster +* `__meta_msk_cluster_arn`: the ARN of the MSK cluster +* `__meta_msk_cluster_state`: the state of the MSK cluster (e.g., ACTIVE, CREATING, DELETING) +* `__meta_msk_cluster_type`: the type of the MSK cluster (e.g., PROVISIONED, SERVERLESS) +* `__meta_msk_cluster_version`: the current version of the MSK cluster +* `__meta_msk_cluster_kafka_version`: the Kafka version running on the cluster +* `__meta_msk_cluster_jmx_exporter_enabled`: whether JMX exporter is enabled on the cluster +* `__meta_msk_cluster_configuration_arn`: the ARN of the MSK configuration +* `__meta_msk_cluster_configuration_revision`: the revision of the MSK configuration +* `__meta_msk_cluster_tag_`: each cluster tag value, keyed by tag name +* `__meta_msk_node_type`: the type of the node (BROKER or CONTROLLER) +* `__meta_msk_node_arn`: the ARN of the node +* `__meta_msk_node_added_time`: the time the node was added to the cluster +* `__meta_msk_node_instance_type`: the instance type of the node +* `__meta_msk_node_attached_eni`: the ID of the attached ENI +* `__meta_msk_broker_id`: the broker ID (broker nodes only) +* `__meta_msk_broker_endpoint_index`: the index of the broker endpoint (broker nodes only) +* `__meta_msk_broker_client_subnet`: the client subnet of the broker (broker nodes only) +* `__meta_msk_broker_client_vpc_ip`: the VPC IP address of the broker (broker nodes only) +* `__meta_msk_broker_node_exporter_enabled`: whether node exporter is enabled on brokers (broker nodes only) +* `__meta_msk_controller_endpoint_index`: the index of the controller endpoint (controller nodes only) + +#### `elasticache` + +The `elasticache` role discovers targets from AWS ElastiCache for both serverless caches and cache clusters. + +**Important**: For cache clusters, one target is created per cache node. Each target includes the cluster-level labels (ARN, status, tags, etc.) and node-specific labels (node ID, endpoint, availability zone, etc.). The `__address__` label is set to the individual node's endpoint address and port. + +For serverless caches, one target is created per serverless cache, with the `__address__` label set to the serverless cache endpoint. + +The IAM credentials used must have the following permissions to discover scrape targets: + +- `elasticache:DescribeServerlessCaches` +- `elasticache:DescribeCacheClusters` +- `elasticache:ListTagsForResource` + +The following meta labels are available on targets during [relabeling](#relabel_config): + +**Common labels (available on all targets):** + +* `__meta_elasticache_deployment_option`: the deployment option - either `serverless` for serverless caches or `node` for cache cluster nodes + +**Serverless Cache labels:** + +* `__meta_elasticache_serverless_cache_arn`: the ARN of the serverless cache +* `__meta_elasticache_serverless_cache_name`: the name of the serverless cache +* `__meta_elasticache_serverless_cache_status`: the status of the serverless cache +* `__meta_elasticache_serverless_cache_engine`: the cache engine (redis or valkey) +* `__meta_elasticache_serverless_cache_full_engine_version`: the full engine version +* `__meta_elasticache_serverless_cache_major_engine_version`: the major engine version +* `__meta_elasticache_serverless_cache_description`: the description of the serverless cache +* `__meta_elasticache_serverless_cache_create_time`: the creation time in RFC3339 format +* `__meta_elasticache_serverless_cache_snapshot_retention_limit`: the snapshot retention limit in days +* `__meta_elasticache_serverless_cache_daily_snapshot_time`: the daily snapshot time +* `__meta_elasticache_serverless_cache_user_group_id`: the user group ID +* `__meta_elasticache_serverless_cache_kms_key_id`: the KMS key ID for encryption at rest +* `__meta_elasticache_serverless_cache_endpoint_address`: the endpoint address +* `__meta_elasticache_serverless_cache_endpoint_port`: the endpoint port +* `__meta_elasticache_serverless_cache_reader_endpoint_address`: the reader endpoint address +* `__meta_elasticache_serverless_cache_reader_endpoint_port`: the reader endpoint port +* `__meta_elasticache_serverless_cache_security_group_id_`: security group IDs (indexed) +* `__meta_elasticache_serverless_cache_subnet_id_`: subnet IDs (indexed) +* `__meta_elasticache_serverless_cache_cache_usage_limit_data_storage_maximum`: maximum data storage in the specified unit +* `__meta_elasticache_serverless_cache_cache_usage_limit_data_storage_minimum`: minimum data storage in the specified unit +* `__meta_elasticache_serverless_cache_cache_usage_limit_data_storage_unit`: unit for data storage (e.g., GB) +* `__meta_elasticache_serverless_cache_cache_usage_limit_ecpu_per_second_maximum`: maximum ECPU per second +* `__meta_elasticache_serverless_cache_cache_usage_limit_ecpu_per_second_minimum`: minimum ECPU per second +* `__meta_elasticache_serverless_cache_tag_`: each serverless cache tag value, keyed by tag name + +**Cache Cluster labels:** + +* `__meta_elasticache_cache_cluster_arn`: the ARN of the cache cluster +* `__meta_elasticache_cache_cluster_cache_cluster_id`: the cache cluster ID +* `__meta_elasticache_cache_cluster_cache_cluster_status`: the status of the cache cluster +* `__meta_elasticache_cache_cluster_engine`: the cache engine (redis or memcached) +* `__meta_elasticache_cache_cluster_engine_version`: the engine version +* `__meta_elasticache_cache_cluster_cache_node_type`: the cache node type (e.g., cache.t3.micro) +* `__meta_elasticache_cache_cluster_num_cache_nodes`: the number of cache nodes +* `__meta_elasticache_cache_cluster_cache_cluster_create_time`: the creation time in RFC3339 format +* `__meta_elasticache_cache_cluster_at_rest_encryption_enabled`: whether encryption at rest is enabled +* `__meta_elasticache_cache_cluster_transit_encryption_enabled`: whether encryption in transit is enabled +* `__meta_elasticache_cache_cluster_transit_encryption_mode`: the transit encryption mode +* `__meta_elasticache_cache_cluster_auth_token_enabled`: whether auth token is enabled +* `__meta_elasticache_cache_cluster_auth_token_last_modified`: the last modification time of auth token +* `__meta_elasticache_cache_cluster_auto_minor_version_upgrade`: whether auto minor version upgrade is enabled +* `__meta_elasticache_cache_cluster_cache_parameter_group`: the cache parameter group name +* `__meta_elasticache_cache_cluster_cache_subnet_group_name`: the cache subnet group name +* `__meta_elasticache_cache_cluster_client_download_landing_page`: the client download landing page URL +* `__meta_elasticache_cache_cluster_ip_discovery`: the IP discovery mode (ipv4 or ipv6) +* `__meta_elasticache_cache_cluster_network_type`: the network type (ipv4, ipv6, or dual_stack) +* `__meta_elasticache_cache_cluster_preferred_availability_zone`: the preferred availability zone +* `__meta_elasticache_cache_cluster_preferred_maintenance_window`: the preferred maintenance window +* `__meta_elasticache_cache_cluster_preferred_outpost_arn`: the preferred outpost ARN +* `__meta_elasticache_cache_cluster_replication_group_id`: the replication group ID (for Redis clusters that are part of a replication group) +* `__meta_elasticache_cache_cluster_replication_group_log_delivery_enabled`: whether log delivery is enabled for the replication group +* `__meta_elasticache_cache_cluster_snapshot_retention_limit`: the snapshot retention limit in days +* `__meta_elasticache_cache_cluster_snapshot_window`: the daily snapshot window +* `__meta_elasticache_cache_cluster_configuration_endpoint_address`: the configuration endpoint address (cluster mode enabled only) +* `__meta_elasticache_cache_cluster_configuration_endpoint_port`: the configuration endpoint port (cluster mode enabled only) +* `__meta_elasticache_cache_cluster_notification_topic_arn`: the SNS topic ARN for notifications +* `__meta_elasticache_cache_cluster_notification_topic_status`: the SNS topic status +* `__meta_elasticache_cache_cluster_log_delivery_configuration_destination_type_`: log delivery destination type (cloudwatch-logs or kinesis-firehose) +* `__meta_elasticache_cache_cluster_log_delivery_configuration_log_format_`: log format (text or json) +* `__meta_elasticache_cache_cluster_log_delivery_configuration_log_type_`: log type (slow-log or engine-log) +* `__meta_elasticache_cache_cluster_log_delivery_configuration_status_`: log delivery status +* `__meta_elasticache_cache_cluster_log_delivery_configuration_message_`: log delivery message +* `__meta_elasticache_cache_cluster_log_delivery_configuration_log_group_`: CloudWatch log group name (cloudwatch-logs destination only) +* `__meta_elasticache_cache_cluster_log_delivery_configuration_delivery_stream_`: Kinesis Firehose delivery stream name (kinesis-firehose destination only) +* `__meta_elasticache_cache_cluster_pending_modified_values_auth_token_status`: pending auth token status +* `__meta_elasticache_cache_cluster_pending_modified_values_cache_node_type`: pending cache node type change +* `__meta_elasticache_cache_cluster_pending_modified_values_engine_version`: pending engine version upgrade +* `__meta_elasticache_cache_cluster_pending_modified_values_num_cache_nodes`: pending number of cache nodes +* `__meta_elasticache_cache_cluster_pending_modified_values_transit_encryption_enabled`: pending transit encryption status +* `__meta_elasticache_cache_cluster_pending_modified_values_transit_encryption_mode`: pending transit encryption mode +* `__meta_elasticache_cache_cluster_pending_modified_values_cache_node_ids_to_remove`: comma-separated list of cache node IDs to be removed +* `__meta_elasticache_cache_cluster_security_group_membership_id_`: security group ID (indexed) +* `__meta_elasticache_cache_cluster_security_group_membership_status_`: security group status (indexed) +* `__meta_elasticache_cache_cluster_node_id`: cache node ID +* `__meta_elasticache_cache_cluster_node_status`: cache node status +* `__meta_elasticache_cache_cluster_node_create_time`: cache node creation time in RFC3339 format +* `__meta_elasticache_cache_cluster_node_availability_zone`: cache node availability zone +* `__meta_elasticache_cache_cluster_node_customer_outpost_arn`: cache node outpost ARN +* `__meta_elasticache_cache_cluster_node_source_cache_node_id`: source cache node ID for replication +* `__meta_elasticache_cache_cluster_node_parameter_group_status`: parameter group status +* `__meta_elasticache_cache_cluster_node_endpoint_address`: cache node endpoint address +* `__meta_elasticache_cache_cluster_node_endpoint_port`: cache node endpoint port +* `__meta_elasticache_cache_cluster_tag_`: each cache cluster tag value, keyed by tag name + +#### `rds` + +The `rds` role discovers targets from [AWS RDS](https://aws.amazon.com/rds/) +database instances within clusters. One target is created for each DB instance +within the specified clusters. The endpoint address and port of each instance is used by default. + +The IAM credentials used must have the `rds:DescribeDBClusters` and `rds:DescribeDBInstances` +permissions to discover scrape targets. + +The following meta labels are available on targets during [relabeling](#relabel_config): + +**Cluster labels:** + +* `__meta_rds_cluster_activity_stream_kinesis_stream_name`: the name of the Amazon Kinesis data stream used for database activity stream +* `__meta_rds_cluster_activity_stream_kms_key_id`: the AWS KMS key identifier used for encrypting the database activity stream +* `__meta_rds_cluster_activity_stream_mode`: the mode of the database activity stream (sync or async) +* `__meta_rds_cluster_activity_stream_status`: the status of the database activity stream +* `__meta_rds_cluster_allocated_storage`: the allocated storage size in gibibytes (GiB) +* `__meta_rds_cluster_arn`: the Amazon Resource Name (ARN) of the DB cluster +* `__meta_rds_cluster_auto_minor_version_upgrade`: whether automatic minor version upgrades are enabled +* `__meta_rds_cluster_automatic_restart_time`: the time when a stopped cluster will be automatically restarted +* `__meta_rds_cluster_aws_backup_recovery_point_arn`: the ARN of the recovery point in AWS Backup +* `__meta_rds_cluster_backtrack_consumed_change_records`: the number of change records stored for backtrack +* `__meta_rds_cluster_backtrack_window`: the target backtrack window in hours +* `__meta_rds_cluster_backup_retention_period`: the number of days for which automated backups are retained +* `__meta_rds_cluster_capacity`: the current capacity of an Aurora Serverless DB cluster +* `__meta_rds_cluster_character_set_name`: the name of the character set +* `__meta_rds_cluster_clone_group_id`: the ID of the clone group +* `__meta_rds_cluster_cluster_create_time`: the time when the DB cluster was created +* `__meta_rds_cluster_cluster_scalability_type`: the scalability type of the cluster +* `__meta_rds_cluster_copy_tags_to_snapshot`: whether tags are copied from the cluster to snapshots +* `__meta_rds_cluster_cross_account_clone`: whether the DB cluster is a cross-account clone +* `__meta_rds_cluster_database_insights_mode`: the mode of Database Insights +* `__meta_rds_cluster_database_name`: the database name +* `__meta_rds_cluster_db_system_id`: the Oracle system ID (Oracle SID) +* `__meta_rds_cluster_deletion_protection`: whether deletion protection is enabled +* `__meta_rds_cluster_earliest_backtrack_time`: the earliest time to which a database can be restored with backtrack +* `__meta_rds_cluster_earliest_restorable_time`: the earliest time to which a database can be restored +* `__meta_rds_cluster_endpoint`: the endpoint of the DB cluster +* `__meta_rds_cluster_engine_lifecycle_support`: the engine lifecycle support value +* `__meta_rds_cluster_engine_mode`: the engine mode of the cluster (provisioned, serverless, etc.) +* `__meta_rds_cluster_engine_version`: the version of the database engine +* `__meta_rds_cluster_engine`: the database engine of the DB cluster +* `__meta_rds_cluster_global_cluster_identifier`: the identifier of the global cluster +* `__meta_rds_cluster_global_write_forwarding_requested`: whether global write forwarding is requested +* `__meta_rds_cluster_global_write_forwarding_status`: the status of global write forwarding +* `__meta_rds_cluster_hosted_zone_id`: the Route 53 hosted zone ID +* `__meta_rds_cluster_http_endpoint_enabled`: whether the HTTP endpoint is enabled +* `__meta_rds_cluster_iam_database_authentication_enabled`: whether the DB cluster has IAM database authentication enabled +* `__meta_rds_cluster_identifier`: the identifier of the DB cluster +* `__meta_rds_cluster_instance_class`: the compute and memory capacity class of the DB cluster +* `__meta_rds_cluster_io_optimized_next_allowed_modification_time`: the time when the next IO optimization configuration change is allowed +* `__meta_rds_cluster_iops`: the provisioned IOPS (I/O operations per second) value +* `__meta_rds_cluster_kms_key_id`: the AWS KMS key identifier for the encrypted cluster +* `__meta_rds_cluster_latest_restorable_time`: the latest time to which a database can be restored +* `__meta_rds_cluster_local_write_forwarding_status`: the status of local write forwarding +* `__meta_rds_cluster_master_username`: the master username +* `__meta_rds_cluster_monitoring_interval`: the interval in seconds between enhanced monitoring metrics collection +* `__meta_rds_cluster_monitoring_role_arn`: the ARN for the IAM role that permits RDS to send enhanced monitoring metrics to CloudWatch +* `__meta_rds_cluster_multi_az`: whether the DB cluster is multi-AZ +* `__meta_rds_cluster_network_type`: the network type (IPV4 or DUAL) +* `__meta_rds_cluster_parameter_group`: the name of the DB cluster parameter group +* `__meta_rds_cluster_percent_progress`: the progress percentage of the DB cluster operation +* `__meta_rds_cluster_performance_insights_enabled`: whether Performance Insights is enabled +* `__meta_rds_cluster_performance_insights_kms_key_id`: the AWS KMS key identifier for encrypting Performance Insights data +* `__meta_rds_cluster_performance_insights_retention_period`: the retention period for Performance Insights data +* `__meta_rds_cluster_port`: the port the DB cluster is listening on +* `__meta_rds_cluster_preferred_backup_window`: the daily time range during which automated backups are created +* `__meta_rds_cluster_preferred_maintenance_window`: the weekly time range during which system maintenance can occur +* `__meta_rds_cluster_publicly_accessible`: whether the DB cluster is publicly accessible +* `__meta_rds_cluster_reader_endpoint`: the reader endpoint of the DB cluster +* `__meta_rds_cluster_replication_source_identifier`: the identifier of the source DB cluster if this is a read replica +* `__meta_rds_cluster_resource_id`: the AWS Region-unique immutable identifier for the DB cluster +* `__meta_rds_cluster_serverless_v2_platform_version`: the platform version of the Aurora Serverless v2 DB cluster +* `__meta_rds_cluster_status`: the status of the DB cluster +* `__meta_rds_cluster_storage_encrypted`: whether the DB cluster is storage encrypted +* `__meta_rds_cluster_storage_encryption_type`: the storage encryption type +* `__meta_rds_cluster_storage_throughput`: the storage throughput in MiBps +* `__meta_rds_cluster_storage_type`: the storage type +* `__meta_rds_cluster_subnet_group`: the name of the subnet group associated with the DB cluster +* `__meta_rds_cluster_tag_`: each tag value of the DB cluster +* `__meta_rds_cluster_upgrade_rollout_order`: the upgrade rollout order + +**Instance labels:** + +* `__meta_rds_instance_activity_stream_engine_native_audit_fields_included`: whether engine-native audit fields are included in the database activity stream +* `__meta_rds_instance_activity_stream_kinesis_stream_name`: the name of the Amazon Kinesis data stream used for the database activity stream +* `__meta_rds_instance_activity_stream_kms_key_id`: the AWS KMS key identifier used for encrypting the database activity stream +* `__meta_rds_instance_activity_stream_mode`: the mode of the database activity stream (sync or async) +* `__meta_rds_instance_activity_stream_policy_status`: the policy status of the database activity stream +* `__meta_rds_instance_activity_stream_status`: the status of the database activity stream +* `__meta_rds_instance_allocated_storage`: the allocated storage size in gibibytes (GiB) +* `__meta_rds_instance_arn`: the Amazon Resource Name (ARN) of the DB instance +* `__meta_rds_instance_auto_minor_version_upgrade`: whether automatic minor version upgrades are enabled +* `__meta_rds_instance_automatic_restart_time`: the time when a stopped instance will be automatically restarted +* `__meta_rds_instance_automation_mode`: the automation mode of the instance +* `__meta_rds_instance_availability_zone`: the availability zone of the DB instance +* `__meta_rds_instance_aws_backup_recovery_point_arn`: the ARN of the recovery point in AWS Backup +* `__meta_rds_instance_backup_retention_period`: the number of days for which automated backups are retained +* `__meta_rds_instance_backup_target`: the backup target (region or outposts) +* `__meta_rds_instance_ca_certificate_identifier`: the identifier of the CA certificate for the DB instance +* `__meta_rds_instance_character_set_name`: the name of the character set +* `__meta_rds_instance_class`: the compute and memory capacity class of the DB instance +* `__meta_rds_instance_copy_tags_to_snapshot`: whether tags are copied from the instance to snapshots +* `__meta_rds_instance_custom_iam_instance_profile`: the instance profile associated with the underlying Amazon EC2 instance +* `__meta_rds_instance_customer_owned_ip_enabled`: whether a customer-owned IP address (CoIP) is enabled +* `__meta_rds_instance_database_insights_mode`: the mode of Database Insights +* `__meta_rds_instance_db_cluster_identifier`: the identifier of the DB cluster this instance is a member of +* `__meta_rds_instance_db_name`: the database name +* `__meta_rds_instance_db_system_id`: the Oracle system ID (Oracle SID) +* `__meta_rds_instance_dedicated_log_volume`: whether the DB instance has a dedicated log volume +* `__meta_rds_instance_deletion_protection`: whether deletion protection is enabled +* `__meta_rds_instance_endpoint_address`: the DNS address of the DB instance +* `__meta_rds_instance_endpoint_hosted_zone_id`: the Route 53 hosted zone ID of the endpoint +* `__meta_rds_instance_endpoint_port`: the port that the DB instance listens on +* `__meta_rds_instance_engine_lifecycle_support`: the engine lifecycle support value +* `__meta_rds_instance_engine_version`: the version of the database engine +* `__meta_rds_instance_engine`: the database engine that the DB instance uses +* `__meta_rds_instance_enhanced_monitoring_resource_arn`: the ARN of the Amazon CloudWatch Logs log stream for enhanced monitoring +* `__meta_rds_instance_iam_database_authentication_enabled`: whether IAM database authentication is enabled +* `__meta_rds_instance_identifier`: the identifier of the DB instance +* `__meta_rds_instance_instance_create_time`: the time when the DB instance was created +* `__meta_rds_instance_iops`: the provisioned IOPS (I/O operations per second) value +* `__meta_rds_instance_is_cluster_writer`: whether the instance is the cluster writer (true/false) +* `__meta_rds_instance_is_storage_config_upgrade_available`: whether a storage configuration upgrade is available +* `__meta_rds_instance_kms_key_id`: the AWS KMS key identifier for the encrypted instance +* `__meta_rds_instance_latest_restorable_time`: the latest time to which a database can be restored +* `__meta_rds_instance_license_model`: the license model information +* `__meta_rds_instance_listener_endpoint_address`: the DNS address of the listener endpoint +* `__meta_rds_instance_listener_endpoint_hosted_zone_id`: the Route 53 hosted zone ID of the listener endpoint +* `__meta_rds_instance_listener_endpoint_port`: the port that the listener endpoint listens on +* `__meta_rds_instance_master_username`: the master username +* `__meta_rds_instance_max_allocated_storage`: the upper limit in gibibytes to which storage can be scaled automatically +* `__meta_rds_instance_monitoring_interval`: the interval in seconds between enhanced monitoring metrics collection +* `__meta_rds_instance_monitoring_role_arn`: the ARN for the IAM role that permits RDS to send enhanced monitoring metrics to CloudWatch +* `__meta_rds_instance_multi_az`: whether the DB instance is a Multi-AZ deployment +* `__meta_rds_instance_multi_tenant`: whether the instance is in a multi-tenant configuration +* `__meta_rds_instance_nchar_character_set_name`: the national character set name +* `__meta_rds_instance_network_type`: the network type (IPV4 or DUAL) +* `__meta_rds_instance_percent_progress`: the progress percentage of the DB instance operation +* `__meta_rds_instance_performance_insights_enabled`: whether Performance Insights is enabled +* `__meta_rds_instance_performance_insights_kms_key_id`: the AWS KMS key identifier for encrypting Performance Insights data +* `__meta_rds_instance_performance_insights_retention_period`: the retention period for Performance Insights data +* `__meta_rds_instance_port`: the port that the DB instance listens on +* `__meta_rds_instance_preferred_backup_window`: the daily time range during which automated backups are created +* `__meta_rds_instance_preferred_maintenance_window`: the weekly time range during which system maintenance can occur +* `__meta_rds_instance_promotion_tier`: the order in which an Aurora replica is promoted to primary instance after a failure +* `__meta_rds_instance_publicly_accessible`: whether the DB instance is publicly accessible +* `__meta_rds_instance_read_replica_source_db_cluster_identifier`: the identifier of the source DB cluster if this instance is a read replica +* `__meta_rds_instance_read_replica_source_db_instance_identifier`: the identifier of the source DB instance if this instance is a read replica +* `__meta_rds_instance_replica_mode`: the replica mode (open-read-only or mounted) +* `__meta_rds_instance_resource_id`: the AWS Region-unique immutable identifier for the DB instance +* `__meta_rds_instance_resume_full_automation_mode_time`: the time when the DB instance will resume full automation +* `__meta_rds_instance_secondary_availability_zone`: the secondary availability zone for Multi-AZ instances +* `__meta_rds_instance_status`: the status of the DB instance +* `__meta_rds_instance_storage_encrypted`: whether the DB instance is storage encrypted +* `__meta_rds_instance_storage_encryption_type`: the storage encryption type +* `__meta_rds_instance_storage_throughput`: the storage throughput in MiBps +* `__meta_rds_instance_storage_type`: the storage type +* `__meta_rds_instance_storage_volume_status`: the status of the storage volume +* `__meta_rds_instance_subnet_group`: the name of the subnet group associated with the DB instance +* `__meta_rds_instance_tag_`: each tag value of the DB instance +* `__meta_rds_instance_tde_credential_arn`: the ARN for the TDE encryption key +* `__meta_rds_instance_timezone`: the time zone of the DB instance +* `__meta_rds_instance_upgrade_rollout_order`: the upgrade rollout order + +See below for the configuration options for AWS discovery: + +```yaml +# The AWS role to use for service discovery. +# Must be one of: ec2, lightsail, ecs, msk, elasticache, or rds. +role: + +# The AWS region. If blank, the region from the instance metadata is used. +[ region: ] + +# Custom endpoint to be used. +[ endpoint: ] + +# AWS access key ID. If blank, the environment variable AWS_ACCESS_KEY_ID is used. +[ access_key: ] + +# AWS secret access key. If blank, the environment variable AWS_SECRET_ACCESS_KEY is used. +[ secret_key: ] + +# Named AWS profile used to authenticate. +[ profile: ] + +# AWS Role ARN, an alternative to using AWS API keys. +[ role_arn: ] + +# Optional External ID that can go along with role_arn. +[ external_id: ] + +# Refresh interval to re-read the targets list. +[ refresh_interval: | default = 60s ] + +# The port to scrape metrics from. If using the public IP address, this must +# instead be specified in the relabeling rule. +[ port: | default = 80 ] + +# Filters can be used optionally to filter the instance list by other criteria (ec2 & rds role only). +# Available filter criteria can be found here: +# EC2: +# - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html +# - Filter API documentation: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Filter.html +# RDS: +# - https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_DescribeDBInstances.html +# - Filter API documentation: https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_Filter.html +filters: + [ - name: + values: , [...] ] + +# List of ECS, ElastiCache, MSK, or RDS cluster identifiers (ecs, elasticache, msk, and rds roles only) to discover. +# A List of ARNs of clusters to discover. If empty, all clusters in the region are discovered. +# This can significantly improve performance when you only need to monitor specific clusters/caches. +[ clusters: [, ...] ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] +``` + ### `` Azure SD configurations allow retrieving scrape targets from Azure VMs. +The discovery requires at least the following permissions: + +* `Microsoft.Compute/virtualMachines/read`: Required for VM discovery +* `Microsoft.Network/networkInterfaces/read`: Required for VM discovery +* `Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read`: Required for scale set (VMSS) discovery +* `Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/read`: Required for scale set (VMSS) discovery + The following meta labels are available on targets during [relabeling](#relabel_config): * `__meta_azure_machine_id`: the machine ID @@ -681,78 +1421,32 @@ subscription_id: # instead be specified in the relabeling rule. [ port: | default = 80 ] -# Authentication information used to authenticate to the Azure API. -# Note that `basic_auth`, `authorization` and `oauth2` options are -# mutually exclusive. -# `password` and `password_file` are mutually exclusive. +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] +``` -# Optional HTTP basic authentication information, currently not support by Azure. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] +### `` -# Optional `Authorization` header configuration, currently not supported by Azure. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration, currently not supported by Azure. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] -``` +Consul SD configurations allow retrieving scrape targets from [Consul's](https://www.consul.io) +service catalog. Discovery uses two Consul API endpoints: -### `` +1. The [Catalog API](https://developer.hashicorp.com/consul/api-docs/catalog) to list services + (used when `services` is empty, or when `tags` or `filter` are set). +2. The [Health API](https://developer.hashicorp.com/consul/api-docs/health) to retrieve service + instances and their health status. -Consul SD configurations allow retrieving scrape targets from [Consul's](https://www.consul.io) -Catalog API. +Because these two APIs have different filtering field schemas, Prometheus exposes separate filter +options for each: `filter` applies to the Catalog API and `health_filter` applies to the Health API. +For example, tags are exposed as `ServiceTags` in the Catalog API but as `Service.Tags` in the +Health API. The following meta labels are available on targets during [relabeling](#relabel_config): * `__meta_consul_address`: the address of the target * `__meta_consul_dc`: the datacenter name for the target * `__meta_consul_health`: the health status of the service -* `__meta_consul_partition`: the admin partition name where the service is registered +* `__meta_consul_partition`: the admin partition name where the service is registered * `__meta_consul_metadata_`: each node metadata key value of the target * `__meta_consul_node`: the node name defined for the target * `__meta_consul_service_address`: the service address of the target @@ -785,14 +1479,18 @@ The following meta labels are available on targets during [relabeling](#relabel_ services: [ - ] -# See https://www.consul.io/api/catalog.html#list-nodes-for-service to know more -# about the possible filters that can be used. +# Filter expression for the Catalog API. See https://developer.hashicorp.com/consul/api-docs/catalog#filtering for syntax. +[ filter: ] + +# Filter expression for the Health API. See https://developer.hashicorp.com/consul/api-docs/health#filtering for syntax. +[ health_filter: ] +# The `tags` and `node_meta` fields are deprecated in favor of `filter` and `health_filter`. # An optional list of tags used to filter nodes for a given service. Services must contain all tags in the list. tags: [ - ] -# Node metadata key/value pairs to filter nodes for a given service. +# Node metadata key/value pairs to filter nodes for a given service. As of Consul 1.14, consider `filter` or `health_filter` instead. [ node_meta: [ : ... ] ] @@ -806,65 +1504,9 @@ tags: # On large setup it might be a good idea to increase this value because the catalog will change all the time. [ refresh_interval: | default = 30s ] -# Authentication information used to authenticate to the consul server. -# Note that `basic_auth`, `authorization` and `oauth2` options are -# mutually exclusive. -# `password` and `password_file` are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` Note that the IP number and port used to scrape the targets is assembled as @@ -882,10 +1524,15 @@ metadata and a single tag). ### `` DigitalOcean SD configurations allow retrieving scrape targets from [DigitalOcean's](https://www.digitalocean.com/) -Droplets API. -This service discovery uses the public IPv4 address by default, by that can be -changed with relabeling, as demonstrated in [the Prometheus digitalocean-sd -configuration file](/documentation/examples/prometheus-digitalocean.yml). +API. +This service discovery supports multiple roles through the `role` parameter. + +One of the following `role` types can be configured to discover targets: + +#### `droplets` + +The `droplets` role discovers targets from DigitalOcean Droplets. The public IPv4 address is used by default, +but may be changed with relabeling. The following meta labels are available on targets during [relabeling](#relabel_config): @@ -903,73 +1550,38 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_digitalocean_tags`: the comma-separated list of tags of the droplet * `__meta_digitalocean_vpc`: the id of the droplet's VPC -```yaml -# Authentication information used to authenticate to the API server. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. - -# Optional HTTP basic authentication information, not currently supported by DigitalOcean. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] +#### `databases` -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] +The `databases` role discovers targets from DigitalOcean Managed Databases. -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] +The following meta labels are available on targets during [relabeling](#relabel_config): -# Whether to enable HTTP2. -[ enable_http2: | default: true ] +* `__meta_digitalocean_db_id`: the id of the database cluster +* `__meta_digitalocean_db_name`: the name of the database cluster +* `__meta_digitalocean_db_engine`: the engine of the database cluster (e.g., `pg`, `mysql`, `redis`, `mongodb`) +* `__meta_digitalocean_db_version`: the version of the engine +* `__meta_digitalocean_db_status`: the status of the database cluster +* `__meta_digitalocean_db_region`: the region of the database cluster +* `__meta_digitalocean_db_size`: the size of the database cluster +* `__meta_digitalocean_db_num_nodes`: the number of nodes in the database cluster +* `__meta_digitalocean_db_host`: the public host of the database cluster +* `__meta_digitalocean_db_private_host`: the private host of the database cluster +* `__meta_digitalocean_db_tag_`: each tag of the database cluster, with its value set to `true` -# TLS configuration. -tls_config: - [ ] +```yaml +# The DigitalOcean role to use for service discovery. +# Must be one of: droplets or databases. +[ role: | default = droplets ] # The port to scrape metrics from. [ port: | default = 80 ] -# The time after which the droplets are refreshed. +# The time after which the targets are refreshed. [ refresh_interval: | default = 60s ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` @@ -1001,34 +1613,6 @@ See below for the configuration options for Docker discovery: # Address of the Docker daemon. host: -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# TLS configuration. -tls_config: - [ ] - # The port to scrape metrics from, when `role` is nodes, and for discovered # tasks and services that don't have published ports. [ port: | default = 80 ] @@ -1037,7 +1621,7 @@ tls_config: [ host_networking_host: | default = "localhost" ] # Sort all non-nil networks in ascending order based on network name and -# get the first network if the container has multiple networks defined, +# get the first network if the container has multiple networks defined, # thus avoiding collecting duplicate targets. [ match_first_network: | default = true ] @@ -1052,39 +1636,9 @@ tls_config: # The time after which the containers are refreshed. [ refresh_interval: | default = 60s ] -# Authentication information used to authenticate to the Docker daemon. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` The [relabeling phase](#relabel_config) is the preferred and more powerful @@ -1193,34 +1747,6 @@ See below for the configuration options for Docker Swarm discovery: # Address of the Docker daemon. host: -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# TLS configuration. -tls_config: - [ ] - # Role of the targets to retrieve. Must be `services`, `tasks`, or `nodes`. role: @@ -1241,39 +1767,9 @@ role: # The time after which the service discovery data is refreshed. [ refresh_interval: | default = 60s ] -# Authentication information used to authenticate to the Docker daemon. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` The [relabeling phase](#relabel_config) is the preferred and more powerful @@ -1341,6 +1837,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_ec2_ipv6_addresses`: comma separated list of IPv6 addresses assigned to the instance's network interfaces, if present * `__meta_ec2_owner_id`: the ID of the AWS account that owns the EC2 instance * `__meta_ec2_platform`: the Operating System platform, set to 'windows' on Windows servers, absent otherwise +* `__meta_ec2_default_ipv6_address`: the first primary IPv6 address found if present, otherwise first non-primary IPv6 address, if present * `__meta_ec2_primary_ipv6_addresses`: comma separated list of the Primary IPv6 addresses of the instance, if present. The list is ordered based on the position of each corresponding network interface in the attachment order. * `__meta_ec2_primary_subnet_id`: the subnet ID of the primary network interface, if available * `__meta_ec2_private_dns_name`: the private DNS name of the instance, if available @@ -1373,6 +1870,9 @@ See below for the configuration options for EC2 discovery: # AWS Role ARN, an alternative to using AWS API keys. [ role_arn: ] +# Optional External ID that can go along with role_arn. +[ external_id: ] + # Refresh interval to re-read the instance list. [ refresh_interval: | default = 60s ] @@ -1388,66 +1888,10 @@ filters: [ - name: values: , [...] ] -# Authentication information used to authenticate to the EC2 API. -# Note that `basic_auth`, `authorization` and `oauth2` options are -# mutually exclusive. -# `password` and `password_file` are mutually exclusive. - -# Optional HTTP basic authentication information, currently not supported by AWS. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration, currently not supported by AWS. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutuall exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration, currently not supported by AWS. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] -``` +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] +``` The [relabeling phase](#relabel_config) is the preferred and more powerful way to filter targets based on arbitrary labels. For users with thousands of @@ -1495,6 +1939,25 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_openstack_tag_`: each metadata item of the instance, with any unsupported characters converted to an underscore. * `__meta_openstack_user_id`: the user account owning the tenant. +#### `loadbalancer` + +The `loadbalancer` role discovers one target per Octavia loadbalancer with a +`PROMETHEUS` listener. The target address defaults to the VIP address +of the load balancer. + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_openstack_loadbalancer_availability_zone`: the availability zone of the OpenStack load balancer. +* `__meta_openstack_loadbalancer_floating_ip`: the floating IP of the OpenStack load balancer. +* `__meta_openstack_loadbalancer_id`: the OpenStack load balancer ID. +* `__meta_openstack_loadbalancer_name`: the OpenStack load balancer name. +* `__meta_openstack_loadbalancer_provider`: the Octavia provider of the OpenStack load balancer. +* `__meta_openstack_loadbalancer_operating_status`: the operating status of the OpenStack load balancer. +* `__meta_openstack_loadbalancer_provisioning_status`: the provisioning status of the OpenStack load balancer. +* `__meta_openstack_loadbalancer_tags`: comma separated list of the OpenStack load balancer. +* `__meta_openstack_loadbalancer_vip`: the VIP of the OpenStack load balancer. +* `__meta_openstack_project_id`: the project (tenant) owning this load balancer. + See below for the configuration options for OpenStack discovery: ```yaml @@ -1675,63 +2138,9 @@ query: # The port to scrape metrics from. [ port: | default = 80 ] -# TLS configuration to connect to the PuppetDB. -tls_config: - [ ] - -# basic_auth, authorization, and oauth2, are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# `Authorization` HTTP header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials with the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` See [this example Prometheus configuration file](/documentation/examples/prometheus-puppetdb.yml) @@ -1745,7 +2154,7 @@ and serves as an interface to plug in custom service discovery mechanisms. It reads a set of files containing a list of zero or more ``s. Changes to all defined files are detected via disk watches -and applied immediately. +and applied immediately. While those individual files are watched for changes, the parent directory is also watched implicitly. This is to handle [atomic @@ -1883,7 +2292,10 @@ The following meta labels are available on all targets during [relabeling](#rela * `__meta_hetzner_server_status`: the status of the server * `__meta_hetzner_public_ipv4`: the public ipv4 address of the server * `__meta_hetzner_public_ipv6_network`: the public ipv6 network (/64) of the server -* `__meta_hetzner_datacenter`: the datacenter of the server + +Note that the `__meta_hetzner_datacenter` label is deprecated for both roles `robot` and `hcloud`: +- For the `robot` role, the replacement label is `__meta_hetzner_robot_datacenter`. +- For the `hcloud` role, the label will be removed after 1 July 2026. For more details, see the [changelog](https://docs.hetzner.cloud/changelog#2025-12-16-phasing-out-datacenters). The labels below are only available for targets with `role` set to `hcloud`: @@ -1891,8 +2303,10 @@ The labels below are only available for targets with `role` set to `hcloud`: * `__meta_hetzner_hcloud_image_description`: the description of the server image * `__meta_hetzner_hcloud_image_os_flavor`: the OS flavor of the server image * `__meta_hetzner_hcloud_image_os_version`: the OS version of the server image -* `__meta_hetzner_hcloud_datacenter_location`: the location of the server -* `__meta_hetzner_hcloud_datacenter_location_network_zone`: the network zone of the server +* `__meta_hetzner_hcloud_location`: the location of the server +* `__meta_hetzner_hcloud_location_network_zone`: the network zone of the server +* `__meta_hetzner_hcloud_datacenter_location`: the location of the server (deprecated in favor of `__meta_hetzner_hcloud_location`) +* `__meta_hetzner_hcloud_datacenter_location_network_zone`: the network zone of the server (deprecated in favor of `__meta_hetzner_hcloud_location_network_zone`) * `__meta_hetzner_hcloud_server_type`: the type of the server * `__meta_hetzner_hcloud_cpu_cores`: the CPU cores count of the server * `__meta_hetzner_hcloud_cpu_type`: the CPU type of the server (shared or dedicated) @@ -1904,6 +2318,7 @@ The labels below are only available for targets with `role` set to `hcloud`: The labels below are only available for targets with `role` set to `robot`: +* `__meta_hetzner_robot_datacenter`: the datacenter of the server * `__meta_hetzner_robot_product`: the product of the server * `__meta_hetzner_robot_cancelled`: the server cancellation status @@ -1912,74 +2327,19 @@ The labels below are only available for targets with `role` set to `robot`: # One of robot or hcloud. role: -# Authentication information used to authenticate to the API server. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. - -# Optional HTTP basic authentication information, required when role is robot -# Role hcloud does not support basic auth. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration, required when role is -# hcloud. Role robot does not support bearer token authentication. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] - # The port to scrape metrics from. [ port: | default = 80 ] # The time after which the servers are refreshed. [ refresh_interval: | default = 60s ] + +# Label selector used to filter the servers when fetching them from the API. See https://docs.hetzner.cloud/#label-selector for more details. +# Only used when role is hcloud. +[ label_selector: ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` @@ -2014,6 +2374,10 @@ Each target has a meta label `__meta_url` during the [relabeling phase](#relabel_config). Its value is set to the URL from which the target was extracted. +There is a list of +[integrations](https://prometheus.io/docs/operating/integrations/#http-service-discovery) with this +discovery mechanism. + ```yaml # URL from which the targets are fetched. url: @@ -2021,65 +2385,9 @@ url: # Refresh interval to re-query the endpoint. [ refresh_interval: | default = 60s ] -# Authentication information used to authenticate to the API server. -# Note that `basic_auth`, `authorization` and `oauth2` options are -# mutually exclusive. -# `password` and `password_file` are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` @@ -2113,74 +2421,15 @@ following meta labels are available on all targets during # The unique ID of the data center. datacenter_id: -# Authentication information used to authenticate to the API server. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. - -# Optional HTTP basic authentication information, required when using IONOS -# Cloud username and password as authentication method. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration, required when using IONOS -# Cloud token as authentication method. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] - # The port to scrape metrics from. [ port: | default = 80 ] # The time after which the servers are refreshed. [ refresh_interval: | default = 60s ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` @@ -2203,6 +2452,7 @@ Available meta labels: * `__meta_kubernetes_node_name`: The name of the node object. * `__meta_kubernetes_node_provider_id`: The cloud provider's name for the node object. +* `__meta_kubernetes_node_condition_`: For every entry in node.Status.Conditions, a label with the condition type in lowercase. Possible values are `true`, `false`, or `unknown`. Examples: `__meta_kubernetes_node_condition_ready`, `__meta_kubernetes_node_condition_memorypressure`, `__meta_kubernetes_node_condition_diskpressure`. * `__meta_kubernetes_node_label_`: Each label from the node object, with any unsupported characters converted to an underscore. * `__meta_kubernetes_node_labelpresent_`: `true` for each label from the node object, with any unsupported characters converted to an underscore. * `__meta_kubernetes_node_annotation_`: Each annotation from the node object. @@ -2265,6 +2515,9 @@ Available meta labels: * `__meta_kubernetes_pod_uid`: The UID of the pod object. * `__meta_kubernetes_pod_controller_kind`: Object kind of the pod controller. * `__meta_kubernetes_pod_controller_name`: Name of the pod controller. +* `__meta_kubernetes_pod_deployment_name`: Name of the deployment the pod belongs to. Requires `attach_metadata: {deployment: true}`. +* `__meta_kubernetes_pod_cronjob_name`: Name of the cronjob the pod belongs to. Requires `attach_metadata: {cronjob: true}`. +* `__meta_kubernetes_pod_job_name`: Name of the job the pod belongs to. Requires `attach_metadata: {job: true}`. #### `endpoints` @@ -2272,6 +2525,9 @@ The `endpoints` role discovers targets from listed endpoints of a service. For e address one target is discovered per port. If the endpoint is backed by a pod, all additional container ports of the pod, not bound to an endpoint port, are discovered as targets as well. +Note that the Endpoints API is [deprecated in Kubernetes v1.33+](https://kubernetes.io/blog/2025/04/24/endpoints-deprecation/), +it is recommended to use EndpointSlices instead and switch to the `endpointslice` role below. + Available meta labels: * `__meta_kubernetes_namespace`: The namespace of the endpoints object. @@ -2298,9 +2554,11 @@ The `endpointslice` role discovers targets from existing endpointslices. For eac address referenced in the endpointslice object one target is discovered. If the endpoint is backed by a pod, all additional container ports of the pod, not bound to an endpoint port, are discovered as targets as well. +The role requires the `discovery.k8s.io/v1` API version (available since Kubernetes v1.21). + Available meta labels: -* `__meta_kubernetes_namespace`: The namespace of the endpoints object. +* `__meta_kubernetes_namespace`: The namespace of the endpointslice object. * `__meta_kubernetes_endpointslice_name`: The name of endpointslice object. * `__meta_kubernetes_endpointslice_label_`: Each label from the endpointslice object, with any unsupported characters converted to an underscore. * `__meta_kubernetes_endpointslice_labelpresent_`: `true` for each label from the endpointslice object, with any unsupported characters converted to an underscore. @@ -2318,7 +2576,7 @@ Available meta labels: * `__meta_kubernetes_endpointslice_endpoint_topology_present_kubernetes_io_hostname`: Flag that shows if the referenced object has a kubernetes.io/hostname annotation. * `__meta_kubernetes_endpointslice_endpoint_hostname`: Hostname of the referenced endpoint. * `__meta_kubernetes_endpointslice_endpoint_node_name`: Name of the Node hosting the referenced endpoint. - * `__meta_kubernetes_endpointslice_endpoint_zone`: Zone the referenced endpoint exists in (only available when using the `discovery.k8s.io/v1` API group). + * `__meta_kubernetes_endpointslice_endpoint_zone`: Zone the referenced endpoint exists in. * `__meta_kubernetes_endpointslice_port`: Port of the referenced endpoint. * `__meta_kubernetes_endpointslice_port_name`: Named port of the referenced endpoint. * `__meta_kubernetes_endpointslice_port_protocol`: Protocol of the referenced endpoint. @@ -2331,6 +2589,8 @@ The `ingress` role discovers a target for each path of each ingress. This is generally useful for blackbox monitoring of an ingress. The address will be set to the host specified in the ingress spec. +The role requires the `networking.k8s.io/v1` API version (available since Kubernetes v1.19). + Available meta labels: * `__meta_kubernetes_namespace`: The namespace of the ingress object. @@ -2362,66 +2622,6 @@ role: # Note that api_server and kube_config are mutually exclusive. [ kubeconfig_file: ] -# Optional authentication information used to authenticate to the API server. -# Note that `basic_auth` and `authorization` options are mutually exclusive. -# password and password_file are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] - # Optional namespace discovery. If omitted, all namespaces are used. namespaces: own_namespace: @@ -2449,8 +2649,27 @@ namespaces: # Optional metadata to attach to discovered targets. If omitted, no additional metadata is attached. attach_metadata: # Attaches node metadata to discovered targets. Valid for roles: pod, endpoints, endpointslice. -# When set to true, Prometheus must have permissions to get Nodes. +# When set to true, Prometheus must have permissions to list/watch Nodes. [ node: | default = false ] +# Attaches namespace metadata to discovered targets. Valid for roles: pod, endpoints, endpointslice, service, ingress. +# When set to true, Prometheus must have permissions to list/watch Namespaces. + [ namespace: | default = false ] +# Attaches deployment metadata to discovered pod targets. Valid for role: pod. +# When set to true, Prometheus must have permissions to list/watch ReplicaSets. +# Enables the __meta_kubernetes_pod_deployment_name label. + [ deployment: | default = false ] +# Attaches job metadata to discovered pod targets. Valid for role: pod. +# When set to true, Prometheus must have permissions to list/watch Jobs. +# Enables the __meta_kubernetes_pod_job_name label. + [ job: | default = false ] +# Attaches cronjob metadata to discovered pod targets. Valid for role: pod. +# When set to true, Prometheus must have permissions to list/watch Jobs. +# Enables the __meta_kubernetes_pod_cronjob_name label. + [ cronjob: | default = false ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` See [this example Prometheus configuration file](/documentation/examples/prometheus-kubernetes.yml) @@ -2463,7 +2682,7 @@ which automates the Prometheus setup on top of Kubernetes. Kuma SD configurations allow retrieving scrape target from the [Kuma](https://kuma.io) control plane. -This SD discovers "monitoring assignments" based on Kuma [Dataplane Proxies](https://kuma.io/docs/latest/documentation/dps-and-data-model), +This SD discovers "monitoring assignments" based on Kuma [Dataplane Proxies](https://kuma.io/docs/latest/production/dp-config/dpp/#data-plane-proxy), via the MADS v1 (Monitoring Assignment Discovery Service) xDS API, and will create a target for each proxy inside a Prometheus-enabled mesh. @@ -2480,7 +2699,7 @@ See below for the configuration options for Kuma MonitoringAssignment discovery: # Address of the Kuma Control Plane's MADS xDS server. server: -# Client id is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. +# Client id is used by Kuma Control Plane to compute Monitoring Assignment for specific Prometheus backend. # This is useful when migrating between multiple Prometheus backends, or having separate backend for each Mesh. # When not specified, system hostname/fqdn will be used if available, if not `prometheus` will be used. [ client_id: ] @@ -2491,66 +2710,9 @@ server: # The time after which the monitoring assignments are refreshed. [ fetch_timeout: | default = 2m ] -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# TLS configuration. -tls_config: - [ ] - -# Authentication information used to authenticate to the Docker daemon. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional the `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials with the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` The [relabeling phase](#relabel_config) is the preferred and more powerful way @@ -2597,6 +2759,9 @@ See below for the configuration options for Lightsail discovery: # AWS Role ARN, an alternative to using AWS API keys. [ role_arn: ] +# Optional External ID that can go along with role_arn. +[ external_id: ] + # Refresh interval to re-read the instance list. [ refresh_interval: | default = 60s ] @@ -2604,65 +2769,9 @@ See below for the configuration options for Lightsail discovery: # instead be specified in the relabeling rule. [ port: | default = 80 ] -# Authentication information used to authenticate to the Lightsail API. -# Note that `basic_auth`, `authorization` and `oauth2` options are -# mutually exclusive. -# `password` and `password_file` are mutually exclusive. - -# Optional HTTP basic authentication information, currently not supported by AWS. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration, currently not supported by AWS. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutuall exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration, currently not supported by AWS. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` @@ -2673,6 +2782,8 @@ This service discovery uses the public IPv4 address by default, by that can be changed with relabeling, as demonstrated in [the Prometheus linode-sd configuration file](/documentation/examples/prometheus-linode.yml). +Linode APIv4 Token must be created with scopes: `linodes:read_only`, `ips:read_only`, and `events:read_only`. + The following meta labels are available on targets during [relabeling](#relabel_config): * `__meta_linode_instance_id`: the id of the linode instance @@ -2689,7 +2800,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_linode_status`: the status of the linode instance * `__meta_linode_tags`: a list of tags of the linode instance joined by the tag separator * `__meta_linode_group`: the display group a linode instance is a member of -* `__meta_linode_gpus`: the number of GPU's of the linode instance +* `__meta_linode_gpus`: the number of GPU's of the linode instance * `__meta_linode_hypervisor`: the virtualization software powering the linode instance * `__meta_linode_backups`: the backup service status of the linode instance * `__meta_linode_specs_disk_bytes`: the amount of storage space the linode instance has access to @@ -2700,71 +2811,10 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_linode_ipv6_ranges`: a list of IPv6 ranges with mask assigned to the linode instance joined by the tag separator ```yaml -# Authentication information used to authenticate to the API server. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. -# Note: Linode APIv4 Token must be created with scopes: 'linodes:read_only', 'ips:read_only', and 'events:read_only' - -# Optional HTTP basic authentication information, not currently supported by Linode APIv4. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional the `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials with the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] # Optional region to filter on. [ region: ] -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] - # The port to scrape metrics from. [ port: | default = 80 ] @@ -2773,6 +2823,10 @@ tls_config: # The time after which the linode instances are refreshed. [ refresh_interval: | default = 60s ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` @@ -2813,67 +2867,9 @@ servers: # It is mutually exclusive with `auth_token` and other authentication mechanisms. [ auth_token_file: ] -# Sets the `Authorization` header on every request with the -# configured username and password. -# This is mutually exclusive with other authentication mechanisms. -# password and password_file are mutually exclusive. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -# NOTE: The current version of DC/OS marathon (v1.11.0) does not support -# standard `Authentication` header, use `auth_token` or `auth_token_file` -# instead. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration for connecting to marathon servers -tls_config: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` By default every app listed in Marathon will be scraped by Prometheus. If not all @@ -2888,8 +2884,7 @@ in the configuration file), which can also be changed using relabeling. ### `` -Nerve SD configurations allow retrieving scrape targets from [AirBnB's Nerve] -(https://github.com/airbnb/nerve) which are stored in +Nerve SD configurations allow retrieving scrape targets from [AirBnB's Nerve](https://github.com/airbnb/nerve) which are stored in [Zookeeper](https://zookeeper.apache.org/). The following meta labels are available on targets during [relabeling](#relabel_config): @@ -2932,74 +2927,18 @@ The following meta labels are available on targets during [relabeling](#relabel_ [ namespace: | default = default ] [ refresh_interval: | default = 60s ] [ region: | default = global ] -[ server: ] +# The URL to connect to the API. +[ server: ] [ tag_separator: | default = ,] -# Authentication information used to authenticate to the nomad server. -# Note that `basic_auth`, `authorization` and `oauth2` options are -# mutually exclusive. -# `password` and `password_file` are mutually exclusive. - -# Optional HTTP basic authentication information. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# TLS configuration. -tls_config: - [ ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` -Serverset SD configurations allow retrieving scrape targets from [Serversets] -(https://github.com/twitter/finagle/tree/develop/finagle-serversets) which are +Serverset SD configurations allow retrieving scrape targets from [Serversets](https://github.com/twitter/finagle/tree/develop/finagle-serversets) which are stored in [Zookeeper](https://zookeeper.apache.org/). Serversets are commonly used by [Finagle](https://twitter.github.io/finagle/) and [Aurora](https://aurora.apache.org/). @@ -3026,6 +2965,93 @@ paths: Serverset data must be in the JSON format, the Thrift format is not currently supported. +### `` + +[STACKIT](https://www.stackit.de/de/) SD configurations allow retrieving +scrape targets from various APIs. + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_stackit_availability_zone`: The availability zone of the server. +* `__meta_stackit_label_`: Each server label, with unsupported characters replaced by underscores. +* `__meta_stackit_labelpresent_`: "true" for each label of the server, with unsupported characters replaced by underscores. +* `__meta_stackit_private_ipv4_`: the private ipv4 address of the server within a given network +* `__meta_stackit_public_ipv4`: the public ipv4 address of the server +* `__meta_stackit_id`: The ID of the target. +* `__meta_stackit_type`: The type or brand of the target. +* `__meta_stackit_name`: The server name. +* `__meta_stackit_status`: The current status of the server. +* `__meta_stackit_power_status`: The power status of the server. + +See below for the configuration options for STACKIT discovery: + +```yaml +# The STACKIT project +project: + +# STACKIT region to use. No automatic discovery of the region is done. +[ region : | default = "eu01" ] + +# Custom API endpoint to be used. Format scheme://host:port +[ endpoint : ] + +# The port to scrape metrics from. +[ port: | default = 80 ] + +# Raw private key string used for authenticating a service account +[ private_key: ] + +# Path to a file containing the raw private key string +[ private_key_path: ] + +# Full JSON-formatted service account key used for authentication +[ service_account_key: ] + +# Path to a file containing the JSON-formatted service account key +[ service_account_key_path: ] + +# Path to a file containing STACKIT credentials. +[ credentials_file_path: ] + +# The time after which the servers are refreshed. +[ refresh_interval: | default = 60s ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] +``` + +A [Service Account Key](https://docs.stackit.cloud/platform/access-and-identity/service-accounts/how-tos/manage-service-account-keys/) can be set through `http_config`. This can be done mapping values from STACKIT Service Account json into oauth2 configuration. + +From a given Service Account json +```json +{ + //.... + "credentials": { + "kid": "6a7c3b36-xxxxxxxx", + "iss": "xxxx@sa.stackit.cloud", + "sub": "af2c2336-xxxxxxxx", + "aud": "https://stackit-service-account-prod.apps.01.cf.eu01.stackit.cloud", + "privateKey": "-----BEGIN PRIVATE KEY-----xxxx" + } +} +``` + +properties can be mapped as: + +```yaml +stackit_sd_config: +- oauth2: + client_id: + client_certificate_key: + client_certificate_key_id: + iss: + audience: + grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer" + token_url: "https://service-account.api.stackit.cloud/token" + signature_algorithm: RS512 +``` + ### `` [Triton](https://github.com/joyent/triton) SD configurations allow retrieving @@ -3131,66 +3157,12 @@ See below for the configuration options for Eureka discovery: # The URL to connect to the Eureka server. server: -# Sets the `Authorization` header on every request with the -# configured username and password. -# password and password_file are mutually exclusive. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configures the scrape request's TLS settings. -tls_config: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - # Refresh interval to re-read the app instance list. [ refresh_interval: | default = 30s ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` See [the Prometheus eureka-sd configuration file](/documentation/examples/prometheus-eureka.yml) @@ -3221,6 +3193,8 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_scaleway_instance_project_id`: project id of the server * `__meta_scaleway_instance_public_ipv4`: the public IPv4 address of the server * `__meta_scaleway_instance_public_ipv6`: the public IPv6 address of the server +* `__meta_scaleway_instance_public_ipv4_addresses`: the public IPv4 addresses of the server +* `__meta_scaleway_instance_public_ipv6_addresses`: the public IPv6 addresses of the server * `__meta_scaleway_instance_region`: the region of the server * `__meta_scaleway_instance_security_group_id`: the ID of the security group of the server * `__meta_scaleway_instance_security_group_name`: the name of the security group of the server @@ -3291,39 +3265,9 @@ tags_filter: # Refresh interval to re-read the targets list. [ refresh_interval: | default = 60s ] -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# TLS configuration. -tls_config: - [ ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` ### `` @@ -3342,82 +3286,30 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_uyuni_proxy_module`: the module name if _Exporter Exporter_ proxy is configured for the target * `__meta_uyuni_scheme`: the protocol scheme used for requests -* `__meta_uyuni_system_id`: the system ID of the client - -See below for the configuration options for Uyuni discovery: - -```yaml -# The URL to connect to the Uyuni server. -server: - -# Credentials are used to authenticate the requests to Uyuni API. -username: -password: - -# The entitlement string to filter eligible systems. -[ entitlement: | default = monitoring_entitled ] - -# The string by which Uyuni group names are joined into the groups label. -[ separator: | default = , ] - -# Refresh interval to re-read the managed targets list. -[ refresh_interval: | default = 60s ] - -# Optional HTTP basic authentication information, currently not supported by Uyuni. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration, currently not supported by Uyuni. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration, currently not supported by Uyuni. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] +* `__meta_uyuni_system_id`: the system ID of the client -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] +See below for the configuration options for Uyuni discovery: -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] +```yaml +# The URL to connect to the Uyuni server. +server: -# Whether to enable HTTP2. -[ enable_http2: | default: true ] +# Credentials are used to authenticate the requests to Uyuni API. +username: +password: -# TLS configuration. -tls_config: - [ ] +# The entitlement string to filter eligible systems. +[ entitlement: | default = monitoring_entitled ] + +# The string by which Uyuni group names are joined into the groups label. +[ separator: | default = , ] + +# Refresh interval to re-read the managed targets list. +[ refresh_interval: | default = 60s ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` See [the Prometheus uyuni-sd configuration file](/documentation/examples/prometheus-uyuni.yml) @@ -3452,74 +3344,60 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_vultr_instance_allowed_bandwidth_gb` : Monthly bandwidth quota in GB. ```yaml -# Authentication information used to authenticate to the API server. -# Note that `basic_auth` and `authorization` options are -# mutually exclusive. -# password and password_file are mutually exclusive. +# The port to scrape metrics from. +[ port: | default = 80 ] -# Optional HTTP basic authentication information, not currently supported by Vultr. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] +# The time after which the instances are refreshed. +[ refresh_interval: | default = 60s ] -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] +``` -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] +### `` -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] +Outscale SD configurations allow retrieving scrape targets from [Outscale Cloud](https://outscale.com/) VMs via the Outscale API (OAPI). -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] +The following meta labels are available on targets during [relabeling](#relabel_config): -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] +* `__meta_outscale_vm_instance_id`: the ID of the VM +* `__meta_outscale_vm_region`: the region of the VM +* `__meta_outscale_vm_subregion`: the subregion of the VM +* `__meta_outscale_vm_state`: the state of the VM +* `__meta_outscale_vm_private_ip`: the private IP address of the VM +* `__meta_outscale_vm_public_ip`: the public IP address of the VM +* `__meta_outscale_vm_tag_`: each tag value; the tag key is sanitized and appended (e.g. tag key `Name` → `__meta_outscale_vm_tag_Name`) -# Whether to enable HTTP2. -[ enable_http2: | default: true ] +Targets use the first address found: private IP, then public IP. This can be changed with relabeling, as demonstrated in [the Prometheus outscale-sd configuration file](/documentation/examples/prometheus-outscale.yml). -# TLS configuration. -tls_config: - [ ] +See below for the configuration options for Outscale discovery: + +```yaml +# Region to use. +[ region: | default = "eu-west-2" ] + +# Access key (20 alphanumeric characters). See https://docs.outscale.com/en/userguide/Creating-an-Access-Key.html +access_key: + +# Secret key (40 characters). Use one of `secret_key` or `secret_key_file`. +[ secret_key: ] + +# Secret key file. +[ secret_key_file: ] + +# API endpoint URL. Defaults to https://api..outscale.com/api/v1 if empty. +[ endpoint: ] # The port to scrape metrics from. [ port: | default = 80 ] -# The time after which the instances are refreshed. +# Refresh interval to re-read the targets list. [ refresh_interval: | default = 60s ] -``` +# HTTP client settings. +[ ] +``` ### `` @@ -3537,6 +3415,11 @@ labels: [ : ... ] ``` +The special labels mentioned in the [relabeling](#relabel_config) section can also be +used here to override the respective settings in the scrape configuration. This is +especially useful when combined with any of the service discovery mechanisms that do not +support these settings directly. + ### `` Relabeling is a powerful tool to dynamically rewrite the label set of a target before @@ -3546,6 +3429,13 @@ in the configuration file. Initially, aside from the configured per-target labels, a target's `job` label is set to the `job_name` value of the respective scrape configuration. + +You can also use special labels like `__address__`, `__scheme__`, `__metrics_path__`, +`__scrape_interval__`, `__scrape_timeout__`, `__convert_classic_histograms_to_nhcb__`, +`__always_scrape_classic_histograms__`, `__scrape_native_histograms__` +to customize the defined targets. These will +override the respective settings in the scrape configuration. + The `__address__` label is set to the `:` address of the target. After relabeling, the `instance` label is set to the value of `__address__` by default if it was not set during relabeling. @@ -3559,6 +3449,26 @@ label is set to the value of the first passed URL parameter called ``, as The `__scrape_interval__` and `__scrape_timeout__` labels are set to the target's interval and timeout, as specified in `scrape_config`. +The `__convert_classic_histograms_to_nhcb__` label is set to the target's +`convert_classic_histograms_to_nhcb` value, as specified in `scrape_config` +(defaulting to the configured global). Setting it during relabeling overrides, +per target, whether classic histograms are converted to native histograms with +custom buckets. Its value must parse as a boolean; a target with an invalid +value is dropped. + +The `__always_scrape_classic_histograms__` label is set to the target's +`always_scrape_classic_histograms` value, as specified in `scrape_config` +(defaulting to the configured global). Setting it during relabeling overrides, +per target, whether a classic histogram is also ingested when it is exposed as +a native histogram. Its value must parse as a boolean; a target with an invalid +value is dropped. + +The `__scrape_native_histograms__` label is set to the target's +`scrape_native_histograms` value, as specified in `scrape_config` (defaulting to +the configured global). Setting it during relabeling overrides, per target, +whether native histograms are scraped. Its value must parse as a boolean; a +target with an invalid value is dropped. + Additional labels prefixed with `__meta_` may be available during the relabeling phase. They are set by the service discovery mechanism that provided the target and vary between mechanisms. @@ -3571,7 +3481,8 @@ input to a subsequent relabeling step), use the `__tmp` label name prefix. This prefix is guaranteed to never be used by Prometheus itself. ```yaml -# The source labels select values from existing labels. Their content is concatenated +# The source_labels tells the rule what labels to fetch from the series. Any +# labels which do not exist get a blank value (""). Their content is concatenated # using the configured separator and matched against the configured regular expression # for the replace, keep, and drop actions. [ source_labels: '[' [, ...] ']' ] @@ -3669,25 +3580,6 @@ through the `__alerts_path__` label. # Configures the protocol scheme used for requests. [ scheme: | default = http ] -# Sets the `Authorization` header on every request with the -# configured username and password. -# password and password_file are mutually exclusive. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - # Optionally configures AWS's Signature Verification 4 signing process to sign requests. # Cannot be set at the same time as basic_auth, authorization, oauth2, azuread or google_iam. # To use the default credentials from the AWS SDK, use `sigv4: {}`. @@ -3707,44 +3599,22 @@ sigv4: # AWS Role ARN, an alternative to using AWS API keys. [ role_arn: ] -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configures the scrape request's TLS settings. -tls_config: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] + # AWS External ID used when assuming a role. + # Can only be used with role_arn. + [ external_id: ] -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] + # Defines the FIPS mode for the AWS STS endpoint. + # Requires Prometheus >= 2.54.0 + # Note: FIPS STS selection should be configured via use_fips_sts_endpoint rather than environment variables. (The problem report that motivated this: AWS_USE_FIPS_ENDPOINT no longer works.) + [ use_fips_sts_endpoint: | default = false ] -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] -# Whether to enable HTTP2. -[ enable_http2: | default: true ] +# List of AWS service discovery configurations. +aws_sd_configs: + [ - ... ] # List of Azure service discovery configurations. azure_sd_configs: @@ -3826,6 +3696,10 @@ nomad_sd_configs: openstack_sd_configs: [ - ... ] +# List of Outscale service discovery configurations. +outscale_sd_configs: + [ - ... ] + # List of OVHcloud service discovery configurations. ovhcloud_sd_configs: [ - ... ] @@ -3842,6 +3716,10 @@ scaleway_sd_configs: serverset_sd_configs: [ - ... ] +# List of STACKIT service discovery configurations. +stackit_sd_configs: + [ - ... ] + # List of Triton service discovery configurations. triton_sd_configs: [ - ... ] @@ -3885,7 +3763,7 @@ url: # * The `prometheus.WriteRequest` represents the message introduced in Remote Write 1.0, which # will be deprecated eventually. # * The `io.prometheus.write.v2.Request` was introduced in Remote Write 2.0 and replaces the former, -# by improving efficiency and sending metadata, created timestamp and native histograms by default. +# by improving efficiency and sending metadata, start timestamp and native histograms by default. # # Before changing this value, consult with your remote storage provider (or test) what message it supports. # Read more on https://prometheus.io/docs/specs/remote_write_spec_2_0/#io-prometheus-write-v2-request @@ -3915,24 +3793,11 @@ write_relabel_configs: # For the `io.prometheus.write.v2.Request` message, this option is noop (always true). [ send_native_histograms: | default = false ] -# Sets the `Authorization` header on every remote write request with the -# configured username and password. -# password and password_file are mutually exclusive. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default = Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] +# When enabled, remote-write will resolve the URL host name via DNS, choose one of the IP addresses at random, and connect to it. +# When disabled, remote-write relies on Go's standard behavior, which is to try to connect to each address in turn. +# The connection timeout applies to the whole operation, i.e. in the latter case it is spread over all attempt. +# This is an experimental feature, and its behavior might still change, or even get removed. +[ round_robin_dns: | default = false ] # Optionally configures AWS's Signature Verification 4 signing process to # sign requests. Cannot be set at the same time as basic_auth, authorization, oauth2, or azuread. @@ -3953,10 +3818,14 @@ sigv4: # AWS Role ARN, an alternative to using AWS API keys. [ role_arn: ] -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth, authorization, sigv4, azuread or google_iam. -oauth2: - [ ] + # AWS External ID used when assuming a role. + # Can only be used with role_arn. + [ external_id: ] + + # Defines the FIPS mode for the AWS STS endpoint. + # Requires Prometheus >= 2.54.0 + # Note: FIPS STS selection should be configured via use_fips_sts_endpoint rather than environment variables. (The problem report that motivated this: AWS_USE_FIPS_ENDPOINT no longer works.) + [ use_fips_sts_endpoint: | default = false ] # Optional AzureAD configuration. # Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or google_iam. @@ -3964,9 +3833,15 @@ azuread: # The Azure Cloud. Options are 'AzurePublic', 'AzureChina', or 'AzureGovernment'. [ cloud: | default = AzurePublic ] - # Azure User-assigned Managed identity. + # Azure Managed Identity. Leave 'client_id' blank to use the default managed identity. [ managed_identity: - [ client_id: ] ] + [ client_id: ] ] + + # Azure Workload Identity. + [ workload_identity: + client_id: + tenant_id: + [ token_file_path: | default = "/var/run/secrets/azure/tokens/azure-identity-token" ] ] # Azure OAuth. [ oauth: @@ -3974,53 +3849,39 @@ azuread: [ client_secret: ] [ tenant_id: ] ] + # Azure Certificate-based authentication. + [ certificate: + client_id: + tenant_id: + certificate_path: + # Optional path to private key file if separate from certificate + [ certificate_key_path: ] + # Optional password for password-protected certificate files (PFX/PKCS12) + [ certificate_password: ] + # Whether to send the certificate chain in the x5c header + [ send_certificate_chain: | default = false ] ] + # Azure SDK auth. # See https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication [ sdk: [ tenant_id: ] ] + # Optional custom OAuth 2.0 scope to request when acquiring tokens. + # If not specified, defaults to the appropriate monitoring scope for the cloud: + # - AzurePublic: https://monitor.azure.com//.default + # - AzureGovernment: https://monitor.azure.us//.default + # - AzureChina: https://monitor.azure.cn//.default + # Use this to authenticate against custom Azure applications or non-standard endpoints. + [ scope: ] + # WARNING: Remote write is NOT SUPPORTED by Google Cloud. This configuration is reserved for future use. # Optional Google Cloud Monitoring configuration. # Cannot be used at the same time as basic_auth, authorization, oauth2, sigv4 or azuread. # To use the default credentials from the Google Cloud SDK, use `google_iam: {}`. google_iam: - # Service account key with monitoring write permessions. + # Service account key with monitoring write permissions. credentials_file: -# Configures the remote write request's TLS settings. -tls_config: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default = false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default = true ] - # Configures the queue used to write to remote storage. queue_config: # Number of samples to buffer per shard before we block reading of more @@ -4062,6 +3923,11 @@ metadata_config: [ send_interval: | default = 1m ] # Maximum number of samples per send. [ max_samples_per_send: | default = 500] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +# enable_http2 defaults to false for remote-write. +[ ] ``` There is a list of @@ -4096,66 +3962,12 @@ headers: # the local storage should have complete data for. [ read_recent: | default = false ] -# Sets the `Authorization` header on every remote read request with the -# configured username and password. -# password and password_file are mutually exclusive. -basic_auth: - [ username: ] - [ password: ] - [ password_file: ] - -# Optional `Authorization` header configuration. -authorization: - # Sets the authentication type. - [ type: | default: Bearer ] - # Sets the credentials. It is mutually exclusive with - # `credentials_file`. - [ credentials: ] - # Sets the credentials to the credentials read from the configured file. - # It is mutually exclusive with `credentials`. - [ credentials_file: ] - -# Optional OAuth 2.0 configuration. -# Cannot be used at the same time as basic_auth or authorization. -oauth2: - [ ] - -# Configures the remote read request's TLS settings. -tls_config: - [ ] - -# Optional proxy URL. -[ proxy_url: ] -# Comma-separated string that can contain IPs, CIDR notation, domain names -# that should be excluded from proxying. IP and domain names can -# contain port numbers. -[ no_proxy: ] -# Use proxy URL indicated by environment variables (HTTP_PROXY, https_proxy, HTTPs_PROXY, https_proxy, and no_proxy) -[ proxy_from_environment: | default: false ] -# Specifies headers to send to proxies during CONNECT requests. -[ proxy_connect_header: - [ : [, ...] ] ] - -# Custom HTTP headers to be sent along with each request. -# Headers that are set by Prometheus itself can't be overwritten. -http_headers: - # Header name. - [ : - # Header values. - [ values: [, ...] ] - # Headers values. Hidden in configuration page. - [ secrets: [, ...] ] - # Files to read header values from. - [ files: [, ...] ] ] - -# Configure whether HTTP requests follow HTTP 3xx redirects. -[ follow_redirects: | default = true ] - -# Whether to enable HTTP2. -[ enable_http2: | default: true ] - # Whether to use the external labels as selectors for the remote read endpoint. [ filter_external_labels: | default = true ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] ``` There is a list of @@ -4166,8 +3978,6 @@ with this feature. `tsdb` lets you configure the runtime-reloadable configuration settings of the TSDB. -NOTE: Out-of-order ingestion is an experimental feature, but you do not need any additional flag to enable it. Setting `out_of_order_time_window` to a positive duration enables it. - ```yaml # Configures how old an out-of-order/out-of-bounds sample can be w.r.t. the TSDB max time. # An out-of-order/out-of-bounds sample is ingested into the TSDB as long as the timestamp @@ -4179,10 +3989,70 @@ NOTE: Out-of-order ingestion is an experimental feature, but you do not need any # that is within the out-of-order window, or (b) too-old, i.e. not in-order # and before the out-of-order window. # -# When out_of_order_time_window is greater than 0, it also affects experimental agent. It allows -# the agent's WAL to accept out-of-order samples that fall within the specified time window relative +# When out_of_order_time_window is greater than 0, it also affects experimental agent. It allows +# the agent's WAL to accept out-of-order samples that fall within the specified time window relative # to the timestamp of the last appended sample for the same series. [ out_of_order_time_window: | default = 0s ] + +# Configures the trigger point for compacting the stale series from the memory into persistent blocks +# and remove those stale series from the memory. +# +# The threshold is a number between 0.0 and 1.0. It represents the ratio of stale series in the memory +# to the total series in the memory. The stale series compaction is triggered when this ratio crosses +# the configured threshold. It may not trigger the stale series compaction if the usual head compaction +# is about to happen soon. +# +# If set to 0, stale series compaction is disabled. +# +# This is an experimental feature, this behaviour could change or be removed in the future. +[ stale_series_compaction_threshold: | default = 0 ] + +# Configures the float chunk encoding to use for new chunks. +# Valid values are 'xor' and 'xor2'. When absent, the encoding follows the +# --enable-feature=xor2-encoding flag: 'xor2' if the flag is set, 'xor' otherwise. +# Setting 'xor' forces standard XOR encoding even when --enable-feature=xor2-encoding is set. +# Setting 'xor2' is only valid when --enable-feature=xor2-encoding is set; +# Prometheus will refuse to reload if 'xor2' is set without the feature flag. +# Setting 'xor' is incompatible with --enable-feature=st-storage (XOR chunks do not store +# start timestamps); Prometheus will refuse to reload in that case too. +# Omitting 'floats' (or the entire 'chunk_encoding' field) is equivalent; the encoding +# follows the --enable-feature=xor2-encoding flag. +# This field is runtime-reloadable. +# When --enable-feature=st-storage is disabled, XOR and XOR2 are compatible +# encodings and in-progress chunks are not cut on an encoding change; the new +# encoding takes effect when the current chunk is next cut for any reason (size, time range, or sample count). +# When --enable-feature=st-storage is enabled, XOR and XOR2 are not compatible +# (XOR chunks do not store start timestamps), so an in-progress chunk is cut +# on the next append after the encoding changes. +[ chunk_encoding: + [ floats: ] ] + +# Configures data retention settings for TSDB. +# +# Note: When retention is changed at runtime, the retention +# settings are updated immediately, but block deletion based on the new retention policy +# occurs during the next block reload cycle. This happens automatically within 1 minute +# or when a compaction completes, whichever comes first. +[ retention: ] : + # How long to retain samples in storage. If neither this option nor the size option + # is set, the retention time defaults to 15d. Setting this to 0 disables time-based retention. + # This option takes precedence over the deprecated command-line flag --storage.tsdb.retention.time. + [ time: ] + + # Maximum number of bytes that can be stored for blocks. A unit is required, + # supported units: B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. + # If set to 0 or not set, size-based retention is disabled. + # This option takes precedence over the deprecated command-line flag --storage.tsdb.retention.size. + [ size: | default = 0 ] + + # Maximum percent of total disk space allowed for storage of blocks. Alternative to `size` and + # behaves the same as if size was calculated by hand as a percentage of the total storage capacity. + # Prometheus will fail to start if this config is enabled, but it fails to query the total storage capacity. + # The total disk space allowed will automatically adapt to volume resize. + # If set to 0 or not set, percentage-based retention is disabled. + # + # This is an experimental feature, this behaviour could change or be removed in the future. + [ percentage: | default = 0 ] ``` ### `` @@ -4225,3 +4095,6 @@ headers: tls_config: [ ] ``` + +If query logging and tracing are both enabled, a traceID and spanID will be injected +into the query log file for use in log/trace correlation. diff --git a/docs/configuration/https.md b/docs/configuration/https.md index bc83e07a380..9a089ca9220 100644 --- a/docs/configuration/https.md +++ b/docs/configuration/https.md @@ -3,8 +3,6 @@ title: HTTPS and authentication sort_rank: 7 --- -# HTTPS and authentication - Prometheus supports basic authentication and TLS. This is **experimental** and might change in the future. @@ -27,7 +25,7 @@ Generic placeholders are defined as follows: A valid example file can be found [here](/documentation/examples/web-config.yml). -``` +```yaml tls_server_config: # Certificate and key files for server to use to authenticate to client. cert_file: diff --git a/docs/configuration/promtool.md b/docs/configuration/promtool.md new file mode 100644 index 00000000000..d127ad080c9 --- /dev/null +++ b/docs/configuration/promtool.md @@ -0,0 +1,175 @@ +--- +title: HTTP configuration for promtool +sort_rank: 6 +--- + +Promtool is a versatile CLI tool for Prometheus that supports validation, debugging, querying, unit testing, tsdb management, pushing data, and experimental PromQL editing. + +Prometheus supports basic authentication and TLS. Since promtool needs to connect to Prometheus, we need to provide the authentication details. To specify those authentication details, use the `--http.config.file` for all requests that need to communicate with Prometheus. +For instance, if you would like to check whether your local Prometheus server is healthy, you would use: +```bash +promtool check healthy --url=http://localhost:9090 --http.config.file=http-config-file.yml +``` + +The file is written in [YAML format](https://en.wikipedia.org/wiki/YAML), defined by the schema described below. +Brackets indicate that a parameter is optional. For non-list parameters the value is set to the specified default. + +The file is read upon every http request, such as any change in the +configuration and the certificates is picked up immediately. + +Generic placeholders are defined as follows: + +* ``: a boolean that can take the values `true` or `false` +* ``: a valid path to a file +* ``: a regular string that is a secret, such as a password +* ``: a regular string + +A valid example file can be found [here](/documentation/examples/promtool-http-config-file.yml). + +```yaml +# Note that `basic_auth` and `authorization` options are mutually exclusive. + +# Sets the `Authorization` header with the configured username and password. +# `username_ref` and `password_ref`refer to the name of the secret within the secret manager. +# `password`, `password_file` and `password_ref` are mutually exclusive. +basic_auth: + [ username: ] + [ username_file: ] + [ username_ref: ] + [ password: ] + [ password_file: ] + [ password_ref: ] + +# Optional the `Authorization` header configuration. +authorization: + # Sets the authentication type. + [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials with the credentials read from the configured file. + # It is mutually exclusive with `credentials`. + [ credentials_file: ] + [ credentials_ref: ] + +# Optional OAuth 2.0 configuration. +# Cannot be used at the same time as basic_auth or authorization. +oauth2: + [ ] + +tls_config: + [ ] + +[ follow_redirects: | default: true ] + +# Whether to enable HTTP2. +[ enable_http2: | default: true ] + +# Optional proxy URL. +[ proxy_url: ] +# Comma-separated string that can contain IPs, CIDR notation, domain names +# that should be excluded from proxying. IP and domain names can +# contain port numbers. +[ no_proxy: ] +[ proxy_from_environment: ] +[ proxy_connect_header: + [ : [ , ... ] ] ] + +# `http_headers` specifies a set of headers that will be injected into each request. +http_headers: + [ :
    ] +``` + +## \ +OAuth 2.0 authentication using the client credentials grant type. +```yaml +# `client_id` and `client_secret` are used to authenticate your +# application with the authorization server in order to get +# an access token. +# `client_secret`, `client_secret_file` and `client_secret_ref` are mutually exclusive. +client_id: +[ client_secret: ] +[ client_secret_file: ] +[ client_secret_ref: ] + +# `scopes` specify the reason for the resource access. +scopes: + [ - ...] + +# The URL to fetch the token from. +token_url: + +# Optional parameters to append to the token URL. +[ endpoint_params: + : ... ] + +# Configures the token request's TLS settings. +tls_config: + [ ] + +# Optional proxy URL. +[ proxy_url: ] +# Comma-separated string that can contain IPs, CIDR notation, domain names +# that should be excluded from proxying. IP and domain names can +# contain port numbers. +[ no_proxy: ] +[ proxy_from_environment: ] +[ proxy_connect_header: + [ : [ , ... ] ] ] +``` + +## +```yaml +# For the following configurations, use either `ca`, `cert` and `key` or `ca_file`, `cert_file` and `key_file` or use `ca_ref`, `cert_ref` or `key_ref`. +# Text of the CA certificate to use for the server. +[ ca: ] +# CA certificate to validate the server certificate with. +[ ca_file: ] +# `ca_ref` is the name of the secret within the secret manager to use as the CA cert. +[ ca_ref: ] + +# Text of the client cert file for the server. +[ cert: ] +# Certificate file for client certificate authentication. +[ cert_file: ] +# `cert_ref` is the name of the secret within the secret manager to use as the client certificate. +[ cert_ref: ] + +# Text of the client key file for the server. +[ key: ] +# Key file for client certificate authentication. +[ key_file: ] +# `key_ref` is the name of the secret within the secret manager to use as the client key. +[ key_ref: ] + +# ServerName extension to indicate the name of the server. +# http://tools.ietf.org/html/rfc4366#section-3.1 +[ server_name: ] + +# Disable validation of the server certificate. +[ insecure_skip_verify: ] + +# Minimum acceptable TLS version. Accepted values: TLS10 (TLS 1.0), TLS11 (TLS +# 1.1), TLS12 (TLS 1.2), TLS13 (TLS 1.3). +# If unset, promtool will use Go default minimum version, which is TLS 1.2. +# See MinVersion in https://pkg.go.dev/crypto/tls#Config. +[ min_version: ] +# Maximum acceptable TLS version. Accepted values: TLS10 (TLS 1.0), TLS11 (TLS +# 1.1), TLS12 (TLS 1.2), TLS13 (TLS 1.3). +# If unset, promtool will use Go default maximum version, which is TLS 1.3. +# See MaxVersion in https://pkg.go.dev/crypto/tls#Config. +[ max_version: ] +``` + +## \ +`header` represents the configuration for a single HTTP header. +```yaml +[ values: + [ - ... ] ] + +[ secrets: + [ - ... ] ] + +[ files: + [ - ... ] ] +``` diff --git a/docs/configuration/recording_rules.md b/docs/configuration/recording_rules.md index 9aa226bbc0b..6d668a4da30 100644 --- a/docs/configuration/recording_rules.md +++ b/docs/configuration/recording_rules.md @@ -1,10 +1,9 @@ --- -title: Recording rules +title: Defining recording rules +nav_title: Recording rules sort_rank: 2 --- -# Defining recording rules - ## Configuring rules Prometheus supports two types of rules which may be configured and then @@ -17,10 +16,6 @@ Rule files use YAML. The rule files can be reloaded at runtime by sending `SIGHUP` to the Prometheus process. The changes are only applied if all rule files are well-formatted. -_Note about native histograms (experimental feature): Native histogram are always -recorded as gauge histograms (for now). Most cases will create gauge histograms -naturally, e.g. after `rate()`._ - ## Syntax-checking rules To quickly check whether a rule file is syntactically correct without starting @@ -38,7 +33,7 @@ When the file is syntactically valid, the checker prints a textual representation of the parsed rules to standard output and then exits with a `0` return status. -If there are any syntax errors or invalid input arguments, it prints an error +If there are any syntax errors or invalid input arguments, it prints an error message to standard error and exits with a `1` return status. ## Recording rules @@ -75,7 +70,8 @@ groups: ``` ### `` -``` + +```yaml # The name of the group. Must be unique within a file. name: @@ -89,6 +85,11 @@ name: # Offset the rule evaluation timestamp of this particular group by the specified duration into the past. [ query_offset: | default = global.rule_query_offset ] +# Labels to add or overwrite before storing the result for its rules. +# Labels defined in will override the key if it has a collision. +labels: + [ : ] + rules: [ - ... ] ``` @@ -97,7 +98,7 @@ rules: The syntax for recording rules is: -``` +```yaml # The name of the time series to output to. Must be a valid metric name. record: @@ -113,7 +114,7 @@ labels: The syntax for alerting rules is: -``` +```yaml # The name of the alert. Must be a valid label value. alert: @@ -142,7 +143,7 @@ annotations: See also the [best practices for naming metrics created by recording rules](https://prometheus.io/docs/practices/rules/#recording-rules). -# Limiting alerts and series +## Limiting alerts and series A limit for alerts produced by alerting rules and series produced recording rules can be configured per-group. When the limit is exceeded, _all_ series produced @@ -151,9 +152,9 @@ the rule, active, pending, or inactive, are cleared as well. The event will be recorded as an error in the evaluation, and as such no stale markers are written. -# Rule query offset +## Rule query offset This is useful to ensure the underlying metrics have been received and stored in Prometheus. Metric availability delays are more likely to occur when Prometheus is running as a remote write target due to the nature of distributed systems, but can also occur when there's anomalies with scraping and/or short evaluation intervals. -# Failed rule evaluations due to slow evaluation +## Failed rule evaluations due to slow evaluation -If a rule group hasn't finished evaluating before its next evaluation is supposed to start (as defined by the `evaluation_interval`), the next evaluation will be skipped. Subsequent evaluations of the rule group will continue to be skipped until the initial evaluation either completes or times out. When this happens, there will be a gap in the metric produced by the recording rule. The `rule_group_iterations_missed_total` metric will be incremented for each missed iteration of the rule group. +If a rule group hasn't finished evaluating before its next evaluation is supposed to start (as defined by the `evaluation_interval`), the next evaluation will be skipped. Subsequent evaluations of the rule group will continue to be skipped until the initial evaluation either completes or times out. When this happens, there will be a gap in the metric produced by the recording rule. The `rule_group_iterations_missed_total` metric will be incremented for each missed iteration of the rule group. diff --git a/docs/configuration/template_examples.md b/docs/configuration/template_examples.md index 672295343fc..bd076b256e0 100644 --- a/docs/configuration/template_examples.md +++ b/docs/configuration/template_examples.md @@ -3,8 +3,6 @@ title: Template examples sort_rank: 4 --- -# Template examples - Prometheus supports templating in the annotations and labels of alerts, as well as in served console pages. Templates have the ability to run queries against the local database, iterate over data, use conditionals, @@ -13,7 +11,7 @@ templating](https://golang.org/pkg/text/template/) system. ## Simple alert field templates -``` +```yaml alert: InstanceDown expr: up == 0 for: 5m @@ -33,7 +31,7 @@ console instead. This displays a list of instances, and whether they are up: -```go +``` {{ range query "up" }} {{ .Labels.instance }} {{ .Value }} {{ end }} @@ -43,7 +41,7 @@ The special `.` variable contains the value of the current sample for each loop ## Display one value -```go +``` {{ with query "some_metric{instance='someinstance'}" }} {{ . | first | value | humanize }} {{ end }} @@ -58,7 +56,7 @@ formatting of results, and linking to the [expression browser](https://prometheu ## Using console URL parameters -```go +``` {{ with printf "node_memory_MemTotal{job='node',instance='%s'}" .Params.instance | query }} {{ . | first | value | humanize1024 }}B {{ end }} @@ -95,7 +93,7 @@ powerful when combined with [console library](template_reference.md#console-templates) support, allowing sharing of templates across consoles. -```go +``` {{/* Define the template */}} {{define "myTemplate"}} do something @@ -107,7 +105,7 @@ sharing of templates across consoles. Templates are limited to one argument. The `args` function can be used to wrap multiple arguments. -```go +``` {{define "myMultiArgTemplate"}} First argument: {{.arg0}} Second argument: {{.arg1}} diff --git a/docs/configuration/template_reference.md b/docs/configuration/template_reference.md index 47df9d1e090..30725a0b043 100644 --- a/docs/configuration/template_reference.md +++ b/docs/configuration/template_reference.md @@ -3,8 +3,6 @@ title: Template reference sort_rank: 5 --- -# Template reference - Prometheus supports templating in the annotations and labels of alerts, as well as in served console pages. Templates have the ability to run queries against the local database, iterate over data, use conditionals, @@ -17,8 +15,8 @@ The primary data structure for dealing with time series data is the sample, defi ```go type sample struct { - Labels map[string]string - Value interface{} + Labels map[string]string + Value interface{} } ``` @@ -57,8 +55,10 @@ If functions are used in a pipeline, the pipeline value is passed as the last ar | humanize1024 | number or string | string | Like `humanize`, but uses 1024 as the base rather than 1000. | | humanizeDuration | number or string | string | Converts a duration in seconds to a more readable format. | | humanizePercentage | number or string | string | Converts a ratio value to a fraction of 100. | -| humanizeTimestamp | number or string | string | Converts a Unix timestamp in seconds to a more readable format. | -| toTime | number or string | *time.Time | Converts a Unix timestamp in seconds to a time.Time. | +| humanizeTimestamp | number or string | string | Converts a Unix timestamp in seconds to a more readable format. | +| toTime | number or string | *time.Time | Converts a Unix timestamp in seconds to a time.Time. | +| toDuration | number or string | *time.Duration | Converts a duration in seconds to a time.Duration. | +| now | none | float64 | Returns the Unix timestamp in seconds at the time of the template evaluation. | Humanizing functions are intended to produce reasonable output for consumption by humans, and are not guaranteed to return the same results between Prometheus @@ -68,7 +68,7 @@ versions. | Name | Arguments | Returns | Notes | | ------------- | ------------- | ------- | ----------- | -| title | string | string | [strings.Title](https://golang.org/pkg/strings/#Title), capitalises first character of each word.| +| title | string | string | [cases.Title](https://pkg.go.dev/golang.org/x/text/cases#Title), capitalises first character of each word.| | toUpper | string | string | [strings.ToUpper](https://golang.org/pkg/strings/#ToUpper), converts all characters to upper case.| | toLower | string | string | [strings.ToLower](https://golang.org/pkg/strings/#ToLower), converts all characters to lower case.| | stripPort | string | string | [net.SplitHostPort](https://pkg.go.dev/net#SplitHostPort), splits string into host and port, then returns only host.| @@ -78,6 +78,7 @@ versions. | tableLink | expr | string | Returns path to tabular ("Table") view in the [expression browser](https://prometheus.io/docs/visualization/browser/) for the expression. | | parseDuration | string | float | Parses a duration string such as "1h" into the number of seconds it represents. | | stripDomain | string | string | Removes the domain part of a FQDN. Leaves port untouched. | +| urlQueryEscape | string | string | [url.QueryEscape](https://pkg.go.dev/net/url#QueryEscape) Escapes the string so it can be safely placed inside a URL query. | ### Others diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 7fc676a2515..af94c414f00 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -1,17 +1,15 @@ --- -title: Unit Testing for Rules +title: Unit testing for rules sort_rank: 6 --- -# Unit Testing for Rules - You can use `promtool` to test your rules. ```shell # For a single test file. ./promtool test rules test.yml -# If you have multiple test files, say test1.yml,test2.yml,test2.yml +# If you have multiple test files, say test1.yml,test2.yml,test3.yml ./promtool test rules test1.yml test2.yml test3.yml ``` @@ -24,6 +22,10 @@ rule_files: [ evaluation_interval: | default = 1m ] +# Setting fuzzy_compare true will very slightly weaken floating point comparisons. +# This will (effectively) ignore differences in the last bit of the mantissa. +[ fuzzy_compare: | default = false ] + # The order in which group names are listed below will be the order of evaluation of # rule groups (at a given evaluation time). The order is guaranteed only for the groups mentioned below. # All the groups need not be mentioned below. @@ -46,6 +48,18 @@ input_series: # Name of the test group [ name: ] +# Start timestamp for the test group. This sets the base time for all samples +# and evaluations in this test group. +# Accepts either a Unix timestamp (e.g., 1609459200) or an RFC3339 formatted +# timestamp (e.g., "2021-01-01T00:00:00Z"). +# Default: 0 (Unix epoch: 1970-01-01 00:00:00 UTC) +# +# When set: +# - All input_series samples are timestamped starting from start_timestamp +# - The eval_time in test cases is relative to start_timestamp +# - The time() function returns start_timestamp + eval_time +[ start_timestamp: | | default = 0 ] + # Unit tests for the above data. # Unit tests for alerting rules. We consider the alerting rules from the input file. @@ -95,20 +109,22 @@ series: # {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5 counter_reset_hint:gauge}} # Native histograms support the same expanding notation as floating point numbers, i.e. 'axn', 'a+bxn' and 'a-bxn'. # All properties are optional and default to 0. The order is not important. The following properties are supported: -# - schema (int): -# Currently valid schema numbers are -4 <= n <= 8. They are all for -# base-2 bucket schemas, where 1 is a bucket boundary in each case, and +# - schema (int): +# Currently valid schema numbers are -53 and -4 <= n <= 8. +# Schema -53 is the custom buckets schema, upper bucket boundaries are defined in custom_values +# like for classic histograms, and you shouldn't use z_bucket, z_bucket_w, n_buckets, n_offset. +# The rest are base-2 standard schemas, where 1.0 is a bucket boundary in each case, and # then each power of two is divided into 2^n logarithmic buckets. Or # in other words, each bucket boundary is the previous boundary times # 2^(2^-n). -# - sum (float): +# - sum (float): # The sum of all observations, including the zero bucket. -# - count (non-negative float): +# - count (non-negative float): # The number of observations, including those that are NaN and including the zero bucket. -# - z_bucket (non-negative float): +# - z_bucket (non-negative float): # The sum of all observations in the zero bucket. -# - z_bucket_w (non-negative float): -# The width of the zero bucket. +# - z_bucket_w (non-negative float): +# The width of the zero bucket. # If z_bucket_w > 0, the zero bucket contains all observations -z_bucket_w <= x <= z_bucket_w. # Otherwise, the zero bucket only contains observations that are exactly 0. # - buckets (list of non-negative floats): @@ -121,6 +137,10 @@ series: # The starting index of the first entry in the negative buckets. # - counter_reset_hint (one of 'unknown', 'reset', 'not_reset' or 'gauge') # The counter reset hint associated with this histogram. Defaults to 'unknown' if not set. +# - custom_values (list of floats in ascending order): +# The upper limits for custom buckets when schema is -53. +# These have the same role as the 'le' numbers in classic histograms. +# Do not append '+Inf' at the end, it is implicit. values: ``` @@ -129,7 +149,8 @@ values: Prometheus allows you to have same alertname for different alerting rules. Hence in this unit testing, you have to list the union of all the firing alerts for the alertname under a single ``. ``` yaml -# The time elapsed from time=0s when the alerts have to be checked. +# The time elapsed from start_timestamp when the alerts have to be checked. +# This is a duration relative to start_timestamp (which defaults to 0). eval_time: # Name of the alert to be tested. @@ -160,7 +181,8 @@ exp_annotations: # Expression to evaluate expr: -# The time elapsed from time=0s when the expression has to be evaluated. +# The time elapsed from start_timestamp when the expression has to be evaluated. +# This is a duration relative to start_timestamp (which defaults to 0). eval_time: # Expected samples at the given evaluation time. @@ -267,3 +289,24 @@ groups: summary: "Instance {{ $labels.instance }} down" description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." ``` + +### Time within tests + +It should be noted that in all tests, either in `alert_test_case` or +`promql_test_case`, the output from all functions related to the current time, +for example the `time()` and `day_of_*()` functions, will output a consistent value +for tests. + +By default, at the start of the test evaluation, `time()` returns 0 (Unix epoch: +January 1, 1970 00:00:00 UTC). The `eval_time` field specifies a duration relative +to `start_timestamp`, so by default `time()` will return a value of `0 + eval_time`. + +You can configure a custom start timestamp for your tests by setting the `start_timestamp` +field in your test group. This field accepts either: +- A Unix timestamp (e.g., `1609459200` for January 1, 2021 00:00:00 UTC) +- An RFC3339 formatted timestamp (e.g., `"2021-01-01T00:00:00Z"`) + +When you set `start_timestamp`: +- All `input_series` samples will be timestamped starting from `start_timestamp` +- The `eval_time` field in test cases is interpreted as a duration relative to `start_timestamp` +- The `time()` function will return `start_timestamp + eval_time` diff --git a/docs/feature_flags.md b/docs/feature_flags.md index 7b07a04d0e2..8b500dba501 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -3,36 +3,17 @@ title: Feature flags sort_rank: 12 --- -# Feature flags - Here is a list of features that are disabled by default since they are breaking changes or are considered experimental. Their behaviour can change in future releases which will be communicated via the [release changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md). You can enable them using the `--enable-feature` flag with a comma separated list of features. They may be enabled by default in future versions. -## Expand environment variables in external labels - -`--enable-feature=expand-external-labels` - -Replace `${var}` or `$var` in the [`external_labels`](configuration/configuration.md#configuration-file) -values according to the values of the current environment variables. References -to undefined variables are replaced by the empty string. -The `$` character can be escaped by using `$$`. - -## Remote Write Receiver - -`--enable-feature=remote-write-receiver` - -The remote write receiver allows Prometheus to accept remote write requests from other Prometheus servers. More details can be found [here](storage.md#overview). - -Activating the remote write receiver via a feature flag is deprecated. Use `--web.enable-remote-write-receiver` instead. This feature flag will be ignored in future versions of Prometheus. - ## Exemplars storage `--enable-feature=exemplar-storage` -[OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) introduces the ability for scrape targets to add exemplars to certain metrics. Exemplars are references to data outside of the MetricSet. A common use case are IDs of program traces. +[OpenMetrics](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#exemplars) introduces the ability for scrape targets to add exemplars to certain metrics. Exemplars are references to data outside of the MetricSet. A common use case are IDs of program traces. Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The config file block [storage](configuration/configuration.md#configuration-file)/[exemplars](configuration/configuration.md#exemplars) can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `trace_id=` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration). @@ -40,14 +21,15 @@ Exemplar storage is implemented as a fixed size circular buffer that stores exem `--enable-feature=memory-snapshot-on-shutdown` -This takes the snapshot of the chunks that are in memory along with the series information when shutting down and stores -it on disk. This will reduce the startup time since the memory state can be restored with this snapshot and m-mapped -chunks without the need of WAL replay. +This takes a snapshot of the chunks that are in memory along with the series information when shutting down and stores it on disk. This will reduce the startup time since the memory state can now be restored with this snapshot +and m-mapped chunks, while a WAL replay from disk is only needed for the parts of the WAL that are not part of the snapshot. ## Extra scrape metrics `--enable-feature=extra-scrape-metrics` +> **Note:** This feature flag is deprecated. Please use the `extra_scrape_metrics` configuration option instead (available at both global and scrape-config level). The feature flag will be removed in a future major version. See the [configuration documentation](configuration/configuration.md) for more details. + When enabled, for each instance scrape, Prometheus stores a sample in the following additional time series: - `scrape_timeout_seconds`. The configured `scrape_timeout` for a target. This allows you to measure each target to find out how close they are to timing out with `scrape_duration_seconds / scrape_timeout_seconds`. @@ -55,164 +37,97 @@ When enabled, for each instance scrape, Prometheus stores a sample in the follow to find out how close they are to reaching the limit with `scrape_samples_post_metric_relabeling / scrape_sample_limit`. Note that `scrape_sample_limit` can be zero if there is no limit configured, which means that the query above can return `+Inf` for targets with no limit (as we divide by zero). If you want to query only for targets that do have a sample limit use this query: `scrape_samples_post_metric_relabeling / (scrape_sample_limit > 0)`. - `scrape_body_size_bytes`. The uncompressed size of the most recent scrape response, if successful. Scrapes failing because `body_size_limit` is exceeded report `-1`, other scrape failures report `0`. -## New service discovery manager +## Per-step stats -`--enable-feature=new-service-discovery-manager` +`--enable-feature=promql-per-step-stats` -When enabled, Prometheus uses a new service discovery manager that does not -restart unchanged discoveries upon reloading. This makes reloads faster and reduces -pressure on service discoveries' sources. +When enabled, passing `stats=all` in a query request returns per-step +statistics. The following sample statistics are included: -Users are encouraged to test the new service discovery manager and report any -issues upstream. +- **totalQueryableSamples** / **totalQueryableSamplesPerStep**: Total number of samples *loaded* during the query. For range-vector functions (e.g. `rate`, `sum_over_time`) evaluated over multiple steps, each step counts the full window (the same point may be counted in multiple steps). +- **samplesRead** / **samplesReadPerStep**: Total number of samples *read* (I/O). For range-vector functions in range queries, only *new* points per step are counted (step 0: full window; later steps: points not seen in the previous step). For other query types, this equals totalQueryableSamples. +- **peakSamples**: Peak number of samples in memory during evaluation (used for the `query.max-samples` limit). -In future releases, this new service discovery manager will become the default and -this feature flag will be ignored. +The Prometheus server exposes two counters for observability: `prometheus_engine_query_samples_total` (samples loaded, full window per step) and `prometheus_engine_query_samples_read_total` (samples read, delta per step for range-vector). -## Prometheus agent +When disabled in either the engine or the query, per-step statistics are not +computed at all. -`--enable-feature=agent` +## Experimental PromQL functions -When enabled, Prometheus runs in agent mode. The agent mode is limited to -discovery, scrape and remote write. +`--enable-feature=promql-experimental-functions` -This is useful when you do not need to query the Prometheus data locally, but -only from a central [remote endpoint](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). +Enables PromQL functions that are considered experimental. These functions +might change their name, syntax, or semantics. They might also get removed +entirely. -## Per-step stats +## Start (Created) Timestamps Zero Injection -`--enable-feature=promql-per-step-stats` +`--enable-feature=created-timestamp-zero-ingestion` -When enabled, passing `stats=all` in a query request returns per-step -statistics. Currently this is limited to totalQueryableSamples. +> NOTE: CreatedTimestamp feature was renamed to StartTimestamp for consistency. The above flag uses old name for stability. -When disabled in either the engine or the query, per-step statistics are not -computed at all. +Enables ingestion of start timestamp. Start timestamps are injected as 0 valued samples when appropriate. See [PromCon talk](https://youtu.be/nWf0BfQ5EEA) for details. -## Auto GOMAXPROCS - -`--enable-feature=auto-gomaxprocs` - -When enabled, GOMAXPROCS variable is automatically set to match Linux container CPU quota. - -## Auto GOMEMLIMIT - -`--enable-feature=auto-gomemlimit` - -When enabled, the GOMEMLIMIT variable is automatically set to match the Linux container memory limit. If there is no container limit, or the process is running outside of containers, the system memory total is used. - -There is also an additional tuning flag, `--auto-gomemlimit.ratio`, which allows controlling how much of the memory is used for Prometheus. The remainder is reserved for memory outside the process. For example, kernel page cache. Page cache is important for Prometheus TSDB query performance. The default is `0.9`, which means 90% of the memory limit will be used for Prometheus. - -## No default scrape port - -`--enable-feature=no-default-scrape-port` - -When enabled, the default ports for HTTP (`:80`) or HTTPS (`:443`) will _not_ be added to -the address used to scrape a target (the value of the `__address_` label), contrary to the default behavior. -In addition, if a default HTTP or HTTPS port has already been added either in a static configuration or -by a service discovery mechanism and the respective scheme is specified (`http` or `https`), that port will be removed. - -## Native Histograms - -`--enable-feature=native-histograms` - -When enabled, Prometheus will ingest native histograms (formerly also known as -sparse histograms or high-res histograms). Native histograms are still highly -experimental. Expect breaking changes to happen (including those rendering the -TSDB unreadable). - -Native histograms are currently only supported in the traditional Prometheus -protobuf exposition format. This feature flag therefore also enables a new (and -also experimental) protobuf parser, through which _all_ metrics are ingested -(i.e. not only native histograms). Prometheus will try to negotiate the -protobuf format first. The instrumented target needs to support the protobuf -format, too, _and_ it needs to expose native histograms. The protobuf format -allows to expose classic and native histograms side by side. With this feature -flag disabled, Prometheus will continue to parse the classic histogram (albeit -via the text format). With this flag enabled, Prometheus will still ingest -those classic histograms that do not come with a corresponding native -histogram. However, if a native histogram is present, Prometheus will ignore -the corresponding classic histogram, with the notable exception of exemplars, -which are always ingested. To keep the classic histograms as well, enable -`scrape_classic_histograms` in the scrape job. - -_Note about the format of `le` and `quantile` label values:_ - -In certain situations, the protobuf parsing changes the number formatting of -the `le` labels of classic histograms and the `quantile` labels of -summaries. Typically, this happens if the scraped target is instrumented with -[client_golang](https://github.com/prometheus/client_golang) provided that -[promhttp.HandlerOpts.EnableOpenMetrics](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp#HandlerOpts) -is set to `false`. In such a case, integer label values are represented in the -text format as such, e.g. `quantile="1"` or `le="2"`. However, the protobuf parsing -changes the representation to float-like (following the OpenMetrics -specification), so the examples above become `quantile="1.0"` and `le="2.0"` after -ingestion into Prometheus, which changes the identity of the metric compared to -what was ingested before via the text format. - -The effect of this change is that alerts, recording rules and dashboards that -directly reference label values as whole numbers such as `le="1"` will stop -working. - -Aggregation by the `le` and `quantile` labels for vectors that contain the old and -new formatting will lead to unexpected results, and range vectors that span the -transition between the different formatting will contain additional series. -The most common use case for both is the quantile calculation via -`histogram_quantile`, e.g. -`histogram_quantile(0.95, sum by (le) (rate(histogram_bucket[10m])))`. -The `histogram_quantile` function already tries to mitigate the effects to some -extent, but there will be inaccuracies, in particular for shorter ranges that -cover only a few samples. - -Ways to deal with this change either globally or on a per metric basis: - -- Fix references to integer `le`, `quantile` label values, but otherwise do -nothing and accept that some queries that span the transition time will produce -inaccurate or unexpected results. -_This is the recommended solution, to get consistently normalized label values._ -Also Prometheus 3.0 is expected to enforce normalization of these label values. -- Use `metric_relabel_config` to retain the old labels when scraping targets. -This should **only** be applied to metrics that currently produce such labels. - - -```yaml - metric_relabel_configs: - - source_labels: - - quantile - target_label: quantile - regex: (\d+)\.0+ - - source_labels: - - le - - __name__ - target_label: le - regex: (\d+)\.0+;.*_bucket -``` +Currently, Prometheus supports start timestamps on the -## OTLP Receiver +* `PrometheusProto` +* `OpenMetrics1.0.0` -`--enable-feature=otlp-write-receiver` -The OTLP receiver allows Prometheus to accept [OpenTelemetry](https://opentelemetry.io/) metrics writes. -Prometheus is best used as a Pull based system, and staleness, `up` metric, and other Pull enabled features -won't work when you push OTLP metrics. +From the above, Prometheus recommends `PrometheusProto`. This is because OpenMetrics 1.0 Start Timestamp information is shared as a `_created` metric and parsing those +are prone to errors and expensive (thus, adding an overhead). You also need to be careful to not pollute your Prometheus with extra `_created` metrics. -## Experimental PromQL functions +Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes the global `scrape_protocols` default configuration option to +`[ PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4 ]`, resulting in negotiating the Prometheus Protobuf protocol first (unless the `scrape_protocols` option is set to a different value explicitly). -`--enable-feature=promql-experimental-functions` +Besides enabling this feature in Prometheus, start timestamps need to be exposed by the application being scraped. -Enables PromQL functions that are considered experimental. These functions -might change their name, syntax, or semantics. They might also get removed -entirely. +## Start timestamp (ST) native storage -## Created Timestamps Zero Injection +`--enable-feature=st-storage` -`--enable-feature=created-timestamp-zero-ingestion` +Enables the storage of start timestamps (ST) per sample, through WAL, TSDB/Agent and Remote-Write 2.0. This option +allows preserving the exact ST value as it was presented from scrape and receive protocols. In the future this feature +is meant to be a replacement of `created-timestamp-zero-ingestion` which injects synthetic 0 samples. + +Currently, Prometheus supports start timestamps on: + +* `PrometheusProto` +* `OpenMetrics1.0.0` + +`PrometheusProto` is recommended, due to efficiency of ST passing. -Enables ingestion of created timestamp. Created timestamps are injected as 0 valued samples when appropriate. See [PromCon talk](https://youtu.be/nWf0BfQ5EEA) for details. +Besides enabling this feature in Prometheus, start timestamps need to be exposed by the application being scraped. -Currently Prometheus supports created timestamps only on the traditional Prometheus Protobuf protocol (WIP for other protocols). As a result, when enabling this feature, the Prometheus protobuf scrape protocol will be prioritized (See `scrape_config.scrape_protocols` settings for more details). +> NOTE: This is an experimental feature with known limitations until fully implemented. +> * It introduces new WAL record type (SamplesV2) that can only be replayed with Prometheus 3.11 or later versions. +> * For persistent storage support (TSDB blocks), you need to manually opt-in for XOR2 chunk format ([`xor2-encoding` flag](#xor2-chunk-encoding)). +> The float chunk encoding must resolve to XOR2 when `st-storage` is active, because XOR chunks do not store start timestamps. +> If the resolved encoding is XOR (that is, `--enable-feature=xor2-encoding` is not set and `chunk_encoding.floats: xor2` is not configured), Prometheus refuses to start and fails the configuration validation with an error rather than continuing to run. +> Likewise, explicitly setting `chunk_encoding.floats: xor` in the config file while `st-storage` is active is rejected at config reload. +> This might change later once we finish experimentation phase with XOR2. +> * ST for native histograms and NHCBs are not yet implemented (see [#18315](https://github.com/prometheus/prometheus/issues/18315)). +> * PromQL use of ST is out of scope of this feature. -Besides enabling this feature in Prometheus, created timestamps need to be exposed by the application being scraped. +## Start timestamp (ST) usage in PromQL functions + +`--enable-feature=use-start-timestamps` + +Enables the use of start timestamps (ST) in PromQL functions such as `rate()`, `irate()`, and `increase()`. This feature doesn't currently work with extended range selectors (`promql-extended-range-selectors`). + +## Start timestamp (ST) synthesis + +`--enable-feature=st-synthesis` + +Enables the synthesis of start timestamps (ST) for cumulative metrics (Counters, Classic Histograms, Native Histograms) when they are not provided by the source. Similar to [the official OpenTelemetry metricstarttimeprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/metricstarttimeprocessor#strategy-subtract-initial-point), it tracks previous values to detect resets and subtracts the initial reference point to synthesize a zero-based timeline from the first sample. + +> NOTE: This is an experimental feature. +> * The first sample is dropped when this feature is turned on to establish the start timestamp reference point. As a result, if a series has only a single point reported, turning this feature on may result in no points being ingested for that series. +> * Synthesis yields accurate Start Timestamp while maintaining accurate counter rates. However, the raw counter values will be different that what's scraped. This is because the first point is dropped and its timestamp is used as the start timestamp for all subsequent points. All subsequent points are normalized against that dropped point (i.e. subtracted by it). Effectively, synthesis create new counter streams with the known start timestamp from the original data. +> * Synthesis works only with scraped data (RW and Otel receiver are not implemented). +> * Synthesis requires ordered samples. As a result, cumulative samples without ST that are out of order will be rejected despite the `tsdb. out_of_order_time_window` setting. +> * If an append fails for a series (e.g., due to out-of-order samples being rejected), the synthesis state for that series is cleared. As a result, the next sample received after the failure will be treated as the first sample again and will be dropped to establish a new reference point. ## Concurrent evaluation of independent rules @@ -226,6 +141,12 @@ This has the potential to improve rule group evaluation latency and resource uti The number of concurrent rule evaluations can be configured with `--rules.max-concurrent-rule-evals`, which is set to `4` by default. +## Serve old Prometheus UI + +Fall back to serving the old (Prometheus 2.x) web UI instead of the new UI. The new UI that was released as part of Prometheus 3.0 is a complete rewrite and aims to be cleaner, less cluttered, and more modern under the hood. However, it is not fully feature complete and battle-tested yet, so some users may still prefer using the old UI. + +`--enable-feature=old-ui` + ## Metadata WAL Records `--enable-feature=metadata-wal-records` @@ -233,8 +154,7 @@ The number of concurrent rule evaluations can be configured with `--rules.max-co When enabled, Prometheus will store metadata in-memory and keep track of metadata changes as WAL records on a per-series basis. -This must be used if -you are also using remote write 2.0 as it will only gather metadata from the WAL. +This must be used if you would like to send metadata using the new remote write 2.0. ## Delay compaction start time @@ -250,7 +170,7 @@ Note that during this delay, the Head continues its usual operations, which incl Despite the delay in compaction, the blocks produced are time-aligned in the same manner as they would be if the delay was not in place. -## Delay __name__ label removal for PromQL engine +## Delay `__name__` label removal for PromQL engine `--enable-feature=promql-delayed-name-removal` @@ -258,10 +178,264 @@ When enabled, Prometheus will change the way in which the `__name__` label is re This allows optionally preserving the `__name__` label via the `label_replace` and `label_join` functions, and helps prevent the "vector cannot contain metrics with the same labelset" error, which can happen when applying a regex-matcher to the `__name__` label. -## UTF-8 Name Support +Note that evaluating parts of the query separately will still trigger the +labelset collision. This commonly happens when analyzing intermediate results +of a query manually or with a tool like PromLens. + +If a query refers to the already removed `__name__` label, its behavior may +change while this feature flag is set. (Example: `sum by (__name__) +(rate({foo="bar"}[5m]))`, see [details on +GitHub](https://github.com/prometheus/prometheus/issues/11397#issuecomment-1451998792).) +These queries are rare to occur and easy to fix. (In the above example, +removing `by (__name__)` doesn't change anything without the feature flag and +fixes the possible problem with the feature flag.) + +It is possible to craft a query that aggregates by `__name__` and puts samples with and without delayed name removal into the same group. In that case, the name is removed from the affected group. Note that this case hardly occurs in queries that fulfill a practical purpose. + +## OTLP Delta Conversion + +`--enable-feature=otlp-deltatocumulative` + +When enabled, Prometheus will convert OTLP metrics from delta temporality to their +cumulative equivalent, instead of dropping them. This cannot be enabled in conjunction with `otlp-native-delta-ingestion`. + +This uses +[deltatocumulative][d2c] +from the OTel collector, using its default settings. + +Delta conversion keeps in-memory state to aggregate delta changes per-series over time. +When Prometheus restarts, this state is lost, starting the aggregation from zero +again. This results in a counter reset in the cumulative series. + +This state is periodically ([`max_stale`][d2c]) cleared of inactive series. + +Enabling this _can_ have negative impact on performance, because the in-memory +state is mutex guarded. Cumulative-only OTLP requests are not affected. + +[d2c]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor + +## PromQL arithmetic expressions in time durations + +`--enable-feature=promql-duration-expr` + +With this flag, arithmetic expressions can be used in time durations in range queries and offset durations. + +In range queries: +``` +rate(http_requests_total[5m * 2]) # 10 minute range +rate(http_requests_total[(5+2) * 1m]) # 7 minute range +``` + +In offset durations: +``` +http_requests_total offset (1h / 2) # 30 minute offset +http_requests_total offset ((2 ^ 3) * 1m) # 8 minute offset +``` + +When using offset with duration expressions, you must wrap the expression in +parentheses. Without parentheses, only the first duration value will be used in +the offset calculation. + +`step()` can be used in duration expressions. +For a **range query**, it resolves to the step width of the range query. +For an **instant query**, it resolves to `0s`. + +`range()` can be used in duration expressions. +For a **range query**, it resolves to the full range of the query (end time - start time). +For an **instant query**, it resolves to `0s`. +This is particularly useful in combination with `@end()` to look back over the entire query range, e.g., `max_over_time(metric[range()] @ end())`. + +`min_of(, )` and `max_of(, )` select between two duration expressions. +`min_of` returns the smaller of the two, which is useful for capping a duration at a maximum value. +`max_of` returns the larger of the two, which is useful for enforcing a minimum value. +For example, `max_of(step(), 5s)` ensures the duration is never shorter than `5s`, while `min_of(range(), 1h)` caps the duration at `1h`. + +**Note**: Duration expressions are not supported in the @ timestamp operator. + +The following operators are supported: + +* `+` - addition +* `-` - subtraction +* `*` - multiplication +* `/` - division +* `%` - modulo +* `^` - exponentiation + +Examples of equivalent durations: + +* `5m * 2` is equivalent to `10m` or `600s` +* `10m - 1m` is equivalent to `9m` or `540s` +* `(5+2) * 1m` is equivalent to `7m` or `420s` +* `1h / 2` is equivalent to `30m` or `1800s` +* `4h % 3h` is equivalent to `1h` or `3600s` +* `(2 ^ 3) * 1m` is equivalent to `8m` or `480s` +* `step() + 1` is equivalent to the query step width increased by 1s. +* `max_of(step(), 5s)` is equivalent to the larger of the query step width and `5s`. +* `min_of(2 * step() + 5s, 5m)` is equivalent to the smaller of twice the query step increased by `5s` and `5m`. + + +## OTLP Native Delta Support + +`--enable-feature=otlp-native-delta-ingestion` + +When enabled, allows for the native ingestion of delta OTLP metrics, storing the raw sample values without conversion. This cannot be enabled in conjunction with `otlp-deltatocumulative`. + +Currently, the StartTimeUnixNano field is ignored, and deltas are given the unknown metric metadata type. + +Delta support is in a very early stage of development and the ingestion and querying process my change over time. For the open proposal see [prometheus/proposals#48](https://github.com/prometheus/proposals/pull/48). + +### Querying + +We encourage users to experiment with deltas and existing PromQL functions; we will collect feedback and likely build features to improve the experience around querying deltas. + +Note that standard PromQL counter functions like `rate()` and `increase()` are designed for cumulative metrics and will produce incorrect results when used with delta metrics. This may change in the future, but for now, to get similar results for delta metrics, you need `sum_over_time()`: + +* `sum_over_time(delta_metric[])`: Calculates the sum of delta values over the specified time range. +* `sum_over_time(delta_metric[]) / `: Calculates the per-second rate of the delta metric. + +These may not work well if the `` is not a multiple of the collection interval of the metric. For example, if you do `sum_over_time(delta_metric[1m]) / 1m` range query (with a 1m step), but the collection interval of a metric is 10m, the graph will show a single point every 10 minutes with a high rate value, rather than 10 points with a lower, constant value. + +### Current gotchas + +* If delta metrics are exposed via [federation](https://prometheus.io/docs/prometheus/latest/federation/), data can be incorrectly collected if the ingestion interval is not the same as the scrape interval for the federated endpoint. + +* It is difficult to figure out whether a metric has delta or cumulative temporality, since there's no indication of temporality in metric names or labels. For now, if you are ingesting a mix of delta and cumulative metrics we advise you to explicitly add your own labels to distinguish them. In the future, we plan to introduce type labels to consistently distinguish metric types and potentially make PromQL functions type-aware (e.g. providing warnings when cumulative-only functions are used with delta metrics). + +* If there are multiple samples being ingested at the same timestamp, only one of the points is kept - the samples are **not** summed together (this is how Prometheus works in general - duplicate timestamp samples are rejected). Any aggregation will have to be done before sending samples to Prometheus. + +## Type and Unit Labels + +`--enable-feature=type-and-unit-labels` + +When enabled, Prometheus will start injecting additional, reserved `__type__` +and `__unit__` labels as designed in the [PROM-39 proposal](https://github.com/prometheus/proposals/pull/39). + +Those labels are sourced from the metadata structures of the existing scrape and ingestion formats +like OpenMetrics Text, Prometheus Text, Prometheus Proto, Remote Write 2 and OTLP. All the user provided labels with +`__type__` and `__unit__` will be overridden. + +PromQL layer will handle those labels the same way `__name__` is handled, e.g. dropped +on certain operations like `-` or `+` and affected by `promql-delayed-name-removal` feature. + +This feature enables important metadata information to be accessible directly with samples and PromQL layer. + +It's especially useful for users who: + +* Want to be able to select metrics based on type or unit. +* Want to handle cases of series with the same metric name and different type and units. + e.g. native histogram migrations or OpenTelemetry metrics from OTLP endpoint, without translation. + +In future more [work is planned](https://github.com/prometheus/prometheus/issues/16610) that will depend on this e.g. rich PromQL UX that helps +when wrong types are used on wrong functions, automatic renames, delta types and more. + +### Behavior with metadata records + +When this feature is enabled and the metadata WAL records exists, in an unlikely situation when type or unit are different across those, +the Prometheus outputs intends to prefer the `__type__` and `__unit__` labels values. For example on Remote Write 2.0, +if the metadata record somehow (e.g. due to bug) says "counter", but `__type__="gauge"` the remote time series will be set to a gauge. + +## Use Uncached IO + +`--enable-feature=use-uncached-io` + +Experimental and only available on Linux. + +When enabled, it makes chunks writing bypass the page cache. Its primary +goal is to reduce confusion around page‐cache behavior and to prevent over‑allocation of +memory in response to misleading cache growth. + +This is currently implemented using direct I/O. + +For more details, see the [proposal](https://github.com/prometheus/proposals/pull/45). + +## XOR2 chunk encoding + +`--enable-feature=xor2-encoding` + +> WARNING: This is highly experimental and risky setting: +> * Chunks encoded with XOR2 **cannot be read by older Prometheus versions** that do not support the encoding. Once enabled and data is written, you need to **manually delete blocks from the disk**, otherwise Prometheus will return error on all queries. +> * We are still experimenting on the final encoding. As of now this encoding can change in any Prometheus version. All your persistent block data will be lost between versions. +> * This encoding is new, meaning downstream tools and LTS systems might not support it yet (e.g. Thanos sidecar uploaded blocks). + +This setting enables the new XOR2 chunk encoding for float samples, which provides better disk compression than the default XOR encoding for typical Prometheus workloads. This format also allows storing Start Timestamp (ST). + +The encoding can also be controlled at each configuration reload via the `chunk_encoding.floats` field in the `storage.tsdb` section of the configuration file. Setting `chunk_encoding.floats: xor` forces standard XOR encoding even when `--enable-feature=xor2-encoding` is set; setting `chunk_encoding.floats: xor2` requires `--enable-feature=xor2-encoding` to be enabled. + +Without [`st-storage`](#start-timestamp-st-native-storage), XOR and XOR2 are compatible encodings, so an encoding change via `chunk_encoding.floats` does not cut the current chunk; the new encoding takes effect when the current chunk is next cut for any reason (size, time range, or sample count). When `st-storage` is also enabled, XOR and XOR2 are not compatible because XOR chunks do not store start timestamps, so the in-progress chunk is cut on the next append after the encoding changes. + +Note that `--enable-feature=st-storage` does not automatically enable XOR2 encoding. +However, setting `chunk_encoding.floats: xor` while `st-storage` is active is rejected at +config reload, because XOR chunks do not store start timestamps. + +## Extended Range Selectors + +`--enable-feature=promql-extended-range-selectors` + +Enables experimental `anchored` and `smoothed` modifiers for PromQL range and instant selectors. These modifiers provide more control over how range boundaries are handled in functions like `rate` and `increase`, especially with missing or irregular data. + +Native Histograms are not yet supported by the extended range selectors. + +### `anchored` + +Uses the most recent sample (within the lookback delta) at the beginning of the range, or alternatively the first sample within the range if there is no sample within the lookback delta. The last sample within the range is also used at the end of the range. No extrapolation or interpolation is applied, so this is useful to get the direct difference between sample values. + +Anchored range selector work with: `resets`, `changes`, `rate`, `increase`, and `delta`. + +Example query: +`increase(http_requests_total[5m] anchored)` + +**Note**: When using the anchored modifier with the increase function, the results returned are integers. + +### `smoothed` + +In range selectors, linearly interpolates values at the range boundaries, using the sample values before and after the boundaries for an improved estimation that is robust against irregular scrapes and missing samples. However, it requires a sample after the evaluation interval to work properly, see note below. + +For instant selectors, values are linearly interpolated at the evaluation timestamp using the samples immediately before and after that point. + +Smoothed range selectors work with: `rate`, `increase`, and `delta`. + +Example query: +`rate(http_requests_total[step()] smoothed)` + +> **Note for alerting and recording rules:** +> The `smoothed` modifier requires samples after the evaluation interval, so using it directly in alerting or recording rules will typically *under-estimate* the result, as future samples are not available at evaluation time. +> To use `smoothed` safely in rules, you **must** apply a `query_offset` to the rule group (see [documentation](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#rule_group)) to ensure the calculation window is fully in the past and all needed samples are available. +> For critical alerting, set the offset to at least one scrape interval; for less critical or more resilient use cases, consider a larger offset (multiple scrape intervals) to tolerate missed scrapes. + +For more details, see the [design doc](https://github.com/prometheus/proposals/blob/main/proposals/2025-04-04_extended-range-selectors-semantics.md). + +**Note**: Extended Range Selectors are not supported for subqueries. + +## Binary operator fill modifiers + +`--enable-feature=promql-binop-fill-modifiers` + +Enables experimental `fill()`, `fill_left()`, and `fill_right()` modifiers for PromQL binary operators. These modifiers allow filling in missing matches on either side of a binary operation with a provided default sample value. + +Example query: + +``` + rate(successful_requests[5m]) ++ fill(0) + rate(failed_requests[5m]) +``` + +See [the fill modifiers documentation](querying/operators.md#filling-in-missing-matches) for more details and examples. + + +## Search API + +`--enable-feature=search-api` -`--enable-feature=utf8-names` +Enables the experimental search API endpoints for discovering metric names, +label names, and label values with fuzzy matching and filtering support. See +[the search API documentation](querying/api.md#searching-metric-names-label-names-and-label-values) +for details. -When enabled, changes the metric and label name validation scheme inside Prometheus to allow the full UTF-8 character set. -By itself, this flag does not enable the request of UTF-8 names via content negotiation. -Users will also have to set `metric_name_validation_scheme` in scrape configs to enable the feature either on the global config or on a per-scrape config basis. +The `--web.search.max-limit` flag (default `10000`) bounds the `limit` query +parameter accepted by the search endpoints. Requests with a higher `limit` are +rejected with HTTP 400. The default response limit (100) is silently clamped +to this maximum, so an operator setting a smaller cap does not break +no-`limit` requests. Setting the flag to `0` disables the cap entirely; this +is **not recommended** for endpoints exposed beyond a trusted network because a +single client can then request the entire index in one response. diff --git a/docs/federation.md b/docs/federation.md index 3344a0ed059..cb0fad7f696 100644 --- a/docs/federation.md +++ b/docs/federation.md @@ -3,20 +3,19 @@ title: Federation sort_rank: 6 --- -# Federation - Federation allows a Prometheus server to scrape selected time series from another Prometheus server. -_Note about native histograms (experimental feature): To scrape native histograms -via federation, the scraping Prometheus server needs to run with native histograms -enabled (via the command line flag `--enable-feature=native-histograms`), implying -that the protobuf format is used for scraping. Should the federated metrics contain -a mix of different sample types (float64, counter histogram, gauge histogram) for -the same metric name, the federation payload will contain multiple metric families -with the same name (but different types). Technically, this violates the rules of -the protobuf exposition format, but Prometheus is nevertheless able to ingest all -metrics correctly._ +_Note about native histograms: To scrape native histograms via federation, the +scraping Prometheus server needs to set `scrape_native_histograms: true` in its +scrape config, implying that the protobuf format is used for scraping. Should +the federated metrics contain a mix of different sample types (floats of any +flavor, counter histograms, or gauge histograms) for the same metric name, the +federation payload will contain multiple metric families with the same name +(but different types – for technical reasons, float samples are always federated +as `untyped`, while histogram samples are federated with their full type +information). Technically, this violates the rules of the protobuf exposition +format, but Prometheus is nevertheless able to ingest all metrics correctly._ ## Use cases diff --git a/docs/getting_started.md b/docs/getting_started.md index e89ac705eeb..35f1a88a7dc 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -3,8 +3,6 @@ title: Getting started sort_rank: 1 --- -# Getting started - This guide is a "Hello World"-style tutorial which shows how to install, configure, and use a simple Prometheus instance. You will download and run Prometheus locally, configure it to scrape itself and an example application, @@ -81,7 +79,7 @@ navigating to its metrics endpoint: Let us explore data that Prometheus has collected about itself. To use Prometheus's built-in expression browser, navigate to -http://localhost:9090/graph and choose the "Table" view within the "Graph" tab. +http://localhost:9090/query and choose the "Graph" tab. As you can gather from [localhost:9090/metrics](http://localhost:9090/metrics), one metric that Prometheus exports about itself is named @@ -115,7 +113,7 @@ For more about the expression language, see the ## Using the graphing interface -To graph expressions, navigate to http://localhost:9090/graph and use the "Graph" +To graph expressions, navigate to http://localhost:9090/query and use the "Graph" tab. For example, enter the following expression to graph the per-second rate of chunks @@ -200,7 +198,7 @@ To record the time series resulting from this expression into a new metric called `job_instance_mode:node_cpu_seconds:avg_rate5m`, create a file with the following recording rule and save it as `prometheus.rules.yml`: -``` +```yaml groups: - name: cpu-node rules: @@ -262,6 +260,9 @@ process ID. ## Shutting down your instance gracefully. While Prometheus does have recovery mechanisms in the case that there is an -abrupt process failure it is recommend to use the `SIGTERM` signal to cleanly -shutdown a Prometheus instance. If you're running on Linux this can be performed -by using `kill -s SIGTERM `, replacing `` with your Prometheus process ID. +abrupt process failure it is recommended to use signals or interrupts for a +clean shutdown of a Prometheus instance. On Linux, this can be done by sending +the `SIGTERM` or `SIGINT` signals to the Prometheus process. For example, you +can use `kill -s `, replacing `` with the signal name +and `` with the Prometheus process ID. Alternatively, you can press the +interrupt character at the controlling terminal, which by default is `^C` (Control-C). diff --git a/docs/http_sd.md b/docs/http_sd.md index 884deb9f3c3..88b03039181 100644 --- a/docs/http_sd.md +++ b/docs/http_sd.md @@ -1,14 +1,13 @@ --- -title: HTTP SD +title: Writing HTTP service discovery +nav_title: HTTP SD sort_rank: 7 --- -# Writing HTTP Service Discovery - Prometheus provides a generic [HTTP Service Discovery](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config), that enables it to discover targets over an HTTP endpoint. -The HTTP Service Discovery is complimentary to the supported service +The HTTP Service Discovery is complementary to the supported service discovery mechanisms, and is an alternative to [File-based Service Discovery](https://prometheus.io/docs/guides/file-sd/#use-file-based-service-discovery-to-discover-scrape-targets). ## Comparison between File-Based SD and HTTP SD @@ -40,8 +39,10 @@ an empty list `[]`. Target lists are unordered. Prometheus caches target lists. If an error occurs while fetching an updated targets list, Prometheus keeps using the current targets list. The targets list -is not saved across restart. The `prometheus_sd_http_failures_total` counter -metric tracks the number of refresh failures. +is not saved across restart. The `prometheus_sd_refresh_failures_total` counter +metric tracks the number of refresh failures and the `prometheus_sd_refresh_duration_seconds` +bucket can be used to track HTTP SD refresh attempts or performance. These metrics are +removed when the underlying scrape job disappears on Prometheus configuration reload. The whole list of targets must be returned on every scrape. There is no support for incremental updates. A Prometheus instance does not send its hostname and it @@ -95,3 +96,7 @@ Examples: } ] ``` + +## HTTP SD integrations + +A list of existing HTTP SD integrations can be found on the [Integrations page](https://prometheus.io/docs/operating/integrations/#http-service-discovery) in the Prometheus documentation. diff --git a/docs/images/prometheus_agent.png b/docs/images/prometheus_agent.png new file mode 100644 index 00000000000..01a9f00f390 Binary files /dev/null and b/docs/images/prometheus_agent.png differ diff --git a/docs/index.md b/docs/index.md index d9d4d2b1527..fff28fa54a0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,6 +13,7 @@ The documentation is available alongside all the project documentation at - [Getting started](getting_started.md) - [Installation](installation.md) +- [Agent Mode](prometheus_agent.md) - [Configuration](configuration/configuration.md) - [Querying](querying/basics.md) - [Storage](storage.md) diff --git a/docs/installation.md b/docs/installation.md index c8e359e7809..fbab67a59d7 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -3,8 +3,6 @@ title: Installation sort_rank: 2 --- -# Installation - ## Using pre-compiled binaries We provide precompiled binaries for most official Prometheus components. Check diff --git a/docs/management_api.md b/docs/management_api.md index 0b496cbaa7a..ecc46fad880 100644 --- a/docs/management_api.md +++ b/docs/management_api.md @@ -3,12 +3,10 @@ title: Management API sort_rank: 8 --- -# Management API - Prometheus provides a set of management APIs to facilitate automation and integration. -### Health check +## Health check ``` GET /-/healthy @@ -18,7 +16,7 @@ HEAD /-/healthy This endpoint always returns 200 and should be used to check Prometheus health. -### Readiness check +## Readiness check ``` GET /-/ready @@ -28,7 +26,7 @@ HEAD /-/ready This endpoint returns 200 when Prometheus is ready to serve traffic (i.e. respond to queries). -### Reload +## Reload ``` PUT /-/reload @@ -40,7 +38,7 @@ This endpoint triggers a reload of the Prometheus configuration and rule files. Alternatively, a configuration reload can be triggered by sending a `SIGHUP` to the Prometheus process. -### Quit +## Quit ``` PUT /-/quit diff --git a/docs/migration.md b/docs/migration.md index cb88bbfd6f7..ec27d06a0cf 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -1,200 +1,257 @@ --- -title: Migration +title: Prometheus 3.0 migration guide +nav_title: Migration sort_rank: 10 --- -# Prometheus 2.0 migration guide - -In line with our [stability promise](https://prometheus.io/blog/2016/07/18/prometheus-1-0-released/#fine-print), -the Prometheus 2.0 release contains a number of backwards incompatible changes. -This document offers guidance on migrating from Prometheus 1.8 to Prometheus 2.0 and newer versions. +In line with our [stability promise](https://prometheus.io/docs/prometheus/latest/stability/), +the Prometheus 3.0 release contains a number of backwards incompatible changes. +This document offers guidance on migrating from Prometheus 2.x to Prometheus 3.0 and newer versions. ## Flags -The format of Prometheus command line flags has changed. Instead of a -single dash, all flags now use a double dash. Common flags (`--config.file`, -`--web.listen-address` and `--web.external-url`) remain but -almost all storage-related flags have been removed. - -Some notable flags which have been removed: - -- `-alertmanager.url` In Prometheus 2.0, the command line flags for configuring - a static Alertmanager URL have been removed. Alertmanager must now be - discovered via service discovery, see [Alertmanager service discovery](#alertmanager-service-discovery). +- The following feature flags have been removed and they have been added to the + default behavior of Prometheus v3: + - `promql-at-modifier` + - `promql-negative-offset` + - `new-service-discovery-manager` + - `expand-external-labels` + - Environment variable references `${var}` or `$var` in external label values + are replaced according to the values of the current environment variables. + - References to undefined variables are replaced by the empty string. + The `$` character can be escaped by using `$$`. + - `no-default-scrape-port` + - Prometheus v3 will no longer add ports to scrape targets according to the + specified scheme. Target will now appear in labels as configured. + - If you rely on scrape targets like + `https://example.com/metrics` or `http://example.com/metrics` to be + represented as `https://example.com/metrics:443` and + `http://example.com/metrics:80` respectively, add them to your target URLs + - `agent` + - Instead use the dedicated `--agent` CLI flag. + - `remote-write-receiver` + - Instead use the dedicated `--web.enable-remote-write-receiver` CLI flag to enable the remote write receiver. + - `auto-gomemlimit` + - Prometheus v3 will automatically set `GOMEMLIMIT` to match the Linux + container memory limit. If there is no container limit, or the process is + running outside of containers, the system memory total is used. To disable + this, `--no-auto-gomemlimit` is available. + - `auto-gomaxprocs` + - Prometheus v3 will automatically set `GOMAXPROCS` to match the Linux + container CPU quota. To disable this, `--no-auto-gomaxprocs` is available. + + Prometheus v3 will log a warning if you continue to pass these to + `--enable-feature`. + +- Starting from v3.9, the feature flag `native-histograms` is a no-op. Native + histograms are a stable feature now, but scraping them has to be enabled via + the `scrape_native_histograms` global or per-scrape configuration option + (added in v3.8). + +## Configuration + +- The scrape job level configuration option `scrape_classic_histograms` has + been renamed to `always_scrape_classic_histograms`. If you use the + `scrape_native_histograms` scrape configuration option to ingest native + histograms and you also want to ingest classic histograms that an endpoint + might expose along with native histograms, be sure to add this configuration + or change your configuration from the old name. +- The `http_config.enable_http2` in `remote_write` items default has been + changed to `false`. In Prometheus v2 the remote write http client would + default to use http2. In order to parallelize multiple remote write queues + across multiple sockets its preferable to not default to http2. + If you prefer to use http2 for remote write you must now set + `http_config.enable_http2: true` in your `remote_write` configuration section. -- `-log.format` In Prometheus 2.0 logs can only be streamed to standard error. - -- `-query.staleness-delta` has been renamed to `--query.lookback-delta`; Prometheus - 2.0 introduces a new mechanism for handling staleness, see [staleness](querying/basics.md#staleness). +## PromQL -- `-storage.local.*` Prometheus 2.0 introduces a new storage engine; as such all - flags relating to the old engine have been removed. For information on the - new engine, see [Storage](#storage). +### Regular expressions match newlines + +The `.` pattern in regular expressions in PromQL matches newline characters. +With this change a regular expressions like `.*` matches strings that include +`\n`. This applies to matchers in queries and relabel configs. + +For example, the following regular expressions now match the accompanying +strings, whereas in Prometheus v2 these combinations didn't match. + - `.*` additionally matches `foo\n` and `Foo\nBar` + - `foo.?bar` additionally matches `foo\nbar` + - `foo.+bar` additionally matches `foo\nbar` + +If you want Prometheus v3 to behave like v2, you will have to change your +regular expressions by replacing all `.` patterns with `[^\n]`, e.g. +`foo[^\n]*`. + +### Range selectors and lookback exclude samples coinciding with the left boundary + +Lookback and range selectors are now left-open and right-closed (previously +left-closed and right-closed), which makes their behavior more consistent. This +change affects queries where the left boundary of a range or the lookback delta +coincides with the timestamp of one or more samples. + +For example, assume we are querying a timeseries with evenly spaced samples +exactly 1 minute apart. Before Prometheus v3, a range query with `5m` would +usually return 5 samples. But if the query evaluation aligns perfectly with a +scrape, it would return 6 samples. In Prometheus v3 queries like this will +always return 5 samples given even spacing. + +This change will typically affect subqueries because their evaluation timing is +naturally perfectly evenly spaced and aligned with timestamps that are multiples +of the subquery resolution. Furthermore, query frontends often align subqueries +to multiples of the step size. In combination, this easily creates a situation +of perfect mutual alignment, often unintended and unknown by the user, so that +the new behavior might come as a surprise. Before Prometheus V3, a subquery of +`foo[1m:1m]` on such a system might have always returned two points, allowing +for rate calculations. In Prometheus V3, however, such a subquery will only +return one point, which is insufficient for a rate or increase calculation, +resulting in No Data returned. + +Such queries will need to be rewritten to extend the window to properly cover +more than one point. In this example, `foo[2m:1m]` would always return two +points no matter the query alignment. The exact form of the rewritten query may +depend on the intended results and there is no universal drop-in replacement for +queries whose behavior has changed. + +Tests are similarly more likely to affected. To fix those either adjust the +expected number of samples or extend the range. + +### holt_winters function renamed + +The `holt_winters` function has been renamed to `double_exponential_smoothing` +and is now guarded by the `promql-experimental-functions` feature flag. +If you want to keep using `holt_winters`, you have to do both of these things: + - Rename `holt_winters` to `double_exponential_smoothing` in your queries. + - Pass `--enable-feature=promql-experimental-functions` in your Prometheus + CLI invocation. + +## Scrape protocols +Prometheus v3 is more strict concerning the Content-Type header received when +scraping. Prometheus v2 would default to the standard Prometheus text protocol +if the target being scraped did not specify a Content-Type header or if the +header was unparsable or unrecognised. This could lead to incorrect data being +parsed in the scrape. Prometheus v3 will now fail the scrape in such cases. + +If a scrape target is not providing the correct Content-Type header the +fallback protocol can be specified using the `fallback_scrape_protocol` +parameter. See [Prometheus scrape_config documentation.](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) + +This is a **breaking change** as scrapes that may have succeeded with Prometheus v2 +may now fail if this fallback protocol is not specified. Make sure your scrape +endpoints respond with one of the supported Content-type headers: + +- `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited` +- `text/plain;version=0.0.4` +- `text/plain;version=1.0.0` +- `application/openmetrics-text;version=0.0.1` +- `application/openmetrics-text;version=1.0.0` -- `-storage.remote.*` Prometheus 2.0 has removed the deprecated remote - storage flags, and will fail to start if they are supplied. To write to - InfluxDB, Graphite, or OpenTSDB use the relevant storage adapter. +## Miscellaneous -## Alertmanager service discovery +### TSDB format and downgrade -Alertmanager service discovery was introduced in Prometheus 1.4, allowing Prometheus -to dynamically discover Alertmanager replicas using the same mechanism as scrape -targets. In Prometheus 2.0, the command line flags for static Alertmanager config -have been removed, so the following command line flag: +The TSDB format has been changed slightly in Prometheus v2.55 in preparation for changes +to the index format. Consequently, a Prometheus v3 TSDB can only be read by a +Prometheus v2.55 or newer. Keep that in mind when upgrading to v3 -- you will be only +able to downgrade to v2.55, not lower, without losing your TSDB persistent data. -``` -./prometheus -alertmanager.url=http://alertmanager:9093/ -``` +As an extra safety measure, you could optionally consider upgrading to v2.55 first and +confirm Prometheus works as expected, before upgrading to v3. -Would be replaced with the following in the `prometheus.yml` config file: +### TSDB storage contract -```yaml -alerting: - alertmanagers: - - static_configs: - - targets: - - alertmanager:9093 -``` +TSDB compatible storage is now expected to return results matching the specified +selectors. This might impact some third party implementations, most likely +implementing `remote_read`. -You can also use all the usual Prometheus service discovery integrations and -relabeling in your Alertmanager configuration. This snippet instructs -Prometheus to search for Kubernetes pods, in the `default` namespace, with the -label `name: alertmanager` and with a non-empty port. +This contract is not explicitly enforced, but can cause undefined behavior. -```yaml -alerting: - alertmanagers: - - kubernetes_sd_configs: - - role: pod - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - relabel_configs: - - source_labels: [__meta_kubernetes_pod_label_name] - regex: alertmanager - action: keep - - source_labels: [__meta_kubernetes_namespace] - regex: default - action: keep - - source_labels: [__meta_kubernetes_pod_container_port_number] - regex: - action: drop -``` +### UTF-8 names -## Recording rules and alerts +Prometheus v3 supports UTF-8 in metric and label names. This means metric and +label names can change after upgrading according to what is exposed by +endpoints. Furthermore, metric and label names that would have previously been +flagged as invalid no longer will be. -The format for configuring alerting and recording rules has been changed to YAML. -An example of a recording rule and alert in the old format: +Users wishing to preserve the original validation behavior can update their +Prometheus yaml configuration to specify the legacy validation scheme: ``` -job:request_duration_seconds:histogram_quantile99 = - histogram_quantile(0.99, sum by (le, job) (rate(request_duration_seconds_bucket[1m]))) - -ALERT FrontendRequestLatency - IF job:request_duration_seconds:histogram_quantile99{job="frontend"} > 0.1 - FOR 5m - ANNOTATIONS { - summary = "High frontend request latency", - } +global: + metric_name_validation_scheme: legacy ``` -Would look like this: +Or on a per-scrape basis: -```yaml -groups: -- name: example.rules - rules: - - record: job:request_duration_seconds:histogram_quantile99 - expr: histogram_quantile(0.99, sum by (le, job) (rate(request_duration_seconds_bucket[1m]))) - - alert: FrontendRequestLatency - expr: job:request_duration_seconds:histogram_quantile99{job="frontend"} > 0.1 - for: 5m - annotations: - summary: High frontend request latency ``` - -To help with the change, the `promtool` tool has a mode to automate the rules conversion. Given a `.rules` file, it will output a `.rules.yml` file in the -new format. For example: - -``` -$ promtool update rules example.rules +scrape_configs: + - job_name: job1 + metric_name_validation_scheme: utf8 + - job_name: job2 + metric_name_validation_scheme: legacy ``` -You will need to use `promtool` from [Prometheus 2.5](https://github.com/prometheus/prometheus/releases/tag/v2.5.0) as later versions no longer contain the above subcommand. - -## Storage - -The data format in Prometheus 2.0 has completely changed and is not backwards -compatible with 1.8 and older versions. To retain access to your historic monitoring data we -recommend you run a non-scraping Prometheus instance running at least version -1.8.1 in parallel with your Prometheus 2.0 instance, and have the new server -read existing data from the old one via the remote read protocol. - -Your Prometheus 1.8 instance should be started with the following flags and an -config file containing only the `external_labels` setting (if any): +### Log message format +Prometheus v3 has adopted `log/slog` over the previous `go-kit/log`. This +results in a change of log message format. An example of the old log format is: ``` -$ ./prometheus-1.8.1.linux-amd64/prometheus -web.listen-address ":9094" -config.file old.yml +ts=2024-10-23T22:01:06.074Z caller=main.go:627 level=info msg="No time or size retention was set so using the default time retention" duration=15d +ts=2024-10-23T22:01:06.074Z caller=main.go:671 level=info msg="Starting Prometheus Server" mode=server version="(version=, branch=, revision=91d80252c3e528728b0f88d254dd720f6be07cb8-modified)" +ts=2024-10-23T22:01:06.074Z caller=main.go:676 level=info build_context="(go=go1.23.0, platform=linux/amd64, user=, date=, tags=unknown)" +ts=2024-10-23T22:01:06.074Z caller=main.go:677 level=info host_details="(Linux 5.15.0-124-generic #134-Ubuntu SMP Fri Sep 27 20:20:17 UTC 2024 x86_64 gigafips (none))" ``` -Prometheus 2.0 can then be started (on the same machine) with the following flags: +a similar sequence in the new log format looks like this: ``` -$ ./prometheus-2.0.0.linux-amd64/prometheus --config.file prometheus.yml +time=2024-10-24T00:03:07.542+02:00 level=INFO source=/home/user/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:640 msg="No time or size retention was set so using the default time retention" duration=15d +time=2024-10-24T00:03:07.542+02:00 level=INFO source=/home/user/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:681 msg="Starting Prometheus Server" mode=server version="(version=, branch=, revision=7c7116fea8343795cae6da42960cacd0207a2af8)" +time=2024-10-24T00:03:07.542+02:00 level=INFO source=/home/user/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:686 msg="operational information" build_context="(go=go1.23.0, platform=linux/amd64, user=, date=, tags=unknown)" host_details="(Linux 5.15.0-124-generic #134-Ubuntu SMP Fri Sep 27 20:20:17 UTC 2024 x86_64 gigafips (none))" fd_limits="(soft=1048576, hard=1048576)" vm_limits="(soft=unlimited, hard=unlimited)" ``` -Where `prometheus.yml` contains in addition to your full existing configuration, the stanza: +### `le` and `quantile` label values +In Prometheus v3, the values of the `le` label of classic histograms and the +`quantile` label of summaries are normalized upon ingestion. In Prometheus v2 +the value of these labels depended on the scrape protocol (protobuf vs text +format) in some situations. This led to label values changing based on the +scrape protocol. E.g. a metric exposed as `my_classic_hist{le="1"}` would be +ingested as `my_classic_hist{le="1"}` via the text format, but as +`my_classic_hist{le="1.0"}` via protobuf. This changed the identity of the +metric and caused problems when querying the metric. +In Prometheus v3 these label values will always be normalized to a float like +representation. I.e. the above example will always result in +`my_classic_hist{le="1.0"}` being ingested into prometheus, no matter via which +protocol. The effect of this change is that alerts, recording rules and +dashboards that directly reference label values as whole numbers such as +`le="1"` will stop working. + +Ways to deal with this change either globally or on a per metric basis: + +- Fix references to integer `le`, `quantile` label values, but otherwise do +nothing and accept that some queries that span the transition time will produce +inaccurate or unexpected results. +_This is the recommended solution._ +- Use `metric_relabel_config` to retain the old labels when scraping targets. +This should **only** be applied to metrics that currently produce such labels. ```yaml -remote_read: - - url: "http://localhost:9094/api/v1/read" + metric_relabel_configs: + - source_labels: + - quantile + target_label: quantile + regex: (\d+)\.0+ + - source_labels: + - le + - __name__ + target_label: le + regex: (\d+)\.0+;.*_bucket ``` -## PromQL - -The following features have been removed from PromQL: - -- `drop_common_labels` function - the `without` aggregation modifier should be used - instead. -- `keep_common` aggregation modifier - the `by` modifier should be used instead. -- `count_scalar` function - use cases are better handled by `absent()` or correct - propagation of labels in operations. - -See [issue #3060](https://github.com/prometheus/prometheus/issues/3060) for more -details. - -## Miscellaneous - -### Prometheus non-root user - -The Prometheus Docker image is now built to [run Prometheus -as a non-root user](https://github.com/prometheus/prometheus/pull/2859). If you -want the Prometheus UI/API to listen on a low port number (say, port 80), you'll -need to override it. For Kubernetes, you would use the following YAML: - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: security-context-demo-2 -spec: - securityContext: - runAsUser: 0 -... -``` - -See [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) -for more details. - -If you're using Docker, then the following snippet would be used: - -``` -docker run -p 9090:9090 prom/prometheus:latest -``` +### Disallow configuring Alertmanager with the v1 API +Prometheus 3 no longer supports Alertmanager's v1 API. Effectively Prometheus 3 +requires [Alertmanager 0.16.0](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0) or later. Users with older Alertmanager +versions or configurations that use `alerting: alertmanagers: [api_version: v1]` +need to upgrade Alertmanager and change their configuration to use `api_version: v2`. -### Prometheus lifecycle +## Prometheus 2.0 migration guide -If you use the Prometheus `/-/reload` HTTP endpoint to [automatically reload your -Prometheus config when it changes](configuration/configuration.md), -these endpoints are disabled by default for security reasons in Prometheus 2.0. -To enable them, set the `--web.enable-lifecycle` flag. +For the migration guide from Prometheus 1.8 to 2.0 please refer to the [Prometheus v2.55 documentation](https://prometheus.io/docs/prometheus/2.55/migration/). diff --git a/docs/prometheus_agent.md b/docs/prometheus_agent.md new file mode 100644 index 00000000000..0d8c3fa94ae --- /dev/null +++ b/docs/prometheus_agent.md @@ -0,0 +1,54 @@ +--- +title: Prometheus Agent Mode +nav_title: Agent Mode +sort_rank: 4 +--- + +## Prometheus Agent Mode + +The Prometheus Agent is an operational mode built into the Prometheus binary with the same scraping APIs, semantics, configuration, and discovery mechanism; this agent mode disables some of Prometheus' usual features(TSDB, alerting, and rule evaluations) and optimizes the binary for scraping and remote writing to remote locations. + +The Prometheus Remote Write protocol forwards(streams) all or a subset of metrics collected by Prometheus to a remote location; you can configure Prometheus to forward some metrics (if you want, with all metadata and exemplars!) to one or more locations that support the Remote Write API. + +With Remote Write, Prometheus still uses a pull model to gather metrics from applications, which gives us an understanding of those different failure modes. After that, we batch samples and series and export, replicate (push) data to the Remote Write endpoints, limiting the number of monitoring unknowns that the central point has. +Streaming data from such a scraper enables Global View use cases by allowing you to store metrics data in a centralized location. This enables the separation of concerns, which is useful when different teams manage applications than the observability or monitoring pipelines. + +The Agent mode optimizes Prometheus for the remote write use case. It disables querying, alerting, and local storage and replaces it with a customized TSDB WAL. Everything else stays the same: scraping logic, service discovery, and related configuration. It can be used as a drop-in replacement for Prometheus if you want to just forward your data to a remote Prometheus server or any other Remote-Write-compliant project. + +In essence, it looks like this: +![Prometheus Agent Remote Write](./images/prometheus_agent.png) + +### Benefits of agent mode + +- Improved efficiency. The customized Agent TSDB WAL removes the data immediately after successful writes. If it cannot reach the remote endpoint, it persists the data temporarily on the disk until the remote endpoint is back online. This is currently limited to a two-hour buffer only, similar to non-agent Prometheus. This means that there is no need to build chunks of data in memory or maintain a full index for querying purposes. Essentially the Agent mode uses a fraction of the resources that a normal Prometheus server would use in a similar situation. +- Agent mode enables easier [horizontal scalability for ingestion](https://prometheus.io/blog/2021/11/16/agent/#the-dream-auto-scalable-metric-ingestion). + +### Downsides of agent mode + +- No local queries. You can not query the local Prometheus instance. +- Recording rules are not possible. You can not pre-summarize data for sending to remote write. Rules must be done remotely. +- No alerting. All alerting must be done by the remote system. + +### How to Use Agent Mode in Detail + +If you show the help output of Prometheus (--help flag), you should see more or less the following: + +``` +usage: prometheus [] + +The Prometheus monitoring server + +Flags: + -h, --help Show context-sensitive help (also try --help-long and --help-man). + (... other flags) + --storage.tsdb.path="data/" + Base path for metrics storage. Use with server mode only. + --storage.agent.path="data-agent/" + Base path for metrics storage. Use with agent mode only. + (... other flags) + --[no-]agent Run Prometheus in 'Agent mode'. +``` + +Use the `--agent` flag to run Prometheus in the Agent mode. The rest of the flags are either for both server and Agent or only for a specific mode. You can see which flag is for which mode by checking the last sentence of a flag's help string. "Use with server mode only" means it's only for server mode. If you don't see any mention like this, it means the flag is shared. + +The Agent mode accepts the same scrape configuration with the same discovery options and remote write options. It also exposes a web UI on port 9095 with disabled query capabilities but shows build info, configuration, targets, and service discovery information as in a normal Prometheus server. diff --git a/docs/querying/api.md b/docs/querying/api.md index efa244fbc8b..e3bb25736b5 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -3,11 +3,25 @@ title: HTTP API sort_rank: 7 --- -# HTTP API - The current stable HTTP API is reachable under `/api/v1` on a Prometheus server. Any non-breaking additions will be added under that endpoint. +## OpenAPI Specification + +An OpenAPI specification for the HTTP API is available at `/api/v1/openapi.yaml`. +By default, it returns OpenAPI 3.1 for broader compatibility. Use `?openapi_version=3.2` +for OpenAPI 3.2, which includes advanced features and endpoints like `/api/v1/notifications/live`. + +This machine-readable specification describes all available endpoints, request parameters, +response formats, and schemas. + +The OpenAPI specification can be used to: + +- Generate client libraries in various programming languages. +- Validate API requests and responses. +- Generate interactive API documentation. +- Test API endpoints. + ## Format overview The API response format is JSON. Every successful API request returns a `2xx` @@ -32,7 +46,7 @@ will be returned in the data field. The JSON response envelope format is as follows: -``` +```json { "status": "success" | "error", "data": , @@ -45,7 +59,7 @@ The JSON response envelope format is as follows: // Only set if there were warnings while executing the request. // There will still be data in the data field. "warnings": [""], - // Only set if there were info-level annnotations while executing the request. + // Only set if there were info-level annotations while executing the request. "infos": [""] } ``` @@ -59,7 +73,7 @@ timestamps are always represented as Unix timestamps in seconds. * ``: Prometheus [time series selectors](basics.md#time-series-selectors) like `http_requests_total` or `http_requests_total{method=~"(GET|POST)"}` and need to be URL-encoded. -* ``: [Prometheus duration strings](basics.md#time-durations). +* ``: [the subset of Prometheus float literals using time units](basics.md#float-literals-and-time-durations). For example, `5m` refers to a duration of 5 minutes. * ``: boolean values (strings `true` and `false`). @@ -86,6 +100,9 @@ URL query parameters: - `time=`: Evaluation timestamp. Optional. - `timeout=`: Evaluation timeout. Optional. Defaults to and is capped by the value of the `-query.timeout` flag. +- `limit=`: Maximum number of returned series. Doesn't affect scalars or strings but truncates the number of series for matrices and vectors. Optional. 0 means disabled. +- `lookback_delta=`: Override the [lookback period](#staleness) just for this query in `duration` format or float number of seconds. Optional. +- `stats=`: Include query statistics in the response. If set to `all`, includes detailed statistics (timings and sample counts). Optional. See [Query statistics](#query-statistics). The current server time is used if the `time` parameter is omitted. @@ -95,7 +112,7 @@ query that may breach server-side URL character limits. The `data` section of the query result has the following format: -``` +```json { "resultType": "matrix" | "vector" | "scalar" | "string", "result": @@ -109,8 +126,11 @@ formats](#expression-query-result-formats). The following example evaluates the expression `up` at the time `2015-07-01T20:10:51.781Z`: +```bash +curl 'http://localhost:9090/api/v1/query?query=up&time=2015-07-01T20:10:51.781Z' +``` + ```json -$ curl 'http://localhost:9090/api/v1/query?query=up&time=2015-07-01T20:10:51.781Z' { "status" : "success", "data" : { @@ -154,6 +174,9 @@ URL query parameters: - `step=`: Query resolution step width in `duration` format or float number of seconds. - `timeout=`: Evaluation timeout. Optional. Defaults to and is capped by the value of the `-query.timeout` flag. +- `limit=`: Maximum number of returned series. Optional. 0 means disabled. +- `lookback_delta=`: Override the [lookback period](#staleness) just for this query in `duration` format or float number of seconds. Optional. +- `stats=`: Include query statistics in the response. If set to `all`, includes detailed statistics (timings and sample counts). Optional. See [Query statistics](#query-statistics). You can URL-encode these parameters directly in the request body by using the `POST` method and `Content-Type: application/x-www-form-urlencoded` header. This is useful when specifying a large @@ -161,7 +184,7 @@ query that may breach server-side URL character limits. The `data` section of the query result has the following format: -``` +```json { "resultType": "matrix", "result": @@ -174,8 +197,11 @@ format](#range-vectors). The following example evaluates the expression `up` over a 30-second range with a query resolution of 15 seconds. +```bash +curl 'http://localhost:9090/api/v1/query_range?query=up&start=2015-07-01T20:10:30.781Z&end=2015-07-01T20:11:00.781Z&step=15s' +``` + ```json -$ curl 'http://localhost:9090/api/v1/query_range?query=up&start=2015-07-01T20:10:30.781Z&end=2015-07-01T20:11:00.781Z&step=15s' { "status" : "success", "data" : { @@ -210,6 +236,20 @@ $ curl 'http://localhost:9090/api/v1/query_range?query=up&start=2015-07-01T20:10 } ``` +### Query statistics + +When the `stats` parameter is set (e.g. `stats=all`), the response `data` includes a `stats` object with the following structure: + +- **timings**: Durations (in seconds) for different phases of query execution (e.g. `evalTotalTime`, `execQueueTime`). +- **samples**: + - **totalQueryableSamples**: Total number of samples *loaded* during the query. For range-vector functions over multiple steps, each step counts the full window. + - **totalQueryableSamplesPerStep**: (Only with `stats=all` and when per-step stats are enabled.) Per-step count of samples loaded; same semantics as `totalQueryableSamples` per step. + - **samplesRead**: Total number of samples *read* (I/O). For range-vector functions in range queries, only new points per step are counted; for other queries this equals `totalQueryableSamples`. + - **samplesReadPerStep**: (Only with `stats=all` and when per-step stats are enabled.) Per-step count of samples read (delta semantics for range-vector). + - **peakSamples**: Peak number of samples in memory during evaluation. + +The server also exposes two Prometheus metrics: `prometheus_engine_query_samples_total` (samples loaded) and `prometheus_engine_query_samples_read_total` (samples read). See [Per-step stats](../feature_flags.md#per-step-stats) for the `promql-per-step-stats` feature flag. + ## Formatting query expressions The following endpoint formats a PromQL expression in a prettified way: @@ -231,14 +271,89 @@ The `data` section of the query result is a string containing the formatted quer The following example formats the expression `foo/bar`: +```bash +curl 'http://localhost:9090/api/v1/format_query?query=foo/bar' +``` + ```json -$ curl 'http://localhost:9090/api/v1/format_query?query=foo/bar' { "status" : "success", "data" : "foo / bar" } ``` +## Parsing a PromQL expressions into a abstract syntax tree (AST) + +This endpoint is **experimental** and might change in the future. It is currently only meant to be used by Prometheus' own web UI, and the endpoint name and exact format returned may change from one Prometheus version to another. It may also be removed again in case it is no longer needed by the UI. + +The following endpoint parses a PromQL expression and returns it as a JSON-formatted AST (abstract syntax tree) representation: + +``` +GET /api/v1/parse_query +POST /api/v1/parse_query +``` + +URL query parameters: + +- `query=`: Prometheus expression query string. + +You can URL-encode these parameters directly in the request body by using the `POST` method and +`Content-Type: application/x-www-form-urlencoded` header. This is useful when specifying a large +query that may breach server-side URL character limits. + +The `data` section of the query result is a string containing the AST of the parsed query expression. + +The following example parses the expression `foo/bar`: + +```bash +curl 'http://localhost:9090/api/v1/parse_query?query=foo/bar' +``` + +```json +{ + "data" : { + "bool" : false, + "lhs" : { + "matchers" : [ + { + "name" : "__name__", + "type" : "=", + "value" : "foo" + } + ], + "name" : "foo", + "offset" : 0, + "startOrEnd" : null, + "timestamp" : null, + "type" : "vectorSelector" + }, + "matching" : { + "card" : "one-to-one", + "include" : [], + "labels" : [], + "on" : false + }, + "op" : "/", + "rhs" : { + "matchers" : [ + { + "name" : "__name__", + "type" : "=", + "value" : "bar" + } + ], + "name" : "bar", + "offset" : 0, + "startOrEnd" : null, + "timestamp" : null, + "type" : "vectorSelector" + }, + "type" : "binaryExpr" + }, + "status" : "success" +} +``` + ## Querying metadata Prometheus offers a set of API endpoints to query metadata about series and their labels. @@ -267,13 +382,18 @@ You can URL-encode these parameters directly in the request body by using the `P or dynamic number of series selectors that may breach server-side URL character limits. The `data` section of the query result consists of a list of objects that -contain the label name/value pairs which identify each series. +contain the label name/value pairs which identify each series. Note that the +`start` and `end` times are approximate and the result may contain label values +for series which have no samples in the given interval. The following example returns all series that match either of the selectors `up` or `process_start_time_seconds{job="prometheus"}`: +```bash +curl -g 'http://localhost:9090/api/v1/series?' --data-urlencode 'match[]=up' --data-urlencode 'match[]=process_start_time_seconds{job="prometheus"}' +``` + ```json -$ curl -g 'http://localhost:9090/api/v1/series?' --data-urlencode 'match[]=up' --data-urlencode 'match[]=process_start_time_seconds{job="prometheus"}' { "status" : "success", "data" : [ @@ -313,13 +433,17 @@ URL query parameters: series from which to read the label names. Optional. - `limit=`: Maximum number of returned series. Optional. 0 means disabled. - -The `data` section of the JSON response is a list of string label names. +The `data` section of the JSON response is a list of string label names. Note +that the `start` and `end` times are approximate and the result may contain +label names for series which have no samples in the given interval. Here is an example. +```bash +curl 'localhost:9090/api/v1/labels' +``` + ```json -$ curl 'localhost:9090/api/v1/labels' { "status": "success", "data": [ @@ -364,22 +488,161 @@ URL query parameters: series from which to read the label values. Optional. - `limit=`: Maximum number of returned series. Optional. 0 means disabled. +The `data` section of the JSON response is a list of string label values. Note +that the `start` and `end` times are approximate and the result may contain +label values for series which have no samples in the given interval. -The `data` section of the JSON response is a list of string label values. -This example queries for all label values for the `job` label: +This example queries for all label values for the `http_status_code` label: + +```bash +curl http://localhost:9090/api/v1/label/http_status_code/values +``` ```json -$ curl http://localhost:9090/api/v1/label/job/values { "status" : "success", "data" : [ - "node", - "prometheus" + "200", + "504" ] } ``` +Label names can optionally be encoded using the Values Escaping method, and is necessary if a name includes the `/` character. To encode a name in this way: + +* Prepend the label with `U__`. +* Letters, numbers, and colons appear as-is. +* Convert single underscores to double underscores. +* For all other characters, use the UTF-8 codepoint as a hex integer, surrounded + by underscores. So ` ` becomes `_20_` and a `.` becomes `_2e_`. + + More information about text escaping can be found in the original UTF-8 [Proposal document](https://github.com/prometheus/proposals/blob/main/proposals/2023-08-21-utf8.md#text-escaping). + +This example queries for all label values for the `http.status_code` label: + +```bash +curl http://localhost:9090/api/v1/label/U__http_2e_status_code/values +``` + +```json +{ + "status" : "success", + "data" : [ + "200", + "404" + ] +} +``` + +### Searching metric names, label names, and label values + +These endpoints are experimental and must be enabled with +`--enable-feature=search-api`. + +The following endpoints provide streamed discovery results for metric names, +label names, and label values: + +``` +GET /api/v1/search/metric_names +POST /api/v1/search/metric_names +GET /api/v1/search/label_names +POST /api/v1/search/label_names +GET /api/v1/search/label_values +POST /api/v1/search/label_values +``` + +These endpoints return newline-delimited JSON with content type +`application/x-ndjson`. The stream contract is: + +- Zero or more **batch lines**, each with a `results` array and an optional + `warnings` array. +- The stream then ends with **either** a **trailer line** (`status`, `has_more`, + optional `warnings`) **or** an **error line** (`status`, `errorType`, `error`) + if iteration failed mid-stream after the first batch was sent. + +```json +{"results":[{"name":"http_requests_total","type":"counter","help":"Total HTTP requests."}]} +{"status":"success","has_more":false} +``` + +If an error occurs **before** streaming starts, the API returns the usual +Prometheus JSON error object with a 4xx/5xx status code. If an error occurs +**after** streaming starts, the stream ends with an NDJSON error line in place +of the trailer. + +Clients must tolerate an abrupt EOF without a trailer (for example, on +transport failure or server shutdown) and must ignore unknown fields in the +trailer for forward compatibility. + +The `has_more` field in the trailer is informational only: this version of +the API does not provide a pagination cursor. To retrieve more results, raise +`limit` (subject to the operator-configured `--web.search.max-limit`) or narrow +the request via `match[]`. A future version of the API may add a cursor. + +Common URL query parameters: + +- `match[]=`: Repeated series selector used to scope the + search. Optional. +- `search[]=`: Repeated search string matched against names or values. + Multiple values use OR semantics. Optional. +- `fuzz_threshold=`: Fuzzy threshold from 0 to 100. Optional. A value + of 0 is the lowest fuzzy threshold. +- `fuzz_alg=`: Matching algorithm. Optional. Default + is `subsequence`. +- `case_sensitive=`: Toggle case-sensitive matching. Optional. +- `sort_by=`: Sort mode. Supported values depend on the endpoint. +- `sort_dir=`: Sort direction. Optional. Only valid with + `sort_by=alpha`. +- `include_score=`: Include the relevance score in each result. Optional. +- `start=`: Start timestamp. Optional. +- `end=`: End timestamp. Optional. +- `limit=`: Maximum number of returned results. Optional. Default is + 100. +- `batch_size=`: Preferred number of results per NDJSON batch. + Optional. Default is 100. + +The `start` and `end` parameters narrow results to the selected time window. +Results may include values from series active slightly outside that window, +because Prometheus stores data in fixed-size blocks (typically 2 hours each). + +Additional parameters for `/api/v1/search/metric_names`: + +- `include_metadata=`: Include metric metadata in each result. +- `sort_by=` + +Additional parameters for `/api/v1/search/label_names`: + +- `sort_by=` + +Additional parameters for `/api/v1/search/label_values`: + +- `label=`: Label name whose values should be searched. Required. +- `sort_by=` + +This example searches metric names for autocomplete: + +```bash +curl -g 'http://localhost:9090/api/v1/search/metric_names?search[]=http_req&sort_by=score&include_metadata=true&limit=5' +``` + +```json +{"results":[{"name":"http_requests_total","type":"counter","help":"Total HTTP requests."}]} +{"status":"success","has_more":false} +``` + +This example searches label values for the `instance` label within the `up` +metric: + +```bash +curl -g 'http://localhost:9090/api/v1/search/label_values?label=instance&match[]=up&search[]=909&sort_by=score' +``` + +```json +{"results":[{"value":"localhost:9090"},{"value":"localhost:9091"}]} +{"status":"success","has_more":true} +``` + ## Querying exemplars This is **experimental** and might change in the future. @@ -396,8 +659,11 @@ URL query parameters: - `start=`: Start timestamp. - `end=`: End timestamp. +```bash +curl -g 'http://localhost:9090/api/v1/query_exemplars?query=test_exemplar_metric_total&start=2020-09-14T15:22:25.479Z&end=2020-09-14T15:23:25.479Z' +``` + ```json -$ curl -g 'http://localhost:9090/api/v1/query_exemplars?query=test_exemplar_metric_total&start=2020-09-14T15:22:25.479Z&end=2020-09-14T15:23:25.479Z' { "status": "success", "data": [ @@ -454,16 +720,16 @@ sample values. JSON does not support special float values such as `NaN`, `Inf`, and `-Inf`, so sample values are transferred as quoted JSON strings rather than raw numbers. -The keys `"histogram"` and `"histograms"` only show up if the experimental -native histograms are present in the response. Their placeholder `` -is explained in detail in its own section below. +The keys `"histogram"` and `"histograms"` only show up if native histograms +are present in the response. Their placeholder `` is explained +in detail in its own section below. ### Range vectors Range vectors are returned as result type `matrix`. The corresponding `result` property has the following format: -``` +```json [ { "metric": { "": "", ... }, @@ -485,7 +751,7 @@ and [`sort_by_label`](functions.md#sort_by_label) have no effect for range vecto Instant vectors are returned as result type `vector`. The corresponding `result` property has the following format: -``` +```json [ { "metric": { "": "", ... }, @@ -499,7 +765,7 @@ Instant vectors are returned as result type `vector`. The corresponding Each series could have the `"value"` key, or the `"histogram"` key, but not both. Series are not guaranteed to be returned in any particular order unless a function -such as [`sort`](functions.md#sort) or [`sort_by_label`](functions.md#sort_by_label)` +such as [`sort`](functions.md#sort) or [`sort_by_label`](functions.md#sort_by_label) is used. ### Scalars @@ -507,7 +773,7 @@ is used. Scalar results are returned as result type `scalar`. The corresponding `result` property has the following format: -``` +```json [ , "" ] ``` @@ -516,7 +782,7 @@ Scalar results are returned as result type `scalar`. The corresponding String results are returned as result type `string`. The corresponding `result` property has the following format: -``` +```json [ , "" ] ``` @@ -524,10 +790,7 @@ String results are returned as result type `string`. The corresponding The `` placeholder used above is formatted as follows. -_Note that native histograms are an experimental feature, and the format below -might still change._ - -``` +```json { "count": "", "sum": "", @@ -547,6 +810,35 @@ Note that with the currently implemented bucket schemas, positive buckets are “open left”, negative buckets are “open right”, and the zero bucket (with a negative left boundary and a positive right boundary) is “closed both”. +## Scrape pools + +The following endpoint returns a list of all configured scrape pools: + +``` +GET /api/v1/scrape_pools +``` + +The `data` section of the JSON response is a list of string scrape pool names. + +```bash +curl http://localhost:9090/api/v1/scrape_pools +``` + +```json +{ + "status": "success", + "data": { + "scrapePools": [ + "prometheus", + "node_exporter", + "blackbox" + ] + } +} +``` + +*New in v2.42* + ## Targets The following endpoint returns an overview of the current state of the @@ -561,8 +853,11 @@ Dropped targets are subject to `keep_dropped_targets` limit, if set. `labels` represents the label set after relabeling has occurred. `discoveredLabels` represent the unmodified labels retrieved during service discovery before relabeling has occurred. +```bash +curl http://localhost:9090/api/v1/targets +``` + ```json -$ curl http://localhost:9090/api/v1/targets { "status": "success", "data": { @@ -593,12 +888,16 @@ $ curl http://localhost:9090/api/v1/targets { "discoveredLabels": { "__address__": "127.0.0.1:9100", + "__always_scrape_classic_histograms__": "false", + "__convert_classic_histograms_to_nhcb__": "false", "__metrics_path__": "/metrics", "__scheme__": "http", "__scrape_interval__": "1m", + "__scrape_native_histograms__": "false", "__scrape_timeout__": "10s", "job": "node" }, + "scrapePool": "node" } ] } @@ -610,8 +909,11 @@ The `state` query parameter allows the caller to filter by active or dropped tar Note that an empty array is still returned for targets that are filtered out. Other values are ignored. +```bash +curl 'http://localhost:9090/api/v1/targets?state=active' +``` + ```json -$ curl 'http://localhost:9090/api/v1/targets?state=active' { "status": "success", "data": { @@ -643,8 +945,11 @@ $ curl 'http://localhost:9090/api/v1/targets?state=active' The `scrapePool` query parameter allows the caller to filter by scrape pool name. +```bash +curl 'http://localhost:9090/api/v1/targets?scrapePool=node_exporter' +``` + ```json -$ curl 'http://localhost:9090/api/v1/targets?scrapePool=node_exporter' { "status": "success", "data": { @@ -674,6 +979,64 @@ $ curl 'http://localhost:9090/api/v1/targets?scrapePool=node_exporter' } ``` +## Relabel steps + +This endpoint is **experimental** and might change in the future. It is currently only meant to be used by Prometheus' own web UI, and the endpoint name and exact format returned may change from one Prometheus version to another. It may also be removed again in case it is no longer needed by the UI. + +The following endpoint returns a step-by-step list of relabeling rules and their effects on a given target's label set. + +``` +GET /api/v1/targets/relabel_steps +``` + +URL query parameters: +- `scrapePool=`: The scrape pool name of the target, used to determine the relabeling rules to apply. Required. +- `labels=`: A JSON object containing the label set of the target before any relabeling is applied. Required. + +The following example returns the relabeling steps for a discovered target in the `prometheus` scrape pool with the label set `{"__address__": "localhost:9090", "job": "prometheus"}`: + +```bash +curl -g 'http://localhost:9090/api/v1/targets/relabel_steps?scrapePool=prometheus&labels={"__address__":"localhost:9090","job":"prometheus"}' +``` + +```json +{ + "data" : { + "steps" : [ + { + "keep" : true, + "output" : { + "__address__" : "localhost:9090", + "env" : "development", + "job" : "prometheus" + }, + "rule" : { + "action" : "replace", + "regex" : "(.*)", + "replacement" : "development", + "separator" : ";", + "target_label" : "env" + } + }, + { + "keep" : false, + "output" : {}, + "rule" : { + "action" : "drop", + "regex" : "localhost:.*", + "replacement" : "$1", + "separator" : ";", + "source_labels" : [ + "__address__" + ] + } + } + ] + }, + "status" : "success" +} +``` + ## Rules The `/rules` API endpoint returns a list of alerting and recording rules that @@ -693,12 +1056,16 @@ URL query parameters: - `rule_name[]=`: only return rules with the given rule name. If the parameter is repeated, rules with any of the provided names are returned. If we've filtered out all the rules of a group, the group is not returned. When the parameter is absent or empty, no filtering is done. - `rule_group[]=`: only return rules with the given rule group name. If the parameter is repeated, rules with any of the provided rule group names are returned. When the parameter is absent or empty, no filtering is done. - `file[]=`: only return rules with the given filepath. If the parameter is repeated, rules with any of the provided filepaths are returned. When the parameter is absent or empty, no filtering is done. -- `exclude_alerts=`: only return rules, do not return active alerts. +- `exclude_alerts=`: only return rules, do not return active alerts. - `match[]=`: only return rules that have configured labels that satisfy the label selectors. If the parameter is repeated, rules that match any of the sets of label selectors are returned. Note that matching is on the labels in the definition of each rule, not on the values after template expansion (for alerting rules). Optional. +- `group_limit=`: The `group_limit` parameter allows you to specify a limit for the number of rule groups that is returned in a single response. If the total number of rule groups exceeds the specified `group_limit` value, the response will include a `groupNextToken` property. You can use the value of this `groupNextToken` property in subsequent requests in the `group_next_token` parameter to paginate over the remaining rule groups. The `groupNextToken` property will not be present in the final response, indicating that you have retrieved all the available rule groups. Please note that there are no guarantees regarding the consistency of the response if the rule groups are being modified during the pagination process. +- `group_next_token`: the pagination token that was returned in previous request when the `group_limit` property is set. The pagination token is used to iteratively paginate over a large number of rule groups. To use the `group_next_token` parameter, the `group_limit` parameter also need to be present. If a rule group that coincides with the next token is removed while you are paginating over the rule groups, a response with status code 400 will be returned. -```json -$ curl http://localhost:9090/api/v1/rules +```bash +curl http://localhost:9090/api/v1/rules +``` +```json { "data": { "groups": [ @@ -761,9 +1128,11 @@ guarantees as the overarching API v1. GET /api/v1/alerts ``` -```json -$ curl http://localhost:9090/api/v1/alerts +```bash +curl http://localhost:9090/api/v1/alerts +``` +```json { "data": { "alerts": [ @@ -785,6 +1154,7 @@ $ curl http://localhost:9090/api/v1/alerts ## Querying target metadata The following endpoint returns metadata about metrics currently scraped from targets. +The endpoint has the limitation that only metadata scraped from targets directly is returned, metadata sent over Remote-Write or OTLP to Prometheus is not included in this endpoint and will not show up on the UI in "Explore Metrics". This is **experimental** and might change in the future. ``` @@ -803,11 +1173,14 @@ contain metric metadata and the target label set. The following example returns all metadata entries for the `go_goroutines` metric from the first two targets with label `job="prometheus"`. -```json +```bash curl -G http://localhost:9091/api/v1/targets/metadata \ --data-urlencode 'metric=go_goroutines' \ --data-urlencode 'match_target={job="prometheus"}' \ --data-urlencode 'limit=2' +``` + +```json { "status": "success", "data": [ @@ -834,11 +1207,14 @@ curl -G http://localhost:9091/api/v1/targets/metadata \ ``` The following example returns metadata for all metrics for all targets with -label `instance="127.0.0.1:9090`. +label `instance="127.0.0.1:9090"`. -```json +```bash curl -G http://localhost:9091/api/v1/targets/metadata \ --data-urlencode 'match_target={instance="127.0.0.1:9090"}' +``` + +```json { "status": "success", "data": [ @@ -887,9 +1263,11 @@ The `data` section of the query result consists of an object where each key is a The following example returns two metrics. Note that the metric `http_requests_total` has more than one object in the list. At least one target has a value for `HELP` that do not match with the rest. -```json +```bash curl -G http://localhost:9090/api/v1/metadata?limit=2 +``` +```json { "status": "success", "data": { @@ -918,9 +1296,11 @@ curl -G http://localhost:9090/api/v1/metadata?limit=2 The following example returns only one metadata entry for each metric. -```json +```bash curl -G http://localhost:9090/api/v1/metadata?limit_per_metric=1 +``` +```json { "status": "success", "data": { @@ -944,9 +1324,11 @@ curl -G http://localhost:9090/api/v1/metadata?limit_per_metric=1 The following example returns metadata only for the metric `http_requests_total`. -```json +```bash curl -G http://localhost:9090/api/v1/metadata?metric=http_requests_total +``` +```json { "status": "success", "data": { @@ -977,8 +1359,11 @@ GET /api/v1/alertmanagers Both the active and dropped Alertmanagers are part of the response. +```bash +curl http://localhost:9090/api/v1/alertmanagers +``` + ```json -$ curl http://localhost:9090/api/v1/alertmanagers { "status": "success", "data": { @@ -1011,8 +1396,11 @@ GET /api/v1/status/config The config is returned as dumped YAML file. Due to limitation of the YAML library, YAML comments are not included. +```bash +curl http://localhost:9090/api/v1/status/config +``` + ```json -$ curl http://localhost:9090/api/v1/status/config { "status": "success", "data": { @@ -1031,8 +1419,11 @@ GET /api/v1/status/flags All values are of the result type `string`. +```bash +curl http://localhost:9090/api/v1/status/flags +``` + ```json -$ curl http://localhost:9090/api/v1/status/flags { "status": "success", "data": { @@ -1058,13 +1449,18 @@ GET /api/v1/status/runtimeinfo The returned values are of different types, depending on the nature of the runtime property. +```bash +curl http://localhost:9090/api/v1/status/runtimeinfo +``` + ```json -$ curl http://localhost:9090/api/v1/status/runtimeinfo { "status": "success", "data": { "startTime": "2019-11-02T17:23:59.301361365+01:00", "CWD": "/", + "hostname" : "DESKTOP-717H17Q", + "serverTime": "2025-01-05T18:27:33Z", "reloadConfigSuccess": true, "lastConfigTime": "2019-11-02T17:23:59+01:00", "timeSeriesCount": 873, @@ -1092,8 +1488,11 @@ GET /api/v1/status/buildinfo All values are of the result type `string`. +```bash +curl http://localhost:9090/api/v1/status/buildinfo +``` + ```json -$ curl http://localhost:9090/api/v1/status/buildinfo { "status": "success", "data": { @@ -1119,9 +1518,11 @@ The following endpoint returns various cardinality statistics about the Promethe GET /api/v1/status/tsdb ``` URL query parameters: -- `limit=`: Limit the number of returned items to a given number for each set of statistics. By default, 10 items are returned. -The `data` section of the query result consists of +- `limit=`: Limit the number of returned items to a given number for each set of statistics. By default, 10 items are returned. The maximum allowed limit is 10000. + +The `data` section of the query result consists of: + - **headStats**: This provides the following data about the head block of the TSDB: - **numSeries**: The number of series. - **chunkCount**: The number of chunks. @@ -1132,8 +1533,11 @@ The `data` section of the query result consists of - **memoryInBytesByLabelName** This will provide a list of the label names and memory used in bytes. Memory usage is calculated by adding the length of all values for a given label name. - **seriesCountByLabelPair** This will provide a list of label value pairs and their series count. +```bash +curl http://localhost:9090/api/v1/status/tsdb +``` + ```json -$ curl http://localhost:9090/api/v1/status/tsdb { "status": "success", "data": { @@ -1187,6 +1591,64 @@ $ curl http://localhost:9090/api/v1/status/tsdb } ``` +*New in v3.6.0* + +### TSDB Blocks + +**NOTE**: This endpoint is **experimental** and might change in the future. The endpoint name and the exact format of the returned data may change between Prometheus versions. The **exact metadata returned** by this endpoint is an implementation detail and may change in future Prometheus versions. + +The following endpoint returns the list of currently loaded TSDB blocks and their metadata. + +``` +GET /api/v1/status/tsdb/blocks +``` + +This endpoint returns the following information for each block: + +- `ulid`: Unique ID of the block. +- `minTime`: Minimum timestamp (in milliseconds) of the block. +- `maxTime`: Maximum timestamp (in milliseconds) of the block. +- `stats`: + - `numSeries`: Number of series in the block. + - `numSamples`: Number of samples in the block. + - `numChunks`: Number of chunks in the block. +- `compaction`: + - `level`: The compaction level of the block. + - `sources`: List of ULIDs of source blocks used to compact this block. +- `version`: The block version. + + +```bash +curl http://localhost:9090/api/v1/status/tsdb/blocks +``` + +```json +{ + "status": "success", + "data": { + "blocks": [ + { + "ulid": "01JZ8JKZY6XSK3PTDP9ZKRWT60", + "minTime": 1750860620060, + "maxTime": 1750867200000, + "stats": { + "numSamples": 13701, + "numSeries": 716, + "numChunks": 716 + }, + "compaction": { + "level": 1, + "sources": [ + "01JZ8JKZY6XSK3PTDP9ZKRWT60" + ] + }, + "version": 1 + } + ] + } +} +``` + *New in v2.15* ### WAL Replay Stats @@ -1197,16 +1659,19 @@ The following endpoint returns information about the WAL replay: GET /api/v1/status/walreplay ``` -**read**: The number of segments replayed so far. -**total**: The total number segments needed to be replayed. -**progress**: The progress of the replay (0 - 100%). -**state**: The state of the replay. Possible states: -- **waiting**: Waiting for the replay to start. -- **in progress**: The replay is in progress. -- **done**: The replay has finished. +- **read**: The number of segments replayed so far. +- **total**: The total number segments needed to be replayed. +- **progress**: The progress of the replay (0 - 100%). +- **state**: The state of the replay. Possible states: + - **waiting**: Waiting for the replay to start. + - **in progress**: The replay is in progress. + - **done**: The replay has finished. + +```bash +curl http://localhost:9090/api/v1/status/walreplay +``` ```json -$ curl http://localhost:9090/api/v1/status/walreplay { "status": "success", "data": { @@ -1222,6 +1687,65 @@ NOTE: This endpoint is available before the server has been marked ready and is *New in v2.28* +### Self Metrics + +**NOTE**: This endpoint is **experimental** and might change in the future. + +The following endpoint returns Prometheus' own instrumentation metrics from its internal client registry as structured JSON. These are the same metrics that are exposed on the `/metrics` endpoint in Prometheus text exposition format, but returned as JSON for programmatic access by the web UI. + +The response uses the standard [ProtoJSON](https://protobuf.dev/programming-guides/json/) representation of the `io.prometheus.client.MetricFamily` protocol buffer message. + +``` +GET /api/v1/status/self_metrics +``` + +URL query parameters: + +- `metric_name_pattern=`: A regular expression filter for metric names (fully anchored, like PromQL label matchers). Only metric families whose names fully match the pattern are returned. For example, `metric_name_pattern=prometheus_tsdb_.*` returns all metric families whose names start with `prometheus_tsdb_`. Optional. When omitted, all metric families are returned. + +Each returned metric family is a ProtoJSON-encoded `MetricFamily` containing: + +- **name**: The metric name. +- **help**: The metric help string. +- **type**: The metric type (`COUNTER`, `GAUGE`, `SUMMARY`, `HISTOGRAM`, `UNTYPED`). +- **unit**: The metric unit, if set (optional). +- **metric**: A list of individual metrics, each containing: + - **label**: A list of `{name, value}` label pairs. + - **gauge**, **counter**, **summary**, **histogram**, or **untyped**: The type-specific metric data. + +```bash +curl 'http://localhost:9090/api/v1/status/self_metrics?metric_name_pattern=prometheus_build_info' +``` + +```json +{ + "status": "success", + "data": [ + { + "name": "prometheus_build_info", + "help": "A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build.", + "type": "GAUGE", + "metric": [ + { + "label": [ + { "name": "branch", "value": "main" }, + { "name": "goarch", "value": "amd64" }, + { "name": "goos", "value": "linux" }, + { "name": "goversion", "value": "go1.26.1-X:nodwarf5" }, + { "name": "revision", "value": "7b5a4090e38d9e1ad7697c7641234f4ed135a6c7" }, + { "name": "tags", "value": "netgo,builtinassets" }, + { "name": "version", "value": "3.11.0-rc.0" } + ], + "gauge": { + "value": 1 + } + } + ] + } + ] +} +``` + ## TSDB Admin APIs These are APIs that expose database functionalities for the advanced user. These APIs are not enabled unless the `--web.enable-admin-api` is set. @@ -1238,8 +1762,11 @@ URL query parameters: - `skip_head=`: Skip data present in the head block. Optional. +```bash +curl -XPOST http://localhost:9090/api/v1/admin/tsdb/snapshot +``` + ```json -$ curl -XPOST http://localhost:9090/api/v1/admin/tsdb/snapshot { "status": "success", "data": { @@ -1271,8 +1798,8 @@ Not mentioning both start and end times would clear all the data for the matched Example: -```json -$ curl -X POST \ +```bash +curl -X POST \ -g 'http://localhost:9090/api/v1/admin/tsdb/delete_series?match[]=up&match[]=process_start_time_seconds{job="prometheus"}' ``` @@ -1292,8 +1819,8 @@ PUT /api/v1/admin/tsdb/clean_tombstones This takes no parameters or body. -```json -$ curl -XPOST http://localhost:9090/api/v1/admin/tsdb/clean_tombstones +```bash +curl -XPOST http://localhost:9090/api/v1/admin/tsdb/clean_tombstones ``` *New in v2.1 and supports PUT from v2.9* @@ -1319,8 +1846,166 @@ is not considered an efficient way of ingesting samples. Use it with caution for specific low-volume use cases. It is not suitable for replacing the ingestion via scraping. -Enable the OTLP receiver by the feature flag -`--enable-feature=otlp-write-receiver`. When enabled, the OTLP receiver +Enable the OTLP receiver by setting +`--web.enable-otlp-receiver`. When enabled, the OTLP receiver endpoint is `/api/v1/otlp/v1/metrics`. *New in v2.47* + +### OTLP Delta + +Prometheus can convert incoming metrics from delta temporality to their cumulative equivalent. +This is done using [deltatocumulative](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) from the OpenTelemetry Collector. + +To enable, pass `--enable-feature=otlp-deltatocumulative`. + +*New in v3.2* + +## Notifications + +The following endpoints provide information about active status notifications concerning the Prometheus server itself. +Notifications are used in the web UI. + +These endpoints are **experimental**. They may change in the future. + +### Active Notifications + +The `/api/v1/notifications` endpoint returns a list of all currently active notifications. + +``` +GET /api/v1/notifications +``` + +Example: + +```bash +curl http://localhost:9090/api/v1/notifications +``` + +``` +{ + "status": "success", + "data": [ + { + "text": "Prometheus is shutting down and gracefully stopping all operations.", + "date": "2024-10-07T12:33:08.551376578+02:00", + "active": true + } + ] +} +``` + +*New in v3.0* + +### Live Notifications + +The `/api/v1/notifications/live` endpoint streams live notifications as they occur, using [Server-Sent Events](https://html.spec.whatwg.org/multipage/server-sent-events.html#server-sent-events). Deleted notifications are sent with `active: false`. Active notifications will be sent when connecting to the endpoint. + +``` +GET /api/v1/notifications/live +``` + +Example: + +```bash +curl http://localhost:9090/api/v1/notifications/live +``` + +``` +data: { + "status": "success", + "data": [ + { + "text": "Prometheus is shutting down and gracefully stopping all operations.", + "date": "2024-10-07T12:33:08.551376578+02:00", + "active": true + } + ] +} +``` + +**Note:** The `/notifications/live` endpoint will return a `204 No Content` response if the maximum number of subscribers has been reached. You can set the maximum number of listeners with the flag `--web.max-notifications-subscribers`, which defaults to 16. + +``` +GET /api/v1/notifications/live +204 No Content +``` + +*New in v3.0* + +### Features + +The following endpoint returns a list of enabled features in the Prometheus server: + +``` +GET /api/v1/features +``` + +This endpoint provides information about which features are currently enabled or disabled in the Prometheus instance. Features are organized into categories such as `api`, `promql`, `promql_functions`, etc. + +The `data` section contains a map where each key is a feature category, and each value is a map of feature names to their enabled status (boolean). + +```bash +curl http://localhost:9090/api/v1/features +``` + +```json +{ + "status": "success", + "data": { + "api": { + "admin": false, + "exclude_alerts": true + }, + "otlp_receiver": { + "delta_conversion": false, + "native_delta_ingestion": false + }, + "prometheus": { + "agent_mode": false, + "auto_reload_config": false + }, + "promql": { + "anchored": false, + "at_modifier": true + }, + "promql_functions": { + "abs": true, + "absent": true + }, + "promql_operators": { + "!=": true, + "!~": true + }, + "rules": { + "concurrent_rule_eval": false, + "keep_firing_for": true + }, + "scrape": { + "start_timestamp_zero_ingestion": false, + "extra_metrics": false + }, + "service_discovery": { + "azure": true, + "consul": true + }, + "templating": { + "args": true, + "externalURL": true + }, + "tsdb": { + "delayed_compaction": false, + "exemplar_storage": false + } + } +} +``` + +**Notes:** + +- All feature names use `snake_case` naming convention +- Features set to `false` may be omitted from the response +- Clients should treat absent features as equivalent to `false` +- Clients must ignore unknown feature names and categories for forward compatibility + +*New in v3.8* diff --git a/docs/querying/basics.md b/docs/querying/basics.md index 81ffb4e0f39..e7f1173af42 100644 --- a/docs/querying/basics.md +++ b/docs/querying/basics.md @@ -4,15 +4,13 @@ nav_title: Basics sort_rank: 1 --- -# Querying Prometheus - Prometheus provides a functional query language called PromQL (Prometheus Query Language) that lets the user select and aggregate time series data in real -time. +time. When you send a query request to Prometheus, it can be an _instant query_, evaluated at one point in time, or a _range query_ at equally-spaced steps between a start and an end time. PromQL works exactly the same -in each cases; the range query is just like an instant query run multiple times at different timestamps. +in each case; the range query is just like an instant query run multiple times at different timestamps. In the Prometheus UI, the "Table" tab is for instant queries and the "Graph" tab is for range queries. @@ -23,6 +21,34 @@ Other programs can fetch the result of a PromQL expression via the [HTTP API](ap This document is a Prometheus basic language reference. For learning, it may be easier to start with a couple of [examples](examples.md). +## Samples + +The value of a sample at a given timestamp returned by PromQL may be a float or +a [native histogram](https://prometheus.io/docs/specs/native_histograms). A +float sample is a simple floating point number, whereas a native histograms +sample contains a full histogram including count, sum, and buckets. + +Note that the term “histogram sample” in the PromQL documentation always refers +to a native histogram. The term "classic histogram" refers to a set of time +series containing float samples with the `_bucket`, `_count`, and `_sum` +suffixes that together describe a histogram. From the perspective of PromQL, +these contain just float samples, there are no “classic histogram samples”. + +Both float samples and histogram samples can have a counter or a gauge “flavor”. +Float samples with a counter or gauge flavor are generally simply called +“counters” or “gauges”, respectively, while their histogram counterparts are +called “counter histograms” or “gauge histograms”. Float samples do not store +their flavor, leaving it to the user to take their flavor into account when +writing PromQL queries. (By convention, time series containing float counters +have a name ending on `_total` to help with the distinction.) + +Since histogram samples “know” their counter or gauge flavor, this allows +reliable warnings about mismatched operations. For example, applying the `rate` +function to gauge floats will most likely produce a +nonsensical result, but the query will be processed without complains. However, +if applied to gauge histograms, the result of the query will be +annotated with a warning. + ## Expression language data types In Prometheus's expression language, an expression or sub-expression can @@ -33,23 +59,35 @@ evaluate to one of four types: * **Scalar** - a simple numeric floating point value * **String** - a simple string value; currently unused -Depending on the use-case (e.g. when graphing vs. displaying the output of an +Depending on the use case (e.g. when graphing vs. displaying the output of an expression), only some of these types are legal as the result of a -user-specified expression. For example, an expression that returns an instant -vector is the only type which can be graphed. +user-specified expression. +For [instant queries](api.md#instant-queries), any of the above data types are allowed as the root of the expression. +[Range queries](api.md#range-queries) only support scalar-typed and instant-vector-typed expressions. + +Both vectors and time series may contain a mix of float samples and histogram +samples. -_Notes about the experimental native histograms:_ +## Reconciliation of histogram bucket layouts -* Ingesting native histograms has to be enabled via a [feature - flag](../feature_flags.md#native-histograms). -* Once native histograms have been ingested into the TSDB (and even after - disabling the feature flag again), both instant vectors and range vectors may - now contain samples that aren't simple floating point numbers (float samples) - but complete histograms (histogram samples). A vector may contain a mix of - float samples and histogram samples. +Native histograms can have different bucket layouts, but they are generally +convertible to compatible versions to apply binary and aggregation operations +to them. Functions acting on range vectors that are applicable to native +histograms also perform such reconciliation. In binary operations this +reconciliation is performed pairwise, in aggregation operations and functions +all histogram samples are reconciled to one compatible bucket layout. + +Not all bucket layouts can be reconciled, if incompatible histograms are +encountered in an operation, the corresponding output vector element is removed +from the result, flagged with a warn-level annotation. +More details can be found in the +[native histogram specification](https://prometheus.io/docs/specs/native_histograms/#compatibility-between-histograms). ## Literals +The following section describes literal values of various kinds. +Note that there is no “histogram literal”. + ### String literals String literals are designated by single quotes, double quotes or backticks. @@ -68,9 +106,10 @@ Example: 'these are unescaped: \n \\ \t' `these are not unescaped: \n ' " \t` -### Float literals +### Float literals and time durations -Scalar float values can be written as literal integer or floating-point numbers in the format (whitespace only included for better readability): +Scalar float values can be written as literal integer or floating-point numbers +in the format (whitespace only included for better readability): [-+]?( [0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? @@ -87,16 +126,53 @@ Examples: 0x8f -Inf NaN - -As of version 2.54, float literals can also be represented using the syntax of time durations, where the time duration is converted into a float value corresponding to the number of seconds the time duration represents. This is an experimental feature and might still change. +Additionally, underscores (`_`) can be used in between decimal or hexadecimal +digits to improve readability. + +Examples: + + 1_000_000 + .123_456_789 + 0x_53_AB_F3_82 + +Float literals are also used to specify durations in seconds. For convenience, +decimal integer numbers may be combined with the following +time units: + +* `ms` – milliseconds +* `s` – seconds – 1s equals 1000ms +* `m` – minutes – 1m equals 60s (ignoring leap seconds) +* `h` – hours – 1h equals 60m +* `d` – days – 1d equals 24h (ignoring so-called daylight saving time) +* `w` – weeks – 1w equals 7d +* `y` – years – 1y equals 365d (ignoring leap days) + +Suffixing a decimal integer number with one of the units above is a different +representation of the equivalent number of seconds as a bare float literal. Examples: - 1s # Equivalent to 1.0 - 2m # Equivalent to 120.0 - 1ms # Equivalent to 0.001 - + 1s # Equivalent to 1. + 2m # Equivalent to 120. + 1ms # Equivalent to 0.001. + -2h # Equivalent to -7200. + +The following examples do _not_ work: + + 0xABm # No suffixing of hexadecimal numbers. + 1.5h # Time units cannot be combined with a floating point. + +Infd # No suffixing of ±Inf or NaN. + +Multiple units can be combined by concatenation of suffixed integers. Units +must be ordered from the longest to the shortest. A given unit must only appear +once per float literal. + +Examples: + + 1h30m # Equivalent to 5400s and thus 5400. + 12h34m56s # Equivalent to 45296s and thus 45296. + 54s321ms # Equivalent to 54.321. ## Time series selectors @@ -109,8 +185,16 @@ single sample value for each at a given timestamp (point in time). In the simpl form, only a metric name is specified, which results in an instant vector containing elements for all time series that have this metric name. +The value returned will be that of the most recent sample at or before the +query's evaluation timestamp (in the case of an +[instant query](api.md#instant-queries)) +or the current step within the query (in the case of a +[range query](api.md#range-queries)). +The [`@` modifier](#modifier) allows overriding the timestamp relative to which +the selection takes place. Time series are only returned if their most recent sample is less than the [lookback period](#staleness) ago. + This example selects all time series that have the `http_requests_total` metric -name: +name, returning the most recent sample for each: http_requests_total @@ -131,7 +215,7 @@ against regular expressions. The following label matching operators exist: * `=~`: Select labels that regex-match the provided string. * `!~`: Select labels that do not regex-match the provided string. -Regex matches are fully anchored. A match of `env=~"foo"` is treated as `env=~"^foo$"`. +[Regex](#regular-expressions) matches are fully anchored. A match of `env=~"foo"` is treated as `env=~"^foo$"`. For example, this selects all `http_requests_total` time series for `staging`, `testing`, and `development` environments and HTTP methods other than `GET`. @@ -158,7 +242,7 @@ and would exclude: http_requests_total{environment="development"} -Multiple matchers can be used for the same label name; they all must pass for a result to be returned. +Multiple matchers can be used for the same label name; they all must pass for a result to be returned. The query: @@ -194,59 +278,25 @@ A workaround for this restriction is to use the `__name__` label: {__name__="on"} # Good! -All regular expressions in Prometheus use [RE2 -syntax](https://github.com/google/re2/wiki/Syntax). - ### Range Vector Selectors Range vector literals work like instant vector literals, except that they -select a range of samples back from the current instant. Syntactically, a [time -duration](#time-durations) is appended in square brackets (`[]`) at the end of -a vector selector to specify how far back in time values should be fetched for -each resulting range vector element. The range is a closed interval, -i.e. samples with timestamps coinciding with either boundary of the range are -still included in the selection. - -In this example, we select all the values we have recorded within the last 5 -minutes for all time series that have the metric name `http_requests_total` and -a `job` label set to `prometheus`: +select a range of samples back from the current instant. Syntactically, a +[float literal](#float-literals-and-time-durations) is appended in square +brackets (`[]`) at the end of a vector selector to specify for how many seconds +back in time values should be fetched for each resulting range vector element. +Commonly, the float literal uses the syntax with one or more time units, e.g. +`[5m]`. The range is a left-open and right-closed interval, i.e. samples with +timestamps coinciding with the left boundary of the range are excluded from the +selection, while samples coinciding with the right boundary of the range are +included in the selection. + +In this example, we select all the values recorded less than 5m ago for all +time series that have the metric name `http_requests_total` and a `job` label +set to `prometheus`: http_requests_total{job="prometheus"}[5m] -### Time Durations - -Time durations are specified as a number, followed immediately by one of the -following units: - -* `ms` - milliseconds -* `s` - seconds -* `m` - minutes -* `h` - hours -* `d` - days - assuming a day always has 24h -* `w` - weeks - assuming a week always has 7d -* `y` - years - assuming a year always has 365d1 - -1 For days in a year, the leap day is ignored, and conversely, for a minute, a leap second is ignored. - -Time durations can be combined by concatenation. Units must be ordered from the -longest to the shortest. A given unit must only appear once in a time duration. - -Here are some examples of valid time durations: - - 5h - 1h30m - 5m - 10s - - -As of version 2.54, time durations can also be represented using the syntax of float literals, implying the number of seconds of the time duration. This is an experimental feature and might still change. - -Examples: - - 1.0 # Equivalent to 1s - 0.001 # Equivalent to 1ms - 120 # Equivalent to 2m - ### Offset modifier The `offset` modifier allows changing the time offset for individual @@ -282,7 +332,7 @@ Note that this allows a query to look ahead of its evaluation time. The `@` modifier allows changing the evaluation time for individual instant and range vectors in a query. The time supplied to the `@` modifier -is a unix timestamp and described with a float literal. +is a Unix timestamp and described with a float literal. For example, the following expression returns the value of `http_requests_total` at `2021-01-04T07:40:00+00:00`: @@ -329,7 +379,7 @@ Note that the `@` modifier allows a query to look ahead of its evaluation time. Subquery allows you to run an instant query for a given range and resolution. The result of a subquery is a range vector. -Syntax: ` '[' ':' [] ']' [ @ ] [ offset ]` +Syntax: ` '[' ':' [] ']' [ @ ] [ offset ]` * `` is optional. Default is the global evaluation interval. @@ -349,6 +399,12 @@ PromQL supports line comments that start with `#`. Example: # This is a comment +## Regular expressions + +All regular expressions in Prometheus use [RE2 syntax](https://github.com/google/re2/wiki/Syntax). + +Regex matches are always fully anchored. + ## Gotchas ### Staleness @@ -358,8 +414,10 @@ independently of the actual present time series data. This is mainly to support cases like aggregation (`sum`, `avg`, and so on), where multiple aggregated time series do not precisely align in time. Because of their independence, Prometheus needs to assign a value at those timestamps for each relevant time -series. It does so by taking the newest sample before this timestamp within the lookback period. -The lookback period is 5 minutes by default. +series. It does so by taking the newest sample that is less than the lookback period ago. +The lookback period is 5 minutes by default, but can be +[set with the `--query.lookback-delta` flag](../command-line/prometheus.md) +or overridden on an individual query via the `lookback_delta` parameter. If a target scrape or rule evaluation no longer returns a sample for a time series that was previously present, this time series will be marked as stale. @@ -371,11 +429,11 @@ as stale, then no value is returned for that time series. If new samples are subsequently ingested for that time series, they will be returned as expected. A time series will go stale when it is no longer exported, or the target no -longer exists. Such time series will disappear from graphs +longer exists. Such time series will disappear from graphs at the times of their latest collected sample, and they will not be returned in queries after they are marked stale. -Some exporters, which put their own timestamps on samples, get a different behaviour: +Some exporters, which put their own timestamps on samples, get a different behaviour: series that stop being exported take the last value for (by default) 5 minutes before disappearing. The `track_timestamps_staleness` setting can change this. diff --git a/docs/querying/examples.md b/docs/querying/examples.md index 8287ff6f628..a732334a957 100644 --- a/docs/querying/examples.md +++ b/docs/querying/examples.md @@ -1,11 +1,9 @@ --- -title: Querying examples +title: Query examples nav_title: Examples sort_rank: 4 --- -# Query examples - ## Simple time series selection Return all time series with the metric `http_requests_total`: @@ -25,14 +23,11 @@ for the same vector, making it a [range vector](../basics/#range-vector-selector Note that an expression resulting in a range vector cannot be graphed directly, but viewed in the tabular ("Console") view of the expression browser. -Using regular expressions, you could select time series only for jobs whose +Using [regular expressions](./basics.md#regular-expressions), you could select time series only for jobs whose name match a certain pattern, in this case, all jobs that end with `server`: http_requests_total{job=~".*server"} -All regular expressions in Prometheus use [RE2 -syntax](https://github.com/google/re2/wiki/Syntax). - To select all HTTP status codes except 4xx ones, you could run: http_requests_total{status!~"4.."} diff --git a/docs/querying/functions.md b/docs/querying/functions.md index c6e22019fc6..5900da4c623 100644 --- a/docs/querying/functions.md +++ b/docs/querying/functions.md @@ -4,37 +4,21 @@ nav_title: Functions sort_rank: 3 --- -# Functions - -Some functions have default arguments, e.g. `year(v=vector(time()) -instant-vector)`. This means that there is one argument `v` which is an instant +Some functions have default arguments, e.g. `year(v=vector(time()) instant-vector)`. This means that there is one argument `v` which is an instant vector, which if not provided it will default to the value of the expression `vector(time())`. -_Notes about the experimental native histograms:_ - -* Ingesting native histograms has to be enabled via a [feature - flag](../feature_flags.md#native-histograms). As long as no native histograms - have been ingested into the TSDB, all functions will behave as usual. -* Functions that do not explicitly mention native histograms in their - documentation (see below) will ignore histogram samples. -* Functions that do already act on native histograms might still change their - behavior in the future. -* If a function requires the same bucket layout between multiple native - histograms it acts on, it will automatically convert them - appropriately. (With the currently supported bucket schemas, that's always - possible.) - ## `abs()` -`abs(v instant-vector)` returns the input vector with all sample values converted to -their absolute value. +`abs(v instant-vector)` returns a vector containing all float samples in the +input vector converted to their absolute value. Histogram samples in the input +vector are ignored silently. ## `absent()` `absent(v instant-vector)` returns an empty vector if the vector passed to it -has any elements (floats or native histograms) and a 1-element vector with the -value 1 if the vector passed to it has no elements. +has any elements (float samples or histogram samples) and a 1-element vector +with the value 1 if the vector passed to it has no elements. This is useful for alerting on when no time series exist for a given metric name and label combination. @@ -56,8 +40,9 @@ of the 1-element output vector from the input vector. ## `absent_over_time()` `absent_over_time(v range-vector)` returns an empty vector if the range vector -passed to it has any elements (floats or native histograms) and a 1-element -vector with the value 1 if the range vector passed to it has no elements. +passed to it has any elements (float samples or histogram samples) and a +1-element vector with the value 1 if the range vector passed to it has no +elements. This is useful for alerting on when no time series exist for a given metric name and label combination for a certain amount of time. @@ -78,8 +63,9 @@ labels of the 1-element output vector from the input vector. ## `ceil()` -`ceil(v instant-vector)` rounds the sample values of all elements in `v` up to -the nearest integer value greater than or equal to v. +`ceil(v instant-vector)` returns a vector containing all float samples in the +input vector rounded up to the nearest integer value greater than or equal to +their original value. Histogram samples in the input vector are ignored silently. * `ceil(+Inf) = +Inf` * `ceil(±0) = ±0` @@ -90,49 +76,62 @@ the nearest integer value greater than or equal to v. For each input time series, `changes(v range-vector)` returns the number of times its value has changed within the provided time range as an instant -vector. +vector. A float sample followed by a histogram sample, or vice versa, counts as +a change. A counter histogram sample followed by a gauge histogram sample with +otherwise exactly the same values, or vice versa, does not count as a change. ## `clamp()` -`clamp(v instant-vector, min scalar, max scalar)` -clamps the sample values of all elements in `v` to have a lower limit of `min` and an upper limit of `max`. +`clamp(v instant-vector, min scalar, max scalar)` clamps the values of all +float samples in `v` to have a lower limit of `min` and an upper limit of +`max`. Histogram samples in the input vector are ignored silently. Special cases: * Return an empty vector if `min > max` -* Return `NaN` if `min` or `max` is `NaN` +* Float samples are clamped to `NaN` if `min` or `max` is `NaN` ## `clamp_max()` -`clamp_max(v instant-vector, max scalar)` clamps the sample values of all -elements in `v` to have an upper limit of `max`. +`clamp_max(v instant-vector, max scalar)` clamps the values of all float +samples in `v` to have an upper limit of `max`. Histogram samples in the input +vector are ignored silently. ## `clamp_min()` -`clamp_min(v instant-vector, min scalar)` clamps the sample values of all -elements in `v` to have a lower limit of `min`. +`clamp_min(v instant-vector, min scalar)` clamps the values of all float +samples in `v` to have a lower limit of `min`. Histogram samples in the input +vector are ignored silently. ## `day_of_month()` -`day_of_month(v=vector(time()) instant-vector)` returns the day of the month -for each of the given times in UTC. Returned values are from 1 to 31. +`day_of_month(v=vector(time()) instant-vector)` interprets float samples in +`v` as timestamps (number of seconds since January 1, 1970 UTC) and returns the +day of the month (in UTC) for each of those timestamps. Returned values are +from 1 to 31. Histogram samples in the input vector are ignored silently. ## `day_of_week()` -`day_of_week(v=vector(time()) instant-vector)` returns the day of the week for -each of the given times in UTC. Returned values are from 0 to 6, where 0 means -Sunday etc. +`day_of_week(v=vector(time()) instant-vector)` interprets float samples in `v` +as timestamps (number of seconds since January 1, 1970 UTC) and returns the day +of the week (in UTC) for each of those timestamps. Returned values are from 0 +to 6, where 0 means Sunday etc. Histogram samples in the input vector are +ignored silently. ## `day_of_year()` -`day_of_year(v=vector(time()) instant-vector)` returns the day of the year for -each of the given times in UTC. Returned values are from 1 to 365 for non-leap years, -and 1 to 366 in leap years. +`day_of_year(v=vector(time()) instant-vector)` interprets float samples in `v` +as timestamps (number of seconds since January 1, 1970 UTC) and returns the day +of the year (in UTC) for each of those timestamps. Returned values are from 1 +to 365 for non-leap years, and 1 to 366 in leap years. Histogram samples in the +input vector are ignored silently. ## `days_in_month()` -`days_in_month(v=vector(time()) instant-vector)` returns number of days in the -month for each of the given times in UTC. Returned values are from 28 to 31. +`days_in_month(v=vector(time()) instant-vector)` interprets float samples in +`v` as timestamps (number of seconds since January 1, 1970 UTC) and returns the +number of days in the month of each of those timestamps (in UTC). Returned +values are from 28 to 31. Histogram samples in the input vector are ignored silently. ## `delta()` @@ -150,36 +149,77 @@ between now and 2 hours ago: delta(cpu_temp_celsius{host="zeus"}[2h]) ``` -`delta` acts on native histograms by calculating a new histogram where each +`delta` acts on histogram samples by calculating a new histogram where each component (sum and count of observations, buckets) is the difference between -the respective component in the first and last native histogram in -`v`. However, each element in `v` that contains a mix of float and native -histogram samples within the range, will be missing from the result vector. +the respective component in the first and last native histogram in `v`. +However, each element in `v` that contains a mix of float samples and histogram +samples within the range will be omitted from the result vector, flagged by a +warn-level annotation. -`delta` should only be used with gauges and native histograms where the -components behave like gauges (so-called gauge histograms). +`delta` should only be used with gauges (for both floats and histograms). ## `deriv()` -`deriv(v range-vector)` calculates the per-second derivative of the time series in a range -vector `v`, using [simple linear regression](https://en.wikipedia.org/wiki/Simple_linear_regression). -The range vector must have at least two samples in order to perform the calculation. When `+Inf` or -`-Inf` are found in the range vector, the slope and offset value calculated will be `NaN`. - -`deriv` should only be used with gauges. +`deriv(v range-vector)` calculates the per-second derivative of each float time +series in the range vector `v`, using [simple linear +regression](https://en.wikipedia.org/wiki/Simple_linear_regression). The range +vector must have at least two float samples in order to perform the +calculation. When `+Inf` or `-Inf` are found in the range vector, the slope and +offset value calculated will be `NaN`. + +`deriv` should only be used with gauges and only works for float samples. +Elements in the range vector that contain only histogram samples are ignored +entirely. For elements that contain a mix of float and histogram samples, only +the float samples are used as input, which is flagged by an info-level +annotation. + +## `end()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`end()` returns the end timestamp of the current query range evaluation as the +number of seconds since January 1, 1970 UTC. For instant queries, this is equal +to the evaluation timestamp. + +## `double_exponential_smoothing()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`double_exponential_smoothing(v range-vector, sf scalar, tf scalar)` produces a +smoothed value for each float time series in the range in `v`. The lower the +smoothing factor `sf`, the more importance is given to old data. The higher the +trend factor `tf`, the more trends in the data is considered. Both `sf` and +`tf` must be between 0 and 1. For additional details, refer to [NIST +Engineering Statistics +Handbook](https://www.itl.nist.gov/div898/handbook/pmc/section4/pmc433.htm). In +Prometheus V2 this function was called `holt_winters`. This caused confusion +since the Holt-Winters method usually refers to triple exponential smoothing. +Double exponential smoothing as implemented here is also referred to as "Holt +Linear". + +`double_exponential_smoothing` should only be used with gauges and only works +for float samples. Elements in the range vector that contain only histogram +samples are ignored entirely. For elements that contain a mix of float and +histogram samples, only the float samples are used as input, which is flagged +by an info-level annotation. ## `exp()` -`exp(v instant-vector)` calculates the exponential function for all elements in `v`. -Special cases are: +`exp(v instant-vector)` calculates the exponential function for all float +samples in `v`. Histogram samples are ignored silently. Special cases are: * `Exp(+Inf) = +Inf` * `Exp(NaN) = NaN` ## `floor()` -`floor(v instant-vector)` rounds the sample values of all elements in `v` down -to the nearest integer value smaller than or equal to v. +`floor(v instant-vector)` returns a vector containing all float samples in the +input vector rounded down to the nearest integer value smaller than or equal +to their original value. Histogram samples in the input vector are ignored silently. * `floor(+Inf) = +Inf` * `floor(±0) = ±0` @@ -188,12 +228,8 @@ to the nearest integer value smaller than or equal to v. ## `histogram_avg()` -_This function only acts on native histograms, which are an experimental -feature. The behavior of this function may change in future versions of -Prometheus, including its removal from PromQL._ - -`histogram_avg(v instant-vector)` returns the arithmetic average of observed values stored in -a native histogram. Samples that are not native histograms are ignored and do +`histogram_avg(v instant-vector)` returns the arithmetic average of observed +values stored in each native histogram sample in `v`. Float samples are ignored and do not show up in the returned vector. Use `histogram_avg` as demonstrated below to compute the average request duration @@ -209,36 +245,42 @@ Which is equivalent to the following query: ## `histogram_count()` and `histogram_sum()` -_Both functions only act on native histograms, which are an experimental -feature. The behavior of these functions may change in future versions of -Prometheus, including their removal from PromQL._ - `histogram_count(v instant-vector)` returns the count of observations stored in -a native histogram. Samples that are not native histograms are ignored and do -not show up in the returned vector. +each native histogram sample in `v`. Float samples are ignored and do not show up in +the returned vector. Similarly, `histogram_sum(v instant-vector)` returns the sum of observations -stored in a native histogram. +stored in each native histogram sample. Use `histogram_count` in the following way to calculate a rate of observations -(in this case corresponding to “requests per second”) from a native histogram: +(in this case corresponding to “requests per second”) from a series of +histogram samples: histogram_count(rate(http_request_duration_seconds[10m])) ## `histogram_fraction()` -_This function only acts on native histograms, which are an experimental -feature. The behavior of this function may change in future versions of -Prometheus, including its removal from PromQL._ - -For a native histogram, `histogram_fraction(lower scalar, upper scalar, v -instant-vector)` returns the estimated fraction of observations between the -provided lower and upper values. Samples that are not native histograms are -ignored and do not show up in the returned vector. +`histogram_fraction(lower scalar, upper scalar, b instant-vector)` returns the +estimated fraction of observations between the provided lower and upper values +for each classic or native histogram contained in `b`. Float samples in `b` are +considered the counts of observations in each bucket of one or more classic +histograms, while native histogram samples in `b` are treated each individually +as a separate histogram. This works in the same way as for `histogram_quantile()`. +(See there for more details.) + +If the provided lower and upper values do not coincide with bucket boundaries, +the calculated fraction is an estimate, using the same interpolation method as for +`histogram_quantile()`. (See there for more details.) Especially with classic +histograms, it is easy to accidentally pick lower or upper values that are very +far away from any bucket boundary, leading to large margins of error. Rather than +using `histogram_fraction()` with classic histograms, it is often a more robust approach +to directly act on the bucket series when calculating fractions. See the +[calculation of the Apdex score](https://prometheus.io/docs/practices/histograms/#apdex-score) +as a typical example. For example, the following expression calculates the fraction of HTTP requests over the last hour that took 200ms or less: - + histogram_fraction(0, 0.2, rate(http_request_duration_seconds[1h])) The error of the estimation depends on the resolution of the underlying native @@ -253,12 +295,17 @@ observations less than or equal 0.2 would be `-Inf` rather than `0`. Whether the provided boundaries are inclusive or exclusive is only relevant if the provided boundaries are precisely aligned with bucket boundaries in the underlying native histogram. In this case, the behavior depends on the schema -definition of the histogram. The currently supported schemas all feature -inclusive upper boundaries and exclusive lower boundaries for positive values -(and vice versa for negative values). Without a precise alignment of -boundaries, the function uses linear interpolation to estimate the -fraction. With the resulting uncertainty, it becomes irrelevant if the -boundaries are inclusive or exclusive. +definition of the histogram. (The usual standard exponential schemas all +feature inclusive upper boundaries and exclusive lower boundaries for positive +values, and vice versa for negative values.) Without a precise alignment of +boundaries, the function uses interpolation to estimate the fraction. With the +resulting uncertainty, it becomes irrelevant if the boundaries are inclusive or +exclusive. + +Special case for native histograms with standard exponential buckets: +`NaN` observations are considered outside of any buckets in this case. +`histogram_fraction(-Inf, +Inf, b)` effectively returns the fraction of +non-`NaN` observations and may therefore be less than 1. ## `histogram_quantile()` @@ -270,10 +317,6 @@ summaries](https://prometheus.io/docs/practices/histograms) for a detailed explanation of φ-quantiles and the usage of the (classic) histogram metric type in general.) -_Note that native histograms are an experimental feature. The behavior of this -function when dealing with native histograms may change in future versions of -Prometheus._ - The float samples in `b` are considered the counts of observations in each bucket of one or more classic histograms. Each float sample must have a label `le` where the label value denotes the inclusive upper bound of the bucket. @@ -284,8 +327,8 @@ type](https://prometheus.io/docs/concepts/metric_types/#histogram) automatically provides time series with the `_bucket` suffix and the appropriate labels. -The native histogram samples in `b` are treated each individually as a separate -histogram to calculate the quantile from. +The (native) histogram samples in `b` are treated each individually as a +separate histogram to calculate the quantile from. As long as no naming collisions arise, `b` may contain a mix of classic and native histograms. @@ -313,7 +356,7 @@ included in the `by` clause. The following expression aggregates the 90th percentile by `job` for classic histograms: histogram_quantile(0.9, sum by (job, le) (rate(http_request_duration_seconds_bucket[10m]))) - + When aggregating native histograms, the expression simplifies to: histogram_quantile(0.9, sum by (job) (rate(http_request_duration_seconds[10m]))) @@ -326,92 +369,135 @@ With native histograms, aggregating everything works as usual without any `by` c histogram_quantile(0.9, sum(rate(http_request_duration_seconds[10m]))) -The `histogram_quantile()` function interpolates quantile values by -assuming a linear distribution within a bucket. +In the (common) case that a quantile value does not coincide with a bucket +boundary, the `histogram_quantile()` function interpolates the quantile value +within the bucket the quantile value falls into. For classic histograms, for +native histograms with custom bucket boundaries, and for the zero bucket of +other native histograms, it assumes a uniform distribution of observations +within the bucket (also called _linear interpolation_). For the +non-zero-buckets of native histograms with a standard exponential bucketing +schema, the interpolation is done under the assumption that the samples within +the bucket are distributed in a way that they would uniformly populate the +buckets in a hypothetical histogram with higher resolution. (This is also +called _exponential interpolation_. See the [native histogram +specification](https://prometheus.io/docs/specs/native_histograms/#interpolation-within-a-bucket) +for more details.) If `b` has 0 observations, `NaN` is returned. For φ < 0, `-Inf` is returned. For φ > 1, `+Inf` is returned. For φ = `NaN`, `NaN` is returned. -The following is only relevant for classic histograms: If `b` contains -fewer than two buckets, `NaN` is returned. The highest bucket must have an -upper bound of `+Inf`. (Otherwise, `NaN` is returned.) If a quantile is located -in the highest bucket, the upper bound of the second highest bucket is -returned. A lower limit of the lowest bucket is assumed to be 0 if the upper -bound of that bucket is greater than -0. In that case, the usual linear interpolation is applied within that -bucket. Otherwise, the upper bound of the lowest bucket is returned for -quantiles located in the lowest bucket. - -You can use `histogram_quantile(0, v instant-vector)` to get the estimated minimum value stored in -a histogram. - -You can use `histogram_quantile(1, v instant-vector)` to get the estimated maximum value stored in -a histogram. - -Buckets of classic histograms are cumulative. Therefore, the following should always be the case: - -* The counts in the buckets are monotonically increasing (strictly non-decreasing). -* A lack of observations between the upper limits of two consecutive buckets results in equal counts -in those two buckets. - -However, floating point precision issues (e.g. small discrepancies introduced by computing of buckets -with `sum(rate(...))`) or invalid data might violate these assumptions. In that case, -`histogram_quantile` would be unable to return meaningful results. To mitigate the issue, -`histogram_quantile` assumes that tiny relative differences between consecutive buckets are happening -because of floating point precision errors and ignores them. (The threshold to ignore a difference -between two buckets is a trillionth (1e-12) of the sum of both buckets.) Furthermore, if there are -non-monotonic bucket counts even after this adjustment, they are increased to the value of the -previous buckets to enforce monotonicity. The latter is evidence for an actual issue with the input -data and is therefore flagged with an informational annotation reading `input to histogram_quantile -needed to be fixed for monotonicity`. If you encounter this annotation, you should find and remove -the source of the invalid data. +Special cases for classic histograms: + +* If `b` contains fewer than two buckets, `NaN` is returned. +* The highest bucket must have an upper bound of `+Inf`. (Otherwise, `NaN` is + returned.) +* If a quantile is located in the highest bucket, the upper bound of the second + highest bucket is returned. +* The lower limit of the lowest bucket is assumed to be 0 if the upper bound of + that bucket is greater than 0. In that case, the usual linear interpolation + is applied within that bucket. Otherwise, the upper bound of the lowest + bucket is returned for quantiles located in the lowest bucket. + +Special cases for native histograms: + +* If a native histogram with standard exponential buckets has `NaN` + observations and the quantile falls into one of the existing exponential + buckets, the result is skewed towards higher values due to `NaN` + observations treated as `+Inf`. This is flagged with an info level + annotation. +* If a native histogram with standard exponential buckets has `NaN` + observations and the quantile falls above all of the existing exponential + buckets, `NaN` is returned. This is flagged with an info level annotation. +* A zero bucket with finite width is assumed to contain no negative + observations if the histogram has observations in positive buckets, but none + in negative buckets. +* A zero bucket with finite width is assumed to contain no positive + observations if the histogram has observations in negative buckets, but none + in positive buckets. + +You can use `histogram_quantile(0, v instant-vector)` to get the estimated +minimum value stored in a histogram. + +You can use `histogram_quantile(1, v instant-vector)` to get the estimated +maximum value stored in a histogram. + +Buckets of classic histograms are cumulative. Therefore, the following should +always be the case: + +* The counts in the buckets are monotonically increasing (strictly + non-decreasing). +* A lack of observations between the upper limits of two consecutive buckets + results in equal counts in those two buckets. + +However, floating point precision issues (e.g. small discrepancies introduced +by computing of buckets with `sum(rate(...))`) or invalid data might violate +these assumptions. In that case, `histogram_quantile` would be unable to return +meaningful results. To mitigate the issue, `histogram_quantile` assumes that +tiny relative differences between consecutive buckets are happening because of +floating point precision errors and ignores them. (The threshold to ignore a +difference between two buckets is a trillionth (1e-12) of the sum of both +buckets.) Furthermore, if there are non-monotonic bucket counts even after this +adjustment, they are increased to the value of the previous buckets to enforce +monotonicity. The latter is evidence for an actual issue with the input data +and is therefore flagged by an info-level annotation reading `input to +histogram_quantile needed to be fixed for monotonicity`. If you encounter this +annotation, you should find and remove the source of the invalid data. + +## `histogram_quantiles()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`histogram_quantiles(v instant-vector, quantile_label string, φ_1 scalar, φ_2 scalar, ...)` calculates multiple (between 1 and 10) φ-quantiles (0 ≤ +φ ≤ 1) from a [classic +histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) or from +a native histogram. Quantile calculation works the same way as in `histogram_quantile()`. +The second argument (a string) specifies the label name that is used to identify different quantiles in the query result. +``` +histogram_quantiles(sum(rate(foo[1m])), "quantile", 0.9, 0.99) +# => {quantile="0.9"} 123 + {quantile="0.99"} 128 +``` ## `histogram_stddev()` and `histogram_stdvar()` -_Both functions only act on native histograms, which are an experimental -feature. The behavior of these functions may change in future versions of -Prometheus, including their removal from PromQL._ - `histogram_stddev(v instant-vector)` returns the estimated standard deviation -of observations in a native histogram, based on the geometric mean of the buckets -where the observations lie. Samples that are not native histograms are ignored and -do not show up in the returned vector. - -Similarly, `histogram_stdvar(v instant-vector)` returns the estimated standard -variance of observations in a native histogram. - -## `holt_winters()` +of observations for each native histogram sample in `v`. For this estimation, all observations +in a bucket are assumed to have the value of the mean of the bucket boundaries. For +the zero bucket and for buckets with custom boundaries, the arithmetic mean is used. +For the usual exponential buckets, the geometric mean is used. Float samples are ignored +and do not show up in the returned vector. -`holt_winters(v range-vector, sf scalar, tf scalar)` produces a smoothed value -for time series based on the range in `v`. The lower the smoothing factor `sf`, -the more importance is given to old data. The higher the trend factor `tf`, the -more trends in the data is considered. Both `sf` and `tf` must be between 0 and -1. - -`holt_winters` should only be used with gauges. +Similarly, `histogram_stdvar(v instant-vector)` returns the estimated +variance of observations for each native histogram sample in `v`. ## `hour()` -`hour(v=vector(time()) instant-vector)` returns the hour of the day -for each of the given times in UTC. Returned values are from 0 to 23. +`hour(v=vector(time()) instant-vector)` interprets float samples in `v` as +timestamps (number of seconds since January 1, 1970 UTC) and returns the hour +of the day (in UTC) for each of those timestamps. Returned values are from 0 +to 23. Histogram samples in the input vector are ignored silently. ## `idelta()` `idelta(v range-vector)` calculates the difference between the last two samples in the range vector `v`, returning an instant vector with the given deltas and -equivalent labels. +equivalent labels. Both samples must be either float samples or histogram +samples. Elements in `v` where one of the last two samples is a float sample +and the other is a histogram sample will be omitted from the result vector, +flagged by a warn-level annotation. -`idelta` should only be used with gauges. +`idelta` should only be used with gauges (for both floats and histograms). ## `increase()` -`increase(v range-vector)` calculates the increase in the -time series in the range vector. Breaks in monotonicity (such as counter -resets due to target restarts) are automatically adjusted for. The -increase is extrapolated to cover the full time range as specified -in the range vector selector, so that it is possible to get a -non-integer result even if a counter increases only by integer -increments. +`increase(v range-vector)` calculates the increase in the time series in the +range vector. Breaks in monotonicity (such as counter resets due to target +restarts) are automatically adjusted for. The increase is extrapolated to cover +the full time range as specified in the range vector selector, so that it is +possible to get a non-integer result even if a counter increases only by +integer increments. The following example expression returns the number of HTTP requests as measured over the last 5 minutes, per time series in the range vector: @@ -420,27 +506,28 @@ over the last 5 minutes, per time series in the range vector: increase(http_requests_total{job="api-server"}[5m]) ``` -`increase` acts on native histograms by calculating a new histogram where each -component (sum and count of observations, buckets) is the increase between -the respective component in the first and last native histogram in -`v`. However, each element in `v` that contains a mix of float and native -histogram samples within the range, will be missing from the result vector. +`increase` acts on histogram samples by calculating a new histogram where each +component (sum and count of observations, buckets) is the increase between the +respective component in the first and last native histogram in `v`. However, +each element in `v` that contains a mix of float samples and histogram samples +within the range, will be omitted from the result vector, flagged by a +warn-level annotation. -`increase` should only be used with counters and native histograms where the -components behave like counters. It is syntactic sugar for `rate(v)` multiplied -by the number of seconds under the specified time range window, and should be -used primarily for human readability. Use `rate` in recording rules so that -increases are tracked consistently on a per-second basis. +`increase` should only be used with counters (for both floats and histograms). +It is syntactic sugar for `rate(v)` multiplied by the number of seconds under +the specified time range window, and should be used primarily for human +readability. Use `rate` in recording rules so that increases are tracked +consistently on a per-second basis. -## `info()` (experimental) +## `info()` _The `info` function is an experiment to improve UX around including labels from [info metrics](https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/#info-metrics). The behavior of this function may change in future versions of Prometheus, -including its removal from PromQL. `info` has to be enabled via the +including its removal from PromQL. `info` has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`._ -`info(v instant-vector, [data-label-selector instant-vector])` finds, for each time +`info(v instant-vector, [data-label-selector instant-vector])` finds, for each time series in `v`, all info series with matching _identifying_ labels (more on this later), and adds the union of their _data_ (i.e., non-identifying) labels to the time series. The second argument `data-label-selector` is optional. @@ -449,6 +536,15 @@ It must start and end with curly braces (`{ ... }`) and may only contain label m The label matchers are used to constrain which info series to consider and which data labels to add to `v`. +If there is no matching info series for a given time series in `v` at a +particular timestamp (e.g. because the info series has gone stale), the +behavior depends on the data label matchers: If the `data-label-selector` +contains any matcher that does not match the empty string (e.g. +`{data=~".+"}`), then that time series is dropped from the result at that +timestamp, because the required enrichment is unavailable. If all matchers +match the empty string (e.g. `{data=~".*"}`), or if no `data-label-selector` +is provided, the time series is returned without enrichment. + Identifying labels of an info series are the subset of labels that uniquely identify the info series. The remaining labels are considered _data labels_ (also called non-identifying). (Note that Prometheus's concept @@ -457,8 +553,8 @@ function, we “logically” define info series identity in a different way than in the conventional Prometheus view.) The identifying labels of an info series are used to join it to regular (non-info) series, i.e. those series that have the same labels as the identifying labels of the info series. The data labels, which are -the ones added to the regular series by the `info` function, effectively encode -metadata key value pairs. (This implies that a change in the data labels +the ones added to the regular series by the `info` function, effectively encode +metadata key value pairs. (This implies that a change in the data labels in the conventional Prometheus view constitutes the end of one info series and the beginning of a new info series, while the “logical” view of the `info` function is that the same info series continues to exist, just with different “data”.) @@ -482,7 +578,7 @@ This query is not only verbose and hard to write, it might also run into an “i If any of the data labels of `target_info` changes, Prometheus sees that as a change of series (as alluded to above, Prometheus just has no native concept of non-identifying labels). If the old `target_info` series is not properly marked as stale (which can happen with certain ingestion paths), -the query above will fail for up to 5m (the lookback delta) because it will find a conflicting +the query above will fail for up to 5m (the lookback delta) because it will find a conflicting match with both the old and the new version of `target_info`. The `info` function not only resolves this conflict in favor of the newer series, it also simplifies the syntax @@ -508,16 +604,24 @@ While `info` normally automatically finds all matching info series, it's possibl restrict them by providing a `__name__` label matcher, e.g. `{__name__="target_info"}`. +Note that if there are any time series in `v` that match the `data-label-selector` (or the default `target_info` if that argument is not specified), they will be treated as info series and will be returned unchanged. + ### Limitations In its current iteration, `info` defaults to considering only info series with the name `target_info`. It also assumes that the identifying info series labels are `instance` and `job`. `info` does support other info series names however, through -`__name__` label matchers. E.g., one can explicitly say to consider both +`__name__` label matchers. E.g., one can explicitly say to consider both `target_info` and `build_info` as follows: `{__name__=~"(target|build)_info"}`. However, the identifying labels always have to be `instance` and `job`. +When only negated `__name__` matchers are provided (e.g. +`{__name__!="target_info"}`), `info` considers all metrics matching +`.+_info` and then applies the negated matchers as filters. This is +because negated matchers alone cannot positively identify which info +metrics to consider. + These limitations are partially defeating the purpose of the `info` function. At the current stage, this is an experiment to find out how useful the approach turns out to be in practice. A final version of the `info` function will indeed @@ -528,7 +632,12 @@ consider all matching info series and with their appropriate identifying labels. `irate(v range-vector)` calculates the per-second instant rate of increase of the time series in the range vector. This is based on the last two data points. Breaks in monotonicity (such as counter resets due to target restarts) are -automatically adjusted for. +automatically adjusted for. Both samples must be either float samples or +histogram samples. Elements in `v` where one of the last two samples is a float +sample and the other is a histogram sample will be omitted from the result +vector, flagged by a warn-level annotation. + +`irate` should only be used with counters (for both floats and histograms). The following example expression returns the per-second rate of HTTP requests looking up to 5 minutes back for the two most recent data points, per time @@ -546,7 +655,7 @@ spikes are hard to read. Note that when combining `irate()` with an [aggregation operator](operators.md#aggregation-operators) (e.g. `sum()`) or a function aggregating over time (any function ending in `_over_time`), -always take a `irate()` first, then aggregate. Otherwise `irate()` cannot detect +always take an `irate()` first, then aggregate. Otherwise `irate()` cannot detect counter resets when your target restarts. ## `label_join()` @@ -566,7 +675,7 @@ label_join(up{job="api-server",src1="a",src2="b",src3="c"}, "foo", ",", "src1", ## `label_replace()` For each timeseries in `v`, `label_replace(v instant-vector, dst_label string, replacement string, src_label string, regex string)` -matches the [regular expression](https://github.com/google/re2/wiki/Syntax) `regex` against the value of the label `src_label`. If it +matches the [regular expression](./basics.md#regular-expressions) `regex` against the value of the label `src_label`. If it matches, the value of the label `dst_label` in the returned timeseries will be the expansion of `replacement`, together with the original labels in the input. Capturing groups in the regular expression can be referenced with `$1`, `$2`, etc. Named capturing groups in the regular expression can be referenced with `$name` (where `name` is the capturing group name). If the regular expression doesn't match then the timeseries is returned unchanged. @@ -584,10 +693,28 @@ This second example has the same effect than the first example, and illustrates label_replace(up{job="api-server",service="a:c"}, "foo", "$name", "service", "(?P.*):(?P.*)") ``` +## `max_of()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`max_of(a scalar, b scalar)` returns the larger of the two scalar values `a` +and `b`. + +## `min_of()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`min_of(a scalar, b scalar)` returns the smaller of the two scalar values `a` +and `b`. + ## `ln()` -`ln(v instant-vector)` calculates the natural logarithm for all elements in `v`. -Special cases are: +`ln(v instant-vector)` calculates the natural logarithm for all float samples +in `v`. Histogram samples in the input vector are ignored silently. Special cases are: * `ln(+Inf) = +Inf` * `ln(0) = -Inf` @@ -596,35 +723,54 @@ Special cases are: ## `log2()` -`log2(v instant-vector)` calculates the binary logarithm for all elements in `v`. -The special cases are equivalent to those in `ln`. +`log2(v instant-vector)` calculates the binary logarithm for all float samples +in `v`. Histogram samples in the input vector are ignored silently. The special cases +are equivalent to those in `ln`. ## `log10()` -`log10(v instant-vector)` calculates the decimal logarithm for all elements in `v`. -The special cases are equivalent to those in `ln`. +`log10(v instant-vector)` calculates the decimal logarithm for all float +samples in `v`. Histogram samples in the input vector are ignored silently. The special +cases are equivalent to those in `ln`. ## `minute()` -`minute(v=vector(time()) instant-vector)` returns the minute of the hour for each -of the given times in UTC. Returned values are from 0 to 59. +`minute(v=vector(time()) instant-vector)` interprets float samples in `v` as +timestamps (number of seconds since January 1, 1970 UTC) and returns the minute +of the hour (in UTC) for each of those timestamps. Returned values are from 0 +to 59. Histogram samples in the input vector are ignored silently. ## `month()` -`month(v=vector(time()) instant-vector)` returns the month of the year for each -of the given times in UTC. Returned values are from 1 to 12, where 1 means -January etc. +`month(v=vector(time()) instant-vector)` interprets float samples in `v` as +timestamps (number of seconds since January 1, 1970 UTC) and returns the month +of the year (in UTC) for each of those timestamps. Returned values are from 1 +to 12, where 1 means January etc. Histogram samples in the input vector are +ignored silently. ## `predict_linear()` `predict_linear(v range-vector, t scalar)` predicts the value of time series `t` seconds from now, based on the range vector `v`, using [simple linear -regression](https://en.wikipedia.org/wiki/Simple_linear_regression). -The range vector must have at least two samples in order to perform the -calculation. When `+Inf` or `-Inf` are found in the range vector, -the slope and offset value calculated will be `NaN`. +regression](https://en.wikipedia.org/wiki/Simple_linear_regression). The range +vector must have at least two float samples in order to perform the +calculation. When `+Inf` or `-Inf` are found in the range vector, the predicted +value will be `NaN`. -`predict_linear` should only be used with gauges. +`predict_linear` should only be used with gauges and only works for float +samples. Elements in the range vector that contain only histogram samples are +ignored entirely. For elements that contain a mix of float and histogram +samples, only the float samples are used as input, which is flagged by an +info-level annotation. + +## `range()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`range()` returns the range duration of the current query range evaluation in +seconds and is equivalent to `end() - start()`. For instant queries, this returns `0`. ## `rate()` @@ -634,7 +780,7 @@ resets due to target restarts) are automatically adjusted for. Also, the calculation extrapolates to the ends of the time range, allowing for missed scrapes or imperfect alignment of scrape cycles with the range's time period. -The following example expression returns the per-second rate of HTTP requests as measured +The following example expression returns the per-second average rate of HTTP requests over the last 5 minutes, per time series in the range vector: ``` @@ -643,13 +789,13 @@ rate(http_requests_total{job="api-server"}[5m]) `rate` acts on native histograms by calculating a new histogram where each component (sum and count of observations, buckets) is the rate of increase -between the respective component in the first and last native histogram in -`v`. However, each element in `v` that contains a mix of float and native -histogram samples within the range, will be missing from the result vector. +between the respective component in the first and last native histogram in `v`. +However, each element in `v` that contains a mix of float and native histogram +samples within the range, will be omitted from the result vector, flagged by a +warn-level annotation. -`rate` should only be used with counters and native histograms where the -components behave like counters. It is best suited for alerting, and for -graphing of slow-moving counters. +`rate` should only be used with counters (for both floats and histograms). It +is best suited for alerting, and for graphing of slow-moving counters. Note that when combining `rate()` with an aggregation operator (e.g. `sum()`) or a function aggregating over time (any function ending in `_over_time`), @@ -664,17 +810,15 @@ decrease in the value between two consecutive float samples is interpreted as a counter reset. A reset in a native histogram is detected in a more complex way: Any decrease in any bucket, including the zero bucket, or in the count of observation constitutes a counter reset, but also the disappearance of any -previously populated bucket, an increase in bucket resolution, or a decrease of -the zero-bucket width. +previously populated bucket, a decrease of the zero-bucket width, or any schema +change that is not a compatible decrease of resolution. -`resets` should only be used with counters and counter-like native -histograms. +`resets` should only be used with counters (for both floats and histograms). -If the range vector contains a mix of float and histogram samples for the same -series, counter resets are detected separately and their numbers added up. The -change from a float to a histogram sample is _not_ considered a counter -reset. Each float sample is compared to the next float sample, and each -histogram is comprared to the next histogram. +A float sample followed by a histogram sample, or vice versa, counts as a +reset. A counter histogram sample followed by a gauge histogram sample, or vice +versa, also counts as a reset (but note that `resets` should not be used on +gauges in the first place, see above). ## `round()` @@ -682,53 +826,82 @@ histogram is comprared to the next histogram. elements in `v` to the nearest integer. Ties are resolved by rounding up. The optional `to_nearest` argument allows specifying the nearest multiple to which the sample values should be rounded. This multiple may also be a fraction. +Histogram samples in the input vector are ignored silently. ## `scalar()` -Given a single-element input vector, `scalar(v instant-vector)` returns the -sample value of that single element as a scalar. If the input vector does not -have exactly one element, `scalar` will return `NaN`. +Given an input vector that contains only one element with a float sample, +`scalar(v instant-vector)` returns the sample value of that float sample as a +scalar. If the input vector does not have exactly one element with a float +sample, `scalar` will return `NaN`. Histogram samples in the input vector are +ignored silently. ## `sgn()` -`sgn(v instant-vector)` returns a vector with all sample values converted to their sign, defined as this: 1 if v is positive, -1 if v is negative and 0 if v is equal to zero. +`sgn(v instant-vector)` returns a vector with all float sample values converted +to their sign, defined as this: 1 if v is positive, -1 if v is negative and 0 +if v is equal to zero. Histogram samples in the input vector are ignored silently. ## `sort()` -`sort(v instant-vector)` returns vector elements sorted by their sample values, -in ascending order. Native histograms are sorted by their sum of observations. +`sort(v instant-vector)` returns vector elements sorted by their float sample +values, in ascending order. Histogram samples in the input vector are ignored silently. -Please note that `sort` only affects the results of instant queries, as range query results always have a fixed output ordering. +Please note that `sort` only affects the results of instant queries, as range +query results always have a fixed output ordering. ## `sort_desc()` Same as `sort`, but sorts in descending order. -Like `sort`, `sort_desc` only affects the results of instant queries, as range query results always have a fixed output ordering. - ## `sort_by_label()` -**This function has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`.** +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** -`sort_by_label(v instant-vector, label string, ...)` returns vector elements sorted by the values of the given labels in ascending order. In case these label values are equal, elements are sorted by their full label sets. +`sort_by_label(v instant-vector, label string, ...)` returns vector elements +sorted by the values of the given labels in ascending order. In case these +label values are equal, elements are sorted by their full label sets. +`sort_by_label` acts on float and histogram samples in the same way. -Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering. +Please note that `sort_by_label` only affects the results of instant queries, as +range query results always have a fixed output ordering. -This function uses [natural sort order](https://en.wikipedia.org/wiki/Natural_sort_order). +`sort_by_label` uses [natural sort +order](https://en.wikipedia.org/wiki/Natural_sort_order). ## `sort_by_label_desc()` -**This function has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`.** +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** Same as `sort_by_label`, but sorts in descending order. -Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering. +## `sqrt()` + +`sqrt(v instant-vector)` calculates the square root of all float samples in +`v`. Histogram samples in the input vector are ignored silently. -This function uses [natural sort order](https://en.wikipedia.org/wiki/Natural_sort_order). +## `start()` -## `sqrt()` +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`start()` returns the start timestamp of the current query range evaluation as the +number of seconds since January 1, 1970 UTC. For instant queries, this is equal +to the evaluation timestamp. + +## `step()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** -`sqrt(v instant-vector)` calculates the square root of all elements in `v`. +`step()` returns the query resolution step as the number of seconds. For instant +queries, this returns `0`. ## `time()` @@ -739,66 +912,96 @@ expression is to be evaluated. ## `timestamp()` `timestamp(v instant-vector)` returns the timestamp of each of the samples of -the given vector as the number of seconds since January 1, 1970 UTC. It also -works with histogram samples. +the given vector as the number of seconds since January 1, 1970 UTC. It acts on +float and histogram samples in the same way. ## `vector()` -`vector(s scalar)` returns the scalar `s` as a vector with no labels. +`vector(s scalar)` converts the scalar `s` to a float sample and returns it as +a single-element instant vector with no labels. ## `year()` -`year(v=vector(time()) instant-vector)` returns the year -for each of the given times in UTC. +`year(v=vector(time()) instant-vector)` returns the year for each of the given +times in UTC. Histogram samples in the input vector are ignored silently. ## `_over_time()` The following functions allow aggregating each series of a given range vector over time and return an instant vector with per-series aggregation results: -* `avg_over_time(range-vector)`: the average value of all points in the specified interval. -* `min_over_time(range-vector)`: the minimum value of all points in the specified interval. -* `max_over_time(range-vector)`: the maximum value of all points in the specified interval. -* `sum_over_time(range-vector)`: the sum of all values in the specified interval. -* `count_over_time(range-vector)`: the count of all values in the specified interval. -* `quantile_over_time(scalar, range-vector)`: the φ-quantile (0 ≤ φ ≤ 1) of the values in the specified interval. -* `stddev_over_time(range-vector)`: the population standard deviation of the values in the specified interval. -* `stdvar_over_time(range-vector)`: the population standard variance of the values in the specified interval. -* `last_over_time(range-vector)`: the most recent point value in the specified interval. +* `avg_over_time(range-vector)`: the average value of all float or histogram samples in the specified interval (see details below). +* `min_over_time(range-vector)`: the minimum value of all float samples in the specified interval. +* `max_over_time(range-vector)`: the maximum value of all float samples in the specified interval. +* `sum_over_time(range-vector)`: the sum of all float or histogram samples in the specified interval (see details below). +* `count_over_time(range-vector)`: the count of all samples in the specified interval. +* `quantile_over_time(scalar, range-vector)`: the φ-quantile (0 ≤ φ ≤ 1) of all float samples in the specified interval. +* `stddev_over_time(range-vector)`: the population standard deviation of all float samples in the specified interval. +* `stdvar_over_time(range-vector)`: the population variance of all float samples in the specified interval. +* `last_over_time(range-vector)`: the most recent sample in the specified interval. * `present_over_time(range-vector)`: the value 1 for any series in the specified interval. If the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions` is set, the following additional functions are available: -* `mad_over_time(range-vector)`: the median absolute deviation of all points in the specified interval. +* `mad_over_time(range-vector)`: the median absolute deviation of all float + samples in the specified interval. +* `ts_of_min_over_time(range-vector)`: the timestamp of the last float sample + that has the minimum value of all float samples in the specified interval. +* `ts_of_max_over_time(range-vector)`: the timestamp of the last float sample + that has the maximum value of all float samples in the specified interval. +* `ts_of_last_over_time(range-vector)`: the timestamp of last sample in the + specified interval. +* `first_over_time(range-vector)`: the oldest sample in the specified interval. +* `ts_of_first_over_time(range-vector)`: the timestamp of earliest sample in the + specified interval. Note that all values in the specified interval have the same weight in the aggregation even if the values are not equally spaced throughout the interval. -`avg_over_time`, `sum_over_time`, `count_over_time`, `last_over_time`, and -`present_over_time` handle native histograms as expected. All other functions -ignore histogram samples. +These functions act on histograms in the following way: + +- `count_over_time`, `first_over_time`, `last_over_time`, and + `present_over_time()` act on float and histogram samples in the same way. +- `avg_over_time()` and `sum_over_time()` act on histogram samples in a way + that corresponds to the respective aggregation operators. If a series + contains a mix of float samples and histogram samples within the range, the + corresponding result is removed entirely from the output vector. Such a + removal is flagged by a warn-level annotation. +- All other functions ignore histogram samples in the following way: Input + ranges containing only histogram samples are silently removed from the + output. For ranges with a mix of histogram and float samples, only the float + samples are processed and the omission of the histogram samples is flagged by + an info-level annotation. + +`first_over_time(m[1m])` differs from `m offset 1m` in that the former will +select the first sample of `m` _within_ the 1m range, where `m offset 1m` will +select the most recent sample within the lookback interval _outside and prior +to_ the 1m offset. This is particularly useful with `first_over_time(m[step()])` +in range queries (available when `--enable-feature=promql-duration-expr` is set) +to ensure that the sample selected is within the range step. ## Trigonometric Functions -The trigonometric functions work in radians: - -* `acos(v instant-vector)`: calculates the arccosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acos)). -* `acosh(v instant-vector)`: calculates the inverse hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acosh)). -* `asin(v instant-vector)`: calculates the arcsine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asin)). -* `asinh(v instant-vector)`: calculates the inverse hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asinh)). -* `atan(v instant-vector)`: calculates the arctangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atan)). -* `atanh(v instant-vector)`: calculates the inverse hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atanh)). -* `cos(v instant-vector)`: calculates the cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cos)). -* `cosh(v instant-vector)`: calculates the hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cosh)). -* `sin(v instant-vector)`: calculates the sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sin)). -* `sinh(v instant-vector)`: calculates the hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sinh)). -* `tan(v instant-vector)`: calculates the tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tan)). -* `tanh(v instant-vector)`: calculates the hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tanh)). +The trigonometric functions work in radians. They ignore histogram samples in +the input vector. + +* `acos(v instant-vector)`: calculates the arccosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Acos)). +* `acosh(v instant-vector)`: calculates the inverse hyperbolic cosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Acosh)). +* `asin(v instant-vector)`: calculates the arcsine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Asin)). +* `asinh(v instant-vector)`: calculates the inverse hyperbolic sine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Asinh)). +* `atan(v instant-vector)`: calculates the arctangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Atan)). +* `atanh(v instant-vector)`: calculates the inverse hyperbolic tangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Atanh)). +* `cos(v instant-vector)`: calculates the cosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Cos)). +* `cosh(v instant-vector)`: calculates the hyperbolic cosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Cosh)). +* `sin(v instant-vector)`: calculates the sine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Sin)). +* `sinh(v instant-vector)`: calculates the hyperbolic sine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Sinh)). +* `tan(v instant-vector)`: calculates the tangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Tan)). +* `tanh(v instant-vector)`: calculates the hyperbolic tangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Tanh)). The following are useful for converting between degrees and radians: -* `deg(v instant-vector)`: converts radians to degrees for all elements in `v`. +* `deg(v instant-vector)`: converts radians to degrees for all float samples in `v`. * `pi()`: returns pi. -* `rad(v instant-vector)`: converts degrees to radians for all elements in `v`. +* `rad(v instant-vector)`: converts degrees to radians for all float samples in `v`. diff --git a/docs/querying/operators.md b/docs/querying/operators.md index f5f217ff636..82b0be78db0 100644 --- a/docs/querying/operators.md +++ b/docs/querying/operators.md @@ -3,17 +3,33 @@ title: Operators sort_rank: 2 --- -# Operators +PromQL supports unary, binary, and aggregation operators. + +## Unary operator + +The only unary operator in PromQL is `-` (unary minus). It can be applied to a +scalar or an instant vector. In the former case, it returns a scalar with +inverted sign. In the latter case, it returns an instant vector with inverted +sign for each element. The sign of a histogram sample is inverted by inverting +the sign of all bucket populations and the count and the sum of observations. +The resulting histogram sample is always considered a gauge histogram. + +NOTE: A histogram with any negative bucket population or a negative count of +observations should only be used as an intermediate result. If such a negative +histogram is the final outcome of a recording rule, the rule evaluation will +fail. Negative histograms cannot be represented by any of the exchange formats +(exposition, remote-write, OTLP), so they cannot ingested into Prometheus in +any way and are only created by PromQL expressions. ## Binary operators -Prometheus's query language supports basic logical and arithmetic operators. -For operations between two instant vectors, the [matching behavior](#vector-matching) -can be modified. +Binary operators cover basic logical and arithmetic operations. For operations +between two instant vectors, the [matching behavior](#vector-matching) can be +modified. ### Arithmetic binary operators -The following binary arithmetic operators exist in Prometheus: +The following binary arithmetic operators exist in PromQL: * `+` (addition) * `-` (subtraction) @@ -23,22 +39,79 @@ The following binary arithmetic operators exist in Prometheus: * `^` (power/exponentiation) Binary arithmetic operators are defined between scalar/scalar, vector/scalar, -and vector/vector value pairs. +and vector/vector value pairs. They follow the usual [IEEE 754 floating point +arithmetic](https://en.wikipedia.org/wiki/IEEE_754), including the handling of +special values like `NaN`, `+Inf`, and `-Inf`. -**Between two scalars**, the behavior is obvious: they evaluate to another +**Between two scalars**, the behavior is straightforward: they evaluate to another scalar that is the result of the operator applied to both scalar operands. **Between an instant vector and a scalar**, the operator is applied to the -value of every data sample in the vector. E.g. if a time series instant vector -is multiplied by 2, the result is another vector in which every sample value of -the original vector is multiplied by 2. The metric name is dropped. +value of every data sample in the vector. + +If the data sample is a float, the operation is performed between that float and the scalar. +For example, if an instant vector of float samples is multiplied by 2, +the result is another vector of float samples in which every sample value of +the original vector is multiplied by 2. + +For vector elements that are histogram samples, the behavior is the +following: + +* For `*`, all bucket populations and the count and the sum of observations are + multiplied by the scalar. If the scalar is negative, the resulting histogram + is considered a gauge histogram. Otherwise, the counter vs. gauge flavor of + the input histogram sample is retained. + +* For `/`, the histogram sample has to be on the left hand side (LHS), followed + by the scalar on the right hand side (RHS). All bucket populations and the + count and the sum of observations are then divided by the scalar. A division + by zero results in a histogram with no regular buckets and the zero bucket + population and the count and sum of observations all set to `+Inf`, `-Inf`, + or `NaN`, depending on their values in the input histogram (positive, + negative, or zero/`NaN`, respectively). If the scalar is negative, the + resulting histogram is considered a gauge histogram. Otherwise, the counter + vs. gauge flavor of the input histogram sample is retained. + +* For `/` with a scalar on the LHS and a histogram sample on the RHS, and + similarly for all other arithmetic binary operators in any combination of a + scalar and a histogram sample, there is no result and the corresponding + element is removed from the resulting vector. Such a removal is flagged by an + info-level annotation. **Between two instant vectors**, a binary arithmetic operator is applied to -each entry in the left-hand side vector and its [matching element](#vector-matching) -in the right-hand vector. The result is propagated into the result vector with the -grouping labels becoming the output label set. The metric name is dropped. Entries -for which no matching entry in the right-hand vector can be found are not part of -the result. +each entry in the LHS vector and its [matching element](#vector-matching) in +the RHS vector. The result is propagated into the result vector with the +grouping labels becoming the output label set. By default, series for which +no matching entry in the opposite vector can be found are not part of the +result. This behavior can be adjusted using [fill modifiers](#filling-in-missing-matches). + +If two float samples are matched, the arithmetic operator is applied to the two +input values. + +If a float sample is matched with a histogram sample, the behavior follows the +same logic as between a scalar and a histogram sample (see above), i.e. `*` and +`/` (the latter with the histogram sample on the LHS) are valid operations, +while all others lead to the removal of the corresponding element from the +resulting vector. + +If two histogram samples are matched, only `+` and `-` are valid operations, +each adding or subtracting all matching bucket populations and the count and +the sum of observations. All other operations result in the removal of the +corresponding element from the output vector, flagged by an info-level +annotation. The `+` and `-` operations should generally only be applied to gauge +histograms, but PromQL allows them for counter histograms, too, to cover +specific use cases, for which special attention is required to avoid problems +with unaligned counter resets. (Certain incompatibilities of counter resets can +be detected by PromQL and are flagged with a warn-level annotations.) Adding +two counter histograms results in a counter histogram. All other combination of +operands and all subtractions result in a gauge histogram. + +**In any arithmetic binary operation involving vectors**, the metric name is +dropped. This occurs even if `__name__` is explicitly mentioned in `on` +(see https://github.com/prometheus/prometheus/issues/16631 for further discussion). + +**For any arithmetic binary operation that may result in a negative +histogram**, take into account the [respective note above](#unary-operator). ### Trigonometric binary operators @@ -46,9 +119,31 @@ The following trigonometric binary operators, which work in radians, exist in Pr * `atan2` (based on https://pkg.go.dev/math#Atan2) -Trigonometric operators allow trigonometric functions to be executed on two vectors using -vector matching, which isn't available with normal functions. They act in the same manner -as arithmetic operators. +Trigonometric operators allow trigonometric functions to be executed on two +vectors using vector matching, which isn't available with normal functions. +They act in the same manner as arithmetic operators. They only operate on float +samples. Operations involving histogram samples result in the removal of the +corresponding vector elements from the output vector, flagged by an +info-level annotation. + +### Histogram trim operators + +The following binary histogram trim operators exist in Prometheus: + +* `/` (trim lower): removes all observations below a threshold value + +Histogram trim operators are defined between vector/scalar and vector/vector value pairs, +where the left hand side is a native histogram (either exponential or NHCB), +and the right hand side is a float threshold value. + +In case the threshold value is not aligned to one of the bucket boundaries of the histogram, +either linear (for NHCB and zero buckets of exponential histogram) or exponential (for non zero +bucket of exponential histogram) interpolation is applied to compute the estimated count +of observations that remain in the bucket containing the threshold. + +In case when some observations get trimmed, the new sum of observation values is recomputed +(approximately) based on the remaining observations. ### Comparison binary operators @@ -72,20 +167,42 @@ operators result in another scalar that is either `0` (`false`) or `1` **Between an instant vector and a scalar**, these operators are applied to the value of every data sample in the vector, and vector elements between which the -comparison result is `false` get dropped from the result vector. If the `bool` -modifier is provided, vector elements that would be dropped instead have the value -`0` and vector elements that would be kept have the value `1`. The metric name -is dropped if the `bool` modifier is provided. +comparison result is false get dropped from the result vector. These +operations only work with float samples in the vector. For histogram samples, +the corresponding element is removed from the result vector, flagged by an +info-level annotation. **Between two instant vectors**, these operators behave as a filter by default, applied to matching entries. Vector elements for which the expression is not true or which do not find a match on the other side of the expression get dropped from the result, while the others are propagated into a result vector with the grouping labels becoming the output label set. -If the `bool` modifier is provided, vector elements that would have been -dropped instead have the value `0` and vector elements that would be kept have -the value `1`, with the grouping labels again becoming the output label set. -The metric name is dropped if the `bool` modifier is provided. + +Matches between two float samples work as usual. + +Matches between a float sample and a histogram sample are invalid, and the +corresponding element is removed from the result vector, flagged by an info-level +annotation. + +Between two histogram samples, `==` and `!=` work as expected, but all other +comparison binary operations are again invalid. + +**In any comparison binary operation involving vectors**, providing the `bool` +modifier changes the behavior in the following ways: + +* Vector elements which find a match on the other side of the expression but for + which the expression is false instead have the value `0`, and vector elements + that do find a match and for which the expression is true have the value `1`. + (Note that elements with no match or invalid operations involving histogram + samples still return no result rather than the value `0`.) +* The metric name is dropped. + +If the `bool` modifier is not provided, then the metric name from the left side +is retained, with some exceptions: + +* If `on` is used, then the metric name is dropped. +* If `group_right` is used, then the metric name from the right side is retained, + to avoid collisions. ### Logical/set binary operators @@ -108,6 +225,9 @@ which do not have matching label sets in `vector1`. `vector1` for which there are no elements in `vector2` with exactly matching label sets. All matching elements in both vectors are dropped. +As these logical/set binary operators do not interact with the sample values, +they work in the same way for float samples and histogram samples. + ## Vector matching Operations between vectors attempt to find a matching element in the right-hand side @@ -116,11 +236,10 @@ matching behavior: One-to-one and many-to-one/one-to-many. ### Vector matching keywords -These vector matching keywords allow for matching between series with different label sets -providing: +These vector matching keywords allow for matching between series with different label sets: -* `on` -* `ignoring` +* `on(