CON-181 Add prometheus metrics to CN

creator-node/package.json

@@ -58,6 +58,7 @@
  "multer": "^1.4.0",
  "pg": "^8.0.3",
  "prettier-config-standard": "^4.0.0",
+ "prom-client": "^14.0.1",
  "promise.any": "^2.0.2",
  "rate-limit-redis": "^1.7.0",
  "sequelize": "^4.44.4",

creator-node/src/components/healthCheck/healthCheckController.js

@@ -131,8 +131,22 @@ const healthCheckController = async (req) => {
  * Controller for `health_check/sync` route, calls
  * syncHealthCheckController
  */
-const syncHealthCheckController = async () => {
+const syncHealthCheckController = async (req) => {
  const response = await syncHealthCheck(serviceRegistry)
+
+ const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry
+ const syncQueueJobsTotalMetric = prometheusRegistry.getMetric(
+ prometheusRegistry.metricNames.SYNC_QUEUE_JOBS_TOTAL_GAUGE
+ )
+ syncQueueJobsTotalMetric.set(
+ { status: 'manual_waiting' },
+ response.manualWaitingCount
+ )
+ syncQueueJobsTotalMetric.set(
+ { status: 'recurring_waiting' },
+ response.recurringWaitingCount
+ )
+
  return successResponse(response)
 }
 

creator-node/src/monitors/filesystem.js

@@ -1,5 +1,6 @@
 const si = require('systeminformation')
 const disk = require('diskusage')
+
 const DiskManager = require('../diskManager')
 
 const getStoragePathSize = async () => {
@@ -11,7 +12,8 @@ const getStoragePathSize = async () => {
 const getStoragePathUsed = async () => {
  const storagePath = DiskManager.getConfigStoragePath()
  const { available, total } = await disk.check(storagePath)
- return total - available
+ const used = total - available
+ return used
 }
 
 // We first check '/var/k8s' in case the service operator has elected to

creator-node/src/routes/prometheusMetricsRoutes.js

@@ -0,0 +1,12 @@
+/**
+ * Exposes Prometheus metrics at `GET /prometheus_metrics`
+ */
+module.exports = function (app) {
+ app.get('/prometheus_metrics', async (req, res) => {
+ const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry
+ const metricData = await prometheusRegistry.getAllMetricData()
+
+ res.setHeader('Content-Type', prometheusRegistry.registry.contentType)
+ return res.end(metricData)
+ })
+}

creator-node/src/routes/tracks.js

@@ -304,8 +304,17 @@ module.exports = function (app) {
  transcodedTrackUUID
  } = req.body
 
+ const prometheusRegistry =
+ req.app.get('serviceRegistry').prometheusRegistry
+ const routePostTracksDurationSecondsMetric = prometheusRegistry.getMetric(
+ prometheusRegistry.metricNames
+ .ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM
+ )
+ const metricEndTimerFn = routePostTracksDurationSecondsMetric.startTimer()
+
  // Input validation
  if (!blockchainTrackId || !blockNumber || !metadataFileUUID) {
+ metricEndTimerFn({ code: 400 })
  return errorResponseBadRequest(
  'Must include blockchainTrackId, blockNumber, and metadataFileUUID.'
  )
@@ -314,6 +323,7 @@ module.exports = function (app) {
  // Error on outdated blocknumber
  const cnodeUser = req.session.cnodeUser
  if (blockNumber < cnodeUser.latestBlockNumber) {
+ metricEndTimerFn({ code: 400 })
  return errorResponseBadRequest(
  `Invalid blockNumber param ${blockNumber}. Must be greater or equal to previously processed blocknumber ${cnodeUser.latestBlockNumber}.`
  )
@@ -325,6 +335,7 @@ module.exports = function (app) {
  where: { fileUUID: metadataFileUUID, cnodeUserUUID }
  })
  if (!file) {
+ metricEndTimerFn({ code: 400 })
  return errorResponseBadRequest(
  `No file db record found for provided metadataFileUUID ${metadataFileUUID}.`
  )
@@ -339,11 +350,13 @@ module.exports = function (app) {
  !Array.isArray(metadataJSON.track_segments) ||
  !metadataJSON.track_segments.length
  ) {
+ metricEndTimerFn({ code: 500 })
  return errorResponseServerError(
  `Malformatted metadataJSON stored for metadataFileUUID ${metadataFileUUID}.`
  )
  }
  } catch (e) {
+ metricEndTimerFn({ code: 500 })
  return errorResponseServerError(
  `No file stored on disk for metadataFileUUID ${metadataFileUUID} at storagePath ${file.storagePath}.`
  )
@@ -357,6 +370,7 @@ module.exports = function (app) {
  metadataJSON.cover_art_sizes
  )
  } catch (e) {
+ metricEndTimerFn({ code: 500 })
  return errorResponseServerError(e.message)
  }
 
@@ -543,10 +557,12 @@ module.exports = function (app) {
 
  await issueAndWaitForSecondarySyncRequests(req)
 
+ metricEndTimerFn({ code: 200 })
  return successResponse()
  } catch (e) {
  req.logger.error(e.message)
  await transaction.rollback()
+ metricEndTimerFn({ code: 500 })
  return errorResponseServerError(e.message)
  }
  })

creator-node/src/serviceRegistry.js

@@ -1,3 +1,7 @@
+const { createBullBoard } = require('@bull-board/api')
+const { BullAdapter } = require('@bull-board/api/bullAdapter')
+const { ExpressAdapter } = require('@bull-board/express')
+
 const { libs: AudiusLibs } = require('@audius/sdk')
 const redisClient = require('./redis')
 const BlacklistManager = require('./blacklistManager')
@@ -6,10 +10,6 @@ const config = require('./config')
 const URSMRegistrationManager = require('./services/URSMRegistrationManager')
 const { logger } = require('./logging')
 const utils = require('./utils')
-const { createBullBoard } = require('@bull-board/api')
-const { BullAdapter } = require('@bull-board/api/bullAdapter')
-const { ExpressAdapter } = require('@bull-board/express')
-
 const MonitoringQueue = require('./monitors/MonitoringQueue')
 const SyncQueue = require('./services/sync/syncQueue')
 const SkippedCIDsRetryQueue = require('./services/sync/skippedCIDsRetryService')
@@ -19,6 +19,7 @@ const TrustedNotifierManager = require('./services/TrustedNotifierManager')
 const ImageProcessingQueue = require('./ImageProcessingQueue')
 const TranscodingQueue = require('./TranscodingQueue')
 const StateMachineManager = require('./services/stateMachineManager')
+const PrometheusRegistry = require('./services/prometheusMonitoring/prometheusRegistry')
 
 /**
  * `ServiceRegistry` is a container responsible for exposing various
@@ -45,6 +46,7 @@ class ServiceRegistry {
  this.blacklistManager = BlacklistManager
  this.monitoringQueue = new MonitoringQueue()
  this.sessionExpirationQueue = new SessionExpirationQueue()
+ this.prometheusRegistry = new PrometheusRegistry()
 
  // below services are initialized separately in below functions `initServices()` and `initServicesThatRequireServer()`
  this.libs = null