Fix CN stack too deep error & probably CPU maxing out

[SidSethi]

Description

Infinite recursion causes stack too deep, blocks event loop (thsi is why requests never go through), and maxes out CPU usage on all node server processes
Solution:

• replace recursive call with infinite while loop
• add delay btwn while loop iterations
• add short circuit for while loop

Loggly link of errors

{"name":"audius_creator_node","clusterWorker":"Worker 2","hostname":"8ef9f0e7d117","pid":116,"requestID":"APF-Z4GMb","requestMethod":"GET","requestHostname":"127.0.0.1","requestUrl":"/prometheus/linux","level":20,"msg":"Begin processing request","time":"2022-12-19T22:06:20.327Z","v":0,"trace_id":"08f5b446fbc3e82968057536535bab81","span_id":"cc279508e7fd5372","trace_flags":"01","resource.span.name":"HTTP GET","resource.span.links":[],"resource.span.attributed":{"http.url":"http://127.0.0.1:3000/prometheus/linux","http.host":"127.0.0.1:3000","net.host.name":"127.0.0.1","http.method":"GET","http.client_ip":"34.132.7.228","http.target":"/prometheus/linux","http.user_agent":"Prometheus/2.33.4","http.flavor":"1.0","net.transport":"ip_tcp"},"resource.service.name":"content-node","resource.service.spid":0,"resource.service.endpoint":"https://creatornode6.staging.audius.co/","logLevel":"debug"}
Exception in PromiseRejectCallback:
/usr/src/app/build/src/diskManager.js:668
return migrateFilesWithNonStandardStoragePaths(5000, prometheusRegistry, logger);
RangeError: Maximum call stack size exceeded
(node:19) UnhandledPromiseRejectionWarning: RangeError: Maximum call stack size exceeded
at stringify (<anonymous>)
at writeChannelMessage (internal/child_process/serialization.js:127:20)
at ChildProcess.target._send (internal/child_process.js:836:17)
at ChildProcess.target.send (internal/child_process.js:734:19)
at Worker.send (internal/cluster/worker.js:46:10)
at ClusterUtilsForPrimary.sendMetricToWorker (/usr/src/app/build/src/utils/cluster/clusterUtilsForPrimary.js:54:24)
at _resetStoragePathMetrics (/usr/src/app/build/src/diskManager.js:602:36)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:646:5)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
at migrateFilesWithNonStandardStoragePaths (/usr/src/app/build/src/diskManager.js:668:12)
(Use `node --trace-warnings ...` to show where the warning was created)
(node:19) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode). (rejection id: 1)
(node:19) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

Tests

Live on staging CN6 and 10 - works

Monitoring - How will this change be monitored? Are there sufficient logs / alerts?

Ensure that disk deletion & storagePath migration jobs still work, and CPU no longer maxes out