cli.js

#!/usr/bin/env node

import Docker from 'dockerode'
import chalk from 'chalk'
import path from 'path'
import os from 'os'
import fs from 'fs-extra'
import got from 'got'
import { pipeline } from 'stream/promises'
import { PassThrough } from 'stream'
import prettyBytes from 'pretty-bytes'
import extract from 'extract-zip'
import tar from 'tar'
import { kebabCase } from 'lodash-es'
import { Listr } from 'listr2'
import { performance } from 'perf_hooks'
import { Duration } from 'luxon'
import keypress from 'keypress'

let dock = null
let diffOutput = []
const diffStack = []
const config = { 
 options: [],
 source: process.cwd(),
 target: null,
 tmpDir: null,
 savePath: null,
 shouldCleanup: false
}
const containers = {
 net: null,
 dbSource: null,
 dbTarget: null,
 appSource: null,
 appTarget: null
}
const sha1reg = /^[0-9a-f]{5,40}$/

/**
 * Prompt the user for a path
 * 
 * @param {Task} task Listr task instance
 * @param {String} msg Prompt message
 * @param {Boolean} mustExist Whether the path must already exist
 * @returns path
 */
async function promptForPath (task, msg, mustExist = true, initial) {
 return task.prompt([
 {
 type: 'input',
 name: 'path',
 message: msg,
 initial,
 async validate (input) {
 if (!input) {
 return 'You must provide a valid path!'
 }
 const proposedPath = path.resolve('.', input)
 if (proposedPath.includes(config.source)) {
 return 'Path must be different than the current datatracker project path!'
 } else if (mustExist && !(await fs.pathExists(proposedPath))) {
 return 'Path is invalid or doesn\'t exist!'
 } else {
 return true
 }
 }
 }
 ])
}

/**
 * Download and Extract a zip archive
 * 
 * @param {Task} task Listr task instance
 * @param {Object} param1 Options
 */
async function downloadExtractZip (task, { msg, url, ext = 'zip', branch }) {
 const archivePath = path.join(config.target, `archive.${ext}`)
 await fs.emptyDir(config.target)
 // Download zip
 try {
 task.title = msg
 const downloadBranchStream = got.stream(url)
 downloadBranchStream.on('downloadProgress', progress => {
 task.output = `${prettyBytes(progress.transferred)} downloaded.`
 })
 await pipeline(
 downloadBranchStream,
 fs.createWriteStream(archivePath)
 )
 task.title = `Downloaded ${ext} archive successfully.`
 } catch (err) {
 throw new Error(`Failed to download ${ext} archive from GitHub. ${err.message}`)
 }

 // Extract zip
 try {
 task.title = `Extracting ${ext} archive contents...`
 if (ext === 'zip') {
 const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dt-'))
 await fs.ensureDir(tmpDir)
 await extract(archivePath, {
 dir: tmpDir,
 onEntry (entry) {
 task.output = entry.fileName
 }
 })
 task.title = 'Moving extracted files to final location...'
 task.output = config.target
 await fs.move(path.join(tmpDir, kebabCase(`datatracker-${branch}`)), config.target, { overwrite: true })
 await fs.remove(tmpDir)
 } else if (ext === 'tgz') {
 await tar.x({
 strip: 1,
 file: archivePath,
 cwd: config.target,
 filter (path) {
 task.output = path
 return true
 }
 })
 }
 task.title = `Extracted ${ext} archive successfully.`
 await fs.remove(archivePath)
 } catch (err) {
 throw new Error(`Failed to extract ${ext} archive contents. ${err.message}`)
 }
}

/**
 * Run a command on a running container
 * 
 * @param {Task} task Listr task instance
 * @param {Docker.Container} container Docker container instance
 * @param {Array<String>} cmd Command to execute
 * @param {Boolean} collectOutput Whether to collect and return the command output
 */
async function executeCommand (task, container, cmd, collectOutput = false, silent = false) {
 const logStack = []
 const errStack = []
 let logFStream = null
 return new Promise(async (resolve, reject) => {
 // Handle stream output
 const logStream = new PassThrough()
 logStream.on('data', chunk => {
 const logLine = chunk.toString('utf8').trim()
 if (logLine && !silent) {
 task.output = logLine
 if (collectOutput) {
 logStack.push(...logLine.split('\n').filter(l => l))
 }
 }
 })
 logStream.on('error', chunk => {
 task.output = chunk.toString('utf8')
 errStack.push(chunk.toString('utf8'))
 })
 if (collectOutput) {
 logFStream = fs.createWriteStream(path.join(config.savePath))
 logStream.pipe(logFStream)
 }
 // Execute command in container
 const execChmod = await container.exec({
 Cmd: cmd,
 AttachStdout: true,
 AttachStderr: true
 })
 const execChmodStream = await execChmod.start()
 // Handle stream close
 execChmodStream.on('close', () => {
 if (collectOutput) {
 logFStream.close()
 }
 if (errStack.length > 0) {
 reject(new Error(errStack))
 } else {
 if (!silent) {
 
 }
 task.output = ''
 resolve(logStack)
 }
 })
 // Pipe container stream to log stream
 container.modem.demuxStream(execChmodStream, logStream, logStream)
 })
}

/**
 * Run cleanup tasks (e.g. stop/remove docker resources)
 * 
 * @param {Boolean} exitAfter Whether to exit the process at the end
 */
async function cleanup (exitAfter = false) {
 try {
 const cleanupTasks = new Listr([
 // ------------------------
 // Stop + Remove Containers
 // ------------------------
 {
 title: 'Stop + remove docker containers',
 task: async (ctx, task) => {
 task.output = 'Stopping containers...'
 try {
 await Promise.allSettled([
 containers.dbSource && containers.dbSource.stop(),
 containers.dbTarget && containers.dbTarget.stop(),
 containers.appSource && containers.appSource.stop(),
 containers.appTarget && containers.appTarget.stop()
 ])
 } catch (err) { }
 task.output = 'Removing containers...'
 try {
 await Promise.allSettled([
 containers.dbSource && containers.dbSource.remove({ v: true }),
 containers.dbTarget && containers.dbTarget.remove({ v: true }),
 containers.appSource && containers.appSource.remove({ v: true }),
 containers.appTarget && containers.appTarget.remove({ v: true })
 ])
 } catch (err) { }
 task.output = 'Removing network...'
 try {
 await containers.net.remove()
 } catch (err) {}
 }
 },
 // --------------------
 // Restore config files
 // --------------------
 {
 title: 'Restore original source settings file',
 task: async (ctx, task) => {
 const sourceSettingsPath = path.join(config.source, 'ietf/settings_local.py')
 if (await fs.pathExists(`${sourceSettingsPath}.bak`)) {
 await fs.move(`${sourceSettingsPath}.bak`, sourceSettingsPath, { overwrite: true })
 task.title = 'Restored original source settings file.'
 } else {
 task.skip('Nothing to restore.')
 }
 }
 },
 {
 title: 'Restore original target settings file',
 task: async (ctx, task) => {
 const targetSettingsPath = path.join(config.target, 'ietf/settings_local.py')
 if (await fs.pathExists(`${targetSettingsPath}.bak`)) {
 await fs.move(`${targetSettingsPath}.bak`, targetSettingsPath, { overwrite: true })
 task.title = 'Restored original target settings file.'
 } else {
 task.skip('Nothing to restore.')
 }
 }
 }
 ], {
 registerSignalListeners: false
 })
 
 await cleanupTasks.run()

 // Cleanup
 if (config.tmpDir) {
 await fs.remove(config.tmpDir)
 }

 } catch (err) {
 console.error(chalk.redBright(err.message))
 process.exit(1)
 }

 if (exitAfter) {
 process.exit(0)
 }
}

async function main () {
 console.clear()
 console.info('╔════════════════════════════╗')
 console.info('║ IETF DATATRACKER DIFF TOOL ║')
 console.info('╚════════════════════════════╝\n')

 try {
 const tasks = new Listr([
 // ----------------------------
 // Connect to Docker Engine API
 // ----------------------------
 {
 title: 'Connect to Docker Engine API',
 task: async (ctx, task) => {
 dock = new Docker()
 await dock.ping()
 task.title = 'Connected to Docker Engine API.'
 }
 },
 // ---------------------------------------------------------------
 // Find base path so that it works from both / and /dev/diff paths
 // ---------------------------------------------------------------
 {
 title: 'Find base datatracker instance base path',
 task: async (ctx, task) => {
 let parentIdx = 0
 while(!(await fs.pathExists(path.join(config.source, 'requirements.txt')))) {
 config.source = path.resolve(config.source, '..')
 parentIdx++
 if (parentIdx > 2) {
 throw new Error('Start the CLI from a valid datatracker project path.')
 }
 }
 task.title = `Using path ${config.source} for source datatracker instance.`
 }
 },
 // --------------------------------------
 // Select comparison datatracker instance
 // --------------------------------------
 {
 title: 'Select diff target',
 task: async (ctx, task) => {
 ctx.targetMode = await task.prompt({
 type: 'select',
 message: 'What do you want to compare against?',
 choices: [
 { name: 'local', message: 'Local folder path...' },
 { name: 'branch', message: 'Remote GitHub branch...' },
 { name: 'tag', message: 'Remote GitHub tag...' },
 { name: 'commit', message: 'Remote GitHub commit hash...' }
 // { name: 'release', message: 'Latest GitHub release', disabled: true }
 ]
 })
 task.title = `Selected diff target: ${ctx.targetMode}`
 }
 },
 // ---------------------------------
 // Fetch target datatracker instance
 // ---------------------------------
 {
 title: 'Fetch diff target',
 task: async (ctx, task) => {
 switch (ctx.targetMode) {
 // MODE: LOCAL
 // ------------------------------------------------
 case 'local': {
 task.title = 'Waiting for diff target path input'
 config.target = await promptForPath(task, 'Enter the local path to the datatracker project to compare against:')
 task.title = `Using path ${config.target} for target datatracker instance.`
 break
 }
 // MODE: REMOTE BRANCH
 // ------------------------------------------------
 case 'branch': {
 // Prompt for branch
 const branches = []
 let branch = 'main'
 try {
 task.title = 'Fetching available remote branches...'
 const branchesResp = await got('https://api.github.com/repos/ietf-tools/datatracker/branches?per_page=100').json()
 if (branchesResp?.length < 1) {
 throw new Error('No remote branches available.')
 }
 branches.push(...branchesResp.map(b => b.name))
 task.output = `Fetched ${branches.length} remote branches.`
 } catch (err) {
 throw new Error(`Failed to fetch branches! ${err.message}`)
 }
 
 branch = await task.prompt([
 {
 type: 'select',
 message: 'Select the remote branch to compare against:',
 choices: branches
 }
 ])
 
 // Prompt for local path where to download branch contents
 config.target = await promptForPath(task, 'Enter a local path where the branch contents will be downloaded to:', false)
 await fs.ensureDir(config.target)
 
 // Download / Extract branch zip
 await downloadExtractZip(task, {
 msg: `Downloading ${branch} branch contents...`,
 url: `https://github.com/ietf-tools/datatracker/archive/refs/heads/${branch}.tar.gz`,
 ext: 'tgz'
 })

 task.title = `Fetched branch ${branch} to ${config.target}`
 break
 }
 // MODE: REMOTE TAG
 // ------------------------------------------------
 case 'tag': {
 // Prompt for tag 
 const tag = await task.prompt([
 {
 type: 'input',
 message: 'Enter the remote repository tag to compare against:'
 }
 ])
 
 // Prompt for local path where to download tag contents
 config.target = await promptForPath(task, 'Enter a local path where the tag contents will be downloaded to:', false)
 await fs.ensureDir(config.target)
 
 // Download / Extract tag tarball
 await downloadExtractZip(task, {
 msg: `Downloading tag ${tag} contents...`,
 url: `https://github.com/ietf-tools/datatracker/archive/refs/tags/${tag}.tar.gz`,
 ext: 'tgz'
 })

 task.title = `Fetched tag ${tag} to ${config.target}`
 break
 }
 // MODE: REMOTE COMMIT
 // ------------------------------------------------
 case 'commit': {
 // Prompt for commit hash 
 const commit = await task.prompt([
 {
 type: 'input',
 message: 'Enter the FULL commit hash to compare against:',
 async validate (input) {
 if (!input) {
 return 'You must provide a hash!'
 } else if (!sha1reg.test(input)) {
 return 'Invalid hash!'
 }
 return true
 }
 }
 ])
 
 // Prompt for local path where to download commit contents
 config.target = await promptForPath(task, 'Enter a local path where the commit contents will be downloaded to:', false)
 await fs.ensureDir(config.target)
 
 // Download / Extract commit tarball
 await downloadExtractZip(task, {
 msg: `Downloading commit ${commit} contents...`,
 url: `https://github.com/ietf-tools/datatracker/archive/${commit}.tar.gz`,
 ext: 'tgz'
 })

 task.title = `Fetched commit ${commit} to ${config.target}`
 break
 }
 // MODE: LATEST RELEASE
 // ------------------------------------------------
 case 'release': {
 task.title = 'Waiting for diff target download location'
 // Prompt for local path where to download release
 config.target = await promptForPath(task, 'Enter a local path where the latest release will be downloaded to:', false)
 await fs.ensureDir(config.target)
 
 // Download / extract latest release
 await downloadExtractZip(task, {
 msg: 'Downloading latest release...',
 url: 'https://github.com/ietf-tools/datatracker/releases/latest/download/release.tar.gz',
 ext: 'tgz'
 })

 task.title = `Fetched latest release to ${config.target}`
 break
 }
 default: {
 throw new Error('Invalid selection. Exiting...')
 }
 }

 // Add missing files not present in branch
 if (!(await fs.pathExists(path.join(config.target, 'dev/diff')))) {
 task.output = `Add missing diff tool files...`
 await fs.ensureDir(path.join(config.target, 'dev/diff'))
 await fs.copy(path.join(config.source, 'dev/diff/prepare.sh'), path.join(config.target, 'dev/diff/prepare.sh'))
 await fs.copy(path.join(config.source, 'dev/diff/settings_local.py'), path.join(config.target, 'dev/diff/settings_local.py'))
 }
 }
 },
 // ------------------------
 // Prompt for crawl options
 // ------------------------
 {
 title: 'Select additional crawl options',
 task: async (ctx, task) => {
 const toggleIndicatorFn = (state, choice) => {
 return choice.enabled ? chalk.greenBright('✔') : chalk.gray('o')
 }
 config.options = await task.prompt([
 {
 type: 'multiselect',
 message: 'Select additional options to enable:',
 hint: '(use <SPACE> to toggle, <ENTER> to confirm)',
 choices: [
 { message: 'Skip HTML Validation', name: '--skip-html-validation', hint: 'Skip HTML Validation', indicator: toggleIndicatorFn },
 { message: 'Fail-fast', name: '--failfast', hint: 'Stop the crawl on the first page failure', indicator: toggleIndicatorFn },
 { message: 'No-Follow', name: '--no-follow', hint: 'Do not follow URLs found in fetched pages, just check the given URLs', indicator: toggleIndicatorFn },
 { message: 'No-Revisit', name: '--no-revisit', hint: 'Don\'t revisit already visited URLs', indicator: toggleIndicatorFn },
 { message: 'Pedantic', name: '--pedantic', hint: 'Stop the crawl on the first error or warning', indicator: toggleIndicatorFn },
 { message: 'Random', name: '--random', hint: 'Crawl URLs randomly', indicator: toggleIndicatorFn },
 { message: 'Validate All', name: '--validate-all', hint: 'Run html 5 validation on all pages, without skipping similar urls', indicator: toggleIndicatorFn },
 { message: 'Verbose', name: '--verbose', hint: 'Be more verbose', indicator: toggleIndicatorFn }
 ]
 }
 ])
 if (config.options.length > 0) {
 task.title = `Selected additional crawl options: ${config.options.join(' ')}`
 }
 }
 },
 // ---------------------------
 // Prompt to save crawl output
 // ---------------------------
 {
 title: 'Save crawl output',
 task: async (ctx, task) => {
 const saveToDisk = await task.prompt({
 type: 'confirm',
 message: 'Save the crawl output to file?'
 })

 if (saveToDisk) {
 config.savePath = await promptForPath(task, 'Enter the path where the crawl output will be saved:', false, path.join(os.homedir(), 'Desktop/crawl-out.txt'))
 task.title = `Crawl output will be saved to ${config.savePath}`
 } else {
 config.tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dt-'))
 config.savePath = path.join(config.tmpDir, 'out.txt')
 task.title = 'Crawl output will not be saved.'
 }
 }
 },
 // ----------------------------
 // Set datatracker config files
 // ----------------------------
 {
 title: 'Set datatracker config files',
 task: async (ctx, task) => {
 // Source
 const sourceSettingsPath = path.join(config.source, 'ietf/settings_local.py')
 if (await fs.pathExists(sourceSettingsPath)) {
 await fs.move(sourceSettingsPath, `${sourceSettingsPath}.bak`, { overwrite: true })
 }
 const cfgSourceRaw = await fs.readFile(path.join(config.source, 'dev/diff/settings_local.py'), 'utf8')
 await fs.outputFile(sourceSettingsPath, cfgSourceRaw.replace('__DBHOST__', 'dt-diff-db-source'))
 // Target
 const targetSettingsPath = path.join(config.target, 'ietf/settings_local.py')
 if (await fs.pathExists(targetSettingsPath)) {
 await fs.move(targetSettingsPath, `${targetSettingsPath}.bak`, { overwrite: true })
 }
 const cfgTargetRaw = await fs.readFile(path.join(config.target, 'dev/diff/settings_local.py'), 'utf8')
 await fs.outputFile(targetSettingsPath, cfgTargetRaw.replace('__DBHOST__', 'dt-diff-db-target'))
 }
 },
 // ------------------
 // Pull latest images
 // ------------------
 {
 title: 'Pull latest docker images',
 task: (ctx, task) => task.newListr([
 {
 title: 'Pulling latest DB docker image...',
 task: async (subctx, subtask) => {
 const dbImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-db:latest')
 await new Promise((resolve, reject) => {
 dock.modem.followProgress(dbImagePullStream, (err, res) => err ? reject(err) : resolve(res))
 })
 subtask.title = `Pulled latest DB docker image successfully.`
 }
 },
 {
 title: 'Pulling latest Datatracker base docker image...',
 task: async (subctx, subtask) => {
 const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest')
 await new Promise((resolve, reject) => {
 dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res))
 })
 subtask.title = `Pulled latest Datatracker base docker image successfully.`
 }
 }
 ], {
 concurrent: true,
 rendererOptions: {
 collapse: false
 }
 })
 },
 // --------------
 // Create network
 // --------------
 {
 title: 'Create docker network',
 task: async (ctx, task) => {
 config.shouldCleanup = true
 containers.net = await dock.createNetwork({
 Name: 'dt-diff-net',
 CheckDuplicate: true
 })
 task.title = 'Created docker network (dt-diff-net).'
 }
 },
 // ----------------------------
 // Create + Start DB containers
 // ----------------------------
 {
 title: 'Create DB docker containers',
 task: (ctx, task) => task.newListr([
 {
 title: 'Creating source DB docker container...',
 task: async (subctx, subtask) => {
 containers.dbSource = await dock.createContainer({
 Image: 'ghcr.io/ietf-tools/datatracker-db:latest',
 name: 'dt-diff-db-source',
 Hostname: 'dbsource',
 HostConfig: {
 NetworkMode: 'dt-diff-net'
 }
 })
 await containers.dbSource.start()
 subtask.title = `Created source DB docker container (dt-diff-db-source) successfully.`
 }
 },
 {
 title: 'Creating target DB docker container...',
 task: async (subctx, subtask) => {
 containers.dbTarget = await dock.createContainer({
 Image: 'ghcr.io/ietf-tools/datatracker-db:latest',
 name: 'dt-diff-db-target',
 Hostname: 'dbtarget',
 HostConfig: {
 NetworkMode: 'dt-diff-net'
 }
 })
 await containers.dbTarget.start()
 subtask.title = `Created target DB docker container (dt-diff-db-target) successfully.`
 }
 }
 ], {
 concurrent: true,
 rendererOptions: {
 collapse: false
 }
 })
 },
 // -------------------------------------
 // Create + Start Datatracker containers
 // -------------------------------------
 {
 title: 'Create Datatracker docker containers',
 task: (ctx, task) => task.newListr([
 {
 title: 'Creating source Datatracker docker container...',
 task: async (subctx, subtask) => {
 containers.appSource = await dock.createContainer({
 Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
 name: 'dt-diff-app-source',
 Tty: true,
 Hostname: 'appsource',
 HostConfig: {
 NetworkMode: 'dt-diff-net'
 }
 })
 await containers.appSource.start()
 subtask.title = `Created source Datatracker docker container (dt-diff-app-source) successfully.`
 }
 },
 {
 title: 'Creating target Datatracker docker container...',
 task: async (subctx, subtask) => {
 containers.appTarget = await dock.createContainer({
 Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
 name: 'dt-diff-app-target',
 Tty: true,
 Hostname: 'apptarget',
 HostConfig: {
 NetworkMode: 'dt-diff-net'
 }
 })
 await containers.appTarget.start()
 subtask.title = `Created target Datatracker docker container (dt-diff-app-target) successfully.`
 }
 }
 ], {
 concurrent: true,
 rendererOptions: {
 collapse: false
 }
 })
 },
 // --------------------------------------------
 // Copy working files to Datatracker containers
 // --------------------------------------------
 {
 title: 'Copy working files to Datatracker docker containers',
 task: (ctx, task) => task.newListr([
 {
 title: 'Copying workfing files into source Datatracker docker container...',
 task: async (subctx, subtask) => {
 const tgzPath = path.join(config.source, 'diff-import.tgz')
 await tar.c({
 gzip: true,
 file: tgzPath,
 cwd: config.source,
 filter (path) {
 if (path.includes('.git') || path.includes('node_modules')) { return false }
 subtask.output = path
 return true
 }
 }, ['.'])
 subtask.output = 'Injecting into container...'
 await containers.appSource.putArchive(tgzPath, {
 path: '/workspace'
 })
 await fs.remove(tgzPath)
 subtask.title = `Imported working files into source Datatracker docker container (dt-diff-app-source) successfully.`
 }
 },
 {
 title: 'Copying working files into target Datatracker docker container...',
 task: async (subctx, subtask) => {
 const tgzPath = path.join(config.target, 'diff-import.tgz')
 await tar.c({
 gzip: true,
 file: tgzPath,
 cwd: config.target,
 filter (path) {
 if (path.includes('.git') || path.includes('node_modules')) { return false }
 subtask.output = path
 return true
 }
 }, ['.'])
 subtask.output = 'Injecting into container...'
 await containers.appTarget.putArchive(tgzPath, {
 path: '/workspace'
 })
 await fs.remove(tgzPath)
 subtask.title = `Imported working files into target Datatracker docker container (dt-diff-app-target) successfully.`
 }
 }
 ], {
 concurrent: true,
 rendererOptions: {
 collapse: false
 }
 })
 },
 // -------------------
 // Run prepare scripts
 // -------------------
 {
 title: 'Prepare Datatracker instances',
 task: (ctx, task) => task.newListr([
 {
 title: 'Preparing source Datatracker instance...',
 task: async (subctx, subtask) => {
 await executeCommand(subtask, containers.appSource, ['bash', '-c', 'chmod +x ./dev/diff/prepare.sh'])
 await executeCommand(subtask, containers.appSource, ['bash', './dev/diff/prepare.sh'])
 subtask.title = `Source Datatracker instance is now ready.`
 }
 },
 {
 title: 'Preparing target Datatracker instance...',
 task: async (subctx, subtask) => {
 await executeCommand(subtask, containers.appTarget, ['bash', '-c', 'chmod +x ./dev/diff/prepare.sh'])
 await executeCommand(subtask, containers.appTarget, ['bash', './dev/diff/prepare.sh'])
 subtask.title = `Run target Datatracker instance - Starting server...`
 executeCommand(subtask, containers.appTarget, ['bash', '-c', './ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local'])
 subtask.title = `Run target Datatracker instance - Waiting for server to accept connections...`
 await executeCommand(subtask, containers.appTarget, ['bash', '-c', '/usr/local/bin/wait-for localhost:8000 -t 300'])
 subtask.title = `Target Datatracker instance is now ready and accepting connections.`
 }
 }
 ], {
 concurrent: true,
 rendererOptions: {
 collapse: false
 }
 })
 },
 // --------------
 // Run crawl tool
 // --------------
 {
 title: 'Run crawl',
 task: async (ctx, task) => {
 task.title = 'Running crawl... (Press F10 to cancel)'
 task.output = 'Starting ./bin/test-crawl... (Results will start appearing soon)'
 
 const startMs = performance.now()

 config.options.push('--settings=ietf.settings_testcrawl')
 config.options.push('--diff http://dt-diff-app-target:8000/')

 // Run crawl
 const errStack = []
 let execChmodStream = null
 let linesScanned = 0
 const execPromise = new Promise(async (resolve, reject) => {
 // Handle stream output
 const logStream = new PassThrough()
 logStream.on('data', chunk => {
 const logLines = chunk.toString('utf8').trim().split('\n').filter(l => l.trim())
 // Check for DIFF mentions
 if (logLines.length > 0) {
 linesScanned += logLines.length
 for (const logLine of logLines) {
 if (logLine.includes('DIFF')) {
 diffStack.push(logLine)
 }
 }
 const currentDur = Duration.fromMillis(performance.now() - startMs)
 task.output = `[${currentDur.toFormat('hh:mm:ss')}] Scanned ${linesScanned} lines. Found ${diffStack.length} DIFF mentions.`
 }
 })
 // Handle error stream
 logStream.on('error', chunk => {
 task.output = chunk.toString('utf8')
 errStack.push(chunk.toString('utf8'))
 })
 // Pipe to file stream
 const logFStream = fs.createWriteStream(path.join(config.savePath))
 logStream.pipe(logFStream)
 // Execute command in container
 const execChmod = await containers.appSource.exec({
 Cmd: ['bash', '-c', `./bin/test-crawl ${config.options.join(' ')}`],
 AttachStdout: true,
 AttachStderr: true
 })
 execChmodStream = await execChmod.start()
 // Handle stream close
 execChmodStream.on('close', () => {
 logFStream.close()
 process.stdin.setRawMode(false)
 process.stdin.resume()
 if (errStack.length > 0) {
 reject(new Error(errStack))
 } else {
 resolve()
 }
 })
 // Pipe container stream to log stream
 containers.appSource.modem.demuxStream(execChmodStream, logStream, logStream)
 })

 // Handle F10 Exit
 keypress(process.stdin)
 process.stdin.on('keypress', (ch, key) => {
 if (key && key.name === 'f10') {
 task.title = 'Run crawl'
 execChmodStream.destroy()
 task.skip()
 }
 })
 process.stdin.setRawMode(true)
 process.stdin.resume()

 return execPromise
 }
 }
 ])

 await tasks.run()

 } catch (err) {
 console.error(chalk.redBright(err.message))
 }

 // ------------------------
 // Cleanup
 // ------------------------

 await cleanup()

 // ------------------------
 // Output results
 // ------------------------

 console.info('\n=====================')
 console.info('RESULTS')
 console.info('=====================\n')

 if (diffStack.length > 0) {
 for (const diffLine of diffStack) {
 console.info(`> ${diffLine}`)
 }
 console.info(chalk.blueBright(`\nFound ${diffStack.length} mention(s) of DIFF.\n`))
 } else {
 console.info(chalk.blueBright(`No mention of DIFF were found.\n`))
 }

 process.exit(0)
}

// Handle user interrupt
// process.once('SIGINT', async () => {
// console.info('Trying to shutdown gracefully...')
// if (config.shouldCleanup) {
// setTimeout(() => { process.exit(1) }, 30000).unref()
// try {
// await cleanup()
// } catch (err) {
// console.warn(`Failed to shutdown gracefully: ${err.message}`)
// }
// }
// process.exit(0)
// })

main()