diff --git a/appveyor.yml b/appveyor.yml index 40dafad..9c3a1e2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,8 +29,8 @@ test_script: ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 chmod -R 777 /appveyor/projects npm install - npm test - npm run test:integration + npm run test -- --ignore-only + npm run test:integration -- --ignore-only artifacts: - path: ./*_sc-core.7z @@ -38,7 +38,9 @@ artifacts: # on successful build on_success: - sh: | - apk add curl jq + if [ ! apk add curl jq ]; then + exit 1 + fi CURR_VER=$(cat package.json | jq -r .version) if curl -s -X GET -H "Authorization: token $deploytoken" https://git.nfp.is/api/v1/repos/$APPVEYOR_REPO_NAME/releases | grep -o "\"name\"\:\"v${CURR_VER}\"" > /dev/null; then echo "Release already exists, nothing to do."; diff --git a/core/application.mjs b/core/application.mjs index 80d46a6..f3ddbe2 100644 --- a/core/application.mjs +++ b/core/application.mjs @@ -1,4 +1,6 @@ import { EventEmitter } from 'events' +import cluster from 'cluster' + import fs from 'fs/promises' import { request } from './client.mjs' import HttpServer from './http.mjs' @@ -21,6 +23,7 @@ export default class Application extends EventEmitter { this.http = new HttpServer(this.config) this.module = null this.running = false + this.workers = {} // Fresh is used to indicate that when we run the application and it fails, // whether the environment we are in was fresh or not. An example would be @@ -38,18 +41,34 @@ export default class Application extends EventEmitter { this.config.heartbeatAttempts = this.config.heartbeatAttempts || 5 this.config.heartbeatAttemptsWait = this.config.heartbeatAttemptsWait || (2 * 1000) this.config.heartbeatPath = this.config.heartbeatPath || '/' + this.config.clusterWaitOnCrash = this.config.clusterWaitOnCrash || (1 * 1000) Object.assign(this, { setInterval: opts.setInterval || setInterval, + setTimeout: opts.setTimeout || setTimeout, fs: opts.fs || fs, + cluster: opts.cluster || cluster }) + this.isSlave = this.cluster.isWorker + this.monitoringCluster = false + + if (this.config.cluster && !this.isSlave) { + for (let i = 1; i <= this.config.cluster; i++) { + this.workers[i] = null + } + } + this.ctx.db.addApplication(name) } startAutoupdater() { if (this.provider.static) return if (this.config.updateEvery === 0) return + if (this.isSlave) { + this.ctx.log.warn('app.startAutoupdater was called from within a slave') + return + } let timer = this.setInterval(() => { this.update().then( @@ -64,15 +83,19 @@ export default class Application extends EventEmitter { timer.unref() } - updateLog(message) { + updateLog(message, level = 'info') { this.ctx.db.data.core[this.name].updater += message - this.ctx.log.info(message) + this.ctx.log[level](message) return message } msgStatic = 'Provider in question is static and so no update required, nothing to do.' update() { + if (this.isSlave) { + this.ctx.log.warn('app.update was called from within a slave') + return null + } if (this.provider.static) { if (this.ctx.db.data.core[this.name].updater !== this.msgStatic) { this.ctx.db.data.core[this.name].updater = '' @@ -122,18 +145,19 @@ export default class Application extends EventEmitter { let latest = null try { - log += this.updateLog(`Checking for latest version at ${new Date().toISOString().replace('T', ' ').split('.')[0]}. `) + '\n' + log += this.updateLog(`Checking for latest version at ${new Date().toISOString().replace('T', ' ').split('.')[0]}. `, 'debug') + '\n' // Get the latest version from our provider latest = await this.provider.getLatestVersion() - log += this.updateLog(`Found ${latest.version}. `) + '\n' - // If the versino matches the latest installed, then there's nothing to do if (this.ctx.db.data.core[this.name].latestInstalled === latest.version) { - this.updateLog('Already up to date, nothing to do. ') + log += this.updateLog(`Found ${latest.version}. `, 'debug') + '\n' + this.updateLog('Already up to date, nothing to do. ', 'debug') return null } + + log += this.updateLog(`Found new update ${latest.version}. `) + '\n' // Make the id for the vesion the version number. Allows for easy lookup // among other nice and simple structure. @@ -241,7 +265,7 @@ export default class Application extends EventEmitter { // For some weird reason, --loglevel=notice is required otherwise // we get practically zero log output. await this.ctx.util.runCommand( - 'npm.cmd', + this.ctx.util.getNpmExecutable(), ['install', '--production', '--no-optional', '--no-package-lock', '--no-audit', '--loglevel=notice'], folder, function(msg) { log += msg } @@ -291,56 +315,128 @@ export default class Application extends EventEmitter { this.module = module } + workerDied(worker) { + if (this.workers[worker.w_id] !== worker) { + return + } + this.workers[worker.w_id] = null + + this.setTimeout(() => { + if (this.workers[worker.w_id]) return + this.startForkProcess(worker.w_id, this.ctx.db.data.core[this.name].active) + }, this.clusterWaitOnCrash) + } + + startForkProcess(i, version) { + this.workers[i] = this.cluster.fork({ + CLUSTER_APP_NAME: this.name, + CLUSTER_APP_VERSION: version, + CLUSTER_APP_INDEX: i, + }) + let pid = this.workers[i].process.pid + this.ctx.log.info(`Creating worker ${i} (${pid})`) + // this.workers[i].once('listening', () => { + // this.ctx.log.info(`Fork ${i} is alive and listening`) + // }) + this.workers[i].once('exit', (code, signal) => { + if (signal) { + // this.ctx.log.info(`Worker ${i} (${pid}) was killed by signal ${signal}`) + } else if (code !== 0) { + this.ctx.log.warn(`Worker ${i} (${pid}) exited with code ${code}`) + } + }) + this.workers[i].w_id = i + this.workers[i].listening = false + this.workers[i].started = new Date() + } + async runVersion(version) { this.ctx.db.data.core[this.name].active = version await this.ctx.db.write() - if (version !== 'static') { - let indexPath = this.ctx.util.getPathFromRoot(`./${this.name}/${version}/index.mjs`) - await this.fs.stat(indexPath).catch((err) => { - return Promise.reject(new Error(`Version was missing index.mjs: ${err.message}`)) + if (this.config.cluster && !this.isSlave) { + for (let i = 1; i <= this.config.cluster; i++) { + this.startForkProcess(i, version) + } + } else { + if (version !== 'static') { + let indexPath = this.ctx.util.getPathFromRoot(`./${this.name}/${version}/index.mjs`) + await this.fs.stat(indexPath).catch((err) => { + return Promise.reject(new Error(`Version was missing index.mjs: ${err.message}`)) + }) + + this.fresh = false + let module = await import(this.ctx.util.getUrlFromRoot(`./${this.name}/${version}/index.mjs`)) + this.registerModule(module, version) + } else { + this.fresh = false + } + + let errTimeout = new Error(`Version timed out (took over ${this.config.startWaitUntilFail}ms) while running start()`) + + await new Promise((res, rej) => { + setTimeout(() => { + rej(errTimeout) + }, this.config.startWaitUntilFail) + + let startRes = this.module.start(this.http, this.config.port, this.ctx) + if (startRes && startRes.then) { + return startRes.then(res, rej) + } + res() }) - this.fresh = false - let module = await import(this.ctx.util.getUrlFromRoot(`./${this.name}/${version}/index.mjs`)) - this.registerModule(module, version) - } else { - this.fresh = false + if (!this.http.active) { + return Promise.reject(new Error(`Version did not call http.createServer()`)) + } } - let errTimeout = new Error(`Version timed out (took over ${this.config.startWaitUntilFail}ms) while running start()`) - - await new Promise((res, rej) => { - setTimeout(() => { - rej(errTimeout) - }, this.config.startWaitUntilFail) - - let startRes = this.module.start(this.http, this.config.port, this.ctx) - if (startRes && startRes.then) { - return startRes.then(res, rej) - } - res() - }) - - if (!this.http.active) { - return Promise.reject(new Error(`Version did not call http.createServer()`)) + if (this.isSlave) { + return } let lastErr = null for (let i = 0; i < this.config.heartbeatAttempts; i++) { try { - await request({ timeout: this.config.heartbeatAttemptsWait }, `http://localhost:${this.config.port}` + this.config.heartbeatPath, null, 0, true) - return + await request({ timeout: this.config.heartbeatTimeout }, `http://localhost:${this.config.port}` + this.config.heartbeatPath, null, 0, true) + lastErr = null + break } catch (err) { lastErr = err + await new Promise((res) => { + setTimeout(res, this.config.heartbeatAttemptsWait) + }) + // this.config.heartbeatAttemptsWait } } - return Promise.reject(new Error(`Version failed to start properly: ${lastErr.message}`)) + if (lastErr) { + let err = new Error(`Version failed to start properly: ${lastErr.message}`) + err.stack = lastErr.stack + return Promise.reject(err) + } + + if (this.config.cluster && !this.isSlave) { + this.__clusterWorkerDied = this.workerDied.bind(this) + this.cluster.on('exit', this.__clusterWorkerDied) + } } closeServer() { + if (this.config.cluster && !this.isSlave) { + if (this.__clusterWorkerDied) { + this.cluster.off('exit', this.__clusterWorkerDied) + } + this.__clusterWorkerDied = null + + for (let i = 1; i <= this.config.cluster; i++) { + if (this.workers[i]) { + this.workers[i].process.kill() + this.workers[i] = null + } + } + } return this.http.closeServer() } } diff --git a/core/client.mjs b/core/client.mjs index 468a63f..7c1b602 100644 --- a/core/client.mjs +++ b/core/client.mjs @@ -2,6 +2,7 @@ import http from 'http' import https from 'https' import stream from 'stream/promises' import fs from 'fs' +import Util from './util.mjs' function resolveRelative(from, to) { const resolvedUrl = new URL(to, new URL(from, 'resolve://')); @@ -35,7 +36,7 @@ export function request(config, path, filePath = null, redirects, fastRaw = fals let req = null - return new Promise(function(resolve, reject) { + let orgErr = new Error(); return new Promise(function(resolve, reject) { if (!path) { return reject(new Error('Request path was empty')) } @@ -52,7 +53,7 @@ export function request(config, path, filePath = null, redirects, fastRaw = fals let timer = setTimeout(function() { timedout = true if (req) { req.destroy() } - reject(new Error(`Request ${path} timed out after ${timeout}ms`)) + reject(Util.combineStack(new Error(`Request ${path} timed out after ${timeout}ms`), orgErr)) }, timeout) req = h.request({ @@ -81,7 +82,7 @@ export function request(config, path, filePath = null, redirects, fastRaw = fals }) }, function(err) { if (err.code === 'ABORT_ERR') return - reject(err) + reject(Util.combineStack(err, orgErr)) }) // let file = fs.createWriteStream(filePath) // res.pipe(file) @@ -113,11 +114,11 @@ export function request(config, path, filePath = null, redirects, fastRaw = fals if (err) { ac.abort() - if (!filePath) return reject(err) + if (!filePath) return reject(Util.combineStack(err, orgErr)) // Do some cleanup in case we were in the middle of downloading file return fs.rm(filePath, function() { - reject(err) + reject(Util.combineStack(err)) }) } // Let the pipeline do the resolving so it can finish flusing before calling resolve @@ -137,11 +138,11 @@ export function request(config, path, filePath = null, redirects, fastRaw = fals if (timedout) return let wrapped = new Error(`Error during request ${path}: ${err.message}`) wrapped.code = err.code - reject(wrapped) + reject(Util.combineStack(wrapped, orgErr)) }) req.on('timeout', function(err) { if (timedout) return - reject(err) + reject(Util.combineStack(err, orgErr)) }) req.end() @@ -155,9 +156,9 @@ export function request(config, path, filePath = null, redirects, fastRaw = fals res.body = JSON.parse(res.body) } catch(e) { if (res.body.indexOf(' { - return app.update().catch(err => { - app.ctx.log.error(err, `Error updating: ${err.message}`) - }) - })) + if (!this.isSlave) { + this.log.info(`Running updater on ${this.applications.length} apps`) + await Promise.all(this.applications.map((app) => { + return app.update().catch(err => { + app.ctx.log.error(err, `Error updating: ${err.message}`) + }) + })) + } let found = false + + if (this.isSlave) { + let app = this.getApplication(process.env.CLUSTER_APP_NAME) + try { + await app.runVersion(process.env.CLUSTER_APP_VERSION) + } catch (err) { + app.ctx.log.fatal(err) + return Promise.reject(err) + } + return + } for (let app of this.applications) { app.startAutoupdater() @@ -136,6 +164,7 @@ export default class Core { application.ctx.log.warn(`Restarting for ${version.version} due to last run failing while not being in fresh state`) return this.restart(`Application ${name} has fresh false while attempting to run ${version.version} with stable -1`) } + await application.closeServer() this._applicationFatalCrash = this.criticalError.bind(this, application, version) @@ -149,6 +178,7 @@ export default class Core { found = true version.stable = 1 await this.db.write() + application.ctx.log.info(`${version.version} is up and running`) break } catch(err) { application.ctx.log.error(err, `Error starting ${version.version}: ${err.message}`) diff --git a/core/db.mjs b/core/db.mjs index cfa23e5..c1be18b 100644 --- a/core/db.mjs +++ b/core/db.mjs @@ -100,14 +100,15 @@ export default function GetDB(config, log, orgFilename = 'db.json') { db.write = function() { return this._write() // Do couple of retries. Sometimes it fails randomly doing atomic writes. - .catch(() => { return setTimeout(20) }) - .then(() => { return this._write() }) - .catch(() => { return setTimeout(50) }) - .then(() => { return this._write() }) - .catch(() => { return setTimeout(100) }) - .then(() => { return this._write() }) + .catch(() => { return setTimeout(20).then(() => { return this._write() }) }) + .catch(() => { return setTimeout(50).then(() => { return this._write() }) }) + .catch(() => { return setTimeout(100).then(() => { return this._write() }) }) .catch((err) => { - this.log.error(err, 'Error saving to db') + try { + db.writeSync() + } catch { + this.log.error(err, 'Error saving to db') + } }) } diff --git a/core/log.mjs b/core/log.mjs index 7c9eaa1..62dbb67 100644 --- a/core/log.mjs +++ b/core/log.mjs @@ -2,6 +2,22 @@ import bunyan from 'bunyan-lite' import { setTimeout } from 'timers/promises' +export function getDefaultStreams() { + if (process.env.NODE_ENV === 'production') { + return [{ + path: 'log.log', + level: 'info', + } + ] + } else { + return [{ + "stream": process.stdout, + "level": "debug" + } + ] + } +} + export default function getLog(name, streams = null, opts = {}) { let settings let ringbuffer = new bunyan.RingBuffer({ limit: 100 }) @@ -15,24 +31,9 @@ export default function getLog(name, streams = null, opts = {}) { }) } - if (process.env.NODE_ENV === 'production') { - settings = { - "name": name, - "streams": streams || [{ - path: 'log.log', - level: 'info', - } - ] - } - } else { - settings = { - "name": name, - "streams": streams || [{ - "stream": process.stdout, - "level": "debug" - } - ] - } + settings = { + name: name, + streams: streams || getDefaultStreams() } let logger @@ -122,6 +123,13 @@ export default function getLog(name, streams = null, opts = {}) { error: function() { return Promise.resolve() }, } } + logger.on('error', function(err) { + if (process.env.NODE_ENV === 'production') { + logger.event.warn(`Error in logger ${name}: ${err.message}`) + } else { + console.log(`Error in logger ${name}: ${err.message}`) + } + }) logger.ringbuffer = ringbuffer logger.ringbufferwarn = ringbufferwarn diff --git a/core/runner.mjs b/core/runner.mjs index 57268df..30f8dec 100644 --- a/core/runner.mjs +++ b/core/runner.mjs @@ -1,6 +1,7 @@ +import cluster from 'cluster' import Util from './util.mjs' import fs from 'fs/promises' -import getLog from './log.mjs' +import getLog, { getDefaultStreams } from './log.mjs' import GetDB from './db.mjs' import Core from './core.mjs' @@ -21,8 +22,14 @@ export async function runner(root_import_meta_url, configname = 'config.json', d throw new Error(`critical error opening ${fullpath}: ${err.message}`) } } + + let streams = getDefaultStreams() + + if (cluster.isWorker) { + streams[0].level = 'error' + } - const log = getLog(config.name) + const log = getLog(config.name, streams) runner.log = log const db = await GetDB(config, log, util.getPathFromRoot('./' + dbname)) diff --git a/core/util.mjs b/core/util.mjs index d9aedf3..417560f 100644 --- a/core/util.mjs +++ b/core/util.mjs @@ -5,6 +5,11 @@ import { spawn, execSync } from 'child_process' import { fileURLToPath, pathToFileURL } from 'url' export default class Util { + static combineStack(err, appendErr) { + err.stack = err.stack + '\nFrom:\n' + appendErr.stack.split('\n').slice(1).join('\n') + '\n' + return err + } + constructor(root_import_meta_url) { this._root_import_meta_url = root_import_meta_url } @@ -79,6 +84,10 @@ export default class Util { if (!valid) continue } + if (config[key].cluster != null && (typeof(config[key].cluster) !== 'number' || config[key].cluster > 100 || config[key].cluster < 0)) + continue + if (config[key].clusterWaitOnCrash != null && (typeof(config[key].clusterWaitOnCrash) !== 'number' || config[key].clusterWaitOnCrash < 10)) + continue out.push(key) } @@ -93,6 +102,13 @@ export default class Util { return util.getPathFromRoot('../bin/7zdec') } + getNpmExecutable() { + if (process.platform === 'win32') { + return 'npm.cmd' + } + return 'npm' + } + verifyConfig(config) { if (!config.name) throw new Error('name is missing in config.json') if (this.getAppNames(config).length === 0) throw new Error('no application was found in config') diff --git a/package.json b/package.json index 87d66a4..5b70819 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.mjs", "scripts": { "dev": "nodemon --watch dev/api --watch core --watch runner.mjs --watch db.mjs --watch log.mjs runner.mjs | bunyan", - "test": "eltro \"test/**/*.test.mjs\" -r dot", + "test": "eltro -t 250 \"test/**/*.test.mjs\" -r dot", "test:integration": "eltro \"test/**/*.test.integration.mjs\" -r list", "test:test": "eltro \"test/application.test.integration.mjs\" -r list", "test:spec": "eltro \"test/**/*.test.mjs\" -r list", @@ -52,6 +52,6 @@ "lowdb": "^3.0.0" }, "devDependencies": { - "eltro": "^1.3.0" + "eltro": "^1.3.1" } } diff --git a/test/application.cluster.test.mjs b/test/application.cluster.test.mjs new file mode 100644 index 0000000..33be88f --- /dev/null +++ b/test/application.cluster.test.mjs @@ -0,0 +1,122 @@ +import { Eltro as t, assert, stub } from 'eltro' +import fs from 'fs/promises' +import Application from '../core/application.mjs' +import Util from '../core/util.mjs' +import lowdb from '../core/db.mjs' +import StaticProvider from '../core/providers/static.mjs' +import { createFakeContext } from './helpers.mjs' +import HttpServer from '../core/http.mjs' + +const util = new Util(import.meta.url) + + +t.describe('#runVersion("version") cluster mode', function() { + const assertPort = 22345 + const assertClusterSize = Math.floor(Math.random() * (8 - 2 + 1) + 2) + const http = new HttpServer() + let handler + let cluster + let workers + let ctx + let app + + t.beforeEach(function() { + return createFakeContext({ testnoexisting: { cluster: assertClusterSize } }, util, null) + .then(function(res) { + workers = [] + ctx = res + let provider = new StaticProvider() + cluster = { + on: stub(), + off: stub(), + fork: stub().returnWith(function() { + let worker = { + on: stub(), + once: stub(), + process: { + kill: stub(), + } + } + workers.push(worker) + return worker + }), + isWorker: false, + } + app = new Application(ctx, provider, 'testnoexisting', { cluster }) + app.config.port = assertPort + app.workerDied = stub() + return app.ctx.db.write() + }) + }) + + t.afterEach(function() { + return Promise.all([ + app.closeServer(), + ]) + }) + + t.before(function() { + handler = function(req, res) { + res.writeHead(200) + res.end('{}') + } + let server = http.createServer(function(req, res) { + req.on('error', function() { }) + res.on('error', function() { }) + res.on('finish', function() { }) + handler(req, res) + }) + + return server.listenAsync(assertPort) + }) + + t.after(function() { + return http.closeServer() + }) + + t.test('should call fork for entire cluster and attach handlers correctly', async function() { + let lastErr = null + for (let i = 1; i <= assertClusterSize; i++) { + assert.strictEqual(app.workers[i], null) + } + handler = function(req, res) { + try { + assert.notOk(cluster.on.called) + assert.notOk(app.__clusterWorkerDied) + lastErr = false + } catch (err) { + lastErr = err + } + res.writeHead(200) + res.end('{}') + } + const assertVersion = 'v11.22.33' + + await app.runVersion(assertVersion) + + assert.strictEqual(cluster.fork.callCount, assertClusterSize) + + for (let i = 1; i <= assertClusterSize; i++) { + assert.ok(app.workers[i]) + assert.strictEqual(app.workers[i], workers[i - 1]) + assert.strictEqual(cluster.fork.getCallN(i)[0].CLUSTER_APP_NAME, app.name) + assert.strictEqual(cluster.fork.getCallN(i)[0].CLUSTER_APP_VERSION, assertVersion) + assert.strictEqual(app.workers[i].w_id, i) + assert.strictEqual(app.workers[i].listening, false) + assert.ok(app.workers[i].started) + assert.ok(new Date() - app.workers[i].started < 100 && new Date() - app.workers[i].started >= 0) + } + + assert.strictEqual(lastErr, false) + assert.strictEqual(cluster.on.callCount, 1) + assert.ok(app.__clusterWorkerDied) + assert.strictEqual(cluster.on.firstCall[0], 'exit') + assert.strictEqual(cluster.on.firstCall[1], app.__clusterWorkerDied) + assert.notStrictEqual(app.workerDied, app.__clusterWorkerDied) + assert.notOk(app.workerDied.called) + + app.__clusterWorkerDied() + + assert.ok(app.workerDied.called) + }) +}) diff --git a/test/application.run.test.mjs b/test/application.run.test.mjs index e4b059f..fbda27a 100644 --- a/test/application.run.test.mjs +++ b/test/application.run.test.mjs @@ -15,7 +15,7 @@ t.describe('#runVersion("static")', function() { const defaultHandler = function(orgHandler) { let handler = orgHandler || function (req, res) { - res.writeHead(204); res.end(JSON.stringify({ a: 1 })) + res.writeHead(200); res.end(JSON.stringify({ a: 1 })) } return function(http, port, ctx) { const server = http.createServer(handler) @@ -36,7 +36,7 @@ t.describe('#runVersion("static")', function() { }) t.afterEach(function() { - return app.http.closeServer() + return app.closeServer() }) t.test('should throw if http is not called', async function() { @@ -98,6 +98,7 @@ t.describe('#runVersion("static")', function() { res.end(JSON.stringify({ a: 1 })) } app.config.heartbeatAttempts = 3 + app.config.heartbeatAttemptsWait = 5 app.registerModule(defaultHandler(handler)) assert.strictEqual(app.fresh, true) @@ -116,7 +117,8 @@ t.describe('#runVersion("static")', function() { called++ } app.config.heartbeatAttempts = 2 - app.config.heartbeatAttemptsWait = 30 + app.config.heartbeatTimeout = 30 + app.config.heartbeatAttemptsWait = 10 app.registerModule(defaultHandler(handler)) assert.strictEqual(app.fresh, true) @@ -129,7 +131,7 @@ t.describe('#runVersion("static")', function() { assert.match(err.message, /time/i) assert.match(err.message, /out/i) assert.match(err.message, /30ms/i) - assert.ok(end - start > app.config.heartbeatAttempts * app.config.heartbeatAttemptsWait) + assert.ok(end - start > app.config.heartbeatAttempts * (app.config.heartbeatTimeout + app.config.heartbeatAttemptsWait)) assert.strictEqual(called, 2) assert.strictEqual(ctx.db.data.core.testapp.active, 'static') }) @@ -145,13 +147,14 @@ t.describe('#runVersion("static")', function() { } } app.config.heartbeatAttempts = 3 + app.config.heartbeatAttemptsWait = 5 app.registerModule(defaultHandler(handler)) let err = await assert.isRejected(app.runVersion('static')) assert.match(err.message, /failed/i) assert.match(err.message, /400/i) - await app.http.closeServer() + await app.closeServer() app.registerModule(defaultHandler(handler)) app.config.heartbeatPath = assertPath @@ -159,6 +162,24 @@ t.describe('#runVersion("static")', function() { assert.strictEqual(ctx.db.data.core.testapp.active, 'static') }) + + t.test('should not check heartbeat if slave', async function() { + let called = 0 + const handler = function(req, res) { + called++ + res.statusCode = 400 + res.end(JSON.stringify({ a: 1 })) + } + app.registerModule(defaultHandler(handler)) + app.isSlave = true + + assert.strictEqual(app.fresh, true) + await app.runVersion('static') + assert.strictEqual(app.fresh, false) + + assert.strictEqual(called, 0) + assert.strictEqual(ctx.db.data.core.testapp.active, 'static') + }) }) t.describe('#runVersion("version")', function() { @@ -190,7 +211,7 @@ t.describe('#runVersion("version")', function() { t.afterEach(function() { return Promise.all([ fs.rm(assertConfig), - app.http.closeServer(), + app.closeServer(), ]) }) diff --git a/test/application.test.integration.mjs b/test/application.test.integration.mjs index 4cf1e59..da28303 100644 --- a/test/application.test.integration.mjs +++ b/test/application.test.integration.mjs @@ -32,7 +32,7 @@ t.after(function() { }) }) -t.timeout(10000).test('should run update and install correctly', async function(){ +t.skip().timeout(10000).test('should run update and install correctly', async function(){ try { await app.update() } catch (err) { diff --git a/test/application.test.mjs b/test/application.test.mjs index b52cb78..cb2ebac 100644 --- a/test/application.test.mjs +++ b/test/application.test.mjs @@ -58,19 +58,84 @@ t.describe('constructor()', function() { assert.strictEqual(app.config.heartbeatAttempts, 5) assert.strictEqual(app.config.heartbeatAttemptsWait, 2 * 1000) assert.strictEqual(app.config.heartbeatPath, '/') + assert.strictEqual(app.config.clusterWaitOnCrash, 1 * 1000) assert.strictEqual(app.ctx.db, ctx.db) assert.strictEqual(app.ctx.app, app) assert.strictEqual(app.ctx.util, ctx.util) assert.strictEqual(app.name, assertName) assert.strictEqual(app.fresh, true) assert.strictEqual(app.running, false) + assert.strictEqual(app.monitoringCluster, false) + assert.deepStrictEqual(app.workers, {}) + assert.strictEqual(app.isSlave, false) assert.ok(app.http) assert.ok(app.http.sockets) assert.strictEqual(typeof(app.http.createServer), 'function') assert.strictEqual(typeof(app.http.closeServer), 'function') }) - + t.test('should support overriding defaults', function() { + const assertTest = { + a: 1, + updateEvery: 10, + startWaitUntilFail: 10, + heartbeatTimeout: 10, + heartbeatAttempts: 10, + heartbeatAttemptsWait: 10, + heartbeatPath: '/asdf', + clusterWaitOnCrash: 10, + } + const assertName = 'test' + ctx.db.config = { + test: assertTest, + app: { b: 2}, + manage: { c: 3 }, + } + + let app = new Application(ctx, {}, assertName) + assert.notStrictEqual(app.config, assertTest) + assert.strictEqual(app.config.a, assertTest.a) + assert.strictEqual(app.config.updateEvery, 10) + assert.strictEqual(app.config.startWaitUntilFail, 10) + assert.strictEqual(app.config.heartbeatTimeout, 10) + assert.strictEqual(app.config.heartbeatAttempts, 10) + assert.strictEqual(app.config.heartbeatAttemptsWait, 10) + assert.strictEqual(app.config.heartbeatPath, '/asdf') + assert.strictEqual(app.config.clusterWaitOnCrash, 10) + }) + + t.test('should fill out workers indexes if is master', function() { + const assertTest = { a: 1, cluster: 2 } + const assertName = 'test' + ctx.db.config = { + test: assertTest, + app: { b: 2}, + manage: { c: 3 }, + } + + let app = new Application(ctx, {}, assertName) + assert.notStrictEqual(app.config, assertTest) + assert.strictEqual(app.config.a, assertTest.a) + assert.deepStrictEqual(app.workers, { + 1: null, + 2: null, + }) + }) + + t.test('should leave workers empty if not master', function() { + const assertTest = { a: 1, cluster: 2 } + const assertName = 'test' + ctx.db.config = { + test: assertTest, + app: { b: 2}, + manage: { c: 3 }, + } + + let app = new Application(ctx, {}, assertName, { cluster: { isWorker: true } }) + assert.notStrictEqual(app.config, assertTest) + assert.strictEqual(app.config.a, assertTest.a) + assert.deepStrictEqual(app.workers, {}) + }) t.test('should not default updateEvery if its zero', function() { const assertTest = { a: 1, updateEvery: 0 } @@ -107,7 +172,7 @@ t.describe('constructor()', function() { }) }) -t.timeout(250).describe('#startAutoupdater()', function() { +t.describe('#startAutoupdater()', function() { let ctx t.beforeEach(function() { @@ -124,6 +189,20 @@ t.timeout(250).describe('#startAutoupdater()', function() { app.startAutoupdater() }) + t.test('should do nothing if isSlave but should warn', async function() { + const stubInterval = stub() + stubInterval.throws(new Error('should not be seen')) + let app = new Application(ctx, { }, 'teststatic', { setInterval: stubInterval }) + app.ctx.log.warn.reset() + assert.notOk(app.ctx.log.warn.called) + app.isSlave = true + + app.startAutoupdater() + + assert.ok(app.ctx.log.warn.called) + assert.match(app.ctx.log.warn.firstCall[0], /slave/i) + }) + t.test('should do nothing if updateEvery is zero', async function() { const stubInterval = stub() stubInterval.throws(new Error('should not be seen')) @@ -222,15 +301,19 @@ t.timeout(250).describe('#startAutoupdater()', function() { }) }) -t.timeout(250).describe('#closeServer()', function() { +t.describe('#closeServer()', function() { let app + let cluster let stubCloseServer t.beforeEach(function() { + cluster = { + off: stub(), + } return createFakeContext() .then(function(res) { let provider = createProvider() - app = new Application(res, provider, 'testapp') + app = new Application(res, provider, 'testapp', { cluster: cluster }) app.http.closeServer = stubCloseServer = stub().resolves() }) }) @@ -244,12 +327,109 @@ t.timeout(250).describe('#closeServer()', function() { assert.strictEqual(err, assertError) }) + t.test('should kill entire cluster if cluster master', async function() { + let workers = {} + let handle = app.__clusterWorkerDied = function() {} + app.config.cluster = Math.floor(Math.random() * (10 - 4 + 1) + 4) + for (let i = 1; i <= app.config.cluster; i++) { + if (i % 2 === 0) { + workers[i] = app.workers[i] = { + process: { + kill: stub().returnWith(function() { + assert.strictEqual(app.__clusterWorkerDied, null) + assert.strictEqual(cluster.off.callCount, 1) + assert.strictEqual(cluster.off.firstCall[0], 'exit') + assert.strictEqual(cluster.off.firstCall[1], handle) + }) + } + } + } else { + workers[i] = app.workers[i] = null + } + } + + await app.closeServer() + + for (let i = 1; i <= app.config.cluster; i++) { + if (workers[i]) { + assert.ok(workers[i].process.kill.called) + assert.strictEqual(app.__clusterWorkerDied, null) + } + } + }) + t.test('otherwise should work fine', async function() { await app.closeServer() }) }) -t.timeout(250).describe('#update()', function() { +t.describe('#workerDied()', function() { + let app + let stubTimeout + let stubFork + + t.beforeEach(function() { + stubTimeout = stub() + return createFakeContext() + .then(function(res) { + let provider = createProvider() + app = new Application(res, provider, 'testapp', { setTimeout: stubTimeout }) + app.startForkProcess = stubFork = stub() + }) + }) + + t.test('should do nothing if worker is not found at index', function() { + const assertWorkerId = 2 + const assertWorker = { a: 1, w_id: assertWorkerId } + stubTimeout.throws(new Error('should not be seen')) + stubFork.throws(new Error('should not be seen')) + app.workers[1] = assertWorker + + app.workerDied(assertWorker) + }) + + t.test('should mark worker as null and call setTimeout', function() { + const assertWorkerId = 2 + const assertTimeoutDuration = 12421 + const assertVersion = 'v11.22.33' + const assertWorker = { a: 1, w_id: assertWorkerId } + app.ctx.db.data.core[app.name].active = assertVersion + app.workers[assertWorkerId] = assertWorker + app.clusterWaitOnCrash = assertTimeoutDuration + + app.workerDied(assertWorker) + + assert.strictEqual(app.workers[assertWorkerId], null) + assert.ok(stubTimeout.called) + assert.strictEqual(typeof(stubTimeout.firstCall[0]), 'function') + assert.strictEqual(stubTimeout.firstCall[1], assertTimeoutDuration) + assert.notOk(stubFork.called) + + stubTimeout.firstCall[0]() + + assert.ok(stubFork.called) + assert.ok(stubFork.firstCall[0], assertWorkerId) + assert.ok(stubFork.firstCall[1], assertVersion) + }) + + t.test('setTimeout should not call startForkProcess if one was already started', function() { + const assertWorkerId = 2 + const assertWorker = { a: 1, w_id: assertWorkerId } + app.workers[assertWorkerId] = assertWorker + + app.workerDied(assertWorker) + + assert.strictEqual(app.workers[assertWorkerId], null) + + app.workers[assertWorkerId] = { a : 2 } + + stubTimeout.firstCall[0]() + + assert.notOk(stubFork.called) + }) +}) + +t.describe('#update()', function() { let ctx let app let provider @@ -321,6 +501,23 @@ t.timeout(250).describe('#update()', function() { assert.strictEqual(stubWrite.callCount, 2) }) + t.test('should do nothing if slave but warn', async function() { + stubWrite.reset().resolves() + + app.isSlave = true + app.ctx.log.warn.reset() + assert.notOk(app.ctx.log.warn.called) + + let result = await app.update() + + assert.strictEqual(result, null) + assert.notOk(ctx.db.data.core.teststatic) + assert.notOk(stubWrite.called) + + assert.ok(app.ctx.log.warn.called) + assert.match(app.ctx.log.warn.firstCall[0], /slave/i) + }) + t.test('multiple calls should be safe', async function() { ctx.db.data.core.testapp.updater = '' @@ -638,7 +835,7 @@ t.timeout(250).describe('#update()', function() { assert.notOk(stubUpdated.called) assert.strictEqual(app.updating, false) assert.strictEqual(err, assertError) - assert.strictEqual(stubRunCommand.firstCall[0], 'npm.cmd') + assert.strictEqual(stubRunCommand.firstCall[0], util.getNpmExecutable()) assert.ok(stubRunCommand.firstCall[1]) assert.strictEqual(stubRunCommand.firstCall[1][0], 'install') assert.ok(stubRunCommand.firstCall[1].includes('--production'), 'should have --production') @@ -845,7 +1042,7 @@ t.timeout(250).describe('#update()', function() { }) }) -t.timeout(250).describe('#registerModule()', function() { +t.describe('#registerModule()', function() { const assertAppName = 'testappregister' let ctx let app diff --git a/test/core.cluster.test.mjs b/test/core.cluster.test.mjs new file mode 100644 index 0000000..7823c02 --- /dev/null +++ b/test/core.cluster.test.mjs @@ -0,0 +1,236 @@ +import { Eltro as t, assert, stub } from 'eltro' +import fs from 'fs/promises' +import Core from '../core/core.mjs' +import Util from '../core/util.mjs' +import { createFakeLog } from './helpers.mjs' +import StaticProvider from '../core/providers/static.mjs' +import lowdb from '../core/db.mjs' + +const util = new Util(import.meta.url) +const log = createFakeLog() +let db + +t.before(function() { + return lowdb({}, log, null).then(function(res) { + db = res + }) +}) + +t.describe('#init()', function() { + const assertProviderName = 'Kyousuu Gakku Gogyou Kikan' + let core + let fakeUtil + let fakeProvider + let fakeProviderConfig + + function FakeProvider(config) { + fakeProvider(config) + this.static = true + this.checkConfig = fakeProviderConfig + } + + t.beforeEach(function() { + log.error.reset() + core = new Core(db, util, log) + core.util = fakeUtil = { + verifyConfig: stub(), + getAppNames: stub().returns([]), + } + fakeProvider = stub() + fakeProviderConfig = stub() + Core.providers.set(assertProviderName, FakeProvider) + core.isSlave = true + }) + + t.after(function() { + return Promise.all([ + fs.rm('./log_test_1.log', { recursive: true, force: true }).catch(function() {}), + ]) + }) + + t.test('should not call provider checkConfig if isSlave', async function() { + const assertAppName = 'Zetsubou' + const assertConfig = { + [assertAppName]: { + provider: assertProviderName, + } + } + db.config = assertConfig + process.env.CLUSTER_APP_NAME = assertAppName + + const assertError = new Error('Shousou') + fakeProviderConfig.rejects(assertError) + fakeUtil.getAppNames.returns([assertAppName]) + + await core.init() + assert.notOk(fakeProviderConfig.called) + }) + + t.test('should only create an application with the environment app name', async function() { + const assertAppName = 'Yasashii Ketsumatsu' + const assertAppSecondaryName = 'Big Time' + const assertTestString = 'Serozore no Omoi' + const assertConfig = { + [assertAppName]: { + provider: assertProviderName, + teststring: assertTestString, + }, + [assertAppSecondaryName]: { + provider: assertProviderName, + teststring: assertTestString, + }, + } + db.config = assertConfig + process.env.CLUSTER_APP_NAME = assertAppName + + fakeUtil.getAppNames.returns([assertAppSecondaryName, assertAppName]) + assert.strictEqual(core.applications.length, 0) + + await core.init() + + assert.strictEqual(core.applications.length, 1) + assert.ok(core.getApplication(assertAppName)) + assert.notOk(core.getApplication(assertAppSecondaryName)) + + let application = core.getApplication(assertAppName) + assert.strictEqual(core.applications[0], application) + assert.strictEqual(application.name, assertAppName) + assert.strictEqual(application.ctx.db, core.db) + assert.strictEqual(application.ctx.util, core.util) + assert.notStrictEqual(application.ctx.log, core.log) + assert.strictEqual(application.ctx.core, core) + assert.strictEqual(application.config.teststring, assertTestString) + assert.ok(application.fresh) + assert.ok(application.provider instanceof FakeProvider) + }) +}) + +t.describe('#run()', function() { + let core + let testAppOneName + let testAppTwoName + let stubRunApplication + + t.beforeEach(function() { + testAppOneName = 'Tenshi' + testAppTwoName = 'no CLOVER' + db.data.core = { + [testAppOneName]: { + versions: [] + }, + [testAppTwoName]: { + versions: [] + }, + } + core = new Core(db, util, log) + core.runApplication = stubRunApplication = stub().resolves() + core.isSlave = true + db.write = stub().resolves() + log.info.reset() + log.warn.reset() + log.error.reset() + for (let name of [testAppOneName, testAppTwoName]) { + let onOrOnce = stub() + let app = { + name: name, + fresh: false, + on: onOrOnce, + once: onOrOnce, + ctx: { + log: { + info: stub(), + warn: stub(), + error: stub(), + fatal: stub(), + }, + }, + runVersion: stub().resolves(), + update: stub().resolves(), + startAutoupdater: stub(), + } + core.applicationMap.set(name, app) + core.applications.push(app) + } + process.env.CLUSTER_APP_NAME = testAppTwoName + }) + + t.test('should not call update', async function() { + const assertFirstError = new Error('Manatsu') + const assertSecondError = new Error('no Photograph') + core.applicationMap.get(testAppOneName).update.rejects(assertFirstError) + core.applicationMap.get(testAppTwoName).update.rejects(assertSecondError) + + await core.run() + + assert.notOk(core.applications[0].update.called) + assert.notOk(core.applications[1].update.called) + }) + + + t.test('app.on updated should not be hooked', async function() { + assert.notOk(core.applicationMap.get(testAppOneName).once.called) + assert.notOk(core.applicationMap.get(testAppTwoName).once.called) + assert.notOk(core.applicationMap.get(testAppOneName).on.called) + assert.notOk(core.applicationMap.get(testAppTwoName).on.called) + + core.runApplication.returnWith(function(app) { + assert.notOk(app.once.called) + assert.notOk(app.on.called) + return Promise.resolve() + }) + + await core.run() + + assert.notOk(core.applicationMap.get(testAppOneName).once.called) + assert.notOk(core.applicationMap.get(testAppTwoName).once.called) + assert.notOk(core.applicationMap.get(testAppOneName).on.called) + assert.notOk(core.applicationMap.get(testAppTwoName).on.called) + }) + + t.test('should not call startAutoupdater', async function() { + stubRunApplication.rejects(new Error('not seen')) + assert.notOk(core.applications[0].startAutoupdater.called) + assert.notOk(core.applications[1].startAutoupdater.called) + + await core.run() + + assert.notOk(core.applications[0].startAutoupdater.called) + assert.notOk(core.applications[1].startAutoupdater.called) + }) + + t.test('should be safe to call multiple times', async function() { + await core.run() + + assert.strictEqual(core.applicationMap.get(testAppOneName).runVersion.callCount, 0) + assert.strictEqual(core.applicationMap.get(testAppTwoName).runVersion.callCount, 1) + + await core.run() + await core.run() + await core.run() + await core.run() + + assert.strictEqual(core.applicationMap.get(testAppOneName).runVersion.callCount, 0) + assert.strictEqual(core.applicationMap.get(testAppTwoName).runVersion.callCount, 1) + }) + + t.test('should call runVersion on selected application', async function() { + const assertFirstError = new Error('Manatsu') + const assertSecondError = new Error('no Photograph') + const assertVersion = 'v11.22.33' + core.applicationMap.get(testAppOneName).runVersion.rejects(assertFirstError) + core.applicationMap.get(testAppTwoName).runVersion.rejects(assertSecondError) + + process.env.CLUSTER_APP_NAME = testAppTwoName + process.env.CLUSTER_APP_VERSION = assertVersion + + let err = await assert.isRejected(core.run()) + + let app = core.applicationMap.get(testAppTwoName) + assert.strictEqual(core.applicationMap.get(testAppOneName).runVersion.callCount, 0) + assert.strictEqual(app.runVersion.callCount, 1) + assert.strictEqual(app.runVersion.firstCall[0], assertVersion) + assert.strictEqual(app.ctx.log.fatal.callCount, 1) + assert.strictEqual(app.ctx.log.fatal.firstCall[0], assertSecondError) + assert.strictEqual(err, assertSecondError) + }) +}) diff --git a/test/core.test.integration.mjs b/test/core.test.integration.mjs index c34a8c7..302baff 100644 --- a/test/core.test.integration.mjs +++ b/test/core.test.integration.mjs @@ -1,6 +1,6 @@ import { Eltro as t, assert} from 'eltro' import fs from 'fs/promises' -import http from 'http' +import HttpServer from '../core/http.mjs' import Util from '../core/util.mjs' import { request } from '../core/client.mjs' import { setTimeout } from 'timers/promises' @@ -10,438 +10,520 @@ import getLog from '../core/log.mjs' const util = new Util(import.meta.url) const port = 61412 +const turnDebuggingOn = false -t.timeout(10000).describe('', function() { - let server = null - let prefix = `http://localhost:${port}/` - let files = [] - let logs = [] - let versions = [] - let processor - let integrationLog = getLog('test.integration', []) - let compressorPath = util.getPathFromRoot('./7za.exe') - if (process.platform !== 'win32') { - compressorPath = util.getPathFromRoot('./7zas') - } +const runners = [ + ['runner.mjs', 'testapp'], + ['runner_cluster.mjs', 'testappcluster'], +] - t.before(function(cb) { - server = http.createServer(function(req, res) { - req.on('error', function(err) { - integrationLog.error(err, 'error') - }) - res.on('error', function(err) { - integrationLog.error(err, 'error') - }) - - integrationLog.info('[SERVER] got request ' + req.url) - - if (req.url === '/releases') { - res.statusCode = 200 - let output = versions.map(x => { - return { - name: x[0], - body: x[1], - assets: [{ - name: x[2], - browser_download_url: prefix + 'files/' + x[2] - }] - } +runners.forEach(function([runnerName, appname]) { + t.timeout(10000).describe(runnerName, function() { + let wasSuccessful = false + let http = null + let server = null + let prefix = `http://localhost:${port}/` + let files = [util.getPathFromRoot('./testappcluster')] + let logs = [] + let allLogs = [] + let versions = [] + let processor + let integrationLog = getLog('test.integration', []) + let compressorPath = util.getPathFromRoot('./7za.exe') + if (process.platform !== 'win32') { + compressorPath = util.getPathFromRoot('./7zas') + } + + t.before(function() { + http = new HttpServer() + + server = http.createServer(function(req, res) { + req.on('error', function(err) { + integrationLog.error(err, 'error') }) - res.end(JSON.stringify(output)); - return - } else if (req.url.startsWith('/files')) { - let filename = req.url.substring(req.url.lastIndexOf('/') + 1) - return fs.open(util.getPathFromRoot('./' + filename)) - .then(function(file) { - pipeline(file.createReadStream(), res, function(err) { - if (err) { - console.log(err) - res.statusCode = 404 - res.end(JSON.stringify({ error: 'unknown url' })) + res.on('error', function(err) { + integrationLog.error(err, 'error') + }) + + integrationLog.info('[SERVER] got request ' + req.url) + + if (req.url === '/releases') { + res.statusCode = 200 + let output = versions.map(x => { + return { + name: x[0], + body: x[1], + assets: [{ + name: x[2], + browser_download_url: prefix + 'files/' + x[2] + }] } }) - }).catch(function(err) { - console.log(err) - res.statusCode = 404 - res.end(JSON.stringify({ error: 'unknown url' })) - }) + res.end(JSON.stringify(output)); + return + } else if (req.url.startsWith('/files')) { + let filename = req.url.substring(req.url.lastIndexOf('/') + 1) + return fs.open(util.getPathFromRoot('./' + filename)) + .then(function(file) { + pipeline(file.createReadStream(), res, function(err) { + if (err) { + console.log(err) + res.statusCode = 404 + res.end(JSON.stringify({ error: 'unknown url' })) + } + }) + }).catch(function(err) { + console.log(err) + res.statusCode = 404 + res.end(JSON.stringify({ error: 'unknown url' })) + }) + } + + res.statusCode = 404 + res.end(JSON.stringify({ error: 'unknown url' })) + }) + + return fs.rm(util.getPathFromRoot('./db.json'), { force: true }) + .then(function() { + return server.listenAsync(port) + }) + }) + + t.after(function() { + if (!turnDebuggingOn && !wasSuccessful) { + for (let i = 0; i < allLogs.length; i++) { + prettyPrintMessage(allLogs[i]) + } } + + return Promise.all(files.map(function(file) { + return fs.rm(file, { force: true, recursive: true }) + })) + .then(function() { + if (processor && !processor.exitCode) { + processor.kill() - res.statusCode = 404 - res.end(JSON.stringify({ error: 'unknown url' })) - }) - - fs.rm(util.getPathFromRoot('./db.json'), { force: true }).then(function() { - server.listen(port, cb) - }, cb) - }) - - t.after(function() { - return Promise.all(files.map(function(file) { - return fs.rm(file, { force: true, recursive: true }) - })) - .then(function() { - if (processor && !processor.exitCode) { - processor.kill() - } - }) - }) - - const version_1_stable = ` - export function start(http, port, ctx) { - const server = http.createServer(function (req, res) { - res.writeHead(200); - res.end(JSON.stringify({ version: 'v1' })) - }) - - return server.listenAsync(port, '0.0.0.0') - .then(() => { - ctx.log.info({ port: port, listening: true }, \`Server is listening on \${port} serving v1\`) + return waitUntilClosed() + } + }).then(function() { + return http.closeServer() }) - } - ` - - const version_2_nolisten = ` - export function start(http, port, ctx) { - } - ` - - const version_3_crashing = ` - export function start(http, port, ctx) { - process.exit(1) - } - ` - - const version_4_stable = ` - export function start(http, port, ctx) { - const server = http.createServer(function (req, res) { - res.writeHead(200); - res.end(JSON.stringify({ version: 'v4' })) }) - - return server.listenAsync(port, '0.0.0.0') - .then(() => { - ctx.log.info({ port: port, listening: true }, \`Server is listening on \${port} serving v4\`) + + const version_1_stable = ` + export function start(http, port, ctx) { + const server = http.createServer(function (req, res) { + res.writeHead(200); + res.end(JSON.stringify({ version: 'v1' })) }) - } - ` - - function file(relative) { - let file = util.getPathFromRoot(relative) - files.push(file) - return file - } - - function log(message) { - let lines = message.split('\n') - for (let line of lines) { - if (!line.trim()) continue - logs.push(line) + + return server.listenAsync(port, '0.0.0.0') + .then(() => { + ctx.log.info({ port: port, listening: true }, \`Server is listening on \${port} serving v1\`) + }) } - } - - function parseLine(line) { - if (line[0] === '{') { - return JSON.parse(line) + ` + + const version_2_nolisten = ` + export function start(http, port, ctx) { } - return { - msg: line + ` + + const version_3_crashing = ` + export function start(http, port, ctx) { + process.exit(1) } - } - - - let logIndex = 0 - function catchupLog(ms = 0) { - if (logs.length > logIndex) { - for (; logIndex < logs.length; logIndex++) { - prettyPrintMessage(logs[logIndex]) + ` + + const version_4_stable = ` + export function start(http, port, ctx) { + const server = http.createServer(function (req, res) { + res.writeHead(200); + res.end(JSON.stringify({ version: 'v4' })) + }) + + return server.listenAsync(port, '0.0.0.0') + .then(() => { + ctx.log.info({ port: port, listening: true }, \`Server is listening on \${port} serving v4\`) + }) + } + ` + + function file(relative) { + let file = util.getPathFromRoot(relative) + files.push(file) + return file + } + + function log(message) { + let lines = message.split('\n') + for (let line of lines) { + if (!line.trim()) continue + logs.push(line) + allLogs.push(line) } } - if (ms > 0) { - return setTimeout(ms) + + function parseLine(line) { + if (line[0] === '{') { + return JSON.parse(line) + } + return { + msg: line + } } - } - - integrationLog.on('newlog', function(record) { - prettyPrintMessage(JSON.stringify(record)) - }) - - let logWaitIndex = 0 - function hasLogLine(regMatch) { - if (logs.length > logWaitIndex) { - for (; logWaitIndex < logs.length; logWaitIndex++) { + + + let logIndex = 0 + function catchupLog(ms = 0) { + if (logs.length > logIndex) { + for (; logIndex < logs.length; logIndex++) { + if (turnDebuggingOn) { + prettyPrintMessage(logs[logIndex]) + } + } + } + if (ms > 0) { + return setTimeout(ms) + } + } + + integrationLog.on('newlog', function(record) { + if (turnDebuggingOn) { + prettyPrintMessage(JSON.stringify(record)) + } + }) + + let logWaitIndex = 0 + function hasLogLine(regMatch) { + if (logs.length > logWaitIndex) { + for (; logWaitIndex < logs.length; logWaitIndex++) { + if (typeof(regMatch) === 'function') { + let res = regMatch(parseLine(logs[logWaitIndex])) + if (res) return true + } + else if (logs[logWaitIndex].match(regMatch)) { + return true + } + } + } + return false + } + + function findInLogs(regMatch) { + for (let i = 0; i < logs.length; i++) { if (typeof(regMatch) === 'function') { - let res = regMatch(parseLine(logs[logWaitIndex])) + let res = regMatch(parseLine(logs[i])) if (res) return true } - else if (logs[logWaitIndex].match(regMatch)) { + else if (logs[i].match(regMatch)) { return true } } } - return false - } - function findInLogs(regMatch) { - for (let i = 0; i < logs.length; i++) { - if (typeof(regMatch) === 'function') { - let res = regMatch(parseLine(logs[i])) - if (res) return true + async function waitUntilListening() { + let listeningLine = null + while (processor.exitCode == null + && !hasLogLine((rec) => { listeningLine = rec; return rec.listening && rec.port })) { + await catchupLog(10) } - else if (logs[i].match(regMatch)) { - return true - } - } - } - - async function waitUntilListening() { - let listeningLine = null - while (processor.exitCode == null - && !hasLogLine((rec) => { listeningLine = rec; return rec.listening && rec.port })) { - await catchupLog(10) - } - catchupLog() - if (listeningLine.listening && listeningLine.port) { - return listeningLine - } else { - return null - } - } - - async function waitUntilClosed(listening) { - while (true) { catchupLog() - try { - await request({}, `http://localhost:${listening.port}/`) - } catch (err) { - break + if (listeningLine.listening && listeningLine.port) { + return listeningLine + } else { + return null } - await setTimeout(25) - } - catchupLog() - - logs.splice(0, logs.length); logIndex = 0; logWaitIndex = 0; console.log('\n-------\n') - } - - function startRunner() { - return util.runCommandBackground('node', ['runner.mjs'], util.getPathFromRoot('./'), log) - } - - t.test('should be fully operational', async function() { - console.log() - - let index = file('./index.mjs') - await fs.writeFile(index, version_1_stable) - await util.runCommand(compressorPath, ['a', file('./v1-sc.7z'), index], util.getPathFromRoot('./testapp')) - - processor = startRunner() - - - while (processor.exitCode == null) { - await catchupLog(10) } - catchupLog() - - let secondLast = parseLine(logs[logs.length - 2]) - let last = parseLine(logs[logs.length - 1]) - assert.match(secondLast.msg, /creating/i) - assert.match(secondLast.msg, /application/i) - assert.match(secondLast.msg, /testapp/i) - assert.match(secondLast.msg, /0 releases/i) - assert.match(last.err.message, /none/i) - assert.match(last.err.message, /successful/i) - - // Reset our log - logs.splice(0, logs.length); logIndex = 0; logWaitIndex = 0; console.log('\n-------\n') - - const assertNameVersion1 = 'v1_ok' - - file(`./testapp/${assertNameVersion1}`) - - versions.splice(0, 0, [assertNameVersion1, 'ok version', 'v1-sc.7z']) - processor = startRunner() - - let listening = await waitUntilListening() - - let checkListening = await request({}, `http://localhost:${listening.port}/`) - assert.strictEqual(checkListening.body.version, 'v1') - - while (!hasLogLine(/core is running/)) { - await catchupLog(10) + async function sendRequestToApplication(listening) { + let lastErr = null + for (let i = 0; i < 3; i++) { + try { + let checkListening = await request({}, `http://localhost:${listening.port}/`) + return checkListening + } catch (err) { + lastErr = err + await setTimeout(150) + } + } + log('-- core.test.integration.mjs crash here --') + log(lastErr.toString()) + throw lastErr } - - catchupLog() - - let db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) - assert.strictEqual(db.core.testapp.active, assertNameVersion1) - assert.strictEqual(db.core.testapp.versions.length, 1) - assert.strictEqual(db.core.testapp.versions[0].stable, 1) - assert.strictEqual(db.core.testapp.versions[0].installed, true) - - // Create our second version - await fs.writeFile(index, version_2_nolisten) - await util.runCommand(compressorPath, ['a', file('./v2-sc.7z'), index], util.getPathFromRoot('./testapp')) - - const assertNameVersion2 = 'v2_nolisten' - file(`./testapp/${assertNameVersion2}`) - versions.splice(0, 0, [assertNameVersion2, 'no listen version', 'v2-sc.7z']) - - // wait a second for it to trigger an update - - await setTimeout(500) - - while (!hasLogLine(/Error starting v2_nolisten/)) { - await catchupLog(10) + + async function waitUntilClosed(listening) { + while (true) { + catchupLog() + try { + await request({}, `http://localhost:${listening.port}/`) + } catch (err) { + break + } + await setTimeout(25) + } + catchupLog() + + logs.splice(0, logs.length); logIndex = 0; logWaitIndex = 0; + if (turnDebuggingOn) { console.log('\n-------\n') } } - while (!hasLogLine(/restart request.*v2_nolisten.*dirty/)) { - await catchupLog(10) + + function startRunner() { + return util.runCommandBackground('node', [runnerName], util.getPathFromRoot('./'), log) } - while (processor.exitCode == null) { - await catchupLog(10) - } - catchupLog() + + t.test('should be fully operational', async function() { + console.log() + if (!turnDebuggingOn) { console.log('Running empty test') } - db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) - assert.strictEqual(db.core.testapp.active, assertNameVersion2) - assert.strictEqual(db.core.testapp.versions.length, 2) - assert.strictEqual(db.core.testapp.versions[0].stable, -1) - assert.strictEqual(db.core.testapp.versions[0].installed, true) - assert.strictEqual(db.core.testapp.versions[1].stable, 1) - assert.strictEqual(db.core.testapp.versions[1].installed, true) + let index = file('./index.mjs') + await fs.writeFile(index, version_1_stable) + await util.runCommand(compressorPath, ['a', file('./v1-sc.7z'), index], util.getPathFromRoot('./testapp')) - // Since application was in dirty state, on next attempt should attempt to - // run v2 again and then falling back to v1 + processor = startRunner() - await waitUntilClosed() - processor = startRunner() - await catchupLog(10) - while (!hasLogLine(/Attempting to run version v2_nolisten/)) { - await catchupLog(10) - } - while (!hasLogLine(/Attempting to run version v1_ok/)) { - await catchupLog(10) - } - - listening = await waitUntilListening() - assert.ok(listening) - checkListening = await request({}, `http://localhost:${listening.port}/`) - assert.strictEqual(checkListening.body.version, 'v1') - - while (!hasLogLine(/core is running/)) { + while (processor.exitCode == null) { + await catchupLog(10) + } + + catchupLog() + + let secondLast = parseLine(logs[logs.length - 2]) + let last = parseLine(logs[logs.length - 1]) + assert.match(secondLast.msg, /creating/i) + assert.match(secondLast.msg, /application/i) + assert.match(secondLast.msg, /testapp/i) + assert.match(secondLast.msg, /0 releases/i) + assert.match(last.err.message, /none/i) + assert.match(last.err.message, /successful/i) + + // Reset our log + logs.splice(0, logs.length); logIndex = 0; logWaitIndex = 0; + if (turnDebuggingOn) { console.log('\n-------\n') } + + const assertNameVersion1 = 'v1_ok' + + if (!turnDebuggingOn) { console.log(`Running update ${assertNameVersion1} test`) } + + file(`./testapp/${assertNameVersion1}`) + + versions.splice(0, 0, [assertNameVersion1, 'ok version', 'v1-sc.7z']) + processor = startRunner() + + let listening = await waitUntilListening() + + let checkListening = await sendRequestToApplication(listening) + assert.strictEqual(checkListening.body.version, 'v1') + + while (!hasLogLine(/core is running/)) { + await catchupLog(10) + if (processor.exitCode !== null) { + throw new Error('Process exited with ' + processor.exitCode) + } + } + + catchupLog() + + let db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) + assert.strictEqual(db.core[appname].active, assertNameVersion1) + assert.strictEqual(db.core[appname].versions.length, 1) + assert.strictEqual(db.core[appname].versions[0].stable, 1) + assert.strictEqual(db.core[appname].versions[0].installed, true) + + // Create our second version + await fs.writeFile(index, version_2_nolisten) + await util.runCommand(compressorPath, ['a', file('./v2-sc.7z'), index], util.getPathFromRoot('./testapp')) + + const assertNameVersion2 = 'v2_nolisten' + if (!turnDebuggingOn) { console.log(`Running update ${assertNameVersion2} test`) } + file(`./testapp/${assertNameVersion2}`) + versions.splice(0, 0, [assertNameVersion2, 'no listen version', 'v2-sc.7z']) + + // wait a second for it to trigger an update + + await setTimeout(500) + + while (!hasLogLine(/Error starting v2_nolisten/)) { + await catchupLog(10) + } + + if (appname !== 'testappcluster') { + while (!hasLogLine(/restart request.*v2_nolisten.*dirty/)) { + await catchupLog(10) + } + + while (processor.exitCode == null) { + await catchupLog(10) + } + catchupLog() + + db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) + assert.strictEqual(db.core[appname].active, assertNameVersion2) + assert.strictEqual(db.core[appname].versions.length, 2) + assert.strictEqual(db.core[appname].versions[0].stable, -1) + assert.strictEqual(db.core[appname].versions[0].installed, true) + assert.strictEqual(db.core[appname].versions[1].stable, 1) + assert.strictEqual(db.core[appname].versions[1].installed, true) + + // Since application was in dirty state, on next attempt should attempt to + // run v2 again and then falling back to v1 + + await waitUntilClosed() + + if (!turnDebuggingOn) { console.log(`Running fresh ${assertNameVersion2} test`) } + processor = startRunner() + await catchupLog(10) + + while (!hasLogLine(/Attempting to run version v2_nolisten/)) { + await catchupLog(10) + } + } + + while (!hasLogLine(/Attempting to run version v1_ok/)) { + await catchupLog(10) + } + + listening = await waitUntilListening() + assert.ok(listening) + checkListening = await sendRequestToApplication(listening) + assert.strictEqual(checkListening.body.version, 'v1') + + while (!hasLogLine(/is up and running/)) { + await setTimeout(10) + } + + db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) + assert.strictEqual(db.core[appname].active, assertNameVersion1) + assert.strictEqual(db.core[appname].versions.length, 2) + assert.strictEqual(db.core[appname].versions[0].stable, -2) + assert.strictEqual(db.core[appname].versions[1].stable, 1) + + assert.ok(findInLogs(/Attempting to run version v2_nolisten/)) + assert.ok(findInLogs(/Error starting v2_nolisten/)) + + processor.kill() + + if (!turnDebuggingOn) { console.log(`Running version stability check test`) } + + await waitUntilClosed() + processor = startRunner() + + listening = await waitUntilListening() + + assert.ok(listening) + + checkListening = await sendRequestToApplication(listening) + assert.strictEqual(checkListening.body.version, 'v1') + + while (!hasLogLine(/is up and running/)) { + await setTimeout(10) + } + + db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) + assert.strictEqual(db.core[appname].active, assertNameVersion1) + assert.strictEqual(db.core[appname].versions.length, 2) + assert.strictEqual(db.core[appname].versions[0].stable, -2) + assert.strictEqual(db.core[appname].versions[1].stable, 1) + + assert.notOk(findInLogs(/Attempting to run version v2_nolisten/)) + assert.notOk(findInLogs(/Error starting v2_nolisten/)) + + // Create our third version + await fs.writeFile(index, version_3_crashing) + await util.runCommand(compressorPath, ['a', file('./v3-sc.7z'), index], util.getPathFromRoot('./testapp')) + + const assertNameVersion3 = 'v3_crash' + if (!turnDebuggingOn) { console.log(`Running update ${assertNameVersion3} test`) } + file(`./testapp/${assertNameVersion3}`) + versions.splice(0, 0, [assertNameVersion3, 'crash version', 'v3-sc.7z']) + + // wait a second for it to trigger an update + + await setTimeout(500) + + while (!hasLogLine(/Attempting to run version v3_crash/)) { + await catchupLog(10) + } + + if (appname !== 'testappcluster') { + while (processor.exitCode == null) { + await catchupLog(10) + } + + db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) + assert.strictEqual(db.core[appname].active, assertNameVersion3) + assert.strictEqual(db.core[appname].versions.length, 3) + assert.strictEqual(db.core[appname].versions[0].stable, -2) + assert.strictEqual(db.core[appname].versions[1].stable, -2) + assert.strictEqual(db.core[appname].versions[2].stable, 1) + + catchupLog() + + // Should recover afterwards + await waitUntilClosed() + processor = startRunner() + + listening = await waitUntilListening() + + assert.ok(listening) + + checkListening = await sendRequestToApplication(listening) + assert.strictEqual(checkListening.body.version, 'v1') + + while (!hasLogLine(/core is running/)) { + await setTimeout(10) + } + } else { + while (!hasLogLine(/Attempting to run version v1_ok/)) { + await catchupLog(10) + } + while (!hasLogLine(/is up and running/)) { + await setTimeout(10) + } + + db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) + assert.strictEqual(db.core[appname].active, assertNameVersion1) + assert.strictEqual(db.core[appname].versions.length, 3) + assert.strictEqual(db.core[appname].versions[0].stable, -2) + assert.strictEqual(db.core[appname].versions[1].stable, -2) + assert.strictEqual(db.core[appname].versions[2].stable, 1) + } + + // Create our fourth version + await fs.writeFile(index, version_4_stable) + await util.runCommand(compressorPath, ['a', file('./v4-sc.7z'), index], util.getPathFromRoot('./testapp')) + + const assertNameVersion4 = 'v4_stable' + if (!turnDebuggingOn) { console.log(`Running update ${assertNameVersion4} test`) } + file(`./testapp/${assertNameVersion4}`) + versions.splice(0, 0, [assertNameVersion4, 'no listen version', 'v4-sc.7z']) + + // wait a second for it to trigger an update + + await setTimeout(500) + + while (!hasLogLine(/Attempting to run version v4_stable/)) { + await catchupLog(10) + } + + while (!hasLogLine(/Server is listening on 31313 serving v4/)) { + await catchupLog(10) + } + + catchupLog() + + checkListening = await sendRequestToApplication(listening) + assert.strictEqual(checkListening.body.version, 'v4') + await setTimeout(10) - } - - db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) - assert.strictEqual(db.core.testapp.active, assertNameVersion1) - assert.strictEqual(db.core.testapp.versions.length, 2) - assert.strictEqual(db.core.testapp.versions[0].stable, -2) - assert.strictEqual(db.core.testapp.versions[1].stable, 1) - - assert.ok(findInLogs(/Attempting to run version v2_nolisten/)) - assert.ok(findInLogs(/Error starting v2_nolisten/)) - - processor.kill() - - await waitUntilClosed() - processor = startRunner() - - listening = await waitUntilListening() - - assert.ok(listening) - - checkListening = await request({}, `http://localhost:${listening.port}/`) - assert.strictEqual(checkListening.body.version, 'v1') - - while (!hasLogLine(/core is running/)) { - await setTimeout(10) - } - - db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) - assert.strictEqual(db.core.testapp.active, assertNameVersion1) - assert.strictEqual(db.core.testapp.versions.length, 2) - assert.strictEqual(db.core.testapp.versions[0].stable, -2) - assert.strictEqual(db.core.testapp.versions[1].stable, 1) - - assert.notOk(findInLogs(/Attempting to run version v2_nolisten/)) - assert.notOk(findInLogs(/Error starting v2_nolisten/)) - - // Create our third version - await fs.writeFile(index, version_3_crashing) - await util.runCommand(compressorPath, ['a', file('./v3-sc.7z'), index], util.getPathFromRoot('./testapp')) - - const assertNameVersion3 = 'v3_crash' - file(`./testapp/${assertNameVersion3}`) - versions.splice(0, 0, [assertNameVersion3, 'crash version', 'v3-sc.7z']) - - // wait a second for it to trigger an update - - await setTimeout(500) - - while (!hasLogLine(/Attempting to run version v3_crash/)) { - await catchupLog(10) - } - - while (processor.exitCode == null) { - await catchupLog(10) - } - - db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) - assert.strictEqual(db.core.testapp.active, assertNameVersion3) - assert.strictEqual(db.core.testapp.versions.length, 3) - assert.strictEqual(db.core.testapp.versions[0].stable, -2) - assert.strictEqual(db.core.testapp.versions[1].stable, -2) - assert.strictEqual(db.core.testapp.versions[2].stable, 1) - - catchupLog() - - // Should recover afterwards - await waitUntilClosed() - processor = startRunner() - - listening = await waitUntilListening() - - assert.ok(listening) - - checkListening = await request({}, `http://localhost:${listening.port}/`) - assert.strictEqual(checkListening.body.version, 'v1') - - while (!hasLogLine(/core is running/)) { - await setTimeout(10) - } - - // Create our fourth version - await fs.writeFile(index, version_4_stable) - await util.runCommand(compressorPath, ['a', file('./v4-sc.7z'), index], util.getPathFromRoot('./testapp')) - - const assertNameVersion4 = 'v4_stable' - file(`./testapp/${assertNameVersion4}`) - versions.splice(0, 0, [assertNameVersion4, 'no listen version', 'v4-sc.7z']) - - // wait a second for it to trigger an update - - await setTimeout(500) - - while (!hasLogLine(/Attempting to run version v4_stable/)) { - await catchupLog(10) - } - - while (!hasLogLine(/Server is listening on 31313 serving v4/)) { - await catchupLog(10) - } - - catchupLog() - - checkListening = await request({}, `http://localhost:${listening.port}/`) - assert.strictEqual(checkListening.body.version, 'v4') - - await setTimeout(10) - - db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) - assert.strictEqual(db.core.testapp.active, assertNameVersion4) - assert.strictEqual(db.core.testapp.versions.length, 4) - assert.strictEqual(db.core.testapp.versions[0].stable, 1) - assert.strictEqual(db.core.testapp.versions[1].stable, -2) + + db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json'))) + assert.strictEqual(db.core[appname].active, assertNameVersion4) + assert.strictEqual(db.core[appname].versions.length, 4) + assert.strictEqual(db.core[appname].versions[0].stable, 1) + assert.strictEqual(db.core[appname].versions[1].stable, -2) + wasSuccessful = true + }) }) }) diff --git a/test/core.test.mjs b/test/core.test.mjs index 71b6536..0c8a1be 100644 --- a/test/core.test.mjs +++ b/test/core.test.mjs @@ -239,6 +239,7 @@ t.describe('#constructor()', function() { assert.strictEqual(core.restart, assertRestarter) assert.deepStrictEqual(core.applications, []) assert.ok(core.applicationMap) + assert.strictEqual(core.isSlave, false) }) }) @@ -965,7 +966,6 @@ t.describe('#runApplication()', function() { assert.notStrictEqual(err, assertError) assert.match(err.message, /found/) }) - t.test('should throw if no version are found', async function() { const assertError = new Error('Daikichi to Rin') @@ -1016,7 +1016,6 @@ t.describe('#runApplication()', function() { assert.ok(stubWrite.called) }) - t.test('should succeed if running a minus one on fresh', async function() { const assertError = new Error('Daikichi to Rin') let count = 0 diff --git a/test/helpers.mjs b/test/helpers.mjs index 233b7d0..5f6b608 100644 --- a/test/helpers.mjs +++ b/test/helpers.mjs @@ -77,7 +77,7 @@ export function prettyPrintMessage(line) { if (line[0] === '{') { try { let rec = JSON.parse(line) - console.log(`[${rec.time.substr(11).replace('Z', '')}] ${style(levels[rec.level], levelcolor[rec.level])}: ${rec.name}: ${style(rec.msg, 'cyan')}`) + console.log(`[${rec.time.substr(11).replace('Z', '')}] ${style(levels[rec.level], levelcolor[rec.level])}: ${rec.name} ${style('(pid: ' + rec.pid + ')', 'grey')}: ${style(rec.msg, 'cyan')}`) if (rec.err && rec.err.message && rec.err.stack) { let err = new Error(rec.err.message) err.stack = rec.err.stack diff --git a/test/runner_cluster.mjs b/test/runner_cluster.mjs new file mode 100644 index 0000000..c8a501c --- /dev/null +++ b/test/runner_cluster.mjs @@ -0,0 +1,36 @@ +import fs from 'fs' +import cluster from 'cluster' +import { runner } from '../core/runner.mjs' + +if (cluster.isPrimary) { + try { + fs.rmSync(util.getPathFromRoot('./db.json')) + } catch {} +} + +runner(import.meta.url, { + name: 'test-runner-cluster', + testappcluster: { + cluster: 2, + port: 31313, + provider: 'git', + url: 'http://localhost:61412/releases', + updateEvery: 0.014, + heartbeatTimeout: 100, + heartbeatAttempts: 3, + heartbeatAttemptsWait: 100, + log: [{ + stream: 'process.stdout', + level: 'info', + }], + } +}, 'db.json') +.then( + function(core) { + core.log.info('core is running') + }, + function(err) { + runner.log.error(err, 'Error starting runner') + process.exit(1) + } +) diff --git a/test/util.test.mjs b/test/util.test.mjs index 09db98b..e31ceea 100644 --- a/test/util.test.mjs +++ b/test/util.test.mjs @@ -195,6 +195,35 @@ t.describe('#getApplications()', function() { assert.deepStrictEqual(util.getAppNames(getBase({ heartbeatAttemptsWait: {} })), []) }) + t.test('should fail to find if clusterWaitOnCrash is defined but not a valid number', function() { + assert.deepStrictEqual(util.getAppNames(getBase()), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: null })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: 5 })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: 15 })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: 1000 })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: 'asdf' })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: '1234' })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: 0 })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: -5 })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: [] })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ clusterWaitOnCrash: {} })), []) + }) + + t.test('should fail to find if cluster is defined but not a valid number', function() { + assert.deepStrictEqual(util.getAppNames(getBase()), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: null })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: 1 })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: 5 })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: 15 })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: 1000 })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: 'asdf' })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: '1234' })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: 0 })), ['app']) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: -5 })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: [] })), []) + assert.deepStrictEqual(util.getAppNames(getBase({ cluster: {} })), []) + }) + t.test('should fail to find if heartbeatPath is defined but not a valid string', function() { assert.deepStrictEqual(util.getAppNames(getBase()), ['app']) assert.deepStrictEqual(util.getAppNames(getBase({ heartbeatPath: null })), ['app'])