More core testing and some development and fixes

2022-02-14 08:15:50 +00:00 · 2022-02-14 08:15:50 +00:00 · e540a54844
parent 4f4bc8cf6a
commit e540a54844
12 changed files with 809 additions and 167 deletions
--- a/core/application.mjs
+++ b/core/application.mjs
@ -65,7 +65,7 @@ export default class Application extends EventEmitter {

  updateLog(message) {
    this.ctx.db.data.core[this.name].updater += message
-    this.ctx.db.log.info(message)
+    this.ctx.log.info(message)
    return message
  }

@ -76,18 +76,25 @@ export default class Application extends EventEmitter {
      if (this.ctx.db.data.core[this.name].updater !== this.msgStatic) {
        this.ctx.db.data.core[this.name].updater = ''
        this.updateLog(this.msgStatic)
-        return this.ctx.db.write()
+        return this.ctx.db.write().then(function() { return null })
      }
-      return Promise.resolve()
+      return Promise.resolve(null)
    }

-    if (this.updating) return
+    if (this.updating) return null
    this.updating = true

    return this._update()
-      .then(() => {
+      .then((result) => {
        this.updating = false
        return this.ctx.db.write()
+          .then(function() { return result })
+      })
+      .then((result) => {
+        if (result) {
+          this.emit('updated', result)
+        }
+        return result
      })
      .catch((err) => {
        this.updating = false
@ -124,7 +131,7 @@ export default class Application extends EventEmitter {
      // If the versino matches the latest installed, then there's nothing to do
      if (this.ctx.db.data.core[this.name].latestInstalled === latest.version) {
        this.updateLog('Already up to date, nothing to do. ')
-        return
+        return null
      }
      
      // Make the id for the vesion the version number. Allows for easy lookup
@ -137,7 +144,7 @@ export default class Application extends EventEmitter {
        // Check if the existing version found was already installed.
        if (found.installed) {
          this.updateLog('Version was already installed, nothing to do. ')
-          return
+          return null
        }

        // We found existing on, update the keys of the one in the databse with
@ -153,12 +160,12 @@ export default class Application extends EventEmitter {
        // listed and avoid at all cost.
        if (latest.failtodownload && latest.failtodownload > 3) {
          this.updateLog('Version failed to download too many times, skipping this version. ')
-          return
+          return null
        }

        if (latest.failtoinstall && latest.failtoinstall > 3) {
          this.updateLog('Version failed to install too many times, skipping this version. ')
-          return
+          return null
        }

        // Combine the logs
@ -269,6 +276,7 @@ export default class Application extends EventEmitter {
    this.ctx.db.data.core[this.name].latestInstalled = latest.version
    latest.installed = true
    latest.log = log
+    return latest
  }

  registerModule(module, version = '') {
--- a/core/core.mjs
+++ b/core/core.mjs
@ -36,6 +36,7 @@ export default class Core {
    if (!util || !(util instanceof Util)) throw new Error('util not instance of Util')
    if (!db || !(db instanceof Low)) throw new Error('db not instance of Low')

+    this.running = false
    this.db = db
    this.util = util
    this.log = log
@ -56,8 +57,6 @@ export default class Core {

    this.log.info(`Found applications: ${names.join(', ')}.`)

-    let lastError = null
-
    for (let name of names) {
      try {
        let provConstructor = Core.providers.get(this.db.config[name].provider)
@ -74,20 +73,22 @@ export default class Core {
        this.applicationMap.set(name, application)
      } catch (err) {
        this.log.error(err, `Error creating application ${name} with provider ${this.db.config[name].provider}: ${err.message}`)
-        lastError = err
      }
    }

    if (names.length && !this.applications.length) {
-      throw lastError
+      return Promise.reject(new Error('None of the application were successful in running'))
    }
  }

  async run() {
+    if (this.running) return
+    this.running = true
+
    this.log.info(`Running updater on ${this.applications.length} apps`)
    await Promise.all(this.applications.map((app) => {
      return app.update().catch(err => {
-        this.log.error(err, `Error updating ${app.name}: ${err.message}`)
+        app.ctx.log.error(err, `Error updating: ${err.message}`)
      })
    }))

@ -101,9 +102,11 @@ export default class Core {
          found = true
        },
        err => {
-          this.log.error(err, `Error running application ${app.name}: ${err.message}`)
+          app.ctx.log.error(err, `Error running: ${err.message}`)
        }
      )
+
+      app.on('updated', this.runApplication.bind(this, app))
    }

    if (!found) {
@ -126,14 +129,16 @@ export default class Core {
      await application.closeServer()

      try {
-        this.log.info(`Attempting to run application ${name} version ${version.version}`)
+        application.ctx.log.info(`Attempting to run version ${version.version}`)
        await application.runVersion(version.version)
        found = true
+        version.stable = 1
+        await this.db.write()
        break
      } catch(err) {
-        version.stable--
+        version.stable = Math.min(version.stable, 0) - 1
        await this.db.write()
-        this.log.error(err, `Error starting ${name} ${version.version}: ${err.message}`)
+        application.ctx.log.error(err, `Error starting ${version.version}: ${err.message}`)
      }
    }

--- a/core/runner.mjs
+++ b/core/runner.mjs
@ -0,0 +1,36 @@
+import Util from './util.mjs'
+import fs from 'fs/promises'
+import getLog from './log.mjs'
+import GetDB from './db.mjs'
+import Core from './core.mjs'
+
+
+export async function runner(root_import_meta_url, configname = 'config.json', dbname = 'db.json') {
+  if (!root_import_meta_url) {
+    throw new Error('ServiceRunner must be called with the full string from "import.meta.url" from a file residing in the root directory')
+  }
+  const util = new Util(root_import_meta_url)
+
+  let config = configname
+  if (typeof(config) === 'string') {
+    let fullpath = util.getPathFromRoot('./' + config)
+
+    try {
+      config = JSON.parse(await fs.readFile(fullpath))
+    } catch (err) {
+      throw new Error(`critical error opening ${fullpath}: ${err.message}`)
+    }
+  }
+  
+  const log = getLog(config.name)
+  runner.log = log
+  const db = await GetDB(config, log, util.getPathFromRoot('./' + dbname))
+  
+  const core = new Core(db, util, log, function() {})
+  await core.init()
+  await core.run()
+
+  return core
+}
+
+runner.log = getLog('runner')
--- a/core/util.mjs
+++ b/core/util.mjs
@ -1,6 +1,7 @@
+import os from 'os'
 import path from 'path'
 import fs from 'fs/promises'
-import { spawn } from 'child_process'
+import { spawn, execSync } from 'child_process'
 import { fileURLToPath, pathToFileURL } from 'url'

 export default class Util {
@ -136,6 +137,9 @@ export default class Util {
      processor.stderr.on('data', function(data) {
        stream(data.toString().replace(/\r\n/g, '\n'))
      })
+      processor.stdin.on('error', function() {
+        clearInterval(timeOuter)
+      })
      processor.on('error', function(err) {
        clearInterval(timeOuter)
        rej(err)
@ -149,4 +153,46 @@ export default class Util {
      })
    })
  }
+
+  runCommandBackground(command, options = [], folder = null, stream = function() {}) {
+    let fullcommand = path.join(folder ? folder : '', command)
+    if (command.indexOf('/') >= 0 || command.indexOf('\\') >= 0) {
+      fullcommand = command
+    }
+    stream(`[Command] ${fullcommand} ${options.join(' ')}\n`)
+    let processor = spawn(command, options, {
+      shell: true,
+      cwd: folder,
+    })
+    let timeOuter = setInterval(function() {
+      try {
+        processor.stdin.write('n\n')
+      } catch {}
+    }, 250)
+    processor.stdout.on('data', function(data) {
+      stream(data.toString().replace(/\r\n/g, '\n'))
+    })
+    processor.stderr.on('data', function(data) {
+      stream(data.toString().replace(/\r\n/g, '\n'))
+    })
+    processor.stdin.on('error', function() {
+      clearInterval(timeOuter)
+    })
+    processor.on('error', function(err) {
+      clearInterval(timeOuter)
+    })
+    processor.on('exit', function (code) {
+      clearInterval(timeOuter)
+    })
+
+    processor._kill = processor.kill
+    processor.kill = function() {
+      if(os.platform() === 'win32'){
+        execSync('taskkill /pid ' + processor.pid + ' /T /F')
+      }else{
+        processor.kill();  
+      }
+    }
+    return processor
+  }
 }
--- a/test/application.test.integration.mjs
+++ b/test/application.test.integration.mjs
@ -32,7 +32,7 @@ t.after(function() {
    })
 })

-t.timeout(10000).test('should run update and install correctly', async function(){
+t.skip().timeout(10000).test('should run update and install correctly', async function(){
  try {
    await app.update()
  } catch (err) {
--- a/test/application.test.mjs
+++ b/test/application.test.mjs
@ -265,10 +265,11 @@ t.timeout(250).describe('#update()', function() {
    provider = new StaticProvider()
    app = new Application(ctx, provider, 'teststatic')

-    stubWrite.reset()
+    stubWrite.reset().resolves()

-    await app.update()
+    let result = await app.update()

+    assert.strictEqual(result, null)
    assert.match(ctx.db.data.core.teststatic.updater, /static/i)
    assert.match(ctx.db.data.core.teststatic.updater, /nothing/i)
    let old = ctx.db.data.core.teststatic.updater
@ -276,16 +277,19 @@ t.timeout(250).describe('#update()', function() {
    assert.strictEqual(stubWrite.callCount, 1)
    
    await app.update()
+    assert.strictEqual(result, null)
    assert.strictEqual(ctx.db.data.core.teststatic.updater, old)
    assert.strictEqual(stubWrite.callCount, 1)

    ctx.db.data.core.teststatic.updater = 'asdf'
    
    await app.update()
+    assert.strictEqual(result, null)
    assert.strictEqual(ctx.db.data.core.teststatic.updater, old)
    assert.strictEqual(stubWrite.callCount, 2)
    
    await app.update()
+    assert.strictEqual(result, null)
    assert.strictEqual(ctx.db.data.core.teststatic.updater, old)
    assert.strictEqual(stubWrite.callCount, 2)
  })
@ -305,10 +309,11 @@ t.timeout(250).describe('#update()', function() {
    assert.strictEqual(app.updating, true)
    assert.strictEqual(provider.getLatestVersion.callCount, 1)

-    app.update()
+    let result = await app.update()
    await setImmediate()
    
    assert.strictEqual(provider.getLatestVersion.callCount, 1)
+    assert.strictEqual(result, null)
  })

  t.test('should check for latest version', async function() {
@ -494,10 +499,13 @@ t.timeout(250).describe('#update()', function() {
  t.test('should not call npm install if stat fails to find index.mjs', async function() {
    const assertError = new Error('File not found')
    const assertTarget = util.getPathFromRoot('./testapp/123456789/index.mjs')
+    const stubUpdated = stub()
    stubRunCommand.rejects(new Error('should not be seen'))
    stubFsStat.rejects(assertError)
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 0)

+    app.once('updated', stubUpdated)
+
    let err = await assert.isRejected(app.update())

    assert.strictEqual(app.updating, false)
@ -519,6 +527,7 @@ t.timeout(250).describe('#update()', function() {

    err = await assert.isRejected(app.update())
    
+    assert.notOk(stubUpdated.called)
    assert.strictEqual(app.updating, false)
    assert.strictEqual(err, assertError)
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 1)
@ -538,10 +547,12 @@ t.timeout(250).describe('#update()', function() {
    })
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 0)

-    await app.update()
+    let result = await new Promise(function(res, rej) {
+      app.once('updated', res)
+      app.update().catch(rej)
+    })

    assert.strictEqual(app.updating, false)
-
    assert.strictEqual(stubFsStat.callCount, 2)
    assert.strictEqual(stubFsStat.secondCall[0], assertTarget)
    assert.ok(stubExtract.called)
@ -552,6 +563,7 @@ t.timeout(250).describe('#update()', function() {
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 1)
    let version = ctx.db.data.core.testapp.versions[0]
    assert.ok(version.log)
+    assert.strictEqual(result, version)
    assert.match(version.log, /package\.json/i)
    assert.match(version.log, /contain/i)
    assert.ok(version.log.endsWith('\n'))
@ -563,8 +575,9 @@ t.timeout(250).describe('#update()', function() {

    provider.downloadVersion.reset()

-    await app.update()
+    result = await app.update()

+    assert.strictEqual(result, null)
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 1)
    assert.notOk(provider.downloadVersion.called)
    assert.match(ctx.db.data.core.testapp.updater, /already/i)
@ -578,6 +591,7 @@ t.timeout(250).describe('#update()', function() {
    const assertError = new Error('Nagisa')
    const assertTarget = util.getPathFromRoot('./testapp/123456789')
    const assertTargetCheck = util.getPathFromRoot('./testapp/123456789/')
+    const stubUpdated = stub()
    provider.getLatestVersion.resolves(assertVersion)
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 0)

@ -590,8 +604,11 @@ t.timeout(250).describe('#update()', function() {
      return Promise.reject(assertError)
    })

+    app.once('updated', stubUpdated)
+
    let err = await assert.isRejected(app.update())

+    assert.notOk(stubUpdated.called)
    assert.strictEqual(app.updating, false)
    assert.strictEqual(err, assertError)
    assert.strictEqual(stubRunCommand.firstCall[0], 'npm.cmd')
@ -641,12 +658,16 @@ t.timeout(250).describe('#update()', function() {
    assert.notStrictEqual(ctx.db.data.core.testapp.latestInstalled, assertVersion.version)
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 0)

-    await app.update()
+    let result = await new Promise(function(res, rej) {
+      app.once('updated', res)
+      app.update().catch(rej)
+    })

    assert.strictEqual(app.updating, false)
    assert.strictEqual(ctx.db.data.core.testapp.latestInstalled, assertVersion.version)
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 1)
    assert.strictEqual(ctx.db.data.core.testapp.versions[0], assertVersion)
+    assert.strictEqual(result, assertVersion)
    assert.ok(assertVersion.log)
    assert.ok(stubWrite.callCount >= 4)
    assert.strictEqual(assertVersion.installed, true)
@ -663,6 +684,7 @@ t.timeout(250).describe('#update()', function() {
    const assertNewLink = 'THE last pain'
    const assertNewFilename = 'The place of hope.7z'
    const assertStable = 100
+    const stubUpdated = stub()
    const oldLog = 'The Smell of Sea\n'
    const assertVersion = { version: '123456789', link: 'httplinkhere', filename: 'test.7z', log: oldLog, stable: assertStable }
    
@ -672,13 +694,19 @@ t.timeout(250).describe('#update()', function() {
    provider.getLatestVersion.resolves({ version: assertVersion.version, link: assertNewLink, filename: assertNewFilename })
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 1)

-    await app.update()
+    app.once('updated', stubUpdated)
+
+    let result = await app.update()
+
+    assert.ok(stubUpdated.called)
+    assert.strictEqual(result, stubUpdated.firstCall[0])
    
    assert.ok(assertVersion.log)
    assert.ok(stubWrite.callCount >= 4)
    assert.strictEqual(app.updating, false)
    assert.strictEqual(ctx.db.data.core.testapp.versions.length, 1)
    assert.strictEqual(ctx.db.data.core.testapp.versions[0], assertVersion)
+    assert.strictEqual(result, assertVersion)
    assert.strictEqual(assertVersion.stable, assertStable)
    assert.ok(assertVersion.log)
    assert.ok(assertVersion.log.startsWith(oldLog))
@ -694,13 +722,23 @@ t.timeout(250).describe('#update()', function() {
  t.test('should do nothing if latestInstalled matches version', async function() {
    const assertError = new Error('should not be seen')
    const assertVersion = { version: '999.888.777.666', filename: 'test.7z' }
+    const stubUpdated = stub()
    provider.getLatestVersion.resolves(assertVersion)
    provider.downloadVersion.rejects(assertError)
    ctx.db.data.core.testapp.updater = ''
    ctx.db.data.core.testapp.latestInstalled = assertVersion.version

-    await app.update()
+    app.once('updated', stubUpdated)
+
+    let result = await app.update()
+
+    assert.strictEqual(result, null)
+    assert.notOk(stubUpdated.called)
+
+    result = await app.update()
    
+    assert.notOk(stubUpdated.called)
+    assert.strictEqual(result, null)
    assert.ok(stubWrite.callCount >= 1)
    assert.strictEqual(app.updating, false)
    assert.notOk(provider.downloadVersion.called)
@ -711,6 +749,7 @@ t.timeout(250).describe('#update()', function() {
  t.test('should do nothing if installed version is found', async function() {
    const assertError = new Error('should not be seen')
    const assertVersion = { version: '999.888.777.666', filename: 'test.7z' }
+    const stubUpdated = stub()
    provider.getLatestVersion.resolves(assertVersion)
    provider.downloadVersion.rejects(assertError)
    ctx.db.data.core.testapp.updater = ''
@ -720,8 +759,11 @@ t.timeout(250).describe('#update()', function() {
    ctx.db.upsert(ctx.db.data.core.testapp.versions, { id: '999.888.777.666', installed: true })
    ctx.db.upsert(ctx.db.data.core.testapp.versions, { id: '333.333.333.333', installed: true })

+    app.once('updated', stubUpdated)
+
    await app.update()
    
+    assert.notOk(stubUpdated.called)
    assert.ok(stubWrite.callCount >= 1)
    assert.strictEqual(app.updating, false)
    assert.notOk(provider.downloadVersion.called)
@ -732,13 +774,18 @@ t.timeout(250).describe('#update()', function() {
  t.test('should do nothing it exists and failtodownload is higher than 3', async function() {
    const assertError = new Error('should not be seen')
    const assertVersion = { version: '999.888.777.666', filename: 'test.7z' }
+    const stubUpdated = stub()
    provider.getLatestVersion.resolves(assertVersion)
    provider.downloadVersion.rejects(assertError)
    ctx.db.data.core.testapp.updater = ''
    ctx.db.upsert(ctx.db.data.core.testapp.versions, { id: '999.888.777.666', version: '999.888.777.666', link: 'httplinkhere', filename: 'test.7z', failtodownload: 4 })

-    await app.update()
+    app.once('updated', stubUpdated)
+
+    let result = await app.update()
    
+    assert.notOk(stubUpdated.called)
+    assert.strictEqual(result, null)
    assert.ok(stubWrite.callCount >= 1)
    assert.strictEqual(app.updating, false)
    assert.notOk(provider.downloadVersion.called)
@ -750,13 +797,18 @@ t.timeout(250).describe('#update()', function() {
  t.test('should do nothing it exists and failtoinstall is higher than 3', async function() {
    const assertError = new Error('should not be seen')
    const assertVersion = { version: '999.888.777.666', filename: 'test.7z' }
+    const stubUpdated = stub()
    provider.getLatestVersion.resolves(assertVersion)
    provider.downloadVersion.rejects(assertError)
    ctx.db.data.core.testapp.updater = ''
    ctx.db.upsert(ctx.db.data.core.testapp.versions, { id: '999.888.777.666', version: '999.888.777.666', link: 'httplinkhere', filename: 'test.7z', failtoinstall: 4 })

-    await app.update()
+    app.once('updated', stubUpdated)
+
+    let result = await app.update()
    
+    assert.notOk(stubUpdated.called)
+    assert.strictEqual(result, null)
    assert.ok(stubWrite.callCount >= 1)
    assert.strictEqual(app.updating, false)
    assert.notOk(provider.downloadVersion.called)
--- a/test/core.test.integration.mjs
+++ b/test/core.test.integration.mjs
@ -3,20 +3,22 @@ import fs from 'fs/promises'
 import http from 'http'
 import Util from '../core/util.mjs'
 import { request } from '../core/client.mjs'
+import { setTimeout } from 'timers/promises'
+import { prettyPrintMessage } from './helpers.mjs'
+import { pipeline } from 'stream'

 const util = new Util(import.meta.url)
 const port = 61412
-const defaultHandler = function(req, res) {
-  res.statusCode = 200
-  res.end('{"a":1}');
-}
-let server = null
-let prefix = `http://localhost:${port}/`
-let handler = defaultHandler
-let files = []
-let logs = []

-t.describe('', function() {
+t.timeout(5000).describe('', function() {
+  let server = null
+  let prefix = `http://localhost:${port}/`
+  let files = []
+  let logs = []
+  let versions = []
+  let requests = []
+  let processor
+
  t.before(function(cb) {
    server = http.createServer(function(req, res) {
      req.on('error', function(err) {
@ -25,32 +27,87 @@ t.describe('', function() {
      res.on('error', function(err) {
        console.log('error', err)
      })
-      handler(req, res)
+
+      requests.push(req.url)
+
+      console.log(req.url)
+
+      if (req.url === '/releases') {
+        res.statusCode = 200
+        let output = versions.map(x => {
+          return {
+            name: x[0],
+            body: x[1],
+            assets: [{
+              name: x[2],
+              browser_download_url: prefix + 'files/' + x[2]
+            }]
+          }
+        })
+        res.end(JSON.stringify(output));
+        return
+      } else if (req.url.startsWith('/files')) {
+        let filename =  req.url.substring(req.url.lastIndexOf('/') + 1)
+        return fs.open(util.getPathFromRoot('./' + filename))
+        .then(function(file) {
+          pipeline(file.createReadStream(), res, function(err) {
+            if (err) {
+              console.log(err)        
+              res.statusCode = 404
+              res.end(JSON.stringify({ error: 'unknown url' }))
+            }
+          })
+        }).catch(function(err) {
+          console.log(err)        
+          res.statusCode = 404
+          res.end(JSON.stringify({ error: 'unknown url' }))
+        })
+      }
+      
+      res.statusCode = 404
+      res.end(JSON.stringify({ error: 'unknown url' }))
    })
-    server.listen(port, cb)
+
+    fs.rm(util.getPathFromRoot('./db.json'), { force: true }).then(function() {
+      server.listen(port, cb)
+    }, cb)
  })

  t.after(function() {
-    return Promise.resolve()
    return Promise.all(files.map(function(file) {
      return fs.rm(file, { force: true, recursive: true })
    }))
+    .then(function() {
+      if (processor) {
+        processor.kill()
+      }
+    })
  })

-  const stable_version_1 = `
+  const version_1_stable = `
  export function start(http, port, ctx) {
    const server = http.createServer(function (req, res) {
      res.writeHead(200);
-      res.end(JSON.stringify({ version: 'stable_version_1' }))
+      res.end(JSON.stringify({ version: 'v1' }))
    })

    return server.listenAsync(port, '0.0.0.0')
      .then(() => {
-        ctx.log.info(\`Server is listening on \${port} serving stable_version_1\`)
+        ctx.log.info({ port: port, listening: true }, \`Server is listening on \${port} serving v1\`)
      })
  }
  `

+  const version_2_nolisten = `
+  export function start(http, port, ctx) {
+  }
+  `
+
+  const version_3_crashing = `
+  export function start(http, port, ctx) {
+  }
+  `
+
  function file(relative) {
    let file = util.getPathFromRoot(relative)
    files.push(file)
@ -58,13 +115,245 @@ t.describe('', function() {
  }

  function log(message) {
-    logs.push(message)
-    console.log(message)
+    let lines = message.split('\n')
+    for (let line of lines) {
+      if (!line.trim()) continue
+      logs.push(line)
+    }
+  }
+
+  function parseLine(line) {
+    if (line[0] === '{') {
+      return JSON.parse(line)
+    }
+    return {
+      msg: line
+    }
+  }
+
+
+  let logIndex = 0
+  function catchupLog() {
+    if (logs.length > logIndex) {
+      for (; logIndex < logs.length; logIndex++) {
+        prettyPrintMessage(logs[logIndex])
+      }
+    }
+  }
+
+  let logWaitIndex = 0
+  function hasLogLine(regMatch) {
+    if (logs.length > logWaitIndex) {
+      for (; logWaitIndex < logs.length; logWaitIndex++) {
+        if (typeof(regMatch) === 'function') {
+          let res = regMatch(parseLine(logs[logWaitIndex]))
+          if (res) return true
+        }
+        else if (logs[logWaitIndex].match(regMatch)) {
+          return true
+        }
+      }
+    }
+    return false
+  }
+  
+  function findInLogs(regMatch) {
+    for (let i = 0; i < logs.length; i++) {
+      if (typeof(regMatch) === 'function') {
+        let res = regMatch(parseLine(logs[i]))
+        if (res) return true
+      }
+      else if (logs[i].match(regMatch)) {
+        return true
+      }
+    }
+  }
+
+  async function waitUntilListening() {
+    let listeningLine = null
+    while (processor.exitCode == null
+        && !hasLogLine((rec) => { listeningLine = rec; return rec.listening && rec.port })) {
+      catchupLog()
+      await setTimeout(10)
+    }
+    catchupLog()
+    return listeningLine 
+  }
+
+  async function waitUntilClosed(listening) {
+    while (true) {
+      catchupLog()
+      try {
+        await request({}, `http://localhost:${listening.port}/`)
+      } catch (err) {
+        break
+      }
+      await setTimeout(25)
+    }
+    catchupLog()
+
+    logs.splice(0, logs.length); logIndex = 0; logWaitIndex = 0; console.log('\n-------\n')
+  }
+
+  function startRunner() {
+    return util.runCommandBackground('node', ['runner.mjs'], util.getPathFromRoot('./'), log)
  }

  t.test('should be fully operational', async function() {
+    console.log()
+
    let index = file('./index.mjs')
-    await fs.writeFile(index, stable_version_1)
-    await util.runCommand(util.get7zipExecutable(), ['a', file('./v1.7z'), index], util.getPathFromRoot('./testapp/'), log)
+    await fs.writeFile(index, version_1_stable)
+    await util.runCommand(util.get7zipExecutable(), ['a', file('./v1-sc.7z'), index], util.getPathFromRoot('./testapp'))
+
+    processor = startRunner()
+
+
+    while (processor.exitCode == null) {
+      catchupLog()
+      await setTimeout(10)
+    }
+
+    catchupLog()
+
+    let secondLast = parseLine(logs[logs.length - 2])
+    let last = parseLine(logs[logs.length - 1])
+    assert.match(secondLast.msg, /creating/i)
+    assert.match(secondLast.msg, /application/i)
+    assert.match(secondLast.msg, /testapp/i)
+    assert.match(secondLast.msg, /0 releases/i)
+    assert.match(last.err.message, /none/i)
+    assert.match(last.err.message, /successful/i)
+
+    // Reset our log
+    logs.splice(0, logs.length); logIndex = 0; logWaitIndex = 0; console.log('\n-------\n')
+
+    const assertNameVersion1 = 'v1_ok'
+
+    file(`./testapp/${assertNameVersion1}`)
+
+    versions.splice(0, 0, [assertNameVersion1, 'ok version', 'v1-sc.7z'])
+    processor = startRunner()
+
+    let listening = await waitUntilListening()
+
+    let checkListening = await request({}, `http://localhost:${listening.port}/`)
+    assert.strictEqual(checkListening.body.version, 'v1')
+
+    while (!hasLogLine(/core is running/)) {
+      await setTimeout(10)
+    }
+
+    let db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json')))
+    assert.strictEqual(db.core.testapp.active, assertNameVersion1)
+    assert.strictEqual(db.core.testapp.versions.length, 1)
+    assert.strictEqual(db.core.testapp.versions[0].stable, 1)
+    assert.strictEqual(db.core.testapp.versions[0].installed, true)
+
+    // Create our second version
+    await fs.writeFile(index, version_2_nolisten)
+    await util.runCommand(util.get7zipExecutable(), ['a', file('./v2-sc.7z'), index], util.getPathFromRoot('./testapp'))
+    
+    const assertNameVersion2 = 'v2_nolisten'
+    file(`./testapp/${assertNameVersion2}`)
+    versions.splice(0, 0, [assertNameVersion2, 'no listen version', 'v2-sc.7z'])
+    requests.splice(0, requests.length)
+    assert.strictEqual(requests.length, 0)
+
+    // wait a second for it to trigger an update
+
+    await setTimeout(500)
+
+    while (requests.length < 2) {
+      catchupLog()
+      await setTimeout(25)
+    }
+
+    while (!hasLogLine(/Error starting v2_nolisten/)) {
+      catchupLog()
+      await setTimeout(10)
+    }
+
+    while (!hasLogLine(/Attempting to run version v1_ok/)) {
+      catchupLog()
+      await setTimeout(10)
+    }
+
+    while (!hasLogLine(/Server is listening on 31313 serving v1/)) {
+      catchupLog()
+      await setTimeout(10)
+    }
+
+    catchupLog()
+
+    checkListening = await request({}, `http://localhost:${listening.port}/`)
+    assert.strictEqual(checkListening.body.version, 'v1')
+
+    await setTimeout(10)
+    
+    db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json')))
+    assert.strictEqual(db.core.testapp.active, assertNameVersion1)
+    assert.strictEqual(db.core.testapp.versions.length, 2)
+    assert.strictEqual(db.core.testapp.versions[0].stable, -1)
+    assert.strictEqual(db.core.testapp.versions[0].installed, true)
+    assert.strictEqual(db.core.testapp.versions[1].stable, 1)
+    assert.strictEqual(db.core.testapp.versions[1].installed, true)
+
+    processor.kill()
+    // Wait for ports to be marked as closed
+    await waitUntilClosed()
+
+    processor = startRunner()
+
+    listening = await waitUntilListening()
+
+    assert.ok(listening)
+
+    checkListening = await request({}, `http://localhost:${listening.port}/`)
+    assert.strictEqual(checkListening.body.version, 'v1')
+
+    while (!hasLogLine(/core is running/)) {
+      await setTimeout(10)
+    }
+
+    db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json')))
+    assert.strictEqual(db.core.testapp.active, assertNameVersion1)
+    assert.strictEqual(db.core.testapp.versions.length, 2)
+    assert.strictEqual(db.core.testapp.versions[0].stable, -2)
+    assert.strictEqual(db.core.testapp.versions[1].stable, 1)
+
+    assert.ok(findInLogs(/Attempting to run version v2_nolisten/))
+    assert.ok(findInLogs(/Error starting v2_nolisten/))
+
+    processor.kill()
+    await waitUntilClosed()
+
+    processor = startRunner()
+
+    listening = await waitUntilListening()
+
+    assert.ok(listening)
+
+    checkListening = await request({}, `http://localhost:${listening.port}/`)
+    assert.strictEqual(checkListening.body.version, 'v1')
+    
+    while (!hasLogLine(/core is running/)) {
+      await setTimeout(10)
+    }
+
+    db = JSON.parse(await fs.readFile(util.getPathFromRoot('./db.json')))
+    assert.strictEqual(db.core.testapp.active, assertNameVersion1)
+    assert.strictEqual(db.core.testapp.versions.length, 2)
+    assert.strictEqual(db.core.testapp.versions[0].stable, -2)
+    assert.strictEqual(db.core.testapp.versions[1].stable, 1)
+
+    assert.notOk(findInLogs(/Attempting to run version v2_nolisten/))
+    assert.notOk(findInLogs(/Error starting v2_nolisten/))
+    
+    while (processor.exitCode == null) {
+      catchupLog()
+      await setTimeout(10)
+    }
+    catchupLog()
  })
 })
--- a/test/core.test.mjs
+++ b/test/core.test.mjs
@ -267,6 +267,7 @@ t.describe('#init()', function() {
  }

  t.beforeEach(function() {
+    log.error.reset()
    core = new Core(db, util, log, function() {})
    core.util = fakeUtil = {
      verifyConfig: stub(),
@ -312,7 +313,10 @@ t.describe('#init()', function() {
    fakeUtil.getAppNames.returns([assertAppName])
    
    let err = await assert.isRejected(core.init())
-    assert.strictEqual(err, assertError)
+    assert.notStrictEqual(err, assertError)
+    assert.strictEqual(log.error.firstCall[0], assertError)
+    assert.match(err.message, /successful/i)
+    assert.match(err.message, /none/i)
    assert.strictEqual(fakeProvider.firstCall[0], assertConfig[assertAppName])
  })

@ -330,7 +334,9 @@ t.describe('#init()', function() {
    fakeUtil.getAppNames.returns([assertAppName])
    
    let err = await assert.isRejected(core.init())
-    assert.strictEqual(err, assertError)
+    assert.strictEqual(log.error.firstCall[0], assertError)
+    assert.match(err.message, /successful/i)
+    assert.match(err.message, /none/i)
    assert.strictEqual(fakeProviderConfig.firstCall[0], assertConfig[assertAppName])
  })

@ -433,6 +439,7 @@ t.describe('#run()', function() {
  let testAppOneName
  let testAppTwoName
  let stubRunApplication
+  let stubOnOrOnce
  let stubWrite

  t.beforeEach(function() {
@ -453,9 +460,19 @@ t.describe('#run()', function() {
    log.warn.reset()
    log.error.reset()
    for (let name of [testAppOneName, testAppTwoName]) {
+      let onOrOnce = stub()
      let app = {
        name: name,
        fresh: false,
+        on: onOrOnce,
+        once: onOrOnce,
+        ctx: {
+          log: {
+            info: stub(),
+            warn: stub(),
+            error: stub()
+          },
+        },
        update: stub().resolves(),
        startAutoupdater: stub(),
      }
@ -472,20 +489,81 @@ t.describe('#run()', function() {

    await core.run()
    
-    assert.strictEqual(log.error.callCount, 2)
-    assert.strictEqual(log.error.firstCall[0], assertFirstError)
-    assert.match(log.error.firstCall[1], new RegExp(testAppOneName))
-    assert.match(log.error.firstCall[1], /updat/)
-    assert.match(log.error.firstCall[1], new RegExp(assertFirstError.message))
-    assert.strictEqual(log.error.secondCall[0], assertSecondError)
-    assert.match(log.error.secondCall[1], new RegExp(testAppTwoName))
-    assert.match(log.error.secondCall[1], /updat/)
-    assert.match(log.error.secondCall[1], new RegExp(assertSecondError.message))
+    assert.strictEqual(core.applications[0].ctx.log.error.callCount, 1)
+    assert.strictEqual(core.applications[0].ctx.log.error.firstCall[0], assertFirstError)
+    assert.match(core.applications[0].ctx.log.error.firstCall[1], /updat/)
+    assert.match(core.applications[0].ctx.log.error.firstCall[1], new RegExp(assertFirstError.message))
+    assert.strictEqual(core.applications[1].ctx.log.error.callCount, 1)
+    assert.strictEqual(core.applications[1].ctx.log.error.firstCall[0], assertSecondError)
+    assert.match(core.applications[1].ctx.log.error.firstCall[1], /updat/)
+    assert.match(core.applications[1].ctx.log.error.firstCall[1], new RegExp(assertSecondError.message))
  })
  

+  t.test('app.on updated should be hooked eventually and should call runApplication', async function() {
+    assert.notOk(core.applicationMap.get(testAppOneName).once.called)
+    assert.notOk(core.applicationMap.get(testAppTwoName).once.called)
+    assert.notOk(core.applicationMap.get(testAppOneName).on.called)
+    assert.notOk(core.applicationMap.get(testAppTwoName).on.called)
+    const assertFirstError = new Error('Manatsu')
+    const assertSecondError = new Error('no Photograph')
+    core.applicationMap.get(testAppOneName).update.returnWith(function() {
+      assert.notOk(core.applicationMap.get(testAppOneName).once.called)
+      assert.notOk(core.applicationMap.get(testAppOneName).on.called)
+      return Promise.reject(assertFirstError)
+    })
+    core.applicationMap.get(testAppTwoName).update.returnWith(function() {
+      assert.notOk(core.applicationMap.get(testAppTwoName).once.called)
+      assert.notOk(core.applicationMap.get(testAppOneName).on.called)
+      return Promise.reject(assertSecondError)
+    })
+    core.runApplication.returnWith(function(app) {
+      assert.notOk(app.once.called)
+      assert.notOk(app.on.called)
+      return Promise.resolve()
+    })
+
+    await core.run()
+
+    assert.ok(core.applicationMap.get(testAppOneName).on.called)
+    assert.ok(core.applicationMap.get(testAppTwoName).on.called)
+    assert.strictEqual(core.applicationMap.get(testAppOneName).on.firstCall[0], 'updated')
+    assert.strictEqual(core.applicationMap.get(testAppTwoName).on.firstCall[0], 'updated')
+    
+    assert.strictEqual(core.applications[0].ctx.log.error.callCount, 1)
+    assert.strictEqual(core.applications[0].ctx.log.error.firstCall[0], assertFirstError)
+    assert.match(core.applications[0].ctx.log.error.firstCall[1], /updat/)
+    assert.match(core.applications[0].ctx.log.error.firstCall[1], new RegExp(assertFirstError.message))
+    assert.strictEqual(core.applications[1].ctx.log.error.callCount, 1)
+    assert.strictEqual(core.applications[1].ctx.log.error.firstCall[0], assertSecondError)
+    assert.match(core.applications[1].ctx.log.error.firstCall[1], /updat/)
+    assert.match(core.applications[1].ctx.log.error.firstCall[1], new RegExp(assertSecondError.message))
+
+   assert.strictEqual(stubRunApplication.callCount, 2)
+
+   assert.strictEqual(stubRunApplication.getCallN(1)[0], core.applications[0])
+   assert.strictEqual(stubRunApplication.getCallN(2)[0], core.applications[1])
+
+   core.runApplication.resolves()
+
+   core.applications[0].on.firstCall[1]()
+   assert.strictEqual(stubRunApplication.callCount, 3)
+   assert.strictEqual(stubRunApplication.getCallN(3)[0], core.applications[0])
+   
+   core.applications[0].on.firstCall[1]()
+   assert.strictEqual(stubRunApplication.callCount, 4)
+   assert.strictEqual(stubRunApplication.getCallN(4)[0], core.applications[0])
+
+   
+   core.applications[1].on.firstCall[1]()
+   assert.strictEqual(stubRunApplication.callCount, 5)
+   assert.strictEqual(stubRunApplication.getCallN(5)[0], core.applications[1])
+  })
+
  t.test('should call startAutoupdater on all applications', async function() {
    stubRunApplication.rejects(new Error('not seen'))
+    assert.notOk(core.applications[0].startAutoupdater.called)
+    assert.notOk(core.applications[1].startAutoupdater.called)

    await assert.isRejected(core.run())
    
@ -493,6 +571,19 @@ t.describe('#run()', function() {
    assert.ok(core.applications[1].startAutoupdater.called)
  })

+  t.test('should be safe to call multiple times', async function() {
+    await core.run()
+
+    assert.strictEqual(stubRunApplication.callCount, 2)
+
+    await core.run()
+    await core.run()
+    await core.run()
+    await core.run()
+
+    assert.strictEqual(stubRunApplication.callCount, 2)
+  })
+
  t.test('should call runApplication on all applications', async function() {
    const assertFirstError = new Error('Manatsu')
    const assertSecondError = new Error('no Photograph')
@ -503,17 +594,14 @@ t.describe('#run()', function() {

    await assert.isRejected(core.run())
    
-    assert.strictEqual(log.error.callCount, 2)
-    assert.strictEqual(stubRunApplication.firstCall[0], core.applications[0])
-    assert.strictEqual(stubRunApplication.secondCall[0], core.applications[1])
-    assert.strictEqual(log.error.firstCall[0], assertFirstError)
-    assert.match(log.error.firstCall[1], new RegExp(testAppOneName))
-    assert.match(log.error.firstCall[1], /run/)
-    assert.match(log.error.firstCall[1], new RegExp(assertFirstError.message))
-    assert.strictEqual(log.error.secondCall[0], assertSecondError)
-    assert.match(log.error.secondCall[1], new RegExp(testAppTwoName))
-    assert.match(log.error.secondCall[1], /run/)
-    assert.match(log.error.secondCall[1], new RegExp(assertSecondError.message))
+    assert.strictEqual(core.applications[0].ctx.log.error.callCount, 1)
+    assert.strictEqual(core.applications[0].ctx.log.error.firstCall[0], assertFirstError)
+    assert.match(core.applications[0].ctx.log.error.firstCall[1], /run/)
+    assert.match(core.applications[0].ctx.log.error.firstCall[1], new RegExp(assertFirstError.message))
+    assert.strictEqual(core.applications[1].ctx.log.error.callCount, 1)
+    assert.strictEqual(core.applications[1].ctx.log.error.firstCall[0], assertSecondError)
+    assert.match(core.applications[1].ctx.log.error.firstCall[1], /run/)
+    assert.match(core.applications[1].ctx.log.error.firstCall[1], new RegExp(assertSecondError.message))
  })
  
  t.test('should throw if all applications fail', async function() {
@ -530,7 +618,8 @@ t.describe('#run()', function() {

    await core.run()
    
-    assert.strictEqual(log.error.callCount, 1)
+    assert.strictEqual(core.applications[0].ctx.log.error.callCount, 1)
+    assert.strictEqual(core.applications[1].ctx.log.error.callCount, 0)
  })
  
  t.test('should not throw if all applications are running', async function() {
@ -561,6 +650,13 @@ t.describe('#runApplication()', function() {
    testApp = {
      name: testAppName,
      fresh: false,
+      ctx: {
+        log: {
+          info: stub(),
+          warn: stub(),
+          error: stub(),
+        },
+      },
      closeServer: stub(),
      runVersion: stub(),
    }
@ -597,12 +693,12 @@ t.describe('#runApplication()', function() {

    let err = await assert.isRejected(core.runApplication(testApp))
    assert.notStrictEqual(err, assertError)
-    assert.ok(log.error.called)
+    assert.notOk(log.error.called)
+    assert.ok(testApp.ctx.log.error.called)

-    assert.strictEqual(log.error.firstCall[0], assertError)
-    assert.match(log.error.firstCall[1], new RegExp(testAppName))
-    assert.match(log.error.firstCall[1], new RegExp(assertVersion))
-    assert.match(log.error.firstCall[1], new RegExp(assertError.message))
+    assert.strictEqual(testApp.ctx.log.error.firstCall[0], assertError)
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp(assertVersion))
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp(assertError.message))
    assert.strict(testApp.runVersion.firstCall[0], assertVersion)
    assert.match(err.message, /no/i)
    assert.match(err.message, /found/i)
@ -630,13 +726,13 @@ t.describe('#runApplication()', function() {

    let err = await assert.isRejected(core.runApplication(testApp))
    assert.notStrictEqual(err, assertError)
-    assert.ok(log.error.called)
+    assert.notOk(log.error.called)
+    assert.ok(testApp.ctx.log.error.called)

-    assert.strictEqual(log.error.callCount, 1)
-    assert.strictEqual(log.error.firstCall[0], assertError)
-    assert.match(log.error.firstCall[1], new RegExp(testAppName))
-    assert.match(log.error.firstCall[1], new RegExp('42'))
-    assert.match(log.error.firstCall[1], new RegExp(assertError.message))
+    assert.strictEqual(testApp.ctx.log.error.callCount, 1)
+    assert.strictEqual(testApp.ctx.log.error.firstCall[0], assertError)
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp('42'))
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp(assertError.message))
    assert.strict(testApp.runVersion.firstCall[0], '42')
  })
  
@ -663,13 +759,13 @@ t.describe('#runApplication()', function() {

    let err = await assert.isRejected(core.runApplication(testApp))
    assert.notStrictEqual(err, assertError)
-    assert.ok(log.error.called)
+    assert.notOk(log.error.called)
+    assert.ok(testApp.ctx.log.error.called)

-    assert.strictEqual(log.error.callCount, 2)
-    assert.strictEqual(log.error.firstCall[0], assertError)
-    assert.match(log.error.firstCall[1], new RegExp(testAppName))
-    assert.match(log.error.firstCall[1], new RegExp('31'))
-    assert.match(log.error.firstCall[1], new RegExp(assertError.message))
+    assert.strictEqual(testApp.ctx.log.error.callCount, 2)
+    assert.strictEqual(testApp.ctx.log.error.firstCall[0], assertError)
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp('31'))
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp(assertError.message))
    assert.strict(testApp.runVersion.firstCall[0], '31')
    assert.strict(testApp.runVersion.firstCall[0], '32')
  })
@ -698,17 +794,16 @@ t.describe('#runApplication()', function() {

    let err = await assert.isRejected(core.runApplication(testApp))
    assert.notStrictEqual(err, assertError)
-    assert.ok(log.error.called)
+    assert.notOk(log.error.called)
+    assert.ok(testApp.ctx.log.error.called)

-    assert.strictEqual(log.error.callCount, 2)
-    assert.strictEqual(log.error.firstCall[0], assertError)
-    assert.match(log.error.firstCall[1], new RegExp(testAppName))
-    assert.match(log.error.firstCall[1], new RegExp('30'))
-    assert.match(log.error.firstCall[1], new RegExp(assertError.message))
-    assert.strictEqual(log.error.secondCall[0], assertError)
-    assert.match(log.error.secondCall[1], new RegExp(testAppName))
-    assert.match(log.error.secondCall[1], new RegExp('32'))
-    assert.match(log.error.secondCall[1], new RegExp(assertError.message))
+    assert.strictEqual(testApp.ctx.log.error.callCount, 2)
+    assert.strictEqual(testApp.ctx.log.error.firstCall[0], assertError)
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp('30'))
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp(assertError.message))
+    assert.strictEqual(testApp.ctx.log.error.secondCall[0], assertError)
+    assert.match(testApp.ctx.log.error.secondCall[1], new RegExp('32'))
+    assert.match(testApp.ctx.log.error.secondCall[1], new RegExp(assertError.message))
    assert.strict(testApp.runVersion.firstCall[0], '30')
    assert.strict(testApp.runVersion.firstCall[0], '32')
  })
@ -736,22 +831,62 @@ t.describe('#runApplication()', function() {

    let err = await assert.isRejected(core.runApplication(testApp))
    assert.notStrictEqual(err, assertError)
-    assert.ok(log.error.called)
+    assert.notOk(log.error.called)
+    assert.ok(testApp.ctx.log.error.called)

-    assert.strictEqual(log.error.callCount, 2)
-    assert.strictEqual(log.error.firstCall[0], assertError)
-    assert.match(log.error.firstCall[1], new RegExp(testAppName))
-    assert.match(log.error.firstCall[1], new RegExp('30'))
-    assert.match(log.error.firstCall[1], new RegExp(assertError.message))
-    assert.match(log.error.secondCall[1], new RegExp(testAppName))
-    assert.match(log.error.secondCall[1], new RegExp('31'))
-    assert.match(log.error.secondCall[1], new RegExp(assertError.message))
+    assert.strictEqual(testApp.ctx.log.error.callCount, 2)
+    assert.strictEqual(testApp.ctx.log.error.firstCall[0], assertError)
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp('30'))
+    assert.match(testApp.ctx.log.error.firstCall[1], new RegExp(assertError.message))
+    assert.match(testApp.ctx.log.error.secondCall[1], new RegExp('31'))
+    assert.match(testApp.ctx.log.error.secondCall[1], new RegExp(assertError.message))

    assert.strictEqual(db.data.core[testAppName].versions[0].stable, -1)
    assert.strictEqual(db.data.core[testAppName].versions[1].stable, -2)

    assert.ok(stubWrite.callCount, 2)
  })
+  
+  t.test('should always go to -1 minimum on crash', async function() {
+    const assertError = new Error('Daikichi to Rin')
+    testApp.runVersion.rejects(assertError)
+    testApp.fresh = false
+    db.data.core[testAppName].versions.push({
+      id: '28',
+      version: 'v28',
+      installed: true,
+      stable: 5,
+    }, {
+      id: '29',
+      version: 'v29',
+      installed: true,
+      stable: 1,
+    }, {
+      id: '30',
+      version: 'v30',
+      installed: true,
+      stable: 0,
+    }, {
+      id: '31',
+      version: 'v31',
+      installed: true,
+      stable: -1,
+    })
+
+    let err = await assert.isRejected(core.runApplication(testApp))
+    assert.notStrictEqual(err, assertError)
+    assert.notOk(log.error.called)
+    assert.ok(testApp.ctx.log.error.called)
+
+    assert.strictEqual(testApp.ctx.log.error.callCount, 3)
+
+    assert.strictEqual(db.data.core[testAppName].versions[0].stable, -1)
+    assert.strictEqual(db.data.core[testAppName].versions[1].stable, -1)
+    assert.strictEqual(db.data.core[testAppName].versions[2].stable, -1)
+    assert.strictEqual(db.data.core[testAppName].versions[3].stable, -1)
+
+    assert.ok(stubWrite.callCount, 2)
+  })

  t.test('should throw if no stable version is found', async function() {
    const assertError = new Error('Daikichi to Rin')
@ -805,7 +940,7 @@ t.describe('#runApplication()', function() {
      id: '70',
      version: 'v70',
      installed: true,
-      stable: 0,
+      stable: 100,
    }, {
      id: '71',
      version: 'v71',
@ -820,5 +955,51 @@ t.describe('#runApplication()', function() {

    await core.runApplication(testApp)
    assert.notOk(log.error.called)
+    assert.notOk(testApp.ctx.log.error.called)
+    assert.strictEqual(testApp.runVersion.callCount, 1)
+    assert.strictEqual(testApp.runVersion.firstCall[0], 'v70')
+    assert.strictEqual(db.data.core[testAppName].versions[0].stable, 1)
+    assert.strictEqual(db.data.core[testAppName].versions[1].stable, 0)
+    assert.strictEqual(db.data.core[testAppName].versions[2].stable, 0)
+    assert.ok(stubWrite.called)
+  })
+  
+  
+  t.test('should succeed if running a minus one on fresh', async function() {
+    const assertError = new Error('Daikichi to Rin')
+    let count = 0
+    testApp.fresh = true
+    testApp.runVersion.returnWith(function() {
+      if (count > 0) return Promise.reject(assertError)
+      count++
+      return Promise.resolve()
+    })
+    testApp.fresh = true
+    db.data.core[testAppName].versions.push({
+      id: '70',
+      version: 'v70',
+      installed: true,
+      stable: -1,
+    }, {
+      id: '71',
+      version: 'v71',
+      installed: true,
+      stable: 0,
+    }, {
+      id: '72',
+      version: 'v72',
+      installed: true,
+      stable: 0,
+    })
+
+    await core.runApplication(testApp)
+    assert.notOk(log.error.called)
+    assert.notOk(testApp.ctx.log.error.called)
+    assert.strictEqual(testApp.runVersion.callCount, 1)
+    assert.strictEqual(testApp.runVersion.firstCall[0], 'v70')
+    assert.strictEqual(db.data.core[testAppName].versions[0].stable, 1)
+    assert.strictEqual(db.data.core[testAppName].versions[1].stable, 0)
+    assert.strictEqual(db.data.core[testAppName].versions[2].stable, 0)
+    assert.ok(stubWrite.called)
  })
 })
--- a/test/helpers.mjs
+++ b/test/helpers.mjs
@ -26,4 +26,65 @@ export function createFakeContext(config = { }, util = new Util(import.meta.url)
      core: { },
    }
  })
-}
+}
+
+var colors = {
+  'bold' : [1, 22],
+  'italic' : [3, 23],
+  'underline' : [4, 24],
+  'inverse' : [7, 27],
+  'white' : [37, 39],
+  'grey' : [90, 39],
+  'black' : [30, 39],
+  'blue' : [34, 39],
+  'cyan' : [36, 39],
+  'green' : [32, 39],
+  'magenta' : [35, 39],
+  'red' : [31, 39],
+  'yellow' : [33, 39]
+};
+
+let levels = {
+  10: 'TRACE',
+  20: 'DEBUG',
+  30: 'INFO',
+  40: 'WARN',
+  50: 'ERROR',
+  60: 'FATAL',
+}
+var levelcolor = {
+  10: 'white',    // TRACE
+  20: 'yellow',   // DEBUG
+  30: 'cyan',     // INFO
+  40: 'magenta',  // WARN
+  50: 'red',      // ERROR
+  60: 'inverse',  // FATAL
+};
+
+function style(str, color) {
+  if (!str)
+    return '';
+  var codes = colors[color];
+  if (codes) {
+    return '\x1B[' + codes[0] + 'm' + str +
+           '\x1B[' + codes[1] + 'm';
+  } else {
+    return str;
+  }
+}
+
+export function prettyPrintMessage(line) {
+  if (line[0] === '{') {
+    try {
+      let rec = JSON.parse(line)
+      console.log(`[${rec.time.substr(11).replace('Z', '')}] ${style(levels[rec.level], levelcolor[rec.level])}: ${rec.name}: ${style(rec.msg, 'cyan')}`)
+      if (rec.err && rec.err.message && rec.err.stack) {
+        let err = new Error(rec.err.message)
+        err.stack = rec.err.stack
+        console.log(err)
+      }
+      return
+    } catch (err){ console.log(err)}
+  }
+  console.log(line)
+}
--- a/test/providers/git.test.integration.mjs
+++ b/test/providers/git.test.integration.mjs
@ -3,7 +3,7 @@ import Util from '../../core/util.mjs'
 import fs from 'fs/promises'
 import GitProvider from '../../core/providers/git.mjs'

-t.describe('test', function() {
+t.skip().describe('test', function() {
  t.after(function() {
    return fs.rm('./test/providers/file.7z')
      .catch(function() { })
--- a/test/runner.mjs
+++ b/test/runner.mjs
@ -1,47 +1,28 @@
 import fs from 'fs'
-import Core from '../core/core.mjs'
-import GetDB from '../core/db.mjs'
-import getLog from '../core/log.mjs'
-import Util from '../core/util.mjs'
-
-const name = 'service-core-runner'
-const util = new Util(import.meta.url)
-const log = getLog(name)
-const config = {
-  name: name,
-  testapp: {
-    port: 31313,
-    provider: 'static',
-  }
-}
+import { runner } from '../core/runner.mjs'

 try {
  fs.rmSync(util.getPathFromRoot('./db.json'))
 } catch {}

-GetDB(config, log, util.getPathFromRoot('./db.json')).then(async function(db) {
-  const core = new Core(db, util, log, function() {})
-
-  console.log(db.data)
-
-  await core.init()
-
-  console.log(db.data)
-
-  db.data.core.testapp.versions.push({
-    id: 'ok',
-    version: 'ok',
-    stable: 0,
-    installed: true,
-  })
-
-  await core.run()
-})
+runner(import.meta.url, {
+  name: 'test-runner',
+  testapp: {
+    port: 31313,
+    provider: 'git',
+    url: 'http://localhost:61412/releases',
+    updateEvery: 0.014,
+    heartbeatTimeout: 500,
+    heartbeatAttempts: 2,
+    heartbeatAttemptsWait: 250,
+  }
+}, 'db.json')
 .then(
-  function() {
-    log.info('core is running')
+  function(core) {
+    core.log.info('core is running')
  },
  function(err) {
-    log.error(err, 'Error starting runner')
+    runner.log.error(err, 'Error starting runner')
    process.exit(1)
-  })
+  }
+)
--- a/test/testapp/ok/index.mjs
+++ b/test/testapp/ok/index.mjs
@ -1,17 +0,0 @@
-export function start(http, port, ctx) {
-  const server = http.createServer(function (req, res) {
-    res.writeHead(200);
-    res.end(JSON.stringify({ version: 'exampleindex' }))
-  })
-
-  return new Promise(function(res, rej) {
-    server.listen(port || 4000, '0.0.0.0', function(err) {
-      if (err) {
-        return rej(err)
-      }
-      ctx.log.event.info(`Server is listening on ${port} serving exampleindex`)
-      ctx.log.info(`Server is listening on ${port} serving exampleindex`)
-      res()
-    })
-  })
-}