Initial development version for testing

.gitea/workflows/deploy.yaml

@@ -0,0 +1,44 @@
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  deploy:
+    runs-on: alpine
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Test
+        run: |
+          npm install
+          npm test
+      - name: Check for new release
+        run: |
+          echo ""
+          echo "------------------------------------"
+          echo ""
+
+          CURR_VER="$(cat package.json | jq -r .name)_v$(cat package.json | jq -r .version)"
+          CURR_NAME="$(cat package.json | jq -r .name) v$(cat package.json | jq -r .version)"
+
+          echo "Checking https://git.nfp.is/api/v1/repos/${{ gitea.repository }}/releases for name ${CURR_NAME}"
+
+          if curl -s -X GET -H "Authorization: token ${{ secrets.deploytoken }}" https://git.nfp.is/api/v1/repos/${{ gitea.repository }}/releases | grep -o "\"name\"\:\"${CURR_NAME}\"" > /dev/null; then
+            echo "Skipping ${{ gitea.job }} since $CURR_NAME already exists";
+            exit;
+          fi
+
+          echo "New release ${CURR_VER} found, running npm install..."
+
+          echo "Creating ${CURR_VER} release on gitea"
+          curl \
+            -X POST \
+            -H "Authorization: token ${{ secrets.deploytoken }}" \
+            -H "Content-Type: application/json" \
+            https://git.nfp.is/api/v1/repos/${{ gitea.repository }}/releases \
+            -d "{\"tag_name\":\"${CURR_VER}\",\"name\":\"${CURR_NAME}\",\"body\":\"Automatic release from Appveyor from ${{ gitea.sha }} :\n\n${{ gitea.event.head_commit.message }}\"}"
+
+          echo "//registry.npmjs.org/:_authToken=${{ secrets.npmtoken }}" > ~/.npmrc
+          echo "Publishing new version to npm"
+          npm publish

.npmrc

@@ -0,0 +1 @@
+package-lock=false

index.mjs

@@ -0,0 +1,80 @@
+import fsSyncOriginal from 'fs'
+import fsPromisesOriginal from 'fs/promises'
+import crypto from 'crypto'
+import path from 'path'
+import os from 'os'
+
+export default class FSCache {
+  constructor(options = {}, fsSync, fsPromises) {
+    this.fsSync = fsSync || fsSyncOriginal
+    this.fsPromises = fsPromises || fsPromisesOriginal
+
+    this.id = crypto.randomBytes(15).toString('base64').replace(/\//g, '-')
+    this.parse_json = options.parse_json ?? true
+    this.prefix = options.prefix ? options.prefix + '-' : ''
+    this.hash_alg = options.hash_alg || 'md5'
+    this.cache_dir = options.cache_dir || path.join(os.tmpdir(), this.id)
+
+    // Verify hash algorithm is supported on this system
+    crypto.createHash(this.hash_alg)
+
+    this.fsSync.mkdirSync(this.cache_dir, { recursive: true })
+  }
+
+  _parseCacheData(data, overwrite = {}) {
+    return overwrite.parse_json ?? this.parse_json ? JSON.parse(data) : data
+  }
+
+  _parseSetData(data, overwrite = {}) {
+    return overwrite.parse_json ?? this.parse_json ? JSON.stringify(data) : data
+  }
+
+  hash(name) {
+    return crypto.hash(this.hash_alg, name)
+  }
+
+  get(name, fallback, opts) {
+    return this.fsPromises.readFile(
+      path.join(this.cache_dir, this.hash(name)),
+      { encoding: 'utf8' }
+    )
+    .then(
+      data => this._parseCacheData(data, opts),
+      err => (fallback)
+    )
+  }
+
+  getSync(name, fallback, opts) {
+    let data;
+
+    try {
+      data = this.fsSync.readFileSync(
+        path.join(this.cache_dir, this.hash(name)),
+        { encoding: 'utf8' }
+      )
+    } catch {
+      return fallback
+    }
+    return this._parseCacheData(data, opts)
+  }
+
+  set(name, data, opts = {}) {
+    try {
+      return this.fsPromises.writeFile(
+        path.join(this.cache_dir, this.hash(name)),
+        this._parseSetData(data, opts),
+        { encoding: opts.encoding || 'utf8' }
+      )
+    } catch (err) {
+      return Promise.reject(err)
+    }
+  }
+
+  setSync(name, data, opts = {}) {
+    this.fsSync.writeFileSync(
+      path.join(this.cache_dir, this.hash(name)),
+      this._parseSetData(data, opts),
+      { encoding: opts.encoding || 'utf8' }
+    )
+  }
+}

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "fs-cache-fast",
+  "version": "1.0.0",
+  "description": "Cache stored on the file system",
+  "main": "index.mjs",
+  "scripts": {
+    "test": "eltro",
+    "test:watch": "eltro -r dot -w test"
+  },
+  "watch": {
+    "test": {
+      "patterns": ["./"],
+      "extensions": "mjs"
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://git.nfp.is/TheThing/fs-cache-fast"
+  },
+  "author": "Jonatan Nilsson",
+  "license": "WTFPL",
+  "devDependencies": {
+    "eltro": "^1.4.5"
+  }
+}

test/fscache.integration.test.mjs

@@ -0,0 +1,75 @@
+import { Eltro as t, assert, spy } from 'eltro'
+import { fileURLToPath } from 'url'
+import fs from 'fs/promises'
+import fsSync from 'fs'
+import path from 'path'
+import Cache from '../index.mjs'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+let cache = new Cache({ cache_dir: path.join(__dirname, 'temp') })
+
+t.before(async function() {
+  for (let file of await fs.readdir(cache.cache_dir)) {
+    if (file !== '.gitkeep') {
+      await fs.rm(path.join(cache.cache_dir, file))
+    }
+  }
+})
+
+t.test('get should work', async function() {
+  const testKey = 'get-test-one'
+
+  assert.deepStrictEqual(await cache.get(testKey, 'HELLO'), 'HELLO')
+  await fs.writeFile(path.join(cache.cache_dir, cache.hash(testKey)), JSON.stringify({ a: 1 }))
+  assert.deepStrictEqual(await cache.get(testKey, 'HELLO'), { a: 1 })
+  assert.deepStrictEqual(await cache.get(testKey, 'HELLO', { parse_json: false }), JSON.stringify({ a: 1 }))
+})
+
+t.test('getSync should work', function() {
+  const testKey = 'get-sync-test-one'
+
+  assert.deepStrictEqual(cache.getSync(testKey, 'HELLO'), 'HELLO')
+  fsSync.writeFileSync(path.join(cache.cache_dir, cache.hash(testKey)), JSON.stringify({ b: 2 }))
+  assert.deepStrictEqual(cache.getSync(testKey, 'HELLO'), { b: 2 })
+  assert.deepStrictEqual(cache.getSync(testKey, 'HELLO', { parse_json: false }), JSON.stringify({ b: 2 }))
+})
+
+t.test('set should work', async function() {
+  const testKey = 'set-test-one'
+  const assertPath = path.join(cache.cache_dir, cache.hash(testKey))
+
+  assert.notOk(fsSync.existsSync(assertPath))
+  await cache.set(testKey, { c: 3 })
+  assert.ok(fsSync.existsSync(assertPath))
+  let content = await fs.readFile(assertPath, { encoding: 'utf8' })
+  assert.strictEqual(content, JSON.stringify({ c: 3 }))
+})
+
+t.test('set should work', function() {
+  const testKey = 'set-sync-test-one'
+  const assertPath = path.join(cache.cache_dir, cache.hash(testKey))
+
+  assert.notOk(fsSync.existsSync(assertPath))
+  cache.setSync(testKey, { d: 4 })
+  assert.ok(fsSync.existsSync(assertPath))
+  let content = fsSync.readFileSync(assertPath, { encoding: 'utf8' })
+  assert.strictEqual(content, JSON.stringify({ d: 4 }))
+})
+
+t.test('should all work together', async function() {
+  const testKey = 'hello world'
+  const assertFallback = 'This is fallback'
+  const assertPath = path.join(cache.cache_dir, cache.hash(testKey))
+
+  assert.notOk(fsSync.existsSync(assertPath))
+  assert.strictEqual(await cache.get(testKey, assertFallback), assertFallback)
+  await cache.set(testKey, { e: 5 })
+  assert.notStrictEqual(await cache.get(testKey, assertFallback), assertFallback)
+  assert.deepStrictEqual(await cache.get(testKey), { e: 5 })
+  assert.notStrictEqual(cache.getSync(testKey, assertFallback), assertFallback)
+  assert.deepStrictEqual(cache.getSync(testKey), { e: 5 })
+  cache.setSync(testKey, { f: 6 })
+  assert.deepStrictEqual(await cache.get(testKey), { f: 6 })
+  assert.deepStrictEqual(cache.getSync(testKey), { f: 6 })
+})

test/fscache.test.mjs

@@ -0,0 +1,339 @@
+import { Eltro as t, assert, spy } from 'eltro'
+import path from 'path'
+import crypto from 'crypto'
+import os from 'os'
+import fsSyncOriginal from 'fs'
+import fsPromisesOriginal from 'fs/promises'
+import Cache from '../index.mjs'
+import { fakeFs, fakeFsPromises, fakeFsSync } from './helper.mjs'
+
+let fsSync
+let fsPromises
+
+t.beforeEach(function() {
+  fsSync = fakeFsSync()
+  fsPromises = fakeFsPromises()
+})
+
+function createCache(opts) {
+  return new Cache(opts, fsSync, fsPromises)
+}
+
+t.describe('#constructor()', function() {
+  t.test('uses default fs', function() {
+    let cache = new Cache({})
+    assert.strictEqual(cache.fsSync, fsSyncOriginal)
+    assert.strictEqual(cache.fsPromises, fsPromisesOriginal)
+  })
+
+  t.test('should be able to override the fs', function() {
+    let cache = createCache({})
+    assert.strictEqual(cache.fsSync, fsSync)
+    assert.strictEqual(cache.fsPromises, fsPromises)
+  })
+
+  t.test('comes with default options', function() {
+    let cache = createCache({})
+    assert.ok(cache.id)
+    assert.strictEqual(cache.parse_json, true)
+    assert.strictEqual(cache.prefix, '')
+    assert.strictEqual(cache.hash_alg, 'md5')
+    assert.strictEqual(cache.cache_dir, path.join(os.tmpdir(), cache.id))
+  })
+
+  t.test('can overwrite options', function() {
+    const assertHash = 'sha256'
+    const assertDir = '/something/else'
+    const assertPrefix = 'blabla'
+    const assertParseJson = false
+
+    let cache = createCache({
+      prefix: assertPrefix,
+      hash_alg: assertHash,
+      cache_dir: assertDir,
+      parse_json: assertParseJson,
+    })
+
+    assert.ok(cache.id)
+    assert.strictEqual(cache.parse_json, assertParseJson)
+    assert.strictEqual(cache.prefix, assertPrefix + '-')
+    assert.strictEqual(cache.hash_alg, assertHash)
+    assert.strictEqual(cache.cache_dir, assertDir)
+  })
+
+  t.test('should create the directory by default', function() {
+    assert.notOk(fsSync.mkdirSync.called)
+    let cache = createCache({})
+    assert.ok(fsSync.mkdirSync.called)
+    assert.strictEqual(fsSync.mkdirSync.firstCall[0], cache.cache_dir)
+    assert.strictEqual(fsSync.mkdirSync.firstCall[1]?.recursive, true)
+  })
+
+  t.test('should check if hash_alg is valid', function() {
+    assert.throws(function() {
+      createCache({ hash_alg: 'dafdsagasdgwa4e' })
+    })
+  })
+})
+
+t.describe('FSCache', function() {
+  t.describe('#hash()', function() {
+    t.test('should use cache hasher to hash string', function() {
+      let cache = createCache({ hash_alg: 'sha256' })
+      assert.strictEqual(cache.hash('asdf'), crypto.hash('sha256', 'asdf'))
+
+      cache = createCache({ hash_alg: 'md5' })
+      assert.strictEqual(cache.hash('asdf'), crypto.hash('md5', 'asdf'))
+    })
+  })
+
+  t.describe('#_parseCacheData()', function() {
+    t.test('should default parse as json', function() {
+      let cache = createCache()
+      let output = cache._parseCacheData('{"hello":"world"}')
+      assert.strictEqual(typeof output, 'object')
+      assert.strictEqual(output.hello, 'world')
+    })
+
+    t.test('can be overwritten in options', function() {
+      let cache = createCache({ parse_json: false })
+      let output = cache._parseCacheData('{"hello":"world"}')
+      assert.strictEqual(typeof output, 'string')
+      assert.strictEqual(output, '{"hello":"world"}')
+    })
+
+    t.test('can be overwritten in parameter', function() {
+      let cache = createCache()
+      let output = cache._parseCacheData('{"hello":"world"}', { parse_json: false })
+      assert.strictEqual(typeof output, 'string')
+      assert.strictEqual(output, '{"hello":"world"}')
+    })
+  })
+
+  t.describe('#_parseSetData()', function() {
+    t.test('should default stringify to json', function() {
+      let cache = createCache()
+      let output = cache._parseSetData({ hello: 'world' })
+      assert.strictEqual(typeof output, 'string')
+      assert.strictEqual(output, '{"hello":"world"}')
+    })
+
+    t.test('can be overwritten in options', function() {
+      let cache = createCache({ parse_json: false })
+      let output = cache._parseSetData('Hello world')
+      assert.strictEqual(typeof output, 'string')
+      assert.strictEqual(output, 'Hello world')
+    })
+
+    t.test('can be overwritten in parameter', function() {
+      let cache = createCache()
+      let output = cache._parseSetData('Hello world', { parse_json: false })
+      assert.strictEqual(typeof output, 'string')
+      assert.strictEqual(output, 'Hello world')
+    })
+  })
+
+  t.describe('#get()', function() {
+    t.test('should call promise readFile and parse result', async function() {
+      const assertKey = 'asdf1234'
+      const assertContent = '{"hello":"world"}'
+      const assertResult = { hello: 'world' }
+      let cache = createCache()
+      fsPromises.readFile.resolves(assertContent)
+      cache._parseCacheData = spy().returns(assertResult)
+
+      let output = await cache.get(assertKey)
+      assert.strictEqual(output, assertResult)
+      assert.ok(fsPromises.readFile.called)
+      assert.strictEqual(fsPromises.readFile.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey)))
+      assert.strictEqual(fsPromises.readFile.firstCall[1]?.encoding, 'utf8')
+      assert.ok(cache._parseCacheData.called)
+      assert.ok(cache._parseCacheData.firstCall[0], assertContent)
+    })
+
+    t.test('should pass extra options to the parser', async function() {
+      const assertOptions = { a: 1 }
+      let cache = createCache()
+      cache._parseCacheData = spy()
+
+      await cache.get('asdf', null, assertOptions)
+
+      assert.ok(cache._parseCacheData.called)
+      assert.ok(cache._parseCacheData.firstCall[1], assertOptions)
+    })
+
+    t.test('should support fallback value if file does not exist', async function() {
+      const assertFallback = { a: 1 }
+      let cache = createCache()
+      fsPromises.readFile.rejects(new Error('asdf'))
+      cache._parseCacheData = spy()
+
+      let output = await cache.get('bla', assertFallback)
+      assert.strictEqual(output, assertFallback)
+      assert.notOk(cache._parseCacheData.called)
+    })
+
+    t.test('parser error should propogate', async function() {
+      const assertError = new Error('Hello')
+      let cache = createCache()
+      cache._parseCacheData = spy().throws(assertError)
+
+      let err = await assert.isRejected(cache.get('asdf'))
+
+      assert.strictEqual(err, assertError)
+    })
+  })
+
+  t.describe('#getSync()', function() {
+    t.test('should call sync readFile and parse result', function() {
+      const assertKey = 'asdf1234'
+      const assertContent = '{"hello":"world"}'
+      const assertResult = { hello: 'world' }
+      let cache = createCache()
+      fsSync.readFileSync.returns(assertContent)
+      cache._parseCacheData = spy().returns(assertResult)
+
+      let output = cache.getSync(assertKey)
+      assert.strictEqual(output, assertResult)
+      assert.ok(fsSync.readFileSync.called)
+      assert.strictEqual(fsSync.readFileSync.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey)))
+      assert.strictEqual(fsSync.readFileSync.firstCall[1]?.encoding, 'utf8')
+      assert.ok(cache._parseCacheData.called)
+      assert.ok(cache._parseCacheData.firstCall[0], assertContent)
+    })
+
+    t.test('should pass extra options to the parser', function() {
+      const assertOptions = { a: 1 }
+      let cache = createCache()
+      cache._parseCacheData = spy()
+
+      cache.getSync('asdf', null, assertOptions)
+
+      assert.ok(cache._parseCacheData.called)
+      assert.ok(cache._parseCacheData.firstCall[1], assertOptions)
+    })
+
+    t.test('should support fallback value if file does not exist', function() {
+      const assertFallback = { a: 1 }
+      let cache = createCache()
+      fsSync.readFileSync.throws(new Error('asdf'))
+      cache._parseCacheData = spy()
+
+      let output = cache.getSync('bla', assertFallback)
+      assert.strictEqual(output, assertFallback)
+      assert.notOk(cache._parseCacheData.called)
+    })
+
+    t.test('parser error should propogate', async function() {
+      const assertError = new Error('Hello')
+      let cache = createCache()
+      cache._parseCacheData = spy().throws(assertError)
+
+      assert.throws(function() {
+        cache.getSync('asdf')
+      }, assertError)
+    })
+  })
+
+  t.describe('#set()', function() {
+    t.test('should call promise writeFile', async function() {
+      const assertKey = 'asdf1234'
+      const assertInput = { hello: 'world' }
+      const assertContent = JSON.stringify(assertInput)
+
+      let cache = createCache()
+      cache._parseSetData = spy().returns(assertContent)
+      await cache.set(assertKey, assertInput)
+
+      assert.ok(fsPromises.writeFile.called)
+      assert.strictEqual(fsPromises.writeFile.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey)))
+      assert.strictEqual(fsPromises.writeFile.firstCall[1], assertContent)
+      assert.strictEqual(fsPromises.writeFile.firstCall[2]?.encoding, 'utf8')
+      assert.ok(cache._parseSetData.called)
+      assert.ok(cache._parseSetData.firstCall[0], assertInput)
+    })
+
+    t.test('should pass extra options to the parser', async function() {
+      const assertOptions = { a: 1 }
+      let cache = createCache()
+      cache._parseSetData = spy()
+
+      await cache.set('asdf', null, assertOptions)
+
+      assert.ok(cache._parseSetData.called)
+      assert.ok(cache._parseSetData.firstCall[1], assertOptions)
+    })
+
+    t.test('should pass extra options to the parser', async function() {
+      const assertEncoding = 'asdf'
+
+      let cache = createCache()
+      await cache.set('asdf', null, { encoding: assertEncoding })
+
+      assert.strictEqual(fsPromises.writeFile.firstCall[2]?.encoding, assertEncoding)
+    })
+
+    t.test('parse error should reject properly', async function() {
+      const assertError = new Error('what is up')
+
+      let cache = createCache()
+      cache._parseSetData = spy().throws(assertError)
+      
+      // make sure it's properly promise wrapped
+      let inbetween = cache.set('asdf')
+      let err = await assert.isRejected(inbetween)
+
+      assert.strictEqual(err, assertError)
+    })
+  })
+
+  t.describe('#setSync()', function() {
+    t.test('should call sync writeFileSync', function() {
+      const assertKey = 'asdf1234'
+      const assertInput = { hello: 'world' }
+      const assertContent = JSON.stringify(assertInput)
+
+      let cache = createCache()
+      cache._parseSetData = spy().returns(assertContent)
+      cache.setSync(assertKey, assertInput)
+
+      assert.ok(fsSync.writeFileSync.called)
+      assert.strictEqual(fsSync.writeFileSync.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey)))
+      assert.strictEqual(fsSync.writeFileSync.firstCall[1], assertContent)
+      assert.strictEqual(fsSync.writeFileSync.firstCall[2]?.encoding, 'utf8')
+      assert.ok(cache._parseSetData.called)
+      assert.ok(cache._parseSetData.firstCall[0], assertInput)
+    })
+
+    t.test('should pass extra options to the parser', function() {
+      const assertOptions = { a: 1 }
+      let cache = createCache()
+      cache._parseSetData = spy()
+
+      cache.setSync('asdf', null, assertOptions)
+
+      assert.ok(cache._parseSetData.called)
+      assert.ok(cache._parseSetData.firstCall[1], assertOptions)
+    })
+
+    t.test('should pass extra options to the parser', function() {
+      const assertEncoding = 'asdf'
+
+      let cache = createCache()
+      cache.setSync('asdf', null, { encoding: assertEncoding })
+
+      assert.strictEqual(fsSync.writeFileSync.firstCall[2]?.encoding, assertEncoding)
+    })
+
+    t.test('parse error should throw', function() {
+      const assertError = new Error('what is up')
+
+      let cache = createCache()
+      cache._parseSetData = spy().throws(assertError)
+
+      assert.throws(function() {
+        cache.setSync('asdf')
+      }, assertError)
+    })
+  })
+})

test/helper.mjs

@@ -0,0 +1,23 @@
+import { spy } from 'eltro'
+
+export function fakeFsSync() {
+  return {
+    mkdirSync: spy(),
+    readFileSync: spy(),
+    writeFileSync: spy(),
+  }
+}
+
+export function fakeFsPromises() {
+  return {
+    readFile: spy().resolves(),
+    writeFile: spy().resolves(),
+  }
+}
+
+export function fakeFs() {
+  return {
+    fsSync: fakeFsSync(),
+    fsPromises: fakeFsPromises(),
+  }
+}

test/temp/0cdd080cf69aa1c180f5ee3c5915cefe

@@ -0,0 +1 @@
+{"d":4}

test/temp/22a8f11d8816ba4ce9825d3452ff2dc0

@@ -0,0 +1 @@
+{"b":2}

test/temp/52037ffdb7cc64358fd69719a86d426a

@@ -0,0 +1 @@
+{"a":1}

test/temp/5eb63bbbe01eeed093cb22bb8f5acdc3

@@ -0,0 +1 @@
+{"f":6}

test/temp/a616fed18d795c4dd4e1c1b3b3bdd044

@@ -0,0 +1 @@
+{"c":3}