From 85e33094cdc34f8ed4abfd597207a3fb6ff2373e Mon Sep 17 00:00:00 2001 From: Jonatan Nilsson Date: Fri, 20 Sep 2024 13:40:01 +0000 Subject: [PATCH] Initial development version for testing --- .gitea/workflows/deploy.yaml | 44 +++ .npmrc | 1 + index.mjs | 80 +++++ package.json | 25 ++ test/fscache.integration.test.mjs | 75 +++++ test/fscache.test.mjs | 339 +++++++++++++++++++++ test/helper.mjs | 23 ++ test/temp/.gitkeep | 0 test/temp/0cdd080cf69aa1c180f5ee3c5915cefe | 1 + test/temp/22a8f11d8816ba4ce9825d3452ff2dc0 | 1 + test/temp/52037ffdb7cc64358fd69719a86d426a | 1 + test/temp/5eb63bbbe01eeed093cb22bb8f5acdc3 | 1 + test/temp/a616fed18d795c4dd4e1c1b3b3bdd044 | 1 + 13 files changed, 592 insertions(+) create mode 100644 .gitea/workflows/deploy.yaml create mode 100644 .npmrc create mode 100644 index.mjs create mode 100644 package.json create mode 100644 test/fscache.integration.test.mjs create mode 100644 test/fscache.test.mjs create mode 100644 test/helper.mjs create mode 100644 test/temp/.gitkeep create mode 100644 test/temp/0cdd080cf69aa1c180f5ee3c5915cefe create mode 100644 test/temp/22a8f11d8816ba4ce9825d3452ff2dc0 create mode 100644 test/temp/52037ffdb7cc64358fd69719a86d426a create mode 100644 test/temp/5eb63bbbe01eeed093cb22bb8f5acdc3 create mode 100644 test/temp/a616fed18d795c4dd4e1c1b3b3bdd044 diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml new file mode 100644 index 0000000..15e0674 --- /dev/null +++ b/.gitea/workflows/deploy.yaml @@ -0,0 +1,44 @@ +on: + push: + branches: + - master + +jobs: + deploy: + runs-on: alpine + steps: + - name: Check out repository code + uses: actions/checkout@v3 + - name: Test + run: | + npm install + npm test + - name: Check for new release + run: | + echo "" + echo "------------------------------------" + echo "" + + CURR_VER="$(cat package.json | jq -r .name)_v$(cat package.json | jq -r .version)" + CURR_NAME="$(cat package.json | jq -r .name) v$(cat package.json | jq -r .version)" + + echo "Checking https://git.nfp.is/api/v1/repos/${{ gitea.repository }}/releases for name ${CURR_NAME}" + + if curl -s -X GET -H "Authorization: token ${{ secrets.deploytoken }}" https://git.nfp.is/api/v1/repos/${{ gitea.repository }}/releases | grep -o "\"name\"\:\"${CURR_NAME}\"" > /dev/null; then + echo "Skipping ${{ gitea.job }} since $CURR_NAME already exists"; + exit; + fi + + echo "New release ${CURR_VER} found, running npm install..." + + echo "Creating ${CURR_VER} release on gitea" + curl \ + -X POST \ + -H "Authorization: token ${{ secrets.deploytoken }}" \ + -H "Content-Type: application/json" \ + https://git.nfp.is/api/v1/repos/${{ gitea.repository }}/releases \ + -d "{\"tag_name\":\"${CURR_VER}\",\"name\":\"${CURR_NAME}\",\"body\":\"Automatic release from Appveyor from ${{ gitea.sha }} :\n\n${{ gitea.event.head_commit.message }}\"}" + + echo "//registry.npmjs.org/:_authToken=${{ secrets.npmtoken }}" > ~/.npmrc + echo "Publishing new version to npm" + npm publish \ No newline at end of file diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..9cf9495 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false \ No newline at end of file diff --git a/index.mjs b/index.mjs new file mode 100644 index 0000000..6be85a4 --- /dev/null +++ b/index.mjs @@ -0,0 +1,80 @@ +import fsSyncOriginal from 'fs' +import fsPromisesOriginal from 'fs/promises' +import crypto from 'crypto' +import path from 'path' +import os from 'os' + +export default class FSCache { + constructor(options = {}, fsSync, fsPromises) { + this.fsSync = fsSync || fsSyncOriginal + this.fsPromises = fsPromises || fsPromisesOriginal + + this.id = crypto.randomBytes(15).toString('base64').replace(/\//g, '-') + this.parse_json = options.parse_json ?? true + this.prefix = options.prefix ? options.prefix + '-' : '' + this.hash_alg = options.hash_alg || 'md5' + this.cache_dir = options.cache_dir || path.join(os.tmpdir(), this.id) + + // Verify hash algorithm is supported on this system + crypto.createHash(this.hash_alg) + + this.fsSync.mkdirSync(this.cache_dir, { recursive: true }) + } + + _parseCacheData(data, overwrite = {}) { + return overwrite.parse_json ?? this.parse_json ? JSON.parse(data) : data + } + + _parseSetData(data, overwrite = {}) { + return overwrite.parse_json ?? this.parse_json ? JSON.stringify(data) : data + } + + hash(name) { + return crypto.hash(this.hash_alg, name) + } + + get(name, fallback, opts) { + return this.fsPromises.readFile( + path.join(this.cache_dir, this.hash(name)), + { encoding: 'utf8' } + ) + .then( + data => this._parseCacheData(data, opts), + err => (fallback) + ) + } + + getSync(name, fallback, opts) { + let data; + + try { + data = this.fsSync.readFileSync( + path.join(this.cache_dir, this.hash(name)), + { encoding: 'utf8' } + ) + } catch { + return fallback + } + return this._parseCacheData(data, opts) + } + + set(name, data, opts = {}) { + try { + return this.fsPromises.writeFile( + path.join(this.cache_dir, this.hash(name)), + this._parseSetData(data, opts), + { encoding: opts.encoding || 'utf8' } + ) + } catch (err) { + return Promise.reject(err) + } + } + + setSync(name, data, opts = {}) { + this.fsSync.writeFileSync( + path.join(this.cache_dir, this.hash(name)), + this._parseSetData(data, opts), + { encoding: opts.encoding || 'utf8' } + ) + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..7654a60 --- /dev/null +++ b/package.json @@ -0,0 +1,25 @@ +{ + "name": "fs-cache-fast", + "version": "1.0.0", + "description": "Cache stored on the file system", + "main": "index.mjs", + "scripts": { + "test": "eltro", + "test:watch": "eltro -r dot -w test" + }, + "watch": { + "test": { + "patterns": ["./"], + "extensions": "mjs" + } + }, + "repository": { + "type": "git", + "url": "https://git.nfp.is/TheThing/fs-cache-fast" + }, + "author": "Jonatan Nilsson", + "license": "WTFPL", + "devDependencies": { + "eltro": "^1.4.5" + } +} diff --git a/test/fscache.integration.test.mjs b/test/fscache.integration.test.mjs new file mode 100644 index 0000000..120bf2d --- /dev/null +++ b/test/fscache.integration.test.mjs @@ -0,0 +1,75 @@ +import { Eltro as t, assert, spy } from 'eltro' +import { fileURLToPath } from 'url' +import fs from 'fs/promises' +import fsSync from 'fs' +import path from 'path' +import Cache from '../index.mjs' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +let cache = new Cache({ cache_dir: path.join(__dirname, 'temp') }) + +t.before(async function() { + for (let file of await fs.readdir(cache.cache_dir)) { + if (file !== '.gitkeep') { + await fs.rm(path.join(cache.cache_dir, file)) + } + } +}) + +t.test('get should work', async function() { + const testKey = 'get-test-one' + + assert.deepStrictEqual(await cache.get(testKey, 'HELLO'), 'HELLO') + await fs.writeFile(path.join(cache.cache_dir, cache.hash(testKey)), JSON.stringify({ a: 1 })) + assert.deepStrictEqual(await cache.get(testKey, 'HELLO'), { a: 1 }) + assert.deepStrictEqual(await cache.get(testKey, 'HELLO', { parse_json: false }), JSON.stringify({ a: 1 })) +}) + +t.test('getSync should work', function() { + const testKey = 'get-sync-test-one' + + assert.deepStrictEqual(cache.getSync(testKey, 'HELLO'), 'HELLO') + fsSync.writeFileSync(path.join(cache.cache_dir, cache.hash(testKey)), JSON.stringify({ b: 2 })) + assert.deepStrictEqual(cache.getSync(testKey, 'HELLO'), { b: 2 }) + assert.deepStrictEqual(cache.getSync(testKey, 'HELLO', { parse_json: false }), JSON.stringify({ b: 2 })) +}) + +t.test('set should work', async function() { + const testKey = 'set-test-one' + const assertPath = path.join(cache.cache_dir, cache.hash(testKey)) + + assert.notOk(fsSync.existsSync(assertPath)) + await cache.set(testKey, { c: 3 }) + assert.ok(fsSync.existsSync(assertPath)) + let content = await fs.readFile(assertPath, { encoding: 'utf8' }) + assert.strictEqual(content, JSON.stringify({ c: 3 })) +}) + +t.test('set should work', function() { + const testKey = 'set-sync-test-one' + const assertPath = path.join(cache.cache_dir, cache.hash(testKey)) + + assert.notOk(fsSync.existsSync(assertPath)) + cache.setSync(testKey, { d: 4 }) + assert.ok(fsSync.existsSync(assertPath)) + let content = fsSync.readFileSync(assertPath, { encoding: 'utf8' }) + assert.strictEqual(content, JSON.stringify({ d: 4 })) +}) + +t.test('should all work together', async function() { + const testKey = 'hello world' + const assertFallback = 'This is fallback' + const assertPath = path.join(cache.cache_dir, cache.hash(testKey)) + + assert.notOk(fsSync.existsSync(assertPath)) + assert.strictEqual(await cache.get(testKey, assertFallback), assertFallback) + await cache.set(testKey, { e: 5 }) + assert.notStrictEqual(await cache.get(testKey, assertFallback), assertFallback) + assert.deepStrictEqual(await cache.get(testKey), { e: 5 }) + assert.notStrictEqual(cache.getSync(testKey, assertFallback), assertFallback) + assert.deepStrictEqual(cache.getSync(testKey), { e: 5 }) + cache.setSync(testKey, { f: 6 }) + assert.deepStrictEqual(await cache.get(testKey), { f: 6 }) + assert.deepStrictEqual(cache.getSync(testKey), { f: 6 }) +}) diff --git a/test/fscache.test.mjs b/test/fscache.test.mjs new file mode 100644 index 0000000..fa0001a --- /dev/null +++ b/test/fscache.test.mjs @@ -0,0 +1,339 @@ +import { Eltro as t, assert, spy } from 'eltro' +import path from 'path' +import crypto from 'crypto' +import os from 'os' +import fsSyncOriginal from 'fs' +import fsPromisesOriginal from 'fs/promises' +import Cache from '../index.mjs' +import { fakeFs, fakeFsPromises, fakeFsSync } from './helper.mjs' + +let fsSync +let fsPromises + +t.beforeEach(function() { + fsSync = fakeFsSync() + fsPromises = fakeFsPromises() +}) + +function createCache(opts) { + return new Cache(opts, fsSync, fsPromises) +} + +t.describe('#constructor()', function() { + t.test('uses default fs', function() { + let cache = new Cache({}) + assert.strictEqual(cache.fsSync, fsSyncOriginal) + assert.strictEqual(cache.fsPromises, fsPromisesOriginal) + }) + + t.test('should be able to override the fs', function() { + let cache = createCache({}) + assert.strictEqual(cache.fsSync, fsSync) + assert.strictEqual(cache.fsPromises, fsPromises) + }) + + t.test('comes with default options', function() { + let cache = createCache({}) + assert.ok(cache.id) + assert.strictEqual(cache.parse_json, true) + assert.strictEqual(cache.prefix, '') + assert.strictEqual(cache.hash_alg, 'md5') + assert.strictEqual(cache.cache_dir, path.join(os.tmpdir(), cache.id)) + }) + + t.test('can overwrite options', function() { + const assertHash = 'sha256' + const assertDir = '/something/else' + const assertPrefix = 'blabla' + const assertParseJson = false + + let cache = createCache({ + prefix: assertPrefix, + hash_alg: assertHash, + cache_dir: assertDir, + parse_json: assertParseJson, + }) + + assert.ok(cache.id) + assert.strictEqual(cache.parse_json, assertParseJson) + assert.strictEqual(cache.prefix, assertPrefix + '-') + assert.strictEqual(cache.hash_alg, assertHash) + assert.strictEqual(cache.cache_dir, assertDir) + }) + + t.test('should create the directory by default', function() { + assert.notOk(fsSync.mkdirSync.called) + let cache = createCache({}) + assert.ok(fsSync.mkdirSync.called) + assert.strictEqual(fsSync.mkdirSync.firstCall[0], cache.cache_dir) + assert.strictEqual(fsSync.mkdirSync.firstCall[1]?.recursive, true) + }) + + t.test('should check if hash_alg is valid', function() { + assert.throws(function() { + createCache({ hash_alg: 'dafdsagasdgwa4e' }) + }) + }) +}) + +t.describe('FSCache', function() { + t.describe('#hash()', function() { + t.test('should use cache hasher to hash string', function() { + let cache = createCache({ hash_alg: 'sha256' }) + assert.strictEqual(cache.hash('asdf'), crypto.hash('sha256', 'asdf')) + + cache = createCache({ hash_alg: 'md5' }) + assert.strictEqual(cache.hash('asdf'), crypto.hash('md5', 'asdf')) + }) + }) + + t.describe('#_parseCacheData()', function() { + t.test('should default parse as json', function() { + let cache = createCache() + let output = cache._parseCacheData('{"hello":"world"}') + assert.strictEqual(typeof output, 'object') + assert.strictEqual(output.hello, 'world') + }) + + t.test('can be overwritten in options', function() { + let cache = createCache({ parse_json: false }) + let output = cache._parseCacheData('{"hello":"world"}') + assert.strictEqual(typeof output, 'string') + assert.strictEqual(output, '{"hello":"world"}') + }) + + t.test('can be overwritten in parameter', function() { + let cache = createCache() + let output = cache._parseCacheData('{"hello":"world"}', { parse_json: false }) + assert.strictEqual(typeof output, 'string') + assert.strictEqual(output, '{"hello":"world"}') + }) + }) + + t.describe('#_parseSetData()', function() { + t.test('should default stringify to json', function() { + let cache = createCache() + let output = cache._parseSetData({ hello: 'world' }) + assert.strictEqual(typeof output, 'string') + assert.strictEqual(output, '{"hello":"world"}') + }) + + t.test('can be overwritten in options', function() { + let cache = createCache({ parse_json: false }) + let output = cache._parseSetData('Hello world') + assert.strictEqual(typeof output, 'string') + assert.strictEqual(output, 'Hello world') + }) + + t.test('can be overwritten in parameter', function() { + let cache = createCache() + let output = cache._parseSetData('Hello world', { parse_json: false }) + assert.strictEqual(typeof output, 'string') + assert.strictEqual(output, 'Hello world') + }) + }) + + t.describe('#get()', function() { + t.test('should call promise readFile and parse result', async function() { + const assertKey = 'asdf1234' + const assertContent = '{"hello":"world"}' + const assertResult = { hello: 'world' } + let cache = createCache() + fsPromises.readFile.resolves(assertContent) + cache._parseCacheData = spy().returns(assertResult) + + let output = await cache.get(assertKey) + assert.strictEqual(output, assertResult) + assert.ok(fsPromises.readFile.called) + assert.strictEqual(fsPromises.readFile.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey))) + assert.strictEqual(fsPromises.readFile.firstCall[1]?.encoding, 'utf8') + assert.ok(cache._parseCacheData.called) + assert.ok(cache._parseCacheData.firstCall[0], assertContent) + }) + + t.test('should pass extra options to the parser', async function() { + const assertOptions = { a: 1 } + let cache = createCache() + cache._parseCacheData = spy() + + await cache.get('asdf', null, assertOptions) + + assert.ok(cache._parseCacheData.called) + assert.ok(cache._parseCacheData.firstCall[1], assertOptions) + }) + + t.test('should support fallback value if file does not exist', async function() { + const assertFallback = { a: 1 } + let cache = createCache() + fsPromises.readFile.rejects(new Error('asdf')) + cache._parseCacheData = spy() + + let output = await cache.get('bla', assertFallback) + assert.strictEqual(output, assertFallback) + assert.notOk(cache._parseCacheData.called) + }) + + t.test('parser error should propogate', async function() { + const assertError = new Error('Hello') + let cache = createCache() + cache._parseCacheData = spy().throws(assertError) + + let err = await assert.isRejected(cache.get('asdf')) + + assert.strictEqual(err, assertError) + }) + }) + + t.describe('#getSync()', function() { + t.test('should call sync readFile and parse result', function() { + const assertKey = 'asdf1234' + const assertContent = '{"hello":"world"}' + const assertResult = { hello: 'world' } + let cache = createCache() + fsSync.readFileSync.returns(assertContent) + cache._parseCacheData = spy().returns(assertResult) + + let output = cache.getSync(assertKey) + assert.strictEqual(output, assertResult) + assert.ok(fsSync.readFileSync.called) + assert.strictEqual(fsSync.readFileSync.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey))) + assert.strictEqual(fsSync.readFileSync.firstCall[1]?.encoding, 'utf8') + assert.ok(cache._parseCacheData.called) + assert.ok(cache._parseCacheData.firstCall[0], assertContent) + }) + + t.test('should pass extra options to the parser', function() { + const assertOptions = { a: 1 } + let cache = createCache() + cache._parseCacheData = spy() + + cache.getSync('asdf', null, assertOptions) + + assert.ok(cache._parseCacheData.called) + assert.ok(cache._parseCacheData.firstCall[1], assertOptions) + }) + + t.test('should support fallback value if file does not exist', function() { + const assertFallback = { a: 1 } + let cache = createCache() + fsSync.readFileSync.throws(new Error('asdf')) + cache._parseCacheData = spy() + + let output = cache.getSync('bla', assertFallback) + assert.strictEqual(output, assertFallback) + assert.notOk(cache._parseCacheData.called) + }) + + t.test('parser error should propogate', async function() { + const assertError = new Error('Hello') + let cache = createCache() + cache._parseCacheData = spy().throws(assertError) + + assert.throws(function() { + cache.getSync('asdf') + }, assertError) + }) + }) + + t.describe('#set()', function() { + t.test('should call promise writeFile', async function() { + const assertKey = 'asdf1234' + const assertInput = { hello: 'world' } + const assertContent = JSON.stringify(assertInput) + + let cache = createCache() + cache._parseSetData = spy().returns(assertContent) + await cache.set(assertKey, assertInput) + + assert.ok(fsPromises.writeFile.called) + assert.strictEqual(fsPromises.writeFile.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey))) + assert.strictEqual(fsPromises.writeFile.firstCall[1], assertContent) + assert.strictEqual(fsPromises.writeFile.firstCall[2]?.encoding, 'utf8') + assert.ok(cache._parseSetData.called) + assert.ok(cache._parseSetData.firstCall[0], assertInput) + }) + + t.test('should pass extra options to the parser', async function() { + const assertOptions = { a: 1 } + let cache = createCache() + cache._parseSetData = spy() + + await cache.set('asdf', null, assertOptions) + + assert.ok(cache._parseSetData.called) + assert.ok(cache._parseSetData.firstCall[1], assertOptions) + }) + + t.test('should pass extra options to the parser', async function() { + const assertEncoding = 'asdf' + + let cache = createCache() + await cache.set('asdf', null, { encoding: assertEncoding }) + + assert.strictEqual(fsPromises.writeFile.firstCall[2]?.encoding, assertEncoding) + }) + + t.test('parse error should reject properly', async function() { + const assertError = new Error('what is up') + + let cache = createCache() + cache._parseSetData = spy().throws(assertError) + + // make sure it's properly promise wrapped + let inbetween = cache.set('asdf') + let err = await assert.isRejected(inbetween) + + assert.strictEqual(err, assertError) + }) + }) + + t.describe('#setSync()', function() { + t.test('should call sync writeFileSync', function() { + const assertKey = 'asdf1234' + const assertInput = { hello: 'world' } + const assertContent = JSON.stringify(assertInput) + + let cache = createCache() + cache._parseSetData = spy().returns(assertContent) + cache.setSync(assertKey, assertInput) + + assert.ok(fsSync.writeFileSync.called) + assert.strictEqual(fsSync.writeFileSync.firstCall[0], path.join(cache.cache_dir, cache.hash(assertKey))) + assert.strictEqual(fsSync.writeFileSync.firstCall[1], assertContent) + assert.strictEqual(fsSync.writeFileSync.firstCall[2]?.encoding, 'utf8') + assert.ok(cache._parseSetData.called) + assert.ok(cache._parseSetData.firstCall[0], assertInput) + }) + + t.test('should pass extra options to the parser', function() { + const assertOptions = { a: 1 } + let cache = createCache() + cache._parseSetData = spy() + + cache.setSync('asdf', null, assertOptions) + + assert.ok(cache._parseSetData.called) + assert.ok(cache._parseSetData.firstCall[1], assertOptions) + }) + + t.test('should pass extra options to the parser', function() { + const assertEncoding = 'asdf' + + let cache = createCache() + cache.setSync('asdf', null, { encoding: assertEncoding }) + + assert.strictEqual(fsSync.writeFileSync.firstCall[2]?.encoding, assertEncoding) + }) + + t.test('parse error should throw', function() { + const assertError = new Error('what is up') + + let cache = createCache() + cache._parseSetData = spy().throws(assertError) + + assert.throws(function() { + cache.setSync('asdf') + }, assertError) + }) + }) +}) diff --git a/test/helper.mjs b/test/helper.mjs new file mode 100644 index 0000000..f19dbb6 --- /dev/null +++ b/test/helper.mjs @@ -0,0 +1,23 @@ +import { spy } from 'eltro' + +export function fakeFsSync() { + return { + mkdirSync: spy(), + readFileSync: spy(), + writeFileSync: spy(), + } +} + +export function fakeFsPromises() { + return { + readFile: spy().resolves(), + writeFile: spy().resolves(), + } +} + +export function fakeFs() { + return { + fsSync: fakeFsSync(), + fsPromises: fakeFsPromises(), + } +} diff --git a/test/temp/.gitkeep b/test/temp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/temp/0cdd080cf69aa1c180f5ee3c5915cefe b/test/temp/0cdd080cf69aa1c180f5ee3c5915cefe new file mode 100644 index 0000000..7959b47 --- /dev/null +++ b/test/temp/0cdd080cf69aa1c180f5ee3c5915cefe @@ -0,0 +1 @@ +{"d":4} \ No newline at end of file diff --git a/test/temp/22a8f11d8816ba4ce9825d3452ff2dc0 b/test/temp/22a8f11d8816ba4ce9825d3452ff2dc0 new file mode 100644 index 0000000..560674f --- /dev/null +++ b/test/temp/22a8f11d8816ba4ce9825d3452ff2dc0 @@ -0,0 +1 @@ +{"b":2} \ No newline at end of file diff --git a/test/temp/52037ffdb7cc64358fd69719a86d426a b/test/temp/52037ffdb7cc64358fd69719a86d426a new file mode 100644 index 0000000..daa5053 --- /dev/null +++ b/test/temp/52037ffdb7cc64358fd69719a86d426a @@ -0,0 +1 @@ +{"a":1} \ No newline at end of file diff --git a/test/temp/5eb63bbbe01eeed093cb22bb8f5acdc3 b/test/temp/5eb63bbbe01eeed093cb22bb8f5acdc3 new file mode 100644 index 0000000..4169add --- /dev/null +++ b/test/temp/5eb63bbbe01eeed093cb22bb8f5acdc3 @@ -0,0 +1 @@ +{"f":6} \ No newline at end of file diff --git a/test/temp/a616fed18d795c4dd4e1c1b3b3bdd044 b/test/temp/a616fed18d795c4dd4e1c1b3b3bdd044 new file mode 100644 index 0000000..daa71dc --- /dev/null +++ b/test/temp/a616fed18d795c4dd4e1c1b3b3bdd044 @@ -0,0 +1 @@ +{"c":3} \ No newline at end of file