From 68eef3a6b6c034d021dea8082fef0e231af60069 Mon Sep 17 00:00:00 2001 From: Jonatan Nilsson Date: Wed, 5 Jan 2022 14:47:51 +0000 Subject: [PATCH] Complete refactor, features and optimization of code Removed large amount of dependencies. Added vips support and automatically resizing based on any criteria. Faster and leaner. Added ability to fetch listing of files in folders. --- api/media/formidable.mjs | 20 +- api/media/routes.mjs | 119 ++++++++- api/media/security.mjs | 62 ++++- api/server.mjs | 6 + config/config.test.json | 10 +- package.json | 5 +- public/.gitkeep | 0 test/helper.client.mjs | 32 ++- test/helper.server.mjs | 2 + test/media/api.test.mjs | 300 ++++++++++++++++++++++- test/media/routes.test.mjs | 462 ++++++++++++++++++++++++++++++++++- test/media/security.test.mjs | 302 ++++++++++++++++++++++- 12 files changed, 1286 insertions(+), 34 deletions(-) delete mode 100644 public/.gitkeep diff --git a/api/media/formidable.mjs b/api/media/formidable.mjs index 856dc47..25123af 100644 --- a/api/media/formidable.mjs +++ b/api/media/formidable.mjs @@ -3,18 +3,29 @@ import { HttpError } from '../error.mjs' import formidable from 'formidable' import config from '../config.mjs' +let lastDateString = '' +let incrementor = 1 + export function uploadFile(ctx, siteName) { return new Promise((res, rej) => { const date = new Date() // Generate 'YYYYMMDD_HHMMSS_' prefix - const prefix = date + let prefix = date .toISOString() .replace(/-/g, '') .replace('T', '_') .replace(/:/g, '') .replace(/\..+/, '_') + // Append xx_ if same date is hit multiple times + if (prefix === lastDateString) { + prefix += incrementor.toString().padStart('2', '0') + '_' + incrementor++ + } else { + lastDateString = prefix + } + var form = new formidable.IncomingForm() form.uploadDir = `./public/${siteName}` form.maxFileSize = config.get('fileSize') @@ -24,6 +35,13 @@ export function uploadFile(ctx, siteName) { if (!files || !files.file) return rej(new HttpError('File in body was missing', 422)) let file = files.file + Object.keys(fields).forEach(function(key) { + try { + fields[key] = JSON.parse(fields[key]) + } catch { } + }) + ctx.req.body = fields + fs.rename(files.file.path, `./public/${siteName}/${prefix}${file.name}`, function(err) { if (err) return rej(err) file.path = `./public/${siteName}/${prefix}${file.name}` diff --git a/api/media/routes.mjs b/api/media/routes.mjs index 7b2ea17..5d91301 100644 --- a/api/media/routes.mjs +++ b/api/media/routes.mjs @@ -1,3 +1,7 @@ +import path from 'path' +import sharp from 'sharp-lite' +import fs from 'fs/promises' +import { HttpError } from '../error.mjs' import * as security from './security.mjs' import * as formidable from './formidable.mjs' @@ -6,19 +10,130 @@ export default class MediaRoutes { Object.assign(this, { security: opts.security || security, formidable: opts.formidable || formidable, + sharp: opts.sharp || sharp, + fs: opts.fs || fs, }) + this.siteCache = new Map() + this.collator = new Intl.Collator('is-IS', { numeric: false, sensitivity: 'accent' }) + } + + init() { + return fs.readdir('./public').then(folders => { + return Promise.all(folders.map(folder => { + return fs.readdir('./public/' + folder) + .then(files => { + return Promise.all(files.map(file => { + return fs.stat(`./public/${folder}/${file}`) + .then(function(stat) { + return { name: file, size: stat.size } + }) + })) + }) + .then(files => { + this.siteCache.set(folder, files) + }) + .catch(function() {}) + })) + }) + } + + filesCacheGet(site) { + let files = this.siteCache.get(site) || [] + return files.sort((a, b) => { + return this.collator.compare(a.name, b.name) + }) + } + + filesCacheAdd(site, filename, size) { + let arr = this.siteCache.get(site) + if (!arr) { + this.siteCache.set(site, arr = []) + } + arr.push({ name: filename, size: size }) + } + + async listFiles(ctx) { + let site = await this.security.verifyToken(ctx) + + ctx.body = this.filesCacheGet(site) + } + + async listPublicFiles(ctx) { + this.security.throwIfNotPublic(ctx.params.site) + + ctx.body = this.filesCacheGet(ctx.params.site) } async upload(ctx) { let site = await this.security.verifyToken(ctx) + ctx.state.site = site - let result = await this.formidable.uploadFile(ctx, site) + let result = await this.formidable.uploadFile(ctx, ctx.state.site) ctx.log.info(`Uploaded ${result.filename}`) + let stat = await this.fs.stat(`./public/${ctx.state.site}/${result.filename}`) + this.filesCacheAdd(ctx.state.site, result.filename, stat.size) + ctx.body = { filename: result.filename, - path: `/${site}/${result.filename}` + path: `/${ctx.state.site}/${result.filename}` } } + + async resize(ctx) { + await this.upload(ctx) + + this.security.verifyBody(ctx) + + let out = { + original: ctx.body, + } + + let keys = Object.keys(ctx.req.body) + + await Promise.all(keys.map(key => { + return Promise.resolve() + .then(async () => { + let item = ctx.req.body[key] + let sharp = this.sharp(`./public/${ctx.state.site}/${ctx.body.filename}`) + .rotate() + + if (item.resize) { + sharp = sharp.resize(item.resize) + } + sharp = sharp[item.format](item[item.format]) + + let target = ctx.body.filename + if (path.extname(target).length > 0) { + target = target.slice(0, -path.extname(target).length) + } + target += `_${key}.${item.format.replace('jpeg', 'jpg')}` + + if (item.out === 'base64') { + let buffer = await sharp.toBuffer() + out[key] = { + base64: `data:image/${item.format};base64,` + buffer.toString('base64'), + } + return + } + await sharp.toFile(`./public/${ctx.state.site}/${target}`) + + let stat = await this.fs.stat(`./public/${ctx.state.site}/${target}`) + this.filesCacheAdd(ctx.state.site, target, stat.size) + + out[key] = { + filename: target, + path: `/${ctx.state.site}/${target}`, + } + }).then( + function() {}, + function(err) { + throw new HttpError(`Error processing ${key}: ${err.message}`, 422) + } + ) + })) + + ctx.body = out + } } diff --git a/api/media/security.mjs b/api/media/security.mjs index e9be16a..7e8a109 100644 --- a/api/media/security.mjs +++ b/api/media/security.mjs @@ -8,11 +8,71 @@ export function verifyToken(ctx) { throw new HttpError('Token is missing in query', 422) } + let org = config.get('sites') + let sites = {} + for (let key in org) { + if (org.hasOwnProperty(key)) { + sites[key] = org[key].keys + } + } + try { - let decoded = decode(token, config.get('sites'), []) + let decoded = decode(token, sites, []) return decoded.iss } catch (err) { ctx.log.error(err, 'Error decoding token: ' + token) throw new HttpError('Token was invalid', 422) } } + +export function throwIfNotPublic(site) { + let sites = config.get('sites') + if (!sites[site] || sites[site].public !== true) { + throw new HttpError(`Requested site ${site} did not exist`, 404) + } +} + +export function verifyBody(ctx) { + let keys = Object.keys(ctx.req.body) + + for (let key of keys) { + if (key === 'original') { + throw new HttpError('Body item with name original is not allowed', 422) + } + let item = ctx.req.body[key] + + if (typeof(item) !== 'object' + || !item + || Array.isArray(item)) { + throw new HttpError(`Body item ${key} was not valid`, 422) + } + + if (typeof(item.format) !== 'string' + || !item.format + || item.format === 'resize' + || item.format === 'out') { + throw new HttpError(`Body item ${key} missing valid format`, 422) + } + + if (typeof(item[item.format]) !== 'object' + || !item[item.format] + || Array.isArray(item[item.format])) { + throw new HttpError(`Body item ${key} options for format ${item.format} was not valid`, 422) + } + + if (item.out != null) { + if (typeof(item.out) !== 'string' + || (item.out !== '' && item.out !== 'file' && item.out !== 'base64') + ) { + throw new HttpError(`Body item ${key} key out was invalid`, 422) + } + } + + if (item.resize != null) { + if (typeof(item.resize) !== 'object' + || Array.isArray(item.resize)) { + throw new HttpError(`Body item ${key} key resize was invalid`, 422) + } + } + } +} diff --git a/api/server.mjs b/api/server.mjs index 27540fb..c5f8231 100644 --- a/api/server.mjs +++ b/api/server.mjs @@ -49,7 +49,13 @@ app.get('/', test.static.bind(test)) app.get('/error', test.error.bind(test)) const media = new MediaRoutes() +media.init().then(function() {}, function(err) { + log.error(err, 'Error initing media') +}) +app.get('/media', [QueryHandler()], media.listFiles.bind(media)) +app.get('/media/:site', [QueryHandler()], media.listPublicFiles.bind(media)) app.post('/media', [QueryHandler()], media.upload.bind(media)) +app.post('/media/resize', [QueryHandler()], media.resize.bind(media)) app.listen(config.get('server:port'), function(a,b) { log.info(`Server listening at ${config.get('server:port')}`) diff --git a/config/config.test.json b/config/config.test.json index 913cb6c..e213c70 100644 --- a/config/config.test.json +++ b/config/config.test.json @@ -9,7 +9,15 @@ }, "sites": { "development": { - "default@HS256": "asdf1234" + "keys": { + "default@HS256": "asdf1234" + } + }, + "existing": { + "public": true, + "keys": { + "default@HS256": "asdf1234" + } } } } diff --git a/package.json b/package.json index d22ac1f..ec3d815 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,10 @@ "homepage": "https://github.com/nfp-projects/storage-upload#readme", "dependencies": { "bunyan-lite": "^1.1.1", - "flaska": "^0.9.7", + "flaska": "^0.9.8", "formidable": "^1.2.2", - "nconf-lite": "^2.0.0" + "nconf-lite": "^2.0.0", + "sharp-lite": "^1.29.5" }, "devDependencies": { "eltro": "^1.2.3" diff --git a/public/.gitkeep b/public/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/test/helper.client.mjs b/test/helper.client.mjs index d5f9bf1..b0fe449 100644 --- a/test/helper.client.mjs +++ b/test/helper.client.mjs @@ -66,23 +66,37 @@ Client.prototype.get = function(url = '/') { return this.customRequest('GET', url, null) } -Client.prototype.upload = function(url, file, method = 'POST') { +const random = (length = 8) => { + // Declare all characters + let chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + // Pick characers randomly + let str = ''; + for (let i = 0; i < length; i++) { + str += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return str; +} + +Client.prototype.upload = function(url, file, method = 'POST', body = {}) { return fs.readFile(file).then(data => { const crlf = '\r\n' const filename = path.basename(file) - const boundary = `---------${Math.random().toString(16)}` - const headers = [ - `Content-Disposition: form-data; name="file"; filename="${filename}"` + crlf, - ] + const boundary = `---------${random(32)}` + const multipartBody = Buffer.concat([ Buffer.from( `${crlf}--${boundary}${crlf}` + - headers.join('') + crlf + `Content-Disposition: form-data; name="file"; filename="${filename}"` + crlf + crlf ), data, - Buffer.from( - `${crlf}--${boundary}--` - ), + Buffer.concat(Object.keys(body).map(function(key) { + return Buffer.from('' + + `${crlf}--${boundary}${crlf}` + + `Content-Disposition: form-data; name="${key}"` + crlf + crlf + + JSON.stringify(body[key]) + ) + })), + Buffer.from(`${crlf}--${boundary}--`), ]) return this.customRequest(method, url, multipartBody, { diff --git a/test/helper.server.mjs b/test/helper.server.mjs index 4ef3533..1455c84 100644 --- a/test/helper.server.mjs +++ b/test/helper.server.mjs @@ -29,8 +29,10 @@ export function resetLog() { export function createContext(opts) { return defaults(opts, { query: new Map(), + params: { }, req: { }, res: { }, + state: {}, log: { log: stub(), warn: stub(), diff --git a/test/media/api.test.mjs b/test/media/api.test.mjs index be02814..c977589 100644 --- a/test/media/api.test.mjs +++ b/test/media/api.test.mjs @@ -1,3 +1,4 @@ +import sharp from 'sharp-lite' import { Eltro as t, assert} from 'eltro' import fs from 'fs/promises' import { fileURLToPath } from 'url' @@ -16,12 +17,12 @@ function resolve(file) { t.describe('Media (API)', () => { const client = new Client() const secret = 'asdf1234' - let testFile + let testFiles = [] t.after(function() { - if (testFile) { - return fs.unlink(resolve(`../../public/${testFile}`)) - } + return Promise.all(testFiles.map(function(file) { + return fs.unlink(resolve(`../../public/${file}`)).catch(function() {}) + })) }) t.timeout(10000).describe('POST /media', function temp() { @@ -86,13 +87,300 @@ t.describe('Media (API)', () => { assert.ok(data.filename) assert.ok(data.path) - testFile = data.path + testFiles.push(data.path) let stats = await Promise.all([ fs.stat(resolve('test.png')), - fs.stat(resolve(`../../public/${testFile}`)), + fs.stat(resolve(`../../public/${data.path}`)), ]) assert.strictEqual(stats[0].size, stats[1].size) + + let img = await sharp(resolve(`../../public/${data.path}`)).metadata() + assert.strictEqual(img.width, 600) + assert.strictEqual(img.height, 700) + assert.strictEqual(img.format, 'png') + }) + }) + + t.timeout(10000).describe('POST /media/resize', function temp() { + t.test('should require authentication', async () => { + resetLog() + assert.strictEqual(server.log.error.callCount, 0) + assert.strictEqual(server.log.warn.callCount, 0) + let err = await assert.isRejected( + client.upload('/media/resize', + resolve('test.png') + ) + ) + + assert.strictEqual(err.status, 422) + assert.match(err.message, /[Tt]oken/) + assert.match(err.message, /[Mm]issing/) + + assert.strictEqual(server.log.error.callCount, 0) + assert.strictEqual(server.log.warn.callCount, 2) + assert.strictEqual(typeof(server.log.warn.firstCall[0]), 'string') + assert.match(server.log.warn.firstCall[0], /[Tt]oken/) + assert.match(server.log.warn.firstCall[0], /[Mm]issing/) + }) + + t.test('should verify token correctly', async () => { + const assertToken = 'asdf.asdf.asdf' + resetLog() + assert.strictEqual(server.log.error.callCount, 0) + assert.strictEqual(server.log.warn.callCount, 0) + assert.strictEqual(server.log.info.callCount, 0) + + let err = await assert.isRejected( + client.upload('/media/resize?token=' + assertToken, + resolve('test.png') + ) + ) + + assert.strictEqual(err.status, 422) + assert.match(err.message, /[Tt]oken/) + assert.match(err.message, /[Ii]nvalid/) + + assert.strictEqual(server.log.error.callCount, 1) + assert.strictEqual(server.log.warn.callCount, 2) + assert.strictEqual(typeof(server.log.warn.firstCall[0]), 'string') + assert.match(server.log.warn.firstCall[0], /[Tt]oken/) + assert.match(server.log.warn.firstCall[0], /[Ii]nvalid/) + assert.ok(server.log.error.lastCall[0] instanceof Error) + assert.match(server.log.error.lastCall[1], new RegExp(assertToken)) + }) + + t.test('should upload file and create file', async () => { + let token = encode(null, { iss: 'development' }, secret) + + let data = await assert.isFulfilled( + client.upload( + `/media/resize?token=${token}`, + resolve('test.png'), + 'POST', + { } + ) + ) + + assert.ok(data) + assert.ok(data.original) + assert.ok(data.original.filename) + assert.ok(data.original.path) + + testFiles.push(data.original.path) + + let stats = await Promise.all([ + fs.stat(resolve('test.png')), + fs.stat(resolve(`../../public/${data.original.path}`)), + ]) + assert.strictEqual(stats[0].size, stats[1].size) + + let img = await sharp(resolve(`../../public/${data.original.path}`)).metadata() + assert.strictEqual(img.width, 600) + assert.strictEqual(img.height, 700) + assert.strictEqual(img.format, 'png') + }) + + t.test('should upload file and create multiple sizes for file', async () => { + let token = encode(null, { iss: 'development' }, secret) + + let data = await assert.isFulfilled( + client.upload( + `/media/resize?token=${token}`, + resolve('test.png'), + 'POST', + { + test1: { + format: 'jpeg', + resize: { + width: 300, + }, + jpeg: { + quality: 80, + mozjpeg: true, + } + }, + test2: { + format: 'png', + resize: { + width: 150, + }, + png: { + compressionLevel: 9, + } + }, + } + ) + ) + + assert.ok(data) + assert.ok(data.original) + assert.ok(data.original.filename) + assert.ok(data.original.path) + assert.ok(data.test1.filename) + assert.ok(data.test1.path) + assert.ok(data.test2.filename) + assert.ok(data.test2.path) + + + testFiles.push(data.original.path) + testFiles.push(data.test1.path) + testFiles.push(data.test2.path) + + let stats = await Promise.all([ + fs.stat(resolve('test.png')), + fs.stat(resolve(`../../public/${data.original.path}`)), + ]) + assert.strictEqual(stats[0].size, stats[1].size) + + let img = await sharp(resolve(`../../public/${data.original.path}`)).metadata() + assert.strictEqual(img.width, 600) + assert.strictEqual(img.height, 700) + assert.strictEqual(img.format, 'png') + + img = await sharp(resolve(`../../public/${data.test1.path}`)).metadata() + assert.strictEqual(img.width, 300) + assert.strictEqual(img.height, 350) + assert.strictEqual(img.format, 'jpeg') + + img = await sharp(resolve(`../../public/${data.test2.path}`)).metadata() + assert.strictEqual(img.width, 150) + assert.strictEqual(img.height, 175) + assert.strictEqual(img.format, 'png') + }) + + + t.test('should upload file and support base64 output', async () => { + let token = encode(null, { iss: 'development' }, secret) + + let data = await assert.isFulfilled( + client.upload( + `/media/resize?token=${token}`, + resolve('test.png'), + 'POST', + { + outtest: { + out: 'base64', + format: 'jpeg', + resize: { + width: 10, + }, + jpeg: { + quality: 80, + mozjpeg: true, + } + }, + } + ) + ) + + assert.ok(data) + assert.ok(data.original) + assert.ok(data.original.filename) + assert.ok(data.original.path) + assert.ok(data.outtest.base64) + + + testFiles.push(data.original.path) + + let stats = await Promise.all([ + fs.stat(resolve('test.png')), + fs.stat(resolve(`../../public/${data.original.path}`)), + ]) + assert.strictEqual(stats[0].size, stats[1].size) + + let img = await sharp(resolve(`../../public/${data.original.path}`)).metadata() + assert.strictEqual(img.width, 600) + assert.strictEqual(img.height, 700) + assert.strictEqual(img.format, 'png') + + let bufferBase64 = Buffer.from(data.outtest.base64.slice(data.outtest.base64.indexOf(',')), 'base64') + + img = await sharp(bufferBase64).metadata() + assert.strictEqual(img.width, 10) + assert.strictEqual(img.height, 12) + assert.strictEqual(img.format, 'jpeg') + }) + }) + + t.describe('GET /media', function() { + t.test('should require authentication', async () => { + resetLog() + assert.strictEqual(server.log.error.callCount, 0) + assert.strictEqual(server.log.warn.callCount, 0) + let err = await assert.isRejected(client.get('/media')) + + assert.strictEqual(err.status, 422) + assert.match(err.message, /[Tt]oken/) + assert.match(err.message, /[Mm]issing/) + + assert.strictEqual(server.log.error.callCount, 0) + assert.strictEqual(server.log.warn.callCount, 2) + assert.strictEqual(typeof(server.log.warn.firstCall[0]), 'string') + assert.match(server.log.warn.firstCall[0], /[Tt]oken/) + assert.match(server.log.warn.firstCall[0], /[Mm]issing/) + }) + + t.test('should verify token correctly', async () => { + const assertToken = 'asdf.asdf.asdf' + resetLog() + assert.strictEqual(server.log.error.callCount, 0) + assert.strictEqual(server.log.warn.callCount, 0) + assert.strictEqual(server.log.info.callCount, 0) + + let err = await assert.isRejected(client.get('/media?token=' + assertToken)) + + assert.strictEqual(err.status, 422) + assert.match(err.message, /[Tt]oken/) + assert.match(err.message, /[Ii]nvalid/) + + assert.strictEqual(server.log.error.callCount, 1) + assert.strictEqual(server.log.warn.callCount, 2) + assert.strictEqual(typeof(server.log.warn.firstCall[0]), 'string') + assert.match(server.log.warn.firstCall[0], /[Tt]oken/) + assert.match(server.log.warn.firstCall[0], /[Ii]nvalid/) + assert.ok(server.log.error.lastCall[0] instanceof Error) + assert.match(server.log.error.lastCall[1], new RegExp(assertToken)) + }) + + t.test('should return list of files in specified folder', async () => { + let token = encode(null, { iss: 'development' }, secret) + + let data = await client.get('/media?token=' + token) + + assert.ok(data.length) + let found = false + for (let file of data) { + if (file.name === '.gitkeep' && file.size === 0) { + found = true + break + } + } + assert.ok(found) + }) + }) + + t.describe('GET /media/:site', function() { + t.test('should give 404 on invalid sites', async () => { + let err = await assert.isRejected(client.get('/media/development')) + + assert.strictEqual(err.status, 404) + + err = await assert.isRejected(client.get('/media/nonexisting')) + + assert.strictEqual(err.status, 404) + + err = await assert.isRejected(client.get('/media/blabla')) + + assert.strictEqual(err.status, 404) + }) + + t.test('should otherwise return list of files for a public site', async () => { + let files = await client.get('/media/existing') + assert.strictEqual(files[0].name, '20220105_101610_test1.jpg') + assert.strictEqual(files[0].size, 15079) + assert.strictEqual(files[1].name, '20220105_101610_test2.png') + assert.strictEqual(files[1].size, 31705) }) }) }) diff --git a/test/media/routes.test.mjs b/test/media/routes.test.mjs index 39d39ec..987f91d 100644 --- a/test/media/routes.test.mjs +++ b/test/media/routes.test.mjs @@ -2,22 +2,187 @@ import { Eltro as t, assert, stub } from 'eltro' import { createContext } from '../helper.server.mjs' import MediaRoutes from '../../api/media/routes.mjs' +import { HttpError } from '../../api/error.mjs' -t.describe('#upload', () => { +t.describe('#filesCacheGet', function() { + t.test('should return all files in public folder', async function() { + const routes = new MediaRoutes() + + await routes.init() + + assert.ok(routes.filesCacheGet('development')) + assert.ok(Array.isArray(routes.filesCacheGet('development'))) + assert.ok(routes.filesCacheGet('existing')) + assert.ok(Array.isArray(routes.filesCacheGet('existing'))) + assert.ok(routes.filesCacheGet('nonexisting')) + assert.ok(Array.isArray(routes.filesCacheGet('nonexisting'))) + + assert.strictEqual(routes.filesCacheGet('development').length, 1) + assert.strictEqual(routes.filesCacheGet('existing').length, 2) + assert.strictEqual(routes.filesCacheGet('nonexisting').length, 0) + + assert.strictEqual(routes.filesCacheGet('development')[0].name, '.gitkeep') + assert.strictEqual(routes.filesCacheGet('development')[0].size, 0) + assert.strictEqual(routes.filesCacheGet('existing')[0].name, '20220105_101610_test1.jpg') + assert.strictEqual(routes.filesCacheGet('existing')[0].size, 15079) + assert.strictEqual(routes.filesCacheGet('existing')[1].name, '20220105_101610_test2.png') + assert.strictEqual(routes.filesCacheGet('existing')[1].size, 31705) + }) + + t.test('should always sort result', async function() { + const routes = new MediaRoutes() + + await routes.init() + + let files = routes.filesCacheGet('existing') + + assert.strictEqual(files[0].name, '20220105_101610_test1.jpg') + assert.strictEqual(files[0].size, 15079) + assert.strictEqual(files[1].name, '20220105_101610_test2.png') + assert.strictEqual(files[1].size, 31705) + + routes.siteCache.get('existing').push({ name: '0000.png', size: 0 }) + + files = routes.filesCacheGet('existing') + assert.strictEqual(files[0].name, '0000.png') + assert.strictEqual(files[0].size, 0) + assert.strictEqual(files[1].name, '20220105_101610_test1.jpg') + assert.strictEqual(files[1].size, 15079) + assert.strictEqual(files[2].name, '20220105_101610_test2.png') + assert.strictEqual(files[2].size, 31705) + }) +}) + +t.describe('#listFiles()', function() { + const stubVerify = stub() + const stubGetCache = stub() + + const routes = new MediaRoutes({ + security: { verifyToken: stubVerify }, + }) + routes.filesCacheGet = stubGetCache + + function reset() { + stubVerify.reset() + stubGetCache.reset() + } + + t.test('should call security correctly', async function() { + reset() + + let ctx = createContext() + const assertError = new Error('temp') + stubVerify.rejects(assertError) + + let err = await assert.isRejected(routes.listFiles(ctx)) + + assert.ok(stubVerify.called) + assert.strictEqual(err, assertError) + assert.strictEqual(stubVerify.firstCall[0], ctx) + }) + + t.test('should call filesCacheGet and return results', async function() { + reset() + + let ctx = createContext() + const assertSiteName = 'benshapiro' + const assertResult = { a: 1 } + stubVerify.resolves(assertSiteName) + stubGetCache.returns(assertResult) + + await routes.listFiles(ctx) + + assert.ok(stubGetCache.called) + assert.strictEqual(stubGetCache.firstCall[0], assertSiteName) + assert.strictEqual(ctx.body, assertResult) + }) +}) + +t.describe('#listPublicFiles()', function() { + const stubSitePublic = stub() + const stubGetCache = stub() + + const routes = new MediaRoutes({ + security: { throwIfNotPublic: stubSitePublic }, + }) + routes.filesCacheGet = stubGetCache + + function reset() { + stubSitePublic.reset() + stubGetCache.reset() + } + + t.test('should call security correctly', async function() { + reset() + + let ctx = createContext() + const assertError = new Error('temp') + const assertSite = 'astalavista' + stubSitePublic.throws(assertError) + ctx.params.site = assertSite + + let err = await assert.isRejected(routes.listPublicFiles(ctx)) + + assert.ok(stubSitePublic.called) + assert.strictEqual(err, assertError) + assert.strictEqual(stubSitePublic.firstCall[0], assertSite) + }) + + t.test('should call filesCacheGet and return results', async function() { + reset() + + let ctx = createContext() + const assertSiteName = 'benshapiro' + const assertResult = { a: 1 } + ctx.params.site = assertSiteName + stubGetCache.returns(assertResult) + + await routes.listPublicFiles(ctx) + + assert.ok(stubGetCache.called) + assert.strictEqual(stubGetCache.firstCall[0], assertSiteName) + assert.strictEqual(ctx.body, assertResult) + }) +}) + +t.describe('#filesCacheAdd', function() { + t.test('should auto-create array of site and add file', async function() { + const routes = new MediaRoutes() + const assertName = 'asdf.png' + const assertSize = 1234 + + await routes.init() + + assert.ok(routes.filesCacheGet('nonexisting')) + assert.ok(Array.isArray(routes.filesCacheGet('nonexisting'))) + assert.strictEqual(routes.filesCacheGet('nonexisting').length, 0) + + routes.filesCacheAdd('nonexisting', assertName, assertSize) + + assert.strictEqual(routes.filesCacheGet('nonexisting').length, 1) + assert.strictEqual(routes.filesCacheGet('nonexisting')[0].name, assertName) + assert.strictEqual(routes.filesCacheGet('nonexisting')[0].size, assertSize) + }) +}) + +t.describe('#upload', function() { const stubVerify = stub() const stubUpload = stub() + const stubStat = stub() const routes = new MediaRoutes({ security: { verifyToken: stubVerify }, formidable: { uploadFile: stubUpload }, + fs: { stat: stubStat }, }) function reset() { stubVerify.reset() stubUpload.reset() + stubStat.reset() } - t.test('should call security correctly', async () => { + t.test('should call security correctly', async function() { reset() let ctx = createContext() @@ -31,7 +196,7 @@ t.describe('#upload', () => { assert.strictEqual(stubVerify.firstCall[0], ctx) }) - t.test('should call upload correctly', async () => { + t.test('should call upload correctly', async function() { reset() let ctx = createContext() @@ -48,17 +213,306 @@ t.describe('#upload', () => { assert.strictEqual(stubUpload.firstCall[1], assertSiteName) }) - t.test('should otherwise set context status to 204 and file in result', async () => { + t.test('should otherwise set context status to 204 and file in result', async function() { reset() let ctx = createContext() + const assertSize = 1241412 const assertFilename = 'asdfsafd' const assertSite = 'mario' stubVerify.resolves(assertSite) stubUpload.resolves({ filename: assertFilename }) + stubStat.resolves({ size: assertSize }) await routes.upload(ctx) assert.strictEqual(ctx.body.filename, assertFilename) assert.strictEqual(ctx.body.path, `/${assertSite}/${assertFilename}`) + + assert.ok(stubStat.called) + assert.strictEqual(stubStat.firstCall[0], `./public/${assertSite}/${assertFilename}`) + + let filesFromCache = routes.filesCacheGet(assertSite) + assert.strictEqual(filesFromCache.length, 1) + assert.strictEqual(filesFromCache[0].name, assertFilename) + assert.strictEqual(filesFromCache[0].size, assertSize) + }) +}) + +t.describe('#resize', function() { + const stubVerifyToken = stub() + const stubVerifyBody = stub() + const stubUpload = stub() + const stubSharp = stub() + const stubSharpResize = stub() + const stubSharpRotate = stub() + const stubSharpToFile = stub() + const stubSharpJpeg = stub() + const stubSharpPng = stub() + const stubSharpToBuffer = stub() + const stubStat = stub() + + const routes = new MediaRoutes({ + security: { + verifyToken: stubVerifyToken, + verifyBody: stubVerifyBody, + }, + fs: { stat: stubStat }, + formidable: { uploadFile: stubUpload }, + sharp: stubSharp, + }) + + function reset() { + stubVerifyToken.reset() + stubVerifyBody.reset() + stubUpload.reset() + let def = { + toFile: stubSharpToFile, + jpeg: stubSharpJpeg, + png: stubSharpPng, + resize: stubSharpResize, + rotate: stubSharpRotate, + toBuffer: stubSharpToBuffer, + } + stubStat.reset() + stubStat.resolves({ size: 0 }) + stubSharp.reset() + stubSharp.returns(def) + stubSharpToFile.reset() + stubSharpToFile.resolves() + stubSharpJpeg.reset() + stubSharpJpeg.returns(def) + stubSharpResize.reset() + stubSharpResize.returns(def) + stubSharpRotate.reset() + stubSharpRotate.returns(def) + stubSharpPng.reset() + stubSharpPng.returns(def) + stubSharpToBuffer.reset() + stubSharpToBuffer.resolves() + routes.siteCache.clear() + } + + t.test('should call security verifyToken correctly', async function() { + reset() + + let ctx = createContext({ req: { body: { }}}) + const assertError = new Error('temp') + stubVerifyToken.rejects(assertError) + + let err = await assert.isRejected(routes.resize(ctx)) + + assert.ok(stubVerifyToken.called) + assert.strictEqual(err, assertError) + assert.strictEqual(stubVerifyToken.firstCall[0], ctx) + }) + + t.test('should call upload correctly', async function() { + reset() + + let ctx = createContext({ req: { body: { }}}) + const assertSiteName = 'benshapiro' + const assertError = new Error('hello') + stubVerifyToken.resolves(assertSiteName) + stubUpload.rejects(assertError) + + let err = await assert.isRejected(routes.resize(ctx)) + + assert.strictEqual(err, assertError) + assert.ok(stubUpload.called) + assert.strictEqual(stubUpload.firstCall[0], ctx) + assert.strictEqual(stubUpload.firstCall[1], assertSiteName) + }) + + t.test('should call security verifyBody correctly', async function() { + reset() + + let ctx = createContext({ req: { body: { }}}) + const assertError = new Error('temp') + stubUpload.resolves({ filename: 'bla' }) + stubVerifyBody.throws(assertError) + + let err = await assert.isRejected(routes.resize(ctx)) + + assert.strictEqual(err, assertError) + assert.ok(stubVerifyBody.called) + assert.strictEqual(stubVerifyBody.firstCall[0], ctx) + }) + + t.test('should otherwise return original in result', async function() { + reset() + + let ctx = createContext({ req: { body: { }}}) + const assertFilename = 'asdfsafd.png' + const assertSite = 'mario' + stubVerifyToken.resolves(assertSite) + stubUpload.resolves({ filename: assertFilename }) + + await routes.resize(ctx) + + assert.notOk(stubSharp.called) + assert.strictEqual(ctx.body.original.filename, assertFilename) + assert.strictEqual(ctx.body.original.path, `/${assertSite}/${assertFilename}`) + }) + + t.test('should call sharp correctly if items are specified', async function() { + reset() + const assertKey = 'asdf' + const assertJpeg = { a: 1 } + const assertFilename = 'asdfsafd' + const assertSite = 'mario' + stubVerifyToken.resolves(assertSite) + stubUpload.resolves({ filename: assertFilename + '.png' }) + + let ctx = createContext({ req: { body: { + [assertKey]: { + format: 'jpeg', + } + }}}) + ctx.req.body[assertKey].jpeg = assertJpeg + + await routes.resize(ctx) + + assert.ok(stubSharp.called) + assert.match(stubSharp.firstCall[0], new RegExp(`\/${assertSite}\/${assertFilename}\.png`)) + assert.ok(stubSharpRotate.called) + assert.ok(stubSharpJpeg.called) + assert.strictEqual(stubSharpJpeg.firstCall[0], assertJpeg) + assert.ok(stubSharpToFile.called) + assert.match(stubSharpToFile.firstCall[0], new RegExp(`\/${assertSite}\/${assertFilename}_${assertKey}\.jpg`)) + assert.notOk(stubSharpPng.called) + assert.notOk(stubSharpResize.called) + assert.strictEqual(ctx.body.original.filename, assertFilename + '.png') + assert.strictEqual(ctx.body.original.path, `/${assertSite}/${assertFilename}.png`) + assert.strictEqual(ctx.body[assertKey].filename, `${assertFilename}_${assertKey}\.jpg`) + assert.strictEqual(ctx.body[assertKey].path, `/${assertSite}/${assertFilename}_${assertKey}.jpg`) + }) + + t.test('should call sharp correctly if items and resize are specified', async function() { + reset() + const assertKey = 'herpderp' + const assertPng = { a: 1 } + const assertResize = { a: 1 } + const assertFilename = 'asdfsafd' + const assertSite = 'mario' + stubVerifyToken.resolves(assertSite) + stubUpload.resolves({ filename: assertFilename + '.png' }) + + let called = 0 + stubStat.returnWith(function() { + called += 10 + return { size: called } + }) + + let ctx = createContext({ req: { body: { + [assertKey]: { + format: 'png', + } + }}}) + ctx.req.body[assertKey].png = assertPng + ctx.req.body[assertKey].resize = assertResize + + await routes.resize(ctx) + + assert.ok(stubSharp.called) + assert.match(stubSharp.firstCall[0], new RegExp(`\/${assertSite}\/${assertFilename}\.png`)) + assert.ok(stubSharpRotate.called) + assert.ok(stubSharpPng.called) + assert.strictEqual(stubSharpPng.firstCall[0], assertPng) + assert.ok(stubSharpToFile.called) + assert.match(stubSharpToFile.firstCall[0], new RegExp(`\/${assertSite}\/${assertFilename}_${assertKey}\.png`)) + assert.notOk(stubSharpJpeg.called) + assert.ok(stubSharpResize.called) + assert.strictEqual(stubSharpResize.firstCall[0], assertResize) + assert.strictEqual(ctx.body.original.filename, assertFilename + '.png') + assert.strictEqual(ctx.body.original.path, `/${assertSite}/${assertFilename}.png`) + assert.strictEqual(ctx.body[assertKey].filename, `${assertFilename}_${assertKey}\.png`) + assert.strictEqual(ctx.body[assertKey].path, `/${assertSite}/${assertFilename}_${assertKey}.png`) + + let filesFromCache = routes.filesCacheGet(assertSite) + assert.strictEqual(filesFromCache.length, 2) + assert.strictEqual(filesFromCache[0].name, `${assertFilename}_${assertKey}\.png`) + assert.strictEqual(filesFromCache[0].size, 20) + assert.strictEqual(filesFromCache[1].name, assertFilename + '.png') + assert.strictEqual(filesFromCache[1].size, 10) + }) + + t.test('should notify which item failed if one fails', async function() { + reset() + const assertValidKey1 = 'herp' + const assertValidKey2 = 'derp' + const assertErrorKey = 'throwmyerr' + const assertErrorMessage = 'some message here' + stubVerifyToken.resolves('asdf') + stubUpload.resolves({ filename: 'file.png' }) + + let called = 0 + stubStat.returnWith(function() { + called += 10 + return { size: called } + }) + + let ctx = createContext({ req: { body: { + [assertValidKey1]: { + format: 'png', + png: { a: 1 }, + }, + [assertValidKey2]: { + format: 'png', + png: { a: 1 }, + }, + [assertErrorKey]: { + format: 'png', + png: { a: 1 }, + }, + }}}) + + stubSharpToFile.returnWith(function(file) { + if (file.match(new RegExp(assertErrorKey))) { + throw new Error(assertErrorMessage) + } + }) + + let err = await assert.isRejected(routes.resize(ctx)) + + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 422) + assert.match(err.message, new RegExp(assertErrorKey)) + assert.match(err.message, new RegExp(assertErrorMessage)) + }) + + t.test('should call sharp correctly and return base64 format if out is base64', async function() { + reset() + const assertKey = 'outtest' + const assertFilename = 'asdfsafd.png' + const assertSite = 'mario' + const assertBase64Data = 'asdf1234' + stubVerifyToken.resolves(assertSite) + stubUpload.resolves({ filename: assertFilename }) + + let ctx = createContext({ req: { body: { + [assertKey]: { + format: 'png', + png: { a: 1 }, + out: 'base64', + } + }}}) + + stubSharpToBuffer.resolves(Buffer.from(assertBase64Data)) + + await routes.resize(ctx) + + assert.ok(stubSharp.called) + assert.notOk(stubSharpToFile.called) + assert.ok(stubSharpToBuffer.called) + assert.ok(ctx.body[assertKey].base64) + assert.ok(ctx.body[assertKey].base64.startsWith(`data:image/png;base64,`)) + let base64 = ctx.body[assertKey].base64 + let bufferBase64 = Buffer.from(base64.slice(base64.indexOf(',')), 'base64') + assert.strictEqual(bufferBase64.toString(), assertBase64Data) + + let filesFromCache = routes.filesCacheGet(assertSite) + assert.strictEqual(filesFromCache.length, 1) + assert.strictEqual(filesFromCache[0].name, assertFilename) }) }) diff --git a/test/media/security.test.mjs b/test/media/security.test.mjs index 533db1f..c33fea2 100644 --- a/test/media/security.test.mjs +++ b/test/media/security.test.mjs @@ -1,16 +1,64 @@ import { Eltro as t, assert} from 'eltro' import { createContext } from '../helper.server.mjs' -import { verifyToken } from '../../api/media/security.mjs' +import { verifyToken, verifyBody, throwIfNotPublic } from '../../api/media/security.mjs' import { HttpError } from '../../api/error.mjs' import encode from '../../api/jwt/encode.mjs' import config from '../../api/config.mjs' +t.describe('#throwIfNotPublic()', function() { + t.before(function() { + config.set('sites', { + justatest: { + }, + justatest2: { + public: false, + }, + justatest3: { + public: true, + }, + }) + }) + + t.test('should throw for sites that do not exist or are null', function() { + let tests = [ + 'justatest', + 'justatest2', + 'nonexisting1', + null, + ] + + tests.forEach(function(test) { + assert.throws(function() { throwIfNotPublic(test) }, function(err) { + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 404) + assert.match(err.message, new RegExp(test)) + assert.match(err.message, /exist/i) + return true + }, `should throw with site ${test}`) + }) + }) + + + t.test('should pass for sites that allow public listing', function() { + let tests = [ + 'justatest3', + ] + + tests.forEach(function(test) { + assert.doesNotThrow(function() { throwIfNotPublic(test) }, `should not throw with site ${test}`) + }) + }) +}) + t.describe('#verifyToken()', function() { t.before(function() { config.set('sites', { justatest: { - 'default@HS512': 'mysharedkey' + keys: { + 'default@HS512': 'mysharedkey', + } }, }) }) @@ -23,8 +71,8 @@ t.describe('#verifyToken()', function() { assert.ok(err instanceof HttpError) assert.ok(err instanceof Error) assert.strictEqual(err.status, 422) - assert.match(err.message, /[Qq]uery/) - assert.match(err.message, /[Tt]oken/) + assert.match(err.message, /query/i) + assert.match(err.message, /token/i) return true }) }) @@ -33,8 +81,8 @@ t.describe('#verifyToken()', function() { assert.ok(err instanceof HttpError) assert.ok(err instanceof Error) assert.strictEqual(err.status, 422) - assert.match(err.message, /[Ii]nvalid/) - assert.match(err.message, /[Tt]oken/) + assert.match(err.message, /invalid/i) + assert.match(err.message, /token/i) return true } @@ -49,7 +97,7 @@ t.describe('#verifyToken()', function() { ctx.query.set('token', 'asdfasdgassdga.asdfasdg.sadfsadfas') assert.throws(function() { verifyToken(ctx) }, assertInvalidToken) - assert.match(ctx.log.error.lastCall[0].message, /[Ii]nvalid/) + assert.match(ctx.log.error.lastCall[0].message, /invalid/i) ctx.query.set('token', encode( { typ: 'JWT', alg: 'HS256' }, @@ -74,7 +122,7 @@ t.describe('#verifyToken()', function() { )) assert.throws(function() { verifyToken(ctx) }, assertInvalidToken) assert.match(ctx.log.error.lastCall[0].message, /HS512/) - assert.match(ctx.log.error.lastCall[0].message, /[vV]erification/) + assert.match(ctx.log.error.lastCall[0].message, /Verification/i) }) t.test('should otherwise return the issuer', function() { @@ -88,3 +136,241 @@ t.describe('#verifyToken()', function() { assert.strictEqual(site, 'justatest') }) }) + +t.describe('#verifyBody()', function() { + t.test('should succeed with empty body', function() { + let ctx = createContext({ req: { body: { } } }) + + verifyBody(ctx) + }) + + t.test('should fail with invalid body', function() { + let ctx = createContext({ req: { body: { + item: {} + } } }) + + let tests = [ + [null, 'null'], + ['', 'empty string'], + ['asdf', 'string'], + [0, 'empty number'], + [123, 'number'], + [[], 'array'], + ] + + tests.forEach(function (check) { + ctx.req.body.item = check[0] + + assert.throws(function() { verifyBody(ctx) }, function(err) { + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 422) + assert.match(err.message, /body/i) + assert.match(err.message, /item/i) + assert.match(err.message, /valid/i) + return true + }, `should fail if body entry is ${check[1]}`) + }) + }) + + t.test('should fail if an item has the name original', function() { + let ctx = createContext({ req: { body: { + original: {} + } } }) + + assert.throws(function() { verifyBody(ctx) }, function(err) { + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 422) + assert.match(err.message, /body/i) + assert.match(err.message, /name/i) + assert.match(err.message, /original/i) + assert.match(err.message, /allowed/i) + return true + }, 'should fail if body item has the name original') + }) + + t.test('should require format string present in item', function() { + let ctx = createContext({ req: { body: { + item: {} + } } }) + + let tests = [ + [undefined, 'undefined'], + [null, 'null'], + ['', 'empty string'], + [{}, 'object'], + [0, 'empty number'], + [123, 'number'], + [[], 'array'], + ['resize', 'not allow resize'], + ['out', 'not allow out'], + ] + + tests.forEach(function (check) { + ctx.req.body.item = { + format: check[0], + } + + assert.throws(function() { verifyBody(ctx) }, function(err) { + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 422) + assert.match(err.message, /body/i) + assert.match(err.message, /format/i) + assert.match(err.message, /missing/i) + return true + }, `should fail if body item format is ${check[1]}`) + }) + }) + + t.test('should require object of same name as format', function() { + let ctx = createContext({ req: { body: { + item: {} + } } }) + + let tests = [ + [undefined, 'undefined'], + [null, 'null'], + ['', 'emptystring'], + ['asdf', 'string'], + [0, 'emptynumber'], + [123, 'number'], + [[], 'array'], + ] + + tests.forEach(function (check) { + ctx = createContext({ req: { body: { + item: {} + } } }) + + ctx.req.body.item.format = check[1] + ctx.req.body.item[check[1]] = check[0] + + assert.throws(function() { verifyBody(ctx) }, function(err) { + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 422) + assert.match(err.message, /body/i) + assert.match(err.message, /format/i) + assert.match(err.message, /options/i) + assert.match(err.message, /valid/i) + return true + }, `should fail if body item format options is ${check[1]}`) + }) + }) + + t.test('should allow empty value or string in out in item', function() { + let ctx = createContext({ req: { body: { + item: { + format: 'test', + test: {}, + } + } } }) + + let tests = [ + [undefined, 'undefined'], + [null, 'null'], + ['', 'empty string'], + ['file', 'allow string file'], + ['base64', 'allow base64'], + ] + + tests.forEach(function (check) { + ctx.req.body.item.out = check[0] + + assert.doesNotThrow(function() { + verifyBody(ctx) + }, `should not throw with ${check[1]} in out`) + }) + }) + + t.test('should fail if out is invalid value', function() { + let ctx = createContext({ req: { body: { + item: { + format: 'test', + test: {}, + } + } } }) + + let tests = [ + [{}, 'object'], + [0, 'empty number'], + [123, 'number'], + [[], 'array'], + ['resize', 'not allow resize'], + ['out', 'not allow out'], + ['example', 'not allow example'], + ] + + tests.forEach(function (check) { + ctx.req.body.item.out = check[0] + + assert.throws(function() { verifyBody(ctx) }, function(err) { + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 422) + assert.match(err.message, /body/i) + assert.match(err.message, /item/i) + assert.match(err.message, /out/i) + assert.match(err.message, /valid/i) + return true + }, `should fail if body item out is ${check[1]}`) + }) + }) + + t.test('should allow empty value or object in resize', function() { + let ctx = createContext({ req: { body: { + item: { + format: 'test', + test: {}, + } + } } }) + + let tests = [ + [undefined, 'undefined'], + [null, 'null'], + [{}, 'object'], + ] + + tests.forEach(function (check) { + ctx.req.body.item.resize = check[0] + + assert.doesNotThrow(function() { + verifyBody(ctx) + }, `should not throw with ${check[1]} in resize`) + }) + }) + + t.test('should fail if resize if specified is invalid', function() { + let ctx = createContext({ req: { body: { + item: { + format: 'test', + test: {}, + } + } } }) + + let tests = [ + ['', 'emptystring'], + ['asdf', 'string'], + [0, 'emptynumber'], + [123, 'number'], + [[], 'array'], + ] + + tests.forEach(function (check) { + ctx.req.body.item.resize = check[0] + + assert.throws(function() { verifyBody(ctx) }, function(err) { + assert.ok(err instanceof HttpError) + assert.ok(err instanceof Error) + assert.strictEqual(err.status, 422) + assert.match(err.message, /body/i) + assert.match(err.message, /item/i) + assert.match(err.message, /resize/i) + assert.match(err.message, /valid/i) + return true + }, `should fail if body item resize is ${check[1]}`) + }) + }) +})