Jonatan Nilsson
d1bab7a929
continuous-integration/appveyor/branch AppVeyor build succeeded
Details
|
||
---|---|---|
api | ||
public | ||
test | ||
.gitignore | ||
.npmrc | ||
7zas | ||
README.md | ||
appveyor.yml | ||
dev.mjs | ||
index.mjs | ||
package.json |
README.md
storage-upload
Micro service for uploading and image resizing files to a storage server.
Usage
storage-upload listens on port 4020 by default. Since the storage upload needs a place to store the uploaded files, it's important to map the /app/public
folder to either locally or some volume.
In addition it's recommended to also map the config file /app/config/config.production.json
which should include a list of accepted sites and the corresponding json web token secret.
docker run -d \
--name storage-upload \
-v /path/to/store/files:/app/public \
-v /path/to/config/file.json:/app/config/config.production.json \
-p 4000:4020
nfpis/storage-upload
Config
The mapped config file should look something like this:
config.production.json
{
"sites": {
"site1": {
"keys": {
"default@HS256": "asdf1234"
}
},
"site2": {
"public": true,
"keys": {
"default@HS256": "asdf1234",
"default@HS512": "asdfasdf"
}
}
}
}
Each entry in sites
has the following properties:
-
keys: Array list of key + hashing algorithm with the corresponding secret key for it.
-
public: Boolean property to indicate this folder allows public folder enumerating.
The server checks the token in the query for supported site name and then verifies the secret match. To generate a token valid to upload to site2
above, you can run something like this:
const jwt = require('jsonwebtoken')
let token = jwt.sign({ iss: 'site2' }, 'site2-secret')
// eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaXRlIjoic2l0ZTIifQ.Ovz7fnTMzaWOLOhnbkMtqHPk20EVqhCD8WDsLKk_Wv0
Hint: Post the above token to https://jwt.io/ and check it out.
Using the above token would save the requested file under /app/public/site2/
folder
API
- POST
/media
- POST
/media/noprefix
- POST
/media/resize
- GET
/media
- GET
/media/:site
- DELETE
/media/:filename
POST /media
Upload requested file to the corresponding folder with auto-generated datetime prefix.
-
URL Params
token=[site-token-here]
-
Data Params
file=@my-file-to-upload.jpg
-
Success Response:
- Code: 200
Content:{ filename: '20171210_115632_my-file-to-upload.jpg', path: '/development/20171210_115632_my-file-to-upload.jpg' }
- Code: 200
-
Sample Call:
curl -X POST -F "file=@my-file-to-upload.jpg" http://localhost:4020/media\?token\=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaXRlIjoidGVzdCJ9.2LAuYwb1bwiMPUWD3gNJKwt9PwLgctleLhYd6sc0FCU
POST /media/noprefix
Upload requested file to the corresponding folder with no prefix, overwriting any existing file if such conflicts happen.
-
URL Params
token=[site-token-here]
-
Data Params
file=@my-file-to-upload.jpg
-
Success Response:
- Code: 200
Content:{ filename: 'my-file-to-upload.jpg', path: '/development/my-file-to-upload.jpg' }
- Code: 200
-
Sample Call:
curl -X POST -F "file=@my-file-to-upload.jpg" http://localhost:4020/media/noprefix\?token\=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaXRlIjoidGVzdCJ9.2LAuYwb1bwiMPUWD3gNJKwt9PwLgctleLhYd6sc0FCU
POST /media/resize
Upload requested file to the corresponding folder and create multiple different resolutions or versions using the requested parameters.
-
URL Params
token=[site-token-here]
-
Data Params
file=@my-file-to-upload.jpg
-
Body Params
{
"medium": {
"format": "png",
"png": {
"compressionLevel": 9
},
"resize": {
"width": 600,
"fit": "inside",
"withoutEnlargement": true
},
},
"small": {
"format": "jpeg",
"jpeg": {
"quality": 80,
"mozjpeg": true
},
"resize": {
"width": 300,
},
},
"tiny": {
"format": "jpeg",
"jpeg": {
"quality": 80,
"mozjpeg": true
},
"blur": 1,
"resize": {
"width": 100,
},
},
}
- Supported operations:
trim
,flatten
,resize
,blur
,extend
For more information about the operations and the parameters they accept, see sharp documentation: https://sharp.pixelplumbing.com/
-
Success Response:
- Code: 200
Content:
- Code: 200
{
"filename": "20220106_102348_test.png",
"path": "/development/20220106_102348_test.png",
"medium": {
"filename": "20220106_102348_test_medium.png",
"path": "/development/20220106_102348_test_medium.png"
},
"small": {
"filename": "20220106_102348_test_small.jpg",
"path": "/development/20220106_102348_test_small.jpg"
},
"tiny": {
"filename": "20220106_102348_test_small.jpg",
"path": "/development/20220106_102348_test_small.jpg"
}
}
-
Sample Call:
curl -X POST -F 'medium={\"format\":\"png\",\"png\":{\"compressionLevel\":9},\"resize\":{\"width\":300}}' -F "file=@test.png" 'http://localhost:4020/media/resize?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaXRlIjoidGVzdCJ9.2LAuYwb1bwiMPUWD3gNJKwt9PwLgctleLhYd6sc0FCU'
GET /media
List all files in the reqested folder
-
URL Params
token=[site-token-here]
-
Success Response:
- Code: 200
Content:[{filename: 'file1.jpg', size:111}, {filename: 'file2.png', size: 222}]
- Code: 200
-
Sample Call:
curl -X GET http://localhost:4020/media\?token\=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaXRlIjoidGVzdCJ9.2LAuYwb1bwiMPUWD3gNJKwt9PwLgctleLhYd6sc0FCU
GET /media/:sitename
List all files in the reqested sitename folder. Please note that this requires the requested site in the config to have "public":true
specified. Otherwise it will always return 404
.
-
Success Response:
- Code: 200
Content:[{filename: 'file1.jpg', size:111}, {filename: 'file2.png', size: 222}]
- Code: 200
-
Sample Call:
curl -X GET http://localhost:4020/media/mypublicsite
DELETE /media/:filename
Remove the filename from the site from the token. Returns 204
if file is found and removed, otherwise returns 422
.
-
URL Params
token=[site-token-here]
-
Success Response:
- Code: 204
-
Sample Call:
curl -X DEL http://localhost:4020/media/20220105_101610_test1.jpg\?token\=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaXRlIjoidGVzdCJ9.2LAuYwb1bwiMPUWD3gNJKwt9PwLgctleLhYd6sc0FCU
Errors
Errors return in the following format: { status:422, message: "error message here" }
Example node.js helper
upload.js
import http from 'http'
import fs from 'fs/promises'
import path from 'path'
import { URL } from 'url'
// taken from isobject npm library
function isObject(val) {
return val != null && typeof val === 'object' && Array.isArray(val) === false
}
function defaults(options, def) {
let out = { }
if (options) {
Object.keys(options || {}).forEach(key => {
out[key] = options[key]
if (Array.isArray(out[key])) {
out[key] = out[key].map(item => {
if (isObject(item)) return defaults(item)
return item
})
} else if (out[key] && typeof out[key] === 'object') {
out[key] = defaults(options[key], def && def[key])
}
})
}
if (def) {
Object.keys(def).forEach(function(key) {
if (typeof out[key] === 'undefined') {
out[key] = def[key]
}
})
}
return out
}
function Client(prefix = 'http://localhost:4020', opts) {
this.options = defaults(opts, {})
this.prefix = prefix
}
Client.prototype.customRequest = function(method = 'GET', path, body, options) {
if (path.slice(0, 4) !== 'http') {
path = this.prefix + path
}
let urlObj = new URL(path)
return new Promise((resolve, reject) => {
const opts = defaults(defaults(options, {
method: method,
timeout: 500,
protocol: urlObj.protocol,
username: urlObj.username,
password: urlObj.password,
host: urlObj.hostname,
port: Number(urlObj.port),
path: urlObj.pathname + urlObj.search,
}))
const req = http.request(opts)
if (body) {
req.write(body)
}
req.on('error', reject)
req.on('timeout', function() { reject(new Error(`Request ${method} ${path} timed out`)) })
req.on('response', res => {
res.setEncoding('utf8')
let output = ''
res.on('data', function (chunk) {
output += chunk.toString()
})
res.on('end', function () {
if (!output) return resolve(null)
try {
output = JSON.parse(output)
} catch (e) {
return reject(new Error(`${e.message} while decoding: ${output}`))
}
if (output.status) {
let err = new Error(`Request failed [${output.status}]: ${output.message}`)
err.body = output
err.status = output.status
return reject(err)
}
resolve(output)
})
})
req.end()
})
}
Client.prototype.get = function(url = '/') {
return this.customRequest('GET', url, null)
}
Client.prototype.del = function(url = '/', body = {}) {
return this.customRequest('DELETE', url, JSON.stringify(body))
}
const random = (length = 8) => {
// Declare all characters
let chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
// Pick characers randomly
let str = '';
for (let i = 0; i < length; i++) {
str += chars.charAt(Math.floor(Math.random() * chars.length));
}
return str;
}
Client.prototype.upload = function(url, file, method = 'POST', body = {}) {
return fs.readFile(file).then(data => {
const crlf = '\r\n'
const filename = path.basename(file)
const boundary = `---------${random(32)}`
const multipartBody = Buffer.concat([
Buffer.from(
`${crlf}--${boundary}${crlf}` +
`Content-Disposition: form-data; name="file"; filename="${filename}"` + crlf + crlf
),
data,
Buffer.concat(Object.keys(body).map(function(key) {
return Buffer.from(''
+ `${crlf}--${boundary}${crlf}`
+ `Content-Disposition: form-data; name="${key}"` + crlf + crlf
+ JSON.stringify(body[key])
)
})),
Buffer.from(`${crlf}--${boundary}--`),
])
return this.customRequest(method, url, multipartBody, {
timeout: 5000,
headers: {
'Content-Type': 'multipart/form-data; boundary=' + boundary,
'Content-Length': multipartBody.length,
},
})
})
}
// const client = new Client()
// client.upload('/media/resize', 'test.png', {
// medium: {
// format: 'jpeg',
// resize: { width: 200 },
// },
// }).then(function(res) {
// console.log('success', res)
// }, function(err) {
// console.log('error', err)
// })