From 840f23908ed99f864b0d21407fd8e89f434dd91f Mon Sep 17 00:00:00 2001 From: Jonatan Nilsson Date: Thu, 11 May 2023 12:26:41 +0000 Subject: [PATCH] saproxy: Created basic proxy service --- saproxy/.npmrc | 1 + saproxy/api/config.mjs | 35 +++++++ saproxy/api/proxy.mjs | 176 ++++++++++++++++++++++++++++++++++ saproxy/api/server.mjs | 109 +++++++++++++++++++++ saproxy/api/static_routes.mjs | 17 ++++ saproxy/build-package.json | 8 ++ saproxy/dev.mjs | 24 +++++ saproxy/index.mjs | 11 +++ saproxy/package.json | 46 +++++++++ 9 files changed, 427 insertions(+) create mode 100644 saproxy/.npmrc create mode 100644 saproxy/api/config.mjs create mode 100644 saproxy/api/proxy.mjs create mode 100644 saproxy/api/server.mjs create mode 100644 saproxy/api/static_routes.mjs create mode 100644 saproxy/build-package.json create mode 100644 saproxy/dev.mjs create mode 100644 saproxy/index.mjs create mode 100644 saproxy/package.json diff --git a/saproxy/.npmrc b/saproxy/.npmrc new file mode 100644 index 0000000..9cf9495 --- /dev/null +++ b/saproxy/.npmrc @@ -0,0 +1 @@ +package-lock=false \ No newline at end of file diff --git a/saproxy/api/config.mjs b/saproxy/api/config.mjs new file mode 100644 index 0000000..89eb3ce --- /dev/null +++ b/saproxy/api/config.mjs @@ -0,0 +1,35 @@ +import Nconf from 'nconf-lite' + +const nconf = new Nconf() + +// Helper method for global usage. +nconf.inTest = () => nconf.get('NODE_ENV') === 'test' + +// Config follow the following priority check order: +// 1. Enviroment variables +// 2. package.json +// 3. config/config.json +// 4. config/config.default.json + +// Load enviroment variables as first priority +nconf.env({ + separator: '__', + whitelist: [ + 'NODE_ENV', + ], + parseValues: true, +}) + + +// Load empty overrides that can be overwritten later +nconf.overrides({}) + +nconf.defaults({ + "NODE_ENV": "development", + "frontend": { + "url": "http://beta01.nfp.moe" + }, +}) + + +export default nconf diff --git a/saproxy/api/proxy.mjs b/saproxy/api/proxy.mjs new file mode 100644 index 0000000..662666e --- /dev/null +++ b/saproxy/api/proxy.mjs @@ -0,0 +1,176 @@ +import https from 'https' +import pipe from 'multipipe' +import { Transform, compose } from 'stream' +import config from './config.mjs' + +export default class Proxy { + constructor(opts = {}) { + Object.assign(this, { + agent: opts.agent || new https.Agent({ + keepAlive: true, + maxSockets: 2, + keepAliveMsecs: 1000 * 60 * 60, + secureProtocol: 'TLSv1_2_method', + rejectUnauthorized: false, + }), + cache: new Map(), + defaultRemap: 'default', + remap: new Map([ + ['default', '/frettatengt/serstok-malefni/taktu-tvaer/'] + ]) + }) + } + + register(server) { + server.flaska.on404(this.proxy.bind(this)) + } + + getTime(time) { + if (!time) return 60 + let m = time.match(/max-age=(\d+)/) + if (!m) return 60 + return Number(m[1]) || 60 + } + + getVaryKey(req, vary) { + if (!vary) { + return 'null' + } + let out = [] + for (let v of vary.split(',')) { + out.push(req.headers[v.trim().toLowerCase()] || 'null') + } + return out.join(', ') + } + + cleanHeaders(headers) { + var set = new Set() + for (let key of Object.keys(headers)) { + if (set.has(key.toLowerCase())) { + delete headers[key] + } else { + set.add(key) + } + } + return headers + } + + cloneHeaders(headers) { + var out = {} + for (let key of Object.keys(headers)) { + if (key !== 'content-length' && key !== 'content-type') { + out[key] = headers[key] + } + } + return out + } + + proxyCache(req, path, time, res) { + let cache = this.cache.get(path) + if (!cache) { + this.cache.set(path, cache = { + path: path, + time: this.getTime(time), + created: new Date(), + status: res.statusCode, + type: res.headers['content-type'], + vary: (res.headers['vary'] || '').toLowerCase(), + data: new Map() + }) + } + + let varyKey = this.getVaryKey(req, cache.vary) + + if (cache.data.has(varyKey)) return res + + let item = { + headers: this.cleanHeaders(res.headers), + data: [], + done: false, + } + cache.data.set(varyKey, item) + + const capture = new Transform({ + decodeStrings: false, + destroy(err, cb) { + if (!err) { + if (item.data.length) { + item.data = Buffer.concat(item.data) + item.done = true + } else { + cache.data.delete(varyKey, item) + } + } + cb(err) + }, + + transform(chunk, encoding, callback) { + item.data.push(chunk) + callback(null, chunk); + }, + }); + + return pipe(res, capture) + } + + getCache(req, path) { + let cache = this.cache.get(path) + if (!cache) return null + if ((new Date() - cache.created) / 1000 > cache.time) { + this.cache.delete(path) + return null + } + let varyKey = this.getVaryKey(req, cache.vary) + let data = cache.data.get(varyKey) + if (!data) return null + if (!data.done) return null + + return { + varyKey: varyKey, + res: cache, + data: cache.data.get(varyKey) + } + } + + async proxy(ctx) { + let url = ctx.req.url + if (!url.startsWith('/_')) { + url = this.remap.get(ctx.req.headers['host']) + if (!url) { + url = this.remap.get(this.defaultRemap) + } + url += ctx.req.url.slice(1) + } + let cache = this.getCache(ctx.req, url) + + if (cache) { + ctx.status = cache.res.status + ctx.type = cache.res.type + ctx.body = cache.data.data + ctx.headers = this.cloneHeaders(cache.data.headers) + console.log('found cache', cache) + return + } + + ctx.req.headers.host = 'www.sa.is' + + var res = await new Promise((res, rej) => { + var req = https.request({ + hostname: 'www.sa.is', + path: url, + method: ctx.req.method, + headers: ctx.req.headers, + agent: this.agent, + }, results => { + res(results) + }) + req.on('error', rej) + req.end() + }) + + ctx.status = res.statusCode + ctx.headers = this.cloneHeaders(res.headers) + ctx.type = res.headers['content-type'] + ctx.body = this.proxyCache(ctx.req, url, ctx.headers['cache-control'], res) + } +} diff --git a/saproxy/api/server.mjs b/saproxy/api/server.mjs new file mode 100644 index 0000000..2c56ecb --- /dev/null +++ b/saproxy/api/server.mjs @@ -0,0 +1,109 @@ +import { Flaska, QueryHandler, JsonHandler } from 'flaska' +import config from './config.mjs' +import StaticRoutes from './static_routes.mjs' +import Proxy from './proxy.mjs' + +export default class Server { + constructor(http, port, core, opts = {}) { + Object.assign(this, opts) + this.http = http + this.port = port + this.core = core + + this.flaskaOptions = { + appendHeaders: { + 'Content-Security-Policy': `default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src * data: blob:; font-src 'self' data:; object-src 'none'; frame-ancestors 'none'`, + }, + log: this.core.log, + nonce: ['script-src'], + nonceCacheLength: 50, + } + + this.init() + } + + init() { } + + runCreateServer() { + // Create our server + this.flaska = new Flaska(this.flaskaOptions, this.http) + + // configure our server + console.log(config.get('NODE_ENV')) + if (config.get('NODE_ENV') === 'development') { + this.flaska.devMode() + } + + this.flaska.onerror(err => { + console.log(err) + ctx.log.error(err) + ctx.status = 500 + }) + + this.flaska.before(function(ctx) { + ctx.state.started = new Date().getTime() + ctx.req.ip = ctx.req.headers['x-forwarded-for'] || ctx.req.connection.remoteAddress + ctx.log = ctx.log.child({ + id: Math.random().toString(36).substring(2, 14), + }) + }) + + let healthChecks = 0 + let healthCollectLimit = 60 * 60 * 12 + + this.flaska.after(function(ctx) { + if (ctx.aborted && ctx.status === 200) { + ctx.status = 299 + } + let ended = new Date().getTime() + var requestTime = ended - ctx.state.started + + let status = '' + let level = 'info' + if (ctx.status >= 400) { + status = ctx.status + ' ' + level = 'warn' + } + if (ctx.status >= 500) { + level = 'error' + } + + if (ctx.url === '/health' || ctx.url === '/api/health') { + healthChecks++ + if (healthChecks >= healthCollectLimit) { + ctx.log[level]({ + duration: Math.round(ended), + status: ctx.status, + }, `<-- ${status}${ctx.method} ${ctx.url} {has happened ${healthChecks} times}`) + healthChecks = 0 + } + return + } + + ctx.log[level]({ + duration: requestTime, + status: ctx.status, + ip: ctx.req.ip, + }, (ctx.aborted ? '[ABORT]' : '<--') + ` ${status}${ctx.method} ${ctx.url}`) + }) + } + + registerRoutes() { + var stat = new StaticRoutes() + var proxy = new Proxy() + stat.register(this) + proxy.register(this) + } + + runStartListen() { + return this.flaska.listenAsync(this.port).then(() => { + this.core.log.info('Server is listening on port ' + this.port) + }) + } + + run() { + this.runCreateServer() + this.registerRoutes() + return this.runStartListen() + } +} diff --git a/saproxy/api/static_routes.mjs b/saproxy/api/static_routes.mjs new file mode 100644 index 0000000..7ce8d01 --- /dev/null +++ b/saproxy/api/static_routes.mjs @@ -0,0 +1,17 @@ +import config from './config.mjs' + +export default class StaticRoutes { + constructor(opts = {}) { + Object.assign(this, { }) + } + + register(server) { + server.flaska.get('/api/health', this.health.bind(this)) + } + + health(ctx) { + ctx.body = { + environment: config.get('NODE_ENV'), + } + } +} diff --git a/saproxy/build-package.json b/saproxy/build-package.json new file mode 100644 index 0000000..f03ccc7 --- /dev/null +++ b/saproxy/build-package.json @@ -0,0 +1,8 @@ +{ + "scripts": { + "build": "echo done;" + }, + "dependencies": { + "service-core": "^3.0.0-beta.17" + } +} diff --git a/saproxy/dev.mjs b/saproxy/dev.mjs new file mode 100644 index 0000000..0d7f32f --- /dev/null +++ b/saproxy/dev.mjs @@ -0,0 +1,24 @@ +import fs from 'fs' +import { ServiceCore } from 'service-core' +import * as index from './index.mjs' + +const port = 4120 + +var core = new ServiceCore('nfp_moe', import.meta.url, port, '') + +let config = { + frontend: { + url: 'http://localhost:' + port + } +} + +try { + config = JSON.parse(fs.readFileSync('./config.json')) +} catch {} + +config.port = port + +core.setConfig(config) +core.init(index).then(function() { + return core.run() +}) \ No newline at end of file diff --git a/saproxy/index.mjs b/saproxy/index.mjs new file mode 100644 index 0000000..4a8dd8e --- /dev/null +++ b/saproxy/index.mjs @@ -0,0 +1,11 @@ +import config from './api/config.mjs' + +export function start(http, port, ctx) { + config.sources[1].store = ctx.config + + return import('./api/server.mjs') + .then(function(module) { + let server = new module.default(http, port, ctx) + return server.run() + }) +} diff --git a/saproxy/package.json b/saproxy/package.json new file mode 100644 index 0000000..c488d52 --- /dev/null +++ b/saproxy/package.json @@ -0,0 +1,46 @@ +{ + "name": "saproxy", + "version": "1.0.0", + "port": 4130, + "description": "Proxy guy for SA", + "main": "index.js", + "directories": { + "test": "test" + }, + "scripts": { + "start": "node --experimental-modules index.mjs", + "dev:server": "node dev.mjs | bunyan", + "dev": "npm-watch dev:server" + }, + "watch": { + "dev:server": { + "patterns": [ + "api/*", + "base/*", + "../base/*" + ], + "extensions": "js,mjs", + "quiet": true, + "inherit": true + } + }, + "repository": { + "type": "git", + "url": "https://git.nfp.is/nfp/nfp_sites.git" + }, + "author": "Jonatan Nilsson", + "license": "WTFPL", + "bugs": { + "url": "https://git.nfp.is/nfp/nfp_sites/issues" + }, + "homepage": "https://git.nfp.is/nfp/nfp_sites", + "dependencies": { + "bunyan-lite": "^1.2.1", + "flaska": "^1.3.3", + "multipipe": "^4.0.0", + "nconf-lite": "^2.0.0" + }, + "devDependencies": { + "service-core": "^3.0.0-beta.17" + } +}