saproxy: Created basic proxy service

saproxy/.npmrc

@@ -0,0 +1 @@
+package-lock=false

saproxy/api/config.mjs

@@ -0,0 +1,35 @@
+import Nconf from 'nconf-lite'
+
+const nconf = new Nconf()
+
+// Helper method for global usage.
+nconf.inTest = () => nconf.get('NODE_ENV') === 'test'
+
+// Config follow the following priority check order:
+// 1. Enviroment variables
+// 2. package.json
+// 3. config/config.json
+// 4. config/config.default.json
+
+// Load enviroment variables as first priority
+nconf.env({
+  separator: '__',
+  whitelist: [
+    'NODE_ENV',
+  ],
+  parseValues: true,
+})
+
+
+// Load empty overrides that can be overwritten later
+nconf.overrides({})
+
+nconf.defaults({
+  "NODE_ENV": "development",
+  "frontend": {
+    "url": "http://beta01.nfp.moe"
+  },
+})
+
+
+export default nconf

saproxy/api/proxy.mjs

@@ -0,0 +1,176 @@
+import https from 'https'
+import pipe from 'multipipe'
+import { Transform, compose } from 'stream'
+import config from './config.mjs'
+
+export default class Proxy {
+  constructor(opts = {}) {
+    Object.assign(this, {
+      agent: opts.agent || new https.Agent({
+        keepAlive: true,
+        maxSockets: 2,
+        keepAliveMsecs: 1000 * 60 * 60,
+        secureProtocol: 'TLSv1_2_method',
+        rejectUnauthorized: false,
+      }),
+      cache: new Map(),
+      defaultRemap: 'default',
+      remap: new Map([
+        ['default', '/frettatengt/serstok-malefni/taktu-tvaer/']
+      ])
+    })
+  }
+
+  register(server) {
+    server.flaska.on404(this.proxy.bind(this))
+  }
+
+  getTime(time) {
+    if (!time) return 60
+    let m = time.match(/max-age=(\d+)/)
+    if (!m) return 60
+    return Number(m[1]) || 60
+  }
+
+  getVaryKey(req, vary) {
+    if (!vary) {
+      return 'null'
+    }
+    let out = []
+    for (let v of vary.split(',')) {
+      out.push(req.headers[v.trim().toLowerCase()] || 'null')
+    }
+    return out.join(', ')
+  }
+
+  cleanHeaders(headers) {
+    var set = new Set()
+    for (let key of Object.keys(headers)) {
+      if (set.has(key.toLowerCase())) {
+        delete headers[key]
+      } else {
+        set.add(key)
+      }
+    }
+    return headers
+  }
+
+  cloneHeaders(headers) {
+    var out = {}
+    for (let key of Object.keys(headers)) {
+      if (key !== 'content-length' && key !== 'content-type') {
+        out[key] = headers[key]
+      }
+    }
+    return out
+  }
+
+  proxyCache(req, path, time, res) {
+    let cache = this.cache.get(path)
+    if (!cache) {
+      this.cache.set(path, cache = {
+        path: path,
+        time: this.getTime(time),
+        created: new Date(),
+        status: res.statusCode,
+        type: res.headers['content-type'],
+        vary: (res.headers['vary'] || '').toLowerCase(),
+        data: new Map()
+      })
+    }
+
+    let varyKey = this.getVaryKey(req, cache.vary)
+
+    if (cache.data.has(varyKey)) return res
+
+    let item = {
+      headers: this.cleanHeaders(res.headers),
+      data: [],
+      done: false,
+    }
+    cache.data.set(varyKey, item)
+
+    const capture = new Transform({
+      decodeStrings: false,
+      destroy(err, cb) {
+        if (!err) {
+          if (item.data.length) {
+            item.data = Buffer.concat(item.data)
+            item.done = true
+          } else {
+            cache.data.delete(varyKey, item)
+          }
+        }
+        cb(err)
+      },
+
+      transform(chunk, encoding, callback) {
+        item.data.push(chunk)
+        callback(null, chunk);
+      },
+    });
+
+    return pipe(res, capture)
+  }
+
+  getCache(req, path) {
+    let cache = this.cache.get(path)
+    if (!cache) return null
+    if ((new Date() - cache.created) / 1000 > cache.time) {
+      this.cache.delete(path)
+      return null
+    }
+    let varyKey = this.getVaryKey(req, cache.vary)
+    let data = cache.data.get(varyKey)
+    if (!data) return null
+    if (!data.done) return null
+
+    return {
+      varyKey: varyKey,
+      res: cache,
+      data: cache.data.get(varyKey)
+    }
+  }
+
+  async proxy(ctx) {
+    let url = ctx.req.url
+    if (!url.startsWith('/_')) {
+      url = this.remap.get(ctx.req.headers['host'])
+      if (!url) {
+        url = this.remap.get(this.defaultRemap)
+      }
+      url += ctx.req.url.slice(1)
+    }
+    let cache = this.getCache(ctx.req, url)
+
+    if (cache) {
+      ctx.status = cache.res.status
+      ctx.type = cache.res.type
+      ctx.body = cache.data.data
+      ctx.headers = this.cloneHeaders(cache.data.headers)
+      console.log('found cache', cache)
+      return
+    }
+
+    ctx.req.headers.host = 'www.sa.is'
+
+    var res = await new Promise((res, rej) => {
+      var req = https.request({
+        hostname: 'www.sa.is',
+        path: url,
+        method: ctx.req.method,
+        headers: ctx.req.headers,
+        agent: this.agent,
+      }, results => {
+        res(results)
+      })
+      req.on('error', rej)
+      req.end()
+    })
+
+    ctx.status = res.statusCode
+    ctx.headers = this.cloneHeaders(res.headers)
+    ctx.type = res.headers['content-type']
+    ctx.body = this.proxyCache(ctx.req, url, ctx.headers['cache-control'], res)
+  }
+}

saproxy/api/server.mjs

@@ -0,0 +1,109 @@
+import { Flaska, QueryHandler, JsonHandler } from 'flaska'
+import config from './config.mjs'
+import StaticRoutes from './static_routes.mjs'
+import Proxy from './proxy.mjs'
+
+export default class Server {
+  constructor(http, port, core, opts = {}) {
+    Object.assign(this, opts)
+    this.http = http
+    this.port = port
+    this.core = core
+
+    this.flaskaOptions = {
+      appendHeaders: {
+        'Content-Security-Policy': `default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src * data: blob:; font-src 'self' data:; object-src 'none'; frame-ancestors 'none'`,
+      },
+      log: this.core.log,
+      nonce: ['script-src'],
+      nonceCacheLength: 50,
+    }
+
+    this.init()
+  }
+
+  init() { }
+
+  runCreateServer() {
+    // Create our server
+    this.flaska = new Flaska(this.flaskaOptions, this.http)
+
+    // configure our server
+    console.log(config.get('NODE_ENV'))
+    if (config.get('NODE_ENV') === 'development') {
+      this.flaska.devMode()
+    }
+
+    this.flaska.onerror(err => {
+      console.log(err)
+      ctx.log.error(err)
+      ctx.status = 500
+    })
+
+    this.flaska.before(function(ctx) {
+      ctx.state.started = new Date().getTime()
+      ctx.req.ip = ctx.req.headers['x-forwarded-for'] || ctx.req.connection.remoteAddress
+      ctx.log = ctx.log.child({
+        id: Math.random().toString(36).substring(2, 14),
+      })
+    })
+    
+    let healthChecks = 0
+    let healthCollectLimit = 60 * 60 * 12
+
+    this.flaska.after(function(ctx) {
+      if (ctx.aborted && ctx.status === 200) {
+        ctx.status = 299
+      }
+      let ended = new Date().getTime()
+      var requestTime = ended - ctx.state.started
+
+      let status = ''
+      let level = 'info'
+      if (ctx.status >= 400) {
+        status = ctx.status + ' '
+        level = 'warn'
+      }
+      if (ctx.status >= 500) {
+        level = 'error'
+      }
+
+      if (ctx.url === '/health' || ctx.url === '/api/health') {
+        healthChecks++
+        if (healthChecks >= healthCollectLimit) {
+          ctx.log[level]({
+            duration: Math.round(ended),
+            status: ctx.status,
+          }, `<-- ${status}${ctx.method} ${ctx.url} {has happened ${healthChecks} times}`)
+          healthChecks = 0
+        }
+        return
+      }
+
+      ctx.log[level]({
+        duration: requestTime,
+        status: ctx.status,
+        ip: ctx.req.ip,
+      }, (ctx.aborted ? '[ABORT]' : '<--') + ` ${status}${ctx.method} ${ctx.url}`)
+    })
+  }
+
+  registerRoutes() {
+    var stat = new StaticRoutes()
+    var proxy = new Proxy()
+    stat.register(this)
+    proxy.register(this)
+  }
+
+  runStartListen() {
+    return this.flaska.listenAsync(this.port).then(() => {
+      this.core.log.info('Server is listening on port ' + this.port)
+    })
+  }
+
+  run() {
+    this.runCreateServer()
+    this.registerRoutes()
+    return this.runStartListen()
+  }
+}

saproxy/api/static_routes.mjs

@@ -0,0 +1,17 @@
+import config from './config.mjs'
+
+export default class StaticRoutes {
+  constructor(opts = {}) {
+    Object.assign(this, { })
+  }
+
+  register(server) {
+    server.flaska.get('/api/health', this.health.bind(this))
+  }
+
+  health(ctx) {
+    ctx.body = {
+      environment: config.get('NODE_ENV'),
+    }
+  }
+}

saproxy/build-package.json

@@ -0,0 +1,8 @@
+{
+  "scripts": {
+    "build": "echo done;"
+  },
+  "dependencies": {
+    "service-core": "^3.0.0-beta.17"
+  }
+}

saproxy/dev.mjs

@@ -0,0 +1,24 @@
+import fs from 'fs'
+import { ServiceCore } from 'service-core'
+import * as index from './index.mjs'
+
+const port = 4120
+
+var core = new ServiceCore('nfp_moe', import.meta.url, port, '')
+
+let config = {
+  frontend: {
+    url: 'http://localhost:' + port
+  }
+}
+
+try {
+  config = JSON.parse(fs.readFileSync('./config.json'))
+} catch {}
+
+config.port = port
+
+core.setConfig(config)
+core.init(index).then(function() {
+  return core.run()
+})

saproxy/index.mjs

@@ -0,0 +1,11 @@
+import config from './api/config.mjs'
+
+export function start(http, port, ctx) {
+  config.sources[1].store = ctx.config
+
+  return import('./api/server.mjs')
+  .then(function(module) {
+    let server = new module.default(http, port, ctx)
+    return server.run()
+  })
+}

saproxy/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "saproxy",
+  "version": "1.0.0",
+  "port": 4130,
+  "description": "Proxy guy for SA",
+  "main": "index.js",
+  "directories": {
+    "test": "test"
+  },
+  "scripts": {
+    "start": "node --experimental-modules index.mjs",
+    "dev:server": "node dev.mjs | bunyan",
+    "dev": "npm-watch dev:server"
+  },
+  "watch": {
+    "dev:server": {
+      "patterns": [
+        "api/*",
+        "base/*",
+        "../base/*"
+      ],
+      "extensions": "js,mjs",
+      "quiet": true,
+      "inherit": true
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://git.nfp.is/nfp/nfp_sites.git"
+  },
+  "author": "Jonatan Nilsson",
+  "license": "WTFPL",
+  "bugs": {
+    "url": "https://git.nfp.is/nfp/nfp_sites/issues"
+  },
+  "homepage": "https://git.nfp.is/nfp/nfp_sites",
+  "dependencies": {
+    "bunyan-lite": "^1.2.1",
+    "flaska": "^1.3.3",
+    "multipipe": "^4.0.0",
+    "nconf-lite": "^2.0.0"
+  },
+  "devDependencies": {
+    "service-core": "^3.0.0-beta.17"
+  }
+}