General development

api/config.mjs

@@ -55,6 +55,7 @@ nconf.defaults({
     "url": "http://beta01.nfp.moe"
   },
   "mssql": {
+    "conn_timeout": 5,
     "floor": 1,
     "ceiling": 2,
     "heartbeatSecs": 20,

api/db.mjs

@@ -0,0 +1,54 @@
+import MSSQL from 'msnodesqlv8'
+import { HttpError } from 'flaska'
+
+export function initPool(core, config) {
+  let pool = new MSSQL.Pool(config)
+
+  core.log.info(config, 'MSSQL database setttings')
+
+  pool.on('open', function() { 
+    core.log.info('MSSQL connection open')
+  })
+
+  let waiting = false
+
+  /*pool.on('error', function(error) {
+    if (error.length) {
+      let msg = 'Error in MSSQL pool\n => ' + error[0].message.trim()
+      for (let i = 1; i < error.length; i++) {
+        msg += '\n => ' + error[i].message.trim()
+      }
+      core.log.error(msg)
+    } else {
+      core.log.error('Error in MSSQL pool')
+      core.log.error(error)
+    }
+
+    if (waiting) { return }
+    core.log.warn('Attempting to connect again in 5 seconds')
+    waiting = true
+    setTimeout(function() {
+      waiting = false
+      console.log('opening')
+      pool.open()
+      console.log('done')
+    }, 5000)
+  })*/
+
+  core.log.info('Attempting to connect to MSSQL server')
+  pool.open()
+
+  return {
+    safeCallProc: function(name, params, options) {
+      return pool.promises.callProc(name, params, options)
+        .catch(function(err) {
+          let message = err.message
+          if (err.lineNumber && err.procName) {
+            message = `Error at ${err.procName}:${err.lineNumber} => ${message}`
+          }
+          throw new HttpError(500, message)
+        })
+    },
+    promises: pool.promises,
+  }
+}

api/file/torrent.mjs

@@ -0,0 +1,105 @@
+import bencode from 'bencode'
+
+/*
+Taken from parse-torrent
+*/
+
+/**
+ * Parse a torrent. Throws an exception if the torrent is missing required fields.
+ * @param  {Buffer|Object} torrent
+ * @return {Object}        parsed torrent
+ */
+export function decodeTorrentFile (torrent) {
+  if (Buffer.isBuffer(torrent)) {
+    torrent = bencode.decode(torrent)
+  }
+
+  // sanity check
+  ensure(torrent.info, 'info')
+  ensure(torrent.info['name.utf-8'] || torrent.info.name, 'info.name')
+  ensure(torrent.info['piece length'], 'info[\'piece length\']')
+  ensure(torrent.info.pieces, 'info.pieces')
+
+  if (torrent.info.files) {
+    torrent.info.files.forEach(file => {
+      ensure(typeof file.length === 'number', 'info.files[0].length')
+      ensure(file['path.utf-8'] || file.path, 'info.files[0].path')
+    })
+  } else {
+    ensure(typeof torrent.info.length === 'number', 'info.length')
+  }
+
+  const result = {
+    info: torrent.info,
+    infoBuffer: bencode.encode(torrent.info),
+    name: (torrent.info['name.utf-8'] || torrent.info.name).toString(),
+    announce: []
+  }
+
+  result.infoHash = sha1.sync(result.infoBuffer)
+  result.infoHashBuffer = Buffer.from(result.infoHash, 'hex')
+
+  if (torrent.info.private !== undefined) result.private = !!torrent.info.private
+
+  if (torrent['creation date']) result.created = new Date(torrent['creation date'] * 1000)
+  if (torrent['created by']) result.createdBy = torrent['created by'].toString()
+
+  if (Buffer.isBuffer(torrent.comment)) result.comment = torrent.comment.toString()
+
+  // announce and announce-list will be missing if metadata fetched via ut_metadata
+  if (Array.isArray(torrent['announce-list']) && torrent['announce-list'].length > 0) {
+    torrent['announce-list'].forEach(urls => {
+      urls.forEach(url => {
+        result.announce.push(url.toString())
+      })
+    })
+  } else if (torrent.announce) {
+    result.announce.push(torrent.announce.toString())
+  }
+
+  // handle url-list (BEP19 / web seeding)
+  if (Buffer.isBuffer(torrent['url-list'])) {
+    // some clients set url-list to empty string
+    torrent['url-list'] = torrent['url-list'].length > 0
+      ? [torrent['url-list']]
+      : []
+  }
+  result.urlList = (torrent['url-list'] || []).map(url => url.toString())
+
+  // remove duplicates by converting to Set and back
+  result.announce = Array.from(new Set(result.announce))
+  result.urlList = Array.from(new Set(result.urlList))
+
+  const files = torrent.info.files || [torrent.info]
+  result.files = files.map((file, i) => {
+    const parts = [].concat(result.name, file['path.utf-8'] || file.path || []).map(p => p.toString())
+    return {
+      path: path.join.apply(null, [path.sep].concat(parts)).slice(1),
+      name: parts[parts.length - 1],
+      length: file.length,
+      offset: files.slice(0, i).reduce(sumLength, 0)
+    }
+  })
+
+  result.length = files.reduce(sumLength, 0)
+
+  const lastFile = result.files[result.files.length - 1]
+
+  result.pieceLength = torrent.info['piece length']
+  result.lastPieceLength = ((lastFile.offset + lastFile.length) % result.pieceLength) || result.pieceLength
+  result.pieces = splitPieces(torrent.info.pieces)
+
+  return result
+}
+
+function splitPieces (buf) {
+  const pieces = []
+  for (let i = 0; i < buf.length; i += 20) {
+    pieces.push(buf.slice(i, i + 20).toString('hex'))
+  }
+  return pieces
+}
+
+function ensure (bool, fieldName) {
+  if (!bool) throw new Error(`Torrent is missing required field: ${fieldName}`)
+}

api/page/model.mjs

@@ -20,7 +20,26 @@ Page model:
 */
 
 export async function getTree(ctx) {
-  let res = await ctx.db.promises.callProc('pages_gettree', [])
-  console.log(res)
-  return {}
+  let res = await ctx.db.safeCallProc('pages_gettree', [])
+  let out = []
+  let children = []
+  let map = new Map()
+  for (let page of res.first) {
+    if (!page.parent_id) {
+      out.push(page)
+    } else {
+      children.push(page)
+    }
+    map.set(page.id, page)
+  }
+  for (let page of children) {
+    let parent = map.get(page.parent_id)
+    if (!parent.children) {
+      parent.children = []
+    }
+    parent.children.push(page)
+  }
+  return {
+    tree: out
+  }
 }

api/serve.mjs

@@ -1,13 +1,25 @@
 import path from 'path'
+import dot from 'dot'
 import { FileResponse, HttpError } from 'flaska'
 import fs from 'fs/promises'
+import fsSync from 'fs'
+
+import { getTree } from './page/model.mjs'
 
 export default class ServeHandler {
   constructor(opts = {}) {
     Object.assign(this, {
       fs: opts.fs || fs,
+      fsSync: opts.fsSync || fsSync,
       root: opts.root,
+      template: null,
+      frontend: opts.frontend || 'http://localhost:4000',
+      version: opts.version || 'version',
     })
+
+    let indexFile = fsSync.readFileSync(path.join(this.root, 'index.html'))
+    this.template = dot.template(indexFile.toString(), { argName: ['headerDescription', 'headerImage', 'headerTitle', 'headerUrl', 'payloadData', 'payloadLinks', 'payloadTree', 'version', 'nonce'] })
+    // console.log(indexFile.toString())
   }
 
   /** GET: /::file */
@@ -18,21 +30,52 @@ export default class ServeHandler {
 
     let file = path.resolve(path.join(this.root, ctx.params.file ? ctx.params.file : 'index.html'))
 
+    if (!ctx.params.file || ctx.params.file === 'index.html') {
+      return this.serveIndex(ctx)
+    }
+
     if (!file.startsWith(this.root)) {
       ctx.status = 404
       ctx.body = 'HTTP 404 Error'
       return
     }
 
-    return this.fs.stat(file).catch((err) => {
-      if (err.code === 'ENOENT') {
-        file = path.resolve(path.join(this.root, 'index.html'))
-        return this.fs.stat(file)
-      }
-      return Promise.reject(err)
-    })
+    return this.fs.stat(file)
     .then(function(stat) {
       ctx.body = new FileResponse(file, stat)
     })
+    .catch((err) => {
+      if (err.code === 'ENOENT') {
+        return this.serveIndex(ctx)
+      }
+      return Promise.reject(err)
+    })
+  }
+ 
+  async serveIndex(ctx) {
+    let payload = {
+      headerDescription: 'Small fansubbing and scanlation group translating and encoding our favourite shows from Japan.',
+      headerImage: this.frontend + '/assets/img/heart.png',
+      headerTitle: 'NFP Moe - Anime/Manga translation group',
+      headerUrl: this.frontend + ctx.url,
+      payloadData: null,
+      payloadLinks: null,
+      payloadTree: null,
+      version: this.version,
+      nonce: ctx.state.nonce,
+    }
+
+    try {
+      payload.payloadTree = JSON.stringify(await getTree(ctx))
+    } catch (e) {
+      ctx.log.error(e)
+    }
+
+    ctx.body = this.template(payload)
+    ctx.type = 'text/html; charset=utf-8'
+  }
+
+  serveErrorPage(ctx) { 
+
   }
 }

api/server.mjs

@@ -1,6 +1,6 @@
 import { Flaska, QueryHandler } from 'flaska'
-import MSSQL from 'msnodesqlv8'
 
+import { initPool } from './db.mjs'
 import config from './config.mjs'
 import PageRoutes from './page/routes.mjs'
 import ServeHandler from './serve.mjs'
@@ -15,30 +15,10 @@ export function run(http, port, core) {
     log: core.log,
     nonce: ['script-src'],
     nonceCacheLength: 50,
-    defaultHeaders: {
-      'Server': 'Flaska',
-      'X-Content-Type-Options': 'nosniff',
-      'Content-Security-Policy': `default-src 'self'; style-src 'self' 'unsafe-inline'; img-src * data: blob:; object-src 'none'; frame-ancestors 'none'; script-src 'self'`,
-      'Cross-Origin-Opener-Policy': 'same-origin',
-      'Cross-Origin-Resource-Policy': 'same-origin',
-      'Cross-Origin-Embedder-Policy': 'require-corp',
-    },
   }, http)
    
   // Create our database pool
-  const pool = new MSSQL.Pool(config.get('mssql'))
-
-  core.log.info(config.get('mssql'), 'MSSQL database setttings')
-
-  pool.on('open', function() {
-    core.log.info('MSSQL connection open')
-  })
-
-  pool.on('error', function(error) {
-    core.log.error(error, 'Error in MSSQL pool')
-  })
-
-  pool.open()
+  let pool = initPool(core, config.get('mssql'))
 
   // configure our server
   if (config.get('NODE_ENV') === 'development') {
@@ -87,6 +67,8 @@ export function run(http, port, core) {
 
   const serve = new ServeHandler({
     root: localUtil.getPathFromRoot('../public'),
+    version: core.app.running,
+    frontend: config.get('frontend:url'),
   })
   flaska.get('/::file', serve.serve.bind(serve))
 

app/api/page.js

@@ -1,6 +1,6 @@
 const common = require('./common')
 
-const Tree = window.__nfptree || []
+const Tree = window.__nfptree?.tree || []
 
 exports.Tree = Tree
 

package.json

@@ -9,12 +9,11 @@
   "scripts": {
     "start": "node --experimental-modules index.mjs",
     "test": "echo \"Error: no test specified\" && exit 1",
-    "bla": "sass --help",
     "build:prod": "sass -s compressed app/app.scss public/assets/app.css && sass -s compressed app/admin.scss public/assets/admin.css && asbundle app/index.js public/assets/app.js && asbundle app/admin.js public/assets/admin.js",
     "build": "sass app/app.scss public/assets/app.css && sass app/admin.scss public/assets/admin.css && asbundle app/index.js public/assets/app.js && asbundle app/admin.js public/assets/admin.js",
     "build:watch": "npm-watch build",
-    "dev:server": "node dev.mjs | bunyan",
-    "dev:server:watch": "npm-watch dev:server",
+    "dev:server": "npm run build && node dev.mjs | bunyan",
+    "dev": "npm-watch dev:server",
     "watch:sass:public": "sass --watch app/app.scss public/assets/app.css",
     "watch:sass:admin": "sass --watch app/admin.scss public/assets/admin.css",
     "prod": "npm run build && npm start"
@@ -22,14 +21,7 @@
   "watch": {
     "dev:server": {
       "patterns": [
-        "api/*"
-      ],
-      "extensions": "js,mjs",
-      "quiet": true,
-      "inherit": true
-    },
-    "build": {
-      "patterns": [
+        "api/*",
         "app/*"
       ],
       "extensions": "js,mjs",
@@ -48,23 +40,18 @@
   },
   "homepage": "https://github.com/nfp-projects/nfp_moe",
   "dependencies": {
-    "bookshelf": "^0.15.1",
-    "dot": "^1.1.2",
-    "flaska": "^1.2.2",
+    "bencode": "^2.0.3",
+    "dot": "^2.0.0-beta.1",
+    "flaska": "^1.2.5",
     "format-link-header": "^2.1.0",
-    "http-errors": "^1.7.2",
-    "json-mask": "^0.3.8",
-    "knex-core": "^0.19.5",
     "msnodesqlv8": "^2.4.7",
     "nconf-lite": "^1.0.1",
-    "parse-torrent": "^7.0.1",
-    "pg": "^8.7.3",
     "striptags": "^3.1.1"
   },
   "devDependencies": {
     "asbundle": "^2.6.1",
-    "mithril": "^2.0.4",
-    "sass": "^1.17.0",
-    "service-core": "^3.0.0-beta.13"
+    "mithril": "^2.2.2",
+    "sass": "^1.52.3",
+    "service-core": "^3.0.0-beta.17"
   }
 }

public/assets/inter.css

@@ -0,0 +1,200 @@
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 100;
+  font-display: swap;
+  src: url("Inter-Thin.woff2?v=3.19") format("woff2"),
+       url("Inter-Thin.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 100;
+  font-display: swap;
+  src: url("Inter-ThinItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-ThinItalic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 200;
+  font-display: swap;
+  src: url("Inter-ExtraLight.woff2?v=3.19") format("woff2"),
+       url("Inter-ExtraLight.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 200;
+  font-display: swap;
+  src: url("Inter-ExtraLightItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-ExtraLightItalic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 300;
+  font-display: swap;
+  src: url("Inter-Light.woff2?v=3.19") format("woff2"),
+       url("Inter-Light.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 300;
+  font-display: swap;
+  src: url("Inter-LightItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-LightItalic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 400;
+  font-display: swap;
+  src: url("Inter-Regular.woff2?v=3.19") format("woff2"),
+       url("Inter-Regular.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 400;
+  font-display: swap;
+  src: url("Inter-Italic.woff2?v=3.19") format("woff2"),
+       url("Inter-Italic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 500;
+  font-display: swap;
+  src: url("Inter-Medium.woff2?v=3.19") format("woff2"),
+       url("Inter-Medium.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 500;
+  font-display: swap;
+  src: url("Inter-MediumItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-MediumItalic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 600;
+  font-display: swap;
+  src: url("Inter-SemiBold.woff2?v=3.19") format("woff2"),
+       url("Inter-SemiBold.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 600;
+  font-display: swap;
+  src: url("Inter-SemiBoldItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-SemiBoldItalic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 700;
+  font-display: swap;
+  src: url("Inter-Bold.woff2?v=3.19") format("woff2"),
+       url("Inter-Bold.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 700;
+  font-display: swap;
+  src: url("Inter-BoldItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-BoldItalic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 800;
+  font-display: swap;
+  src: url("Inter-ExtraBold.woff2?v=3.19") format("woff2"),
+       url("Inter-ExtraBold.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 800;
+  font-display: swap;
+  src: url("Inter-ExtraBoldItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-ExtraBoldItalic.woff?v=3.19") format("woff");
+}
+
+@font-face {
+  font-family: 'Inter';
+  font-style:  normal;
+  font-weight: 900;
+  font-display: swap;
+  src: url("Inter-Black.woff2?v=3.19") format("woff2"),
+       url("Inter-Black.woff?v=3.19") format("woff");
+}
+@font-face {
+  font-family: 'Inter';
+  font-style:  italic;
+  font-weight: 900;
+  font-display: swap;
+  src: url("Inter-BlackItalic.woff2?v=3.19") format("woff2"),
+       url("Inter-BlackItalic.woff?v=3.19") format("woff");
+}
+
+/* -------------------------------------------------------
+Variable font.
+Usage:
+
+  html { font-family: 'Inter', sans-serif; }
+  @supports (font-variation-settings: normal) {
+    html { font-family: 'Inter var', sans-serif; }
+  }
+*/
+@font-face {
+  font-family: 'Inter var';
+  font-weight: 100 900;
+  font-display: swap;
+  font-style: normal;
+  font-named-instance: 'Regular';
+  src: url("Inter-roman.var.woff2?v=3.19") format("woff2");
+}
+@font-face {
+  font-family: 'Inter var';
+  font-weight: 100 900;
+  font-display: swap;
+  font-style: italic;
+  font-named-instance: 'Italic';
+  src: url("Inter-italic.var.woff2?v=3.19") format("woff2");
+}
+
+
+/* --------------------------------------------------------------------------
+[EXPERIMENTAL] Multi-axis, single variable font.
+
+Slant axis is not yet widely supported (as of February 2019) and thus this
+multi-axis single variable font is opt-in rather than the default.
+
+When using this, you will probably need to set font-variation-settings
+explicitly, e.g.
+
+  * { font-variation-settings: "slnt" 0deg }
+  .italic { font-variation-settings: "slnt" 10deg }
+
+*/
+@font-face {
+  font-family: 'Inter var experimental';
+  font-weight: 100 900;
+  font-display: swap;
+  font-style: oblique 0deg 10deg;
+  src: url("Inter.var.woff2?v=3.19") format("woff2");
+}

public/index.html

@@ -2,41 +2,37 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <title>{{=it.title}}</title>
+  <title>{{=headerTitle}}</title>
   <base href="/">
-  <meta name="description" content="{{=it.description}}">
+  <meta name="description" content="{{=headerDescription}}">
   <meta name="twitter:card" value="summary">
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <meta property="og:type" content="website" />
-  <meta property="og:url" content="{{=it.url}}" />
-  <meta property="og:image" content="{{=it.image}}" />
-  <meta property="og:description" content="{{=it.description}}" />
-  <meta property="og:title" content="{{=it.title}}" />
-  {{? it.image === '/assets/img/heart.jpg' }}
+  <meta property="og:url" content="{{=headerUrl}}" />
+  <meta property="og:image" content="{{=headerImage}}" />
+  <meta property="og:description" content="{{=headerDescription}}" />
+  <meta property="og:title" content="{{=headerTitle}}" />
+  {{? headerImage === '/assets/img/heart.jpg' }}
     <meta id="ogimagewidth" property="og:image:width" content="400" />
     <meta id="ogimageheight" property="og:image:height" content="500" />
-  {{?? true }}
   {{? }}
 
   <link rel="icon" type="image/png" href="/assets/img/favicon.png">
-  <link rel="Stylesheet" href="/assets/app.css?v={{=it.v}}" type="text/css" />
-  <link rel="preconnect" href="https://cdn-nfp.global.ssl.fastly.net" />
-  <meta name="google-signin-client_id" content="1076074914074-3no1difo1jq3dfug3glfb25pn1t8idud.apps.googleusercontent.com">
+  <link rel="Stylesheet" href="/assets/app.css?v={{=version}}" type="text/css" />
+  <link rel="preconnect" href="https://cdn.nfp.is" />
 </head>
 <body class="daymode">
-  <script type="text/javascript">
+  <script type="text/javascript" nonce="{{=nonce}}">
     if (localStorage.getItem('darkmode')) {document.body.className = 'darkmodeon';}
-    window.__nfptree = {{=it.tree}};
-    window.__nfpfeatured = {{=it.featured}};
-    window.__nfpdata = {{=it.data}};
-    window.__nfpsubdata = {{=it.subdata}};
-    window.__nfplinks = {{=it.links}};
+    window.__nfptree = {{=payloadTree}};
+    window.__nfpdata = {{=payloadData}};
+    window.__nfplinks = {{=payloadLinks}};
   </script>
   <div class="maincontainer">
     <div id="nav"></div>
     <main id="main"></main>
     <footer id="footer"></footer>
   </div>
-  <script type="text/javascript" src="/assets/app.js?v={{=it.v}}"></script>
+  <script type="text/javascript" src="/assets/app.js?v={{=version}}" nonce="{{=nonce}}"></script>
 </body>
 </html>

temp.sql

@@ -0,0 +1,24 @@
+use nfp_sites;
+go
+
+-- Object_definition(object_id)
+
+Declare @sql varchar(max) ;
+SELECT @sql=Object_definition(object_id)
+FROM   sys.procedures
+WHERE  name = 'spe_get_my_data' and SCHEMA_NAME(schema_id) = 'dbo';
+SET @sql=REPLACE(@sql, 'CREATE PROCEDURE [dbo].', 'ALTER PROCEDURE [nfp_moe].');
+print @sql
+exec (@sql)
+print '-----------------------------';
+
+
+SELECT *
+FROM   sys.procedures
+WHERE  name like 'spe_%' and SCHEMA_NAME(schema_id) = 'dbo';
+
+
+SELECT *
+FROM   sys.procedures where SCHEMA_NAME(schema_id) = 'dbo';
+
+EXEC sp_helptext 'dbo.spe_get_schema';