From cbd1939a74de5a6ef44385f539ebea04c67a9d29 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 1 Aug 2019 19:43:20 +0200 Subject: [PATCH 01/11] Stop pretending there's documentation Remove docs scripts, documentation.yml and the documentation dependency. --- documentation.yml | 10 ---------- package.json | 3 --- 2 files changed, 13 deletions(-) delete mode 100644 documentation.yml diff --git a/documentation.yml b/documentation.yml deleted file mode 100644 index f5ff75a1..00000000 --- a/documentation.yml +++ /dev/null @@ -1,10 +0,0 @@ -toc: - - name: Master Server API - - clusterioMaster - - name: Type declarations - - itemStack - - name: Internal modules - description: | - Here are some helper functions for navigation. - - modManager - - getFactorioLocale \ No newline at end of file diff --git a/package.json b/package.json index 2f817d0f..7ba7bd53 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,6 @@ "test": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{,!(node_modules)/**/}*.spec.js\" -R spec || echo .", "start": "echo Run node client.js start [instance] to start slaves && node master.js", "cover": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{,!(node_modules)/**/}*.spec.js\" -R spec || echo . && codecov -t f7b38085-36eb-4d72-84e6-64721910171e", - "docs": "documentation build index.js --config documentation.yml -f html -o docs", - "livedocs": "documentation serve --watch *.js lib/** --config documentation.yml", "install": "node lib/npmPostinstall.js", "bundle-dependencies": "bundle-dependencies", "snyk-protect": "snyk protect", @@ -75,7 +73,6 @@ "underscore": "^1.8.3" }, "devDependencies": { - "documentation": "^5.4.0", "istanbul": "^0.4.5", "mocha": "^3.5.3", "mocha-sinon": "^2.0.0", From 6264bd988881f2078151290f12c88836c2d40106 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 1 Aug 2019 19:58:49 +0200 Subject: [PATCH 02/11] Update node-factorio-api The 0.3.5 release of node-factorio-api removed the dependency to documentation, which itself pulls in another 500 dependencies. Update to 0.3.5 to avoid having this pulled in. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7ba7bd53..093dc0ae 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "moment": "^2.22.2", "ncp": "^2.0.0", "needle": "^2.2.4", - "node-factorio-api": "^0.3.1-alpha", + "node-factorio-api": "^0.3.5", "node-forge": "^0.7.1", "prom-client": "^10.2.3", "rcon-client-fork": "^3.2.0", From 0e849488e256dd1c60cb283e934d03ad90d6181d Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 1 Aug 2019 20:25:41 +0200 Subject: [PATCH 03/11] Remove snyk It's not supposed to be installed as a dependency in the first place, and this seems like the wrong solution to there being too many stinking dependencies to keep track the security of. --- .snyk | 45 --------------------------------------------- package.json | 8 ++------ 2 files changed, 2 insertions(+), 51 deletions(-) delete mode 100644 .snyk diff --git a/.snyk b/.snyk deleted file mode 100644 index d7f0b403..00000000 --- a/.snyk +++ /dev/null @@ -1,45 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.13.3 -ignore: {} -# patches apply the minimum changes required to fix a vulnerability -patch: - 'npm:lodash:20180130': - - html5-validator > caporal > cli-table2 > lodash: - patched: '2018-07-03T06:07:55.532Z' - 'npm:hawk:20160119': - - node-factorio-api > documentation > highlight.js > gear-lib > less > request > hawk: - patched: '2019-02-25T06:48:21.845Z' - 'npm:http-signature:20150122': - - node-factorio-api > documentation > highlight.js > gear-lib > less > request > http-signature: - patched: '2019-02-25T06:48:21.845Z' - 'npm:mime:20170907': - - node-factorio-api > documentation > highlight.js > gear-lib > mime: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > less > mime: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > less > request > form-data > mime: - patched: '2019-02-25T06:48:21.845Z' - 'npm:minimatch:20160620': - - node-factorio-api > documentation > highlight.js > gear > liftoff > findup-sync > glob > minimatch: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > gear > liftoff > findup-sync > glob > minimatch: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > glob > minimatch: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > jslint > glob > minimatch: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > jshint > cli > glob > minimatch: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > jshint > minimatch: - patched: '2019-02-25T06:48:21.845Z' - 'npm:request:20160119': - - node-factorio-api > documentation > highlight.js > gear-lib > less > request: - patched: '2019-02-25T06:48:21.845Z' - 'npm:tunnel-agent:20170305': - - node-factorio-api > documentation > highlight.js > gear-lib > less > request > tunnel-agent: - patched: '2019-02-25T06:48:21.845Z' - 'npm:uglify-js:20151024': - - node-factorio-api > documentation > highlight.js > gear-lib > handlebars > uglify-js: - patched: '2019-02-25T06:48:21.845Z' - - node-factorio-api > documentation > highlight.js > gear-lib > uglify-js: - patched: '2019-02-25T06:48:21.845Z' diff --git a/package.json b/package.json index 093dc0ae..fdf26872 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,7 @@ "start": "echo Run node client.js start [instance] to start slaves && node master.js", "cover": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{,!(node_modules)/**/}*.spec.js\" -R spec || echo . && codecov -t f7b38085-36eb-4d72-84e6-64721910171e", "install": "node lib/npmPostinstall.js", - "bundle-dependencies": "bundle-dependencies", - "snyk-protect": "snyk protect", - "prepublish": "npm run snyk-protect" + "bundle-dependencies": "bundle-dependencies" }, "keywords": [ "factorio" @@ -63,7 +61,6 @@ "sanitizer": "^0.1.3", "simple-git": "^1.96.0", "single-line-log": "^1.1.2", - "snyk": "^1.134.2", "socket.io": "^2.0.4", "socket.io-client": "^2.0.4", "socket.io-prometheus": "^0.2.1", @@ -80,6 +77,5 @@ "nock": "^9.1.6", "sinon": "^2.4.1", "supertest": "^3.0.0" - }, - "snyk": true + } } From 13a069a798c3f25748aa5abea90cf23dd1e78df8 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 2 Aug 2019 14:30:18 +0200 Subject: [PATCH 04/11] Reimplement hashFile locally Replace the 10 MB of dependencies that is hash-files with a homebrew Promise based implementation of hashFile that uses crypto and fs. This somehow also ends up being the same amount of code as the one using hash-files. --- client.js | 54 +++++++++++++++++----------------------------------- lib/hash.js | 22 +++++++++++++++++++++ package.json | 1 - 3 files changed, 39 insertions(+), 38 deletions(-) create mode 100644 lib/hash.js diff --git a/client.js b/client.js index adcc8cab..67b8bd04 100644 --- a/client.js +++ b/client.js @@ -8,7 +8,6 @@ const syncRequest = require('sync-request'); const request = require("request"); const ncp = require('ncp').ncp; const Rcon = require('rcon-client-fork').Rcon; -const hashFiles = require('hash-files'); const _ = require('underscore'); const deepmerge = require("deepmerge"); const getMac = require('getmac').getMac; @@ -20,6 +19,7 @@ const objectOps = require("lib/objectOps.js"); const fileOps = require("lib/fileOps"); const stringUtils = require("lib/stringUtils.js"); const configManager = require("lib/manager/configManager.js"); +const hashFile = require('lib/hash').hashFile; // argument parsing const args = require('minimist')(process.argv.slice(2)); @@ -773,42 +773,22 @@ function hashMods(instanceName, callback) { if(!callback) { throw new Error("ERROR in function hashMods NO CALLBACK"); } - function callback2(hash, modName){ - hashedMods[hashedMods.length] = { - modName: modName, - hash: hash, - }; - // check if this callback has ran once for each mod - if(hashedMods.length == instanceMods.length) { - callback(hashedMods); - } - //console.log(modname); - } - let hashedMods = []; - /*let mods = fs.readdirSync("./sharedMods/")*/ - let instanceMods = fs.readdirSync(config.instanceDirectory+"/"+instanceName+"/mods/"); - if(instanceMods.length == 0){ - callback({}); // there are no mods installed, return an empty object - } - for(let o=0;o ( + {modName: name, hash: hash} + )); } } - for(let i=0; i { + // Remove null entries from hashMod + callback(hashes.filter(entry => entry !== null)); + }); } diff --git a/lib/hash.js b/lib/hash.js new file mode 100644 index 00000000..f0bdfd7a --- /dev/null +++ b/lib/hash.js @@ -0,0 +1,22 @@ +const crypto = require('crypto'); +const fs = require('fs'); + +/** + * Returns a promise that resolves to the SHA1 hash of the file given by path + */ +exports.hashFile = function(path) { + return new Promise(function(resolve, reject) { + let hasher = crypto.createHash('sha1') + hasher.setEncoding('hex') + hasher.on('finish', function() { + resolve(hasher.read()); + }); + + let stream = fs.createReadStream(path); + stream.on('error', function(error) { + // the docs doesn't say anything about what's passed on error + reject(error); + }); + stream.pipe(hasher); + }); +} diff --git a/package.json b/package.json index fdf26872..a906cbd7 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "follow-redirects": "^1.5.0", "fs-extra": "^6.0.1", "getmac": "^1.4.0", - "hash-files": "^1.1.1", "html5-validator": "^1.0.7", "ini": "^1.3.5", "jsonwebtoken": "^8.2.1", From d0afcbf0ac05adead5f5bbb648e71756762a84af Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 2 Aug 2019 14:34:32 +0200 Subject: [PATCH 05/11] Move html5-validator to devDependencies html5-validator is only used for testing the master server web interface and does not need to be included on production installs. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a906cbd7..20eddedd 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "follow-redirects": "^1.5.0", "fs-extra": "^6.0.1", "getmac": "^1.4.0", - "html5-validator": "^1.0.7", "ini": "^1.3.5", "jsonwebtoken": "^8.2.1", "lib": "file:./lib", @@ -69,6 +68,7 @@ "underscore": "^1.8.3" }, "devDependencies": { + "html5-validator": "^1.0.7", "istanbul": "^0.4.5", "mocha": "^3.5.3", "mocha-sinon": "^2.0.0", From 200c8dbdeb0c433a2810eae2f1b97dff61441c5f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 2 Aug 2019 14:37:20 +0200 Subject: [PATCH 06/11] Use --only=production on npm install Add --only=production flag to Dockerfile and the README entries describing how to install nodes. This will avoid pulling in a lot of dependencies that are not needed for running master/slaves. --- Dockerfile | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 399c1259..b754a329 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:9.11.2 RUN apt-get update && apt install git curl tar -y RUN mkdir factorioClusterio -RUN git clone -b master https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install +RUN git clone -b master https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install --only=production RUN cd factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz WORKDIR factorioClusterio diff --git a/README.md b/README.md index 99a9995e..68efdcbe 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Master and all slaves: cd factorioClusterio curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 tar -xf factorio.tar.gz - npm install + npm install --only=production cp config.json.dist config.json node ./lib/npmPostinstall.js @@ -283,7 +283,7 @@ node client.js list ``` git pull -npm install +npm install --only=production ``` 3. Download the latest version of the factorioClusterioMod from its github repo From ca0281ae5862c311da7357b999c2c42584473805 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 3 Aug 2019 12:45:06 +0200 Subject: [PATCH 07/11] Remove pointless micro-optimization This saved 30ms of the startup on my machine. Even if it saved a second it's still pretty pointless as we're not depending on the master server startup being super fast. --- .gitignore | 2 -- master.js | 4 ---- package.json | 1 - 3 files changed, 7 deletions(-) diff --git a/.gitignore b/.gitignore index 043bbd5c..50b1c745 100644 --- a/.gitignore +++ b/.gitignore @@ -62,7 +62,5 @@ sharedPlugins/scenario_loader/scenarios # File used for marking plugins as disabled sharedPlugins/*/DISABLED -\.cache-require-paths\.json - # IDE .idea/ diff --git a/master.js b/master.js index 02f3e36f..db1ef173 100644 --- a/master.js +++ b/master.js @@ -26,10 +26,6 @@ process.on('unhandledRejection', r => console.log(r)); // configgy stuff debug = false; -// reduces loadtime from 1700 ms to 1200 ms on my i3 + SSD -// tested by logging Date.now() here and when the webserver started listening -require('cache-require-paths'); - // argument parsing const args = require('minimist')(process.argv.slice(2)); diff --git a/package.json b/package.json index 20eddedd..baa0a308 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "bcrypt-promise": "^2.0.0", "bcryptjs": "^2.4.3", "body-parser": "^1.18.2", - "cache-require-paths": "^0.3.0", "cli": "^1.0.1", "cli-interact": "^0.1.9", "codecov": "^3.0.1", From 66c0238a3d29d5ef4c047e33e11ab3d4847f1eb5 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 3 Aug 2019 19:01:26 +0200 Subject: [PATCH 08/11] Remove undescore A 50 kb library is not needed to make a function run once, especially not when it doesn't work. _.once does not modify the function passed to it to only run once it returns a new function that only runs once. Since the result of once was discarded, it had no effect. Remove underscore and implement instanceManagement running once properly with 3 lines of code. --- client.js | 6 ++++-- package.json | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 67b8bd04..6434ba22 100644 --- a/client.js +++ b/client.js @@ -8,7 +8,6 @@ const syncRequest = require('sync-request'); const request = require("request"); const ncp = require('ncp').ncp; const Rcon = require('rcon-client-fork').Rcon; -const _ = require('underscore'); const deepmerge = require("deepmerge"); const getMac = require('getmac').getMac; const rmdirSync = require('rmdir-sync'); @@ -563,8 +562,11 @@ write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n } // ensure instancemanagement only ever runs once -_.once(instanceManagement); +var _instanceInitialized; async function instanceManagement(instanceconfig) { + if (_instanceInitialized) return; + _instanceInitialized = true; + console.log("Started instanceManagement();"); /* Open websocket connection to master */ diff --git a/package.json b/package.json index baa0a308..da2dd9ae 100644 --- a/package.json +++ b/package.json @@ -63,8 +63,7 @@ "socket.io-prometheus": "^0.2.1", "sync-request": "^4.0.1", "tail": "^2.0.2", - "typescript-collections": "^1.3.2", - "underscore": "^1.8.3" + "typescript-collections": "^1.3.2" }, "devDependencies": { "html5-validator": "^1.0.7", From 1e8fb851225b21257cf542237d4f758401f91e27 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Aug 2019 01:11:26 +0200 Subject: [PATCH 09/11] Remove bcryptjs Remove unused dependency bcryptjs. --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index da2dd9ae..769527c1 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "base64url": "^3.0.0", "bcrypt": "^2.0.1", "bcrypt-promise": "^2.0.0", - "bcryptjs": "^2.4.3", "body-parser": "^1.18.2", "cli": "^1.0.1", "cli-interact": "^0.1.9", From a452749caab2e7551b5433644d9e7d20dfde8b97 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Aug 2019 01:30:40 +0200 Subject: [PATCH 10/11] Remove app-root-path Resolve the relative path using path.relative instead of some hacky string substitutions. --- lib/factorioServer/index.js | 1 - lib/factorioServer/index.spec.js | 5 ++--- package.json | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/factorioServer/index.js b/lib/factorioServer/index.js index 0dd4c602..b6e175f8 100644 --- a/lib/factorioServer/index.js +++ b/lib/factorioServer/index.js @@ -15,7 +15,6 @@ const child_process = require("child_process"); const path = require("path"); const mkdirp = require("mkdirp"); -const appRootPath = require("app-root-path"); const fileOps = require("lib/fileOps"); diff --git a/lib/factorioServer/index.spec.js b/lib/factorioServer/index.spec.js index 0a63fbcb..1caf2afb 100644 --- a/lib/factorioServer/index.spec.js +++ b/lib/factorioServer/index.spec.js @@ -2,7 +2,6 @@ const assert = require("assert"); const fs = require("fs"); const path = require("path"); -const appRootPath = require("app-root-path"); const parallel = require("mocha.parallel"); const mkdirp = require("mkdirp"); @@ -75,7 +74,7 @@ describe("class factorioServer/index.js", ()=>{ describe(".createMap(mapName.zip)", ()=>{ it("creates a map file as instanceDirectory/saves/map.zip", async function(){ this.timeout(20000); // these test often takes a lot of time in the map creation step (4500 ms on i3 dev system) - let relativePath = __dirname.replace(appRootPath, "").substr(1); + let relativePath = path.relative('.', __dirname); let server = new factorioServer({ instanceDirectory: path.join(relativePath, "_testData_createMap"), instanceName: "Unit testing instance", @@ -93,7 +92,7 @@ describe("class factorioServer/index.js", ()=>{ }); it("supports custom filenames", async function(){ this.timeout(20000); - let relativePath = __dirname.replace(appRootPath, "").substr(1); + let relativePath = path.relative('.', __dirname); let server = new factorioServer({ instanceDirectory: path.join(relativePath, "_testData_createMap_2"), instanceName: "Unit testing instance", diff --git a/package.json b/package.json index 769527c1..a81d6f63 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ "author": "Danielv123", "license": "ISC", "dependencies": { - "app-root-path": "^2.0.1", "as-table": "^1.0.31", "averaged-timeseries": "^1.4.0", "base64url": "^3.0.0", From bc7f3804c8de40f0c5dea12b9dd8623decca914a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Aug 2019 02:14:19 +0200 Subject: [PATCH 11/11] Remove right-pad Remove unused dependency right-pad. --- lib/manager/modManager.js | 1 - package.json | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/manager/modManager.js b/lib/manager/modManager.js index b4fc2279..999c81d8 100644 --- a/lib/manager/modManager.js +++ b/lib/manager/modManager.js @@ -3,7 +3,6 @@ Provides tools to manage mods in a clusterio install @module modManager */ -const rightpad = require("right-pad"); const asTable = require("as-table").configure({delimiter: ' | '}); const moment = require("moment"); const syncRequest = require("sync-request"); diff --git a/package.json b/package.json index a81d6f63..08fa33f3 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,6 @@ "prom-client": "^10.2.3", "rcon-client-fork": "^3.2.0", "request": "^2.83.0", - "right-pad": "^1.0.1", "rmdir-sync": "^1.0.1", "sanitizer": "^0.1.3", "simple-git": "^1.96.0",