From a9a382b9938e9d6f9ddc8d6bce6f0e6a5a9c6d03 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 4 Sep 2019 18:33:20 +0200 Subject: [PATCH 01/13] Mark package as 2.0.0 development version Change the version in package.json to 2.0.0-alpha --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ed4a182e..3083b21f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clusterio", - "version": "1.7.0", + "version": "2.0.0-alpha", "description": "Interserver communication for factorio", "repository": "https://github.com/clusterio/factorioClusterio", "main": "master.js", From 6bde8a22194e03fce9ffb45610d8f9909e7fad9b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 3 Sep 2019 20:18:36 +0200 Subject: [PATCH 02/13] Remove .js suffix from node invocations Remove the .js suffix from help texts of node invocations as well as actual invocations as this is not necessary. Removing it opens up the possibility of restructuring the code into a directory with the same name. --- CLI tools/cluster_rcon.js | 2 +- Dockerfile | 4 ++-- README.md | 28 ++++++++++++++-------------- client.js | 24 ++++++++++++------------ master.js | 6 +++--- package.json | 4 ++-- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/CLI tools/cluster_rcon.js b/CLI tools/cluster_rcon.js index 72808013..becb0e08 100644 --- a/CLI tools/cluster_rcon.js +++ b/CLI tools/cluster_rcon.js @@ -3,7 +3,7 @@ const cli = require("cli"); const config = require("./../config.json"); let options = cli.parse() -cli.setUsage(`node cluster_rcon.js /c game.player.force = "player"`) +cli.setUsage(`node cluster_rcon /c game.player.force = "player"`) if(process.argv.length < 3){ cli.getUsage() process.exit(1) diff --git a/Dockerfile b/Dockerfile index 4aefeac4..6db56d48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ WORKDIR factorioClusterio RUN mkdir instances sharedMods RUN cp config.json.dist config.json -RUN node client.js download +RUN node client download LABEL maintainer "Sir3lit@gmail.com" @@ -18,4 +18,4 @@ VOLUME /factorioClusterio/instances VOLUME /factorioClusterio/sharedMods VOLUME /factorioClusterio/sharedPlugins -CMD RCONPORT="$RCONPORT" FACTORIOPORT="$FACTORIOPORT" MODE="$MODE" node $MODE\.js start $INSTANCE +CMD RCONPORT="$RCONPORT" FACTORIOPORT="$FACTORIOPORT" MODE="$MODE" node $MODE start $INSTANCE diff --git a/README.md b/README.md index 88dbcf95..09797799 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ Master and all slaves: tar -xf factorio.tar.gz npm install --only=production cp config.json.dist config.json - node ./lib/npmPostinstall.js + node ./lib/npmPostinstall downloads and installs nodejs, git and clusterio. To specify a version, change "latest" in the link to a version number like 0.14.21. @@ -131,26 +131,26 @@ Pretty much all the blank fields should be filled in, except on the master where **Master** - node master.js + node master Or with pm2 (it's recommened to run it without pm2 first): - pm2 start master.js --name master + pm2 start master --name master **Server Host** To download the mod for all its non vanilla features and items, (optional, but very recommended) - node client.js manage shared mods add clusterio + node client manage shared mods add clusterio To create a new instance (its own save, set of mods and config files) - node client.js start [instancename] + node client start [instancename] To launch an instance with pm2 - pm2 start --name [instancename] client.js -- start [instancename] + pm2 start --name [instancename] client -- start [instancename] use `nano config.json` to change settings. @@ -196,9 +196,9 @@ reboot when you are done, then proceed to the next steps. *reboots matter* 3. Follow the instructions in the bat file 3. Some of the instructions are outdated. If you get stuck somewhere, look at the Ubuntu section. -4. Run `node client.js manage shared mods add clusterio` +4. Run `node client manage shared mods add clusterio` -5. type `node master.js` to start the server +5. type `node master` to start the server **Server Host** @@ -210,7 +210,7 @@ reboot when you are done, then proceed to the next steps. *reboots matter* 3.5 Some of the instructions are outdated. If you get stuck somewhere, look at the Ubuntu section. -4. Type `node client.js start [instancename]` to create a new instance. +4. Type `node client start [instancename]` to create a new instance. To connect to a master server running on a remote machine, open config.json with your favourite text editor (notepad++). You can also set it up to use the official server browser. @@ -268,19 +268,19 @@ TLDR: the tested fix is: ### Other fixes for other potential problems: -Sometimes the install fails. Try `node ./lib/npmPostinstall.js` to complete it. +Sometimes the install fails. Try `node ./lib/npmPostinstall` to complete it. ## Cheatsheet **To create a new instance/start it** - node client.js start [instanceName] + node client start [instanceName] **Other instance management tools:** ``` -node client.js delete [instanceName] -node client.js list +node client delete [instanceName] +node client list ``` **To update clusterio to the latest version:** @@ -296,5 +296,5 @@ npm install --only=production 3. Download the latest version of the factorioClusterioMod from its github repo ``` -node client.js manage shared mods add clusterio +node client manage shared mods add clusterio ``` diff --git a/client.js b/client.js index 147fe83c..543baa0d 100644 --- a/client.js +++ b/client.js @@ -105,15 +105,15 @@ function needleOptionsWithTokenAuthHeader(config) { function printUsage() { console.error("Usage: "); - console.error("node client.js start [instance name]"); - console.error("node client.js list"); - console.error("node client.js delete [instance]"); + console.error("node client start [instance name]"); + console.error("node client list"); + console.error("node client delete [instance]"); console.error("To download the latest version of the Clusterio lua mod, do"); - console.error("node client.js manage shared mods download clusterio"); + console.error("node client manage shared mods download clusterio"); console.error("To download a clusterio plugin, do"); - console.error("node client.js manage shared plugins install https://github.com/Danielv123/playerManager"); + console.error("node client manage shared plugins install https://github.com/Danielv123/playerManager"); console.error("For more management options, do"); - console.error("node client.js manage"); + console.error("node client manage"); } async function listInstances(config) { @@ -133,14 +133,14 @@ async function listInstances(config) { async function manage(config, instance) { // console.log("Launching mod manager"); - //const fullUsage = 'node client.js manage [instance, "shared"] ["mods", "config"] ...'; + //const fullUsage = 'node client manage [instance, "shared"] ["mods", "config"] ...'; function usage(instance, tool, action){ if(tool && tool == "mods"){ - console.log('node client.js manage '+instance.name+' '+tool+' ["list", "search", "add", "remove"]'); + console.log('node client manage '+instance.name+' '+tool+' ["list", "search", "add", "remove"]'); } else if(tool && tool == "plugins") { - console.log(`node client.js manage ${instance.name} ${tool} ["list", "add", "remove"]`); + console.log(`node client manage ${instance.name} ${tool} ["list", "add", "remove"]`); } else { - console.log('node client.js manage '+(instance && instance.name || '[instance, "shared"]') +' '+ (tool || '["mods", "plugins"]') + ' ...'); + console.log('node client manage '+(instance && instance.name || '[instance, "shared"]') +' '+ (tool || '["mods", "plugins"]') + ' ...'); } } const tool = process.argv[4] || ""; @@ -197,7 +197,7 @@ async function manage(config, instance) { async function deleteInstance(instance) { if (instance === undefined) { - console.error("Usage: node client.js delete [instance]"); + console.error("Usage: node client delete [instance]"); process.exit(1); } else if (fs.existsSync(instance.path())) { fileOps.deleteFolderRecursiveSync(instance.path()); // TODO: Check if this can cause i-craft users to format their server by using wrong paths @@ -546,7 +546,7 @@ async function startClient() { await downloadMod(); } else if (command == "start" && instance === undefined) { console.error("ERROR: No instanceName provided!"); - console.error("Usage: node client.js start [instanceName]"); + console.error("Usage: node client start [instanceName]"); process.exit(0); } else if (command == "start" && !fs.existsSync(instance.path())) { await createInstance(config, args, instance); diff --git a/master.js b/master.js index 541752b7..2606e639 100644 --- a/master.js +++ b/master.js @@ -7,7 +7,7 @@ combinator signals. @author Danielv123 @example -node master.js +node master */ // Attempt updating @@ -421,7 +421,7 @@ POST endpoint for storing items in master's inventory. @alias api/place @param {itemStack} itemStack the number and type of items to store (see typedef) @param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance -@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name] +@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name] @returns {string} status either "success" or "failure" */ app.post("/api/place", authenticate.middleware, function(req, res) { @@ -491,7 +491,7 @@ POST endpoint to remove items from DB when client orders items. @alias api/remove @param {itemStack} itemStack the name of and the number of items to remove (see typedef) @param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance -@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client.js start [name] +@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name] @returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages. */ app.post("/api/remove", authenticate.middleware, function(req, res) { diff --git a/package.json b/package.json index 3083b21f..e1aa09f2 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,8 @@ "cover": "nyc npm test", "fast-cover": "FAST_TEST=y nyc npm test", "ci-cover": "nyc --reporter=lcovonly npm run-script test", - "start": "echo Run node client.js start [instance] to start slaves && node master.js", - "install": "node lib/npmPostinstall.js", + "start": "echo Run node client start [instance] to start slaves && node master", + "install": "node lib/npmPostinstall", "bundle-dependencies": "bundle-dependencies" }, "nyc": { From ff3fc3b94be8331f7d4029cefc1be7810dcf0fc2 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 3 Sep 2019 19:35:25 +0200 Subject: [PATCH 03/13] Remove .js suffix in require paths Remove the .js suffix in require paths to make requires consistent across the project. --- client.js | 4 ++-- lib/clusterTools/index.js | 2 +- lib/isFactorioCommand/index.js | 2 +- lib/manager/modManager.js | 2 +- master.js | 10 +++++----- routes/api/remove.js | 2 +- sharedPlugins/clusterioMod/index.js | 2 +- sharedPlugins/inventoryImports/index.js | 4 ++-- sharedPlugins/inventoryImports/interfacingFunctions.js | 2 +- sharedPlugins/researchSync/test/index.js | 2 +- sharedPlugins/serverManager/index.js | 2 +- sharedPlugins/serverManager/masterPlugin.js | 4 ++-- test/lib/generateSSLcert.js | 2 +- test/master.js | 2 +- 14 files changed, 21 insertions(+), 21 deletions(-) diff --git a/client.js b/client.js index 543baa0d..5e3165bf 100644 --- a/client.js +++ b/client.js @@ -13,7 +13,7 @@ const asTable = require("as-table").configure({delimiter: ' | '}); const util = require("util"); // internal libraries -const objectOps = require("lib/objectOps.js"); +const objectOps = require("lib/objectOps"); const fileOps = require("lib/fileOps"); const pluginManager = require("lib/manager/pluginManager"); const modManager = require("lib/manager/modManager"); @@ -594,7 +594,7 @@ async function instanceManagement(config, instance, instanceconfig, server) { } if(pluginConfig.binary == "nodePackage" && pluginConfig.enabled){ // require plugin class and execute it - let pluginClass = require(path.resolve(pluginConfig.pluginPath, "index.js")); + let pluginClass = require(path.resolve(pluginConfig.pluginPath, "index")); plugins[i] = new pluginClass(combinedConfig, async function(data, callback){ if(data && data.toString('utf8')[0] != "/") { console.log("Clusterio | "+ pluginsToLoad[i].name + " | " + data.toString('utf8')); diff --git a/lib/clusterTools/index.js b/lib/clusterTools/index.js index eee7cb1b..09351555 100644 --- a/lib/clusterTools/index.js +++ b/lib/clusterTools/index.js @@ -12,5 +12,5 @@ module.exports = ({config} = {}) => ({ arg1 is the instanceID Promise resolves with a string, either the displayName or instanceID if not found. */ - getInstanceName: require("./getInstanceName.js")(config), + getInstanceName: require("./getInstanceName")(config), }) diff --git a/lib/isFactorioCommand/index.js b/lib/isFactorioCommand/index.js index 5d4aeb9b..408a2086 100644 --- a/lib/isFactorioCommand/index.js +++ b/lib/isFactorioCommand/index.js @@ -1 +1 @@ -module.exports = require("./isFactorioCommand.js"); \ No newline at end of file +module.exports = require("./isFactorioCommand"); diff --git a/lib/manager/modManager.js b/lib/manager/modManager.js index 606cef05..dc5da90b 100644 --- a/lib/manager/modManager.js +++ b/lib/manager/modManager.js @@ -36,7 +36,7 @@ modManager.listMods(config, instance); */ async function listMods(config, instance) { return new Promise((resolve, reject) => { - const instances = require("./../getInstances.js")(config).sync(); + const instances = require("./../getInstances")(config).sync(); if (instances.includes(instance.name)) { factorioAPI.init(false, instance.path("instanceMods"), instance.path("saves"), config.factorio_version); factorioAPI.authenticate({ diff --git a/master.js b/master.js index 2606e639..3acb2be3 100644 --- a/master.js +++ b/master.js @@ -11,7 +11,7 @@ node master */ // Attempt updating -// const updater = require("./updater.js"); +// const updater = require("./updater"); // updater.update().then(console.log); // argument parsing @@ -32,7 +32,7 @@ let config = {}; // homebrew modules const generateSSLcert = require("lib/generateSSLcert"); -const pluginManager = require("lib/manager/pluginManager.js"); +const pluginManager = require("lib/manager/pluginManager"); const database = require("lib/database"); const factorio = require("lib/factorio"); @@ -71,8 +71,8 @@ app.use(function(req, res, next){ next(); }); -require("./routes.js")(app); -require("./routes/api/getPictures.js")(app); +require("./routes")(app); +require("./routes/api/getPictures")(app); // Set folder to serve static content from (the website) app.use(express.static('static')); @@ -480,7 +480,7 @@ app.post("/api/place", authenticate.middleware, function(req, res) { res.send("failure"); } }); -const routes_api_remove = require("./routes/api/remove.js"); +const routes_api_remove = require("./routes/api/remove"); let neuralDole; /** diff --git a/routes/api/remove.js b/routes/api/remove.js index 4de5ba18..382dade9 100644 --- a/routes/api/remove.js +++ b/routes/api/remove.js @@ -1,6 +1,6 @@ const averagedTimeSeries = require("averaged-timeseries") -const doleNN = require("lib/dole_nn_base.js") +const doleNN = require("lib/dole_nn_base") const Prometheus = require('prom-client'); diff --git a/sharedPlugins/clusterioMod/index.js b/sharedPlugins/clusterioMod/index.js index ce4b57a7..64a11ce4 100644 --- a/sharedPlugins/clusterioMod/index.js +++ b/sharedPlugins/clusterioMod/index.js @@ -1,7 +1,7 @@ const needle = require("needle"); const fs = require("fs-extra"); -const objectOps = require("lib/objectOps.js"); +const objectOps = require("lib/objectOps"); const fileOps = require("lib/fileOps"); const pluginConfig = require("./config"); diff --git a/sharedPlugins/inventoryImports/index.js b/sharedPlugins/inventoryImports/index.js index 40dd929e..282dbf45 100644 --- a/sharedPlugins/inventoryImports/index.js +++ b/sharedPlugins/inventoryImports/index.js @@ -1,6 +1,6 @@ const needle = require("needle"); -const isJson = require("./isJson.js"); -const pluginConfig = require("./config.js"); +const isJson = require("./isJson"); +const pluginConfig = require("./config"); const functions = require("./interfacingFunctions"); diff --git a/sharedPlugins/inventoryImports/interfacingFunctions.js b/sharedPlugins/inventoryImports/interfacingFunctions.js index 870185d8..9ea57750 100644 --- a/sharedPlugins/inventoryImports/interfacingFunctions.js +++ b/sharedPlugins/inventoryImports/interfacingFunctions.js @@ -1,4 +1,4 @@ -const objectOps = require("lib/objectOps.js"); +const objectOps = require("lib/objectOps"); const needle = require("needle"); const pluginConfig = require("./config") const isJson = require("./isJson") diff --git a/sharedPlugins/researchSync/test/index.js b/sharedPlugins/researchSync/test/index.js index aa0b9a6c..00f8788f 100644 --- a/sharedPlugins/researchSync/test/index.js +++ b/sharedPlugins/researchSync/test/index.js @@ -3,7 +3,7 @@ const nock = require("nock"); const isFactorioCommand = require("lib/isFactorioCommand"); -const researchSync = require("../index.js"); +const researchSync = require("../index"); describe("researchSync/index.js", function() { it("exports a single class (or at least a function)", function() { diff --git a/sharedPlugins/serverManager/index.js b/sharedPlugins/serverManager/index.js index aafa32a7..3208b305 100644 --- a/sharedPlugins/serverManager/index.js +++ b/sharedPlugins/serverManager/index.js @@ -8,7 +8,7 @@ module.exports = class remoteCommands { this.messageInterface = messageInterface; this.config = mergedConfig; this.socket = extras.socket; - const pluginManager = require("lib/manager/pluginManager.js"); + const pluginManager = require("lib/manager/pluginManager"); // const modManager = require("lib/manager/modManager"); this.socket.on("hello", () => { this.socket.emit("registerServerManager"); diff --git a/sharedPlugins/serverManager/masterPlugin.js b/sharedPlugins/serverManager/masterPlugin.js index 981dbf00..6c458389 100644 --- a/sharedPlugins/serverManager/masterPlugin.js +++ b/sharedPlugins/serverManager/masterPlugin.js @@ -22,10 +22,10 @@ class masterPlugin { // this.banlist = getDatabaseSync("database/banlist.json").banlist || []; // initialize web API - require("./js/api-endpoints.js")(this); + require("./js/api-endpoints")(this); // expose UI elements embedded in the master - this.ui = require("./js/ui.js").injectThis(this).ui; + this.ui = require("./js/ui").injectThis(this).ui; this.serverSockets = {}; diff --git a/test/lib/generateSSLcert.js b/test/lib/generateSSLcert.js index 6f69efae..dbed9058 100644 --- a/test/lib/generateSSLcert.js +++ b/test/lib/generateSSLcert.js @@ -2,7 +2,7 @@ var assert = require("assert"); var fs = require("fs-extra"); var fileOps = require("lib/fileOps"); -var generateSSLcert = require("lib/generateSSLcert.js") +var generateSSLcert = require("lib/generateSSLcert") describe("generateSSLcert.js(options)", ()=>{ it("Creates a folder with a .crt and .key file in it", async function() { diff --git a/test/master.js b/test/master.js index 8153ec7d..79dbf410 100644 --- a/test/master.js +++ b/test/master.js @@ -4,7 +4,7 @@ const validateHTML = require('html5-validator'); const parallel = require('mocha.parallel'); const jwt = require("jsonwebtoken"); -const master = require('../master.js'); +const master = require('../master'); const authenticate = require('lib/authenticate'); const database = require('lib/database'); From bc444b79c24066047febf28f71ac3a99df17e385 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 28 Oct 2019 14:18:30 +0100 Subject: [PATCH 04/13] Remove broken client download command Remove the client download command which would download pre-releases. The mod add command is a complete replacement for it. --- CHANGELOG.md | 2 ++ Dockerfile | 2 +- client.js | 23 ----------------------- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5af51bc8..2adbb981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ Version 2.0.0 - The game port, RCON port and RCON password instance config entries are are now null by default, indicating that a random one will be generated every time the instance is started. +- Removed the broken client download command as this would download + pre-releases. ### Breaking Changes diff --git a/Dockerfile b/Dockerfile index 6db56d48..040399de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ WORKDIR factorioClusterio RUN mkdir instances sharedMods RUN cp config.json.dist config.json -RUN node client download +RUN node client manage shared mods add clusterio LABEL maintainer "Sir3lit@gmail.com" diff --git a/client.js b/client.js index 5e3165bf..bbf4987f 100644 --- a/client.js +++ b/client.js @@ -1,10 +1,8 @@ const fs = require('fs-extra'); const Tail = require('tail').Tail; -const https = require('follow-redirects').https; const needle = require("needle"); const child_process = require('child_process'); const path = require('path'); -const syncRequest = require('sync-request'); const request = require("request"); const deepmerge = require("deepmerge"); const getMac = require('getmac').getMac; @@ -209,25 +207,6 @@ async function deleteInstance(instance) { } } -async function downloadMod() { - console.log("Downloading mods..."); - // get JSON data about releases - let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"factorioClusterio"}}); - let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url; - let name = JSON.parse(res.getBody())[0].assets[0].name; - if(url) { - console.log(url); - let file = fs.createWriteStream("sharedMods/"+name); - https.get(url, function(response) { - response.on('end', function () { - console.log("Downloaded "+name); - process.exit(0); - }); - response.pipe(file); - }).end(); - } -} - /** * Create and update symlinks for shared mods in an instance * @@ -542,8 +521,6 @@ async function startClient() { // process.exit(0); } else if (command == "delete") { await deleteInstance(instance); - } else if (command == "download") { - await downloadMod(); } else if (command == "start" && instance === undefined) { console.error("ERROR: No instanceName provided!"); console.error("Usage: node client start [instanceName]"); From e4aae886140401c5e4dde3286575594d92ceb7a9 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 28 Oct 2019 18:43:11 +0100 Subject: [PATCH 05/13] Move clusterioMod specific code into the plugin Move item database, tests, endpoint definitions for /api/place, /api/remove, /api/inventory, /api/inventoryAsObject, and the web interface view for the storage page into the clusterioMod plugin. --- CHANGELOG.md | 4 + master.js | 284 ++---------------- .../clusterioMod/dole.js | 22 +- .../clusterioMod}/dole_nn_base.js | 0 sharedPlugins/clusterioMod/masterPlugin.js | 155 +++++----- sharedPlugins/clusterioMod/routes.js | 241 +++++++++++++++ .../clusterioMod/static}/storage.js | 2 +- sharedPlugins/clusterioMod/test/missing.js | 2 + sharedPlugins/clusterioMod/test/routes.js | 110 +++++++ sharedPlugins/clusterioMod/views/storage.ejs | 49 +++ test/master.js | 90 ------ test/missing.js | 1 - views/index.ejs | 34 +-- views/nav.ejs | 3 - 14 files changed, 524 insertions(+), 473 deletions(-) rename routes/api/remove.js => sharedPlugins/clusterioMod/dole.js (92%) rename {lib => sharedPlugins/clusterioMod}/dole_nn_base.js (100%) create mode 100644 sharedPlugins/clusterioMod/routes.js rename {static => sharedPlugins/clusterioMod/static}/storage.js (99%) create mode 100644 sharedPlugins/clusterioMod/test/routes.js create mode 100644 sharedPlugins/clusterioMod/views/storage.ejs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2adbb981..844e5665 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,10 @@ Version 2.0.0 time the instance is started. - Removed the broken client download command as this would download pre-releases. +- Moved the item database and HTTP definitions for /api/place, /api/remove, + /api/inventory, /api/inventoryAsObject and the web interface view for the + storage page into the clusterioMod plugin. If you disable this plugin + then these things will not be available. ### Breaking Changes diff --git a/master.js b/master.js index 3acb2be3..a154a6d9 100644 --- a/master.js +++ b/master.js @@ -17,7 +17,6 @@ node master // argument parsing const args = require('minimist')(process.argv.slice(2)); -const averagedTimeSeries = require("averaged-timeseries"); const deepmerge = require("deepmerge"); const path = require("path"); const fs = require("fs-extra"); @@ -99,21 +98,6 @@ const prometheusWsUsageCounter = new Prometheus.Counter({ help: 'Websocket traffic', labelNames: ["connectionType", "instanceID"], }); -const prometheusExportGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'export_gauge', - help: 'Items exported by instance', - labelNames: ["instanceID", "itemName"], -}); -const prometheusImportGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'import_gauge', - help: 'Items imported by instance', - labelNames: ["instanceID", "itemName"], -}); -const prometheusDoleFactorGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'dole_factor_gauge', - help: 'The current dole division factor for this item', - labelNames: ["itemName"], -}); const prometheusPlayerCountGauge = new Prometheus.Gauge({ name: prometheusPrefix+'player_count_gauge', help: 'Amount of players connected to this cluster', @@ -124,11 +108,6 @@ const prometheusUPSGauge = new Prometheus.Gauge({ help: 'UPS of the current server', labelNames: ["instanceID", "instanceName"], }); -const prometheusMasterInventoryGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'master_inventory_gauge', - help: 'Amount of items stored on master', - labelNames: ["itemName"], -}); /** GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format. @memberof clusterioMaster @@ -156,10 +135,14 @@ function registerMoreMetrics(){ if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0); }catch(e){} } - // inventory - for (let [key, count] of db.items._items) { - prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0); + + // plugins + for (let plugin of masterPlugins) { + if (plugin.main && plugin.main.onMetrics && typeof plugin.main.onMetrics == "function") { + plugin.main.onMetrics(); + } } + // Slave count let numberOfActiveSlaves = 0; for(let slave of db.slaves.values()){ @@ -175,26 +158,10 @@ async function loadDatabase(databaseDirectory) { let slavesPath = path.resolve(databaseDirectory, "slaves.json"); console.log(`Loading ${slavesPath}`); db.slaves = await database.loadJsonAsMap(slavesPath); - - let itemsPath = path.resolve(databaseDirectory, "items.json"); - console.log(`Loading ${itemsPath}`); - try { - let content = await fs.readFile(itemsPath); - db.items = new database.ItemDatabase(JSON.parse(content)); - - } catch (err) { - if (err.code === "ENOENT") { - console.log("Creating new item database"); - db.items = new database.ItemDatabase(); - - } else { - throw err; - } - } } -// store slaves and inventory in a .json full of JSON data +// store slaves in a .json full of JSON data async function shutdown() { console.log('Ctrl-C...'); let exitStartTime = Date.now(); @@ -208,14 +175,6 @@ async function shutdown() { console.error(`Slave database too large, not saving (${db.slaves.size}`); } - if (db.items && db.items.size < 50000) { - let file = path.resolve(config.databaseDirectory, "items.json"); - console.log(`writing ${file}`); - let content = JSON.stringify(db.items.serialize()); - await fs.outputFile(file, content); - } else if (db.items) { - console.error(`Item database too large, not saving (${db.items.size})`); - } for(let i in masterPlugins){ let plugin = masterPlugins[i]; if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ @@ -410,213 +369,14 @@ app.post("/api/rconPasswords", authenticate.middleware, function(req, res) { res.send(slaveCache.cache); }); -var recievedItemStatisticsBySlaveID = {}; -var sentItemStatisticsBySlaveID = {}; -// -/** -POST endpoint for storing items in master's inventory. - -@memberof clusterioMaster -@instance -@alias api/place -@param {itemStack} itemStack the number and type of items to store (see typedef) -@param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance -@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name] -@returns {string} status either "success" or "failure" -*/ -app.post("/api/place", authenticate.middleware, function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let x; - try { - x = JSON.parse(req.body); - } catch (e) { - x = req.body; - } - if(!x.instanceName) { - x.instanceName = "unknown"; - } - if(!x.instanceID) { - x.instanceID = "unknown"; - for (let [instanceID, slave] of db.slaves) { - if(slave.instanceName == req.body.instanceName) { - x.instanceID = instanceID; - break; - } - } - } - if(x.instanceID - && x.instanceName - && !isNaN(Number(x.count)) - && x.name - && typeof x.name == "string"){ - if(config.logItemTransfers){ - console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")"); - } - // gather statistics - let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID]; - if(recievedItemStatistics === undefined){ - recievedItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }, console.log); - recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics; - } - recievedItemStatistics.add({ - key:req.body.name, - value:req.body.count, - }); - prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); - // save items we get - - var count = Number(req.body.count); - db.items.addItem(req.body.name, count); - - // Attempt confirming - res.send("success"); - } else { - res.send("failure"); - } -}); -const routes_api_remove = require("./routes/api/remove"); -let neuralDole; - -/** -POST endpoint to remove items from DB when client orders items. - -@memberof clusterioMaster -@instance -@alias api/remove -@param {itemStack} itemStack the name of and the number of items to remove (see typedef) -@param {string} [itemStack.instanceID="unknown"] the unique/instanceID which is a numerical value for an instance -@param {string} [itemStack.instanceName="unknown"] the name of an instance for identification in statistics, as provided when launching it. ex node client start [name] -@returns {itemStack} the number of items actually removed, may be lower than what was asked for due to shortages. -*/ -app.post("/api/remove", authenticate.middleware, function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - // save items we get - var object = req.body; - if(!object.instanceID) { - object.instanceID = "unknown" - } - if(!object.instanceName) { - object.instanceName = "unknown"; - } - if (db.slaves.has(String(object.instanceID))) { - object.instanceName = db.slaves.get(String(object.instanceID)).instanceName; - } - let itemCount = db.items.getItemCount(object.name); - if ( - (config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" ) - && object.name != "electricity" - ) { - if(config.logItemTransfers){ - console.log('failure could not find ' + object.name); - } - res.send({name:object.name, count:0}); - } else if(config.disableFairItemDistribution){ - // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing. - let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount); - db.items.removeItem(object.name, numberToRemove); - res.send({count: numberToRemove, name: object.name}); - - // track statistics and do graphing things - prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0); - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }, console.log); - } - sentItemStatistics.add({ - key:object.name, - value:numberToRemove, - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; - } else if(config.useNeuralNetDoleDivider){ - // use fancy neural net to calculate a "fair" dole division rate. - neuralDole.divider({ - res, - object, - config, - sentItemStatisticsBySlaveID, - prometheusImportGauge - }) - } else { - // Use dole division. Makes it really slow to drain out the last little bit. - routes_api_remove.doleDivider({ - itemCount, - object, - db, - sentItemStatisticsBySlaveID, - config, - prometheusDoleFactorGauge, - prometheusImportGauge, - req,res, - }) - } -}); - -/** -GET endpoint to read the masters current inventory of items. - -@memberof clusterioMaster -@instance -@alias api/inventory -@returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}] -*/ -// endpoint for getting an inventory of what we got -app.get("/api/inventory", function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - // Check it and send it - var inventory = []; - for (let [name, count] of db.items._items) { - inventory.push({ name, count }); - } - res.send(JSON.stringify(inventory)); -}); -/** -GET endpoint to read the masters inventory as an object with key:value pairs - -@memberof clusterioMaster -@instance -@alias api/inventoryAsObject -@returns {object} JSON {"iron-plate":100, "copper-plate":5} -*/ -app.get("/api/inventoryAsObject", function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - // Check it and send it - res.send(JSON.stringify(db.items.serialise())); -}); - -/** - GET endpoint to read the masters inventory as an object with key:value pairs - - @memberof clusterioMaster - @instance - @alias api/modData - @returns {object} JSON - */ - // wrap a request in an promise - async function downloadPage(url) { - return new Promise((resolve, reject) => { - request(url, (error, response, body) => { - resolve(body); - }); +async function downloadPage(url) { + return new Promise((resolve, reject) => { + request(url, (error, response, body) => { + resolve(body); }); - } + }); +} app.get("/api/modmeta", async function(req, res) { endpointHitCounter.labels(req.route.path).inc(); @@ -832,7 +592,10 @@ async function getPlugins(){ path: pluginConfig.pluginPath, socketio: io, express: app, + db, Prometheus, + prometheusPrefix, + endpointHitCounter, }), pluginConfig, }); @@ -900,12 +663,6 @@ async function startServer() { fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { expiresIn: 86400*365 // expires in 1 year })); - setInterval(async function() { - let file = path.resolve(config.databaseDirectory, "items.json"); - console.log(`autosaving ${file}`); - let content = JSON.stringify(db.items.serialize()); - await fs.outputFile(file, content); - },config.autosaveInterval || 60000); process.on('SIGINT', shutdown); // ctrl + c process.on('SIGHUP', shutdown); // terminal closed @@ -921,13 +678,6 @@ async function startServer() { await loadDatabase(config.databaseDirectory); authenticate.setAuthSecret(config.masterAuthSecret); - // Only initialize neural network when it's enabled, otherwise it might override gauge - if(config.useNeuralNetDoleDivider) { - neuralDole=new routes_api_remove.neuralDole({ - db, gaugePrefix: prometheusPrefix - }); - } - // Make sure we're actually going to listen on a port let httpPort = args.masterPort || config.masterPort; let httpsPort = args.sslPort || config.sslPort; diff --git a/routes/api/remove.js b/sharedPlugins/clusterioMod/dole.js similarity index 92% rename from routes/api/remove.js rename to sharedPlugins/clusterioMod/dole.js index 382dade9..3ddbf7f7 100644 --- a/routes/api/remove.js +++ b/sharedPlugins/clusterioMod/dole.js @@ -1,6 +1,6 @@ const averagedTimeSeries = require("averaged-timeseries") -const doleNN = require("lib/dole_nn_base") +const doleNN = require("./dole_nn_base") const Prometheus = require('prom-client'); @@ -19,7 +19,7 @@ class neuralDole { } constructor({ - db, gaugePrefix + items, gaugePrefix }){ // Set some storage variables for the dole divider this.prometheusNNDoleGauge = new Prometheus.Gauge({ @@ -27,8 +27,8 @@ class neuralDole { help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given', labelNames: ["itemName"], }); - this.db = db - this.itemsLastTick = new Map(db.items._items) + this.items = items + this.itemsLastTick = new Map(items._items) this.dole = {} this.carry = {} this.lastRequest = {} @@ -36,7 +36,7 @@ class neuralDole { this.debt= {} setInterval(()=>{ - for (let [name, count] of this.db.items._items) { + for (let [name, count] of this.items._items) { let magicData = doleNN.Tick( count, this.dole[name], @@ -51,7 +51,7 @@ class neuralDole { // DONE handle magicData[1] for graphing for our users this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0); } - this.itemsLastTick = new Map(this.db.items._items) + this.itemsLastTick = new Map(this.items._items) }, 1000) } divider({ @@ -63,7 +63,7 @@ class neuralDole { }){ let magicData = doleNN.Dose( object.count, // numReq - this.db.items.getItemCount(object.name), + this.items.getItemCount(object.name), this.itemsLastTick.get(object.name) || 0, this.dole[object.name], this.carry[object.name+" "+object.instanceID] || 0, @@ -85,9 +85,9 @@ class neuralDole { this.debt[object.name+" "+object.instanceID] = magicData[3] // Remove item from DB and send it - this.db.items.removeItem(object.name, magicData[0]); + this.items.removeItem(object.name, magicData[0]); if(config.logItemTransfers){ - console.log("removed: " + object.name + " " + magicData[0] + " . " + this.db.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName); + console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName); } let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; if(sentItemStatistics === undefined){ @@ -116,7 +116,7 @@ _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this function doleDivider({ itemCount, object, - db, + items, sentItemStatisticsBySlaveID, config, prometheusDoleFactorGauge, @@ -140,7 +140,7 @@ function doleDivider({ if (config.logItemTransfers) { console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName); } - db.items.removeItem(object.name, object.count) + items.removeItem(object.name, object.count) let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; if(sentItemStatistics === undefined){ sentItemStatistics = new averagedTimeSeries({ diff --git a/lib/dole_nn_base.js b/sharedPlugins/clusterioMod/dole_nn_base.js similarity index 100% rename from lib/dole_nn_base.js rename to sharedPlugins/clusterioMod/dole_nn_base.js diff --git a/sharedPlugins/clusterioMod/masterPlugin.js b/sharedPlugins/clusterioMod/masterPlugin.js index e02ca4b0..7151b257 100644 --- a/sharedPlugins/clusterioMod/masterPlugin.js +++ b/sharedPlugins/clusterioMod/masterPlugin.js @@ -1,83 +1,104 @@ +"use strict"; const fs = require("fs-extra"); const path = require("path"); const Express = require("express"); const ejs = require("ejs"); +const database = require("lib/database"); +const routes = require("./routes"); + + class masterPlugin { - constructor({config, pluginConfig, pluginPath, socketio, express}){ + constructor({ + config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter, + }) { this.config = config; this.pluginConfig = pluginConfig; this.pluginPath = pluginPath; this.io = socketio; - this.app = express; - - // load databases - this.items = getDatabaseSync("database/items.json"); - - this.clients = {}; - this.slaves = {}; - - // initialize web API - // require("./js/api-endpoints.js")(this); - - // expose UI elements embedded in the master - // this.ui = require("./js/ui.js").ui; - - this.io.on("connection", socket => { - - }); - } - async broadcastCommand(command){ - let returnValues = []; - for(let instanceID in this.slaves){ - let slave = this.slaves[instanceID]; - slave.emit("runCommand", { - // commandID:Math.random(), - command, - }); - }; - return returnValues; - } - findInArray(key, value, array){ - let indexes = []; - for(let i in array){ - if(array[i][key] && array[i][key] === value) indexes.push(i); - } - return indexes; - } - async onExit(){ - // await saveDatabase("database/items.json", this.items); - return; - } -} -module.exports = masterPlugin; -function arrayRemoveDuplicates(array){ - let newArray = []; - array.forEach(value => { - if(!newArray.includes(value)) newArray.push(value); - }); - return newArray; -} -function getDatabaseSync(path){ - let db; - try { - db = JSON.parse(fs.readFileSync(path, "utf8")); - } catch(e){ - db = {}; + this.express = express; + this.db = db, + this.Prometheus = Prometheus; + this.prometheusPrefix = prometheusPrefix; + this.endpointHitCounter = endpointHitCounter; + + this.prometheusMasterInventoryGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'master_inventory_gauge', + help: 'Amount of items stored on master', + labelNames: ["itemName"], + }); + + this.ui = { + sidebar: [ + { + getHtml: () => ` +
+ Storage +
`, + }, + ], + }; + } + + onMetrics() { + if (this.items) { + for (let [key, count] of this.items._items) { + this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0); + } + } + } + + async onLoadFinish() { + this.items = await loadDatabase(this.config); + this.autosaveId = setInterval(async function() { + let file = path.join(this.config.databaseDirectory, "items.json"); + console.log(`autosaving ${file}`); + let content = JSON.stringify(db.items.serialize()); + await fs.outputFile(file, content); + },this.config.autosaveInterval || 60000); + + routes.addApiRoutes( + this.express, this.config, this.db, this.items, + this.Prometheus, this.prometheusPrefix, this.endpointHitCounter, + ); + + routes.addWebRoutes(this.express); + } + + async onExit(){ + clearInterval(this.autosaveId); + await saveDatabase(this.config, this.items); } - return db; } -async function saveDatabase(path, database){ - if(!path){ - throw new Error("No path provided!"); - } else if(!database){ - throw new Error("No database provided!"); - } else { - try { - await fs.writeFile(path, JSON.stringify(database, null, 4)); - } catch(e){ - throw new Error("Unable to write to database! "+path); + +async function loadDatabase(config) { + let itemsPath = path.resolve(config.databaseDirectory, "items.json"); + console.log(`Loading ${itemsPath}`); + try { + let content = await fs.readFile(itemsPath); + return new database.ItemDatabase(JSON.parse(content)); + + } catch (err) { + if (err.code === "ENOENT") { + console.log("Creating new item database"); + return new database.ItemDatabase(); + + } else { + throw err; } } } + +async function saveDatabase(config, items) { + if (items && items.size < 50000) { + let file = path.resolve(config.databaseDirectory, "items.json"); + console.log(`writing ${file}`); + let content = JSON.stringify(items.serialize()); + await fs.outputFile(file, content); + } else if (items) { + console.error(`Item database too large, not saving (${items.size})`); + } +} + +module.exports = masterPlugin; diff --git a/sharedPlugins/clusterioMod/routes.js b/sharedPlugins/clusterioMod/routes.js new file mode 100644 index 00000000..c087093a --- /dev/null +++ b/sharedPlugins/clusterioMod/routes.js @@ -0,0 +1,241 @@ +var recievedItemStatisticsBySlaveID = {}; +var sentItemStatisticsBySlaveID = {}; + +const averagedTimeSeries = require("averaged-timeseries"); +const express = require("express"); +const path = require("path"); + +const authenticate = require("lib/authenticate"); + +const dole = require("./dole"); + + +function addWebRoutes(app) { + app.use('/clusterioMod', express.static(path.join(__dirname, 'static'))); + app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); }); +} + +function addApiRoutes( + app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter, +) { + const prometheusExportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'export_gauge', + help: 'Items exported by instance', + labelNames: ["instanceID", "itemName"], + }); + const prometheusImportGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'import_gauge', + help: 'Items imported by instance', + labelNames: ["instanceID", "itemName"], + }); + const prometheusDoleFactorGauge = new Prometheus.Gauge({ + name: prometheusPrefix+'dole_factor_gauge', + help: 'The current dole division factor for this item', + labelNames: ["itemName"], + }); + + // Only initialize neural network when it's enabled, otherwise it might override gauge + let neuralDole; + if(config.useNeuralNetDoleDivider) { + neuralDole=new dole.neuralDole({ + db, gaugePrefix: prometheusPrefix + }); + } + + /** + * POST endpoint for storing items in master's inventory. + + * @memberof clusterioMaster + * @instance + * @alias api/place + * @param {itemStack} itemStack the number and type of items to store (see typedef) + * @param {string} [itemStack.instanceID="unknown"] + * the unique/instanceID which is a numerical value for an instance + * @param {string} [itemStack.instanceName="unknown"] + * the name of an instance for identification in statistics, as + * provided when launching it. ex node client start [name] + * @returns {string} status either "success" or "failure" + */ + app.post("/api/place", authenticate.middleware, function(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + let x; + try { + x = JSON.parse(req.body); + } catch (e) { + x = req.body; + } + if(!x.instanceName) { + x.instanceName = "unknown"; + } + if(!x.instanceID) { + x.instanceID = "unknown"; + for (let [instanceID, slave] of db.slaves) { + if(slave.instanceName == req.body.instanceName) { + x.instanceID = instanceID; + break; + } + } + } + if(x.instanceID + && x.instanceName + && !isNaN(Number(x.count)) + && x.name + && typeof x.name == "string"){ + if(config.logItemTransfers){ + console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")"); + } + // gather statistics + let recievedItemStatistics = recievedItemStatisticsBySlaveID[x.instanceID]; + if(recievedItemStatistics === undefined){ + recievedItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics; + } + recievedItemStatistics.add({ + key:req.body.name, + value:req.body.count, + }); + prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); + // save items we get + var count = Number(req.body.count); + items.addItem(req.body.name, count); + + // Attempt confirming + res.send("success"); + } else { + res.send("failure"); + } + }); + + /** + * POST endpoint to remove items from DB when client orders items. + * + * @memberof clusterioMaster + * @instance + * @alias api/remove + * @param {itemStack} itemStack the name of and the number of items to remove (see typedef) + * @param {string} [itemStack.instanceID="unknown"] + * the unique/instanceID which is a numerical value for an instance + * @param {string} [itemStack.instanceName="unknown"] + * the name of an instance for identification in statistics, as + * provided when launching it. ex node client start [name] + * @returns {itemStack} + * the number of items actually removed, may be lower than what was + * asked for due to shortages. + */ + app.post("/api/remove", authenticate.middleware, function(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + // save items we get + var object = req.body; + if(!object.instanceID) { + object.instanceID = "unknown" + } + if(!object.instanceName) { + object.instanceName = "unknown"; + } + if (db.slaves.has(String(object.instanceID))) { + object.instanceName = db.slaves.get(String(object.instanceID)).instanceName; + } + let itemCount = items.getItemCount(object.name); + if ( + (config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" ) + && object.name != "electricity" + ) { + if(config.logItemTransfers){ + console.log('failure could not find ' + object.name); + } + res.send({name:object.name, count:0}); + } else if(config.disableFairItemDistribution){ + // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing. + let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount); + items.removeItem(object.name, numberToRemove); + res.send({count: numberToRemove, name: object.name}); + + // track statistics and do graphing things + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0); + let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; + if(sentItemStatistics === undefined){ + sentItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + } + sentItemStatistics.add({ + key:object.name, + value:numberToRemove, + }); + //console.log(sentItemStatistics.data) + sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; + } else if(config.useNeuralNetDoleDivider){ + // use fancy neural net to calculate a "fair" dole division rate. + neuralDole.divider({ + res, + object, + config, + sentItemStatisticsBySlaveID, + prometheusImportGauge + }) + } else { + // Use dole division. Makes it really slow to drain out the last little bit. + dole.doleDivider({ + itemCount, + object, + db, + sentItemStatisticsBySlaveID, + config, + prometheusDoleFactorGauge, + prometheusImportGauge, + req,res, + }) + } + }); + + /** + * GET endpoint to read the masters current inventory of items. + * + * @memberof clusterioMaster + * @instance + * @alias api/inventory + * @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}] + */ + app.get("/api/inventory", function(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + // Check it and send it + var inventory = []; + for (let [name, count] of items._items) { + inventory.push({ name, count }); + } + res.send(JSON.stringify(inventory)); + }); + + /** + * GET endpoint to read the masters inventory as an object with key:value pairs + * + * @memberof clusterioMaster + * @instance + * @alias api/inventoryAsObject + * @returns {object} JSON {"iron-plate":100, "copper-plate":5} + */ + app.get("/api/inventoryAsObject", function(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + // Check it and send it + res.send(JSON.stringify(items.serialise())); + }); +} + +module.exports = { + addWebRoutes, + addApiRoutes, +} diff --git a/static/storage.js b/sharedPlugins/clusterioMod/static/storage.js similarity index 99% rename from static/storage.js rename to sharedPlugins/clusterioMod/static/storage.js index 0e3c68a7..0a8f561f 100644 --- a/static/storage.js +++ b/sharedPlugins/clusterioMod/static/storage.js @@ -65,7 +65,7 @@ function updateInventory() { drawcontents(data); } } - xmlhttp.open("GET", "api/inventory", true); + xmlhttp.open("GET", "/api/inventory", true); xmlhttp.send(); } if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) { diff --git a/sharedPlugins/clusterioMod/test/missing.js b/sharedPlugins/clusterioMod/test/missing.js index 9993114b..5b080fef 100644 --- a/sharedPlugins/clusterioMod/test/missing.js +++ b/sharedPlugins/clusterioMod/test/missing.js @@ -3,3 +3,5 @@ require('../index'); require('../masterPlugin'); +require('../dole'); +require('../dole_nn_base'); diff --git a/sharedPlugins/clusterioMod/test/routes.js b/sharedPlugins/clusterioMod/test/routes.js new file mode 100644 index 00000000..1e5a6e20 --- /dev/null +++ b/sharedPlugins/clusterioMod/test/routes.js @@ -0,0 +1,110 @@ +const assert = require("assert").strict; +const express = require("express"); +const bodyParser = require("body-parser"); +const request = require("supertest"); +const Prometheus = require('prom-client'); +const jwt = require("jsonwebtoken"); + +const authenticate = require('lib/authenticate'); +const database = require("lib/database"); + +const routes = require("../routes"); + + +const testSecret = "TestSecretDoNotUse"; +authenticate.setAuthSecret(testSecret); + +const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 }); + +describe("clusterioMod endpoints", function() { + const app = express(); + app.use(bodyParser.json({ limit: '10mb', })); + app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true })); + const config = { + itemStats: { maxEntries:60, entriesPerSecond: 1 / 60 }, + disableFairItemDistribution: true, + }; + const items = new database.ItemDatabase(); + const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] }); + routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge); + + let persistentMaster = request(app); + describe("#POST /api/place", function() { + it("adds an itemStack to the masters inventory", function() { + return persistentMaster.post("/api/place").send({ + name:"steel-plate", + count:20, + instanceName:"unitTest" + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.equal(res.text, "success", "something went wrong with the request") + }); + }); + }); + describe("#GET /api/inventory", function() { + it("returns the masters current inventory", function() { + return persistentMaster.get("/api/inventory").expect(200).then(function(res) { + let inventory = JSON.parse(res.text); + assert.equal(typeof inventory, "object", "Inventory should be an object"); + assert(inventory.length >= 1, "There should be at least 1 entry in the inventory"); + + let contains20SteelPlate = false; + inventory.forEach(itemStack => { + if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true; + }); + assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory") + }); + }); + }); + describe("#POST /api/remove", function() { + it("returns an itemStack of how many items were removed", function() { + return persistentMaster.post("/api/remove").send({ + name:"steel-plate", + count:10 + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`); + }); + }); + it("works correctly for items named addItem or removeItem", async function() { + function add(item) { + return persistentMaster.post("/api/place").send({ + name: item, + count: 10 + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`); + }) + } + + function remove(item) { + return persistentMaster.post("/api/remove").send({ + name: item, + count: 10 + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.deepEqual(res.body, {"name":item,"count":10}); + }); + } + + await Promise.all([add("addItem"), add("removeItem")]); + await Promise.all([remove("addItem"), remove("removeItem")]); + }); + it("returns an empty itemStack if you don't have any of the item you request", function() { + return persistentMaster.post("/api/remove").send({ + name:"imaginaryItem", + count:999999 + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for"); + assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere"); + assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object"); + }); + }); + }); +}); diff --git a/sharedPlugins/clusterioMod/views/storage.ejs b/sharedPlugins/clusterioMod/views/storage.ejs new file mode 100644 index 00000000..c274964b --- /dev/null +++ b/sharedPlugins/clusterioMod/views/storage.ejs @@ -0,0 +1,49 @@ + + + <%- include("head") %> + + + + +<%- include("nav") %> +
+
+ +
+ +
+
+ + + + + + + + + +
IconItem IDQuantity
+
+
+
+
+
+
+
+
+ + + + diff --git a/test/master.js b/test/master.js index 79dbf410..d4dd484d 100644 --- a/test/master.js +++ b/test/master.js @@ -2,25 +2,14 @@ const assert = require('assert').strict; const request = require('supertest'); const validateHTML = require('html5-validator'); const parallel = require('mocha.parallel'); -const jwt = require("jsonwebtoken"); const master = require('../master'); -const authenticate = require('lib/authenticate'); -const database = require('lib/database'); master._db.slaves = new Map(); -master._db.items = new database.ItemDatabase(); // Fields required in the config master._config.factorioDirectory = "factorio"; -master._config.itemStats = { maxEntries:60, entriesPerSecond: 1 / 60 }; -master._config.disableFairItemDistribution = true; - -const testSecret = "TestSecretDoNotUse"; -authenticate.setAuthSecret(testSecret); - -const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 }); async function get(path) { return await request(master.app).get(path).expect(200); @@ -54,83 +43,4 @@ describe('Master testing', function() { it(`sends some HTML when accessing ${path}`, () => getValidate(path)); } }); - let persistentMaster = request(master.app); - describe("#POST /api/place", function() { - it("adds an itemStack to the masters inventory", function() { - return persistentMaster.post("/api/place").send({ - name:"steel-plate", - count:20, - instanceName:"unitTest" - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.text, "success", "something went wrong with the request") - }); - }); - }); - describe("#GET /api/inventory", function() { - it("returns the masters current inventory", function() { - return persistentMaster.get("/api/inventory").expect(200).then(function(res) { - let inventory = JSON.parse(res.text); - assert.equal(typeof inventory, "object", "Inventory should be an object"); - assert(inventory.length >= 1, "There should be at least 1 entry in the inventory"); - - let contains20SteelPlate = false; - inventory.forEach(itemStack => { - if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true; - }); - assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory") - }); - }); - }); - describe("#POST /api/remove", function() { - it("returns an itemStack of how many items were removed", function() { - return persistentMaster.post("/api/remove").send({ - name:"steel-plate", - count:10 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`); - }); - }); - it("works correctly for items named addItem or removeItem", async function() { - function add(item) { - return persistentMaster.post("/api/place").send({ - name: item, - count: 10 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`); - }) - } - - function remove(item) { - return persistentMaster.post("/api/remove").send({ - name: item, - count: 10 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.deepEqual(res.body, {"name":item,"count":10}); - }); - } - - await Promise.all([add("addItem"), add("removeItem")]); - await Promise.all([remove("addItem"), remove("removeItem")]); - }); - it("returns an empty itemStack if you don't have any of the item you request", function() { - return persistentMaster.post("/api/remove").send({ - name:"imaginaryItem", - count:999999 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for"); - assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere"); - assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object"); - }); - }); - }); }); diff --git a/test/missing.js b/test/missing.js index 59938262..edb63b13 100644 --- a/test/missing.js +++ b/test/missing.js @@ -2,7 +2,6 @@ // required here in order to track coverage for them. require('lib/clusterTools'); -require('lib/dole_nn_base'); require('lib/getInstances'); require('lib/manager/instanceManager'); require('lib/manager/modManager'); diff --git a/views/index.ejs b/views/index.ejs index 336848b8..3c01a242 100644 --- a/views/index.ejs +++ b/views/index.ejs @@ -1,7 +1,6 @@ <%- include("head") %> - @@ -10,40 +9,9 @@
-
- -
-
- - - - - - - - - -
IconItem IDQuantity
-
-
-
-
-
-
- - - diff --git a/views/nav.ejs b/views/nav.ejs index 5a560649..661cf96c 100644 --- a/views/nav.ejs +++ b/views/nav.ejs @@ -5,9 +5,6 @@

Clusterio

-
- Storage -
Slaves
<% From 9a2b2c9f8d0fc015301100a2520fa709b0d1d72a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 28 Oct 2019 19:01:43 +0100 Subject: [PATCH 06/13] Speed up generateSSLcert test Significantly speed up the generateSSLtest by reducing the bits in the key from 2048 to 512. --- test/lib/generateSSLcert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/generateSSLcert.js b/test/lib/generateSSLcert.js index dbed9058..dbe3d1e3 100644 --- a/test/lib/generateSSLcert.js +++ b/test/lib/generateSSLcert.js @@ -12,7 +12,7 @@ describe("generateSSLcert.js(options)", ()=>{ assert(!await fs.exists(certPath)); // if it already exsits, the test might pass because of leftover data (which is bad) await generateSSLcert({ - // bits:512, // supported, but I just use the default of 2048 for this test + bits: 512, // This is too small for real world usage, but faster to test sslCertPath: certPath, sslPrivKeyPath: privKeyPath, doLogging: false, From 72b16de227f01bdcabbcc716fead01139ae6f798 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 3 Sep 2019 20:44:59 +0200 Subject: [PATCH 07/13] Remove logging from averaged-timeseries The logging implemented in averaged-timeseries consists solely of printing "Logging enabled!". Remove this pointless logging. --- sharedPlugins/clusterioMod/dole.js | 4 ++-- sharedPlugins/clusterioMod/routes.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sharedPlugins/clusterioMod/dole.js b/sharedPlugins/clusterioMod/dole.js index 3ddbf7f7..896d7c89 100644 --- a/sharedPlugins/clusterioMod/dole.js +++ b/sharedPlugins/clusterioMod/dole.js @@ -95,7 +95,7 @@ class neuralDole { maxEntries: config.itemStats.maxEntries, entriesPerSecond: config.itemStats.entriesPerSecond, mergeMode: "add", - }, console.log); + }); } sentItemStatistics.add({ key:object.name, @@ -147,7 +147,7 @@ function doleDivider({ maxEntries: config.itemStats.maxEntries, entriesPerSecond: config.itemStats.entriesPerSecond, mergeMode: "add", - }, console.log); + }); } sentItemStatistics.add({ key:object.name, diff --git a/sharedPlugins/clusterioMod/routes.js b/sharedPlugins/clusterioMod/routes.js index c087093a..db7ee7a4 100644 --- a/sharedPlugins/clusterioMod/routes.js +++ b/sharedPlugins/clusterioMod/routes.js @@ -93,7 +93,7 @@ function addApiRoutes( maxEntries: config.itemStats.maxEntries, entriesPerSecond: config.itemStats.entriesPerSecond, mergeMode: "add", - }, console.log); + }); recievedItemStatisticsBySlaveID[x.instanceID] = recievedItemStatistics; } recievedItemStatistics.add({ @@ -166,7 +166,7 @@ function addApiRoutes( maxEntries: config.itemStats.maxEntries, entriesPerSecond: config.itemStats.entriesPerSecond, mergeMode: "add", - }, console.log); + }); } sentItemStatistics.add({ key:object.name, From 3fdc27238ba4542805183e05419ddfe6c46782e7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 4 Sep 2019 14:34:47 +0200 Subject: [PATCH 08/13] Make _doleDivisionFactor local Turn _doleDivisionFactor into a variable local to the module instead of implicitly global. --- sharedPlugins/clusterioMod/dole.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sharedPlugins/clusterioMod/dole.js b/sharedPlugins/clusterioMod/dole.js index 896d7c89..5f1f197b 100644 --- a/sharedPlugins/clusterioMod/dole.js +++ b/sharedPlugins/clusterioMod/dole.js @@ -112,7 +112,8 @@ class neuralDole { } } -_doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. +// If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. +let _doleDivisionFactor = {}; function doleDivider({ itemCount, object, From 467249579d39923f64d8c578641e8219bc0d41fc Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 6 Sep 2019 14:40:01 +0200 Subject: [PATCH 09/13] Add missing changelog entry for f4ca5d9 Add note in the changelog describing the feature added by f4ca5d9. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 844e5665..28fb75b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Version 2.0.0 - Added plugins directory to the views path. This makes it possible for plugins to render their own ejs views or pages in their own folders by using paths of the format "pluginName/path/to/page-or-view". +- Added instanceName entry to the config passed to instance plugins. - Replaced the per instance copy of the shared Factorio mods with symlinks. On Windows hard links are used instead due to the privileges requirements of symlinks. From 6f5534c45bbb36c00159bc5294e91a5ddc0d93db Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 10 Sep 2019 00:50:09 +0200 Subject: [PATCH 10/13] Test for numeric strings in addItem and removeItem Add tests asserting that numeric strings passed to the addItem and removeItem methods to ItemDatabes throws. --- test/lib/database.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/lib/database.js b/test/lib/database.js index 512de4ad..8956fed3 100644 --- a/test/lib/database.js +++ b/test/lib/database.js @@ -176,6 +176,10 @@ describe("lib/database", function() { () => items.addItem('a', NaN), new Error("count must be a number") ); + assert.throws( + () => items.addItem('a', "1"), + new Error("count must be a number") + ); }); }); @@ -206,6 +210,10 @@ describe("lib/database", function() { () => items.removeItem('a', 'b'), new Error("count must be a number") ); + assert.throws( + () => items.removeItem('a', "1"), + new Error("count must be a number") + ); }); }); }); From cb8d7e240fc52f8d623af21c11acd44aaa37ec1a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 28 Oct 2019 19:16:34 +0100 Subject: [PATCH 11/13] Remove more remanent remote map code Remove JavaScript code exported on the web interface that relates to the removed remote map plugin. --- static/lib/entityDrawRules.js | 345 ---------------------------------- static/lib/utility.js | 44 ----- 2 files changed, 389 deletions(-) delete mode 100644 static/lib/entityDrawRules.js diff --git a/static/lib/entityDrawRules.js b/static/lib/entityDrawRules.js deleted file mode 100644 index 903bc722..00000000 --- a/static/lib/entityDrawRules.js +++ /dev/null @@ -1,345 +0,0 @@ - -import spritesheetJson from "./../pictures/spritesheet.js"; - -export var entityDrawRules = { - "accumulator": { - spritesheet: spritesheetJson.frames["images/factorio/entity/accumulator/accumulator.png"], - sizeInTiles: { - x:3.5, - y:3.5, - }, - positionOffset: { - x:-1, - y:-2, - } - }, - "centrifuge": { - spritesheet: spritesheetJson.frames["images/factorio/entity/centrifuge/centrifuge-A.png.0"], - sizeInTiles: { - x:4, - y:4, - }, - positionOffset: { - x:-1.5, - y:-1, - }, - }, - "stone-furnace": { - spritesheet: spritesheetJson.frames["images/factorio/entity/stone-furnace/stone-furnace.png"], - sizeInTiles: { - x:3, - y:2.2, - }, - positionOffset: { - x:-0.9, - y:-1, - }, - }, - "steel-furnace": { - spritesheet: spritesheetJson.frames["images/factorio/entity/steel-furnace/steel-furnace.png"], - sizeInTiles: { - x:2.5, - y:2.5, - }, - positionOffset: { - x:-1.2, - y:-1.1, - }, - }, - "roboport": { - spritesheet: spritesheetJson.frames["images/factorio/entity/roboport/roboport-base.png"], - sizeInTiles: { - x:5, - y:4.5, - }, - positionOffset: { - x:-2, - y:-2.2, - }, - }, - "substation": { - spritesheet: spritesheetJson.frames["images/factorio/entity/substation/substation.png.0"], - sizeInTiles: { - x:3, - y:3, - }, - positionOffset: { - x:-1, - y:-1.8, - }, - }, - "big-electric-pole": { - spritesheet: spritesheetJson.frames["images/factorio/entity/big-electric-pole/big-electric-pole.png.0"], - sizeInTiles: { - x:5, - y:5, - }, - positionOffset: { - x:-1, - y:-3.5, - }, - }, - "electric-mining-drill": { - spritesheet: [{ - spritesheet: spritesheetJson.frames["images/factorio/entity/electric-mining-drill/electric-mining-drill-N.png.0"], - },{ - spritesheet: spritesheetJson.frames["images/factorio/entity/electric-mining-drill/electric-mining-drill-E.png.0"], - },{ - spritesheet: spritesheetJson.frames["images/factorio/entity/electric-mining-drill/electric-mining-drill-S.png.0"], - },{ - spritesheet: spritesheetJson.frames["images/factorio/entity/electric-mining-drill/electric-mining-drill-W.png.0"], - }, - ], - sizeInTiles: { - x:3, - y:3, - }, - positionOffset: { - x:-1, - y:-1, - }, - }, - "straight-rail": { - sizeInTiles: { - x:2, - y:2, - }, - positionOffset: { - x:-1, - y:-1, - } - }, -}; -let template3x3entity = { - sizeInTiles: { - x:3, - y:3, - }, - positionOffset: { - x:-1, - y:-1, - }, -}; -for(let i = 1; i <= 3; i++){ - let name = "assembling-machine-"+i; - // entityDrawRules[name] = - entityDrawRules[name] = { - spritesheet: spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png.0"], - sizeInTiles: { - x:3.9, - y:3.9, - }, - positionOffset: { - x:-1.4, - y:-1.2, - }, - } - // template3x3entity; -}; -entityDrawRules["radar"] = { - spritesheet: [], - sizeInTiles: { - x:4, - y:4, - }, - positionOffset: { - x:-0.8, - y:-1.7, - }, -} -for(let i = 0; i < 64; i++){ - // animate the radar - entityDrawRules.radar.spritesheet.push({spritesheet: spritesheetJson.frames["images/factorio/entity/radar/radar.png."+i]}); -} -["beacon", "electric-furnace"].forEach(name => { - entityDrawRules[name] = template3x3entity; -}); -entityDrawRules["lab"] = { - spritesheet: spritesheetJson.frames["images/factorio/entity/lab/lab.png.0"], - sizeInTiles: { - x:3.5, - y:3, - }, - positionOffset: { - x:-1, - y:-0.9, - }, -} -entityDrawRules["chemical-plant"] = { - spritesheet: [{ - spritesheet: spritesheetJson.frames["images/factorio/entity/chemical-plant/chemical-plant.png.0"], - sizeInTiles: { - x:3.5, - y:3.5, - }, - positionOffset: { - x:-1.3, - y:-1, - }, - },{ - spritesheet: spritesheetJson.frames["images/factorio/entity/chemical-plant/chemical-plant.png.1"], - sizeInTiles: { - x:3.7, - y:4, - }, - positionOffset: { - x:-1.3, - y:-1.5, - }, - },{ - spritesheet: spritesheetJson.frames["images/factorio/entity/chemical-plant/chemical-plant.png.2"], - sizeInTiles: { - x:3.5, - y:3.2, - }, - positionOffset: { - x:-1.3, - y:-1, - }, - },{ - spritesheet: spritesheetJson.frames["images/factorio/entity/chemical-plant/chemical-plant.png.3"], - sizeInTiles: { - x:3.7, - y:3.6, - }, - positionOffset: { - x:-1.45, - y:-1.3, - }, - }, - ], -}; -[{name: "solar-panel", ref:"images/factorio/entity/solar-panel/solar-panel.png"}].forEach(ent => { - entityDrawRules[ent.name] = { - spritesheet: spritesheetJson.frames[ent.ref], - sizeInTiles: { - x:3, - y:3, - }, - positionOffset: { - x:-1, - y:-1, - }, - } -}); - -["transport-belt", "fast-transport-belt", "express-transport-belt"].forEach(name => { - entityDrawRules[name] = { - spritesheet: [{ - spritesheet: [ // Up - // spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png.0"], // 0 to 31 - ], - sizeInTiles: { - x:1.2,y:1.4, - }, - positionOffset: { - x:-0.1,y:-0, - } - },{ - spritesheet: [ // Right - // spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png.0"], // 0 to 31 - ], - sizeInTiles: { - x:1.2,y:1.4, - }, - positionOffset: { - x:-0.1,y:-0.2, - } - },{ - spritesheet: [ // Down - // spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png.27"], // 32 to 63 - ], - flip: "y", - sizeInTiles: { - x:1.3,y:1.6, - }, - positionOffset: { - x:-0.15,y:-1.6, - } - },{ - spritesheet: [ // Left - // spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png.27"], // 32 to 63 - ], - flip: "x", - sizeInTiles: { - x:1.3,y:1.6, - }, - positionOffset: { - x:-1.35,y:-0.25, - } - }], - } - for(let i = 0; i < 16; i++){ - if(name == "transport-belt"){ - entityDrawRules[name].spritesheet[3].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+i]); - entityDrawRules[name].spritesheet[2].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+(i+16)]); - entityDrawRules[name].spritesheet[1].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+i]); - entityDrawRules[name].spritesheet[0].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+(i+16)]); - - } else { - entityDrawRules[name].spritesheet[3].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+i]); - entityDrawRules[name].spritesheet[2].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+(i+32)]); - entityDrawRules[name].spritesheet[1].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+i]); - entityDrawRules[name].spritesheet[0].spritesheet.push(spritesheetJson.frames["images/factorio/entity/"+name+"/"+name+".png."+(i+32)]); - } - } -}); -["splitter", "fast-splitter", "express-splitter"].forEach(splitter => { - entityDrawRules[splitter] = { - rotOffset:180, - } -}); -["underground-belt", "fast-underground-belt", "express-underground-belt"].forEach(underneathie => { - entityDrawRules[underneathie] = { - rotOffset: 270, - } -}); -["inserter", "fast-inserter", "stack-inserter", "filter-inserter", "burner-inserter", 'long-handed-inserter', 'stack-filter-inserter'].forEach(inserter => { - entityDrawRules[inserter] = { - rotOffset:90, - } -}); -entityDrawRules["oil-refinery"] = { - spritesheet: [ - { - spritesheet: spritesheetJson.frames["images/factorio/entity/oil-refinery/oil-refinery.png.0"], - }, { - spritesheet: spritesheetJson.frames["images/factorio/entity/oil-refinery/oil-refinery.png.1"], - }, { - spritesheet: spritesheetJson.frames["images/factorio/entity/oil-refinery/oil-refinery.png.2"], - }, { - spritesheet: spritesheetJson.frames["images/factorio/entity/oil-refinery/oil-refinery.png.3"], - }], -} -// correct image size for the oil refinery frames (same for all of them on this one) -entityDrawRules["oil-refinery"].spritesheet.forEach(object => { - object.sizeInTiles = { - x:10, - y:7.5, - } - object.positionOffset = { - x:-2, - y:-2.7, - } -}); -// tree sizes -["tree-02", -"tree-02-red", -"tree-03", -"tree-04", -"tree-05", -"tree-07", -"tree-09", -"tree-09-brown", -"tree-09-red"].forEach(tree => { - entityDrawRules[tree] = { - sizeInTiles: { - x: 3, - y: 4, - }, - positionOffset: { - x: -0.6, - y: -2.6, - } - } -}); diff --git a/static/lib/utility.js b/static/lib/utility.js index 930f9653..f9f909ba 100644 --- a/static/lib/utility.js +++ b/static/lib/utility.js @@ -7,47 +7,3 @@ export function getParameterByName(name, url) { if(!results[2]) return ''; return decodeURIComponent(results[2].replace(/\+/g, " ")); } - -// ctx, entityImages[name].img, xPos, yPos, size.x, size.y, rotation, sprWidth, sprHeight, offLeft, offTop -export function drawImageWithRotation(ctx, image, x, y, w, h, degrees, sprWidth, sprHeight, offLeft, offTop, flipX, flipY){ - ctx.save(); - ctx.translate(x+w/2, y+h/2); - ctx.rotate(degrees*Math.PI/180.0); - ctx.translate(-x-w/2, -y-h/2); - if(flipX) { - x -= x*2+sprWidth; - ctx.scale(-1, 1); - } - if(flipY){ - y -= y*2+sprHeight; - ctx.scale(1, -1); - } - - if(isImageOk(image) && sprWidth != undefined && sprHeight != undefined && offLeft != undefined && offTop != undefined){ - // console.log(sprWidth+" "+sprHeight+" "+offLeft+" "+offTop+" "+w+" "+h) - ctx.drawImage(image, offLeft, offTop, sprWidth, sprHeight, x, y, w, h); - } else if(isImageOk(image)){ - ctx.drawImage(image, x, y, w, h); - } else { - // console.log("Image is not OK! "+image.src); - } - ctx.restore(); -} -export function isImageOk(img) { - // During the onload event, IE correctly identifies any images that - // weren't downloaded as not complete. Others should too. Gecko-based - // browsers act like NS4 in that they report this incorrectly. - if (!img.complete) { - return false; - } - - // However, they do have two very useful properties: naturalWidth and - // naturalHeight. These give the true size of the image. If it failed - // to load, either of these should be zero. - if (typeof img.naturalWidth != "undefined" && img.naturalWidth == 0) { - return false; - } - - // No other way of checking: assume it's ok. - return true; -} \ No newline at end of file From be02e5701218b7564bca8ae3ff779a5d0c1d837c Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 28 Oct 2019 19:28:38 +0100 Subject: [PATCH 12/13] Remove mkdirp dependency Replace usage of mkdirp with fs-extra.mkdirp and remove the mkdirp dependency and the unused mkdirp-promise dependency. --- lib/npmPostinstall.js | 5 ++--- package.json | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/npmPostinstall.js b/lib/npmPostinstall.js index a57c2f82..bdd539bf 100644 --- a/lib/npmPostinstall.js +++ b/lib/npmPostinstall.js @@ -1,12 +1,11 @@ -var mkdirp = require("mkdirp"); var http = require('http'); var https = require('https'); -var fs = require('fs'); +var fs = require('fs-extra'); function makeDirectory(path){ return new Promise((resolve, reject) => { - mkdirp(path, function(err){ + fs.mkdirp(path, function(err){ if(err){ console.error("Unable to mkdirp "+path); console.error(err); diff --git a/package.json b/package.json index e1aa09f2..960e0a8b 100644 --- a/package.json +++ b/package.json @@ -53,8 +53,6 @@ "lib": "file:./lib", "luamin": "^1.0.4", "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "mkdirp-promise": "^5.0.1", "moment": "^2.22.2", "needle": "^2.4.0", "node-factorio-api": "^0.3.8", From 283d47d88b666dd8ae4e7eb60fcaa5ce0be7b67f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 25 Oct 2019 00:17:14 +0200 Subject: [PATCH 13/13] Debug ci failure Output factorio integration log to the travis log. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 31d07777..49ad77c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,4 +7,5 @@ before_install: - npm install -g codecov script: - npm run-script ci-cover + - cat test/temp/integration/log.txt - codecov -f coverage/lcov.info