diff --git a/lib/database.js b/lib/database.js new file mode 100644 index 00000000..69f7ae93 --- /dev/null +++ b/lib/database.js @@ -0,0 +1,231 @@ +/** + * Implementation of master databases + * + * @module + * @author Hornwitser + */ + +"use strict"; +const fs = require("fs-extra"); + + +/** + * Converts a Map with only string keys to a JavaScript object. + * + * @param {Map} map - the Map to convert to an object. + * @return {object} - Object with the mapping's key-values as properties. + * @throws {Error} if there are non-string keys in map. + */ +function mapToObject(map) { + let obj = {}; + for (let [key, value] of map) { + if (typeof key !== 'string') { + throw new Error( + `Expected all keys to be string but got ${typeof key}` + ); + } + + obj[key] = value; + } + return obj; +} + +/** + * Load JSON file as a Map + * + * Loads the JSON file specified by path an converts it into a JavaScript Map. + * Returns an empty Map if the file does not exist. + * + * @param {string} filePath - The path to the JSON file to load. + * @throws {Error} If JSON file did not contain an object. + * @throws {Error} If an error occured reading the file. + */ +async function loadJsonAsMap(filePath) { + try { + let content = await fs.readFile(filePath); + let parsed = JSON.parse(content); + + if (parsed instanceof Array) { + throw new Error(`Expected object but got array`); + } + + if (parsed === null) { + throw new Error(`Expected object but got null`); + } + + if (typeof parsed !== "object") { + throw new Error(`Expected object but got ${typeof parsed}`); + } + + return new Map(Object.entries(parsed)); + + } catch (err) { + if (err.code == "ENOENT") { + // File does not exist, return empty map + return new Map(); + + } else { + throw err; + } + } +} + +/** + * Save a Map as a JSON file + * + * Save the content of a key-value Map with only string keys as a JSON file. + * If the directory the file is to be saved into doesn't exist it will be + * created. + * + * @param {string} filePath - The path to the JSON file that will be saved. + * @param {Map} map - Map with only string keys to save. + * @throws {Error} if there are non-string keys in map. + * @throws {Error} if an error occured writing to the file. + */ +async function saveMapAsJson(filePath, map) { + let obj = mapToObject(map); + await fs.outputFile(filePath, JSON.stringify(obj, null, 4)); +} + + +function checkName(name) { + if (typeof name !== "string") { + throw new Error("name must be a string"); + } +} + +function checkCount(count) { + if (typeof count !== "number" || isNaN(count)) { + throw new Error("count must be a number"); + } +} + + +/** + * Item database + * + * Stores counts of items for the master server. Items that have not been + * stored in the database are treated as having a stored count of zero. When + * serialized the database discards entries with a zero counts, and when + * deserialized the content is verified. + */ +class ItemDatabase { + + /** + * Create a new item database + * + * @param {object|undefined} serialized - An object from a previous call to + * {@link module:lib/database~ItemDatabase#serialize} to restore the + * database from. An empty database will be created if this parameter + * is left undefined. + */ + constructor(serialized) { + this._items = new Map(); + + // Verify the content of the serialized database + if (serialized !== undefined) { + for (let [name, count] of Object.entries(serialized)) { + checkName(name); + checkCount(count); + + this._items.set(name, count); + } + } + } + + /** + * Serialize item database + * + * Serialize the item database into a plain JavaScript object that can be + * turned inta a string with JSON.stringify(). + * + * @returns {object} Serialized representation of the database + */ + serialize() { + let obj = {}; + for (let [name, count] of this._items) { + if (count !== 0) { + obj[name] = count; + } + } + return obj; + } + + /** + * Approximate size of the items database. + * + * Does not account for entries being zero in the item database. + */ + get size() { + return this._items.size; + } + + + /** + * Get the count of an item stored in the database + * + * Returns the count of a given item that has been stored in to the + * database. If the item has not been previously stored into the database + * then 0 is returned. + * + * @param {string} name - The name of the item to get the count of. + * @returns {number} The count of the item stored. + */ + getItemCount(name) { + checkName(name); + + if (!this._items.has(name)) { + return 0 + } + + return this._items.get(name); + } + + /** + * Adds count of an item to the database + * + * Add count copies of the item specified by name to the item database. + * + * @param {string} name - The name of the item to add. + * @param {number} count - The count of item to remove. + */ + addItem(name, count) { + checkName(name); + checkCount(count); + + if (this._items.has(name)) { + count += this._items.get(name); + } + + this._items.set(name, count); + }; + + /** + * Removes count of an item from the database. + * + * Remove count copies of the item specified by name from the item database + * If count is greater than the currently stored count the stored item + * count will become negative. + * + * @param {string} name - The name of the item to remove. + * @param {number} count - The count of items to remove. + */ + removeItem(name, count) { + checkName(name); + checkCount(count); + + count = -count; + if (this._items.has(name)) { + count += this._items.get(name); + } + + this._items.set(name, count); + }; +} + +module.exports = { + mapToObject, + loadJsonAsMap, + saveMapAsJson, + ItemDatabase, +} diff --git a/lib/database.spec.js b/lib/database.spec.js new file mode 100644 index 00000000..512de4ad --- /dev/null +++ b/lib/database.spec.js @@ -0,0 +1,212 @@ +const assert = require("assert").strict; +const fs = require("fs-extra"); + +const database = require("lib/database"); + +describe("lib/database", function() { + describe("mapToObject()", function() { + it("should throw on non-string key", function() { + assert.throws( + () => database.mapToObject(new Map([[1, 1]])), + new Error("Expected all keys to be string but got number") + ); + assert.throws( + () => database.mapToObject(new Map([[Symbol(), 1]])), + new Error("Expected all keys to be string but got symbol") + ); + }); + + it("should throw convert a map to an object", function() { + assert.deepEqual( + database.mapToObject(new Map([['a', 1], ['b', true]])), + {'a': 1, 'b': true} + ); + }); + }); + + describe("loadJsonAsMap()", function() { + let badTypes = ['null', 'array', 'number', 'string', 'boolean']; + + for (let type of badTypes) { + it(`should reject on ${type} JSON`, async function() { + await assert.rejects( + database.loadJsonAsMap(`test/json/${type}.json`), + new Error(`Expected object but got ${type}`) + ); + }); + } + + it("should work on empty object JSON", async function() { + assert.deepEqual( + await database.loadJsonAsMap('test/json/object.json'), + new Map() + ); + }); + + it("should work on object JSON", async function() { + assert.deepEqual( + await database.loadJsonAsMap('test/json/load_map.json'), + new Map([['a', 1], ['b', true]]) + ); + }); + + it("should give an empty Map for non-existant file", async function() { + assert.deepEqual( + await database.loadJsonAsMap('test/json/does-not-exist'), + new Map() + ); + }); + }); + + describe("saveMapAsJson()", function() { + it("should save a mapping as JSON", async function() { + let testFile = 'test/json/save_map.json'; + async function deleteTestFile() { + try { + await fs.unlink(testFile); + } catch (err) { + /* istanbul ignore if */ + if (err.code !== 'ENOENT') { + throw err; + } + } + } + + await deleteTestFile(); + await database.saveMapAsJson( + testFile, new Map([['c', {}], ['d', "foo"]]) + ); + + assert.equal( + await fs.readFile(testFile, {encoding: 'utf-8'}), + '{\n "c": {},\n "d": "foo"\n}' + ); + await deleteTestFile(); + }); + }); + + describe("class ItemDatabase", function() { + describe("constructor()", function() { + it("should create an empty database with no args", function() { + let items = new database.ItemDatabase(); + assert.deepEqual(items._items, new Map()); + }); + + it("should restore the passed serialized database", function() { + let items = new database.ItemDatabase({'a': 1, 'b': 2}); + assert.deepEqual(items._items, new Map([['a', 1], ['b', 2]])); + }); + + it("should throw on invalid serialized database", function() { + assert.throws( + () => new database.ItemDatabase({'a': NaN}), + new Error("count must be a number") + ); + + assert.throws( + () => new database.ItemDatabase({'a': 'a'}), + new Error("count must be a number") + ); + }); + }); + + describe(".serialize()", function() { + it("should return a serialized database", function() { + let items = new database.ItemDatabase({'a': 10}); + assert.deepEqual(items.serialize(), {'a': 10}); + }); + + it("should remove zero count entries", function() { + let items = new database.ItemDatabase({'a': 0}); + assert.deepEqual(items.serialize(), {}); + }); + }); + + describe(".size", function() { + it("should give an approximate size of the database", function() { + let items = new database.ItemDatabase({'a': 10}); + assert.equal(items.size, 1); + }); + }); + + describe(".getItemCount()", function() { + it("should return the count of the given item", function() { + let items = new database.ItemDatabase({'a': 10}); + assert.equal(items.getItemCount('a'), 10); + }); + + it("should return zero if item does not exist", function() { + let items = new database.ItemDatabase(); + assert.equal(items.getItemCount('b'), 0); + }); + + it("should throw on invalid name", function() { + let items = new database.ItemDatabase(); + assert.throws( + () => items.getItemCount(2), + new Error("name must be a string") + ); + }); + }); + + describe(".addItem()", function() { + it("should add a new item", function() { + let items = new database.ItemDatabase(); + items.addItem('a', 10); + assert.deepEqual(items._items, new Map([['a', 10]])); + }) + + it("should add an existing item", function() { + let items = new database.ItemDatabase({'a': 10}); + items.addItem('a', 10); + assert.deepEqual(items._items, new Map([['a', 20]])); + }) + + it("should throw on invalid name", function() { + let items = new database.ItemDatabase(); + assert.throws( + () => items.addItem(2, 10), + new Error("name must be a string") + ); + }); + + it("should throw on invalid count", function() { + let items = new database.ItemDatabase(); + assert.throws( + () => items.addItem('a', NaN), + new Error("count must be a number") + ); + }); + }); + + describe(".removeItem()", function() { + it("should remove an existing item", function() { + let items = new database.ItemDatabase({'a': 20}); + items.removeItem('a', 10); + assert.deepEqual(items._items, new Map([['a', 10]])); + }) + + it("should turn a non-existing item negative", function() { + let items = new database.ItemDatabase(); + items.removeItem('a', 10); + assert.deepEqual(items._items, new Map([['a', -10]])); + }) + + it("should throw on invalid name", function() { + let items = new database.ItemDatabase(); + assert.throws( + () => items.removeItem(2, 10), + new Error("name must be a string") + ); + }); + + it("should throw on invalid count", function() { + let items = new database.ItemDatabase(); + assert.throws( + () => items.removeItem('a', 'b'), + new Error("count must be a number") + ); + }); + }); + }); +}); diff --git a/master.js b/master.js index abd61b28..c8d96d59 100644 --- a/master.js +++ b/master.js @@ -34,6 +34,7 @@ const config = require(args.config || './config'); // homebrew modules const generateSSLcert = require("lib/generateSSLcert"); const getFactorioLocale = require("lib/getFactorioLocale"); +const database = require("lib/database"); // homemade express middleware for token auth const authenticate = require("lib/authenticate"); @@ -67,7 +68,7 @@ app.use(function(req, res, next){ res.locals.res = res; res.locals.req = req; res.locals.masterPlugins = masterPlugins; - res.locals.slaves = slaves; + res.locals.slaves = db.slaves; res.locals.moment = moment; next(); }); @@ -146,27 +147,25 @@ app.get('/metrics', (req, res) => { }); function registerMoreMetrics(){ - for(let instanceID in slaves){ + for (let [instanceID, slave] of db.slaves) { // playercount try{ - prometheusPlayerCountGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].playerCount) || 0); + prometheusPlayerCountGauge.labels(instanceID, slave.instanceName).set(Number(slave.playerCount) || 0); }catch(e){} // UPS try{ - prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.UPS) || 60); - if(slaves[instanceID].meta.tick && typeof slaves[instanceID].meta.tick === "number") prometheusUPSGauge.labels(instanceID, slaves[instanceID].instanceName).set(Number(slaves[instanceID].meta.tick) || 0); + prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.UPS) || 60); + if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0); }catch(e){} } // inventory - for(let key in db.items){ - if(typeof db.items[key] == "number" || typeof db.items[key] == "string"){ - prometheusMasterInventoryGauge.labels(key).set(Number(db.items[key]) || 0); - } + for (let [key, count] of db.items._items) { + prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0); } // Slave count let numberOfActiveSlaves = 0; - for(let instance in slaves){ - if(Date.now() - Number(slaves[instance].time) < 1000 * 30) numberOfActiveSlaves++; + for(let slave of db.slaves.values()){ + if(Date.now() - Number(slave.time) < 1000 * 30) numberOfActiveSlaves++; } prometheusConnectedInstancesCounter.set(numberOfActiveSlaves); } @@ -174,49 +173,28 @@ function registerMoreMetrics(){ // set up database const db = {}; -(function(){ - try{ - let x = fs.statSync(path.resolve(config.databaseDirectory, "slaves.json")); - console.log(`loading slaves from path.resolve(config.databaseDirectory, "slaves.json")`); - slaves = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "slaves.json"))); - } catch (e){ - slaves = {}; - } - try{ - x = fs.statSync(path.resolve(config.databaseDirectory, "items.json")); - console.log(`loading items from ${path.resolve(config.databaseDirectory, "items.json")}`); - db.items = JSON.parse(fs.readFileSync(path.resolve(config.databaseDirectory, "items.json"))); - } catch (e){ - db.items = {}; - } -})(); +async function loadDatabase(databaseDirectory) { + let slavesPath = path.resolve(databaseDirectory, "slaves.json"); + console.log(`Loading ${slavesPath}`); + db.slaves = await database.loadJsonAsMap(slavesPath); + + let itemsPath = path.resolve(databaseDirectory, "items.json"); + console.log(`Loading ${itemsPath}`); + try { + let content = await fs.readFile(itemsPath); + db.items = new database.ItemDatabase(JSON.parse(content)); + + } catch (err) { + if (err.code === "ENOENT") { + console.log("Creating new item database"); + db.items = new database.ItemDatabase(); -db.items.addItem = function(object) { - if(object.name == "addItem" || object.name == "removeItem") { - console.error("Fuck you, that would screw everything up if you named your item that."); - return false; - } else { - if(this[object.name] && Number(this[object.name]) != NaN){ - this[object.name] = Number(this[object.name]) + Number(object.count); } else { - this[object.name] = Number(object.count); + throw err; } - return true; } -}; -db.items.removeItem = function(object) { - if(object.name == "addItem" || object.name == "removeItem") { - console.error("Fuck you, that would screw everything up if you named your item that."); - return false; - } else { - if(this[object.name] && Number(this[object.name]) != NaN){ - this[object.name] = Number(this[object.name]) - Number(object.count); - } else { - this[object.name] = 0; - } - return true; - } -}; +} + // store slaves and inventory in a .json full of JSON data async function shutdown() { @@ -224,17 +202,21 @@ async function shutdown() { let exitStartTime = Date.now(); try { // set insane limit to slave length, if its longer than this we are probably being ddosed or something - if(slaves && Object.keys(slaves).length < 2000000){ - console.log("saving to slaves.json"); - fs.writeFileSync(path.resolve(config.databaseDirectory, "slaves.json"), JSON.stringify(slaves)); - } else if(slaves) { - console.log("Slave database too large, not saving ("+Object.keys(slaves).length+")"); + if (db.slaves && db.slaves.size < 2000000) { + let file = path.resolve(config.databaseDirectory, "slaves.json"); + console.log(`writing ${file}`); + await database.saveMapAsJson(file, db.slaves); + } else if (db.slaves) { + console.error(`Slave database too large, not saving (${db.slaves.size}`); } - if(db.items && Object.keys(db.items).length < 50000){ - console.log("saving to items.json"); - fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items)); - } else if(slaves) { - console.log("Item database too large, not saving ("+Object.keys(slaves).length+")"); + + if (db.items && db.items.size < 50000) { + let file = path.resolve(config.databaseDirectory, "items.json"); + console.log(`writing ${file}`); + let content = JSON.stringify(db.items.serialize()); + await fs.outputFile(file, content); + } else if (db.items) { + console.error(`Item database too large, not saving (${db.items.size})`); } for(let i in masterPlugins){ let plugin = masterPlugins[i]; @@ -266,15 +248,16 @@ app.post("/api/getID", authenticate.middleware, function(req,res) { // time us a unix timestamp we can use to check for how long the server has been unresponsive // we should save that somewhere and give appropriate response if(req.body){ - if(!slaves[req.body.unique]){ - slaves[req.body.unique] = {}; + if (!db.slaves.has(req.body.unique)) { + db.slaves.set(req.body.unique, {}); } - for(k in req.body){ + let slave = db.slaves.get(req.body.unique); + for(let k in req.body){ if(k != "meta" && req.body.hasOwnProperty(k)){ - slaves[req.body.unique][k] = req.body[k]; + slave[k] = req.body[k]; } } - // console.log("Slave registered: " + slaves[req.body.unique].mac + " : " + slaves[req.body.unique].serverPort+" at " + slaves[req.body.unique].publicIP + " with name " + slaves[req.body.unique].instanceName); + // console.log("Slave registered: " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP + " with name " + slave.instanceName); } }); /** @@ -289,16 +272,17 @@ app.post("/api/editSlaveMeta", authenticate.middleware, function(req,res) { // {instanceID, pass, meta:{x,y,z}} if(req.body && req.body.instanceID && req.body.meta){ - if(slaves[req.body.instanceID]){ - if(!slaves[req.body.instanceID].meta) { - slaves[req.body.instanceID].meta = {}; + if (db.slaves.has(req.body.instanceID)) { + let slave = db.slaves.get(req); + if(!slave.meta) { + slave.meta = {}; } - slaves[req.body.instanceID].meta = deepmerge(slaves[req.body.instanceID].meta, req.body.meta, {clone:true}); + slave.meta = deepmerge(slave.meta, req.body.meta, {clone:true}); let metaPortion = JSON.stringify(req.body.meta); if(metaPortion.length > 50) { metaPortion = metaPortion.substring(0,20) + "..."; } - // console.log("Updating slave "+slaves[req.body.instanceID].instanceName+": " + slaves[req.body.instanceID].mac + " : " + slaves[req.body.instanceID].serverPort+" at " + slaves[req.body.instanceID].publicIP, metaPortion); + // console.log("Updating slave "+slave.instanceName+": " + slave.mac + " : " + slave.serverPort+" at " + slave.publicIP, metaPortion); res.sendStatus(200); } else { res.send("ERROR: Invalid instanceID or password") @@ -316,12 +300,13 @@ app.post("/api/getSlaveMeta", function (req, res) { endpointHitCounter.labels(req.route.path).inc(); console.log("body", req.body); if(req.body && req.body.instanceID){ - if (slaves[req.body.instanceID]) { + if (db.slaves.has(req.body.instanceID)) { + let slave = db.slaves.get(req.body.instanceID); console.log("returning meta for ", req.body.instanceID); - if(!slaves[req.body.instanceID].meta) { - slaves[req.body.instanceID].meta = {}; + if(!slave.meta) { + slave.meta = {}; } - res.send(JSON.stringify(slaves[req.body.instanceID].meta)) + res.send(JSON.stringify(slave.meta)) } else { res.status(404); res.send('{"status": 404, "info": "Slave not registered"}') @@ -387,7 +372,7 @@ let slaveCache = { }; function getSlaves() { if(!slaveCache.cache || Date.now() - slaveCache.timestamp > 5000) { - let copyOfSlaves = JSON.parse(JSON.stringify(slaves)); + let copyOfSlaves = JSON.parse(JSON.stringify(database.mapToObject(db.slaves))); slaveCache.cache = copyOfSlaves; slaveCache.timestamp = Date.now(); } @@ -460,11 +445,12 @@ app.post("/api/place", authenticate.middleware, function(req, res) { } if(!x.instanceID) { x.instanceID = "unknown"; - Object.keys(slaves).forEach(instanceID => { - if(slaves[instanceID].instanceName == req.body.instanceName) { + for (let [instanceID, slave] of db.slaves) { + if(slave.instanceName == req.body.instanceName) { x.instanceID = instanceID; + break; } - }); + } } if(x.instanceID && x.instanceName @@ -490,10 +476,7 @@ app.post("/api/place", authenticate.middleware, function(req, res) { }); prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); // save items we get - db.items.addItem({ - name:req.body.name, - count:req.body.count - }); + db.items.addItem(req.body.name, req.body.count); // Attempt confirming res.send("success"); @@ -502,12 +485,8 @@ app.post("/api/place", authenticate.middleware, function(req, res) { } }); const routes_api_remove = require("./routes/api/remove.js"); -var neuralDole = null; +let neuralDole; -if(config.useNeuralNetDoleDivider)//Only initialize neural network when it's enabled, otherwise it might override gauge - neuralDole=new routes_api_remove.neuralDole({ - items: db.items, gaugePrefix: prometheusPrefix - }); /** POST endpoint to remove items from DB when client orders items. @@ -531,22 +510,22 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { if(!object.instanceName) { object.instanceName = "unknown"; } - if(slaves[object.instanceID]) object.instanceName = slaves[object.instanceID].instanceName; - let item = db.items[object.name]; - // console.dir(doc); - if((item === undefined) - ||(( - config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" ) - && object.name != "electricity") - ){ + if (db.slaves.has(object.instanceID)) { + object.instanceName = db.slaves.get(object.instanceID).instanceName; + } + let itemCount = db.items.getItemCount(object.name); + if ( + (config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" ) + && object.name != "electricity" + ) { if(config.logItemTransfers){ console.log('failure could not find ' + object.name); } res.send({name:object.name, count:0}); } else if(config.disableFairItemDistribution){ // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing. - let numberToRemove = Math.min(Math.abs(Number(object.count)),Number(item)); - db.items.removeItem({count: numberToRemove, name: object.name}); + let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount); + db.items.removeItem(object.name, numberToRemove); res.send({count: numberToRemove, name: object.name}); // track statistics and do graphing things @@ -577,7 +556,7 @@ app.post("/api/remove", authenticate.middleware, function(req, res) { } else { // Use dole division. Makes it really slow to drain out the last little bit. routes_api_remove.doleDivider({ - item, + itemCount, object, db, sentItemStatisticsBySlaveID, @@ -604,10 +583,8 @@ app.get("/api/inventory", function(req, res) { res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); // Check it and send it var inventory = []; - for(let key in db.items){ - if(typeof db.items[key] == "number" || typeof db.items[key] == "string"){ - inventory.push({name:key, count:db.items[key]}); - } + for (let [name, count] of db.items._items) { + inventory.push({ name, count }); } res.send(JSON.stringify(inventory)); }); @@ -624,7 +601,7 @@ app.get("/api/inventoryAsObject", function(req, res) { res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); // Check it and send it - res.send(JSON.stringify(db.items)); + res.send(JSON.stringify(db.items.serialise())); }); /** @@ -906,8 +883,11 @@ async function startServer() { fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { expiresIn: 86400*365 // expires in 1 year })); - setInterval(()=>{ - fs.writeFileSync(path.resolve(config.databaseDirectory, "items.json"), JSON.stringify(db.items)); + setInterval(async function() { + let file = path.resolve(config.databaseDirectory, "items.json"); + console.log(`autosaving ${file}`); + let content = JSON.stringify(db.items.serialize()); + await fs.outputFile(file, content); },config.autosaveInterval || 60000); process.on('SIGINT', shutdown); // ctrl + c process.on('SIGHUP', shutdown); // terminal closed @@ -921,8 +901,16 @@ async function startServer() { // mod downloads app.use(express.static(masterModFolder)); + await loadDatabase(config.databaseDirectory); authenticate.setAuthSecret(config.masterAuthSecret); + // Only initialize neural network when it's enabled, otherwise it might override gauge + if(config.useNeuralNetDoleDivider) { + neuralDole=new routes_api_remove.neuralDole({ + db, gaugePrefix: prometheusPrefix + }); + } + // Make sure we're actually going to listen on a port let httpPort = args.masterPort || config.masterPort; let httpsPort = args.sslPort || config.sslPort; @@ -973,6 +961,9 @@ async function startServer() { module.exports = { app, + + // For testing only + _db: db, } if (module === require.main) { diff --git a/master.spec.js b/master.spec.js index 6c229611..31d490f5 100644 --- a/master.spec.js +++ b/master.spec.js @@ -7,9 +7,12 @@ const jwt = require("jsonwebtoken"); const master = require('./master.js'); const config = require('./config.json'); const authenticate = require('lib/authenticate'); +const database = require('lib/database'); authenticate.setAuthSecret(config.masterAuthSecret); +master._db.slaves = new Map(); +master._db.items = new database.ItemDatabase(); const token = jwt.sign( { id: "api" }, config.masterAuthSecret, { expiresIn: 600 }); @@ -25,7 +28,7 @@ async function getValidate(path) { assert(filtered.length === 0, "there are HTML errors on the page, please fix: "+JSON.stringify(validation.messages, null, 4)); } -describe('Master server endpoint testing', function() { +describe('Master testing', function() { describe('#GET /api/getFactorioLocale', function() { it('should get the basegame factorio locale', async function() { let res = await get('/api/getFactorioLocale'); @@ -86,19 +89,31 @@ describe('Master server endpoint testing', function() { assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`); }); }); - it("returns an empty itemStack if you try to request addItem or removeItem", function() { - function remove() { - return persistentMaster.post("/api/remove").send({ - name:"addItem", - count:10 + it("works correctly for items named addItem or removeItem", async function() { + function add(item) { + return persistentMaster.post("/api/place").send({ + name: item, + count: 10 }).set( "X-Access-Token", token ).expect(200).then(function(res) { - assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0"); + assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`); + }) + } + + function remove(item) { + return persistentMaster.post("/api/remove").send({ + name: item, + count: 10 + }).set( + "X-Access-Token", token + ).expect(200).then(function(res) { + assert.deepEqual(res.body, {"name":item,"count":10}); }); } - return Promise.all([remove(), remove()]); + await Promise.all([add("addItem"), add("removeItem")]); + await Promise.all([remove("addItem"), remove("removeItem")]); }); it("returns an empty itemStack if you don't have any of the item you request", function() { return persistentMaster.post("/api/remove").send({ diff --git a/routes/api/remove.js b/routes/api/remove.js index e9d1952c..4de5ba18 100644 --- a/routes/api/remove.js +++ b/routes/api/remove.js @@ -19,7 +19,7 @@ class neuralDole { } constructor({ - items, gaugePrefix + db, gaugePrefix }){ // Set some storage variables for the dole divider this.prometheusNNDoleGauge = new Prometheus.Gauge({ @@ -27,8 +27,8 @@ class neuralDole { help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given', labelNames: ["itemName"], }); - this.items = items - this.itemsLastTick = JSON.parse(JSON.stringify(items)) + this.db = db + this.itemsLastTick = new Map(db.items._items) this.dole = {} this.carry = {} this.lastRequest = {} @@ -36,15 +36,11 @@ class neuralDole { this.debt= {} setInterval(()=>{ - for(let name in this.items){ - // The items object have 2 functions tied to it - addItem and removeItem. A for loop will reveal those properties. - if(typeof this.items[name] !== "number") continue - - let count = this.items[name] + for (let [name, count] of this.db.items._items) { let magicData = doleNN.Tick( count, this.dole[name], - this.itemsLastTick[name], + this.itemsLastTick.get(name), this.getRequestStats(name,5) ) this.stats[name]=this.stats[name] || [] @@ -55,7 +51,7 @@ class neuralDole { // DONE handle magicData[1] for graphing for our users this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0); } - this.itemsLastTick = JSON.parse(JSON.stringify(this.items)) + this.itemsLastTick = new Map(this.db.items._items) }, 1000) } divider({ @@ -67,8 +63,8 @@ class neuralDole { }){ let magicData = doleNN.Dose( object.count, // numReq - this.items[object.name], - this.itemsLastTick[object.name] || 0, + this.db.items.getItemCount(object.name), + this.itemsLastTick.get(object.name) || 0, this.dole[object.name], this.carry[object.name+" "+object.instanceID] || 0, this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0, @@ -87,46 +83,40 @@ class neuralDole { this.dole[object.name] = magicData[1] this.carry[object.name+" "+object.instanceID] = magicData[2] this.debt[object.name+" "+object.instanceID] = magicData[3] - - // Remove item from DB and send it - if(this.items.removeItem({count: magicData[0], name: object.name})){ - if(config.logItemTransfers){ - console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items[object.name] + " and sent to " + object.instanceID + " | " + object.instanceName); - } - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }, console.log); - } - sentItemStatistics.add({ - key:object.name, - value:magicData[0], - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; - prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0); - res.send({ - name: object.name, - count: magicData[0], - }) - } else { - res.send({ - name: object.name, - count: 0, - }) - } + // Remove item from DB and send it + this.db.items.removeItem(object.name, magicData[0]); + if(config.logItemTransfers){ + console.log("removed: " + object.name + " " + magicData[0] + " . " + this.db.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName); + } + let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; + if(sentItemStatistics === undefined){ + sentItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + } + sentItemStatistics.add({ + key:object.name, + value:magicData[0], + }); + //console.log(sentItemStatistics.data) + sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; + prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0); + + res.send({ + name: object.name, + count: magicData[0], + }) } } _doleDivisionFactor = {}; //If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. function doleDivider({ - item, + itemCount, object, - db, + db, sentItemStatisticsBySlaveID, config, prometheusDoleFactorGauge, @@ -139,34 +129,35 @@ function doleDivider({ const originalCount = Number(object.count) || 0; object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation; object.count = Math.round(object.count); - if(item.length > 40) console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${item} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${Number(item) > Number(object.count)?'stocked':'short'}.`); - - // Update existing items if item name already exists - if(Number(item) > Number(object.count)) { + if (itemCount > 40) { + console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`); + } + + // Update existing items if item name already exists + if (itemCount > object.count) { //If successful, increase dole _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1; - if(config.logItemTransfers){ - console.log("removed: " + object.name + " " + object.count + " . " + item + " and sent to " + object.instanceID + " | " + object.instanceName); - } - if(db.items.removeItem({count: object.count, name: object.name})){ - let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; - if(sentItemStatistics === undefined){ - sentItemStatistics = new averagedTimeSeries({ - maxEntries: config.itemStats.maxEntries, - entriesPerSecond: config.itemStats.entriesPerSecond, - mergeMode: "add", - }, console.log); - } - sentItemStatistics.add({ - key:object.name, - value:object.count, - }); - //console.log(sentItemStatistics.data) - sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; - } - + if (config.logItemTransfers) { + console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName); + } + db.items.removeItem(object.name, object.count) + let sentItemStatistics = sentItemStatisticsBySlaveID[object.instanceID]; + if(sentItemStatistics === undefined){ + sentItemStatistics = new averagedTimeSeries({ + maxEntries: config.itemStats.maxEntries, + entriesPerSecond: config.itemStats.entriesPerSecond, + mergeMode: "add", + }, console.log); + } + sentItemStatistics.add({ + key:object.name, + value:object.count, + }); + //console.log(sentItemStatistics.data) + sentItemStatisticsBySlaveID[object.instanceID] = sentItemStatistics; + prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); - prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(object.count) || 0); + prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0); res.send({count: object.count, name: object.name}); } else { // if we didn't have enough, attempt giving out a smaller amount next time diff --git a/sharedPlugins/serverManager/static/index.html b/sharedPlugins/serverManager/static/index.html index 45d4af12..5af40484 100644 --- a/sharedPlugins/serverManager/static/index.html +++ b/sharedPlugins/serverManager/static/index.html @@ -13,10 +13,10 @@ html = ""; if(req.query && req.query.instanceID - && slaves[req.query.instanceID] - && slaves[req.query.instanceID].mods + && locals.slaves.get(req.query.instanceID) + && locals.slaves.get(req.query.instanceID).mods && serverManager.serverSockets[req.query.instanceID]){ - let slave = slaves[req.query.instanceID]; + let slave = locals.slaves.get(req.query.instanceID); html += `