From 21d9fdff7ba54acea3fc9fbf6529526d129c97b0 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Dec 2019 21:14:26 +0100 Subject: [PATCH 01/11] Remove old plugin system Remove the old client/instance fused plugin system along with instanceManagement in client.js and getPlugins in master.js. --- CHANGELOG.md | 2 + client.js | 137 ----------------------------------- lib/manager/pluginManager.js | 125 -------------------------------- master.js | 39 +--------- package.json | 1 - test/missing.js | 1 - 6 files changed, 3 insertions(+), 302 deletions(-) delete mode 100644 lib/manager/pluginManager.js diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b64fc8..8074d6fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,8 @@ Version 2.0.0 - Removed the hello event from the socket connection handshake. Breaks playerManager, trainTeleports, serverSelect, and discordChat - Changed the format of database/slaves.json. +- Removed the output file subscription system. Breaks inventoryImports, + playerManager, trainTeleports, serverSelect and researchSync. Version 1.2.4 diff --git a/client.js b/client.js index d533b5a2..f7bfbe45 100644 --- a/client.js +++ b/client.js @@ -1,27 +1,15 @@ const fs = require('fs-extra'); -const Tail = require('tail').Tail; -const needle = require("needle"); -const child_process = require('child_process'); const path = require('path'); -const request = require("request"); -const deepmerge = require("deepmerge"); -const ioClient = require("socket.io-client"); -const util = require("util"); const yargs = require("yargs"); const version = require("./package").version; // internal libraries -const objectOps = require("lib/objectOps"); const fileOps = require("lib/fileOps"); -const pluginManager = require("lib/manager/pluginManager"); const hashFile = require('lib/hash').hashFile; const factorio = require("lib/factorio"); -const schema = require("lib/schema"); const link = require("lib/link"); const errors = require("lib/errors"); -// Uhm... -var global = {}; /** * Keeps track of the runtime parameters of an instance @@ -186,7 +174,6 @@ class Instance extends link.Link{ clientPort: this.server.rconPort, clientPassword: this.server.rconPassword, } - //instanceManagement(slaveConfig, this, compatConfig, this.server, socket); // XXX async function }); await this.server.start(saveName); @@ -739,130 +726,6 @@ async function startClient() { */ } -// ensure instancemanagement only ever runs once -var _instanceInitialized; -async function instanceManagement(slaveConfig, instance, instanceconfig, server, socket) { - if (_instanceInitialized) return; - _instanceInitialized = true; - - let compatUrl = slaveConfig.masterUrl; - if (compatUrl.endsWith("/")) { - compatUrl = compatUrl.slice(0, -1); - } - let compatConfig = { - name: slaveConfig.name, - masterURL: compatUrl, - masterAuthToken: slaveConfig.masterToken, - publicIP: slaveConfig.publicAddress, - instanceDirectory: slaveConfig.instancesDir, - factorioDirectory: slaveConfig.factorioDir, - } - - console.log("Started instanceManagement();"); - - // load plugins and execute onLoad event - let pluginsToLoad = await pluginManager.getPlugins(); - let plugins = []; - - for(let i = 0; i < pluginsToLoad.length; i++){ - let pluginLoadStarted = Date.now(); - let combinedConfig = deepmerge(instanceconfig,compatConfig,{clone:true}); - combinedConfig.instanceName = instance.name; - let pluginConfig = pluginsToLoad[i]; - - if(!global.subscribedFiles) { - global.subscribedFiles = {}; - } - if (pluginConfig.enabled) { - // require plugin class and execute it - let pluginClass = require(path.resolve(pluginConfig.pluginPath, "index")); - plugins[i] = new pluginClass(combinedConfig, async function(data, callback){ - if(data && data.toString('utf8')[0] != "/") { - console.log("Clusterio | "+ pluginsToLoad[i].name + " | " + data.toString('utf8')); - return true; - } else if (data && data.toString('utf8')[0] == "/"){ - let result = await server.sendRcon(data.toString('utf8')); - if (typeof callback === "function") { - callback(result); - } - return result; - } - }, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat. - socket, // socket.io connection to master (and ES6 destructuring, yay) - }); - if(plugins[i].factorioOutput && typeof plugins[i].factorioOutput === "function"){ - // when factorio logs a line, send it to the plugin. This includes things like autosaves, chat, errors etc - server.on('stdout', data => plugins[i].factorioOutput(data.toString())); - } - if(pluginConfig.scriptOutputFileSubscription && typeof pluginConfig.scriptOutputFileSubscription == "string"){ - if(global.subscribedFiles[pluginConfig.scriptOutputFileSubscription]) { - // please choose a unique file to subscribe to. If you need plugins to share this interface, set up a direct communication - // between those plugins instead. - throw "FATAL ERROR IN " + pluginConfig.name + " FILE ALREADY SUBSCRIBED " + pluginConfig.scriptOutputFileSubscription; - } - - let outputPath = instance.path( - "script-output", - pluginConfig.scriptOutputFileSubscription - ); - if (!fs.existsSync(outputPath)) { - // Do something - fs.writeFileSync(outputPath, ""); - } - global.subscribedFiles[pluginConfig.scriptOutputFileSubscription] = true; - console.log("Clusterio | Registered file subscription on "+outputPath); - - - if(!pluginConfig.fileReadDelay || pluginConfig.fileReadDelay == 0) { - // only wipe the file on restart for now, should most likely be rotated during runtime too - fs.writeFileSync(outputPath, ""); - let tail = new Tail(outputPath); - tail.on("line", function (data) { - plugins[i].scriptOutput(data); - }); - } else { - fs.watch(outputPath, fileChangeHandler); - // run once in case a plugin wrote out information before the plugin loaded fully - // delay, so the socket got enough time to connect - setTimeout(() => { - fileChangeHandler(false, pluginConfig.scriptOutputFileSubscription); - }, 500); - - // send file contents to plugin for processing - function fileChangeHandler(eventType, filename) { - if (filename != null) { - setTimeout( - () => { - // get array of lines in file - let stuff = fs.readFileSync(instance.path("script-output", filename), "utf8").split("\n"); - - // if you found anything, reset the file - if (stuff[0]) { - fs.writeFileSync(instance.path("script-output", filename), ""); - } - for (let o = 0; o < stuff.length; o++) { - if (stuff[o] && !stuff[o].includes('\u0000\u0000')) { - try { - plugins[i].scriptOutput(stuff[o]); - } catch (e) { - console.error(e) - } - } - } - }, - pluginConfig.fileReadDelay || 0 - ); - } - } - } - } - console.log(`Clusterio | Loaded plugin ${pluginsToLoad[i].name} in ${Date.now() - pluginLoadStarted}ms`); - } else { - // this plugin doesn't have a client portion. Maybe it runs on the master only? - } - } -} // END OF INSTANCE START --------------------------------------------------------------------- - // string, function // returns [{modName:string,hash:string}, ... ] function hashMods(instance, callback) { diff --git a/lib/manager/pluginManager.js b/lib/manager/pluginManager.js deleted file mode 100644 index f0d03ca3..00000000 --- a/lib/manager/pluginManager.js +++ /dev/null @@ -1,125 +0,0 @@ -const fs = require("fs-extra"); -const path = require("path"); -const git = require("simple-git")("./sharedPlugins"); - - -async function listPlugins() { - let plugins = await this.getPlugins(); - plugins.forEach(plug => { - console.log(`${plug.name}`.padEnd(22).substr(0,22)+` - ${plug.description}`.substr(0,80-22)); - }); -} - -async function getPlugins() { - let directories = await fs.readdir("./sharedPlugins"); - let returnArray = []; - for(let i in directories){ - let dir = directories[i]; - let pluginContents = await fs.readdir(path.join("./sharedPlugins", dir)); - // console.log(pluginContents) - if(pluginContents.includes("config.js") || pluginContents.includes("config.json")){ - try{ - var pluginConfig = require(path.join("./../../sharedPlugins", dir, "config")); - pluginConfig.pluginPath = path.join("./sharedPlugins", dir); - }catch(e){ - console.log(e) - } - - // check if plugin is enabled - try{ - await fs.stat(path.join(pluginConfig.pluginPath, "DISABLED")) - pluginConfig.enabled = false; - }catch(e){ - pluginConfig.enabled = true; - } - returnArray.push(pluginConfig); - } - } - return returnArray; -} - -function addPlugin(path) { - return new Promise((resolve, reject) => { - if(path.includes("git")){ // we are decently sure this is a git repo - git.clone(path, data => resolve(data)) - } - }); -} - -async function removePlugin(name) { - let plugins = await this.getPlugins(); - for(let i in plugins){ - let plug = plugins[i]; - if(plug.name.includes(name)){ - // console.log(`Removing plugin ${plug.name}`); - await fs.remove(plug.pluginPath); - return { - ok:true, - msg:`Deleted plugin ${plug.name}` - } - } - } - return { - ok:false, - msg:`Plugin matching "${name}" not found`, - } -} - -async function enablePlugin(name, instance) { - let plugins = await this.getPlugins(); - let plugin = plugins[findInArray("name", name, plugins)[0]]; - if(plugin){ - let files = await fs.readdir(plugin.pluginPath); - if(files.includes("DISABLED")){ - await fs.unlink(path.join(plugin.pluginPath, "DISABLED")); - return { - ok:true, - msg:`Globally enabled plugin ${name}`, - } - } else { - return { - ok:true, - msg:`Plugin ${name} already enabled`, - } - } - } else { - return { - ok:false, - msg:`Found no excact match for plugin ${name}`, - } - } -} - -async function disablePlugin(name, instance) { - let plugins = await this.getPlugins(); - let plugin = plugins[findInArray("name", name, plugins)[0]]; - if(plugin){ - await fs.writeFile(path.join(plugin.pluginPath, "DISABLED"), ""); - return { - ok:true, - msg:`Plugin ${name} is now disabled on this host`, - } - } else { - return { - ok:false, - msg:`Found no excact match for plugin ${name}`, - } - } -} - -function findInArray(key, value, array) { - let indexes = []; - for(let i in array){ - if(array[i][key] && array[i][key] === value) indexes.push(i); - } - return indexes; -} - -module.exports = { - listPlugins, - getPlugins, - addPlugin, - removePlugin, - enablePlugin, - disablePlugin, -}; diff --git a/master.js b/master.js index c2970929..593f7b88 100644 --- a/master.js +++ b/master.js @@ -32,7 +32,6 @@ let config = {}; // homebrew modules const generateSSLcert = require("lib/generateSSLcert"); -const pluginManager = require("lib/manager/pluginManager"); const database = require("lib/database"); const factorio = require("lib/factorio"); const schema = require("lib/schema"); @@ -502,46 +501,10 @@ io.on('connection', function (socket) { var masterPlugins = []; async function pluginManagement(){ let startPluginLoad = Date.now(); - masterPlugins = await getPlugins(); + // masterPlugins = await getPlugins(); console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms"); } -async function getPlugins(){ - let plugins = []; - let pluginsToLoad = await pluginManager.getPlugins(); - for(let i = 0; i < pluginsToLoad.length; i++){ - let pluginStartedLoading = Date.now(); // just for logging - let stats = await fs.stat(pluginsToLoad[i].pluginPath); - if(stats.isDirectory()){ - let pluginConfig = pluginsToLoad[i]; - if(pluginConfig.masterPlugin && pluginConfig.enabled){ - let masterPlugin = require(path.resolve(pluginConfig.pluginPath, pluginConfig.masterPlugin)); - plugins.push({ - main:new masterPlugin({ - config, - pluginConfig, - path: pluginConfig.pluginPath, - socketio: io, - express: app, - db, - Prometheus, - prometheusPrefix, - endpointHitCounter, - }), - pluginConfig, - }); - - console.log("Loaded plugin "+pluginConfig.name+" in "+(Date.now() - pluginStartedLoading)+"ms"); - } - } - } - for(let i in plugins){ - let plugin = plugins[i]; - if(plugin.main.onLoadFinish && typeof plugin.main.onLoadFinish == "function") await plugin.main.onLoadFinish({plugins}); - } - return plugins; -} - /** * Calls listen on server capturing any errors that occurs * binding to the port. diff --git a/package.json b/package.json index a56560de..85995813 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "socket.io-client": "^2.2.0", "socket.io-prometheus": "^0.2.1", "sync-request": "^4.0.1", - "tail": "^2.0.2", "typescript-collections": "^1.3.2", "yargs": "^14.2.0" }, diff --git a/test/missing.js b/test/missing.js index b93e9446..836dd6af 100644 --- a/test/missing.js +++ b/test/missing.js @@ -2,7 +2,6 @@ // required here in order to track coverage for them. require('lib/manager/modManager'); -require('lib/manager/pluginManager'); require('lib/npmPostinstall.js'); require('lib/typedef'); require('../clusterctl.js'); From e5115df28b1d993433e8c60b448800068967df6f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Dec 2019 21:37:27 +0100 Subject: [PATCH 02/11] Implement broadcast for events Add broadcastTo property to events that broadcasts them to all entities of the specified target that's downstream to the connection it was sent to. --- client.js | 17 +++++++++++++++++ lib/link/messages.js | 33 ++++++++++++++++++++++++++++++++- master.js | 10 ++++++++++ test/lib/link/messages.js | 6 ++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index f7bfbe45..9bd28fb6 100644 --- a/client.js +++ b/client.js @@ -300,6 +300,17 @@ class InstanceConnection extends link.Link { async forwardEventToMaster(message, event) { event.send(this.slave, message.data); } + + async broadcastEventToInstance(message, event) { + for (let instanceConnection of this.slave.instanceConnections.values()) { + if (instanceConnection === this) { + continue; // Do not broadcast back to the source + } + + console.log("Broadcasting event"); + event.send(instanceConnection, message.data); + } + } } class SlaveConnector extends link.SocketIOClientConnector { @@ -377,6 +388,12 @@ class Slave extends link.Link { return await request.send(instanceConnection, message.data); } + async broadcastEventToInstance(message, event) { + for (let instanceConnection of this.instanceConnections.values()) { + event.send(instanceConnection, message.data); + } + } + async createInstanceRequestHandler(message) { let { id, options } = message.data; if (this.instanceInfos.has(id)) { diff --git a/lib/link/messages.js b/lib/link/messages.js index 7a597221..a385262f 100644 --- a/lib/link/messages.js +++ b/lib/link/messages.js @@ -313,19 +313,30 @@ class Event { * @param {?string} forwardTo - * Optional target to forward this request to. Only 'master' is * currently supported. + * @param {?string} broadcastTo - + * Optional target to broadcast this event to. Only 'instance' is + * currently supported. An event will only be broadcast towards + * links that are downstream, so sending an event set to instance + * broadcast that's sent to a slave will only be broadcast to that + * slave's instances. * @param {Object} eventProperties - * Mapping of property values to JSON schema specifications for the * properties that are valid in the data payload of this event. */ - constructor({ type, links, forwardTo = null, eventProperties = {} }) { + constructor({ type, links, forwardTo = null, broadcastTo = null, eventProperties = {} }) { this.type = type; this.links = links; this.forwardTo = forwardTo; + this.broadcastTo = broadcastTo; if (forwardTo !== 'master' && forwardTo !== null) { throw new Error(`Invalid forwardTo value ${forwardTo}`); } + if (broadcastTo !== 'instance' && broadcastTo !== null) { + throw new Error(`Invalid broadcastTo value ${broadcastTo}`); + } + this.eventType = type + '_event'; this._eventValidator = schema.compile({ $schema: "http://json-schema.org/draft-07/schema#", @@ -349,6 +360,9 @@ class Event { * If forwardTo was set to 'master' the `forwardEventToMaster` method on * the link is used as the default handler. * + * If broadcastTo was set to 'instance' the `broadcastEventToInstance` + * method on the link is invoked before the handler. BUT HOW? + * * Does nothing if the link is not a target for this event. * * @param {Link} link - The link to attach to. @@ -367,6 +381,23 @@ class Event { } } + if ( + this.broadcastTo === 'instance' + && [ + 'instance-slave', 'slave-master', 'control-master', 'master-slave' + ].includes(`${link.target}-${link.source}`) + ) { + if (!handler) { + handler = link.broadcastEventToInstance; + } else { + let originalHandler = handler; + handler = async (message, event) => { + await link.broadcastEventToInstance(message, event); + await originalHandler.call(link, message, event); + } + } + } + if (!handler) { throw new Error(`Missing handler for ${this.eventType} on ${link.source}-${link.target} link`); } diff --git a/master.js b/master.js index 593f7b88..244e8f73 100644 --- a/master.js +++ b/master.js @@ -267,6 +267,16 @@ class BaseConnection extends link.Link { return await request.send(connection, message.data); } + + async broadcastEventToInstance(message, event) { + for (let slaveConnection of slaveConnections.values()) { + if (slaveConnection === this) { + continue; // Do not broadcast back to the source + } + + event.send(slaveConnection, message.data); + } + } } let controlConnections = new Array(); diff --git a/test/lib/link/messages.js b/test/lib/link/messages.js index 3f9ab742..21d03629 100644 --- a/test/lib/link/messages.js +++ b/test/lib/link/messages.js @@ -115,6 +115,12 @@ describe("lib/link/messages", function() { new Error("Invalid forwardTo value invalid") ); }); + it("should throw on invalid broadcastTo", function() { + assert.throws( + () => new link.Event({ broadcastTo: 'invalid'}), + new Error("Invalid broadcastTo value invalid") + ); + }); }); describe(".attach()", function() { From 513651393bdae06eb42cb96af0e778476e9363a8 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 14 Dec 2019 17:31:02 +0100 Subject: [PATCH 03/11] Implement forwarding to instances for events Implement forward to instance for Event messages. --- client.js | 24 ++++++++++++++++++++++++ lib/link/messages.js | 21 +++++++++++++++------ master.js | 10 ++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/client.js b/client.js index 9bd28fb6..324d04be 100644 --- a/client.js +++ b/client.js @@ -297,6 +297,20 @@ class InstanceConnection extends link.Link { link.attachAllMessages(this); } + async forwardEventToInstance(message, event) { + let instanceId = message.data.instance_id; + if (!this.slave.instanceInfos.has(instanceId)) { + // Instance is probably on another slave + await this.slave.forwardEventToMaster(message, event); + return; + } + + let instanceConnection = this.slave.instanceConnections.get(instanceId); + if (!instanceConnection) { return; } + + event.send(instanceConnection, message.data); + } + async forwardEventToMaster(message, event) { event.send(this.slave, message.data); } @@ -388,6 +402,16 @@ class Slave extends link.Link { return await request.send(instanceConnection, message.data); } + async forwardEventToInstance(message, event) { + let instanceId = message.data.instance_id; + if (!this.instanceInfos.has(instanceId)) { return; } + + let instanceConnection = this.instanceConnections.get(instanceId); + if (!instanceConnection) { return; } + + event.send(instanceConnection, message.data); + } + async broadcastEventToInstance(message, event) { for (let instanceConnection of this.instanceConnections.values()) { event.send(instanceConnection, message.data); diff --git a/lib/link/messages.js b/lib/link/messages.js index a385262f..bd76e4c9 100644 --- a/lib/link/messages.js +++ b/lib/link/messages.js @@ -311,8 +311,9 @@ class Event { * Links this event is valid on. Takes an array of strings * containing "-" specifications. * @param {?string} forwardTo - - * Optional target to forward this request to. Only 'master' is - * currently supported. + * Optional target to forward this request to. Supported values are + * 'master' and 'instance'. If set to 'instance' an instance_id + * property is implicity added to eventProperties. * @param {?string} broadcastTo - * Optional target to broadcast this event to. Only 'instance' is * currently supported. An event will only be broadcast towards @@ -329,7 +330,12 @@ class Event { this.forwardTo = forwardTo; this.broadcastTo = broadcastTo; - if (forwardTo !== 'master' && forwardTo !== null) { + if (forwardTo === 'instance') { + eventProperties = Object.assign( + {"instance_id": { type: "integer" }}, eventProperties + ); + + } else if (forwardTo !== 'master' && forwardTo !== null) { throw new Error(`Invalid forwardTo value ${forwardTo}`); } @@ -358,10 +364,11 @@ class Event { * for the event. * * If forwardTo was set to 'master' the `forwardEventToMaster` method on - * the link is used as the default handler. + * the link is used as the default handler. If it was set to 'instance' + * the `forwardEventToInstance` method on the link is used by default. * * If broadcastTo was set to 'instance' the `broadcastEventToInstance` - * method on the link is invoked before the handler. BUT HOW? + * method on the link is invoked before the handler. * * Does nothing if the link is not a target for this event. * @@ -376,7 +383,9 @@ class Event { if (this.links.includes(`${link.target}-${link.source}`)) { // Use forwarder if handler is not present. if (!handler) { - if (this.forwardTo === 'master') { + if (this.forwardTo === 'instance') { + handler = link.forwardEventToInstance; + } else if (this.forwardTo === 'master') { handler = link.forwardEventToMaster; } } diff --git a/master.js b/master.js index 244e8f73..e1aaa713 100644 --- a/master.js +++ b/master.js @@ -268,6 +268,16 @@ class BaseConnection extends link.Link { return await request.send(connection, message.data); } + async forwardEventToInstance(message, event) { + let instance = db.instance.get(message.data.instance_id); + if (!instance) { return; } + + let connection = slaveConnection.get(instance.slaveId); + if (!connection) { return; } + + event.send(connection, message.data); + } + async broadcastEventToInstance(message, event) { for (let slaveConnection of slaveConnections.values()) { if (slaveConnection === this) { From bcceae687f9a8a54bb7b3f57a5664e3405c61ab3 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 14 Dec 2019 17:48:56 +0100 Subject: [PATCH 04/11] Reimplement plugin system Implement a new plugin system based subclassing pre-defined base clases and communicating with pre-defined link messages. --- CHANGELOG.md | 8 + client.js | 58 ++++++- docs/writing-plugins.md | 303 ++++++++++++++++++++++++++++++++++ lib/errors.js | 10 ++ lib/plugin.js | 126 ++++++++++++++ master.js | 63 +++++-- package.json | 4 +- test/client.js | 23 +-- test/integration/clusterio.js | 3 +- test/lib/plugin.js | 124 ++++++++++++++ 10 files changed, 688 insertions(+), 34 deletions(-) create mode 100644 docs/writing-plugins.md create mode 100644 lib/plugin.js create mode 100644 test/lib/plugin.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 8074d6fd..f8eab618 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ Version 2.0.0 - Rewritten the communication between slaves and the master from scratch. The new system is based on a WebSocket connection between the slaves and the master server and provides efficient validated bi-directonal communication. +- Rewritten the plugin system from scratch. Plugins now inherit from + a base class and use the same WebSocket connection Clusterio uses to + communicate. ### Changes @@ -100,6 +103,11 @@ Version 2.0.0 - Changed the format of database/slaves.json. - Removed the output file subscription system. Breaks inventoryImports, playerManager, trainTeleports, serverSelect and researchSync. +- Removed the factorioOutput hook from instance plugins. The onOutput + hook provides parsed output instead. +- Removed the onLoadFinish hook from master plugins. +- Moved plugins from the sharedPlugins directory to plugins directory. +- Implemented a new plugin system that replaces the old. Breaks all plugins. Version 1.2.4 diff --git a/client.js b/client.js index 324d04be..1b3ccabd 100644 --- a/client.js +++ b/client.js @@ -8,6 +8,7 @@ const fileOps = require("lib/fileOps"); const hashFile = require('lib/hash').hashFile; const factorio = require("lib/factorio"); const link = require("lib/link"); +const plugin = require("lib/plugin"); const errors = require("lib/errors"); @@ -20,6 +21,8 @@ class Instance extends link.Link{ link.attachAllMessages(this); this._dir = dir; + this.plugins = new Map(); + // This is expected to change with the config system rewrite this.config = { id: instanceConfig.id, @@ -41,11 +44,34 @@ class Instance extends link.Link{ this.server.on('output', (output) => { link.messages.instanceOutput.send(this, { instance_id: this.config.id, output }) + + for (let [name, plugin] of this.plugins) { + plugin.onOutput(output).catch(err => { + console.error(`Plugin ${name} raised error in onOutput:`, err); + }); + } }); } - async init() { + async init(pluginInfos) { await this.server.init(); + + // load plugins + for (let pluginInfo of pluginInfos) { + if (!pluginInfo.enabled || !pluginInfo.instanceEntrypoint) { + continue; + } + + // require plugin class and initialize it + let pluginLoadStarted = Date.now(); + let { InstancePlugin } = require(`./plugins/${pluginInfo.name}/${pluginInfo.instanceEntrypoint}`); + let instancePlugin = new InstancePlugin(pluginInfo, this); + await instancePlugin.init(); + this.plugins.set(pluginInfo.name, instancePlugin); + plugin.attachPluginMessages(this, pluginInfo, instancePlugin); + + console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`); + } } /** @@ -133,8 +159,8 @@ class Instance extends link.Link{ // plugins that are not disabled. This will most likely change in the // future when the plugin refactor is done. let modules = []; - for (let pluginName of await fs.readdir("sharedPlugins")) { - let pluginDir = path.join("sharedPlugins", pluginName); + for (let pluginName of await fs.readdir("plugins")) { + let pluginDir = path.join("plugins", pluginName); if (await fs.pathExists(path.join(pluginDir, "DISABLED"))) { continue; } @@ -177,6 +203,10 @@ class Instance extends link.Link{ }); await this.server.start(saveName); + + for (let pluginInstance of this.plugins.values()) { + await pluginInstance.onStart(); + } } /** @@ -185,6 +215,10 @@ class Instance extends link.Link{ async stop() { // XXX this needs more thought to it if (this.server._state === "running") { + for (let pluginInstance of this.plugins.values()) { + await pluginInstance.onStop(); + } + await this.server.stop(); } } @@ -295,6 +329,10 @@ class InstanceConnection extends link.Link { super('slave', 'instance', connector); this.slave = slave; link.attachAllMessages(this); + + for (let pluginInfo of slave.pluginInfos) { + plugin.attachPluginMessages(this, pluginInfo, null); + } } async forwardEventToInstance(message, event) { @@ -355,9 +393,15 @@ class SlaveConnector extends link.SocketIOClientConnector { class Slave extends link.Link { // I don't like God classes, but the alternative of putting all this state // into global variables is not much better. - constructor(connector, slaveConfig) { + constructor(connector, slaveConfig, pluginInfos) { super('slave', 'master', connector); link.attachAllMessages(this); + + this.pluginInfos = pluginInfos; + for (let pluginInfo of pluginInfos) { + plugin.attachPluginMessages(this, pluginInfo, null); + } + this.config = { id: slaveConfig.id, name: slaveConfig.name, @@ -449,7 +493,7 @@ class Slave extends link.Link { let instance = new Instance( connectionClient, instanceInfo.path, this.config.factorioDir, instanceInfo.config ); - await instance.init(); + await instance.init(this.pluginInfos); // XXX: race condition on multiple simultanious calls this.instanceConnections.set(instanceId, instanceConnection); @@ -709,7 +753,6 @@ async function startClient() { const slaveConfig = JSON.parse(await fs.readFile(args.config)); await fs.ensureDir(slaveConfig.instanceDirectory); - await fs.ensureDir("sharedPlugins"); await fs.ensureDir("sharedMods"); // Set the process title, shows up as the title of the CMD window on windows @@ -737,8 +780,9 @@ async function startClient() { return; } + let pluginInfos = await plugin.getPluginInfos("plugins"); let slaveConnector = new SlaveConnector(slaveConfig); - let slave = new Slave(slaveConnector, slaveConfig); + let slave = new Slave(slaveConnector, slaveConfig, pluginInfos); // Handle interrupts let secondSigint = false diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md new file mode 100644 index 00000000..b1d19ac9 --- /dev/null +++ b/docs/writing-plugins.md @@ -0,0 +1,303 @@ +Writing Plugins +=============== + +Plugins for Clusterio are classes written in JavaScript that run under +Node.js. The plugin classes have pre-defined hooks that are called +during various stages and operations of Clusterio. + + +Plugin Structure +---------------- + +The basic file structure of a plugin is the following. + + plugin_name/ + +- info.js + +- package.json + +- master.js + +- instance.js + +- test/ + | +- plugin.js + +- lua/ + +- plugin.lua + +The most important file is the `info.js` file. Without it the plugin +will not recognized by Clusterio. Here's an example of it: + + const link = require('lib/link'); // For messages + + module.exports = { + name: "foo_frobber", + title: "Foo Frobber", + description: "Does advance frobnication", + version: "0.8.1", + instanceEntrypoint: "instance", + masterEntrypoint: "master", + messages: { + /* See below */ + }, + }; + +The following properties are recognized: + +**name**: + Internal name of the plugin. Must be the same as the plugin's + directory name + +**title**: + Name of the plugin as shown to users. Currently not used. + +**description**: + Brief description of what the plugin does. Currently not used. + +**version**: + Semver compatible version string of the plugin. Currently not used. + +**instanceEntrypoint**: + Path to a Node.js module relative to the plugin directory which + contains the InstancePlugin class definition for this plugin. This + is an optional paramater. A plugin can be for instances only but + for it to be able to send any messages to other instances it will + still have to be loaded on the master server. + +**masterEntrypoint**: + Path to a Node.js module relative to the plugin directory which + contains the MasterPlugin class definiton for this plugin. This is + an optional parameter. A plugin can be made that only runs on the + master server. + +**messages**: + Object with link messages definitions for this plugin. See guide + for [defining link messages](#defining-link-messages) below. + +While there is no standard for how to organize a plugin it's recommended +to put the MasterPlugin class definition into master.js and the +InstancePlugin class definition into instance.js. You can put them into +whatever file you want (even the same one for both) + +For both instanceEntrypoint and masterEntrypoint the path should not end +with .js and it should use forward slashes for directory sepparators if +any. + + +Defining the plugin class +------------------------- + +The plugin class should derive from its respective base class defined in +`lib/plugin`. For example to define a MasterPlugin class the following +code can be used: + + const plugin = require('lib/plugin'); + + class MasterPlugin extends plugin.BaseMasterPlugin { + async init() { + this.foo = 42; + await this.startFrobnication(); + } + + // ... + } + + module.exports = { + MasterPlugin, + } + +For the instance plugin it's exactly the same except "Master" is +replaced with "Instance". The available hooks that you can override is +documented on the base class [in lib/plugin.js](../lib/plugin.js). + +It's best to avoid defining a constructor, but if you insist on defining +one forward all arguments to be the base class. E.g.: + + constructor(...args) { + super(...args); + + // Code here + } + +The arguments passed may change, and attempting to modify them will +result in unpredicatable behaviour. The async init method always called +immediatly after the constructor so there's little reason to do this. + + +Defining Link Messages +---------------------- + +You will most likely have to communicate with the master or other +instances in your plugin for it to do anything useful. For this there's +a WebSocket communication channel established between the slaves and the +master server that plugins can define their own messages to send over +it. This channel is bi-directional and all messages send over it are +validated with a JSON schema (see [this guide][guide] for an +introduction to writing JSON schema.) + +[guide]: https://json-schema.org/learn/getting-started-step-by-step.html + +There are currently two kinds of messages that can be defined. Events +and requests. Events are simple one way notifications that invoke a +handler on target it's sent to. Requests are pairs of request and +response messages where the request is sent to the target and the +response is the reply back from the target. The requests are similar to +HTTP requests only that both parties of a link may innitiate one. + + +### Defining Events + +Events are defined as properties of the messages object exported by +`info.js` that map to instances of the `Event` class from `lib/link`. +The name of the property correspond to the handler invoked on the plugin +class. The Event constructor takes an object of properties that define +the event, for example the following could be defined in `info.js`: + + messages: { + startFrobnication: new link.Event({ + type: 'foo_frobber:start_frobnication', + links: ['master-slave', 'slave-instance'], + forwardTo: 'instance', + eventProperties: { + "frobnication_type": { type: "string" }, + }, + }), + }, + +This specifies an event that can be sent from the master to a slave, +and from a slave to an instance. And that the event must contain the +property `frobnication_type` with a string value in the data payload. +It will also be forwarded by slaves to a specific instance. + +The following properties are recognized by the Event constructor: + +**type**: + The message type sent over the wire. This can be any string but + it must be unique across all plugins and it's recommended that it is + of format `plugin_name:message_name`. The suffix `_event` will be + appened to the type. + +**links**: + An array of strings describing which links this event can be sent + over. Direction matters, `'master-slave'` means the event can be + sent from the master to slave, but can't be sent back the other way + unless `'slave-master'` is also present in the links array. + +**forwardTo**: + Target to forward event to. Can be either `'master'` to indicate a + slave should forward it to the master server or `'instance'` to + indicate it should be forwarded to the instances specified by the + `instance_id` event property. This works by using a default handler + for the event at the links that forward it. + +**broadcastTo**: + Target to broadcast this message towards. Currently only + `'instance'` is supported and means the event will be broadcast to + all instances downstream of the target it's sent to, but not back + from where it came from. This means that sending an event to a + slave from an instance will cause it to be broadcast to all + instances of that slave except for the instance it came from. + +**eventProperties**: + Object with properties mapping to a JSON schema of that property + that specifies what's valid to send in the event. This is + equivalent to using the `properties` keyword in JSON schema except + that the properties specified are implicitly required and additional + properties are not allowed. See [this guide][guide] for an + introduction to writing JSON schemas + +The forwardTo and broadcastTo can be combined such that specifying +`'master'` as the forwardTo value and `'instance'` as the broadcastTo +value will cause the event to be broadcast to all instances in the +cluster. For this to work you will need to specify instance-slave, +slave-master and master-slave, slave-instance as the links. + +Keep in mind when forwarding events that if the target an event is being +forwarded to is not online the event will be dropped. If you need a +confirmation that the message was received use a request. + + +### Definining Requests + +Requests are defined as properties of the messages object exported by +`info.js` that map to instances of the `Request` class from `lib/link`. +The name of the property correspond to the handler invoked on the plugin +class. The Request constructor takes an object of properties that define +the event, for example the following could be defined in `info.js`: + + messages: { + reportFrobnication: new link.Request({ + type: 'foo_frobber:report_frobnication', + links: ['master-slave', 'slave-instance'], + forwardTo: 'instance', + requestProperties: { + "verbosity": { type: "integer" }, + }, + responseProperties: { + "report": { + type: "array", + items: { type: "string" }, + }, + }, + }), + }, + +This specifies a request that can be sent from the master to a slave, +and from a slave to an instance. The request data must contain the +property `verbosity` with an integer number as the value as well as the +`instance_id` property (implied by `forwardTo: 'instance'`), and the +response sent must contain a `report` property mapping to an array of +strings. When received by a slave it will also be forwarded to the +instance specified by `instance_id`. + +The following properties are recognized by the Request constructor: + +**type**: + The message type sent over the wire. This can be any string but + it must be unique across all plugins and it's recommended that it is + of format `plugin_name:message_name`. The suffix `_request` will be + appened to the type for the request message sent, and the suffix + `_response` will be appended to the type for the response. + +**links**: + An array of strings describing which links this request can be sent + over. Direction matters, `'master-slave'` means the request can be + sent from the master to slave and the slave can reply to the master, + but the slave can't sent a request to the master unless + `'slave-master'` is also present in the links array. + +**forwardTo**: + Target to forward request to. Can be either `'master'` to indicate + a slave should forward it to the master server when receiving it + from an instance or `'instance'` to indicate it should be forwarded + to the instances specified by the `instance_id` event property. + This works by using a default handler for the event at the links + that forward it. + +**requestProperties**: + Object with properties mapping to a JSON schema of that property + that specifies what's valid to send in the request. This is + equivalent to using the `properties` keyword in JSON schema except + that the properties specified are implicitly required and additional + properties are not allowed. See [this guide][guide] for an + introduction to writing JSON schemas + +**responseProperties**: + Same as the requestProperties only for the response sent back by the + target. + + +Sending Link Messages +--------------------- + +Link messages are sent by calling the `.send()` method on the +Event/Request instance with the link you want to send it over and the +data to send. For instance plugins the link is the instance itself +which is accessible through the `.instance` property on the plugin. The +`.info` property of the plugin class exposes the data exported from the +plugin's `info.js` module. In other words: + + // In an InstancePlugin class + async frobnicate() { + this.info.messages.exampleEvent.send(this.instance, { foo: "bar" }); + } + +For the Request class the send method is async and returns the response +data received from the target it was sent to, or throws an error if the +request failed. diff --git a/lib/errors.js b/lib/errors.js index 9e9676f9..45dd309a 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -15,10 +15,20 @@ class StartupError extends Error { } // Errors outside of our control. class EnvironmentError extends Error { } +// Errors caused by plugins +class PluginError extends Error { + constructor(pluginName, original) { + super(`PluginError: ${original.message}`); + this.pluginName = pluginName; + this.original = original; + } +} + module.exports = { CommandError, RequestError, InvalidMessage, StartupError, EnvironmentError, + PluginError, } diff --git a/lib/plugin.js b/lib/plugin.js new file mode 100644 index 00000000..f0b5c8fc --- /dev/null +++ b/lib/plugin.js @@ -0,0 +1,126 @@ +"use strict"; +const fs = require("fs-extra"); +const path = require("path"); + +const errors = require("lib/errors"); + + +/** + * Base class for instance plugins + */ +class BaseInstancePlugin { + constructor(info, instance) { + this.info = info; + this.instance = instance; + } + + /** + * Called immediately after the class is instantiated + */ + async init() { } + + /** + * Called after the Factorio server is started + */ + async onStart() { } + + /** + * Called before the Factorio server is stopped + * + * This will not be called if for example the Factorio server crashes or + * is killed. + */ + async onStop() { } + + /** + * Called when the instance is shutting down + */ + async onExit() { } + + /** + * Called when the Factorio outputs a line + * + * Called when the Factorio server outputs a line on either stdout or + * stderr. The output is a parsed form of the line and contains the + * following properties (those marked with ? are optional): + * + * - source: Where the message came from, one of "stdout" and "stderr". + * - format: Timestamp format, one of "date", "seconds" and "none". + * - time?: Timestamp of the message. Not present if format is "none". + * - type: Type of message, one of "log", "action" and "generic". + * - level?: Log level for "log" type. i.e "Info" normally. + * - file?: File reported for "log" type. + * - action?: Kind of action for "action" type. i.e "CHAT" for chat. + * - message: Main content of the line. + */ + async onOutput(output) { } +} + +class BaseMasterPlugin { + constructor(info) { + this.info = info; + } + + /** + * Called immediately after the class is instantiated + */ + async init() { } + + /** + * Called when the master server is shutting down + */ + async onExit() { } +} + +async function getPluginInfos(baseDir) { + let plugins = []; + for (let pluginDir of await fs.readdir(baseDir)) { + let pluginInfo; + + try { + // Note: Require path is relative to this module + pluginInfo = require(path.posix.join('..', baseDir, pluginDir, 'info')); + + } catch (err) { + if (err.code === 'MODULE_NOT_FOUND') { + continue; + } else { + throw new errors.PluginError(pluginDir, err); + } + } + + if (pluginInfo.name !== pluginDir) { + throw new errors.EnvironmentError( + `Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})` + ); + } + + pluginInfo.enabled = !await fs.exists(path.join(baseDir, pluginDir, "DISABLED")); + plugins.push(pluginInfo); + } + return plugins; +} + +function attachPluginMessages(link, pluginInfo, plugin) { + let messageDefinitions = pluginInfo.messages || []; + for (let [name, messageFormat] of Object.entries(messageDefinitions)) { + let handler = name + messageFormat.constructor.name + 'Handler'; + if (plugin === null || !plugin[handler]) { + messageFormat.attach(link); + + } else { + messageFormat.attach(link, async function(message, format) { + return await plugin[handler](message, format, this); + }); + } + } +} + + +module.exports = { + BaseInstancePlugin, + BaseMasterPlugin, + + getPluginInfos, + attachPluginMessages, +}; diff --git a/master.js b/master.js index e1aaa713..3e01741d 100644 --- a/master.js +++ b/master.js @@ -37,6 +37,7 @@ const factorio = require("lib/factorio"); const schema = require("lib/schema"); const link = require("lib/link"); const errors = require("lib/errors"); +const plugin = require("lib/plugin"); // homemade express middleware for token auth const authenticate = require("lib/authenticate"); @@ -69,7 +70,6 @@ app.set('views', ['views', 'sharedPlugins']); app.use(function(req, res, next){ res.locals.res = res; res.locals.req = req; - res.locals.masterPlugins = masterPlugins; res.locals.slaves = db.slaves; res.locals.moment = moment; next(); @@ -142,10 +142,8 @@ function registerMoreMetrics(){ } // plugins - for (let plugin of masterPlugins) { - if (plugin.main && plugin.main.onMetrics && typeof plugin.main.onMetrics == "function") { - plugin.main.onMetrics(); - } + for (let masterPlugin of masterPlugins.values()) { + masterPlugin.onMetrics(); } // Slave count @@ -186,13 +184,10 @@ async function shutdown() { await saveMap(config.databaseDirectory, "slaves.json", db.slaves); await saveMap(config.databaseDirectory, "instances.json", db.instances); - for(let i in masterPlugins){ - let plugin = masterPlugins[i]; - if(plugin.main && plugin.main.onExit && typeof plugin.main.onExit == "function"){ - let startTime = Date.now(); - await plugin.main.onExit(); - console.log("Plugin "+plugin.pluginConfig.name+" exited in "+(Date.now()-startTime)+"ms"); - } + for (let [pluginName, masterPlugin] of masterPlugins) { + let startTime = Date.now(); + await masterPlugin.onExit(); + console.log(`Plugin ${pluginName} exited in ${Date.now() - startTime}ms`); } console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms"); process.exit(0); @@ -252,6 +247,9 @@ class BaseConnection extends link.Link { constructor(target, connector) { super('master', target, connector); link.attachAllMessages(this); + for (let masterPlugin of masterPlugins.values()) { + plugin.attachPluginMessages(this, masterPlugin.info, masterPlugin); + } } async forwardRequestToInstance(message, request) { @@ -518,13 +516,43 @@ io.on('connection', function (socket) { }); // handle plugins on the master -var masterPlugins = []; +var masterPlugins = new Map(); async function pluginManagement(){ let startPluginLoad = Date.now(); - // masterPlugins = await getPlugins(); + masterPlugins = await loadPlugins(); console.log("All plugins loaded in "+(Date.now() - startPluginLoad)+"ms"); } +async function loadPlugins() { + let plugins = new Map(); + for (pluginInfo of await plugin.getPluginInfos("plugins")) { + if (!pluginInfo.enabled) { + continue; + } + + let pluginLoadStarted = Date.now(); + let MasterPlugin = plugin.BaseMasterPlugin; + if (pluginInfo.masterEntrypoint) { + ({ MasterPlugin } = require(`./plugins/${pluginInfo.name}/${pluginInfo.masterEntrypoint}`)); + } + + let masterPlugin = new MasterPlugin(pluginInfo); + await masterPlugin.init(); + plugins.set(pluginInfo.name, masterPlugin); + + console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`); + /* + main:new masterPlugin({ + TODO? + config, socketio: io, express: app, + db, Prometheus, prometheusPrefix, endpointHitCounter, + }), + pluginConfig, + });*/ + } + return plugins; +} + /** * Calls listen on server capturing any errors that occurs * binding to the port. @@ -666,6 +694,13 @@ I version of clusterio. Expect things to break. I | Unable to to start master server | +----------------------------------+` ); + } else if (err instanceof errors.PluginError) { + console.error(` +Error: ${err.pluginName} plugin threw an unexpected error + during startup, please report it to the plugin author. +--------------------------------------------------------------` + ); + err = err.original; } else { console.error(` +---------------------------------------------------------------+ diff --git a/package.json b/package.json index 85995813..fcb79778 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "repository": "https://github.com/clusterio/factorioClusterio", "main": "master.js", "scripts": { - "test": "mocha --exit --check-leaks -R spec --recursive test sharedPlugins/*/test", + "test": "mocha --exit --check-leaks -R spec --recursive test plugins/*/test", "fast-test": "FAST_TEST=y npm test", "cover": "nyc npm test", "fast-cover": "FAST_TEST=y nyc npm test", @@ -17,7 +17,7 @@ "nyc": { "exclude": [ "test/**", - "sharedPlugins/*/test/**" + "plugins/*/test/**" ] }, "keywords": [ diff --git a/test/client.js b/test/client.js index d6775bda..e47a73ce 100644 --- a/test/client.js +++ b/test/client.js @@ -8,17 +8,20 @@ const client = require("../client"); describe("Client testing", function() { describe("class Instance", function() { let instance = new client._Instance(new link.VirtualConnector(), "dir", "factorioDir", { name:"foo" }); - it("should give the name on .name", function() { - assert.equal(instance.name, "foo"); - }) + describe(".name", function() { + it("should give the name of the instance", function() { + assert.equal(instance.name, "foo"); + }) + }); - it("should give the path to it on .path()", function() { - assert.equal(instance.path(), "dir"); - }) - - it("should join path on .path(...parts)", function() { - assert.equal(instance.path("bar"), path.join("dir", "bar")); - }) + describe(".path()", function() { + it("should give the path when called without arguments", function() { + assert.equal(instance.path(), "dir"); + }) + it("should join path with arguments", function() { + assert.equal(instance.path("bar"), path.join("dir", "bar")); + }) + }); }); describe("checkFilename()", function() { diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index f4b16077..3bd00e3b 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -3,6 +3,7 @@ const fs = require("fs-extra"); const path = require("path"); const link = require("lib/link"); +const plugin = require("lib/plugin"); const master = require("../../master"); const client = require("../../client"); const clusterctl = require("../../clusterctl"); @@ -62,7 +63,7 @@ describe("Integration of Clusterio", function() { masterURL: "http://invalid", masterAuthToken: "invalid", publicIP: "invalid", - }); + }, await plugin.getPluginInfos("plugins")); testSlaveConnection = new master._SlaveConnection({ agent: "test", version, name: "slave", id: 4}, slaveServer); master._slaveConnections.set(4, testSlaveConnection); }); diff --git a/test/lib/plugin.js b/test/lib/plugin.js new file mode 100644 index 00000000..eaf63b4e --- /dev/null +++ b/test/lib/plugin.js @@ -0,0 +1,124 @@ +const assert = require('assert').strict; +const fs = require('fs-extra'); +const path = require('path'); + +const mock = require('../mock'); +const link = require('lib/link'); +const plugin = require('lib/plugin'); +const errors = require('lib/errors'); + + +describe("lib/plugin", function() { + describe("class BaseInstancePlugin", function() { + let instancePlugin; + it("should be constructible", async function() { + instancePlugin = new plugin.BaseInstancePlugin(); + await instancePlugin.init(); + }) + it("should define defaults for hooks", async function() { + await instancePlugin.onStart(); + await instancePlugin.onStop(); + await instancePlugin.onExit(); + await instancePlugin.onOutput({}); + }) + }); + + describe("class BaseMasterPlugin", function() { + let masterPlugin; + it("should be constructible", async function() { + masterPlugin = new plugin.BaseMasterPlugin(); + await masterPlugin.init(); + }) + it("should define defaults for hooks", async function() { + await masterPlugin.onExit(); + }) + }); + + describe("getPluginInfos()", function() { + let baseDir = path.join('test', 'temp', 'plugin'); + let emptyDir = path.join(baseDir, 'emptyDir'); + let emptyPlugin = path.join(baseDir, 'emptyPlugin'); + let testPlugin = path.join(baseDir, 'testPlugin'); + let disabledPlugin = path.join(baseDir, 'disabledPlugin'); + let brokenPlugin = path.join(baseDir, 'brokenPlugin'); + let invalidPlugin = path.join(baseDir, 'invalidPlugin'); + before(async function() { + await fs.ensureDir(emptyDir); + await fs.ensureDir(path.join(emptyPlugin, 'empty')); + + async function writePlugin(pluginPath, name, infoName = name) { + await fs.outputFile( + path.join(pluginPath, name, 'info.js'), + `module.exports = { name: "${infoName}" };` + ); + } + + await writePlugin(testPlugin, 'test'); + await writePlugin(disabledPlugin, 'disabled'); + await fs.outputFile(path.join(disabledPlugin, 'disabled', 'DISABLED'), ""); + await writePlugin(brokenPlugin, 'broken'); + await fs.outputFile(path.join(brokenPlugin, 'broken', 'info.js'), "Syntax Error"); + await writePlugin(invalidPlugin, 'invalid', 'wrong'); + }); + + it("should return an empty array for an empty directory", async function() { + assert.deepEqual(await plugin.getPluginInfos(emptyDir), []); + }); + it("should ignore plugin dirs without info module", async function() { + assert.deepEqual(await plugin.getPluginInfos(emptyPlugin), []); + }); + it("should discover test plugin", async function() { + assert.deepEqual( + await plugin.getPluginInfos(testPlugin), + [{ name: 'test', enabled: true }] + ); + }); + it("should discover disabled plugin", async function() { + assert.deepEqual( + await plugin.getPluginInfos(disabledPlugin), + [{ name: 'disabled', enabled: false }] + ); + }); + it("should discover disabled plugin", async function() { + assert.deepEqual( + await plugin.getPluginInfos(disabledPlugin), + [{ name: 'disabled', enabled: false }] + ); + }); + it("should reject on broken plugin", async function() { + await assert.rejects( + plugin.getPluginInfos(brokenPlugin), + { message: "PluginError: Unexpected identifier" } + ); + }); + it("should reject on invalid plugin", async function() { + await assert.rejects( + plugin.getPluginInfos(invalidPlugin), + { message: `Plugin dir ${invalidPlugin}/invalid does not match the name of the plugin (wrong)` } + ); + }); + + after(async function() { + await fs.unlink(path.join(brokenPlugin, 'broken', 'info.js')); + }); + }); + + describe("attachPluginMessages()", function() { + let mockLink = new link.Link('source', 'target', new mock.MockConnector()); + let mockEvent = new link.Event({ type: 'test', links: ['target-source'] }); + it("should accept pluginInfo without messages", function() { + plugin.attachPluginMessages(mockLink, {}, null); + }); + it("should attach handler for the given message", function() { + let mockEventEventHandler = function() {}; + plugin.attachPluginMessages(mockLink, { messages: { mockEvent }}, { mockEventEventHandler }); + assert(mockLink._handlers.get('test_event'), "handler was not registered"); + }); + it("should throw if missing handler for the given message", function() { + assert.throws( + () => plugin.attachPluginMessages(mockLink, { messages: { mockEvent }}, {}), + new Error("Missing handler for test_event on source-target link") + ); + }); + }); +}); From b71c392b32fc3ba59204fc0f5c6469303ec929bb Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 14 Dec 2019 17:51:54 +0100 Subject: [PATCH 05/11] Port globalChat to new plugin system Port the globalChat plugin to the new plugin system. Since there's no config system yet the config options for it is removed. --- .gitignore | 1 - CHANGELOG.md | 3 + config.json.dist | 2 - plugins/global_chat/info.js | 23 ++++++ plugins/global_chat/instance.js | 58 +++++++++++++++ plugins/global_chat/test/plugin.js | 91 ++++++++++++++++++++++++ sharedPlugins/globalChat/config.js | 9 --- sharedPlugins/globalChat/index.js | 52 -------------- sharedPlugins/globalChat/test/missing.js | 4 -- test/mock.js | 26 ++++++- 10 files changed, 199 insertions(+), 70 deletions(-) create mode 100644 plugins/global_chat/info.js create mode 100644 plugins/global_chat/instance.js create mode 100644 plugins/global_chat/test/plugin.js delete mode 100644 sharedPlugins/globalChat/config.js delete mode 100644 sharedPlugins/globalChat/index.js delete mode 100644 sharedPlugins/globalChat/test/missing.js diff --git a/.gitignore b/.gitignore index bc8664ff..8803d5f3 100644 --- a/.gitignore +++ b/.gitignore @@ -50,7 +50,6 @@ sharedPlugins/* !sharedPlugins/inventoryImports/ !sharedPlugins/UPSdisplay/ !sharedPlugins/researchSync/ -!sharedPlugins/globalChat/ !sharedPlugins/clusterioMod/ !sharedPlugins/statisticsExporter/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f8eab618..7ff5b68b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,9 @@ Version 2.0.0 - Removed oddball limits to slaves.json size. - Moved slave specific configuration into its own configuration file. - Removed unused binary option from plugin config. +- Removed info and shout command from globalChat plugin +- Removed mirrorAllChat and enableCrossServerShout configuration options for + globalChat plugin. ### Breaking Changes diff --git a/config.json.dist b/config.json.dist index 5481789e..8b8bdb99 100644 --- a/config.json.dist +++ b/config.json.dist @@ -40,8 +40,6 @@ "useNeuralNetDoleDivider": true, "msBetweenCommands": 10, "allowRemoteCommandExecution": true, - "enableCrossServerShout": true, - "mirrorAllChat": false, "__comment7": "Autosave interval for items in the master storage in milliseconds. Defaults to one minute when set to 0 or not set.", "autosaveInterval": 600000, "__comment8": "This setting only affects the master. It makes every import for things that arent 'electricity' return a maximum of 0. Effectively disables teleport chests.", diff --git a/plugins/global_chat/info.js b/plugins/global_chat/info.js new file mode 100644 index 00000000..228bd554 --- /dev/null +++ b/plugins/global_chat/info.js @@ -0,0 +1,23 @@ +let link = require("lib/link"); +let plugin = require("lib/plugin"); + +module.exports = { + name: "global_chat", + title: "Global Chat", + description: "Forwards chat between instances.", + version: "2.0.0-alpha", + instanceEntrypoint: "instance", + + messages: { + chat: new link.Event({ + type: 'global_chat:chat', + links: ['instance-slave', 'slave-master', 'master-slave', 'slave-instance'], + forwardTo: 'master', + broadcastTo: 'instance', + eventProperties: { + "instance_name": { type: "string" }, + "content": { type: "string" }, + }, + }), + }, +} diff --git a/plugins/global_chat/instance.js b/plugins/global_chat/instance.js new file mode 100644 index 00000000..8d85d1c1 --- /dev/null +++ b/plugins/global_chat/instance.js @@ -0,0 +1,58 @@ +const plugin = require("lib/plugin"); + + +/** + * Removes gps and train tags from messags + */ +function removeTags(content) { + return content.replace(/(\[gps=-?\d+,-?\d+\]|\[train=\d+\])/g, ""); +} + +/** + * Escapes a string for inclusion into a lua string + */ +function escapeLuaString(content) { + return content + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/'/g, "\\'") + .replace(/\0/g, "\\0") + .replace(/\n/g, "\\n") + .replace(/\r/g, "\\r") + ; +} + +class InstancePlugin extends plugin.BaseInstancePlugin { + constructor(...args) { + super(...args); + } + + async init() { + // XXX Nothing? + } + + async chatEventHandler(message) { + // TODO check if cross server chat is enabled + let content = `[${message.data.instance_name}] ${removeTags(message.data.content)}`; + await this.instance.server.sendRcon(`/sc game.print('${escapeLuaString(content)}')`); + } + + async onOutput(output) { + if (output.type === "action" && output.action === "CHAT") { + this.info.messages.chat.send(this.instance, { + instance_name: this.instance.name, + content: output.message, + }); + } + } + + // TODO implement info command in lua? +} + +module.exports = { + InstancePlugin, + + // For testing only + _removeTags: removeTags, + _escapeLuaString: escapeLuaString, +} diff --git a/plugins/global_chat/test/plugin.js b/plugins/global_chat/test/plugin.js new file mode 100644 index 00000000..0e1d3de0 --- /dev/null +++ b/plugins/global_chat/test/plugin.js @@ -0,0 +1,91 @@ +const assert = require('assert').strict; + +const mock = require('../../../test/mock'); +const lines = require('../../../test/lib/factorio/lines'); +const instance = require('../instance'); +const info = require('../info'); + + +describe("global_chat plugin", function() { + describe("removeTags()", function() { + it("should pass through an ordinary string", function() { + assert.equal(instance._removeTags("string"), "string"); + }); + it("should strip out gps tag", function() { + assert.equal(instance._removeTags("Look at [gps=12,-4]"), "Look at "); + }); + it("should strip out train tag", function() { + assert.equal(instance._removeTags("Train [train=1235]"), "Train "); + }); + }); + + describe("escapeLuaSrting", function() { + it("should escape backslashes", function() { + assert.equal(instance._escapeLuaString("\\a"), "\\\\a"); + }); + it("should escape double quotes", function() { + assert.equal(instance._escapeLuaString('"a"'), '\\"a\\"'); + }); + it("should escape single quotes", function() { + assert.equal(instance._escapeLuaString("'a'"), "\\'a\\'"); + }); + it("should escape nul bytes", function() { + assert.equal(instance._escapeLuaString("a\0b"), "a\\0b"); + }); + it("should escape newlines", function() { + assert.equal(instance._escapeLuaString("a\nb"), "a\\nb"); + }); + it("should escape carriage return", function() { + assert.equal(instance._escapeLuaString("a\rb"), "a\\rb"); + }); + it("should escape all combined", function() { + assert.equal(instance._escapeLuaString("a\\b\"c'd\0e\nf\rg"), "a\\\\b\\\"c\\'d\\0e\\nf\\rg"); + }); + }); + + describe("class InstancePlugin", function() { + let instancePlugin; + + before(async function() { + instancePlugin = new instance.InstancePlugin(info, new mock.MockInstance()); + await instancePlugin.init(); + }); + + describe(".chatEventHandler()", function() { + it("should send received chat as command", async function() { + instancePlugin.instance.server.rconCommands = []; + await instancePlugin.chatEventHandler({ data: { instance_name: "test", content: "User: message" } }); + assert.deepEqual( + instancePlugin.instance.server.rconCommands, + ["/sc game.print('[test] User: message')"], + ); + }); + }); + describe(".onOutput()", function() { + it("should forward chat", async function() { + let count = 0; + for (let [_, output] of lines.testLines) { + if (output.type === "action" && output.action === "CHAT") { + instancePlugin.instance.connector.sentMessages = []; + await instancePlugin.onOutput(output); + assert(instancePlugin.instance.connector.sentMessages.length, "message was not sent"); + count++; + } + } + assert(count > 0, "no lines were tested"); + }); + it("should ignore regular output", async function() { + let count = 0; + for (let [_, output] of lines.testLines) { + if (output.type !== "action" || output.action !== "CHAT") { + instancePlugin.instance.connector.sentMessages = []; + await instancePlugin.onOutput(output); + assert(!instancePlugin.instance.connector.sentMessages.length, "message was sent"); + count++; + } + } + assert(count > 0, "no lines were tested"); + }); + }) + }); +}); diff --git a/sharedPlugins/globalChat/config.js b/sharedPlugins/globalChat/config.js deleted file mode 100644 index 277032f6..00000000 --- a/sharedPlugins/globalChat/config.js +++ /dev/null @@ -1,9 +0,0 @@ -/* - Clusterio plugin to allow for chat between instances. -*/ -module.exports = { - // Name of package. For display somewhere I guess. - name: "globalChat", - version: "1.0.0", - description: "Clusterio plugin to allow for chat between instances.", -} \ No newline at end of file diff --git a/sharedPlugins/globalChat/index.js b/sharedPlugins/globalChat/index.js deleted file mode 100644 index 6eb03bf3..00000000 --- a/sharedPlugins/globalChat/index.js +++ /dev/null @@ -1,52 +0,0 @@ -module.exports = class remoteCommands { - constructor(mergedConfig, messageInterface, extras){ - this.messageInterface = messageInterface; - this.config = mergedConfig; - this.socket = extras.socket; - this.instances = {}; - - this.socket.emit("registerChatReciever"); - - this.socket.on("gameChat", async data => { - if(data.instanceID && data.instanceID != this.config.unique.toString()){ - if(data.data.includes("[CHAT]") - // if this isn't used as an info command or something (reduce spam) - && !data.data.includes("!info") - // check if we are allowed to cross chat at all, default to true - && (mergedConfig.enableCrossServerShout == undefined || mergedConfig.enableCrossServerShout) - // check if we are supposed to relay this message - && (mergedConfig.mirrorAllChat || data.data.toLowerCase().includes("/shout") || data.data.toLowerCase().includes("!shout"))){ - let words = data.data.trim().split(" "); - words.shift(); - words.shift(); - words.shift(); - let message = words.join(" ").replace(/[";]/g, " ").replace(/[']/g,"").replace("/shout ", "").replace("!shout ","").replace(/\[gps=.*?\]/g, "").replace(/\[train=.*?\]/g, ""); - - if(data.data.includes("[gps=")){ - //Nothing - }else{ - this.messageInterface("/silent-command game.print('[" + data.instanceName + "] " + message + "')"); - } - } - } - }); - } - async factorioOutput(data){ - try{ - this.socket.emit("gameChat", { - instanceID: this.config.unique.toString(), - instanceName: this.config.name, - data, - }); - if(data.includes("[CHAT]") && data.includes("!info")){ - let infoMessage = [ - "=== Server info ===", - "Name: "+await this.getInstanceName(this.config.unique)+" ("+this.config.unique+")", - ]; - infoMessage.forEach((line, i) => { - setTimeout(()=>this.messageInterface("/silent-command game.print('"+line+"')"),i*200); - }); - } - }catch(e){console.log(e)} - } -} diff --git a/sharedPlugins/globalChat/test/missing.js b/sharedPlugins/globalChat/test/missing.js deleted file mode 100644 index 923d82bf..00000000 --- a/sharedPlugins/globalChat/test/missing.js +++ /dev/null @@ -1,4 +0,0 @@ -// These are libraries that does not have any units test, they are -// required here in order to track coverage for them. - -require('../index'); diff --git a/test/mock.js b/test/mock.js index d771a48a..3a1f8f09 100644 --- a/test/mock.js +++ b/test/mock.js @@ -1,5 +1,7 @@ const events = require("events"); +const link = require("lib/link"); + class MockSocket { constructor() { @@ -23,7 +25,7 @@ class MockSocket { close() { this.closeCalled = true; } -}; +} class MockConnector extends events.EventEmitter { constructor() { @@ -50,9 +52,29 @@ class MockConnector extends events.EventEmitter { disconnect() { this.disconnectCalled = true; } -}; +} + +class MockServer { + constructor() { + this.rconCommands = []; + } + + async sendRcon(command) { + this.rconCommands.push(command); + } +} + +class MockInstance extends link.Link { + constructor() { + super('instance', 'slave', new MockConnector()); + this.server = new MockServer(); + this.name = "test"; + } +} + module.exports = { MockSocket, MockConnector, + MockInstance, }; From 8b9bb59357269c6c5ad5232c463d5b654efc08a2 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 14 Dec 2019 17:53:34 +0100 Subject: [PATCH 06/11] Fix fast-test Fix incorrect locations for slowTest in clusterio integration tests and add slowTest to the patch integration tests. Fixes fast-test being broken. --- test/integration/clusterio.js | 10 +++++----- test/integration/patch.js | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 3bd00e3b..38710887 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -88,39 +88,39 @@ describe("Integration of Clusterio", function() { }); describe("create-save", function() { - slowTest(this); it("creates a save", async function() { + slowTest(this); await clusterctl._commands.get("create-save").run({instance: "test"}, testControl); }); }); describe("start-instance", function() { - slowTest(this); it("starts the instance", async function() { + slowTest(this); await clusterctl._commands.get("start-instance").run({instance: "test"}, testControl); // TODO check that the instance actually stopped }); }); describe("send-rcon", function() { - slowTest(this); it("sends the command", async function() { + slowTest(this); await clusterctl._commands.get("send-rcon").run({instance: "test", command: "test"}, testControl); // TODO check that the command was received }); }); describe("stop-instance", function() { - slowTest(this); it("stops the instance", async function() { + slowTest(this); await clusterctl._commands.get("stop-instance").run({instance: "test"}, testControl); // TODO check that the instance actually stopped }); }); describe("delete-instance", function() { - slowTest(this); it("deletes the instance", async function() { + slowTest(this); await clusterctl._commands.get("delete-instance").run({instance: "test"}, testControl); assert(!await fs.exists(path.join(instancesDir, "test")), "Instance was not deleted"); }); diff --git a/test/integration/patch.js b/test/integration/patch.js index 87edc342..25e8b1f8 100644 --- a/test/integration/patch.js +++ b/test/integration/patch.js @@ -10,9 +10,17 @@ const factorio = require("lib/factorio"); require("./server") describe("Integration of lib/factorio/patch", function() { + // Mark that this test depeneds on a test that takes a lot of time. + function slowTest(test) { + if (process.env.FAST_TEST) { + test.skip(); + } + } + describe("patch()", function() { let savePath = path.join("test", "temp", "integration", "saves", "test.zip"); it("should patch a freeplay game", async function() { + slowTest(this); await factorio.patch(savePath, [{ name: "test", files: [{ path: "test.lua", content: "-- test" }] @@ -23,6 +31,7 @@ describe("Integration of lib/factorio/patch", function() { assert.equal(content, "-- test"); }); it("should remove old modules in a save", async function() { + slowTest(this); await factorio.patch(savePath, [{ name: "test", files: [] }]); let zip = await jszip.loadAsync(await fs.readFile(savePath)); assert.equal(zip.file("test/modules/test.lua"), null); From 1f6574a89c705af8247caaa610397dd6e33d5360 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 14 Dec 2019 18:03:40 +0100 Subject: [PATCH 07/11] Improve master shutdown Close active connections and http servers during master shutdown and let node exit on its own. If this fails with an error set output streams to blocking and report it then shutdown with process.exit. Also add double SIGINT handling to immediately shutdown. --- master.js | 86 ++++++++++++++++++++++++++++++++++++++++++++-------- package.json | 1 + 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/master.js b/master.js index 3e01741d..1a7cb3fd 100644 --- a/master.js +++ b/master.js @@ -24,6 +24,7 @@ const crypto = require('crypto'); const base64url = require('base64url'); const moment = require("moment"); const request = require("request"); +const setBlocking = require("set-blocking"); const events = require("events"); const version = require("./package").version; @@ -49,6 +50,8 @@ const cookieParser = require('cookie-parser'); const bodyParser = require("body-parser"); const fileUpload = require('express-fileupload'); var app = express(); +var httpServer; +var httpsServer; app.use(cookieParser()); app.use(bodyParser.json({ @@ -176,9 +179,11 @@ async function saveMap(databaseDirectory, file, map) { } -// store slaves in a .json full of JSON data +/** + * Innitiate shutdown of master server + */ async function shutdown() { - console.log('Ctrl-C...'); + console.log('Shutting down'); let exitStartTime = Date.now(); try { await saveMap(config.databaseDirectory, "slaves.json", db.slaves); @@ -189,12 +194,37 @@ async function shutdown() { await masterPlugin.onExit(); console.log(`Plugin ${pluginName} exited in ${Date.now() - startTime}ms`); } - console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms"); - process.exit(0); - } catch(e) { - console.log(e); - console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms"); - process.exit(1) + + for (let controlConnection of controlConnections) { + controlConnection.connector.close("shutdown"); + } + + for (let slaveConnection of slaveConnections.values()) { + slaveConnection.connector.close("shutdown"); + } + + if (httpServer) { + console.log("Stopping HTTP server"); + await new Promise(resolve => httpServer.close(resolve)); + } + + if (httpsServer) { + console.log("Stopping HTTPS server"); + await new Promise(resolve => httpsServer.close(resolve)); + } + + console.log(`Clusterio cleanly exited in ${Date.now() - exitStartTime}ms`); + + } catch(err) { + setBlocking(true); + console.error(` ++--------------------------------------------------------------------+ +| Unexpected error occured while shutting down master, please report | +| it to https://github.com/clusterio/factorioClusterio/issues | ++--------------------------------------------------------------------+` + ); + console.error(err); + process.exit(1); } } @@ -468,6 +498,19 @@ class SocketIOServerConnector extends events.EventEmitter { return this._seq++; } + /** + * Close the connection with the given reason. + * + * Sends a close message and disconnects the connector. + */ + close(reason) { + this.send('close', { reason }); + this.disconnect(); + } + + /** + * Immediatly close the connection + */ disconnect() { this._socket.disconnect(true); } @@ -495,7 +538,7 @@ io.on('connection', function (socket) { let connection = slaveConnections.get(data.id); if (connection) { console.log(`SOCKET | disconecting existing connection for slave ${data.id}`); - connection.close("Registered from another connection"); + connection.connector.close("Registered from another connection"); } console.log(`SOCKET | registered slave ${data.id} version ${data.version}`); @@ -602,8 +645,25 @@ async function startServer() { fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { expiresIn: 86400*365 // expires in 1 year })); - process.on('SIGINT', shutdown); // ctrl + c - process.on('SIGHUP', shutdown); // terminal closed + + let secondSigint = false + process.on('SIGINT', () => { + if (secondSigint) { + console.log("Caught second interrupt, terminating immediately"); + process.exit(1); + } + + secondSigint = true; + console.log("Caught interrupt signal, shutting down"); + shutdown(); + }); + + // terminal closed + process.on('SIGHUP', () => { + // No graceful cleanup, no warning out (stdout is likely closed.) + // Don't close the terminal with the clusterio master in it. + process.exit(1); + }); config.databaseDirectory = args.databaseDirectory || config.databaseDirectory || "./database"; await fs.ensureDir(config.databaseDirectory); @@ -635,7 +695,7 @@ async function startServer() { // Only start listening for connections after all plugins have loaded if (httpPort) { - let httpServer = require("http").Server(app); + httpServer = require("http").Server(app); await listen(httpServer, httpPort); io.attach(httpServer); console.log("Listening for HTTP on port %s...", httpServer.address().port); @@ -653,7 +713,7 @@ async function startServer() { ); } - let httpsServer = require("https").createServer({ + httpsServer = require("https").createServer({ key: privateKey, cert: certificate }, app) diff --git a/package.json b/package.json index fcb79778..98913802 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "rcon-client": "^4.2.0", "request": "^2.83.0", "sanitizer": "^0.1.3", + "set-blocking": "^2.0.0", "simple-git": "^1.96.0", "single-line-log": "^1.1.2", "socket.io": "^2.2.0", From b4c782410101e8c23cba4b309336d5efdc93d50b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 18 Dec 2019 23:08:33 +0100 Subject: [PATCH 08/11] Add homebrew Prometheus client Add an implementation of a Prometheus client. This is because the prom-client implementation seems to have missunderstood how Prometheus is supposed to work and doesn't allow updating collectors via asynchronous callback. --- lib/prometheus.js | 432 +++++++++++++++++++++++++++++++ test/lib/prometheus.js | 565 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 997 insertions(+) create mode 100644 lib/prometheus.js create mode 100644 test/lib/prometheus.js diff --git a/lib/prometheus.js b/lib/prometheus.js new file mode 100644 index 00000000..2f1386c8 --- /dev/null +++ b/lib/prometheus.js @@ -0,0 +1,432 @@ +class Metric { + constructor(type, name, help, labels = []) { + if (typeof name !== 'string') { + throw new Error("Expected name to be a string"); + } + + if (typeof help !== 'string') { + throw new Error("Expected help to be a string"); + } + + if (!/^[a-zA-Z_:][a-zA-Z0-9_:]*$/.test(name)) { + throw new Error(`Invalid name '${name}'`); + } + + for (let name of labels) { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + throw new Error(`Invalid label '${name}'`); + } + } + + this.name = name; + this.help = help; + this.type = type; + this.labels = labels; + } +} + +let defaultRegistry; + +class Collector { + constructor(register = true) { + if (register) { + defaultRegistry.register(this); + } + } + + async* collect() { } +} + +function escapeLabelValue(value) { + return value + .replace(/\\/g, "\\\\") + .replace(/\n/g, "\\n") + .replace(/"/g, "\\\"") + ; +} + +function labelsToKey(labels, metricLabels) { + let items = [] + if (labels.length === 1 && typeof labels[0] === "object") { + labels = labels[0]; + for (let name of metricLabels) { + if (!Object.hasOwnProperty.call(labels, name)) { + throw new Error(`Missing label '${name}'`); + } + if (typeof labels[name] !== "string") { + throw new Error( + `Expected value for label '${name}' to be a string` + ); + } + items.push(`${name}="${escapeLabelValue(labels[name])}"`); + } + + for (let name of Object.keys(labels)) { + if (!metricLabels.includes(name)) { + throw new Error(`Extra label '${name}'`); + } + } + + } else { + if (metricLabels.length > labels.length) { + throw new Error(`Missing label '${metricLabels[labels.length]}'`); + } + + if (labels.length > metricLabels.length) { + throw new Error(`Extra positional label`); + } + + for (let i=0; i < metricLabels.length; ++i) { + if (typeof labels[i] !== "string") { + throw new Error( + `Expected value for label '${metricLabels[i]}' to be a string` + ); + } + items.push(`${metricLabels[i]}="${escapeLabelValue(labels[i])}"`); + } + } + + return items.join(','); +} + +function keyToLabels(key) { + let labels = new Map(); + + if (key !== '') for (let pair of key.split(',')) { + let [name, value] = pair.split('=', 2); + labels.set(name, value + .slice(1, -1) + .replace(/\\"/g, "\"") + .replace(/\\n/g, "\n") + .replace(/\\\\/g, "\\") + ); + } + + return labels; +} + +class ValueCollector extends Collector { + constructor(type, name, help, options, childClass) { + let labels = []; + let register = true; + let callback = null; + for (let [name, value] of Object.entries(options)) { + if (name === 'labels') { labels = value; } + else if (name === 'register') { register = value; } + else if (name === 'callback') { callback = value; } + else { throw new Error(`Unrecognized option '${name}'`); } + } + + // Make sure we don't register to the default registry if metric throws. + let metric = new Metric(type, name, help, labels); + + super(register); + + this.callback = callback; + this.metric = metric; + this._values = new Map(); + this._children = new Map(); + this._childClass = childClass; + this._defaultChild = labels.length ? null : this.labels({}); + } + + async* collect() { + if (this.callback) { await this.callback(this); } + yield { metric: this.metric, samples: this._values }; + } + + get() { + return this._defaultChild.get(); + } + + labels(...labels) { + let key = labelsToKey(labels, this.metric.labels); + let child = this._children.get(key); + if (child === undefined) { + child = new this._childClass(this._values, key); + this._children.set(key, child); + } + return child; + } + + remove(...labels) { + let key = labelsToKey(labels, this.metric.labels); + if (key === '') { + throw new Error("labels cannot be empty"); + } + this._children.delete(key); + this._values.delete(key); + } + + removeAll(labels) { + if (!Object.keys(labels).length) { + throw new Error("labels cannot be empty"); + } + + function removeMatchingLabels(mapping, labels) { + for (let key of mapping.keys()) { + let candidate = keyToLabels(key); + let hasLabels = Object.entries(labels).every(([name, value]) => { + return candidate.get(name) === value; + }); + if (hasLabels) { + mapping.delete(key) + } + } + } + + removeMatchingLabels(this._values, labels); + removeMatchingLabels(this._children, labels); + } + + clear() { + if (!this.metric.labels.length) { + throw new Error("Cannot clear unlabeled metric"); + } + this._children = new Map(); + this._values = new Map(); + } +} + +class ValueCollectorChild { + constructor(values, key) { + this._values = values; + this._key = key; + + this._values.set(key, 0); + } + + get() { + return this._values.get(this._key); + } +} + +class CounterChild extends ValueCollectorChild { + inc(value = 1) { + // Note: Inverted to also catch NaN + if (!(value >= 0)) { + throw new Error("Expected value to be a positive number"); + } + + this._values.set(this._key, this._values.get(this._key) + value) + } +} + +class Counter extends ValueCollector { + constructor(name, help, options = {}) { + super("counter", name, help, options, CounterChild); + } + + inc(value = 1) { + this._defaultChild.inc(value); + } +} + +class GaugeChild extends ValueCollectorChild { + inc(value = 1) { + this._values.set(this._key, this._values.get(this._key) + value) + } + + dec(value = 1) { + this._values.set(this._key, this._values.get(this._key) - value) + } + + set(value) { + this._values.set(this._key, value) + } + + setToCurrentTime() { + this._values.set(this._key, Date.now() / 1000) + } +} + +class Gauge extends ValueCollector { + constructor(name, help, options = {}) { + super("gauge", name, help, options, GaugeChild); + } + + inc(value = 1) { + this._defaultChild.inc(value); + } + + dec(value = 1) { + this._defaultChild.dec(value); + } + + set(value) { + this._defaultChild.set(value); + } + + setToCurrentTime() { + this._defaultChild.setToCurrentTime(); + } +} + +class CollectorRegistry { + constructor() { + this.collectors = []; + } + + async* collect() { + for (let collector of this.collectors) { + for await (let result of collector.collect()) { + yield result; + } + } + } + + register(metric) { + let index = this.collectors.lastIndexOf(metric); + if (index !== -1) { + throw new Error( + "Collector is already registered in this registry." + ); + } + + this.collectors.push(metric); + } + + unregister(metric) { + let index = this.collectors.lastIndexOf(metric); + if (index === -1) { + throw new Error( + "Collector is not registered in this registry." + ); + } + + this.collectors.splice(index, 1); + } +} + +defaultRegistry = new CollectorRegistry(); + +function escapeHelp(help) { + return help + .replace(/\\/g, "\\\\") + .replace(/\n/g, "\\n") + ; +} + +function formatValue(value) { + if (value === Infinity) { + return '+Inf'; + } else if (value === -Infinity) { + return '-Inf'; + } else { + return value.toString(); + } +} + +async function* expositionLines(resultsIterator) { + let first = true; + for await (let result of resultsIterator) { + if (first) { + first = false; + } else { + yield `\n` + } + + yield `# HELP ${result.metric.name} ${escapeHelp(result.metric.help)}\n`; + yield `# TYPE ${result.metric.name} ${result.metric.type}\n`; + + for (let [key, value] of result.samples) { + if (key === '') { + yield `${result.metric.name} ${formatValue(value)}\n`; + } else { + yield `${result.metric.name}{${key}} ${formatValue(value)}\n`; + } + } + } +} + +async function exposition(resultsIterator = defaultRegistry.collect()) { + let exposition = ''; + for await (let line of expositionLines(resultsIterator)) { + exposition += line; + } + return exposition; +} + +// HTTP Content-Type for the exposition format that's implemented +exposition.contentType = 'text/plain; version=0.0.4'; + +function serializeResult(result, options = {}) { + let addLabels = null; + let metricName = result.metric.name; + let metricHelp = result.metric.help; + for (let [name, value] of Object.entries(options)) { + if (name === 'addLabels') { addLabels = value; } + else if (name === 'metricName') { metricName = value; } + else if (name === 'metricHelp') { metricHelp = value; } + else { throw new Error(`Unrecognized option '${name}'`); } + } + + let samples; + if (addLabels === null) { + samples = result.samples; + addLabels = {}; + + } else { + samples = new Map(); + let key = labelsToKey([addLabels], [...Object.keys(addLabels)]); + for (let [labels, value] of result.samples.entries()) { + if (labels === '') { samples.set(key, value); } + else { samples.set(`${labels},${key}`, value); } + } + } + + return { + metric: { + type: result.metric.type, + name: metricName, + help: metricHelp, + labels: [...result.metric.labels, ...Object.keys(addLabels)], + }, + samples: [...samples.entries()], + }; +} + +function deserializeResult(serializedResult) { + return { + metric: new Metric( + serializedResult.metric.type, + serializedResult.metric.name, + serializedResult.metric.help, + serializedResult.metric.labels, + ), + samples: new Map(serializedResult.samples), + } +} + +let defaultCollectors = {}; +defaultCollectors.processCpuSecondsTotal = new Gauge( + 'process_cpu_seconds', "Total user and system CPU time spent in seconds.", + { + callback: async function() { + let usage = process.cpuUsage(); + this.set((usage.user + usage.system) / 1e6); + }, + }, +); + +defaultCollectors.processStartTimeSeconds = new Gauge( + 'process_start_time_seconds', + "Start time of the process since unix epoch in seconds." +); +defaultCollectors.processStartTimeSeconds.setToCurrentTime(); + + +module.exports = { + Counter, + Gauge, + CollectorRegistry, + Collector, + ValueCollector, + Metric, + + exposition, + defaultRegistry, + defaultCollectors, + serializeResult, + deserializeResult, +}; diff --git a/test/lib/prometheus.js b/test/lib/prometheus.js new file mode 100644 index 00000000..4b2fd911 --- /dev/null +++ b/test/lib/prometheus.js @@ -0,0 +1,565 @@ +const assert = require('assert').strict; + +const prometheus = require('lib/prometheus'); + + +describe("lib/prometheus", function() { + describe("class CollectorRegistry", function() { + let registry; + beforeEach(function() { + registry = new prometheus.CollectorRegistry(); + }) + + describe(".register()", function() { + it("should register a collector", function() { + let collector = new prometheus.Collector(false); + registry.register(collector); + assert( + registry.collectors.includes(collector), + "Collector was not registered" + ); + }); + it("should throw when already registered", function() { + let collector = new prometheus.Collector(false); + registry.register(collector); + assert.throws( + () => registry.register(collector), + new Error("Collector is already registered in this registry.") + ); + }); + it("should allow being regitstered in another register", function() { + let otherCollectorRegistry = new prometheus.CollectorRegistry(); + let collector = new prometheus.Collector(false); + registry.register(collector); + otherCollectorRegistry.register(collector); + assert( + registry.collectors.includes(collector), + "Collector was not registered in first register" + ); + assert( + otherCollectorRegistry.collectors.includes(collector), + "Collector was not registered in second register" + ); + }); + }); + describe(".unregister()", function() { + it("should removed a registered collector", function() { + let collector = new prometheus.Collector(false); + registry.register(collector); + registry.unregister(collector); + assert( + !registry.collectors.includes(collector), + "Collector was not removed" + ); + }); + it("should throw when not registered", function() { + let collector = new prometheus.Collector(false); + assert.throws( + () => registry.unregister(collector), + new Error("Collector is not registered in this registry.") + ); + }); + }); + + describe("defaultRegistry", function() { + it("should have collectors registered to it by default", function() { + let collector = new prometheus.Collector(); + assert( + prometheus.defaultRegistry.collectors.includes(collector), + "collector was not registered to defaultRegistry" + ); + prometheus.defaultRegistry.unregister(collector); + }); + it("should have the default collectors registered to it", function() { + let collectors = Object.entries(prometheus.defaultCollectors); + for (let [name, collector] of collectors) { + assert( + prometheus.defaultRegistry.collectors.includes(collector), + `${name} is not registered` + ); + } + }); + it("should allow default collectors to be unregistered", function() { + let collectors = Object.values(prometheus.defaultCollectors); + for (let collector of collectors) { + prometheus.defaultRegistry.unregister(collector); + } + assert( + prometheus.defaultRegistry.collectors.length == 0, + "Collectors left over after removing the default ones" + ); + }); + }); + }); + + describe("class Collector", function() { + it("should implement .collect()", async function() { + let collector = new prometheus.Collector(false); + for await (let result of collector.collect()) { + assert.fail("Should not return any results"); + } + }); + }); + + describe("class Counter", function() { + let counter; + beforeEach(function() { + counter = new prometheus.Counter('test', "Help", { register: false }); + }); + + describe("constructor", function() { + it("should be initialized to 0", function() { + assert.equal(counter._values.get(''), 0); + }); + }); + describe(".inc()", function() { + it("should increment the counter by 1 by default", function() { + counter.inc(); + assert.equal(counter._values.get(''), 1); + }); + it("should increment the counter by specified value", function() { + counter.inc(3.2); + assert.equal(counter._values.get(''), 3.2); + }); + it("should throw if value is negative", function() { + assert.throws( + () => counter.inc(-1), + new Error("Expected value to be a positive number") + ); + }); + it("should throw if value is negative infinity", function() { + assert.throws( + () => counter.inc(-Infinity), + new Error("Expected value to be a positive number") + ); + }); + it("should throw if value is NaN", function() { + assert.throws( + () => counter.inc(NaN), + new Error("Expected value to be a positive number") + ); + }); + }); + }); + + describe("class Gauge", function() { + let gauge; + beforeEach(function() { + gauge = new prometheus.Gauge('test', "Help", { register: false }); + }); + + describe("constructor", function() { + it("should be initialized to 0", function() { + assert.equal(gauge._values.get(''), 0); + }); + }); + describe(".inc()", function() { + it("should increment the gauge by 1 by default", function() { + gauge.inc(); + assert.equal(gauge._values.get(''), 1); + }); + it("should increment the gauge by specified value", function() { + gauge.inc(3.2); + assert.equal(gauge._values.get(''), 3.2); + }); + it("should decrement on negative value", function() { + gauge.inc(-2); + assert.equal(gauge._values.get(''), -2); + }); + }); + describe(".dec()", function() { + it("should decrement the gauge by 1 by default", function() { + gauge.dec(); + assert.equal(gauge._values.get(''), -1); + }); + it("should decrement the gauge by specified value", function() { + gauge.dec(3.2); + assert.equal(gauge._values.get(''), -3.2); + }); + it("should increment on negative value", function() { + gauge.dec(-2); + assert.equal(gauge._values.get(''), 2); + }); + }); + describe(".set()", function() { + it("should set the gauge to the specified value", function() { + gauge.set(7); + assert.equal(gauge._values.get(''), 7); + }); + }); + describe(".setToCurrentTime()", function() { + it("should set the gauge to current unix time in seconds", function() { + gauge.setToCurrentTime(); + assert( + Math.abs(gauge._values.get('') - Date.now() / 1000) < 1, + "value set is not close to the current unix time" + ); + }); + }); + }); + + describe("class ValueCollector", function() { + describe("constructor", function() { + it("should throw on invalid metric name", function() { + for (let name of ["", " ", "0a", "$a"]) { + assert.throws( + () => new prometheus.Gauge( + name, "Help", { register: false } + ), + new Error(`Invalid name '${name}'`) + ) + } + }); + it("should throw on invalid label names", function() { + for (let name of ["", " ", "0a", "$a", "a:"]) { + assert.throws( + () => new prometheus.Gauge( + 'test', "Help", { register: false, labels: [name] } + ), + new Error(`Invalid label '${name}'`) + ) + } + }); + it("should throw on missing name", function() { + assert.throws( + () => new prometheus.Gauge(), + new Error("Expected name to be a string") + ); + }); + it("should throw on missing help", function() { + assert.throws( + () => new prometheus.Gauge("test"), + new Error("Expected help to be a string") + ); + }); + it("should throw on unrecoginzed option", function() { + assert.throws( + () => new prometheus.Gauge("test", "help", { invalid: 1 }), + new Error("Unrecognized option 'invalid'") + ); + }); + }); + + describe(".get()", function() { + it("should return the current value", function() { + let gauge = new prometheus.Gauge('test', "Help", { register: false }); + gauge.set(11); + assert.equal(gauge.get(), 11); + }); + }); + + describe(".labels()", function() { + let gauge; + beforeEach(function() { + gauge = new prometheus.Gauge( + 'test', "Help", { register: false, labels: ['a', 'b'] } + ); + }); + it("should require all labels set at creation", function() { + assert.throws( + () => gauge.labels({a: "1"}), + new Error("Missing label 'b'") + ); + assert.throws( + () => gauge.labels({b: "1"}), + new Error("Missing label 'a'") + ); + assert.throws( + () => gauge.labels({a: '1', b: '2', c: '3'}), + new Error("Extra label 'c'") + ); + assert.throws( + () => gauge.labels('1'), + new Error("Missing label 'b'") + ); + assert.throws( + () => gauge.labels('1', '2', '3'), + new Error("Extra positional label") + ); + }); + it("should throw if label value is not a string", function() { + assert.throws( + () => gauge.labels({a: "1", b: 2}), + new Error("Expected value for label 'b' to be a string") + ); + assert.throws( + () => gauge.labels("1", 2), + new Error("Expected value for label 'b' to be a string") + ); + }); + it("should initialize a value for the label set", function() { + gauge.labels({ a: '1', b: '2'}); + assert(gauge._values.has('a="1",b="2"'), "value was not set"); + gauge.labels('4', '5'); + assert(gauge._values.has('a="4",b="5"'), "value was not set"); + }); + it("should return a child supporting the usual methods", function() { + let child = gauge.labels({ a: '1', b: '2'}); + child.get(); + child.inc(); + child.dec(); + child.setToCurrentTime(); + child = gauge.labels('7', '8'); + child.get(); + child.inc(); + child.dec(); + child.setToCurrentTime(); + }); + it("should return the same child for the same labels", function() { + let child = gauge.labels('1', '2'); + assert.equal(gauge.labels('1', '2'), child); + }); + }); + describe(".remove()", function() { + it("should remove a label set created with .label()", function() { + let gauge = new prometheus.Gauge( + 'test', "Help", { register: false, labels: ['a', 'b'] } + ); + gauge.labels({ a: '1', b: '2'}); + gauge.remove({ a: '1', b: '2'}); + assert(!gauge._values.has('a="1",b="2"'), "value was not removed"); + }); + it("should throw when removing the empty label set", function() { + let gauge = new prometheus.Gauge('test', "Help", { register: false }); + assert.throws( + () => gauge.remove({}), new Error("labels cannot be empty") + ); + }); + }); + describe(".removeAll()", function() { + it("should remove a label set created with .label()", function() { + let gauge = new prometheus.Gauge( + 'test', "Help", { register: false, labels: ['a', 'b'] } + ); + gauge.labels('1', '2'); + gauge.labels('1', '3'); + gauge.labels('4', '6'); + gauge.removeAll({a: '1'}); + assert(gauge._values.size <= 1, "Matching labels remained") + assert( + gauge._values.has('a="4",b="6"'), + "Non-matching label was removed" + ); + }); + it("should throw when removing the empty label set", function() { + let gauge = new prometheus.Gauge('test', "Help", { register: false }); + assert.throws( + () => gauge.removeAll({}), new Error("labels cannot be empty") + ); + }); + it("should handle special characters", function() { + let gauge = new prometheus.Gauge( + 'test', "Help", { register: false, labels: ['a', 'b'] } + ) + gauge.labels('x\\\\\n\"\"\n\\', '2'); + gauge.removeAll({a: 'x\\\\\n\"\"\n\\'}); + assert(gauge._values.size === 0, "Matching labels remained") + }); + }); + describe(".clear()", function() { + it("should remove all label sets created with .label()", function() { + let gauge = new prometheus.Gauge( + 'test', "Help", { register: false, labels: ['a', 'b'] } + ); + gauge.labels({ a: '1', b: '2'}); + gauge.labels({ a: '3', b: '4'}); + gauge.labels({ a: '5', b: '6'}); + gauge.clear(); + assert(gauge._values.size == 0, "values was not cleared"); + }); + it("should throw when used on an unlabled metric", function() { + let gauge = new prometheus.Gauge('test', "Help", { register: false }); + assert.throws( + () => gauge.clear(), new Error("Cannot clear unlabeled metric") + ); + }); + }); + }); + + describe("defaultCollectors", function() { + describe("processStartTimeSeconds", function() { + it("should give a close point in time", async function() { + let results = [] + let collector = prometheus.defaultCollectors.processStartTimeSeconds; + for await (let result of collector.collect()) { + results.push(result); + } + + assert( + results.length == 1, + "collector did not give exactly one result" + ); + + assert( + Math.abs(results[0].samples.get('') - Date.now() / 1000) < 1, + "value set is not close to the current unix time" + ); + }); + }); + describe("processCpuSecondsTotal", function() { + it("should give a low value", async function() { + let results = [] + let collector = prometheus.defaultCollectors.processCpuSecondsTotal; + for await (let result of collector.collect()) { + results.push(result); + } + + assert( + results.length == 1, + "collector did not give exactly one result" + ); + + let value = results[0].samples.get('') + assert( + value < 5 && value > 0, + `process time is not between 0 and 5 seconds (${value})` + ); + }); + }); + }); + + describe("exposition()", function() { + it("should format simple metrics", async function() { + let registry = new prometheus.CollectorRegistry(); + let counter = new prometheus.Counter('a', "Help", { register: false }); + registry.register(counter); + let gauge = new prometheus.Gauge('b', "Help", { register: false }); + registry.register(gauge); + let exposition = await prometheus.exposition(registry.collect()); + assert.equal( + exposition, + '# HELP a Help\n' + + '# TYPE a counter\n' + + 'a 0\n' + + '\n' + + '# HELP b Help\n' + + '# TYPE b gauge\n' + + 'b 0\n' + ); + }); + it("should format labeled metrics", async function() { + let registry = new prometheus.CollectorRegistry(); + let gauge = new prometheus.Gauge( + 'test', "Help", { register: false, labels: ['a', 'b']} + ); + gauge.labels({a: '1', b: '2'}); + gauge.labels({b: 'y', a: 'x'}); + registry.register(gauge); + let exposition = await prometheus.exposition(registry.collect()); + assert.equal( + exposition, + '# HELP test Help\n' + + '# TYPE test gauge\n' + + 'test{a="1",b="2"} 0\n' + + 'test{a="x",b="y"} 0\n' + ); + }); + it("should format Infinity and NaN", async function() { + let registry = new prometheus.CollectorRegistry(); + let gauge = new prometheus.Gauge( + 'test', "Help", { register: false, labels: ['n']} + ); + gauge.labels({n: '1'}).set(Infinity); + gauge.labels({n: '2'}).set(-Infinity); + gauge.labels({n: '3'}).set(NaN); + registry.register(gauge); + let exposition = await prometheus.exposition(registry.collect()); + assert.equal( + exposition, + '# HELP test Help\n' + + '# TYPE test gauge\n' + + 'test{n="1"} +Inf\n' + + 'test{n="2"} -Inf\n' + + 'test{n="3"} NaN\n' + ); + }); + }); + + let serializedResult = { + metric: { type: 'type', name: 'name', help: "Help", labels: ['a', 'b'] }, + samples: [ ['a="1",b="2"', 1], ['a="3",b="4"', 5], ] + }; + let deserializedResult = { + metric: new prometheus.Metric('type', 'name', "Help", ['a', 'b']), + samples: new Map([['a="1",b="2"', 1], ['a="3",b="4"', 5]]), + }; + describe("serializeResult()", function() { + it("should convert a result into plain object", function() { + assert.deepEqual( + prometheus.serializeResult(deserializedResult), + serializedResult + ) + }); + it("should support adding labels", function() { + assert.deepEqual( + prometheus.serializeResult({ + metric: new prometheus.Metric('type', 'name', "Help"), + samples: new Map([['', 2]]), + }, { addLabels: { c: '8' }}), + { + metric: {type: 'type', name: 'name', help: "Help", labels: ['c']}, + samples: [['c="8"', 2]], + } + ); + assert.deepEqual( + prometheus.serializeResult({ + metric: new prometheus.Metric('type', 'name', "Help", ['a']), + samples: new Map([['a="4"', 3]]), + }, { addLabels: { c: '8' }}), + { + metric: {type: 'type', name: 'name', help: "Help", labels: ['a', 'c']}, + samples: [['a="4",c="8"', 3]], + } + ); + }); + it("should support redefining name and help", function() { + assert.deepEqual( + prometheus.serializeResult({ + metric: new prometheus.Metric('type', 'name', "Help"), + samples: new Map([['', 2]]), + }, { metricName: 'new', }), + { + metric: {type: 'type', name: 'new', help: "Help", labels: []}, + samples: [['', 2]], + } + ); + assert.deepEqual( + prometheus.serializeResult({ + metric: new prometheus.Metric('type', 'name', "Help"), + samples: new Map([['', 2]]), + }, { metricHelp: "old", }), + { + metric: {type: 'type', name: 'name', help: "old", labels: []}, + samples: [['', 2]], + } + ); + }); + it("should throw on unrecoginzed option", function() { + assert.throws( + () => new prometheus.serializeResult({ + metric: new prometheus.Metric('type', 'name', "Help"), + samples: new Map([['', 3]]), + }, { invalid: true }), + new Error("Unrecognized option 'invalid'") + ); + }); + }); + + describe("deserializeResult()", function() { + it("should convert a plain object into a result", function() { + assert.deepEqual( + prometheus.deserializeResult(serializedResult), + deserializedResult + ) + }); + }); + + describe("post test checks", function() { + it("should not have any collectors in the default registry", function() { + assert( + prometheus.defaultRegistry.collectors.length == 0, + "Collectors left over by test code" + ); + }); + }); +}); From 8229d1955c98bead4aecb763ea476604d4a2bc3d Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 20 Dec 2019 02:31:51 +0100 Subject: [PATCH 09/11] Skip broken Prometheus client tests Skip Prometheus client tests that assume the environment is isolated from the rest of the tests. --- test/lib/prometheus.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/lib/prometheus.js b/test/lib/prometheus.js index 4b2fd911..afecec92 100644 --- a/test/lib/prometheus.js +++ b/test/lib/prometheus.js @@ -80,6 +80,7 @@ describe("lib/prometheus", function() { } }); it("should allow default collectors to be unregistered", function() { + this.skip(); // XXX Doesn't work in Clusterio's shared testing env let collectors = Object.values(prometheus.defaultCollectors); for (let collector of collectors) { prometheus.defaultRegistry.unregister(collector); @@ -379,6 +380,7 @@ describe("lib/prometheus", function() { describe("defaultCollectors", function() { describe("processStartTimeSeconds", function() { it("should give a close point in time", async function() { + this.skip(); // XXX Doesn't work in Clusterio's shared testing env let results = [] let collector = prometheus.defaultCollectors.processStartTimeSeconds; for await (let result of collector.collect()) { @@ -398,6 +400,7 @@ describe("lib/prometheus", function() { }); describe("processCpuSecondsTotal", function() { it("should give a low value", async function() { + this.skip(); // XXX Doesn't work in Clusterio's shared testing env let results = [] let collector = prometheus.defaultCollectors.processCpuSecondsTotal; for await (let result of collector.collect()) { @@ -556,6 +559,7 @@ describe("lib/prometheus", function() { describe("post test checks", function() { it("should not have any collectors in the default registry", function() { + this.skip(); // XXX Doesn't work in Clusterio's shared testing env assert( prometheus.defaultRegistry.collectors.length == 0, "Collectors left over by test code" From 26eefb0476f5104f7e697610a16c345cbb1849f4 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 20 Dec 2019 01:27:40 +0100 Subject: [PATCH 10/11] Rewrite statistics export Add automatic export of metrics from slaves to the master server. And rewrite the statistics export plugin to use this new system. --- .gitignore | 1 - CHANGELOG.md | 16 +++ client.js | 51 +++++++ docs/contributing.md | 2 + docs/writing-plugins.md | 49 +++++++ lib/link/messages.js | 8 ++ lib/plugin.js | 34 +++++ master.js | 130 ++++++++---------- package.json | 3 - plugins/statistics_exporter/info.js | 9 ++ plugins/statistics_exporter/instance.js | 62 +++++++++ plugins/statistics_exporter/lua/export.lua | 39 ++++++ plugins/statistics_exporter/slave.js | 0 sharedPlugins/statisticsExporter/config.js | 9 -- sharedPlugins/statisticsExporter/index.js | 40 ------ .../statisticsExporter/lua/export.lua | 30 ---- .../statisticsExporter/masterPlugin.js | 67 --------- .../statisticsExporter/test/missing.js | 5 - test/mock.js | 5 + 19 files changed, 334 insertions(+), 226 deletions(-) create mode 100644 plugins/statistics_exporter/info.js create mode 100644 plugins/statistics_exporter/instance.js create mode 100644 plugins/statistics_exporter/lua/export.lua create mode 100644 plugins/statistics_exporter/slave.js delete mode 100644 sharedPlugins/statisticsExporter/config.js delete mode 100644 sharedPlugins/statisticsExporter/index.js delete mode 100644 sharedPlugins/statisticsExporter/lua/export.lua delete mode 100644 sharedPlugins/statisticsExporter/masterPlugin.js delete mode 100644 sharedPlugins/statisticsExporter/test/missing.js diff --git a/.gitignore b/.gitignore index 8803d5f3..aa612a11 100644 --- a/.gitignore +++ b/.gitignore @@ -51,7 +51,6 @@ sharedPlugins/* !sharedPlugins/UPSdisplay/ !sharedPlugins/researchSync/ !sharedPlugins/clusterioMod/ -!sharedPlugins/statisticsExporter/ # File used for marking plugins as disabled sharedPlugins/*/DISABLED diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff5b68b..303ffd78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ Version 2.0.0 a base class and use the same WebSocket connection Clusterio uses to communicate. +### Features + +- Added export of pollution statitics. + ### Changes - Fixed sslCert and sslPrivKey entries being ignored in the config. @@ -111,6 +115,18 @@ Version 2.0.0 - Removed the onLoadFinish hook from master plugins. - Moved plugins from the sharedPlugins directory to plugins directory. - Implemented a new plugin system that replaces the old. Breaks all plugins. +- Removed express metric http_request_duration_milliseconds metric. +- Removed socket.io metrics socket_io_connected, socket_io_connect_total, + socket_io_disconnect_total, socket_io_events_received_total, + socket_io_events_sent_total, socket_io_recieve_bytes, and + socket_io_transmit_bytes, +- Removed clusterio_connected_instaces_gauge and added + clusterio_master_connected_clients_count in its place. +- Renamed clusterio_player_count_gauge to clusterio_instance_player_count +- Removed clusterio_UPS_gauge and added clusterio_instance_game_ticks_total in + its place. +- Renamed clusterio_endpoint_hit_gauge to clusterio_http_enpoint_hits_total +- Renamed clusterio_statistics_gauge to clusterio_instance_force_flows Version 1.2.4 diff --git a/client.js b/client.js index 1b3ccabd..444d6c0d 100644 --- a/client.js +++ b/client.js @@ -10,6 +10,7 @@ const factorio = require("lib/factorio"); const link = require("lib/link"); const plugin = require("lib/plugin"); const errors = require("lib/errors"); +const prometheus = require('lib/prometheus'); /** @@ -221,6 +222,30 @@ class Instance extends link.Link{ await this.server.stop(); } + + // Clear metrics this instance is exporting + for (let collector of prometheus.defaultRegistry.collectors) { + if ( + collector instanceof prometheus.ValueCollector + && collector.metric.labels.includes('instance_id') + ) { + collector.removeAll({ instance_id: String(this.config.id) }); + } + } + } + + async getMetricsRequestHandler() { + let results = [] + for (let pluginInstance of this.plugins.values()) { + let pluginResults = await pluginInstance.onMetrics(); + if (pluginResults !== undefined) { + for await (let result of pluginResults) { + results.push(prometheus.serializeResult(result)) + } + } + } + + return { results }; } async startInstanceRequestHandler() { @@ -500,6 +525,32 @@ class Slave extends link.Link { return instanceConnection; } + async getMetricsRequestHandler() { + let requests = []; + for (let instanceConnection of this.instanceConnections.values()) { + requests.push(link.messages.getMetrics.send(instanceConnection)); + } + + let results = []; + for (let response of await Promise.all(requests)) { + results.push(...response.results); + } + + for await (let result of prometheus.defaultRegistry.collect()) { + if (result.metric.name.startsWith('process_')) { + results.push(prometheus.serializeResult(result, { + addLabels: { 'slave_id': String(this.config.id) }, + metricName: result.metric.name.replace('process_', 'clusterio_slave_'), + })); + + } else { + results.push(prometheus.serializeResult(result)); + } + } + + return { results }; + } + async startInstanceRequestHandler(message, request) { let instanceConnection = await this._connectInstance(message.data.instance_id); return await request.send(instanceConnection, message.data); diff --git a/docs/contributing.md b/docs/contributing.md index 67a8c5ba..159fec0c 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -170,6 +170,8 @@ indents. Should you be modifiying such code that is mixing spaces and tabs as indending you should update the indenting to use tabs _on the code you change_ while leaving the rest of the code untouched. +Lua code should be indented with 4 spaces. + ### Line Length diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index b1d19ac9..86b450d6 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -301,3 +301,52 @@ plugin's `info.js` module. In other words: For the Request class the send method is async and returns the response data received from the target it was sent to, or throws an error if the request failed. + + +Collecting Statistics +--------------------- + +Clusterio comes with it's own Prometheus client implementation, one part +due to Not Invented Here and another part due to collectors in +prom-client being difficult to get to work nicely with collecting data +from plugins optionally loaded at runtime on different computers. + +In its simplest form collecting data from a plugins consists of defining +the metric and updating it somewhere in the plugin code. For example: + + const { Counter } = require('lib/prometheus'); + + const fooMetric = new Counter( + 'clusterio_foo_metric', "Measures the level of foo", + ); + + // Somewhere in the master plugin code + fooMetric.inc(); + +This works for master plugins, and the metric will be automatically +available through the /metric HTTP endpoint. For metrics that are +per-instance you must define an `instance_id` label and set it +accordingly, for example: + + const { Counter } = require('lib/prometheus'); + + const barMetric = new Gauge( + 'clusterio_bar_metric', "Bar instance level", + { labels: ['instance_id'] } + ); + + // Somewhere in the instance plugin code + barMetric.labels(String(this.config.id)).set(someValue); + +Metrics are automatically registered to the default registry, and this +default registry is automatically polled by the master server on slaves. +This means that it's important that you place the definition of the +metric at module level so that it's not created more than once over the +lifetime of a slave. Since the metrics remember their values and would +continue to be exported after an instance is shutdown there's code at +instance shutdown that removes all the values where the `instance_id` +label matches the id of the instance shut down. + +For statistics you need to update on collection there's an `onMetrics` +hook on both master and instance plugins that is runned before the +metrics in the default registry is collected. diff --git a/lib/link/messages.js b/lib/link/messages.js index bd76e4c9..49dc37ae 100644 --- a/lib/link/messages.js +++ b/lib/link/messages.js @@ -292,6 +292,14 @@ messages.sendRcon = new Request({ } }); +messages.getMetrics = new Request({ + type: 'statistics_exporter:get_metrics', + links: ['master-slave', 'slave-instance'], + responseProperties: { + "results": { type: "array" }, + }, +}); + /** * Represents an event sent over the link diff --git a/lib/plugin.js b/lib/plugin.js index f0b5c8fc..aae39ed9 100644 --- a/lib/plugin.js +++ b/lib/plugin.js @@ -19,6 +19,23 @@ class BaseInstancePlugin { */ async init() { } + /** + * Called before collecting Prometheus metrics + * + * Invoked before the default metrics of prometheus is collected. Note + * that since this is done while replying to /metrics this should not + * take a long time, otherwise the requst will time out. + * + * Although you can return the results of collecting your prometheus + * collectors directly, all gauges in the default registry is already + * automatically collected. It's therefore recommended to only update + * the Gauges/Counters/etc in onMetric and let Clusterio deal will + * collecting the values. + * + * @returns an async iterator of prometheus metric results or undefined. + */ + async onMetrics() { } + /** * Called after the Factorio server is started */ @@ -66,6 +83,23 @@ class BaseMasterPlugin { */ async init() { } + /** + * Called before collecting Prometheus metrics + * + * Invoked before the default metrics of prometheus is collected. Note + * that since this is done while replying to /metrics this should not + * take a long time, otherwise the requst will time out. + * + * Although you can return the results of collecting your prometheus + * collectors directly, all gauges in the default registry is already + * automatically collected. It's therefore recommended to only update + * the Gauges/Counters/etc in onMetric and let Clusterio deal will + * collecting the values. + * + * @returns an async iterator of prometheus metric results or undefined. + */ + async onMetrics() { } + /** * Called when the master server is shutting down */ diff --git a/master.js b/master.js index 1a7cb3fd..c7f0fd9b 100644 --- a/master.js +++ b/master.js @@ -39,6 +39,7 @@ const schema = require("lib/schema"); const link = require("lib/link"); const errors = require("lib/errors"); const plugin = require("lib/plugin"); +const prometheus = require("lib/prometheus"); // homemade express middleware for token auth const authenticate = require("lib/authenticate"); @@ -83,79 +84,69 @@ require("./routes/api/getPictures")(app); // Set folder to serve static content from (the website) app.use(express.static('static')); -// set up logging software -const prometheusPrefix = "clusterio_"; -const Prometheus = require('prom-client'); -const expressPrometheus = require('express-prometheus-request-metrics'); -Prometheus.collectDefaultMetrics({ timeout: 10000 }); // collects RAM usage etc every 10 s +const endpointHitCounter = new prometheus.Gauge( + 'clusterio_master_http_endpoint_hits_total', "How many requests a particular endpoint has gotten", + { labels: ['route'] } +); -// collect express request durations ms -app.use(expressPrometheus(Prometheus)); - -const endpointHitCounter = new Prometheus.Gauge({ - name: prometheusPrefix+'endpoint_hit_gauge', - help: "How many requests a particular endpoint has gotten", - labelNames: ['route'], -}); -const prometheusConnectedInstancesCounter = new Prometheus.Gauge({ - name: prometheusPrefix+'connected_instaces_gauge', - help: "How many instances are currently connected to this master server", -}); -const prometheusWsUsageCounter = new Prometheus.Counter({ - name: prometheusPrefix+'websocket_usage_counter', - help: 'Websocket traffic', - labelNames: ["connectionType", "instanceID"], -}); -const prometheusPlayerCountGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'player_count_gauge', - help: 'Amount of players connected to this cluster', - labelNames: ["instanceID", "instanceName"], -}); -const prometheusUPSGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'UPS_gauge', - help: 'UPS of the current server', - labelNames: ["instanceID", "instanceName"], -}); -/** -GET Prometheus metrics endpoint. Returns performance and usage metrics in a prometheus readable format. -@memberof clusterioMaster -@instance -@alias /metrics -*/ -app.get('/metrics', (req, res) => { +// Prometheus polling endpoint +async function getMetrics(req, res, next) { endpointHitCounter.labels(req.route.path).inc(); - res.set('Content-Type', Prometheus.register.contentType); - /// gather some static metrics - registerMoreMetrics(); - res.end(Prometheus.register.metrics()); -}); - -function registerMoreMetrics(){ - for (let [instanceID, slave] of db.slaves) { - // playercount - try{ - prometheusPlayerCountGauge.labels(instanceID, slave.instanceName).set(Number(slave.playerCount) || 0); - }catch(e){} - // UPS - try{ - prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.UPS) || 60); - if(slave.meta.tick && typeof slave.meta.tick === "number") prometheusUPSGauge.labels(instanceID, slave.instanceName).set(Number(slave.meta.tick) || 0); - }catch(e){} - } - - // plugins + let results = [] for (let masterPlugin of masterPlugins.values()) { - masterPlugin.onMetrics(); + let pluginResults = await masterPlugin.onMetrics(); + if (pluginResults !== undefined) { + for await (let result of pluginResults) { + results.push(result) + } + } } - // Slave count - let numberOfActiveSlaves = 0; - for(let slave of db.slaves.values()){ - if(Date.now() - Number(slave.time) < 1000 * 30) numberOfActiveSlaves++; + let requests = []; + for (let slaveConnection of slaveConnections.values()) { + // XXX this needs a timeout + requests.push(link.messages.getMetrics.send(slaveConnection)); } - prometheusConnectedInstancesCounter.set(numberOfActiveSlaves); + + for await (let result of await prometheus.defaultRegistry.collect()) { + results.push(result); + } + + let resultMap = new Map(); + for (let response of await Promise.all(requests)) { + for (let result of response.results) { + if (!resultMap.has(result.metric.name)) { + resultMap.set(result.metric.name, result); + + } else { + // Merge metrics received by multiple slaves + let stored = resultMap.get(result.metric.name); + stored.samples.push(...result.samples); + } + } + } + + for (let result of resultMap.values()) { + results.push(prometheus.deserializeResult(result)); + } + + let text = await prometheus.exposition(results); + res.set('Content-Type', prometheus.exposition.contentType); + res.send(text); } +app.get('/metrics', (req, res, next) => getMetrics(req, res, next).catch(next)); + + +const masterConnectedClientsCount = new prometheus.Gauge( + 'clusterio_master_connected_clients_count', "How many clients are currently connected to this master server", + { + labels: ['type'], callback: async function(gauge) { + gauge.labels("slave").set(slaveConnections.size); + gauge.labels("control").set(controlConnections.length); + }, + }, +); // set up database const db = {}; @@ -269,9 +260,6 @@ app.get("/api/getFactorioLocale", function(req,res){ // socket.io connection for slaves var io = require("socket.io")({}); -const ioMetrics = require("socket.io-prometheus"); -ioMetrics(io); - class BaseConnection extends link.Link { constructor(target, connector) { @@ -408,7 +396,7 @@ class SlaveConnection extends BaseConnection { }); this.connector.on('message', () => { - prometheusWsUsageCounter.labels('message', "other").inc(); + // XXX prometheusWsUsageCounter.labels('message', "other").inc(); }); this.connector.on('disconnect', () => { @@ -525,7 +513,7 @@ io.on('connection', function (socket) { // Handle socket handshake socket.once('message', (payload) => { - prometheusWsUsageCounter.labels('message', "other").inc(); + // XXX prometheusWsUsageCounter.labels('message', "other").inc(); if (!schema.clientHandshake(payload)) { console.log(`SOCKET | closing ${socket.handshake.address} after receiving invalid handshake`, payload); connector.send('close', { reason: "Invalid handshake" }); @@ -588,7 +576,7 @@ async function loadPlugins() { main:new masterPlugin({ TODO? config, socketio: io, express: app, - db, Prometheus, prometheusPrefix, endpointHitCounter, + db, }), pluginConfig, });*/ diff --git a/package.json b/package.json index 98913802..e69f172d 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,6 @@ "ejs": "^2.5.7", "express": "^4.16.2", "express-fileupload": "1.1.5", - "express-prometheus-request-metrics": "github:burnash/express-prometheus-request-metrics", "follow-redirects": "^1.5.0", "fs-extra": "^8.1.0", "ini": "^1.3.5", @@ -55,7 +54,6 @@ "needle": "^2.4.0", "node-factorio-api": "^0.3.8", "node-forge": "^0.7.1", - "prom-client": "^10.2.3", "rcon-client": "^4.2.0", "request": "^2.83.0", "sanitizer": "^0.1.3", @@ -64,7 +62,6 @@ "single-line-log": "^1.1.2", "socket.io": "^2.2.0", "socket.io-client": "^2.2.0", - "socket.io-prometheus": "^0.2.1", "sync-request": "^4.0.1", "typescript-collections": "^1.3.2", "yargs": "^14.2.0" diff --git a/plugins/statistics_exporter/info.js b/plugins/statistics_exporter/info.js new file mode 100644 index 00000000..a1bf60da --- /dev/null +++ b/plugins/statistics_exporter/info.js @@ -0,0 +1,9 @@ +let link = require("lib/link"); + +module.exports = { + name: "statistics_exporter", + title: "Prometheus Statistics Export", + description: "Exports statistics to Prometheus", + version: "2.0.0-alpha", + instanceEntrypoint: "instance", +} diff --git a/plugins/statistics_exporter/instance.js b/plugins/statistics_exporter/instance.js new file mode 100644 index 00000000..ff6644b8 --- /dev/null +++ b/plugins/statistics_exporter/instance.js @@ -0,0 +1,62 @@ +const plugin = require('lib/plugin'); +const { Gauge } = require('lib/prometheus'); + + +const instancePlayerCount = new Gauge( + 'clusterio_instance_player_count', "Amount of players connected to this cluster", + { labels: ['instance_id'] } +); +const instanceGameTicksTotal = new Gauge( + 'clusterio_instance_game_ticks_total', "Game tick an instance has progressed to", + { labels: ['instance_id'] } +); +const instanceForceFlowStatistics = new Gauge( + 'clusterio_instance_force_flow_statistics', "Items/fluids/enemies/buildings produced/built/killed by a force", + { labels: ['instance_id', 'force', 'statistic', 'direction', 'name'] }, +); +const instanceGameFlowStatistics = new Gauge( + 'clusterio_instance_game_flow_statistics', "Pollution produced/consumed in the game", + { labels: ['instance_id', 'statistic', 'direction', 'name'] }, +); + + +class InstancePlugin extends plugin.BaseInstancePlugin { + async onMetrics() { + let string = await this.instance.server.sendRcon('/sc remote.call("statistics_exporter", "export")'); + let stats = JSON.parse(string); + + let instanceId = this.instance.config.id; + instanceGameTicksTotal.labels(String(instanceId)).set(stats.game_tick); + instancePlayerCount.labels(String(instanceId)).set(stats.player_count); + + for (let [forceName, flowStatistics] of Object.entries(stats.force_flow_statistics)) { + for (let [statisticName, statistic] of Object.entries(flowStatistics)) { + for (let [direction, counts] of Object.entries(statistic)) { + for (let [item, value] of Object.entries(counts)) { + instanceForceFlowStatistics.labels( + String(instanceId), forceName, statisticName, direction, item + ).set(value); + } + } + } + } + + for (let [direction, counts] of Object.entries(stats.game_flow_statistics.pollution_statistics)) { + for (let [item, value] of Object.entries(counts)) { + instanceGameFlowStatistics.labels( + String(instanceId), "pollution_statistics", direction, item + ).set(value); + } + } + } +} + +module.exports = { + InstancePlugin, + + // For testing only + _instancePlayerCount: instancePlayerCount, + _instanceGameTicksTotal: instanceGameTicksTotal, + _instanceForceFlowStatistics: instanceForceFlowStatistics, + _instanceGameFlowStatistics: instanceGameFlowStatistics, +} diff --git a/plugins/statistics_exporter/lua/export.lua b/plugins/statistics_exporter/lua/export.lua new file mode 100644 index 00000000..babb44da --- /dev/null +++ b/plugins/statistics_exporter/lua/export.lua @@ -0,0 +1,39 @@ +local statistics = { + "item_production_statistics", + "fluid_production_statistics", + "kill_count_statistics", + "entity_build_count_statistics", +} + +local function export() + local stats = { + game_tick = game.tick, + player_count = #game.connected_players, + game_flow_statistics = { + pollution_statistics = { + input = game.pollution_statistics.input_counts, + output = game.pollution_statistics.output_counts, + }, + }, + force_flow_statistics = {} + } + for _, force in pairs(game.forces) do + local flow_statistics = {} + for _, statName in pairs(statistics) do + flow_statistics[statName] = { + input = force[statName].input_counts, + output = force[statName].output_counts, + } + end + stats.force_flow_statistics[force.name] = flow_statistics + end + + + rcon.print(game.table_to_json(stats)) +end + +return { + add_remote_interface = function() + remote.add_interface("statistics_exporter", { export = export }) + end +} diff --git a/plugins/statistics_exporter/slave.js b/plugins/statistics_exporter/slave.js new file mode 100644 index 00000000..e69de29b diff --git a/sharedPlugins/statisticsExporter/config.js b/sharedPlugins/statisticsExporter/config.js deleted file mode 100644 index 9ebc0b4a..00000000 --- a/sharedPlugins/statisticsExporter/config.js +++ /dev/null @@ -1,9 +0,0 @@ -/* - Clusterio plugin for gathering ingame statistics -*/ -module.exports = { - name: "statisticsExporter", - version: "1.0.0", - description: "Gathers ingame stats such as production, consumption, kills and buildings placed", - masterPlugin: "masterPlugin.js" -} diff --git a/sharedPlugins/statisticsExporter/index.js b/sharedPlugins/statisticsExporter/index.js deleted file mode 100644 index f6dbebd0..00000000 --- a/sharedPlugins/statisticsExporter/index.js +++ /dev/null @@ -1,40 +0,0 @@ -const needle = require("needle"); - - -module.exports = class remoteCommands { - constructor(mergedConfig, messageInterface, extras){ - this.messageInterface = messageInterface; - this.config = mergedConfig; - - const needleOptionsWithTokenAuthHeader = { - headers: { - 'x-access-token': this.config.masterAuthToken - }, - }; - - // Gather and submit statistics every 15 seconds - setInterval(async () => { - let stats = await this.getStats(); - stats.forEach(force => { - // I have no idea why I need to do JSON.stringify before passing it to needle.post, - // but if you do forceData: force.data, it doesn't work... - let request = { - instanceID: this.config.unique, - force: force.forceName, - forceData: JSON.stringify(force.data), - }; - needle.post(`${mergedConfig.masterURL}/api/logStats`, - request, - needleOptionsWithTokenAuthHeader, (err, resp) => { - if(!err && resp.body && resp.body.ok == false) console.log("Error calling /api/logStats: " + resp.body); - } - ); - }); - }, 15000); - } - - async getStats() { - let string = await this.messageInterface('/sc remote.call("statisticsExporter", "export")'); - return JSON.parse(string); - } -} diff --git a/sharedPlugins/statisticsExporter/lua/export.lua b/sharedPlugins/statisticsExporter/lua/export.lua deleted file mode 100644 index 76c2c756..00000000 --- a/sharedPlugins/statisticsExporter/lua/export.lua +++ /dev/null @@ -1,30 +0,0 @@ -local statistics = { - "item_production_statistics", - "fluid_production_statistics", - "kill_count_statistics", - "entity_build_count_statistics", -} - -local function export() - local stats = {} - for _, force in pairs(game.forces) do - local data = {} - for _, statName in pairs(statistics) do - data[statName] = { - input = force[statName].input_counts, - output = force[statName].output_counts, - } - end - table.insert(stats, { - forceName = force.name, - data = data, - }) - end - rcon.print(game.table_to_json(stats)) -end - -return { - add_remote_interface = function() - remote.add_interface("statisticsExporter", { export = export }) - end -} diff --git a/sharedPlugins/statisticsExporter/masterPlugin.js b/sharedPlugins/statisticsExporter/masterPlugin.js deleted file mode 100644 index d65a8e20..00000000 --- a/sharedPlugins/statisticsExporter/masterPlugin.js +++ /dev/null @@ -1,67 +0,0 @@ -const fs = require("fs-extra"); -const path = require("path"); - -class masterPlugin { - constructor({config, socketio, express, Prometheus}){ - const authenticate = require("lib/authenticate"); - - this.config = config; - this.io = socketio; - const app = express; - - const prometheusStatisticsGauge = new Prometheus.Gauge({ - name: 'clusterio_statistics_gauge', - help: 'Items produced/built/killed by a force', - labelNames: ["instanceID", "force", "statistic", "direction", "itemName"], - }); - - /** --- TODO: Replace this with a simpler Prometheus exporter. Requires changes to the mod as well. - POST endpoint to log production graph statistics. Should contain an instanceID (also reffered to as "unique") - and of course the timeSeries data organized by category. - - @memberof clusterioMaster - @instance - @alias api/logStats - @param {object} JSON {instanceID: "string", force: "string" data: {"item_production_statistics": {"input":{"iron-plate":150}, "output":{}}, ...}} - @returns {string} failure - */ - app.post("/api/logStats", authenticate.middleware, function(req,res) { - let data = req.body.forceData; - if(typeof req.body.forceData == "string") { - data = JSON.parse(req.body.forceData); - } - // endpointHitCounter.labels(req.route.path).inc(); - if(typeof req.body == "object" - && req.body.instanceID - && typeof req.body.instanceID == "string" - && req.body.instanceID.length < 100 - && data - && typeof data == "object" - && !Array.isArray(data)){ - try{ - Object.keys(data).forEach(statisticName => { - Object.keys(data[statisticName]).forEach(direction => { - let statHash = data[statisticName][direction]; - Object.keys(statHash).forEach(item => { - let value = Number(statHash[item]); - if(value) prometheusStatisticsGauge.labels(req.body.instanceID, req.body.force, statisticName, direction, item).set(value); - }); - }); - }); - res.send({ok:true, msg:`Statistics successfully reported at ${Date.now()}`}); - }catch(e){ - res.send({ok:false, msg:"Statistics reporter crashed, see sharedPlugins/statisticsExporter/masterPlugin.js"}); - console.log(e) - }; - } else { - console.log(`/api/logStats: Invalid request. instanceID: ${req.body.instanceID}, dataIsArray: ` - + `${Array.isArray(data)}, ${typeof req.body.instanceID}, ${typeof data}`); - res.send({ - ok:false, - msg:"Invalid request parameters" - }); - } - }); - } -} -module.exports = masterPlugin; diff --git a/sharedPlugins/statisticsExporter/test/missing.js b/sharedPlugins/statisticsExporter/test/missing.js deleted file mode 100644 index 9993114b..00000000 --- a/sharedPlugins/statisticsExporter/test/missing.js +++ /dev/null @@ -1,5 +0,0 @@ -// These are libraries that does not have any units test, they are -// required here in order to track coverage for them. - -require('../index'); -require('../masterPlugin'); diff --git a/test/mock.js b/test/mock.js index 3a1f8f09..33b50f9f 100644 --- a/test/mock.js +++ b/test/mock.js @@ -57,10 +57,12 @@ class MockConnector extends events.EventEmitter { class MockServer { constructor() { this.rconCommands = []; + this.rconCommandResults = new Map(); } async sendRcon(command) { this.rconCommands.push(command); + return this.rconCommandResults.get(command) || ''; } } @@ -69,6 +71,9 @@ class MockInstance extends link.Link { super('instance', 'slave', new MockConnector()); this.server = new MockServer(); this.name = "test"; + this.config = { + id: 7357, + } } } From 1873f9bda8e484b10b736569f4ca19d393138916 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 20 Dec 2019 02:14:58 +0100 Subject: [PATCH 11/11] Remove UPSdisplay plugin Remove the redundant UPSdisplay plugin. UPS statistics is gathered by the statistics_exporter plugin. --- .gitignore | 1 - CHANGELOG.md | 2 ++ sharedPlugins/UPSdisplay/config.js | 12 ------- sharedPlugins/UPSdisplay/index.js | 45 ------------------------ sharedPlugins/UPSdisplay/test/missing.js | 4 --- 5 files changed, 2 insertions(+), 62 deletions(-) delete mode 100644 sharedPlugins/UPSdisplay/config.js delete mode 100644 sharedPlugins/UPSdisplay/index.js delete mode 100644 sharedPlugins/UPSdisplay/test/missing.js diff --git a/.gitignore b/.gitignore index aa612a11..94d6d05f 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,6 @@ saves !sharedPlugins/ sharedPlugins/* !sharedPlugins/inventoryImports/ -!sharedPlugins/UPSdisplay/ !sharedPlugins/researchSync/ !sharedPlugins/clusterioMod/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 303ffd78..59430008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ Version 2.0.0 - Removed info and shout command from globalChat plugin - Removed mirrorAllChat and enableCrossServerShout configuration options for globalChat plugin. +- Removed UPSdisplay plugin. UPS statistics is exported by the statistics + exporter plugin. ### Breaking Changes diff --git a/sharedPlugins/UPSdisplay/config.js b/sharedPlugins/UPSdisplay/config.js deleted file mode 100644 index 7abaacf9..00000000 --- a/sharedPlugins/UPSdisplay/config.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - Example config file for example Clusterio plugin -*/ - -module.exports = { - // Name of package. For display somewhere I guess. - name: "UPSdisplay", - version: "2.0.0", - description: "Internal clusterio plugin to get the worlds UPS and display it on master", - // We'll send everything in this file to your stdin. Beware. - scriptOutputFileSubscription: "UPSdisplay.txt", -} \ No newline at end of file diff --git a/sharedPlugins/UPSdisplay/index.js b/sharedPlugins/UPSdisplay/index.js deleted file mode 100644 index f15a4ea2..00000000 --- a/sharedPlugins/UPSdisplay/index.js +++ /dev/null @@ -1,45 +0,0 @@ -const needle = require("needle"); - - - -class UPSdisplay { - constructor(slaveConfig, msgInterface, extras){ - this.config = slaveConfig; - this.messageInterface = msgInterface; - this.socket = extras.socket; - this.slave = extras.slave; - - this.historicalTicks = []; - setInterval(() => { - this.messageInterface("/silent-command rcon.print(game.tick)", data => { - this.tick = Number(data); - this.historicalTicks[this.historicalTicks.length] = {tick:Number(data), timestamp:Date.now()}; - if(this.historicalTicks.length > 30){ - this.historicalTicks.shift(); // delete last element in array (position 0); - } - }); - },1000); - setInterval(() => { - if(this.historicalTicks && this.historicalTicks[0]){ - let timePeriod = this.historicalTicks[this.historicalTicks.length-1].timestamp - this.historicalTicks[0].timestamp; - let ticksInPeriod = this.historicalTicks[this.historicalTicks.length-1].tick - this.historicalTicks[0].tick; - let UPS = Math.round(ticksInPeriod / (timePeriod/1000)); - // console.log("UPS: " + UPS); - try{ - needle.post( - this.config.masterURL+'/api/editSlaveMeta', - {instanceID: this.config.unique, password: this.config.clientPassword, meta: {UPS:UPS, tick:this.tick}}, - {headers: {'x-access-token': this.config.masterAuthToken}}, - function(err, resp) { - // success? - } - ); - } catch (err){ - console.log(err); - } - } - }, 5000); - this.messageInterface("/silent-command game.print('UPSdisplay enabled')"); - } -} -module.exports = UPSdisplay; \ No newline at end of file diff --git a/sharedPlugins/UPSdisplay/test/missing.js b/sharedPlugins/UPSdisplay/test/missing.js deleted file mode 100644 index 923d82bf..00000000 --- a/sharedPlugins/UPSdisplay/test/missing.js +++ /dev/null @@ -1,4 +0,0 @@ -// These are libraries that does not have any units test, they are -// required here in order to track coverage for them. - -require('../index');