From 5ef6d020547a27f82d3242572954c19759c35368 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 6 Jan 2020 06:37:39 +0100 Subject: [PATCH 01/11] Add serialize library to clusterio module Add library for serializing items and inventories to the clusterio module. This will provide a common codebase for plugins like playerManager and trainTeleports to use in order to serialize inventories to the master server. --- modules/clusterio/serialize.lua | 262 ++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 modules/clusterio/serialize.lua diff --git a/modules/clusterio/serialize.lua b/modules/clusterio/serialize.lua new file mode 100644 index 00000000..1714e194 --- /dev/null +++ b/modules/clusterio/serialize.lua @@ -0,0 +1,262 @@ +-- Library for serializing items in Factorio +-- Based on code from playerManager and trainTeleports +local serialize = {} + +-- Equipment Grids are serialized into an array of equipment entries +-- where ench entry is a table with the following fields: +-- n: name +-- p: position (array of 2 numbers corresponding to x and y) +-- s: shield (optional) +-- e: energy (optional) +-- If the equipment is a burner the following is also present: +-- i: burner inventory +-- r: result inventory +-- b: curently burning (optional) +-- f: remaining_burning_fuel (optional) +function serialize.serialize_equipment_grid(grid) + local serialized = {} + local processed = {} + for y = 0, grid.height - 1 do + for x = 0, grid.width - 1 do + local equipment = grid.get({x, y}) + if equipment ~= nil then + local pos = equipment.position + local combined_pos = pos.x + pos.y * grid.width + 1 + if not processed[combined_pos] then + processed[combined_pos] = true + local entry = { + n = equipment.name, + p = {pos.x, pos.y}, + } + if equipment.shield > 0 then entry.s = equipment.shield end + if equipment.energy > 0 then entry.e = equipment.energy end + -- TODO: Test with Industrial Revolution + if equipment.burner then + local burner = equipment.burner + entry.i = serialize.serialize_inventory(burner.inventory) + entry.r = serialize.serialize_inventory(burner.burnt_result_inventory) + if burner.curently_burning then + entry.b = {} + serialize.serialize_item_stack(burner.curently_burning, entry.b) + entry.f = burner.remaining_burning_fuel + end + end + table.insert(serialized, entry) + end + end + end + end + return serialized +end + +function serialize.deserialize_equipment_grid(grid, serialized) + grid.clear() + for _, entry in ipairs(serialized) do + local equipment = grid.put({ + name = entry.n, + position = entry.p, + }) + if equipment then + if entry.s then equipment.shield = entry.s end + if entry.e then equipment.energy = entry.e end + if entry.i then + if entry.b then serialize.deserialize_item_stack(burner.currently_burning, entry.b) end + if entry.f then burner.remaining_burning_fuel = entry.f end + serialize.deserialize_inventory(burner.burnt_result_inventory, entry.r) + serialize.deserialize_inventory(burner.inventory, entry.i) + end + end + end +end + +-- Item stacks are serialized into a table with the following fields: +-- n: name +-- c: count +-- h: health (optional) +-- d: durability (optional) +-- a: ammo count (optional) +-- l: label (optional) +-- g: equipment grid (optional) +-- i: item inventory (optional) +-- If the item stack is exportable it has the following property instead +-- e: export string +-- Label is a table with the following fields: +-- t: label text (optional) +-- c: color (optional) +-- a: allow manual label change +function serialize.serialize_item_stack(slot, entry) + if + slot.is_blueprint + or slot.is_blueprint_book + or slot.is_upgrade_item + or slot.is_deconstruction_item + or slot.is_item_with_tags + then + local call_success, call_return = pcall(slot.export_stack) + if not call_success then + print("Error: '" .. call_return .. "' thrown exporting '" .. slot.name .. "'") + else + entry.e = call_return + end + + return + end + + entry.n = slot.name + entry.c = slot.count + if slot.health < 1 then entry.h = slot.health end + if slot.durability then entry.d = slot.durability end + if slot.type == "ammo" then entry.a = slot.ammo end + if slot.is_item_with_label then + local label = {} + if slot.label then label.t = slot.label end + if slot.label_color then label.c = slot.label_color end + label.a = slot.allow_manual_label_change + entry.l = label + end + + if slot.grid then + entry.g = serialize.serialize_equipment_grid(slot.grid) + end + + if slot.is_item_with_inventory then + local sub_inventory = slot.get_inventory(defines.inventory.item_main) + entry.i = serialize.serialize_inventory(sub_inventory) + end +end + +function serialize.deserialize_item_stack(slot, entry) + if entry.e then + local success = slot.import_stack(entry.e) + if success == 1 then + print("Error: import of '" .. entry.e .. "' succeeded with errors") + elseif success == -1 then + print("Error: import of '" .. entry.e .. "' failed") + end + + return + end + + local item_stack = { + name = entry.n, + count = entry.c, + } + if entry.h then item_stack.health = entry.h end + if entry.d then item_stack.durability = entry.d end + if entry.a then item_stack.ammo = entry.a end + + local call_success, call_return = pcall(slot.set_stack, item_stack) + if not call_success then + print("Error: '" .. call_return .. "' thrown setting stack ".. serpent.line(entry)) + + elseif not call_return then + print("Error: Failed to set stack " .. serpent.line(entry)) + + else + if entry.l then + -- TODO test this with AAI's unit-remote-control + local label = entry.l + if label.t then slot.label = label.t end + if label.c then slot.label_color = label.c end + slot.allow_manual_label_change = label.a + end + if entry.g then + serialize.deserialize_equipment_grid(slot.grid, entry.g) + end + if entry.i then + local sub_inventory = slot.get_inventory(defines.inventory.item_main) + serialize.deserialize_inventory(sub_inventory, entry.i) + end + end +end + +-- Inventories are serialized into a table with the following fields: +-- i: array of item stack or exportable item entries +-- b: bar position (optional) +-- Each item entry has the following fields +-- s: index (optional, equals to previous plus one if not present) +-- r: repeat count (optional) +-- f: slot filter (optional) +-- Pluss all the fields for item stacks (see deserialize_item_stack) +-- It's also possible that the slot is empty but has a slot filter. +function serialize.serialize_inventory(inventory) + local serialized = {} + if inventory.hasbar() and inventory.getbar() <= #inventory then + serialized.b = inventory.getbar() + end + + serialized.i = {} + local previous_index = 0 + local previous_serialized = nil + for i = 1, #inventory do + local item = {} + local slot = inventory[i] + if inventory.supports_filters() then + item.f = inventory.get_filter(i) + end + + if slot.valid_for_read then + serialize.serialize_item_stack(slot, item) + end + + if item.n or item.f or item.e then + local item_serialized = game.table_to_json(item) + if item_serialized == previous_serialized then + local previous_item = serialized.i[#serialized.i] + previous_item.r = (previous_item.r or 0) + 1 + previous_index = i + + else + if i ~= previous_index + 1 then + item.s = i + end + + previous_index = i + previous_serialized = item_serialized + table.insert(serialized.i, item) + end + + else + -- Either an empty slot or serilization failed + previous_index = 0 + previous_serialized = nil + end + end + + return serialized +end + +function serialize.deserialize_inventory(inventory, serialized) + if serialized.b and inventory.hasbar() then + inventory.setbar(serialized.b) + end + + local last_slot_index = 0 + for _, entry in ipairs(serialized.i) do + local base_index = entry.s or last_slot_index + 1 + + local repeat_count = entry.r or 0 + for offset = 0, repeat_count do + -- XXX what if the inventory is smaller on this instance? + local index = base_index + offset + local slot = inventory[index] + if entry.f then + local call_success, call_return = pcall(inventory.set_filter, index, entry.f) + if not call_success then + print("Error: '" .. call_return .. "' thrown setting filter " .. entry.f) + + elseif not call_return then + print("Error: Failed to set filter " .. entry.f) + end + end + + if entry.n or entry.e then + serialize.deserialize_item_stack(slot, entry) + end + end + last_slot_index = base_index + repeat_count + end +end + + +return serialize From 1e6b807ae3166a0564cd9dced9c0eb2d26a2feb7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 12 Mar 2020 15:39:25 +0100 Subject: [PATCH 02/11] Add Clusterio Library mod Add a library mod for wrapping the Clusterio remote API and providing related utilities. This eases development of Clusterio related Lua code by providing the same interface for both mods and modules and also makes the serialize library available without having to duplicate it. It is not possible to implement this through save patching as mods cannot load Lua code from the level code like they can with code from other mods. --- build-mod.js | 92 +++++++++++++++++++++ docs/developing-for-clusterio.md | 137 ++++++++++++++++++++++++++++--- mod/api.lua | 67 +++++++++++++++ mod/changelog.txt | 5 ++ mod/info.json | 9 ++ modules/clusterio/api.lua | 36 ++------ modules/clusterio/impl.lua | 40 +++++---- package.json | 1 + 8 files changed, 328 insertions(+), 59 deletions(-) create mode 100644 build-mod.js create mode 100644 mod/api.lua create mode 100644 mod/changelog.txt create mode 100644 mod/info.json diff --git a/build-mod.js b/build-mod.js new file mode 100644 index 00000000..ef47f8a4 --- /dev/null +++ b/build-mod.js @@ -0,0 +1,92 @@ +"use strict"; +const events = require("events"); +const fs = require("fs-extra"); +const JSZip = require("jszip"); +const klaw = require("klaw"); +const path = require("path"); +const yargs = require("yargs"); + + +// Location of mod source files +const sourceDir = "mod"; + +// Paths to additional files to put into the mod's folder. +const additionalFiles = [ + path.join("modules", "clusterio", "serialize.lua"), +]; + +async function main() { + const args = yargs + .scriptName("build") + .options({ + 'clean': { describe: "Remove previous builds", type: 'boolean', default: false }, + 'build': { describe: "Build mod", type: 'boolean', default: true }, + 'pack': { describe: "Pack into zip file", type: 'boolean', default: true }, + 'output-dir': { describe: "Path to output built mod", nargs: 1, type: 'string', default: "dist" }, + }) + .argv + ; + + let info = JSON.parse(await fs.readFile(path.join(sourceDir, "info.json"))); + + if (args.clean) { + let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/ + for (let entry of await fs.readdir(args.outputDir)) { + let match = splitter.exec(entry); + if (match) { + let [, name, version] = match; + if (name === info.name) { + let modPath = path.join(args.outputDir, entry); + console.log(`Removing ${modPath}`); + await fs.remove(modPath); + } + } + } + } + + if (args.build) { + await fs.ensureDir(args.outputDir); + let modName = `${info.name}_${info.version}`; + + if (args.pack) { + let zip = new JSZip(); + let walker = klaw(sourceDir) + .on('data', item => { + if (item.stats.isFile()) { + // On Windows the path created uses backslashes as the directory sepparator + // but the zip file needs to use forward slashes. We can't use the posix + // version of relative here as it doesn't work with Windows style paths. + let basePath = path.relative(sourceDir, item.path).replace(/\\/g, "/"); + zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path)); + } + }) + ; + await events.once(walker, 'end'); + + for (let filePath of additionalFiles) { + zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath)); + } + + let modPath = path.join(args.outputDir, `${modName}.zip`); + console.log(`Writing ${modPath}`); + let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath)); + await events.once(writeStream, 'finish'); + + } else { + let modDir = path.join(args.outputDir, modName); + if (await fs.exists(modDir)) { + console.log(`Removing existing build ${modDir}`); + await fs.remove(modDir); + } + console.log(`Building ${modDir}`); + await fs.copy(sourceDir, modDir); + for (let filePath of additionalFiles) { + await fs.copy(filePath, path.join(modDir, path.basename(filePath))); + } + } + } +} + +if (module === require.main) { + main().catch(err => { console.log(err) }); +} diff --git a/docs/developing-for-clusterio.md b/docs/developing-for-clusterio.md index 4ada6fd8..5e2273fe 100644 --- a/docs/developing-for-clusterio.md +++ b/docs/developing-for-clusterio.md @@ -14,7 +14,7 @@ Contents - [event_handler interface](#event_handler-interface) - [Communicating with Clusterio](#communicating-with-clusterio) - [Clusterio Modules](#clusterio-modules) - - [Clusterio Module API](#clusterio-module-api) +- [Clusterio Lua API](#clusterio-lua-api) - [Clusterio Plugins](#clusterio-plugins) @@ -216,7 +216,8 @@ The following entries are supported in the module.json file: supported. The events for dependencies is invoked before the events of the dependent, and starting an instance will fail if the dependencies cannot be satisfied. If not specified it will default - to depending on `clusterio`. + to depending on `clusterio`. Make sure to add `clusterio` to your + dependencies if you add other dependencies and depend on the Lua API. - `require`: Lua files to require in `control.lua`. This should only really be needed if you want to make a global function available for use in @@ -250,21 +251,133 @@ module. Because modules can be patched into an existing game you cannot rely on the `on_init` callback to be called in Clusterio Modules. Nor can you rely on the `on_configuration_changed` callback, as this is not called -when level code changes. The `clusterio` module provides the custom +when level code changes. The Clusterio Lua API provides the custom `on_server_startup` event that can be used as a substitute, see the next section. -### Clusterio Module API +Clusterio Lua API +----------------- -There's a builtin clusterio module that provides some tools for getting -instance info and communicating to plugins, script events to listen for -and a Factorio remote call API. The available interfaces is documented -in [modules/clusterio/api.lua](../modules/clusterio/api.lua). To use -this API from another module, require it using something along the lines -of `local clusterio_api = require("modules/clusterio/api")`. Make sure -to add `clusterio` as a dependency in your module.json if you use the -dependency entry in it. +Clusterio provides a save patched module as well as a regular Factorio +mod for interfacing with Clusterio. This includes tools sending data to +plugins, script events to listen for, and an item serialization module. + + +### clusterio_api library + +**require path** +From a module +`local clusterio_api = require("modules/clusterio/api")` +From a mod +`local clusterio_api = require("__clusterio_lib__/api")` + +Provides the main interface to Clusterio from within the game. When +using the mod version you must call `init` before it'll function +properly. + +#### init + +Mod version only + +Initialize the API. This should be called from both the `on_init` and +`on_load` events in your mod, and before this is called certain features +will not be available, most notably the events table. + +#### clusterio_api.events + +Table of events raised by clusterio. If the game is started outside of +Clusterio then none of these events will be raised. + +In the mod version this table does not exist before the `init` function +is called. + +##### on_instance_updated + +Raised after the name and id of an instance has been updated. This may +occur even if the id and name didn't change. + +Event data: +- `instance_id`: The id of the instance. +- `instance_name`: the name of the instance. + +##### on_server_startup + +Raised when Clusterio is starting up the server. It can be used to +initialize clusterio related features in a mod, and as a stand-in for +the `on_init` and `on_configuration_changed` event for modules. It is +invoked on the first tick the server runs after the save has been +patched, before most other events. + +Use this event to initialize and and/or migrate the data structures +you need. Keep in mind that both mods and Clusterio modules can switch +from any version to any version so it should be able to handle both +forwards and backwards migrations gracefully. + +#### clusterio_api.send_json(channel, data) + +Send json data to Clusterio over the given channel. The `data` argument +should be table that can be serialized with `game.table_to_json`. +Clusterio plugins can listen to channels and will receive an event with +the data sent here. See the [Communicating with Factorio +section](writing-plugins.md#communicating-with-factorio) in the Writing +Plugins document for more information. + +If the game is started outside of Clusterio the data will be sent, but +since there's no code following stdout to pick it up it will be lost. + +**Note**: Payloads greater than 4 MB will cause stuttering in the game. + +**Note**: This is not a binary safe way of sending data. Strings +embedded into the tables sent must be valid UTF-8 text. + +Parameters: +- `channel`: string identifying which channel to send it on. +- `data`: table that can be converted to JSON with game.table_to_json + + +### serialize library + +**require path** +From a module +`local serialize = require("modules/clusterio/serialize")` +From a mod +`local serialize = require("__clusterio_lib__/serialize")` + +Serializes inventories and item stacks into a table that in turn can be +converted into JSON with `game.table_to_json`. Properly handles the +extra data for most items. + +#### serialize.serialize_equipment_grid(LuaEquipmentGrid) + +Returns a table containing a serialized representation of the equipment +grid passed. + +#### serialize.deserialize_equipment_grid(LuaEquipmentGrid, serialized_grid) + +Deserialize a previously serialized equipment grid into the target +LuaEquipmentGrid. Overwrites all content in the target. + +#### serialize.serialize_item_stack(LuaItemStack, destination_table) + +Serialize the LuaItemStack passed into the target destination_table. +The destination_table should be a new empty table. + +#### serialize.deserialize_item_stack(LuaItemStack, serialized_stack) + +Deserialize a previously serialized item stack into the target +LuaItemStack. Overwrites the content of the item stack. + +#### serialize.serialize_inventory(LuaInventory) + +Returns a table containing a serialized representation of the inventory +passed. + +#### serialize.deserialize_inventory(LuaInventory, serialized_inventory) + +Deserialize a previously serialized inventory into the target +LuaInventory. Overwrites slots that has content in the serialized +inventory. Clusterio Plugins diff --git a/mod/api.lua b/mod/api.lua new file mode 100644 index 00000000..c800b035 --- /dev/null +++ b/mod/api.lua @@ -0,0 +1,67 @@ +-- Important: Keep this API in sync with modules/clusterio/api.lua +local api = {} + + +local initialized = false +function api.init() + if initialized then + return + end + initialized = true + + if remote.interfaces.clusterio_api then + api.events = remote.call("clusterio_api", "get_events") + else + local null_event = script.generate_event_name() + api.events = { + on_instance_updated = null_event, + on_server_startup = null_event, + } + end +end + +local function call_api(fn, ...) + if remote.interfaces.clusterio_api and remote.interfaces.clusterio_api[fn] then + return remote.call("clusterio_api", fn, ...) + else + return nil + end +end + +function api.get_instance_name() + return call_api("get_instance_name") +end + +function api.get_instance_id() + return call_api("get_instance_id") +end + + +function api.send_json(channel, data) + if not remote.interfaces.clusterio_api then + return + end + + -- Escape bad characters. The question mark is used for separating the + -- channel name from the payload. + channel = channel:gsub("([\x00-\x1f?\\])", function(match) + return "\\x" .. string.format("%02x", match:byte()) + end) + + data = game.table_to_json(data) + + -- If there's more than about 4kB of data users running with the Windows + -- console open will start to experience stuttering, otherwise we could + -- output about 1 MB of data at a time through stdout. + if #data < 4000 then + print("\f$ipc:" .. channel .. "?j" .. data) + else + local file_no = remote.call("clusterio_api", "get_file_no") + local file_name = "clst_" .. file_no .. ".json" + game.write_file(file_name, data, false, 0) + print("\f$ipc:" .. channel .. "?f" .. file_name) + end +end + + +return api diff --git a/mod/changelog.txt b/mod/changelog.txt new file mode 100644 index 00000000..50e3310e --- /dev/null +++ b/mod/changelog.txt @@ -0,0 +1,5 @@ +--------------------------------------------------------------------------------------------------- +Version: 0.1.0 + Major Features: + - API for mods to interact with Clusterio + - Serialization library for converting items to JSON compatible tables. diff --git a/mod/info.json b/mod/info.json new file mode 100644 index 00000000..26a80f92 --- /dev/null +++ b/mod/info.json @@ -0,0 +1,9 @@ +{ + "name": "clusterio_lib", + "version": "0.1.0", + "title": "Clusterio Library", + "author": "Clusterio Team", + "homepage": "https://github.com/clusterio/factorioClusterio", + "description": "Library for interfacing with Clusterio.", + "factorio_version": "0.17" +} diff --git a/modules/clusterio/api.lua b/modules/clusterio/api.lua index 2132f4fa..f0e932dd 100644 --- a/modules/clusterio/api.lua +++ b/modules/clusterio/api.lua @@ -1,42 +1,20 @@ +-- Important: Keep this API in sync with mod/api.lua local api = {} --- Table of events raised by clusterio api.events = { - -- Raised after the name and id of an instance has been updated - -- This may occur even if the id and name didn't change. - - -- Event data: - -- instance_id: The id of the instance. - -- instance_name: the name of the instance. on_instance_updated = script.generate_event_name(), - - -- Raised when the server is starting up - -- This event serves as the stand-in for the on_init and - -- on_configuration_changed events for modules. It is invoked as early - -- as possible, before most but not all other events, every time the - -- server is started up. - - -- Use this event to initialize and and/or migrate the data structures - -- you need. Keep in mind that Clusterio can switch from any version of - -- your module to any version of your module so it should be able to - -- handle both forwards and backwards migrations. on_server_startup = script.generate_event_name(), } --- Send a table as json to Clusterio --- Send data to Clusterio over the given channel. Clusterio plugins can --- listen to channels and will receive an event with the data sent here. --- See docs/writing-plugins.md for more information. +function api.get_instance_name() + return global.clusterio.instance_name +end --- Note: Payloads greater than 4 MB will cause stuttering. +function api.get_instance_id() + return global.clusterio.instance_id +end --- Note: This is not a binary safe way of sending data. Strings embedded --- into the tables sent must be valid UTF-8 text. - --- Parameters: --- channel: string identifying which channel to send it on. --- data: table that can be converted to json with game.table_to_json function api.send_json(channel, data) -- Escape bad characters. The question mark is used for separating the diff --git a/modules/clusterio/impl.lua b/modules/clusterio/impl.lua index d4989221..3ae9593f 100644 --- a/modules/clusterio/impl.lua +++ b/modules/clusterio/impl.lua @@ -32,26 +32,30 @@ function clusterio_private.update_instance(new_id, new_name) end -function impl.add_remote_interface() - remote.add_interface('clusterio_api', { - -- Returns a table of events raised by clusterio - get_events = function() - return api.events - end, +-- This is not part of the add_remote_interface callback to ensure it is +-- available when the clusterio_lib mod is loaded. The reason this is +-- neccessary is that on_init for newly added mods happen before on_load +-- for existing mods, and the add_remote_interface callback is done in +-- on_load. See https://forums.factorio.com/viewtopic.php?f=25&t=81552 for +-- more details. +remote.add_interface('clusterio_api', { + get_events = function() + return api.events + end, - -- Returns the instance id the game is run under - -- This may change over time and/or be nil. - get_instance_id = function() - return global.clusterio.instance_id - end, + get_instance_id = function() + return global.clusterio.instance_id + end, - -- Returns the name of the instance the game runs under. - -- This may change over time and/or be nil. - get_instance_name = function() - return global.clusterio.instance_name - end, - }) -end + get_instance_name = function() + return global.clusterio.instance_name + end, + + get_file_no = function() + global.clusterio_file_no = (global.clusterio_file_no or 0) + 1 + return global.clusterio_file_no + end, +}) return impl diff --git a/package.json b/package.json index a4afb539..2b258eb0 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ }, "devDependencies": { "html5-validator": "^1.2.1", + "klaw": "^3.0.0", "mocha": "^6.2.0", "mocha-sinon": "^2.1.0", "mocha.parallel": "^0.15.5", From 6f22942cc5197d0d8b14dd227b63615be4f1bcaa Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 21:37:46 +0100 Subject: [PATCH 03/11] Fix incorrect instance lookups in plugins Fix mistaken lookups missing the instance part, or trying to get the id from the removed instance.id property. --- docs/writing-plugins.md | 4 ++-- plugins/statistics_exporter/instance.js | 2 +- test/mock.js | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index e5be14cb..e955503e 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -226,7 +226,7 @@ plugin you listen for an event named ipc-channel_name in order to get data sent by `send_json`. For example in the plugin code: async init() { - this.server.on('ipc-my_plugin_foo', content => + this.instance.server.on('ipc-my_plugin_foo', content => this.handleFoo(content).catch(err => console.log( "Error handling foo:", foo )) @@ -478,7 +478,7 @@ accordingly, for example: ); // Somewhere in the instance plugin code - barMetric.labels(String(this.config.id)).set(someValue); + barMetric.labels(String(this.instance.config.get("instance.id"))).set(someValue); Metrics are automatically registered to the default registry, and this default registry is automatically polled by the master server on slaves. diff --git a/plugins/statistics_exporter/instance.js b/plugins/statistics_exporter/instance.js index aa158231..eee2e0ab 100644 --- a/plugins/statistics_exporter/instance.js +++ b/plugins/statistics_exporter/instance.js @@ -30,7 +30,7 @@ class InstancePlugin extends plugin.BaseInstancePlugin { throw new Error(`Error parsing statistics JSON: ${err.message}, content "${string}"`); } - let instanceId = this.instance.config.id; + let instanceId = this.instance.config.get("instance.id"); instanceGameTicksTotal.labels(String(instanceId)).set(stats.game_tick); instancePlayerCount.labels(String(instanceId)).set(stats.player_count); diff --git a/test/mock.js b/test/mock.js index 33b50f9f..9410bd4c 100644 --- a/test/mock.js +++ b/test/mock.js @@ -72,7 +72,10 @@ class MockInstance extends link.Link { this.server = new MockServer(); this.name = "test"; this.config = { - id: 7357, + get: (name) => { + if (name === "instance.id") { return 7357 } + throw Error("Not implemented"); + } } } } From 5681ef9733a927b8faefeb45f29fbcb1370b5c47 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 21:40:33 +0100 Subject: [PATCH 04/11] Fix plugin enabled config being inverted Fix plugins being loaded when enable was set to false, and not loaded when set to true. --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index 09a00c6d..20d1d6e5 100644 --- a/client.js +++ b/client.js @@ -53,7 +53,7 @@ class Instance extends link.Link{ // load plugins for (let pluginInfo of pluginInfos) { - if (!pluginInfo.instanceEntrypoint || this.config.group(pluginInfo.name).get("enabled")) { + if (!pluginInfo.instanceEntrypoint || !this.config.group(pluginInfo.name).get("enabled")) { continue; } From 6a8346186fe23000ad58dc508b8cfdcef7ef69f7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 21:49:03 +0100 Subject: [PATCH 05/11] Implemented forwarding of requests to the master Implement the forwardTo function for requests to the master server. --- client.js | 4 ++++ docs/writing-plugins.md | 4 ++-- lib/link/messages.js | 13 +++++++++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/client.js b/client.js index 20d1d6e5..d4b18ff7 100644 --- a/client.js +++ b/client.js @@ -358,6 +358,10 @@ class InstanceConnection extends link.Link { } } + async forwardRequestToMaster(message, request) { + return await request.send(this.slave, message.data); + } + async forwardEventToInstance(message, event) { let instanceId = message.data.instance_id; let instanceConnection = this.slave.instanceConnections.get(instanceId); diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index e955503e..16d8865d 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -408,8 +408,8 @@ The following properties are recognized by the Request constructor: Target to forward request to. Can be either `'master'` to indicate a slave should forward it to the master server when receiving it from an instance or `'instance'` to indicate it should be forwarded - to the instances specified by the `instance_id` event property. - This works by using a default handler for the event at the links + to the instances specified by the `instance_id` request property. + This works by using a default handler for the request at the links that forward it. **requestProperties**: diff --git a/lib/link/messages.js b/lib/link/messages.js index edb87dc1..e651e5cb 100644 --- a/lib/link/messages.js +++ b/lib/link/messages.js @@ -20,9 +20,9 @@ class Request { * Links this request is valid on. Takes an array of strings * containing "-" specifications. * @param {?string} forwardTo - - * Optional target to forward this request to. Only 'instance' is - * currently supported and adds instance_id into the - * requestProperties. + * Optional target to forward this request to. 'instance' add + * instance_id into the requestProperties and forward to the given + * instance. 'master' forwards it to the master server. * @param {Object} requestProperties - * Mapping of property values to JSON schema specifications for the * properties that are valid in the data payload of this requst. @@ -46,7 +46,7 @@ class Request { {"instance_id": { type: "integer" }}, requestProperties ); - } else if (forwardTo !== null) { + } else if (forwardTo !== "master" && forwardTo !== null) { throw new Error(`Invalid forwardTo value ${forwardTo}`); } @@ -99,6 +99,9 @@ class Request { * If forwardTo was set to 'instance' the `forwardRequestToInstance` * method on the link is used as the default handler. * + * If forwardTo was set to 'master' the `forwardRequesToMaster` method + * on the link is used as the default handler. + * * Does nothing if the link is neither a source nor a target for this * request. * @@ -121,6 +124,8 @@ class Request { if (!handler) { if (this.forwardTo === 'instance') { handler = link.forwardRequestToInstance; + } else if (this.forwardTo === "master") { + handler = link.forwardRequestToMaster; } } From ddd2027739b0f8bafb36f470236d2a77320d805b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 21:52:39 +0100 Subject: [PATCH 06/11] Handle errors during instance startup Catch errors and shut down the instance in question if the startup failed. This happens if for example the Factorio server errors out on a script during startup. --- client.js | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/client.js b/client.js index d4b18ff7..725d185f 100644 --- a/client.js +++ b/client.js @@ -567,14 +567,33 @@ class Slave extends link.Link { } async startInstanceRequestHandler(message, request) { - let instanceConnection = await this._connectInstance(message.data.instance_id); - return await request.send(instanceConnection, message.data); + let instanceId = message.data.instance_id; + let instanceConnection = await this._connectInstance(instanceId); + try { + return await request.send(instanceConnection, message.data); + + } catch (err) { + await this.stopInstance(instanceId); + throw err; + } } async createSaveRequestHandler(message, request) { let instanceId = message.data.instance_id; let instanceConnection = await this._connectInstance(instanceId); - await request.send(instanceConnection, message.data); + try { + await request.send(instanceConnection, message.data); + this.instanceConnections.delete(instanceId); + + } catch (err) { + await this.stopInstance(instanceId); + throw err; + } + } + + async stopInstance(instanceId) { + let instanceConnection = this.instanceConnections.get(instanceId); + await link.messages.stopInstance.send(instanceConnection, { instance_id: instanceId }); this.instanceConnections.delete(instanceId); } From cb1d744060d040c2cc9cc6a8ee0b1de729d9de2e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 21:55:45 +0100 Subject: [PATCH 07/11] Handle errors during master plugin initialization Catch errors thrown when constructing or initializing master pluigns and wrap them in PluginError to ensure that they are properly reported as plugin errors. --- master.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/master.js b/master.js index 92ebfcdc..cd932901 100644 --- a/master.js +++ b/master.js @@ -670,13 +670,18 @@ async function loadPlugins(pluginInfos) { let pluginLoadStarted = Date.now(); let MasterPlugin = plugin.BaseMasterPlugin; - if (pluginInfo.masterEntrypoint) { - ({ MasterPlugin } = require(`./plugins/${pluginInfo.name}/${pluginInfo.masterEntrypoint}`)); - } + try { + if (pluginInfo.masterEntrypoint) { + ({ MasterPlugin } = require(`./plugins/${pluginInfo.name}/${pluginInfo.masterEntrypoint}`)); + } - let masterPlugin = new MasterPlugin(pluginInfo); - await masterPlugin.init(); - plugins.set(pluginInfo.name, masterPlugin); + let masterPlugin = new MasterPlugin(pluginInfo); + await masterPlugin.init(); + plugins.set(pluginInfo.name, masterPlugin); + + } catch (err) { + throw new errors.PluginError(pluginInfo.name, err) + } console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`); /* From 090fbd6dff0455b08195f384e645d290518b77d2 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 21:58:56 +0100 Subject: [PATCH 08/11] Expose master interfaces to plugin Expose Express.js app, master config, instance and slave database, slave connections and metrics to the master plugins. --- lib/plugin.js | 4 +++- master.js | 12 +++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/plugin.js b/lib/plugin.js index 4764327a..e08fff22 100644 --- a/lib/plugin.js +++ b/lib/plugin.js @@ -74,8 +74,10 @@ class BaseInstancePlugin { } class BaseMasterPlugin { - constructor(info) { + constructor(info, master, metrics) { this.info = info; + this.master = master; + this.metrics = metrics; } /** diff --git a/master.js b/master.js index cd932901..fe207c1c 100644 --- a/master.js +++ b/master.js @@ -675,7 +675,9 @@ async function loadPlugins(pluginInfos) { ({ MasterPlugin } = require(`./plugins/${pluginInfo.name}/${pluginInfo.masterEntrypoint}`)); } - let masterPlugin = new MasterPlugin(pluginInfo); + let masterPlugin = new MasterPlugin( + pluginInfo, { app, config: masterConfig, db, slaveConnections }, { endpointHitCounter } + ); await masterPlugin.init(); plugins.set(pluginInfo.name, masterPlugin); @@ -684,14 +686,6 @@ async function loadPlugins(pluginInfos) { } console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`); - /* - main:new masterPlugin({ - TODO? - config, socketio: io, express: app, - db, - }), - pluginConfig, - });*/ } return plugins; } From d9fe408c42dfd638eb7bd92cc36bcfbd21520570 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 22:01:53 +0100 Subject: [PATCH 09/11] Fix master web interface Fix the breaking changes to the plugin struture and remove slave information that is no longer present. --- master.js | 3 ++- views/nav.ejs | 16 +++++++--------- views/nodes.ejs | 22 ++++++++-------------- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/master.js b/master.js index fe207c1c..853e5050 100644 --- a/master.js +++ b/master.js @@ -67,12 +67,13 @@ app.use(compression()); // dynamic HTML generations with EJS app.set('view engine', 'ejs'); -app.set('views', ['views', 'sharedPlugins']); +app.set('views', ['views', 'plugins']); // give ejs access to some interesting information app.use(function(req, res, next){ res.locals.res = res; res.locals.req = req; + res.locals.masterPlugins = masterPlugins; res.locals.slaves = db.slaves; res.locals.moment = moment; next(); diff --git a/views/nav.ejs b/views/nav.ejs index 661cf96c..a6b300a2 100644 --- a/views/nav.ejs +++ b/views/nav.ejs @@ -8,17 +8,15 @@ <% -for(let o in locals.masterPlugins) { - let plug = locals.masterPlugins[o]; +for (let plug of locals.masterPlugins.values()) { if ( - plug.main - && plug.main.ui - && typeof plug.main.ui.sidebar == "object" - && Array.isArray(plug.main.ui.sidebar) + plug + && plug.ui + && typeof plug.ui.sidebar == "object" + && Array.isArray(plug.ui.sidebar) ) { - let sidebarEntries = plug.main.ui.sidebar; - for (let i in sidebarEntries) { - let button = sidebarEntries[i]; + let sidebarEntries = plug.ui.sidebar; + for (let button of sidebarEntries) { if (button.getHtml && typeof button.getHtml == "function") { %> <%- button.getHtml() %><% } diff --git a/views/nodes.ejs b/views/nodes.ejs index 1e860c23..31606dbb 100644 --- a/views/nodes.ejs +++ b/views/nodes.ejs @@ -17,27 +17,21 @@ for (let [id, slave] of locals.slaves) { %>
  • ID: <%= id %>
  • -
  • Last seen: <%= moment(Number(slave.time)).format("DD.MM.YY, hh:mm") %>
  • -
  • Online players: <%= slave.playerCount %>
  • -
  • IP: <%= slave.publicIP %>:<%= slave.serverPort %>
  • -
  • Host: <%= slave.mac %>
<% - for (let o in locals.masterPlugins) { - let plug = locals.masterPlugins[o]; + for (let plug of locals.masterPlugins.values()) { if ( - plug.main - && plug.main.ui - && typeof plug.main.ui.slaveListing == "object" - && Array.isArray(plug.main.ui.slaveListing) + plug + && plug.ui + && typeof plug.ui.slaveListing == "object" + && Array.isArray(plug.ui.slaveListing) ) { - let slaveListingEntries = plug.main.ui.slaveListing; - for (let i in slaveListingEntries) { - let button = slaveListingEntries[i]; + let slaveListingEntries = plug.ui.slaveListing; + for (let button of slaveListingEntries) { if (button.getHtml && typeof button.getHtml == "function") { %> <% } From 737c9924b9e4540ebdb0df8067d6b521b9862ff5 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2020 22:09:22 +0100 Subject: [PATCH 10/11] Port and rename clusterioMod plugin Port the clusterioMod plugin to the new plugin system under the new subspace_storage name. --- .../subspace_storage}/dole.js | 86 +++---- .../subspace_storage}/dole_nn_base.js | 0 plugins/subspace_storage/info.js | 106 +++++++++ plugins/subspace_storage/instance.js | 94 ++++++++ plugins/subspace_storage/master.js | 217 ++++++++++++++++++ plugins/subspace_storage/routes.js | 55 +++++ plugins/subspace_storage/static/storage.js | 83 +++++++ .../subspace_storage}/test/missing.js | 4 +- plugins/subspace_storage/test/routes.js | 44 ++++ .../subspace_storage}/views/storage.ejs | 10 +- sharedPlugins/clusterioMod/config.js | 10 - sharedPlugins/clusterioMod/index.js | 194 ---------------- sharedPlugins/clusterioMod/masterPlugin.js | 101 -------- sharedPlugins/clusterioMod/routes.js | 208 ----------------- sharedPlugins/clusterioMod/static/storage.js | 208 ----------------- sharedPlugins/clusterioMod/test/routes.js | 109 --------- test/integration/clusterio.js | 3 + 17 files changed, 641 insertions(+), 891 deletions(-) rename {sharedPlugins/clusterioMod => plugins/subspace_storage}/dole.js (63%) rename {sharedPlugins/clusterioMod => plugins/subspace_storage}/dole_nn_base.js (100%) create mode 100644 plugins/subspace_storage/info.js create mode 100644 plugins/subspace_storage/instance.js create mode 100644 plugins/subspace_storage/master.js create mode 100644 plugins/subspace_storage/routes.js create mode 100644 plugins/subspace_storage/static/storage.js rename {sharedPlugins/clusterioMod => plugins/subspace_storage}/test/missing.js (77%) create mode 100644 plugins/subspace_storage/test/routes.js rename {sharedPlugins/clusterioMod => plugins/subspace_storage}/views/storage.ejs (75%) delete mode 100644 sharedPlugins/clusterioMod/config.js delete mode 100644 sharedPlugins/clusterioMod/index.js delete mode 100644 sharedPlugins/clusterioMod/masterPlugin.js delete mode 100644 sharedPlugins/clusterioMod/routes.js delete mode 100644 sharedPlugins/clusterioMod/static/storage.js delete mode 100644 sharedPlugins/clusterioMod/test/routes.js diff --git a/sharedPlugins/clusterioMod/dole.js b/plugins/subspace_storage/dole.js similarity index 63% rename from sharedPlugins/clusterioMod/dole.js rename to plugins/subspace_storage/dole.js index 4d732e85..7f2f2b0d 100644 --- a/sharedPlugins/clusterioMod/dole.js +++ b/plugins/subspace_storage/dole.js @@ -1,6 +1,18 @@ const doleNN = require("./dole_nn_base") -const Prometheus = require('prom-client'); +const prometheus = require("lib/prometheus"); + + +const prometheusNNDoleGauge = new prometheus.Gauge( + "clusterio_nn_dole_gauge", + "Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given", + { labels: ["resource"] } +); +const prometheusDoleFactorGauge = new prometheus.Gauge( + "clusterio_dole_factor_gauge", + "The current dole division factor for this resource", + { labels: ["resource"] } +); class neuralDole { @@ -17,14 +29,9 @@ class neuralDole { } constructor({ - items, gaugePrefix + items }){ // Set some storage variables for the dole divider - this.prometheusNNDoleGauge = new Prometheus.Gauge({ - name: gaugePrefix+'nn_dole_gauge', - help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given', - labelNames: ["itemName"], - }); this.items = items this.itemsLastTick = new Map(items._items) this.dole = {} @@ -32,32 +39,26 @@ class neuralDole { this.lastRequest = {} this.stats=[] this.debt= {} + } + doMagic() { + for (let [name, count] of this.items._items) { + let magicData = doleNN.Tick( + count, + this.dole[name], + this.itemsLastTick.get(name), + this.getRequestStats(name,5) + ) + this.stats[name]=this.stats[name] || [] + this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect + if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats - setInterval(()=>{ - for (let [name, count] of this.items._items) { - let magicData = doleNN.Tick( - count, - this.dole[name], - this.itemsLastTick.get(name), - this.getRequestStats(name,5) - ) - this.stats[name]=this.stats[name] || [] - this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect - if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats - - this.dole[name] = magicData[0] - // DONE handle magicData[1] for graphing for our users - this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0); - } - this.itemsLastTick = new Map(this.items._items) - }, 1000) + this.dole[name] = magicData[0] + // DONE handle magicData[1] for graphing for our users + prometheusNNDoleGauge.labels(name).set(magicData[1] || 0); + } + this.itemsLastTick = new Map(this.items._items) } - divider({ - res, - object, - masterConfig, - prometheusImportGauge - }){ + divider(object) { let magicData = doleNN.Dose( object.count, // numReq this.items.getItemCount(object.name), @@ -83,29 +84,18 @@ class neuralDole { // Remove item from DB and send it this.items.removeItem(object.name, magicData[0]); - if (masterConfig.get('log_item_transfers')) { - console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName); - } - prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0); - res.send({ - name: object.name, - count: magicData[0], - }) + return magicData[0]; } } // If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down. let _doleDivisionFactor = {}; function doleDivider({ - itemCount, object, items, - masterConfig, - prometheusDoleFactorGauge, - prometheusImportGauge, - req,res, }){ + let itemCount = items.getItemCount(object.name); const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase @@ -120,19 +110,15 @@ function doleDivider({ if (itemCount > object.count) { //If successful, increase dole _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1; - if (masterConfig.get('log_item_transfers')) { - console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName); - } items.removeItem(object.name, object.count) prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); - prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0); - res.send({count: object.count, name: object.name}); + return object.count; } else { // if we didn't have enough, attempt giving out a smaller amount next time _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2); prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0); - res.send({name:object.name, count:0}); + return 0; //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")"); } } diff --git a/sharedPlugins/clusterioMod/dole_nn_base.js b/plugins/subspace_storage/dole_nn_base.js similarity index 100% rename from sharedPlugins/clusterioMod/dole_nn_base.js rename to plugins/subspace_storage/dole_nn_base.js diff --git a/plugins/subspace_storage/info.js b/plugins/subspace_storage/info.js new file mode 100644 index 00000000..dbbfc683 --- /dev/null +++ b/plugins/subspace_storage/info.js @@ -0,0 +1,106 @@ +const link = require("lib/link"); +const config = require("lib/config"); + +class MasterConfigGroup extends config.PluginConfigGroup {} +MasterConfigGroup.groupName = "subspace_storage"; +MasterConfigGroup.define({ + name: "autosave_interval", + title: "Autosave Interval", + description: "Interval the storage is autosaved at in seconds.", + type: "number", + initial_value: 60, +}); +MasterConfigGroup.define({ + name: "division_method", + title: "Division Method", + description: "Method for dividing resource requests between instances.", + type: "string", + enum: ["simple", "dole", "neural_dole"], + initial_value: "simple", +}); +MasterConfigGroup.define({ + name: "log_item_transfers", + title: "Log Item Trassfers", + description: "Spam master console with item transfers done.", + type: "boolean", + initial_value: false, +}); +MasterConfigGroup.finalize(); + +class InstanceConfigGroup extends config.PluginConfigGroup {} +InstanceConfigGroup.groupName = "subspace_storage"; +InstanceConfigGroup.define({ + name: "log_item_transfers", + title: "Log Item Trassfers", + description: "Spam slave console with item transfers done.", + type: "boolean", + initial_value: false, +}); +InstanceConfigGroup.finalize(); + +// JSON schema for subspace storage items +const items = { + type: "array", + items: { + type: "array", + minItems: 2, + maxItems: 2, + items: [ + { type: "string" }, + { type: "integer" }, + ], + }, +} + +module.exports = { + name: "subspace_storage", + title: "Subspace Storage", + description: "Provides shared storage across instances for the Subspace Storage mod", + version: "2.0.0-alpha", + instanceEntrypoint: "instance", + InstanceConfigGroup, + + masterEntrypoint: "master", + MasterConfigGroup, + + messages: { + // XXX this should be a request to be reliable + place: new link.Event({ + type: "subspace_storage:place", + links: ["instance-slave", "slave-master"], + forwardTo: "master", + eventProperties: { + "instance_id": { type: "integer" }, + "items": items, + }, + }), + remove: new link.Request({ + type: "subspace_storage:remove", + links: ["instance-slave", "slave-master"], + forwardTo: "master", + requestProperties: { + "instance_id": { type: "integer" }, + "items": items, + }, + responseProperties: { + "items": items, + }, + }), + getStorage: new link.Request({ + type: "subspace_storage:get_storage", + links: ["instance-slave", "slave-master"], + forwardTo: "master", + responseProperties: { + "items": items, + }, + }), + updateStorage: new link.Event({ + type: "subspace_storage:update_storage", + links: ["master-slave", "slave-instance"], + broadcastTo: "instance", + eventProperties: { + "items": items, + }, + }), + }, +} diff --git a/plugins/subspace_storage/instance.js b/plugins/subspace_storage/instance.js new file mode 100644 index 00000000..5e7c9848 --- /dev/null +++ b/plugins/subspace_storage/instance.js @@ -0,0 +1,94 @@ +const fs = require("fs-extra"); + +const plugin = require("lib/plugin"); +const luaTools = require("lib/luaTools"); + +function unexpectedError(err) { + console.log("Unexpected error in subspace_storage") + console.log("------------------------------------") + console.log(err); +} + +class InstancePlugin extends plugin.BaseInstancePlugin { + async init() { + this.instance.server.on("ipc-subspace_storage:output", (output) => { + this.provideItems(output).catch(unexpectedError); + }); + this.instance.server.on("ipc-subspace_storage:orders", (orders) => { + this.requestItems(orders).catch(unexpectedError); + }); + } + + async onStart() { + this.pingId = setInterval(() => { + this.instance.server.sendRcon( + "/sc __subspace_storage__ global.ticksSinceMasterPinged = 0" + ).catch(unexpectedError); + }, 5000); + + let response = await this.info.messages.getStorage.send(this.instance); + let itemsJson = luaTools.escapeString(JSON.stringify(response.items)); + await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`); + } + + async onStop() { + clearInterval(this.pingId); + } + + // provide items -------------------------------------------------------------- + async provideItems(items) { + this.info.messages.place.send(this.instance, { + items, + instance_id: this.instance.config.get("instance.id"), + }); + + if (this.instance.config.get("subspace_storage.log_item_transfers")) { + console.log("Exported the following to master:") + console.log(items); + } + } + + // request items -------------------------------------------------------------- + async requestItems(items) { + // Request the items all at once + let response = await this.info.messages.remove.send(this.instance, { + instance_id: this.instance.config.get("instance.id"), + items, + }); + + if (!response.items.length) { + return; + } + + if (this.instance.config.get("subspace_storage.log_item_transfers")) { + console.log("Imported following from master:") + console.log(response.items); + } + + let itemsJson = luaTools.escapeString(JSON.stringify(response.items)); + await this.instance.server.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`); + } + + // combinator signals --------------------------------------------------------- + async updateStorageEventHandler(message, event) { + let items = message.data.items; + + // XXX this should be done on the lua side + // Ensure counts don't overflow + for (let item of items) { + if (item[1] > 0x7ffffff) { + item[1] = 0x7ffffff; + } + } + + // XXX this should be moved to instance/clusterio api + items.push(["signal-unixtime", Math.floor(Date.now()/1000)]); + + let itemsJson = luaTools.escapeString(JSON.stringify(items)); + await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`); + } +} + +module.exports = { + InstancePlugin, +}; diff --git a/plugins/subspace_storage/master.js b/plugins/subspace_storage/master.js new file mode 100644 index 00000000..8fbd78dd --- /dev/null +++ b/plugins/subspace_storage/master.js @@ -0,0 +1,217 @@ +"use strict"; +const fs = require("fs-extra"); +const path = require("path"); + +const database = require("lib/database"); +const plugin = require("lib/plugin"); +const prometheus = require("lib/prometheus"); + +const routes = require("./routes"); +const dole = require("./dole"); + + +const prometheusExportGauge = new prometheus.Gauge( + "clusterio_export_gauge", + "Resources exported by instance", + { labels: ["instance_id", "resource"] } +); +const prometheusImportGauge = new prometheus.Gauge( + "clusterio_import_gauge", + "Resources imported by instance", + { labels: ["instance_id", "resource"] } +); + + +class MasterPlugin extends plugin.BaseMasterPlugin { + async init() { + this.prometheusMasterInventoryGauge = new prometheus.Gauge( + "clusterio_master_inventory_gauge", + "Amount of resources stored on master", + { labels: ["resource"] } + ); + + this.ui = { + sidebar: [ + { + getHtml: () => ` + `, + }, + ], + }; + + this.items = await loadDatabase(this.master.config); + this.itemsLastUpdate = new Map(this.items._items.entries()); + this.autosaveId = setInterval(() => { + saveDatabase(this.master.config, this.items).catch(err => { + console.log("Unexpected error autosaving items in subspace_storage") + console.log("-----------------------------------------------------") + console.log(err); + }); + }, this.master.config.get("subspace_storage.autosave_interval") * 1000); + + this.neuralDole = new dole.neuralDole({ items: this.items }); + this.doleMagicId = setInterval(() => { + if (this.master.config.get("subspace_storage.division_method") === "neural_dole") { + this.neuralDole.doMagic() + } + }, 1000); + + routes.addApiRoutes(this.master.app, this.items, this.metrics.endpointHitCounter); + routes.addWebRoutes(this.master.app); + } + + updateStorage() { + let itemsToUpdate = new Map(); + for (let [name, count] of this.items._items) { + if (this.itemsLastUpdate.get(name) === count) { + continue; + } + itemsToUpdate.set(name, count); + } + + if (!itemsToUpdate.size) { + return; + } + + itemsToUpdate = [...itemsToUpdate.entries()]; + for (let slaveConnection of this.master.slaveConnections.values()) { + this.info.messages.updateStorage.send(slaveConnection, { items: itemsToUpdate }); + } + this.itemsLastUpdate = new Map(this.items._items.entries()); + } + + async getStorageRequestHandler(message) { + return { items: [...this.items._items.entries()] }; + } + + async placeEventHandler(message) { + let instanceId = message.data.instance_id; + + for (let item of message.data.items) { + this.items.addItem(item[0], item[1]); + prometheusExportGauge.labels(String(instanceId), item[0]).inc(item[1]); + } + + this.updateStorage(); + + if (this.master.config.get("subspace_storage.log_item_transfers")) { + console.log(`Imported the following from ${message.data.instance_id}:`) + console.log(message.data.items); + } + } + + async removeRequestHandler(message) { + let method = this.master.config.get("subspace_storage.division_method"); + let instanceId = message.data.instance_id; + + let itemsRemoved = []; + if (method === "simple") { + // Give out as much items as possible until there are 0 left. This + // might lead to one slave getting all the items and the rest nothing. + for (let item of message.data.items) { + let count = this.items.getItemCount(item[0]); + let toRemove = Math.min(count, item[1]); + if (toRemove > 0) { + this.items.removeItem(item[0], toRemove); + itemsRemoved.push([item[0], toRemove]); + } + } + + } else { + let instanceConfig = this.master.db.instances.get(instanceId); + let instanceName = instanceConfig ? instanceConfig.get("instance.name") : "unkonwn"; + + // use fancy neural net to calculate a "fair" dole division rate. + if (method === "neural_dole") { + for (let item of message.data.items) { + let count = neuralDole.divider({ name: item[0], count: item[1], instanceId, instanceName }); + if (count > 0) { + itemsRemoved.push([item[0], count]); + } + } + + // Use dole division. Makes it really slow to drain out the last little bit. + } else if (method === "dole") { + for (let item of message.data.items) { + let count = dole.doleDivider({ + object: { name: item[0], count: item[1], instanceId, instanceName}, + items: this.items, + }); + if (count > 0) { + itemsRemoved.push([item[0], count]); + } + } + + // Should not be possible + } else { + throw Error(`Unkown division_method ${method}`); + } + } + + if (itemsRemoved.length) { + for (let item of itemsRemoved) { + prometheusImportGauge.labels(String(instanceId), item[0]).inc(item[1]); + } + + this.updateStorage(); + + if (itemsRemoved.length && this.master.config.get("subspace_storage.log_item_transfers")) { + console.log(`Exported the following to ${instanceId}`); + console.log(itemsRemoved); + } + } + + return { + items: itemsRemoved, + } + } + + onMetrics() { + if (this.items) { + for (let [key, count] of this.items._items) { + this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0); + } + } + } + + async onExit() { + clearInterval(this.autosaveId); + clearInterval(this.doleMagicId); + await saveDatabase(this.master.config, this.items); + } +} + +async function loadDatabase(config) { + let itemsPath = path.resolve(config.get("master.database_directory"), "items.json"); + console.log(`Loading ${itemsPath}`); + try { + let content = await fs.readFile(itemsPath); + return new database.ItemDatabase(JSON.parse(content)); + + } catch (err) { + if (err.code === "ENOENT") { + console.log("Creating new item database"); + return new database.ItemDatabase(); + + } else { + throw err; + } + } +} + +async function saveDatabase(masterConfig, items) { + if (items && items.size < 50000) { + let file = path.resolve(masterConfig.get("master.database_directory"), "items.json"); + console.log(`writing ${file}`); + let content = JSON.stringify(items.serialize()); + await fs.outputFile(file, content); + } else if (items) { + console.error(`Item database too large, not saving (${items.size})`); + } +} + +module.exports = { + MasterPlugin, +}; diff --git a/plugins/subspace_storage/routes.js b/plugins/subspace_storage/routes.js new file mode 100644 index 00000000..9bcfdc2a --- /dev/null +++ b/plugins/subspace_storage/routes.js @@ -0,0 +1,55 @@ +const express = require("express"); +const path = require("path"); + +const authenticate = require("lib/authenticate"); +const prometheus = require("lib/prometheus"); + + +function addWebRoutes(app) { + app.use("/subspace_storage", express.static(path.join(__dirname, "static"))); + app.get("/subspace_storage/storage", (req, res) => { res.render("subspace_storage/views/storage"); }); +} + +function addApiRoutes(app, items, endpointHitCounter) { + + /** + * GET endpoint to read the masters current inventory of items. + * + * @memberof clusterioMaster + * @instance + * @alias api/inventory + * @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}] + */ + app.get("/api/inventory", function(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + // Check it and send it + var inventory = []; + for (let [name, count] of items._items) { + inventory.push({ name, count }); + } + res.send(JSON.stringify(inventory)); + }); + + /** + * GET endpoint to read the masters inventory as an object with key:value pairs + * + * @memberof clusterioMaster + * @instance + * @alias api/inventoryAsObject + * @returns {object} JSON {"iron-plate":100, "copper-plate":5} + */ + app.get("/api/inventoryAsObject", function(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + // Check it and send it + res.send(JSON.stringify(items.serialise())); + }); +} + +module.exports = { + addWebRoutes, + addApiRoutes, +} diff --git a/plugins/subspace_storage/static/storage.js b/plugins/subspace_storage/static/storage.js new file mode 100644 index 00000000..9a99819d --- /dev/null +++ b/plugins/subspace_storage/static/storage.js @@ -0,0 +1,83 @@ +// function to draw data we recieve from ajax requests +_lastData = []; +function drawcontents(data) { + data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box. + _lastData = data; + + // allow searching for multiple criteria separated by space + var searchArgs = document.querySelector("#search").value; + searchArgs = searchArgs.trim(); + searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')'; + const search = new RegExp(searchArgs, 'i'); + data = data.filter(function(item) { + return search.test(item.name); + }) + + sortByKey(data, "count"); + + const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically + const rows = table.children; + + //update existing rows or create new ones + data.forEach(function(item, i) { + // format count to have .s in it for large number readability (1,000,000) + item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); + + let row = rows[i]; + if(!row) { + row = document.createElement('tr'); + row.innerHTML = ""; + table.appendChild(row); + } + + const img = row.querySelector('img'); + const imgName = getImageFromName(item.name); + if(img.getAttribute('src') !== imgName) { + img.setAttribute('src',imgName); + } + + const name = row.querySelector('.name'); + if(name.textContent !== item.name) { + name.textContent = item.name; + } + + const count = row.querySelector('.count'); + if(count.textContent !== ''+item.count) { + count.textContent = item.count; + } + }); + + //remove excess rows, for example, after filtering + while (data.length < rows.length) { + table.removeChild(rows[data.length]); + } +} + +// get cluster inventory from master +function updateInventory() { + var xmlhttp = new XMLHttpRequest(); + xmlhttp.onreadystatechange = function() { + if(xmlhttp.readyState == 4 && xmlhttp.status == 200) { + let data = JSON.parse(xmlhttp.responseText); + drawcontents(data); + } + } + xmlhttp.open("GET", "/api/inventory", true); + xmlhttp.send(); +} +if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) { + setInterval(updateInventory, 500); +} else { + updateInventory(); +} + +// function to sort arrays of objects after a keys value +function sortByKey(array, key) { + array.sort(function(a, b) { + return b[key] - a[key]; + }); +} + +document.querySelector("#search").addEventListener('input', function() { + drawcontents(); +}); diff --git a/sharedPlugins/clusterioMod/test/missing.js b/plugins/subspace_storage/test/missing.js similarity index 77% rename from sharedPlugins/clusterioMod/test/missing.js rename to plugins/subspace_storage/test/missing.js index 5b080fef..7eadd460 100644 --- a/sharedPlugins/clusterioMod/test/missing.js +++ b/plugins/subspace_storage/test/missing.js @@ -1,7 +1,7 @@ // These are libraries that does not have any units test, they are // required here in order to track coverage for them. -require('../index'); -require('../masterPlugin'); +require('../instance'); +require('../master'); require('../dole'); require('../dole_nn_base'); diff --git a/plugins/subspace_storage/test/routes.js b/plugins/subspace_storage/test/routes.js new file mode 100644 index 00000000..7fceeef4 --- /dev/null +++ b/plugins/subspace_storage/test/routes.js @@ -0,0 +1,44 @@ +const assert = require("assert").strict; +const express = require("express"); +const bodyParser = require("body-parser"); +const request = require("supertest"); +const jwt = require("jsonwebtoken"); + +const authenticate = require('lib/authenticate'); +const database = require("lib/database"); +const prometheus = require("lib/prometheus"); + +const routes = require("../routes"); + + +const testSecret = "TestSecretDoNotUse"; +authenticate.setAuthSecret(testSecret); + +const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 }); + +describe("subspace_storage endpoints", function() { + const app = express(); + app.use(bodyParser.json({ limit: '10mb', })); + app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true })); + const items = new database.ItemDatabase(); + items.addItem("steel-plate", 20); + const gauge = new prometheus.Gauge("test_gauge", "test", { labels: ["route"] }); + routes.addApiRoutes(app, items, gauge); + + let persistentMaster = request(app); + describe("#GET /api/inventory", function() { + it("returns the masters current inventory", function() { + return persistentMaster.get("/api/inventory").expect(200).then(function(res) { + let inventory = JSON.parse(res.text); + assert.equal(typeof inventory, "object", "Inventory should be an object"); + assert(inventory.length >= 1, "There should be at least 1 entry in the inventory"); + + let contains20SteelPlate = false; + inventory.forEach(itemStack => { + if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true; + }); + assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory") + }); + }); + }); +}); diff --git a/sharedPlugins/clusterioMod/views/storage.ejs b/plugins/subspace_storage/views/storage.ejs similarity index 75% rename from sharedPlugins/clusterioMod/views/storage.ejs rename to plugins/subspace_storage/views/storage.ejs index c274964b..da151dd7 100644 --- a/sharedPlugins/clusterioMod/views/storage.ejs +++ b/plugins/subspace_storage/views/storage.ejs @@ -1,7 +1,7 @@ <%- include("head") %> - + @@ -31,18 +31,10 @@
-
-
-
-
-
- diff --git a/sharedPlugins/clusterioMod/config.js b/sharedPlugins/clusterioMod/config.js deleted file mode 100644 index 3ffae1eb..00000000 --- a/sharedPlugins/clusterioMod/config.js +++ /dev/null @@ -1,10 +0,0 @@ -/* - Clusterio plugin to allow for chat between instances. -*/ -module.exports = { - // Name of package. For display somewhere I guess. - name: "clusterioMod", - version: "1.0.0", - description: "Server side handling of functionality in the clusterio lua mod", - masterPlugin: "masterPlugin.js", -} diff --git a/sharedPlugins/clusterioMod/index.js b/sharedPlugins/clusterioMod/index.js deleted file mode 100644 index 3a845b63..00000000 --- a/sharedPlugins/clusterioMod/index.js +++ /dev/null @@ -1,194 +0,0 @@ -const needle = require("needle"); -const fs = require("fs-extra"); - -const objectOps = require("lib/objectOps"); -const fileOps = require("lib/fileOps"); - -const pluginConfig = require("./config"); - - -function ensureFileSync(path) { - if (!fs.existsSync(path)) { - fs.outputFileSync(path, ""); - } -} - -module.exports = class remoteCommands { - constructor(mergedConfig, messageInterface, extras){ - // Ugly global - global.confirmedOrders = []; - - this.messageInterface = messageInterface; - this.config = mergedConfig; - this.socket = extras.socket; - const that = this; - const instance = process.argv[3]; - const instancedirectory = this.config.instanceDirectory + '/' + instance; - - const needleOptionsWithTokenAuthHeader = { - headers: { - 'x-access-token': this.config.masterAuthToken - }, - }; - - ensureFileSync(instancedirectory + "/script-output/output.txt"); - ensureFileSync(instancedirectory + "/script-output/orders.txt"); - - // provide items -------------------------------------------------------------- - // trigger when something happens to output.txt - fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) { - // get array of lines in file - let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n"); - // if you found anything, reset the file - if (items[0]) { - fs.writeFileSync(instancedirectory + "/script-output/output.txt", ""); - } - for (let i = 0; i < items.length; i++) { - if (items[i]) { - let g = items[i].split(" "); - g[0] = g[0].replace("\u0000", ""); - // console.log("exporting " + JSON.stringify(g)); - // send our entity and count to the master for him to keep track of - needle.post(that.config.masterURL + '/api/place', { - name: g[0], - count: g[1], - instanceName: instance, // name of instance - instanceID: that.config.unique, // a hash computed from the randomly generated rcon password - }, needleOptionsWithTokenAuthHeader, function (err, resp, body) { - if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]); - if(that.config.logItemTransfers){ - if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`); - } - }); - } - } - }); - - // request items -------------------------------------------------------------- - setInterval(function () { - // get array of lines in file - let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n"); - // if we actually got anything from the file, proceed and reset file - if (items[0]) { - fs.writeFileSync(instancedirectory + "/script-output/orders.txt", ""); - // prepare a package of all our requested items in a more transfer friendly format - var preparedPackage = {}; - for (let i = 0; i < items.length; i++) { - (function (i) { - if (items[i]) { - items[i] = items[i].split(" "); - items[i][0] = items[i][0].replace("\u0000", ""); - items[i][0] = items[i][0].replace(",", ""); - if (preparedPackage[items[i][0]]) { - // if we have buffered some already, sum the new items - if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") { - preparedPackage[items[i][0]] = { - "name": items[i][0], - "count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]), - "instanceName":instance, - "instanceID":that.config.unique, - }; - // else just add em in without summing - } else if (typeof Number(items[i][1]) == "number") { - preparedPackage[items[i][0]] = { - "name": items[i][0], - "count": Number(items[i][1]), - "instanceName":instance, - "instanceID":that.config.unique, - }; - } - // this condition will NEVER be triggered but we know how that goes - } else if (typeof Number(items[i][1]) == "number") { - preparedPackage[items[i][0]] = { - "name": items[i][0], - "count": Number(items[i][1]), - "instanceName":instance, - "instanceID":that.config.unique, - }; - } - } - })(i); - } - // request our items, one item at a time - for (let i = 0; i < Object.keys(preparedPackage).length; i++) { - // console.log(preparedPackage[Object.keys(preparedPackage)[i]]); - function callback(err, response, body) { - if (response && response.body && typeof response.body == "object") { - // buffer confirmed orders - confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count} - if(that.config.logItemTransfers){ - console.log(`Imported ${response.body.count} ${response.body.name} from master`); - } - } - } - needle.post( - that.config.masterURL + '/api/remove', - preparedPackage[Object.keys(preparedPackage)[i]], - needleOptionsWithTokenAuthHeader, - callback - ); - } - // if we got some confirmed orders - // console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders)); - //if (!(confirmedOrders.length>0)){return;} - let cmd="local t={"; - for(let i=0;i { - needle.get(that.config.masterURL + '/api/inventory', function (err, response, body) { - if(err){ - console.error("Unable to get JSON master/api/inventory, master might be unaccessible"); - } else if (response && response.body) { - // Take the inventory we (hopefully) got and turn it into the format LUA accepts - if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver - try { - var inventory = JSON.parse(response.body); - var inventoryFrame = {}; - for (let i = 0; i < inventory.length; i++) { - inventoryFrame[inventory[i].name] = Number(inventory[i].count); - if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){ - inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice - } - } - inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000); - // console.log("RCONing inventory! " + JSON.stringify(inventoryFrame)); - let first = true; - let cmd="local s={"; - for (let key in inventoryFrame) - { - cmd+='["'+key+'"]='+inventoryFrame[key]+","; - } - if (!(cmd==="local s={")){ - that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')"); - } - that.messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')"); - } catch (e){ - console.log(e); - } - } - }); - }, 1550); - // Make sure world has its worldID - setTimeout(function(){ - messageInterface("/silent-command remote.call('clusterio','setWorldID',"+that.config.unique+")") - }, 20000); - } - async factorioOutput(data){ - - } -} -async function sleep(s){ - return new Promise((resolve, reject) => { - setTimeout(resolve, s*1000); - }); -} diff --git a/sharedPlugins/clusterioMod/masterPlugin.js b/sharedPlugins/clusterioMod/masterPlugin.js deleted file mode 100644 index efbd75ce..00000000 --- a/sharedPlugins/clusterioMod/masterPlugin.js +++ /dev/null @@ -1,101 +0,0 @@ -"use strict"; -const fs = require("fs-extra"); -const path = require("path"); -const Express = require("express"); -const ejs = require("ejs"); - -const database = require("lib/database"); -const routes = require("./routes"); - - -class masterPlugin { - constructor({ - config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter, - }) { - this.config = config; - this.pluginConfig = pluginConfig; - this.pluginPath = pluginPath; - this.io = socketio; - - this.express = express; - this.db = db, - this.Prometheus = Prometheus; - this.prometheusPrefix = prometheusPrefix; - this.endpointHitCounter = endpointHitCounter; - - this.prometheusMasterInventoryGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'master_inventory_gauge', - help: 'Amount of items stored on master', - labelNames: ["itemName"], - }); - - this.ui = { - sidebar: [ - { - getHtml: () => ` - `, - }, - ], - }; - } - - onMetrics() { - if (this.items) { - for (let [key, count] of this.items._items) { - this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0); - } - } - } - - async onLoadFinish() { - this.items = await loadDatabase(this.config); - this.autosaveId = setInterval(async () => { - await saveDatabase(this.config, this.items); - },this.config.autosaveInterval || 60000); - - routes.addApiRoutes( - this.express, this.config, this.db, this.items, - this.Prometheus, this.prometheusPrefix, this.endpointHitCounter, - ); - - routes.addWebRoutes(this.express); - } - - async onExit(){ - clearInterval(this.autosaveId); - await saveDatabase(this.config, this.items); - } -} - -async function loadDatabase(config) { - let itemsPath = path.resolve(config.databaseDirectory, "items.json"); - console.log(`Loading ${itemsPath}`); - try { - let content = await fs.readFile(itemsPath); - return new database.ItemDatabase(JSON.parse(content)); - - } catch (err) { - if (err.code === "ENOENT") { - console.log("Creating new item database"); - return new database.ItemDatabase(); - - } else { - throw err; - } - } -} - -async function saveDatabase(config, items) { - if (items && items.size < 50000) { - let file = path.resolve(config.databaseDirectory, "items.json"); - console.log(`writing ${file}`); - let content = JSON.stringify(items.serialize()); - await fs.outputFile(file, content); - } else if (items) { - console.error(`Item database too large, not saving (${items.size})`); - } -} - -module.exports = masterPlugin; diff --git a/sharedPlugins/clusterioMod/routes.js b/sharedPlugins/clusterioMod/routes.js deleted file mode 100644 index bd0baece..00000000 --- a/sharedPlugins/clusterioMod/routes.js +++ /dev/null @@ -1,208 +0,0 @@ -const express = require("express"); -const path = require("path"); - -const authenticate = require("lib/authenticate"); - -const dole = require("./dole"); - - -function addWebRoutes(app) { - app.use('/clusterioMod', express.static(path.join(__dirname, 'static'))); - app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); }); -} - -function addApiRoutes( - app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter, -) { - const prometheusExportGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'export_gauge', - help: 'Items exported by instance', - labelNames: ["instanceID", "itemName"], - }); - const prometheusImportGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'import_gauge', - help: 'Items imported by instance', - labelNames: ["instanceID", "itemName"], - }); - const prometheusDoleFactorGauge = new Prometheus.Gauge({ - name: prometheusPrefix+'dole_factor_gauge', - help: 'The current dole division factor for this item', - labelNames: ["itemName"], - }); - - // Only initialize neural network when it's enabled, otherwise it might override gauge - let neuralDole; - if(config.useNeuralNetDoleDivider) { - neuralDole=new dole.neuralDole({ - db, gaugePrefix: prometheusPrefix - }); - } - - /** - * POST endpoint for storing items in master's inventory. - - * @memberof clusterioMaster - * @instance - * @alias api/place - * @param {itemStack} itemStack the number and type of items to store (see typedef) - * @param {string} [itemStack.instanceID="unknown"] - * the unique/instanceID which is a numerical value for an instance - * @param {string} [itemStack.instanceName="unknown"] - * the name of an instance for identification in statistics, as - * provided when launching it. ex node client start [name] - * @returns {string} status either "success" or "failure" - */ - app.post("/api/place", authenticate.middleware, function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - let x; - try { - x = JSON.parse(req.body); - } catch (e) { - x = req.body; - } - if(!x.instanceName) { - x.instanceName = "unknown"; - } - if(!x.instanceID) { - x.instanceID = "unknown"; - for (let [instanceID, slave] of db.slaves) { - if(slave.instanceName == req.body.instanceName) { - x.instanceID = instanceID; - break; - } - } - } - if(x.instanceID - && x.instanceName - && !isNaN(Number(x.count)) - && x.name - && typeof x.name == "string"){ - if(config.logItemTransfers){ - console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")"); - } - // gather statistics - prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0); - // save items we get - var count = Number(req.body.count); - items.addItem(req.body.name, count); - - // Attempt confirming - res.send("success"); - } else { - res.send("failure"); - } - }); - - /** - * POST endpoint to remove items from DB when client orders items. - * - * @memberof clusterioMaster - * @instance - * @alias api/remove - * @param {itemStack} itemStack the name of and the number of items to remove (see typedef) - * @param {string} [itemStack.instanceID="unknown"] - * the unique/instanceID which is a numerical value for an instance - * @param {string} [itemStack.instanceName="unknown"] - * the name of an instance for identification in statistics, as - * provided when launching it. ex node client start [name] - * @returns {itemStack} - * the number of items actually removed, may be lower than what was - * asked for due to shortages. - */ - app.post("/api/remove", authenticate.middleware, function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - // save items we get - var object = req.body; - if(!object.instanceID) { - object.instanceID = "unknown" - } - if(!object.instanceName) { - object.instanceName = "unknown"; - } - if (db.slaves.has(String(object.instanceID))) { - object.instanceName = db.slaves.get(String(object.instanceID)).instanceName; - } - let itemCount = items.getItemCount(object.name); - if ( - (config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" ) - && object.name != "electricity" - ) { - if(config.logItemTransfers){ - console.log('failure could not find ' + object.name); - } - res.send({name:object.name, count:0}); - } else if(config.disableFairItemDistribution){ - // Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing. - let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount); - items.removeItem(object.name, numberToRemove); - res.send({count: numberToRemove, name: object.name}); - - // track statistics and do graphing things - prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0); - } else if(config.useNeuralNetDoleDivider){ - // use fancy neural net to calculate a "fair" dole division rate. - neuralDole.divider({ - res, - object, - config, - prometheusImportGauge - }) - } else { - // Use dole division. Makes it really slow to drain out the last little bit. - dole.doleDivider({ - itemCount, - object, - db, - config, - prometheusDoleFactorGauge, - prometheusImportGauge, - req,res, - }) - } - }); - - /** - * GET endpoint to read the masters current inventory of items. - * - * @memberof clusterioMaster - * @instance - * @alias api/inventory - * @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}] - */ - app.get("/api/inventory", function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - // Check it and send it - var inventory = []; - for (let [name, count] of items._items) { - inventory.push({ name, count }); - } - res.send(JSON.stringify(inventory)); - }); - - /** - * GET endpoint to read the masters inventory as an object with key:value pairs - * - * @memberof clusterioMaster - * @instance - * @alias api/inventoryAsObject - * @returns {object} JSON {"iron-plate":100, "copper-plate":5} - */ - app.get("/api/inventoryAsObject", function(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - // Check it and send it - res.send(JSON.stringify(items.serialise())); - }); -} - -module.exports = { - addWebRoutes, - addApiRoutes, -} diff --git a/sharedPlugins/clusterioMod/static/storage.js b/sharedPlugins/clusterioMod/static/storage.js deleted file mode 100644 index 0a8f561f..00000000 --- a/sharedPlugins/clusterioMod/static/storage.js +++ /dev/null @@ -1,208 +0,0 @@ -// function to draw data we recieve from ajax requests -_lastData = []; -function drawcontents(data) { - data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box. - _lastData = data; - - // allow searching for multiple criteria separated by space - var searchArgs = document.querySelector("#search").value; - searchArgs = searchArgs.trim(); - searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')'; - const search = new RegExp(searchArgs, 'i'); - data = data.filter(function(item) { - return search.test(item.name); - }) - - sortByKey(data, "count"); - - const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically - const rows = table.children; - - //update existing rows or create new ones - data.forEach(function(item, i) { - // format count to have .s in it for large number readability (1,000,000) - item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); - - let row = rows[i]; - if(!row) { - row = document.createElement('tr'); - row.innerHTML = ""; - table.appendChild(row); - } - - const img = row.querySelector('img'); - const imgName = getImageFromName(item.name); - if(img.getAttribute('src') !== imgName) { - img.setAttribute('src',imgName); - } - - const name = row.querySelector('.name'); - if(name.textContent !== item.name) { - name.textContent = item.name; - row.onclick = ()=>{ - makeItemGraphs(item.name); - } - } - - const count = row.querySelector('.count'); - if(count.textContent !== ''+item.count) { - count.textContent = item.count; - } - }); - - //remove excess rows, for example, after filtering - while (data.length < rows.length) { - table.removeChild(rows[data.length]); - } -} - -// get cluster inventory from master -function updateInventory() { - var xmlhttp = new XMLHttpRequest(); - xmlhttp.onreadystatechange = function() { - if(xmlhttp.readyState == 4 && xmlhttp.status == 200) { - let data = JSON.parse(xmlhttp.responseText); - drawcontents(data); - } - } - xmlhttp.open("GET", "/api/inventory", true); - xmlhttp.send(); -} -if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) { - setInterval(updateInventory, 500); -} else { - updateInventory(); -} - -// function to sort arrays of objects after a keys value -function sortByKey(array, key) { - array.sort(function(a, b) { - return b[key] - a[key]; - }); -} - -document.querySelector("#search").addEventListener('input', function() { - drawcontents(); -}); - -// Make graph of cluster IO items -/*getJSON("/api/getStats", function(err, data){ - -}*/ -function getItemStatData(item, callback){ - var xhr = new XMLHttpRequest(); - var url = "/api/getStats"; - xhr.open("POST", url, true); - xhr.setRequestHeader("Content-type", "application/json"); - xhr.onreadystatechange = function () { - if (xhr.readyState === 4 && xhr.status === 200) { - var json = JSON.parse(xhr.responseText); - console.log(json.email + ", " + json.password); - callback(json); - } - }; - var requestData = JSON.stringify({ - "statistic": "place", - "itemName": "shit", - }); - xhr.send(data); -} - -function makeItemGraphs(itemName){ - getJSON("/api/slaves", function(err, slaves){ - if(err) throw err; - if(typeof slaves == "string"){ - slaves = JSON.parse(slaves); - } - let instanceIDs = Object.keys(slaves); - let HTML = "" - // Draw in/out chart for unregistered slaves as well - instanceIDs.unshift("unknown"); - instanceIDs.forEach(function(instanceID){ - if(instanceID == "unknown" || Date.now() - slaves[instanceID].time < 120000 || JSON.parse(localStorage.settings)["Display offline slaves"]){ - HTML += '
'; - ["place", "remove"].forEach(statistic => { - console.log("Making chart for "+instanceID+"'s "+statistic+" statistic"); - - setTimeout(function(){ - let xmlhttp = new XMLHttpRequest(); // new HttpRequest instance - xmlhttp.open("POST", "/api/getStats"); - xmlhttp.setRequestHeader("Content-Type", "application/json"); - xmlhttp.onreadystatechange = function(){ - console.log(xmlhttp.status); - if(xmlhttp.readyState === XMLHttpRequest.DONE && xmlhttp.status === 200) { - // console.log(xmlhttp.responseText); - if(isJSON(xmlhttp.responseText) && JSON.parse(xmlhttp.responseText).statusForDebugging === undefined){ - let slaveDisplayName = "unknown"; - try { - slaveDisplayName = slaves[instanceID].instanceName; - } catch(e) {} - let whatGraphIsIt = "Item IO from "; - // if(statistic == "remove") whatGraphIsIt = "Items taken from cluster by"; - - let chartData = JSON.parse(xmlhttp.response).data; - switch(statistic){ - case "place": - chartData.name = "Items exported:"; - break; - case "remove": - chartData.name = "Items imported:"; - break; - } - drawChartWhenCalledTwice(chartData, whatGraphIsIt + " " + slaveDisplayName); - //drawChart(statistic+"contentGraph"+instanceID, [JSON.parse(xmlhttp.response).data], `${whatGraphIsIt} ${slaveDisplayName} (${instanceID}) ${itemName}`); - // console.log(JSON.parse(xmlhttp.response).data); - } - } - } - xmlhttp.send(JSON.stringify({statistic:statistic, itemName:itemName, instanceID:instanceID})); - },1); - }); - let chartLines = []; - function drawChartWhenCalledTwice(data, slaveDisplayName){ - chartLines.push(data); - if(chartLines.length >= 2){ - drawChart("contentGraph"+instanceID, chartLines, `${slaveDisplayName} (${instanceID}) ${itemName}`); - } - } - } - }); - document.querySelector("#graphDisplay > #storageGraphContainer").innerHTML = HTML; - }); -} -let chartsByID = {}; -function drawChart(selector, chartData, title) { - // selector is ID of element, ex "chartContainer" or "-123199123" - console.log(chartData) - chartsByID[selector] = new CanvasJS.Chart(selector, { - title:{ - text: title || "Production graph" - }, - toolTip:{ - shared: true, - // content: "{name}: {y}", - }, - zoomEnabled: true, - axisY:{ - includeZero: true, - }, - legend: { - cursor: "pointer", - itemclick: function (e) { - //console.log("legend click: " + e.dataPointIndex); - //console.log(e); - if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) { - e.dataSeries.visible = false; - } else { - e.dataSeries.visible = true; - } - e.chart.render(); - } - }, - data: chartData - }); - chart = chartsByID[selector]; - // console.log(chart) - chart.render(); - return chart; -} diff --git a/sharedPlugins/clusterioMod/test/routes.js b/sharedPlugins/clusterioMod/test/routes.js deleted file mode 100644 index a2872284..00000000 --- a/sharedPlugins/clusterioMod/test/routes.js +++ /dev/null @@ -1,109 +0,0 @@ -const assert = require("assert").strict; -const express = require("express"); -const bodyParser = require("body-parser"); -const request = require("supertest"); -const Prometheus = require('prom-client'); -const jwt = require("jsonwebtoken"); - -const authenticate = require('lib/authenticate'); -const database = require("lib/database"); - -const routes = require("../routes"); - - -const testSecret = "TestSecretDoNotUse"; -authenticate.setAuthSecret(testSecret); - -const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 }); - -describe("clusterioMod endpoints", function() { - const app = express(); - app.use(bodyParser.json({ limit: '10mb', })); - app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true })); - const config = { - disableFairItemDistribution: true, - }; - const items = new database.ItemDatabase(); - const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] }); - routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge); - - let persistentMaster = request(app); - describe("#POST /api/place", function() { - it("adds an itemStack to the masters inventory", function() { - return persistentMaster.post("/api/place").send({ - name:"steel-plate", - count:20, - instanceName:"unitTest" - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.text, "success", "something went wrong with the request") - }); - }); - }); - describe("#GET /api/inventory", function() { - it("returns the masters current inventory", function() { - return persistentMaster.get("/api/inventory").expect(200).then(function(res) { - let inventory = JSON.parse(res.text); - assert.equal(typeof inventory, "object", "Inventory should be an object"); - assert(inventory.length >= 1, "There should be at least 1 entry in the inventory"); - - let contains20SteelPlate = false; - inventory.forEach(itemStack => { - if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true; - }); - assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory") - }); - }); - }); - describe("#POST /api/remove", function() { - it("returns an itemStack of how many items were removed", function() { - return persistentMaster.post("/api/remove").send({ - name:"steel-plate", - count:10 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`); - }); - }); - it("works correctly for items named addItem or removeItem", async function() { - function add(item) { - return persistentMaster.post("/api/place").send({ - name: item, - count: 10 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`); - }) - } - - function remove(item) { - return persistentMaster.post("/api/remove").send({ - name: item, - count: 10 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.deepEqual(res.body, {"name":item,"count":10}); - }); - } - - await Promise.all([add("addItem"), add("removeItem")]); - await Promise.all([remove("addItem"), remove("removeItem")]); - }); - it("returns an empty itemStack if you don't have any of the item you request", function() { - return persistentMaster.post("/api/remove").send({ - name:"imaginaryItem", - count:999999 - }).set( - "X-Access-Token", token - ).expect(200).then(function(res) { - assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for"); - assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere"); - assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object"); - }); - }); - }); -}); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index debd0714..2815856d 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -86,6 +86,9 @@ describe("Integration of Clusterio", function() { it("creates the instance", async function() { await clusterctl._commands.get("create-instance").run({id: 44, name: "test"}, testControl); assert(master._db.instances.has(44), "Instance was not created"); + + // XXX breaks tests. + master._db.instances.get(44).set("subspace_storage.enabled", false); }); }); From 9dac57b36b0ec93c9e74e52f0b196f7aa41b5f59 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 15 Mar 2020 17:41:52 +0100 Subject: [PATCH 11/11] Port Research Sync plugin Rewriwe the researchSync plugin into plugins/research_sync. --- plugins/research_sync/info.js | 80 ++++ plugins/research_sync/instance.js | 81 ++++ plugins/research_sync/master.js | 184 +++++++++ plugins/research_sync/module/module.json | 4 + plugins/research_sync/module/sync.lua | 215 ++++++++++ plugins/research_sync/test/missing.js | 5 + sharedPlugins/researchSync/config.js | 12 - sharedPlugins/researchSync/dumpResearch.lua | 24 -- sharedPlugins/researchSync/enableResearch.lua | 33 -- sharedPlugins/researchSync/index.js | 385 ------------------ sharedPlugins/researchSync/test/index.js | 128 ------ sharedPlugins/researchSync/updateProgress.lua | 33 -- test/integration/clusterio.js | 1 + 13 files changed, 570 insertions(+), 615 deletions(-) create mode 100644 plugins/research_sync/info.js create mode 100644 plugins/research_sync/instance.js create mode 100644 plugins/research_sync/master.js create mode 100644 plugins/research_sync/module/module.json create mode 100644 plugins/research_sync/module/sync.lua create mode 100644 plugins/research_sync/test/missing.js delete mode 100644 sharedPlugins/researchSync/config.js delete mode 100644 sharedPlugins/researchSync/dumpResearch.lua delete mode 100644 sharedPlugins/researchSync/enableResearch.lua delete mode 100644 sharedPlugins/researchSync/index.js delete mode 100644 sharedPlugins/researchSync/test/index.js delete mode 100644 sharedPlugins/researchSync/updateProgress.lua diff --git a/plugins/research_sync/info.js b/plugins/research_sync/info.js new file mode 100644 index 00000000..2e184014 --- /dev/null +++ b/plugins/research_sync/info.js @@ -0,0 +1,80 @@ +const link = require("lib/link"); + + +// schema used for syncing technologies to and from the master +const technologies = { + type: "array", + items: { + type: "array", + minItems: 4, + maxItems: 4, + items: [ + { type: "string" }, + { type: "integer" }, + { type: ["null", "number"] }, + { type: "boolean" }, + ] + }, +}; + +module.exports = { + name: "research_sync", + title: "Research Sync", + description: "Synchronises technology research progress between instances.", + version: "2.0.0-alpha", + instanceEntrypoint: "instance", + masterEntrypoint: "master", + + messages: { + contribution: new link.Event({ + type: "research_sync:contribution", + links: ["instance-slave", "slave-master"], + forwardTo: "master", + eventProperties: { + "name": { type: "string" }, + "level": { type: "integer" }, + "contribution": { type: "number" }, + }, + }), + progress: new link.Event({ + type: "research_sync:progress", + links: ["master-slave", "slave-instance"], + broadcastTo: "instance", + eventProperties: { + "technologies": { + type: "array", + items: { + additionalProperties: false, + required: ["name", "level", "progress"], + properties: { + "name": { type: "string" }, + "level": { type: "integer" }, + "progress": { type: "number" }, + }, + }, + }, + }, + }), + finished: new link.Event({ + type: "research_sync:finished", + links: ["instance-slave", "slave-master", "master-slave", "slave-instance"], + forwardTo: "master", + broadcastTo: "instance", + eventProperties: { + "name": { type: "string" }, + "level": { type: "integer" }, + }, + }), + syncTechnologies: new link.Request({ + type: "research_sync:sync_technologies", + links: ["instance-slave", "slave-master"], + forwardTo: "master", + requestProperties: { + "technologies": technologies, + }, + responseProperties: { + "technologies": technologies, + }, + }), + }, +} diff --git a/plugins/research_sync/instance.js b/plugins/research_sync/instance.js new file mode 100644 index 00000000..ffbeb3a0 --- /dev/null +++ b/plugins/research_sync/instance.js @@ -0,0 +1,81 @@ +const needle = require("needle"); + +const plugin = require("lib/plugin"); +const luaTools = require("lib/luaTools"); + + +function unexpectedError(err) { + console.log("Unexpected error in research_sync") + console.log("---------------------------------") + console.log(err); +} + +class InstancePlugin extends plugin.BaseInstancePlugin { + async init() { + this.instance.server.on("ipc-research_sync:contribution", (tech) => { + this.researchContribution(tech).catch(unexpectedError); + }); + this.instance.server.on("ipc-research_sync:finished", (tech) => { + this.researchFinished(tech).catch(unexpectedError); + }); + } + + async researchContribution(tech) { + this.info.messages.contribution.send(this.instance, tech); + } + + async progressEventHandler(message) { + let techsJson = luaTools.escapeString(JSON.stringify(message.data.technologies)); + await this.instance.server.sendRcon(`/sc research_sync.update_progress("${techsJson}")`, true); + } + + async researchFinished(tech) { + this.info.messages.finished.send(this.instance, tech); + } + + async finishedEventHandler(message) { + let { name, level, researched } = message.data; + await this.instance.server.sendRcon( + `/sc research_sync.research_technology("${luaTools.escapeString(name)}", ${level})`, true + ); + } + + async onStart() { + let dumpJson = await this.instance.server.sendRcon("/sc research_sync.dump_technologies()"); + let techsToSend = []; + let instanceTechs = new Map(); + for (let tech of JSON.parse(dumpJson)) { + techsToSend.push([ + tech.name, + tech.level, + tech.progress || null, + tech.researched, + ]); + instanceTechs.set(tech.name, tech); + } + + let response = await this.info.messages.syncTechnologies.send(this.instance, { technologies: techsToSend }); + let techsToSync = []; + for (let masterTech of response.technologies) { + let [name, level, progress, researched] = masterTech; + let instanceTech = instanceTechs.get(name); + if ( + !instanceTech + || instanceTech.level !== level + || (instanceTech.progress || null) !== progress + || instanceTech.researched !== researched + ) { + techsToSync.push(masterTech); + } + } + + if (techsToSync.length) { + let syncJson = luaTools.escapeString(JSON.stringify(techsToSync)); + await this.instance.server.sendRcon(`/sc research_sync.sync_technologies("${syncJson}")`, true); + } + } +} + +module.exports = { + InstancePlugin, +} diff --git a/plugins/research_sync/master.js b/plugins/research_sync/master.js new file mode 100644 index 00000000..deaf10c4 --- /dev/null +++ b/plugins/research_sync/master.js @@ -0,0 +1,184 @@ +const fs = require("fs-extra"); +const path = require("path"); + +const plugin = require("lib/plugin"); + + +class MasterPlugin extends plugin.BaseMasterPlugin { + async init() { + this.technologies = await loadTechnologies(this.master.config); + this.lastProgressBroadcast = Date.now(); + this.progressBroadcastId = null; + this.progressToBroadcast = new Set(); + } + + async onExit() { + if (this.progressBroadcastId) { + clearTimeout(this.progressBroadcastId); + } + await saveTechnologies(this.master.config, this.technologies); + } + + registerProgress() { + // Rate limit progress broadcasts to one per second + if (Date.now() < this.lastProgressBroadcast + 1000) { + if (!this.progressBroadcastId) { + this.progressBroadcastId = setTimeout(() => { + this.broadcastProgress(); + this.progressBroadcastId = null; + }, 2000); + } + + } else { + if (this.progressBroadcastId) { + clearTimeout(this.progressBroadcastId); + } + this.broadcastProgress(); + } + } + + broadcastProgress() { + let techs = [] + for (let name of this.progressToBroadcast) { + let tech = this.technologies.get(name); + if (tech.progress) { + techs.push({ name, level: tech.level, progress: tech.progress }); + } + } + this.progressToBroadcast.clear(); + + this.lastProgressBroadcast = Date.now(); + if (techs.length) { + for (let slaveConnection of this.master.slaveConnections.values()) { + this.info.messages.progress.send(slaveConnection, { technologies: techs }); + } + } + } + + async contributionEventHandler(message) { + let { name, level, contribution } = message.data; + let tech = this.technologies.get(name); + if (!tech) { + tech = { level, progress: 0, researched: false }; + this.technologies.set(name, tech) + + // Ignore contribution to already researched technologies + } else if (tech.level > level || tech.level === level && tech.researched) { + return; + } + + let newProgress = tech.progress + contribution; + if (newProgress < 1) { + tech.progress = newProgress; + this.progressToBroadcast.add(name); + this.registerProgress(); + + } else { + tech.researched = true; + tech.progress = null; + this.progressToBroadcast.delete(name); + + for (let slaveConnection of this.master.slaveConnections.values()) { + this.info.messages.finished.send(slaveConnection, { + name, + level: tech.level + }); + } + } + } + + async finishedEventHandler(message) { + let { name, level } = message.data; + let tech = this.technologies.get(name); + if (!tech || tech.level <= level) { + this.progressToBroadcast.delete(name); + this.technologies.set(name, { level, progress: null, researched: true }) + } + } + + async syncTechnologiesRequestHandler(message) { + function baseLevel(name) { + let match = /-(\d+)$/.exec(name); + if (!match) { + return 1; + } else { + return Number.parseInt(match[1], 10); + } + } + + for (let instanceTech of message.data.technologies) { + let [name, level, progress, researched] = instanceTech; + let tech = this.technologies.get(name); + if (!tech) { + this.technologies.set(name, { level, progress, researched }); + if (progress) { + this.progressToBroadcast.add(name); + } else if (researched || baseLevel(name) != level) { + for (let slaveConnection of this.master.slaveConnections.values()) { + this.info.messages.finished.send(slaveConnection, { name, level }); + } + } + + } else { + if (tech.level > level || tech.level === level && tech.researched) { + continue; + } + + if (tech.level < level || researched) { + tech.level = level; + tech.progress = progress; + tech.researched = researched; + for (let slaveConnection of this.master.slaveConnections.values()) { + this.info.messages.finished.send(slaveConnection, { name, level }); + } + + if (progress) { + this.progressToBroadcast.add(name); + } else { + this.progressToBroadcast.delete(name); + } + + } else if (tech.progress < progress) { + tech.progress = progress; + this.progressToBroadcast.add(name); + } + } + } + this.registerProgress(); + + let technologies = []; + for (let [name, tech] of this.technologies) { + technologies.push([ name, tech.level, tech.progress, tech.researched ]) + } + + return { technologies } + } +} + +async function loadTechnologies(masterConfig) { + let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json"); + console.log(`Loading ${filePath}`); + try { + return new Map(JSON.parse(await fs.readFile(filePath))); + + } catch (err) { + if (err.code === "ENOENT") { + console.log("Creating new technologies database"); + return new Map(); + + } else { + throw err; + } + } +} + +async function saveTechnologies(masterConfig, technologies) { + let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json"); + console.log(`writing ${filePath}`); + await fs.outputFile(filePath, JSON.stringify([...technologies.entries()], null, 4)); +} + + +module.exports = { + MasterPlugin +} diff --git a/plugins/research_sync/module/module.json b/plugins/research_sync/module/module.json new file mode 100644 index 00000000..c2b055e9 --- /dev/null +++ b/plugins/research_sync/module/module.json @@ -0,0 +1,4 @@ +{ + "name": "research_sync", + "load": ["sync.lua"] +} diff --git a/plugins/research_sync/module/sync.lua b/plugins/research_sync/module/sync.lua new file mode 100644 index 00000000..f7c88aae --- /dev/null +++ b/plugins/research_sync/module/sync.lua @@ -0,0 +1,215 @@ +local clusterio_api = require("modules/clusterio/api") + +local sync = {} + + +local function get_technology_progress(tech) + if tech == tech.force.current_research then + return tech.force.research_progress + else + return tech.force.get_saved_technology_progress(tech.name) + end +end + +local function set_technology_progress(tech, progress) + if tech == tech.force.current_research then + tech.force.research_progress = progress + else + tech.force.set_saved_technology_progress(tech.name, progress) + end +end + +sync.events = {} +sync.events[clusterio_api.events.on_server_startup] = function(event) + if not global.research_sync then + global.research_sync = { + technologies = {}, + } + end + + -- Used when syncing completed technologies from the master + global.research_sync.ignore_research_finished = false + + local force = game.forces["player"] + for _, tech in pairs(force.technologies) do + if tech.enabled then + local progress = get_technology_progress(tech) + global.research_sync.technologies[tech.name] = { + level = tech.level, + researched = tech.researched, + progress = progress, + } + end + end +end + +local function get_contribution(tech) + local progress = get_technology_progress(tech) + if not progress then + return 0, nil + end + + local prev_tech = global.research_sync.technologies[tech.name] + if prev_tech.progress and prev_tech.level == tech.level then + return progress - prev_tech.progress, progress + else + return progress, progress + end +end + +local function send_contribution(tech) + local contribution, progress = get_contribution(tech) + if contribution ~= 0 then + clusterio_api.send_json("research_sync:contribution", { + name = tech.name, + level = tech.level, + contribution = contribution, + }) + global.research_sync.technologies[tech.name].progress = progress + end +end + +sync.events[defines.events.on_research_started] = function(event) + local tech = event.last_research + if tech then + send_contribution(tech) + end +end + +sync.events[defines.events.on_research_finished] = function(event) + if global.research_sync.ignore_research_finished then + return + end + + local tech = event.research + global.research_sync.technologies[tech.name] = { + level = tech.level, + researched = tech.researched, + } + + local level = tech.level + if not tech.researched then + level = level - 1 + end + + clusterio_api.send_json("research_sync:finished", { + name = tech.name, + level = level, + }) +end + +sync.on_nth_tick = {} +sync.on_nth_tick[79] = function(event) + local tech = game.forces["player"].current_research + if tech then + send_contribution(tech) + end +end + +research_sync = {} +function research_sync.dump_technologies() + local force = game.forces["player"] + + local techs = {} + for _, tech in pairs(force.technologies) do + if tech.enabled then + table.insert(techs, { + name = tech.name, + level = tech.level, + progress = get_technology_progress(tech), + researched = tech.researched, + }) + end + end + + if #techs == 0 then + rcon.print("[]") + else + rcon.print(game.table_to_json(techs)) + end +end + +function research_sync.sync_technologies(data) + local force = game.forces["player"] + + local nameIndex = 1 + local levelIndex = 2 + local progressIndex = 3 + local researchedIndex = 4 + + global.research_sync.ignore_research_finished = true + for _, tech_data in pairs(game.json_to_table(data)) do + local tech = force.technologies[tech_data[nameIndex]] + if tech and tech.enabled and tech.level <= tech_data[levelIndex] then + tech.level = tech_data[levelIndex] + local progress + if tech_data[researchedIndex] then + tech.researched = true + progress = nil + elseif tech_data[progressIndex] then + send_contribution(tech) + progress = tech_data[progressIndex] + set_technology_progress(tech, progress) + else + progress = get_technology_progress(tech) + end + + global.research_sync.technologies[tech.name] = { + level = tech.level, + researched = tech.researched, + progress = progress, + } + end + end + global.research_sync.ignore_research_finished = false +end + +function research_sync.update_progress(data) + local techs = game.json_to_table(data) + local force = game.forces["player"] + + for _, masterTech in ipairs(techs) do + local tech = force.technologies[masterTech.name] + if tech and tech.enabled and tech.level == masterTech.level then + send_contribution(tech) + set_technology_progress(tech, masterTech.progress) + global.research_sync.technologies[tech.name] = { + level = tech.level, + progress = masterTech.progress + } + end + end +end + +function research_sync.research_technology(name, level) + local force = game.forces["player"] + local tech = force.technologies[name] + if not tech or not tech.enabled or tech.level > level then + return + end + + global.research_sync.ignore_research_finished = true + if tech == force.current_research and tech.level == level then + force.research_progress = 1 + + elseif tech.level < level or tech.level == level and not tech.researched then + tech.level = level + tech.researched = true + + if tech.name:find("-%d+$") then + game.print {"", "Researched ", {"technology-name." .. tech.name:gsub("-%d+$", "")}, level} + else + game.print {"", "Researched ", {"technology-name." .. tech.name}} + end + game.play_sound { path = "utility/research_completed" } + end + global.research_sync.ignore_research_finished = false + + global.research_sync.technologies[tech.name] = { + level = tech.level, + researched = tech.researched, + } +end + + +return sync diff --git a/plugins/research_sync/test/missing.js b/plugins/research_sync/test/missing.js new file mode 100644 index 00000000..c0b5b5ab --- /dev/null +++ b/plugins/research_sync/test/missing.js @@ -0,0 +1,5 @@ +// These are libraries that does not have any units test, they are +// required here in order to track coverage for them. + +require('../instance'); +require('../master'); diff --git a/sharedPlugins/researchSync/config.js b/sharedPlugins/researchSync/config.js deleted file mode 100644 index 8d1210dd..00000000 --- a/sharedPlugins/researchSync/config.js +++ /dev/null @@ -1,12 +0,0 @@ - - -module.exports = { - // Name of package. For display somewhere I guess. - name: "researchSync", - version: "1.0.0", - arguments: ["index.js"], - description: "syncs research across servers", - // We'll send everything in this file to your stdin. Beware. - scriptOutputFileSubscription: "researchSync.txt", - fileReadDelay: 50 -} diff --git a/sharedPlugins/researchSync/dumpResearch.lua b/sharedPlugins/researchSync/dumpResearch.lua deleted file mode 100644 index 27ec7946..00000000 --- a/sharedPlugins/researchSync/dumpResearch.lua +++ /dev/null @@ -1,24 +0,0 @@ - -local data = {}; -local force = game.forces["player"]; - -for _, tech in pairs(force.technologies) do - local progress; - if tech == force.current_research then - progress = force.research_progress - else - progress = force.get_saved_technology_progress(tech.name); - end - local infinite = tech.enabled == true and tech.prototype.max_level == 4294967295; - local tech_data = { - tech.name, - tostring(tech.researched), - tostring(tech.level), - tostring(progress), - tostring(infinite), - } - tech_data = table.concat(tech_data, ':') - table.insert(data, tech_data); -end - -game.write_file("researchSync.txt", table.concat(data, "\n"), false, 0) diff --git a/sharedPlugins/researchSync/enableResearch.lua b/sharedPlugins/researchSync/enableResearch.lua deleted file mode 100644 index 8c8b7329..00000000 --- a/sharedPlugins/researchSync/enableResearch.lua +++ /dev/null @@ -1,33 +0,0 @@ - -local function updateTech() - script.raise_event(defines.events.on_research_finished, {research=game.forces['player'].technologies['{tech_name}'], by_script=true}) - if {notify} then - if 1 == {tech_infinite} then - game.print("Infinite technology {tech_name} level {tech_level} unlocked") - else - game.print("Technology {tech_name} researched") - end - game.play_sound({path="utility/research_completed"}) - end -end - -local force = game.forces['player'] -local tech = force.technologies['{tech_name}'] -if tech then - if 1 == {tech_infinite} then - if (tech.level ~= level) then - tech.level = {tech_level} - if tech == force.current_research then - force.research_progress = {tech_progress} - else - force.set_saved_technology_progress(tech.name, {tech_progress}); - end - updateTech() - end - else - if not tech.researched then - tech.researched = true - updateTech() - end - end -end diff --git a/sharedPlugins/researchSync/index.js b/sharedPlugins/researchSync/index.js deleted file mode 100644 index 5e555cac..00000000 --- a/sharedPlugins/researchSync/index.js +++ /dev/null @@ -1,385 +0,0 @@ -const fs = require('fs'); -const path = require('path') -const needle = require("needle"); - - -function time() { - let d = new Date() - return `${d.getUTCHours()}:${d.getUTCMinutes()}:${d.getUTCSeconds()}.${d.getUTCMilliseconds()}` -} - -class ResearchSync { - constructor(slaveConfig, messageInterface, extras = {}){ - this.config = slaveConfig - this.messageInterface = messageInterface - - this.functions = { - dumpResearch: this.loadFunc("dumpResearch.lua"), - enableResearch: this.loadFunc("enableResearch.lua"), - updateProgress: this.loadFunc("updateProgress.lua") - } - - this.log_folder = './logs' - if (!fs.existsSync(this.log_folder)) - fs.mkdirSync(this.log_folder); - this.log_file = path.join(this.log_folder, `${this.config.instanceName}-research.log`) - - this.research = {} - this.prev_research = {} - this.initial_request_own_data( - () => this.setup_sync_task(extras) - ) - } - - log(data) { - try { - console.log(`researchSync: ${data}`) - fs.appendFileSync(this.log_file, `${time()}: ${data}\n`); - } catch (e) { - console.error(e) - } - } - - error(data) { - try { - console.error(`researchSync: ${data}`) - fs.appendFileSync(this.log_file, `${time()}: ${data}\n`); - } catch (e) { - console.error(e) - } - } - - initial_request_own_data(callback) { - const url = `${this.config.masterURL}/api/getSlaveMeta` - const data = { - instanceID: this.config.unique, - password: this.config.clientPassword, - } - const options = { - headers: {'x-access-token': this.config.masterAuthToken}, - json: true, - compressed: true - } - needle.post(url, data, options, (err, res, techs) => { - if (err) { - this.log(`Can't get own slave data:`) - this.error(err) - return - } - if (res.statusCode === 404) { - this.log('slave is not registered yet. Delaying for 5 secs') - setTimeout( - () => this.initial_request_own_data(callback), - 5000 - ) - return - } - if (res.statusCode !== 200) { - this.log(`Can't get own slave data:`) - this.error(`status code ${res.statusCode}, ${res.body}`) - return - } - if(techs.toString()!="") { - techs = JSON.parse(techs) - if (typeof techs.research === 'object') - this.research = techs.research - this.log('techs imported from master') - } else { - this.log('no techs imported from master, since there are none yet') - } - callback() - }) - } - - setup_sync_task(extras) { - const timeout = extras.researchSyncPollInterval || 5000 - setInterval(() => this.sync_task(), timeout); - } - - sync_task() { - this.messageInterface(this.functions.dumpResearch); - setTimeout(this.request_cluster_data.bind(this), 2000); - } - - request_cluster_data() { - const slaves_data_url = `${this.config.masterURL}/api/slaves` - needle.get(slaves_data_url, {compressed:true}, this.sync_researches.bind(this)) - } - - sync_researches(err, resp, slaves_data) { - if (err) { - this.messageInterface("Unable to post JSON master/api/slaves, master might be unreachable"); - return false; - } - if (resp.statusCode !== 200) { - this.messageInterface("got error when calling slaves", resp.statusCode, resp.body); - return; - } - - slaves_data = Object.values(slaves_data) - slaves_data = slaves_data.filter( - slave_data => slave_data.unique !== this.config.unique.toString() - && slave_data.meta && slave_data.meta.research - ) - - this.clear_contribution_to_researched_techs() - let cluster_techs = this.get_cluster_techs(slaves_data) - this.recount_cluster_research_progress(slaves_data, cluster_techs) - - let to_research = this.filter_researched_techs(cluster_techs) - let to_update_progress = this.filter_updated_techs(cluster_techs, to_research) - - this.research_technologies(to_research) - this.update_technologies_progress(to_update_progress) - - this.print_own_contribution() - - needle.post(this.config.masterURL + '/api/editSlaveMeta', { - instanceID: this.config.unique, - password: this.config.clientPassword, - meta: {research: this.research} - }, {headers: {'x-access-token': this.config.masterAuthToken}, json: true, compressed:true}, (err, resp) => { - if (err) - this.error(err) - }) - } - - clear_contribution_to_researched_techs() { - for (let [name, tech] of Object.entries(this.research)) { - if (!Object.hasOwnProperty.call(this.prev_research, name)) { - continue; - } - - let researched; - if (tech.infinite) { - researched = this.prev_research[name].level < tech.level; - } else { - researched = this.prev_research[name].researched < tech.researched; - } - - if (researched) { - tech.contribution = 0; - } - } - } - - get_cluster_techs(slavesData) { - let cluster_techs = {} - for (let slave_data of slavesData) { - let node_researches = slave_data.meta.research - for (let [name, node_tech] of Object.entries(node_researches)) { - if (isNaN(node_tech.researched) || isNaN(node_tech.level) || isNaN(node_tech.infinite)) - continue - - if (Object.hasOwnProperty.call(cluster_techs, name)) { - if (cluster_techs[name].infinite === 1 && cluster_techs[name].level < node_tech.level) { - cluster_techs[name].level = node_tech.level - } else if (node_tech.researched > cluster_techs[name].researched) { - cluster_techs[name].researched = 1 - } - } else { - cluster_techs[name] = node_tech - } - } - } - return cluster_techs - } - - recount_cluster_research_progress(slaves_data, cluster_researches) { - for (let [name, tech] of Object.entries(cluster_researches)) { - if (Object.hasOwnProperty.call(this.research, name)) { - tech.progress = this.research[name].contribution - } else { - tech.progress = 0; - } - } - - for (let slave_data of slaves_data) { - for (let [name, tech] of Object.entries(slave_data.meta.research)) { - if (!Object.hasOwnProperty.call(cluster_researches, name)) { - continue; - } - - if ( - isNaN(cluster_researches[name].progress) - || isNaN(tech.contribution) - || isNaN(tech.level) - || isNaN(cluster_researches[name].level) - ) { - continue; - } - - if (cluster_researches[name].level === tech.level) { - cluster_researches[name].progress += tech.contribution; - } - } - } - - for (let [name, tech] of Object.entries(cluster_researches)) { - if (Object.hasOwnProperty.call(this.research, name) && tech.progress > 1) { - tech.progress = null - tech.researched = 1 - this.research[name].contribution = 0 - tech.contribution = 0 - if (this.research[name].level >= tech.level) - tech.level = this.research[name].level + 1 - } - } - } - - filter_researched_techs(cluster_researches) { - let result = {}; - for (let [name, tech] of Object.entries(this.research)) { - if (!Object.hasOwnProperty.call(cluster_researches, name)) { - continue; - } - - if (isNaN(cluster_researches[name].researched) || isNaN(cluster_researches[name].level)) - continue - - let researched - if (tech.infinite) { - researched = tech.level < cluster_researches[name].level; - } else { - researched = tech.researched < cluster_researches[name].researched; - } - - if (researched) - result[name] = cluster_researches[name] - } - return result; - } - - filter_updated_techs(cluster_techs, to_research) { - let result = {} - for (let [name, tech] of Object.entries(this.research)) { - if (Object.hasOwnProperty.call(to_research, name) || !Object.hasOwnProperty.call(cluster_techs, name)) { - continue; - } - if (isNaN(cluster_techs[name].progress)) - continue - - // Ignore researched techs - if (!tech.infinite && tech.researched) { - continue; - } - - // Do not update progress for infinite techs with a local level that's higher. - if (tech.infinite && tech.level > cluster_techs[name].level) { - continue; - } - - if (tech.progress < cluster_techs[name].progress) - result[name] = cluster_techs[name] - } - return result - } - - research_technologies(to_research) { - for (let name of Object.keys(to_research)) - if (!Object.hasOwnProperty.call(this.research, name)) { - delete to_research[name]; - } - - const notify = Object.keys(to_research).length === 1 - for (let [name, tech] of Object.entries(to_research)) { - this.research[name].contribution = 0 - this.research[name].progress = null - let command = this.functions.enableResearch; - command = command.replace(/{tech_name}/g, name); - command = command.replace(/{tech_level}/g, tech.level); - command = command.replace(/{tech_infinite}/g, tech.infinite); - command = command.replace(/{tech_progress}/g, tech.progress) - command = command.replace(/{notify}/g, notify); - this.messageInterface(command); - let log_message = tech.infinite - ? `Unlocking infinite research ${name} at level ${this.research[name].level}` - : `Unlocking research ${name}` - this.log(log_message); - this.messageInterface(log_message); - this.research[name] = tech; - } - } - - update_technologies_progress(to_update) { - for (let [name, tech] of Object.entries(to_update)) { - if (!Object.hasOwnProperty.call(this.research, name)) { - continue; - } - let progress = this.research[name].progress - if (progress === null) - progress = 'nil' - let command = this.functions.updateProgress - command = command.replace(/{tech_name}/g, name) - command = command.replace(/{last_check_progress}/g, progress) - command = command.replace(/{new_progress}/g, tech.progress) - this.messageInterface(command); - this.log( - `Updating ${name}: ${this.research[name].progress} += ${tech.progress - this.research[name].progress}` - ); - this.research[name].progress = tech.progress - } - } - - print_own_contribution() { - for (let [name, tech] of Object.entries(this.research)) { - if (!Object.hasOwnProperty.call(this.prev_research, name)) { - continue; - } - let diff = this.research[name].contribution - this.prev_research[name].contribution - if (Math.abs(diff) > Number.EPSILON * 1000) - this.log(`Own research ${name}: ${this.research[name].progress} += ${diff}`) - } - } - - loadFunc(path, silent=true) { - let command = fs.readFileSync("sharedPlugins/researchSync/" + path,'utf-8') - command = command.replace(/\r?\n|\r/g,' ') - command = (silent ? '/silent-command ' : '/c ') + command - return command; - } - scriptOutput(data) { - let [name, researched, level, progress, infinite] = data.split(":") - researched = +(researched === 'true') - infinite = +(infinite === 'true') - level = parseInt(level); - if (progress === 'nil') - progress = null - else - progress = parseFloat(progress) - - if (isNaN(level) || isNaN(researched)) - return - - if (Object.hasOwnProperty.call(this.research, name)) { - this.prev_research[name] = this.research[name]; - } else { - this.prev_research[name] = { - researched: null, - level: null, - progress: null, - contribution: 0, - infinite, - } - } - this.research[name] = { - researched, - level, - progress, - contribution: this.prev_research[name].contribution, - infinite - } - if (this.prev_research[name].progress && this.research[name].progress) { - // this.prev_research[name].progress gets updated to overall cluster progress - // therefore contribution should be own research progress change over sync interval - let contribution = this.research[name].progress - this.prev_research[name].progress - this.research[name].contribution += contribution - } - if (Math.abs(this.research[name].contribution) < Number.EPSILON * 1000) { - // if contribution should be 0 but because of floating-point precision is e.g. 2.2564e-18 - this.research[name].contribution = 0 - } - } -} - -module.exports = ResearchSync; diff --git a/sharedPlugins/researchSync/test/index.js b/sharedPlugins/researchSync/test/index.js deleted file mode 100644 index 00f8788f..00000000 --- a/sharedPlugins/researchSync/test/index.js +++ /dev/null @@ -1,128 +0,0 @@ -const assert = require("assert").strict; -const nock = require("nock"); - -const isFactorioCommand = require("lib/isFactorioCommand"); - -const researchSync = require("../index"); - -describe("researchSync/index.js", function() { - it("exports a single class (or at least a function)", function() { - assert.equal(typeof researchSync, "function"); - }); - - const config = { - instanceName: "test", - masterURL: "http://localhost:8080", - unique: 99, - clientPassword: "password", - masterAuthToken: "masterToken", - }; - - const scope = nock('http://localhost:8080') - .post('/api/getSlaveMeta') - .reply(200, '{}') - .persist() - ; - - describe("class researchSync()", function() { - it(".filter_researched_techs(researces) returns research that needs enabling on the local instance", function() { - let reSync = new researchSync(config, function(){}); - - let local = { - a:{infinite: true, researched: true, level: 0}, - b:{infinite: true, researched: true, level: 0}, - c:{infinite: true, researched: false, level: 0}, - d:{infinite: true, researched: false, level: 10}, - }; - - let remote = { - a:{infinite: true, researched: true, level: 0}, - b:{infinite: true, researched: true, level: 1}, - c:{infinite: true, researched: false, level: 0}, - e:{infinite: true, researched: false, level: 50}, - }; - reSync.research = local; - let diffResult = reSync.filter_researched_techs(remote); - console.log(diffResult); - - assert.equal(diffResult.a, undefined); - - assert.notEqual(diffResult.b, undefined); - assert.equal(diffResult.b.researched, true); - assert.equal(diffResult.b.level, 1); - - assert.equal(diffResult.d, undefined); - - // filter_researched_techs is broken - this.skip(); - assert.notEqual(diffResult.e, undefined); - assert.equal(diffResult.e.researched, false); - assert.equal(diffResult.e.level, 50); - }); - it(".pollResearch() dumps a long Lua command", function() { - // pollResearch doesn't exist - this.skip(); - let reSync = new researchSync(config, io); - let ioRecieved; - function io(str){ - ioRecieved = true; - assert(isFactorioCommand(str), "pollResearch should run a command"); - } - - assert(!ioRecieved); - - reSync.pollResearch(); - - assert(ioRecieved); - }); - describe(".scriptOutput(data) handles file writes from factorio", function() { - it("wants a key:value pair of a single research, parses and saves it", function() { - let reSync = new researchSync(config, console.log); - let researches = [ - 'automation:false:0:nil', - 'automation-2:false:0:nil', - 'automation-3:false:0:nil', - 'electronics:false:0:nil', - 'advanced-electronics:false:0:nil', - 'advanced-electronics-2:false:0:nil', - 'circuit-network:false:0:nil', - 'explosives:false:0:nil', - 'logistics:false:0:nil', - 'logistics-2:false:0:nil', - 'logistics-3:false:0:nil', - 'optics:false:0:nil', - 'laser:false:0:nil', - 'solar-energy:false:0:nil', - 'turrets:false:0:nil', - 'laser-turrets:false:0:nil', - 'stone-walls:false:0:nil', - 'gates:false:0:nil', - 'engine:false:0:nil', - ] - researches.forEach(research => { - reSync.scriptOutput(research); - assert(!isFactorioCommand("/c x = "+research)); - }); - assert.equal(Object.keys(reSync.research).length, researches.length); - }); - }); - it("regularily polls and syncs research (configurable delay)", function(done) { - let reSync = new researchSync(config, io, { - researchSyncPollInterval: 100, - }); - let ioRecieved; - function io(str){ - ioRecieved = true; - assert(isFactorioCommand(str), "pollResearch should run a command"); - } - function waitForIo(){ - if(!ioRecieved){ - setTimeout(waitForIo,50); - } else { - done(); - } - } - waitForIo(); - }); - }); -}); diff --git a/sharedPlugins/researchSync/updateProgress.lua b/sharedPlugins/researchSync/updateProgress.lua deleted file mode 100644 index 0d07bb84..00000000 --- a/sharedPlugins/researchSync/updateProgress.lua +++ /dev/null @@ -1,33 +0,0 @@ - -local function get_tech_progress(force, tech) - if tech == force.current_research then - return force.research_progress - else - return force.get_saved_technology_progress(tech.name); - end -end -local function set_tech_progress(force, tech, progress) - if tech == force.current_research then - force.research_progress = progress - else - force.set_saved_technology_progress(tech.name, progress); - end -end - -local force = game.forces['player'] -local tech_name = '{tech_name}' -local tech = force.technologies[tech_name] -local last_check_progress = {last_check_progress} -local new_progress = {new_progress} - -local progress_delta = get_tech_progress(force, tech) -if last_check_progress ~= nil then - progress_delta = progress_delta - last_check_progress -end -if progress_delta ~= nil then - new_progress = new_progress + progress_delta -end -if new_progress > 1 then - new_progress = 1 -end -set_tech_progress(force, tech, new_progress) diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 2815856d..ffd5a869 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -89,6 +89,7 @@ describe("Integration of Clusterio", function() { // XXX breaks tests. master._db.instances.get(44).set("subspace_storage.enabled", false); + master._db.instances.get(44).set("research_sync.enabled", false); }); });