Merge pull request #278 from Hornwitser/plugin-rewrite

Plugin rewrite and code improvements
This commit is contained in:
Hornwitser authored and GitHub committed 2020-03-21 19:38:46 +01:00
commit 941ceab935
47 files changed
+1879 -1618

No files matched your search

+92
View File
@@ -0,0 +1,92 @@
"use strict";
const events = require("events");
const fs = require("fs-extra");
const JSZip = require("jszip");
const klaw = require("klaw");
const path = require("path");
const yargs = require("yargs");
// Location of mod source files
const sourceDir = "mod";
// Paths to additional files to put into the mod's folder.
const additionalFiles = [
path.join("modules", "clusterio", "serialize.lua"),
];
async function main() {
const args = yargs
.scriptName("build")
.options({
'clean': { describe: "Remove previous builds", type: 'boolean', default: false },
'build': { describe: "Build mod", type: 'boolean', default: true },
'pack': { describe: "Pack into zip file", type: 'boolean', default: true },
'output-dir': { describe: "Path to output built mod", nargs: 1, type: 'string', default: "dist" },
})
.argv
;
let info = JSON.parse(await fs.readFile(path.join(sourceDir, "info.json")));
if (args.clean) {
let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/
for (let entry of await fs.readdir(args.outputDir)) {
let match = splitter.exec(entry);
if (match) {
let [, name, version] = match;
if (name === info.name) {
let modPath = path.join(args.outputDir, entry);
console.log(`Removing ${modPath}`);
await fs.remove(modPath);
}
}
}
}
if (args.build) {
await fs.ensureDir(args.outputDir);
let modName = `${info.name}_${info.version}`;
if (args.pack) {
let zip = new JSZip();
let walker = klaw(sourceDir)
.on('data', item => {
if (item.stats.isFile()) {
// On Windows the path created uses backslashes as the directory sepparator
// but the zip file needs to use forward slashes. We can't use the posix
// version of relative here as it doesn't work with Windows style paths.
let basePath = path.relative(sourceDir, item.path).replace(/\\/g, "/");
zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
}
})
;
await events.once(walker, 'end');
for (let filePath of additionalFiles) {
zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath));
}
let modPath = path.join(args.outputDir, `${modName}.zip`);
console.log(`Writing ${modPath}`);
let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
await events.once(writeStream, 'finish');
} else {
let modDir = path.join(args.outputDir, modName);
if (await fs.exists(modDir)) {
console.log(`Removing existing build ${modDir}`);
await fs.remove(modDir);
}
console.log(`Building ${modDir}`);
await fs.copy(sourceDir, modDir);
for (let filePath of additionalFiles) {
await fs.copy(filePath, path.join(modDir, path.basename(filePath)));
}
}
}
}
if (module === require.main) {
main().catch(err => { console.log(err) });
}
+27 -4
View File
@@ -53,7 +53,7 @@ class Instance extends link.Link{
// load plugins
for (let pluginInfo of pluginInfos) {
if (!pluginInfo.instanceEntrypoint || this.config.group(pluginInfo.name).get("enabled")) {
if (!pluginInfo.instanceEntrypoint || !this.config.group(pluginInfo.name).get("enabled")) {
continue;
}
@@ -358,6 +358,10 @@ class InstanceConnection extends link.Link {
}
}
async forwardRequestToMaster(message, request) {
return await request.send(this.slave, message.data);
}
async forwardEventToInstance(message, event) {
let instanceId = message.data.instance_id;
let instanceConnection = this.slave.instanceConnections.get(instanceId);
@@ -563,14 +567,33 @@ class Slave extends link.Link {
}
async startInstanceRequestHandler(message, request) {
let instanceConnection = await this._connectInstance(message.data.instance_id);
return await request.send(instanceConnection, message.data);
let instanceId = message.data.instance_id;
let instanceConnection = await this._connectInstance(instanceId);
try {
return await request.send(instanceConnection, message.data);
} catch (err) {
await this.stopInstance(instanceId);
throw err;
}
}
async createSaveRequestHandler(message, request) {
let instanceId = message.data.instance_id;
let instanceConnection = await this._connectInstance(instanceId);
await request.send(instanceConnection, message.data);
try {
await request.send(instanceConnection, message.data);
this.instanceConnections.delete(instanceId);
} catch (err) {
await this.stopInstance(instanceId);
throw err;
}
}
async stopInstance(instanceId) {
let instanceConnection = this.instanceConnections.get(instanceId);
await link.messages.stopInstance.send(instanceConnection, { instance_id: instanceId });
this.instanceConnections.delete(instanceId);
}
+125 -12
View File
@@ -14,7 +14,7 @@ Contents
- [event_handler interface](#event_handler-interface)
- [Communicating with Clusterio](#communicating-with-clusterio)
- [Clusterio Modules](#clusterio-modules)
- [Clusterio Module API](#clusterio-module-api)
- [Clusterio Lua API](#clusterio-lua-api)
- [Clusterio Plugins](#clusterio-plugins)
@@ -216,7 +216,8 @@ The following entries are supported in the module.json file:
supported. The events for dependencies is invoked before the events
of the dependent, and starting an instance will fail if the
dependencies cannot be satisfied. If not specified it will default
to depending on `clusterio`.
to depending on `clusterio`. Make sure to add `clusterio` to your
dependencies if you add other dependencies and depend on the Lua API.
- `require`:
Lua files to require in `control.lua`. This should only really be
needed if you want to make a global function available for use in
@@ -250,21 +251,133 @@ module.
Because modules can be patched into an existing game you cannot rely on
the `on_init` callback to be called in Clusterio Modules. Nor can you
rely on the `on_configuration_changed` callback, as this is not called
when level code changes. The `clusterio` module provides the custom
when level code changes. The Clusterio Lua API provides the custom
`on_server_startup` event that can be used as a substitute, see the next
section.
### Clusterio Module API
Clusterio Lua API
-----------------
There's a builtin clusterio module that provides some tools for getting
instance info and communicating to plugins, script events to listen for
and a Factorio remote call API. The available interfaces is documented
in [modules/clusterio/api.lua](../modules/clusterio/api.lua). To use
this API from another module, require it using something along the lines
of `local clusterio_api = require("modules/clusterio/api")`. Make sure
to add `clusterio` as a dependency in your module.json if you use the
dependency entry in it.
Clusterio provides a save patched module as well as a regular Factorio
mod for interfacing with Clusterio. This includes tools sending data to
plugins, script events to listen for, and an item serialization module.
### clusterio_api library
**require path**
From a module
`local clusterio_api = require("modules/clusterio/api")`
From a mod
`local clusterio_api = require("__clusterio_lib__/api")`
Provides the main interface to Clusterio from within the game. When
using the mod version you must call `init` before it'll function
properly.
#### init
<sub>Mod version only</sub>
Initialize the API. This should be called from both the `on_init` and
`on_load` events in your mod, and before this is called certain features
will not be available, most notably the events table.
#### clusterio_api.events
Table of events raised by clusterio. If the game is started outside of
Clusterio then none of these events will be raised.
In the mod version this table does not exist before the `init` function
is called.
##### on_instance_updated
Raised after the name and id of an instance has been updated. This may
occur even if the id and name didn't change.
Event data:
- `instance_id`: The id of the instance.
- `instance_name`: the name of the instance.
##### on_server_startup
Raised when Clusterio is starting up the server. It can be used to
initialize clusterio related features in a mod, and as a stand-in for
the `on_init` and `on_configuration_changed` event for modules. It is
invoked on the first tick the server runs after the save has been
patched, before most other events.
Use this event to initialize and and/or migrate the data structures
you need. Keep in mind that both mods and Clusterio modules can switch
from any version to any version so it should be able to handle both
forwards and backwards migrations gracefully.
#### clusterio_api.send_json(channel, data)
Send json data to Clusterio over the given channel. The `data` argument
should be table that can be serialized with `game.table_to_json`.
Clusterio plugins can listen to channels and will receive an event with
the data sent here. See the [Communicating with Factorio
section](writing-plugins.md#communicating-with-factorio) in the Writing
Plugins document for more information.
If the game is started outside of Clusterio the data will be sent, but
since there's no code following stdout to pick it up it will be lost.
**Note**: Payloads greater than 4 MB will cause stuttering in the game.
**Note**: This is not a binary safe way of sending data. Strings
embedded into the tables sent must be valid UTF-8 text.
Parameters:
- `channel`: string identifying which channel to send it on.
- `data`: table that can be converted to JSON with game.table_to_json
### serialize library
**require path**
From a module
`local serialize = require("modules/clusterio/serialize")`
From a mod
`local serialize = require("__clusterio_lib__/serialize")`
Serializes inventories and item stacks into a table that in turn can be
converted into JSON with `game.table_to_json`. Properly handles the
extra data for most items.
#### serialize.serialize_equipment_grid(LuaEquipmentGrid)
Returns a table containing a serialized representation of the equipment
grid passed.
#### serialize.deserialize_equipment_grid(LuaEquipmentGrid, serialized_grid)
Deserialize a previously serialized equipment grid into the target
LuaEquipmentGrid. Overwrites all content in the target.
#### serialize.serialize_item_stack(LuaItemStack, destination_table)
Serialize the LuaItemStack passed into the target destination_table.
The destination_table should be a new empty table.
#### serialize.deserialize_item_stack(LuaItemStack, serialized_stack)
Deserialize a previously serialized item stack into the target
LuaItemStack. Overwrites the content of the item stack.
#### serialize.serialize_inventory(LuaInventory)
Returns a table containing a serialized representation of the inventory
passed.
#### serialize.deserialize_inventory(LuaInventory, serialized_inventory)
Deserialize a previously serialized inventory into the target
LuaInventory. Overwrites slots that has content in the serialized
inventory.
Clusterio Plugins
+4 -4
View File
@@ -226,7 +226,7 @@ plugin you listen for an event named ipc-channel_name in order to get
data sent by `send_json`. For example in the plugin code:
async init() {
this.server.on('ipc-my_plugin_foo', content =>
this.instance.server.on('ipc-my_plugin_foo', content =>
this.handleFoo(content).catch(err => console.log(
"Error handling foo:", foo
))
@@ -408,8 +408,8 @@ The following properties are recognized by the Request constructor:
Target to forward request to. Can be either `'master'` to indicate
a slave should forward it to the master server when receiving it
from an instance or `'instance'` to indicate it should be forwarded
to the instances specified by the `instance_id` event property.
This works by using a default handler for the event at the links
to the instances specified by the `instance_id` request property.
This works by using a default handler for the request at the links
that forward it.
**requestProperties**:
@@ -478,7 +478,7 @@ accordingly, for example:
);
// Somewhere in the instance plugin code
barMetric.labels(String(this.config.id)).set(someValue);
barMetric.labels(String(this.instance.config.get("instance.id"))).set(someValue);
Metrics are automatically registered to the default registry, and this
default registry is automatically polled by the master server on slaves.
+9 -4
View File
@@ -20,9 +20,9 @@ class Request {
* Links this request is valid on. Takes an array of strings
* containing "<source>-<target>" specifications.
* @param {?string} forwardTo -
* Optional target to forward this request to. Only 'instance' is
* currently supported and adds instance_id into the
* requestProperties.
* Optional target to forward this request to. 'instance' add
* instance_id into the requestProperties and forward to the given
* instance. 'master' forwards it to the master server.
* @param {Object<string, Object>} requestProperties -
* Mapping of property values to JSON schema specifications for the
* properties that are valid in the data payload of this requst.
@@ -46,7 +46,7 @@ class Request {
{"instance_id": { type: "integer" }}, requestProperties
);
} else if (forwardTo !== null) {
} else if (forwardTo !== "master" && forwardTo !== null) {
throw new Error(`Invalid forwardTo value ${forwardTo}`);
}
@@ -99,6 +99,9 @@ class Request {
* If forwardTo was set to 'instance' the `forwardRequestToInstance`
* method on the link is used as the default handler.
*
* If forwardTo was set to 'master' the `forwardRequesToMaster` method
* on the link is used as the default handler.
*
* Does nothing if the link is neither a source nor a target for this
* request.
*
@@ -121,6 +124,8 @@ class Request {
if (!handler) {
if (this.forwardTo === 'instance') {
handler = link.forwardRequestToInstance;
} else if (this.forwardTo === "master") {
handler = link.forwardRequestToMaster;
}
}
+3 -1
View File
@@ -74,8 +74,10 @@ class BaseInstancePlugin {
}
class BaseMasterPlugin {
constructor(info) {
constructor(info, master, metrics) {
this.info = info;
this.master = master;
this.metrics = metrics;
}
/**
+15 -15
View File
@@ -67,12 +67,13 @@ app.use(compression());
// dynamic HTML generations with EJS
app.set('view engine', 'ejs');
app.set('views', ['views', 'sharedPlugins']);
app.set('views', ['views', 'plugins']);
// give ejs access to some interesting information
app.use(function(req, res, next){
res.locals.res = res;
res.locals.req = req;
res.locals.masterPlugins = masterPlugins;
res.locals.slaves = db.slaves;
res.locals.moment = moment;
next();
@@ -670,23 +671,22 @@ async function loadPlugins(pluginInfos) {
let pluginLoadStarted = Date.now();
let MasterPlugin = plugin.BaseMasterPlugin;
if (pluginInfo.masterEntrypoint) {
({ MasterPlugin } = require(`./plugins/${pluginInfo.name}/${pluginInfo.masterEntrypoint}`));
try {
if (pluginInfo.masterEntrypoint) {
({ MasterPlugin } = require(`./plugins/${pluginInfo.name}/${pluginInfo.masterEntrypoint}`));
}
let masterPlugin = new MasterPlugin(
pluginInfo, { app, config: masterConfig, db, slaveConnections }, { endpointHitCounter }
);
await masterPlugin.init();
plugins.set(pluginInfo.name, masterPlugin);
} catch (err) {
throw new errors.PluginError(pluginInfo.name, err)
}
let masterPlugin = new MasterPlugin(pluginInfo);
await masterPlugin.init();
plugins.set(pluginInfo.name, masterPlugin);
console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`);
/*
main:new masterPlugin({
TODO?
config, socketio: io, express: app,
db,
}),
pluginConfig,
});*/
}
return plugins;
}
+67
View File
@@ -0,0 +1,67 @@
-- Important: Keep this API in sync with modules/clusterio/api.lua
local api = {}
local initialized = false
function api.init()
if initialized then
return
end
initialized = true
if remote.interfaces.clusterio_api then
api.events = remote.call("clusterio_api", "get_events")
else
local null_event = script.generate_event_name()
api.events = {
on_instance_updated = null_event,
on_server_startup = null_event,
}
end
end
local function call_api(fn, ...)
if remote.interfaces.clusterio_api and remote.interfaces.clusterio_api[fn] then
return remote.call("clusterio_api", fn, ...)
else
return nil
end
end
function api.get_instance_name()
return call_api("get_instance_name")
end
function api.get_instance_id()
return call_api("get_instance_id")
end
function api.send_json(channel, data)
if not remote.interfaces.clusterio_api then
return
end
-- Escape bad characters. The question mark is used for separating the
-- channel name from the payload.
channel = channel:gsub("([\x00-\x1f?\\])", function(match)
return "\\x" .. string.format("%02x", match:byte())
end)
data = game.table_to_json(data)
-- If there's more than about 4kB of data users running with the Windows
-- console open will start to experience stuttering, otherwise we could
-- output about 1 MB of data at a time through stdout.
if #data < 4000 then
print("\f$ipc:" .. channel .. "?j" .. data)
else
local file_no = remote.call("clusterio_api", "get_file_no")
local file_name = "clst_" .. file_no .. ".json"
game.write_file(file_name, data, false, 0)
print("\f$ipc:" .. channel .. "?f" .. file_name)
end
end
return api
+5
View File
@@ -0,0 +1,5 @@
---------------------------------------------------------------------------------------------------
Version: 0.1.0
Major Features:
- API for mods to interact with Clusterio
- Serialization library for converting items to JSON compatible tables.
+9
View File
@@ -0,0 +1,9 @@
{
"name": "clusterio_lib",
"version": "0.1.0",
"title": "Clusterio Library",
"author": "Clusterio Team",
"homepage": "https://github.com/clusterio/factorioClusterio",
"description": "Library for interfacing with Clusterio.",
"factorio_version": "0.17"
}
+7 -29
View File
@@ -1,42 +1,20 @@
-- Important: Keep this API in sync with mod/api.lua
local api = {}
-- Table of events raised by clusterio
api.events = {
-- Raised after the name and id of an instance has been updated
-- This may occur even if the id and name didn't change.
-- Event data:
-- instance_id: The id of the instance.
-- instance_name: the name of the instance.
on_instance_updated = script.generate_event_name(),
-- Raised when the server is starting up
-- This event serves as the stand-in for the on_init and
-- on_configuration_changed events for modules. It is invoked as early
-- as possible, before most but not all other events, every time the
-- server is started up.
-- Use this event to initialize and and/or migrate the data structures
-- you need. Keep in mind that Clusterio can switch from any version of
-- your module to any version of your module so it should be able to
-- handle both forwards and backwards migrations.
on_server_startup = script.generate_event_name(),
}
-- Send a table as json to Clusterio
-- Send data to Clusterio over the given channel. Clusterio plugins can
-- listen to channels and will receive an event with the data sent here.
-- See docs/writing-plugins.md for more information.
function api.get_instance_name()
return global.clusterio.instance_name
end
-- Note: Payloads greater than 4 MB will cause stuttering.
function api.get_instance_id()
return global.clusterio.instance_id
end
-- Note: This is not a binary safe way of sending data. Strings embedded
-- into the tables sent must be valid UTF-8 text.
-- Parameters:
-- channel: string identifying which channel to send it on.
-- data: table that can be converted to json with game.table_to_json
function api.send_json(channel, data)
-- Escape bad characters. The question mark is used for separating the
+22 -18
View File
@@ -32,26 +32,30 @@ function clusterio_private.update_instance(new_id, new_name)
end
function impl.add_remote_interface()
remote.add_interface('clusterio_api', {
-- Returns a table of events raised by clusterio
get_events = function()
return api.events
end,
-- This is not part of the add_remote_interface callback to ensure it is
-- available when the clusterio_lib mod is loaded. The reason this is
-- neccessary is that on_init for newly added mods happen before on_load
-- for existing mods, and the add_remote_interface callback is done in
-- on_load. See https://forums.factorio.com/viewtopic.php?f=25&t=81552 for
-- more details.
remote.add_interface('clusterio_api', {
get_events = function()
return api.events
end,
-- Returns the instance id the game is run under
-- This may change over time and/or be nil.
get_instance_id = function()
return global.clusterio.instance_id
end,
get_instance_id = function()
return global.clusterio.instance_id
end,
-- Returns the name of the instance the game runs under.
-- This may change over time and/or be nil.
get_instance_name = function()
return global.clusterio.instance_name
end,
})
end
get_instance_name = function()
return global.clusterio.instance_name
end,
get_file_no = function()
global.clusterio_file_no = (global.clusterio_file_no or 0) + 1
return global.clusterio_file_no
end,
})
return impl
+262
View File
@@ -0,0 +1,262 @@
-- Library for serializing items in Factorio
-- Based on code from playerManager and trainTeleports
local serialize = {}
-- Equipment Grids are serialized into an array of equipment entries
-- where ench entry is a table with the following fields:
-- n: name
-- p: position (array of 2 numbers corresponding to x and y)
-- s: shield (optional)
-- e: energy (optional)
-- If the equipment is a burner the following is also present:
-- i: burner inventory
-- r: result inventory
-- b: curently burning (optional)
-- f: remaining_burning_fuel (optional)
function serialize.serialize_equipment_grid(grid)
local serialized = {}
local processed = {}
for y = 0, grid.height - 1 do
for x = 0, grid.width - 1 do
local equipment = grid.get({x, y})
if equipment ~= nil then
local pos = equipment.position
local combined_pos = pos.x + pos.y * grid.width + 1
if not processed[combined_pos] then
processed[combined_pos] = true
local entry = {
n = equipment.name,
p = {pos.x, pos.y},
}
if equipment.shield > 0 then entry.s = equipment.shield end
if equipment.energy > 0 then entry.e = equipment.energy end
-- TODO: Test with Industrial Revolution
if equipment.burner then
local burner = equipment.burner
entry.i = serialize.serialize_inventory(burner.inventory)
entry.r = serialize.serialize_inventory(burner.burnt_result_inventory)
if burner.curently_burning then
entry.b = {}
serialize.serialize_item_stack(burner.curently_burning, entry.b)
entry.f = burner.remaining_burning_fuel
end
end
table.insert(serialized, entry)
end
end
end
end
return serialized
end
function serialize.deserialize_equipment_grid(grid, serialized)
grid.clear()
for _, entry in ipairs(serialized) do
local equipment = grid.put({
name = entry.n,
position = entry.p,
})
if equipment then
if entry.s then equipment.shield = entry.s end
if entry.e then equipment.energy = entry.e end
if entry.i then
if entry.b then serialize.deserialize_item_stack(burner.currently_burning, entry.b) end
if entry.f then burner.remaining_burning_fuel = entry.f end
serialize.deserialize_inventory(burner.burnt_result_inventory, entry.r)
serialize.deserialize_inventory(burner.inventory, entry.i)
end
end
end
end
-- Item stacks are serialized into a table with the following fields:
-- n: name
-- c: count
-- h: health (optional)
-- d: durability (optional)
-- a: ammo count (optional)
-- l: label (optional)
-- g: equipment grid (optional)
-- i: item inventory (optional)
-- If the item stack is exportable it has the following property instead
-- e: export string
-- Label is a table with the following fields:
-- t: label text (optional)
-- c: color (optional)
-- a: allow manual label change
function serialize.serialize_item_stack(slot, entry)
if
slot.is_blueprint
or slot.is_blueprint_book
or slot.is_upgrade_item
or slot.is_deconstruction_item
or slot.is_item_with_tags
then
local call_success, call_return = pcall(slot.export_stack)
if not call_success then
print("Error: '" .. call_return .. "' thrown exporting '" .. slot.name .. "'")
else
entry.e = call_return
end
return
end
entry.n = slot.name
entry.c = slot.count
if slot.health < 1 then entry.h = slot.health end
if slot.durability then entry.d = slot.durability end
if slot.type == "ammo" then entry.a = slot.ammo end
if slot.is_item_with_label then
local label = {}
if slot.label then label.t = slot.label end
if slot.label_color then label.c = slot.label_color end
label.a = slot.allow_manual_label_change
entry.l = label
end
if slot.grid then
entry.g = serialize.serialize_equipment_grid(slot.grid)
end
if slot.is_item_with_inventory then
local sub_inventory = slot.get_inventory(defines.inventory.item_main)
entry.i = serialize.serialize_inventory(sub_inventory)
end
end
function serialize.deserialize_item_stack(slot, entry)
if entry.e then
local success = slot.import_stack(entry.e)
if success == 1 then
print("Error: import of '" .. entry.e .. "' succeeded with errors")
elseif success == -1 then
print("Error: import of '" .. entry.e .. "' failed")
end
return
end
local item_stack = {
name = entry.n,
count = entry.c,
}
if entry.h then item_stack.health = entry.h end
if entry.d then item_stack.durability = entry.d end
if entry.a then item_stack.ammo = entry.a end
local call_success, call_return = pcall(slot.set_stack, item_stack)
if not call_success then
print("Error: '" .. call_return .. "' thrown setting stack ".. serpent.line(entry))
elseif not call_return then
print("Error: Failed to set stack " .. serpent.line(entry))
else
if entry.l then
-- TODO test this with AAI's unit-remote-control
local label = entry.l
if label.t then slot.label = label.t end
if label.c then slot.label_color = label.c end
slot.allow_manual_label_change = label.a
end
if entry.g then
serialize.deserialize_equipment_grid(slot.grid, entry.g)
end
if entry.i then
local sub_inventory = slot.get_inventory(defines.inventory.item_main)
serialize.deserialize_inventory(sub_inventory, entry.i)
end
end
end
-- Inventories are serialized into a table with the following fields:
-- i: array of item stack or exportable item entries
-- b: bar position (optional)
-- Each item entry has the following fields
-- s: index (optional, equals to previous plus one if not present)
-- r: repeat count (optional)
-- f: slot filter (optional)
-- Pluss all the fields for item stacks (see deserialize_item_stack)
-- It's also possible that the slot is empty but has a slot filter.
function serialize.serialize_inventory(inventory)
local serialized = {}
if inventory.hasbar() and inventory.getbar() <= #inventory then
serialized.b = inventory.getbar()
end
serialized.i = {}
local previous_index = 0
local previous_serialized = nil
for i = 1, #inventory do
local item = {}
local slot = inventory[i]
if inventory.supports_filters() then
item.f = inventory.get_filter(i)
end
if slot.valid_for_read then
serialize.serialize_item_stack(slot, item)
end
if item.n or item.f or item.e then
local item_serialized = game.table_to_json(item)
if item_serialized == previous_serialized then
local previous_item = serialized.i[#serialized.i]
previous_item.r = (previous_item.r or 0) + 1
previous_index = i
else
if i ~= previous_index + 1 then
item.s = i
end
previous_index = i
previous_serialized = item_serialized
table.insert(serialized.i, item)
end
else
-- Either an empty slot or serilization failed
previous_index = 0
previous_serialized = nil
end
end
return serialized
end
function serialize.deserialize_inventory(inventory, serialized)
if serialized.b and inventory.hasbar() then
inventory.setbar(serialized.b)
end
local last_slot_index = 0
for _, entry in ipairs(serialized.i) do
local base_index = entry.s or last_slot_index + 1
local repeat_count = entry.r or 0
for offset = 0, repeat_count do
-- XXX what if the inventory is smaller on this instance?
local index = base_index + offset
local slot = inventory[index]
if entry.f then
local call_success, call_return = pcall(inventory.set_filter, index, entry.f)
if not call_success then
print("Error: '" .. call_return .. "' thrown setting filter " .. entry.f)
elseif not call_return then
print("Error: Failed to set filter " .. entry.f)
end
end
if entry.n or entry.e then
serialize.deserialize_item_stack(slot, entry)
end
end
last_slot_index = base_index + repeat_count
end
end
return serialize
+1
View File
@@ -69,6 +69,7 @@
},
"devDependencies": {
"html5-validator": "^1.2.1",
"klaw": "^3.0.0",
"mocha": "^6.2.0",
"mocha-sinon": "^2.1.0",
"mocha.parallel": "^0.15.5",
+80
View File
@@ -0,0 +1,80 @@
const link = require("lib/link");
// schema used for syncing technologies to and from the master
const technologies = {
type: "array",
items: {
type: "array",
minItems: 4,
maxItems: 4,
items: [
{ type: "string" },
{ type: "integer" },
{ type: ["null", "number"] },
{ type: "boolean" },
]
},
};
module.exports = {
name: "research_sync",
title: "Research Sync",
description: "Synchronises technology research progress between instances.",
version: "2.0.0-alpha",
instanceEntrypoint: "instance",
masterEntrypoint: "master",
messages: {
contribution: new link.Event({
type: "research_sync:contribution",
links: ["instance-slave", "slave-master"],
forwardTo: "master",
eventProperties: {
"name": { type: "string" },
"level": { type: "integer" },
"contribution": { type: "number" },
},
}),
progress: new link.Event({
type: "research_sync:progress",
links: ["master-slave", "slave-instance"],
broadcastTo: "instance",
eventProperties: {
"technologies": {
type: "array",
items: {
additionalProperties: false,
required: ["name", "level", "progress"],
properties: {
"name": { type: "string" },
"level": { type: "integer" },
"progress": { type: "number" },
},
},
},
},
}),
finished: new link.Event({
type: "research_sync:finished",
links: ["instance-slave", "slave-master", "master-slave", "slave-instance"],
forwardTo: "master",
broadcastTo: "instance",
eventProperties: {
"name": { type: "string" },
"level": { type: "integer" },
},
}),
syncTechnologies: new link.Request({
type: "research_sync:sync_technologies",
links: ["instance-slave", "slave-master"],
forwardTo: "master",
requestProperties: {
"technologies": technologies,
},
responseProperties: {
"technologies": technologies,
},
}),
},
}
+81
View File
@@ -0,0 +1,81 @@
const needle = require("needle");
const plugin = require("lib/plugin");
const luaTools = require("lib/luaTools");
function unexpectedError(err) {
console.log("Unexpected error in research_sync")
console.log("---------------------------------")
console.log(err);
}
class InstancePlugin extends plugin.BaseInstancePlugin {
async init() {
this.instance.server.on("ipc-research_sync:contribution", (tech) => {
this.researchContribution(tech).catch(unexpectedError);
});
this.instance.server.on("ipc-research_sync:finished", (tech) => {
this.researchFinished(tech).catch(unexpectedError);
});
}
async researchContribution(tech) {
this.info.messages.contribution.send(this.instance, tech);
}
async progressEventHandler(message) {
let techsJson = luaTools.escapeString(JSON.stringify(message.data.technologies));
await this.instance.server.sendRcon(`/sc research_sync.update_progress("${techsJson}")`, true);
}
async researchFinished(tech) {
this.info.messages.finished.send(this.instance, tech);
}
async finishedEventHandler(message) {
let { name, level, researched } = message.data;
await this.instance.server.sendRcon(
`/sc research_sync.research_technology("${luaTools.escapeString(name)}", ${level})`, true
);
}
async onStart() {
let dumpJson = await this.instance.server.sendRcon("/sc research_sync.dump_technologies()");
let techsToSend = [];
let instanceTechs = new Map();
for (let tech of JSON.parse(dumpJson)) {
techsToSend.push([
tech.name,
tech.level,
tech.progress || null,
tech.researched,
]);
instanceTechs.set(tech.name, tech);
}
let response = await this.info.messages.syncTechnologies.send(this.instance, { technologies: techsToSend });
let techsToSync = [];
for (let masterTech of response.technologies) {
let [name, level, progress, researched] = masterTech;
let instanceTech = instanceTechs.get(name);
if (
!instanceTech
|| instanceTech.level !== level
|| (instanceTech.progress || null) !== progress
|| instanceTech.researched !== researched
) {
techsToSync.push(masterTech);
}
}
if (techsToSync.length) {
let syncJson = luaTools.escapeString(JSON.stringify(techsToSync));
await this.instance.server.sendRcon(`/sc research_sync.sync_technologies("${syncJson}")`, true);
}
}
}
module.exports = {
InstancePlugin,
}
+184
View File
@@ -0,0 +1,184 @@
const fs = require("fs-extra");
const path = require("path");
const plugin = require("lib/plugin");
class MasterPlugin extends plugin.BaseMasterPlugin {
async init() {
this.technologies = await loadTechnologies(this.master.config);
this.lastProgressBroadcast = Date.now();
this.progressBroadcastId = null;
this.progressToBroadcast = new Set();
}
async onExit() {
if (this.progressBroadcastId) {
clearTimeout(this.progressBroadcastId);
}
await saveTechnologies(this.master.config, this.technologies);
}
registerProgress() {
// Rate limit progress broadcasts to one per second
if (Date.now() < this.lastProgressBroadcast + 1000) {
if (!this.progressBroadcastId) {
this.progressBroadcastId = setTimeout(() => {
this.broadcastProgress();
this.progressBroadcastId = null;
}, 2000);
}
} else {
if (this.progressBroadcastId) {
clearTimeout(this.progressBroadcastId);
}
this.broadcastProgress();
}
}
broadcastProgress() {
let techs = []
for (let name of this.progressToBroadcast) {
let tech = this.technologies.get(name);
if (tech.progress) {
techs.push({ name, level: tech.level, progress: tech.progress });
}
}
this.progressToBroadcast.clear();
this.lastProgressBroadcast = Date.now();
if (techs.length) {
for (let slaveConnection of this.master.slaveConnections.values()) {
this.info.messages.progress.send(slaveConnection, { technologies: techs });
}
}
}
async contributionEventHandler(message) {
let { name, level, contribution } = message.data;
let tech = this.technologies.get(name);
if (!tech) {
tech = { level, progress: 0, researched: false };
this.technologies.set(name, tech)
// Ignore contribution to already researched technologies
} else if (tech.level > level || tech.level === level && tech.researched) {
return;
}
let newProgress = tech.progress + contribution;
if (newProgress < 1) {
tech.progress = newProgress;
this.progressToBroadcast.add(name);
this.registerProgress();
} else {
tech.researched = true;
tech.progress = null;
this.progressToBroadcast.delete(name);
for (let slaveConnection of this.master.slaveConnections.values()) {
this.info.messages.finished.send(slaveConnection, {
name,
level: tech.level
});
}
}
}
async finishedEventHandler(message) {
let { name, level } = message.data;
let tech = this.technologies.get(name);
if (!tech || tech.level <= level) {
this.progressToBroadcast.delete(name);
this.technologies.set(name, { level, progress: null, researched: true })
}
}
async syncTechnologiesRequestHandler(message) {
function baseLevel(name) {
let match = /-(\d+)$/.exec(name);
if (!match) {
return 1;
} else {
return Number.parseInt(match[1], 10);
}
}
for (let instanceTech of message.data.technologies) {
let [name, level, progress, researched] = instanceTech;
let tech = this.technologies.get(name);
if (!tech) {
this.technologies.set(name, { level, progress, researched });
if (progress) {
this.progressToBroadcast.add(name);
} else if (researched || baseLevel(name) != level) {
for (let slaveConnection of this.master.slaveConnections.values()) {
this.info.messages.finished.send(slaveConnection, { name, level });
}
}
} else {
if (tech.level > level || tech.level === level && tech.researched) {
continue;
}
if (tech.level < level || researched) {
tech.level = level;
tech.progress = progress;
tech.researched = researched;
for (let slaveConnection of this.master.slaveConnections.values()) {
this.info.messages.finished.send(slaveConnection, { name, level });
}
if (progress) {
this.progressToBroadcast.add(name);
} else {
this.progressToBroadcast.delete(name);
}
} else if (tech.progress < progress) {
tech.progress = progress;
this.progressToBroadcast.add(name);
}
}
}
this.registerProgress();
let technologies = [];
for (let [name, tech] of this.technologies) {
technologies.push([ name, tech.level, tech.progress, tech.researched ])
}
return { technologies }
}
}
async function loadTechnologies(masterConfig) {
let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
console.log(`Loading ${filePath}`);
try {
return new Map(JSON.parse(await fs.readFile(filePath)));
} catch (err) {
if (err.code === "ENOENT") {
console.log("Creating new technologies database");
return new Map();
} else {
throw err;
}
}
}
async function saveTechnologies(masterConfig, technologies) {
let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
console.log(`writing ${filePath}`);
await fs.outputFile(filePath, JSON.stringify([...technologies.entries()], null, 4));
}
module.exports = {
MasterPlugin
}
+4
View File
@@ -0,0 +1,4 @@
{
"name": "research_sync",
"load": ["sync.lua"]
}
+215
View File
@@ -0,0 +1,215 @@
local clusterio_api = require("modules/clusterio/api")
local sync = {}
local function get_technology_progress(tech)
if tech == tech.force.current_research then
return tech.force.research_progress
else
return tech.force.get_saved_technology_progress(tech.name)
end
end
local function set_technology_progress(tech, progress)
if tech == tech.force.current_research then
tech.force.research_progress = progress
else
tech.force.set_saved_technology_progress(tech.name, progress)
end
end
sync.events = {}
sync.events[clusterio_api.events.on_server_startup] = function(event)
if not global.research_sync then
global.research_sync = {
technologies = {},
}
end
-- Used when syncing completed technologies from the master
global.research_sync.ignore_research_finished = false
local force = game.forces["player"]
for _, tech in pairs(force.technologies) do
if tech.enabled then
local progress = get_technology_progress(tech)
global.research_sync.technologies[tech.name] = {
level = tech.level,
researched = tech.researched,
progress = progress,
}
end
end
end
local function get_contribution(tech)
local progress = get_technology_progress(tech)
if not progress then
return 0, nil
end
local prev_tech = global.research_sync.technologies[tech.name]
if prev_tech.progress and prev_tech.level == tech.level then
return progress - prev_tech.progress, progress
else
return progress, progress
end
end
local function send_contribution(tech)
local contribution, progress = get_contribution(tech)
if contribution ~= 0 then
clusterio_api.send_json("research_sync:contribution", {
name = tech.name,
level = tech.level,
contribution = contribution,
})
global.research_sync.technologies[tech.name].progress = progress
end
end
sync.events[defines.events.on_research_started] = function(event)
local tech = event.last_research
if tech then
send_contribution(tech)
end
end
sync.events[defines.events.on_research_finished] = function(event)
if global.research_sync.ignore_research_finished then
return
end
local tech = event.research
global.research_sync.technologies[tech.name] = {
level = tech.level,
researched = tech.researched,
}
local level = tech.level
if not tech.researched then
level = level - 1
end
clusterio_api.send_json("research_sync:finished", {
name = tech.name,
level = level,
})
end
sync.on_nth_tick = {}
sync.on_nth_tick[79] = function(event)
local tech = game.forces["player"].current_research
if tech then
send_contribution(tech)
end
end
research_sync = {}
function research_sync.dump_technologies()
local force = game.forces["player"]
local techs = {}
for _, tech in pairs(force.technologies) do
if tech.enabled then
table.insert(techs, {
name = tech.name,
level = tech.level,
progress = get_technology_progress(tech),
researched = tech.researched,
})
end
end
if #techs == 0 then
rcon.print("[]")
else
rcon.print(game.table_to_json(techs))
end
end
function research_sync.sync_technologies(data)
local force = game.forces["player"]
local nameIndex = 1
local levelIndex = 2
local progressIndex = 3
local researchedIndex = 4
global.research_sync.ignore_research_finished = true
for _, tech_data in pairs(game.json_to_table(data)) do
local tech = force.technologies[tech_data[nameIndex]]
if tech and tech.enabled and tech.level <= tech_data[levelIndex] then
tech.level = tech_data[levelIndex]
local progress
if tech_data[researchedIndex] then
tech.researched = true
progress = nil
elseif tech_data[progressIndex] then
send_contribution(tech)
progress = tech_data[progressIndex]
set_technology_progress(tech, progress)
else
progress = get_technology_progress(tech)
end
global.research_sync.technologies[tech.name] = {
level = tech.level,
researched = tech.researched,
progress = progress,
}
end
end
global.research_sync.ignore_research_finished = false
end
function research_sync.update_progress(data)
local techs = game.json_to_table(data)
local force = game.forces["player"]
for _, masterTech in ipairs(techs) do
local tech = force.technologies[masterTech.name]
if tech and tech.enabled and tech.level == masterTech.level then
send_contribution(tech)
set_technology_progress(tech, masterTech.progress)
global.research_sync.technologies[tech.name] = {
level = tech.level,
progress = masterTech.progress
}
end
end
end
function research_sync.research_technology(name, level)
local force = game.forces["player"]
local tech = force.technologies[name]
if not tech or not tech.enabled or tech.level > level then
return
end
global.research_sync.ignore_research_finished = true
if tech == force.current_research and tech.level == level then
force.research_progress = 1
elseif tech.level < level or tech.level == level and not tech.researched then
tech.level = level
tech.researched = true
if tech.name:find("-%d+$") then
game.print {"", "Researched ", {"technology-name." .. tech.name:gsub("-%d+$", "")}, level}
else
game.print {"", "Researched ", {"technology-name." .. tech.name}}
end
game.play_sound { path = "utility/research_completed" }
end
global.research_sync.ignore_research_finished = false
global.research_sync.technologies[tech.name] = {
level = tech.level,
researched = tech.researched,
}
end
return sync
+5
View File
@@ -0,0 +1,5 @@
// These are libraries that does not have any units test, they are
// required here in order to track coverage for them.
require('../instance');
require('../master');
+1 -1
View File
@@ -30,7 +30,7 @@ class InstancePlugin extends plugin.BaseInstancePlugin {
throw new Error(`Error parsing statistics JSON: ${err.message}, content "${string}"`);
}
let instanceId = this.instance.config.id;
let instanceId = this.instance.config.get("instance.id");
instanceGameTicksTotal.labels(String(instanceId)).set(stats.game_tick);
instancePlayerCount.labels(String(instanceId)).set(stats.player_count);
@@ -1,6 +1,18 @@
const doleNN = require("./dole_nn_base")
const Prometheus = require('prom-client');
const prometheus = require("lib/prometheus");
const prometheusNNDoleGauge = new prometheus.Gauge(
"clusterio_nn_dole_gauge",
"Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given",
{ labels: ["resource"] }
);
const prometheusDoleFactorGauge = new prometheus.Gauge(
"clusterio_dole_factor_gauge",
"The current dole division factor for this resource",
{ labels: ["resource"] }
);
class neuralDole {
@@ -17,14 +29,9 @@ class neuralDole {
}
constructor({
items, gaugePrefix
items
}){
// Set some storage variables for the dole divider
this.prometheusNNDoleGauge = new Prometheus.Gauge({
name: gaugePrefix+'nn_dole_gauge',
help: 'Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given',
labelNames: ["itemName"],
});
this.items = items
this.itemsLastTick = new Map(items._items)
this.dole = {}
@@ -32,32 +39,26 @@ class neuralDole {
this.lastRequest = {}
this.stats=[]
this.debt= {}
}
doMagic() {
for (let [name, count] of this.items._items) {
let magicData = doleNN.Tick(
count,
this.dole[name],
this.itemsLastTick.get(name),
this.getRequestStats(name,5)
)
this.stats[name]=this.stats[name] || []
this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
setInterval(()=>{
for (let [name, count] of this.items._items) {
let magicData = doleNN.Tick(
count,
this.dole[name],
this.itemsLastTick.get(name),
this.getRequestStats(name,5)
)
this.stats[name]=this.stats[name] || []
this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
this.dole[name] = magicData[0]
// DONE handle magicData[1] for graphing for our users
this.prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
}
this.itemsLastTick = new Map(this.items._items)
}, 1000)
this.dole[name] = magicData[0]
// DONE handle magicData[1] for graphing for our users
prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
}
this.itemsLastTick = new Map(this.items._items)
}
divider({
res,
object,
masterConfig,
prometheusImportGauge
}){
divider(object) {
let magicData = doleNN.Dose(
object.count, // numReq
this.items.getItemCount(object.name),
@@ -83,29 +84,18 @@ class neuralDole {
// Remove item from DB and send it
this.items.removeItem(object.name, magicData[0]);
if (masterConfig.get('log_item_transfers')) {
console.log("removed: " + object.name + " " + magicData[0] + " . " + this.items.getItemCount(object.name) + " and sent to " + object.instanceID + " | " + object.instanceName);
}
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(magicData[0]) || 0);
res.send({
name: object.name,
count: magicData[0],
})
return magicData[0];
}
}
// If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
let _doleDivisionFactor = {};
function doleDivider({
itemCount,
object,
items,
masterConfig,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
}){
let itemCount = items.getItemCount(object.name);
const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
@@ -120,19 +110,15 @@ function doleDivider({
if (itemCount > object.count) {
//If successful, increase dole
_doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
if (masterConfig.get('log_item_transfers')) {
console.log("removed: " + object.name + " " + object.count + " . " + itemCount + " and sent to " + object.instanceID + " | " + object.instanceName);
}
items.removeItem(object.name, object.count)
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
prometheusImportGauge.labels(object.instanceID, object.name).inc(object.count || 0);
res.send({count: object.count, name: object.name});
return object.count;
} else {
// if we didn't have enough, attempt giving out a smaller amount next time
_doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
res.send({name:object.name, count:0});
return 0;
//console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
}
}
+106
View File
@@ -0,0 +1,106 @@
const link = require("lib/link");
const config = require("lib/config");
class MasterConfigGroup extends config.PluginConfigGroup {}
MasterConfigGroup.groupName = "subspace_storage";
MasterConfigGroup.define({
name: "autosave_interval",
title: "Autosave Interval",
description: "Interval the storage is autosaved at in seconds.",
type: "number",
initial_value: 60,
});
MasterConfigGroup.define({
name: "division_method",
title: "Division Method",
description: "Method for dividing resource requests between instances.",
type: "string",
enum: ["simple", "dole", "neural_dole"],
initial_value: "simple",
});
MasterConfigGroup.define({
name: "log_item_transfers",
title: "Log Item Trassfers",
description: "Spam master console with item transfers done.",
type: "boolean",
initial_value: false,
});
MasterConfigGroup.finalize();
class InstanceConfigGroup extends config.PluginConfigGroup {}
InstanceConfigGroup.groupName = "subspace_storage";
InstanceConfigGroup.define({
name: "log_item_transfers",
title: "Log Item Trassfers",
description: "Spam slave console with item transfers done.",
type: "boolean",
initial_value: false,
});
InstanceConfigGroup.finalize();
// JSON schema for subspace storage items
const items = {
type: "array",
items: {
type: "array",
minItems: 2,
maxItems: 2,
items: [
{ type: "string" },
{ type: "integer" },
],
},
}
module.exports = {
name: "subspace_storage",
title: "Subspace Storage",
description: "Provides shared storage across instances for the Subspace Storage mod",
version: "2.0.0-alpha",
instanceEntrypoint: "instance",
InstanceConfigGroup,
masterEntrypoint: "master",
MasterConfigGroup,
messages: {
// XXX this should be a request to be reliable
place: new link.Event({
type: "subspace_storage:place",
links: ["instance-slave", "slave-master"],
forwardTo: "master",
eventProperties: {
"instance_id": { type: "integer" },
"items": items,
},
}),
remove: new link.Request({
type: "subspace_storage:remove",
links: ["instance-slave", "slave-master"],
forwardTo: "master",
requestProperties: {
"instance_id": { type: "integer" },
"items": items,
},
responseProperties: {
"items": items,
},
}),
getStorage: new link.Request({
type: "subspace_storage:get_storage",
links: ["instance-slave", "slave-master"],
forwardTo: "master",
responseProperties: {
"items": items,
},
}),
updateStorage: new link.Event({
type: "subspace_storage:update_storage",
links: ["master-slave", "slave-instance"],
broadcastTo: "instance",
eventProperties: {
"items": items,
},
}),
},
}
+94
View File
@@ -0,0 +1,94 @@
const fs = require("fs-extra");
const plugin = require("lib/plugin");
const luaTools = require("lib/luaTools");
function unexpectedError(err) {
console.log("Unexpected error in subspace_storage")
console.log("------------------------------------")
console.log(err);
}
class InstancePlugin extends plugin.BaseInstancePlugin {
async init() {
this.instance.server.on("ipc-subspace_storage:output", (output) => {
this.provideItems(output).catch(unexpectedError);
});
this.instance.server.on("ipc-subspace_storage:orders", (orders) => {
this.requestItems(orders).catch(unexpectedError);
});
}
async onStart() {
this.pingId = setInterval(() => {
this.instance.server.sendRcon(
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0"
).catch(unexpectedError);
}, 5000);
let response = await this.info.messages.getStorage.send(this.instance);
let itemsJson = luaTools.escapeString(JSON.stringify(response.items));
await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`);
}
async onStop() {
clearInterval(this.pingId);
}
// provide items --------------------------------------------------------------
async provideItems(items) {
this.info.messages.place.send(this.instance, {
items,
instance_id: this.instance.config.get("instance.id"),
});
if (this.instance.config.get("subspace_storage.log_item_transfers")) {
console.log("Exported the following to master:")
console.log(items);
}
}
// request items --------------------------------------------------------------
async requestItems(items) {
// Request the items all at once
let response = await this.info.messages.remove.send(this.instance, {
instance_id: this.instance.config.get("instance.id"),
items,
});
if (!response.items.length) {
return;
}
if (this.instance.config.get("subspace_storage.log_item_transfers")) {
console.log("Imported following from master:")
console.log(response.items);
}
let itemsJson = luaTools.escapeString(JSON.stringify(response.items));
await this.instance.server.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`);
}
// combinator signals ---------------------------------------------------------
async updateStorageEventHandler(message, event) {
let items = message.data.items;
// XXX this should be done on the lua side
// Ensure counts don't overflow
for (let item of items) {
if (item[1] > 0x7ffffff) {
item[1] = 0x7ffffff;
}
}
// XXX this should be moved to instance/clusterio api
items.push(["signal-unixtime", Math.floor(Date.now()/1000)]);
let itemsJson = luaTools.escapeString(JSON.stringify(items));
await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`);
}
}
module.exports = {
InstancePlugin,
};
+217
View File
@@ -0,0 +1,217 @@
"use strict";
const fs = require("fs-extra");
const path = require("path");
const database = require("lib/database");
const plugin = require("lib/plugin");
const prometheus = require("lib/prometheus");
const routes = require("./routes");
const dole = require("./dole");
const prometheusExportGauge = new prometheus.Gauge(
"clusterio_export_gauge",
"Resources exported by instance",
{ labels: ["instance_id", "resource"] }
);
const prometheusImportGauge = new prometheus.Gauge(
"clusterio_import_gauge",
"Resources imported by instance",
{ labels: ["instance_id", "resource"] }
);
class MasterPlugin extends plugin.BaseMasterPlugin {
async init() {
this.prometheusMasterInventoryGauge = new prometheus.Gauge(
"clusterio_master_inventory_gauge",
"Amount of resources stored on master",
{ labels: ["resource"] }
);
this.ui = {
sidebar: [
{
getHtml: () => `
<div class="nav-item mr-1">
<a class="nav-link align-middle" href="/subspace_storage/storage">Storage</a>
</div>`,
},
],
};
this.items = await loadDatabase(this.master.config);
this.itemsLastUpdate = new Map(this.items._items.entries());
this.autosaveId = setInterval(() => {
saveDatabase(this.master.config, this.items).catch(err => {
console.log("Unexpected error autosaving items in subspace_storage")
console.log("-----------------------------------------------------")
console.log(err);
});
}, this.master.config.get("subspace_storage.autosave_interval") * 1000);
this.neuralDole = new dole.neuralDole({ items: this.items });
this.doleMagicId = setInterval(() => {
if (this.master.config.get("subspace_storage.division_method") === "neural_dole") {
this.neuralDole.doMagic()
}
}, 1000);
routes.addApiRoutes(this.master.app, this.items, this.metrics.endpointHitCounter);
routes.addWebRoutes(this.master.app);
}
updateStorage() {
let itemsToUpdate = new Map();
for (let [name, count] of this.items._items) {
if (this.itemsLastUpdate.get(name) === count) {
continue;
}
itemsToUpdate.set(name, count);
}
if (!itemsToUpdate.size) {
return;
}
itemsToUpdate = [...itemsToUpdate.entries()];
for (let slaveConnection of this.master.slaveConnections.values()) {
this.info.messages.updateStorage.send(slaveConnection, { items: itemsToUpdate });
}
this.itemsLastUpdate = new Map(this.items._items.entries());
}
async getStorageRequestHandler(message) {
return { items: [...this.items._items.entries()] };
}
async placeEventHandler(message) {
let instanceId = message.data.instance_id;
for (let item of message.data.items) {
this.items.addItem(item[0], item[1]);
prometheusExportGauge.labels(String(instanceId), item[0]).inc(item[1]);
}
this.updateStorage();
if (this.master.config.get("subspace_storage.log_item_transfers")) {
console.log(`Imported the following from ${message.data.instance_id}:`)
console.log(message.data.items);
}
}
async removeRequestHandler(message) {
let method = this.master.config.get("subspace_storage.division_method");
let instanceId = message.data.instance_id;
let itemsRemoved = [];
if (method === "simple") {
// Give out as much items as possible until there are 0 left. This
// might lead to one slave getting all the items and the rest nothing.
for (let item of message.data.items) {
let count = this.items.getItemCount(item[0]);
let toRemove = Math.min(count, item[1]);
if (toRemove > 0) {
this.items.removeItem(item[0], toRemove);
itemsRemoved.push([item[0], toRemove]);
}
}
} else {
let instanceConfig = this.master.db.instances.get(instanceId);
let instanceName = instanceConfig ? instanceConfig.get("instance.name") : "unkonwn";
// use fancy neural net to calculate a "fair" dole division rate.
if (method === "neural_dole") {
for (let item of message.data.items) {
let count = neuralDole.divider({ name: item[0], count: item[1], instanceId, instanceName });
if (count > 0) {
itemsRemoved.push([item[0], count]);
}
}
// Use dole division. Makes it really slow to drain out the last little bit.
} else if (method === "dole") {
for (let item of message.data.items) {
let count = dole.doleDivider({
object: { name: item[0], count: item[1], instanceId, instanceName},
items: this.items,
});
if (count > 0) {
itemsRemoved.push([item[0], count]);
}
}
// Should not be possible
} else {
throw Error(`Unkown division_method ${method}`);
}
}
if (itemsRemoved.length) {
for (let item of itemsRemoved) {
prometheusImportGauge.labels(String(instanceId), item[0]).inc(item[1]);
}
this.updateStorage();
if (itemsRemoved.length && this.master.config.get("subspace_storage.log_item_transfers")) {
console.log(`Exported the following to ${instanceId}`);
console.log(itemsRemoved);
}
}
return {
items: itemsRemoved,
}
}
onMetrics() {
if (this.items) {
for (let [key, count] of this.items._items) {
this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
}
}
}
async onExit() {
clearInterval(this.autosaveId);
clearInterval(this.doleMagicId);
await saveDatabase(this.master.config, this.items);
}
}
async function loadDatabase(config) {
let itemsPath = path.resolve(config.get("master.database_directory"), "items.json");
console.log(`Loading ${itemsPath}`);
try {
let content = await fs.readFile(itemsPath);
return new database.ItemDatabase(JSON.parse(content));
} catch (err) {
if (err.code === "ENOENT") {
console.log("Creating new item database");
return new database.ItemDatabase();
} else {
throw err;
}
}
}
async function saveDatabase(masterConfig, items) {
if (items && items.size < 50000) {
let file = path.resolve(masterConfig.get("master.database_directory"), "items.json");
console.log(`writing ${file}`);
let content = JSON.stringify(items.serialize());
await fs.outputFile(file, content);
} else if (items) {
console.error(`Item database too large, not saving (${items.size})`);
}
}
module.exports = {
MasterPlugin,
};
+55
View File
@@ -0,0 +1,55 @@
const express = require("express");
const path = require("path");
const authenticate = require("lib/authenticate");
const prometheus = require("lib/prometheus");
function addWebRoutes(app) {
app.use("/subspace_storage", express.static(path.join(__dirname, "static")));
app.get("/subspace_storage/storage", (req, res) => { res.render("subspace_storage/views/storage"); });
}
function addApiRoutes(app, items, endpointHitCounter) {
/**
* GET endpoint to read the masters current inventory of items.
*
* @memberof clusterioMaster
* @instance
* @alias api/inventory
* @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
*/
app.get("/api/inventory", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
var inventory = [];
for (let [name, count] of items._items) {
inventory.push({ name, count });
}
res.send(JSON.stringify(inventory));
});
/**
* GET endpoint to read the masters inventory as an object with key:value pairs
*
* @memberof clusterioMaster
* @instance
* @alias api/inventoryAsObject
* @returns {object} JSON {"iron-plate":100, "copper-plate":5}
*/
app.get("/api/inventoryAsObject", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
res.send(JSON.stringify(items.serialise()));
});
}
module.exports = {
addWebRoutes,
addApiRoutes,
}
@@ -0,0 +1,83 @@
// function to draw data we recieve from ajax requests
_lastData = [];
function drawcontents(data) {
data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box.
_lastData = data;
// allow searching for multiple criteria separated by space
var searchArgs = document.querySelector("#search").value;
searchArgs = searchArgs.trim();
searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')';
const search = new RegExp(searchArgs, 'i');
data = data.filter(function(item) {
return search.test(item.name);
})
sortByKey(data, "count");
const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically
const rows = table.children;
//update existing rows or create new ones
data.forEach(function(item, i) {
// format count to have .s in it for large number readability (1,000,000)
item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
let row = rows[i];
if(!row) {
row = document.createElement('tr');
row.innerHTML = "<td><img width=32 height=32></td><td class=name></td><td class=count></td>";
table.appendChild(row);
}
const img = row.querySelector('img');
const imgName = getImageFromName(item.name);
if(img.getAttribute('src') !== imgName) {
img.setAttribute('src',imgName);
}
const name = row.querySelector('.name');
if(name.textContent !== item.name) {
name.textContent = item.name;
}
const count = row.querySelector('.count');
if(count.textContent !== ''+item.count) {
count.textContent = item.count;
}
});
//remove excess rows, for example, after filtering
while (data.length < rows.length) {
table.removeChild(rows[data.length]);
}
}
// get cluster inventory from master
function updateInventory() {
var xmlhttp = new XMLHttpRequest();
xmlhttp.onreadystatechange = function() {
if(xmlhttp.readyState == 4 && xmlhttp.status == 200) {
let data = JSON.parse(xmlhttp.responseText);
drawcontents(data);
}
}
xmlhttp.open("GET", "/api/inventory", true);
xmlhttp.send();
}
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
setInterval(updateInventory, 500);
} else {
updateInventory();
}
// function to sort arrays of objects after a keys value
function sortByKey(array, key) {
array.sort(function(a, b) {
return b[key] - a[key];
});
}
document.querySelector("#search").addEventListener('input', function() {
drawcontents();
});
@@ -1,7 +1,7 @@
// These are libraries that does not have any units test, they are
// required here in order to track coverage for them.
require('../index');
require('../masterPlugin');
require('../instance');
require('../master');
require('../dole');
require('../dole_nn_base');
+44
View File
@@ -0,0 +1,44 @@
const assert = require("assert").strict;
const express = require("express");
const bodyParser = require("body-parser");
const request = require("supertest");
const jwt = require("jsonwebtoken");
const authenticate = require('lib/authenticate');
const database = require("lib/database");
const prometheus = require("lib/prometheus");
const routes = require("../routes");
const testSecret = "TestSecretDoNotUse";
authenticate.setAuthSecret(testSecret);
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
describe("subspace_storage endpoints", function() {
const app = express();
app.use(bodyParser.json({ limit: '10mb', }));
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
const items = new database.ItemDatabase();
items.addItem("steel-plate", 20);
const gauge = new prometheus.Gauge("test_gauge", "test", { labels: ["route"] });
routes.addApiRoutes(app, items, gauge);
let persistentMaster = request(app);
describe("#GET /api/inventory", function() {
it("returns the masters current inventory", function() {
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
let inventory = JSON.parse(res.text);
assert.equal(typeof inventory, "object", "Inventory should be an object");
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
let contains20SteelPlate = false;
inventory.forEach(itemStack => {
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
});
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
});
});
});
});
@@ -1,7 +1,7 @@
<!doctype HTML>
<head>
<%- include("head") %>
<script defer src="/clusterioMod/storage.js"></script>
<script defer src="/subspace_storage/storage.js"></script>
</head>
<body>
@@ -31,18 +31,10 @@
</table>
</div>
</div>
<div id="graphDisplay">
<div id="storageGraphContainer">
<div id="contentGraph" class="storageGraph" style="width: 100%;"></div>
</div>
</div>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
crossorigin="anonymous"></script>
-10
View File
@@ -1,10 +0,0 @@
/*
Clusterio plugin to allow for chat between instances.
*/
module.exports = {
// Name of package. For display somewhere I guess.
name: "clusterioMod",
version: "1.0.0",
description: "Server side handling of functionality in the clusterio lua mod",
masterPlugin: "masterPlugin.js",
}
-194
View File
@@ -1,194 +0,0 @@
const needle = require("needle");
const fs = require("fs-extra");
const objectOps = require("lib/objectOps");
const fileOps = require("lib/fileOps");
const pluginConfig = require("./config");
function ensureFileSync(path) {
if (!fs.existsSync(path)) {
fs.outputFileSync(path, "");
}
}
module.exports = class remoteCommands {
constructor(mergedConfig, messageInterface, extras){
// Ugly global
global.confirmedOrders = [];
this.messageInterface = messageInterface;
this.config = mergedConfig;
this.socket = extras.socket;
const that = this;
const instance = process.argv[3];
const instancedirectory = this.config.instanceDirectory + '/' + instance;
const needleOptionsWithTokenAuthHeader = {
headers: {
'x-access-token': this.config.masterAuthToken
},
};
ensureFileSync(instancedirectory + "/script-output/output.txt");
ensureFileSync(instancedirectory + "/script-output/orders.txt");
// provide items --------------------------------------------------------------
// trigger when something happens to output.txt
fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) {
// get array of lines in file
let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n");
// if you found anything, reset the file
if (items[0]) {
fs.writeFileSync(instancedirectory + "/script-output/output.txt", "");
}
for (let i = 0; i < items.length; i++) {
if (items[i]) {
let g = items[i].split(" ");
g[0] = g[0].replace("\u0000", "");
// console.log("exporting " + JSON.stringify(g));
// send our entity and count to the master for him to keep track of
needle.post(that.config.masterURL + '/api/place', {
name: g[0],
count: g[1],
instanceName: instance, // name of instance
instanceID: that.config.unique, // a hash computed from the randomly generated rcon password
}, needleOptionsWithTokenAuthHeader, function (err, resp, body) {
if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]);
if(that.config.logItemTransfers){
if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`);
}
});
}
}
});
// request items --------------------------------------------------------------
setInterval(function () {
// get array of lines in file
let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n");
// if we actually got anything from the file, proceed and reset file
if (items[0]) {
fs.writeFileSync(instancedirectory + "/script-output/orders.txt", "");
// prepare a package of all our requested items in a more transfer friendly format
var preparedPackage = {};
for (let i = 0; i < items.length; i++) {
(function (i) {
if (items[i]) {
items[i] = items[i].split(" ");
items[i][0] = items[i][0].replace("\u0000", "");
items[i][0] = items[i][0].replace(",", "");
if (preparedPackage[items[i][0]]) {
// if we have buffered some already, sum the new items
if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") {
preparedPackage[items[i][0]] = {
"name": items[i][0],
"count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]),
"instanceName":instance,
"instanceID":that.config.unique,
};
// else just add em in without summing
} else if (typeof Number(items[i][1]) == "number") {
preparedPackage[items[i][0]] = {
"name": items[i][0],
"count": Number(items[i][1]),
"instanceName":instance,
"instanceID":that.config.unique,
};
}
// this condition will NEVER be triggered but we know how that goes
} else if (typeof Number(items[i][1]) == "number") {
preparedPackage[items[i][0]] = {
"name": items[i][0],
"count": Number(items[i][1]),
"instanceName":instance,
"instanceID":that.config.unique,
};
}
}
})(i);
}
// request our items, one item at a time
for (let i = 0; i < Object.keys(preparedPackage).length; i++) {
// console.log(preparedPackage[Object.keys(preparedPackage)[i]]);
function callback(err, response, body) {
if (response && response.body && typeof response.body == "object") {
// buffer confirmed orders
confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count}
if(that.config.logItemTransfers){
console.log(`Imported ${response.body.count} ${response.body.name} from master`);
}
}
}
needle.post(
that.config.masterURL + '/api/remove',
preparedPackage[Object.keys(preparedPackage)[i]],
needleOptionsWithTokenAuthHeader,
callback
);
}
// if we got some confirmed orders
// console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders));
//if (!(confirmedOrders.length>0)){return;}
let cmd="local t={";
for(let i=0;i<confirmedOrders.length;i++)
{
cmd+='["'+confirmedOrders[i].name+'"]='+confirmedOrders[i].count+',';
}
if (!(cmd==="local t={")){
that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
}
confirmedOrders=[];
}
}, 1000);
// COMBINATOR SIGNALS ---------------------------------------------------------
// get inventory from Master and RCON it to our slave
setInterval(() => {
needle.get(that.config.masterURL + '/api/inventory', function (err, response, body) {
if(err){
console.error("Unable to get JSON master/api/inventory, master might be unaccessible");
} else if (response && response.body) {
// Take the inventory we (hopefully) got and turn it into the format LUA accepts
if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver
try {
var inventory = JSON.parse(response.body);
var inventoryFrame = {};
for (let i = 0; i < inventory.length; i++) {
inventoryFrame[inventory[i].name] = Number(inventory[i].count);
if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){
inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice
}
}
inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000);
// console.log("RCONing inventory! " + JSON.stringify(inventoryFrame));
let first = true;
let cmd="local s={";
for (let key in inventoryFrame)
{
cmd+='["'+key+'"]='+inventoryFrame[key]+",";
}
if (!(cmd==="local s={")){
that.messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
}
that.messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')");
} catch (e){
console.log(e);
}
}
});
}, 1550);
// Make sure world has its worldID
setTimeout(function(){
messageInterface("/silent-command remote.call('clusterio','setWorldID',"+that.config.unique+")")
}, 20000);
}
async factorioOutput(data){
}
}
async function sleep(s){
return new Promise((resolve, reject) => {
setTimeout(resolve, s*1000);
});
}
-101
View File
@@ -1,101 +0,0 @@
"use strict";
const fs = require("fs-extra");
const path = require("path");
const Express = require("express");
const ejs = require("ejs");
const database = require("lib/database");
const routes = require("./routes");
class masterPlugin {
constructor({
config, pluginConfig, pluginPath, socketio, express, db, Prometheus, prometheusPrefix, endpointHitCounter,
}) {
this.config = config;
this.pluginConfig = pluginConfig;
this.pluginPath = pluginPath;
this.io = socketio;
this.express = express;
this.db = db,
this.Prometheus = Prometheus;
this.prometheusPrefix = prometheusPrefix;
this.endpointHitCounter = endpointHitCounter;
this.prometheusMasterInventoryGauge = new Prometheus.Gauge({
name: prometheusPrefix+'master_inventory_gauge',
help: 'Amount of items stored on master',
labelNames: ["itemName"],
});
this.ui = {
sidebar: [
{
getHtml: () => `
<div class="nav-item mr-1">
<a class="nav-link align-middle" href="/clusterioMod/storage">Storage</a>
</div>`,
},
],
};
}
onMetrics() {
if (this.items) {
for (let [key, count] of this.items._items) {
this.prometheusMasterInventoryGauge.labels(key).set(Number(count) || 0);
}
}
}
async onLoadFinish() {
this.items = await loadDatabase(this.config);
this.autosaveId = setInterval(async () => {
await saveDatabase(this.config, this.items);
},this.config.autosaveInterval || 60000);
routes.addApiRoutes(
this.express, this.config, this.db, this.items,
this.Prometheus, this.prometheusPrefix, this.endpointHitCounter,
);
routes.addWebRoutes(this.express);
}
async onExit(){
clearInterval(this.autosaveId);
await saveDatabase(this.config, this.items);
}
}
async function loadDatabase(config) {
let itemsPath = path.resolve(config.databaseDirectory, "items.json");
console.log(`Loading ${itemsPath}`);
try {
let content = await fs.readFile(itemsPath);
return new database.ItemDatabase(JSON.parse(content));
} catch (err) {
if (err.code === "ENOENT") {
console.log("Creating new item database");
return new database.ItemDatabase();
} else {
throw err;
}
}
}
async function saveDatabase(config, items) {
if (items && items.size < 50000) {
let file = path.resolve(config.databaseDirectory, "items.json");
console.log(`writing ${file}`);
let content = JSON.stringify(items.serialize());
await fs.outputFile(file, content);
} else if (items) {
console.error(`Item database too large, not saving (${items.size})`);
}
}
module.exports = masterPlugin;
-208
View File
@@ -1,208 +0,0 @@
const express = require("express");
const path = require("path");
const authenticate = require("lib/authenticate");
const dole = require("./dole");
function addWebRoutes(app) {
app.use('/clusterioMod', express.static(path.join(__dirname, 'static')));
app.get('/clusterioMod/storage', (req, res) => { res.render('clusterioMod/views/storage'); });
}
function addApiRoutes(
app, config, db, items, Prometheus, prometheusPrefix, endpointHitCounter,
) {
const prometheusExportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'export_gauge',
help: 'Items exported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusImportGauge = new Prometheus.Gauge({
name: prometheusPrefix+'import_gauge',
help: 'Items imported by instance',
labelNames: ["instanceID", "itemName"],
});
const prometheusDoleFactorGauge = new Prometheus.Gauge({
name: prometheusPrefix+'dole_factor_gauge',
help: 'The current dole division factor for this item',
labelNames: ["itemName"],
});
// Only initialize neural network when it's enabled, otherwise it might override gauge
let neuralDole;
if(config.useNeuralNetDoleDivider) {
neuralDole=new dole.neuralDole({
db, gaugePrefix: prometheusPrefix
});
}
/**
* POST endpoint for storing items in master's inventory.
* @memberof clusterioMaster
* @instance
* @alias api/place
* @param {itemStack} itemStack the number and type of items to store (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {string} status either "success" or "failure"
*/
app.post("/api/place", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
let x;
try {
x = JSON.parse(req.body);
} catch (e) {
x = req.body;
}
if(!x.instanceName) {
x.instanceName = "unknown";
}
if(!x.instanceID) {
x.instanceID = "unknown";
for (let [instanceID, slave] of db.slaves) {
if(slave.instanceName == req.body.instanceName) {
x.instanceID = instanceID;
break;
}
}
}
if(x.instanceID
&& x.instanceName
&& !isNaN(Number(x.count))
&& x.name
&& typeof x.name == "string"){
if(config.logItemTransfers){
console.log("added: " + req.body.name + " " + req.body.count+" from "+x.instanceName+" ("+x.instanceID+")");
}
// gather statistics
prometheusExportGauge.labels(x.instanceID, req.body.name).inc(Number(req.body.count) || 0);
// save items we get
var count = Number(req.body.count);
items.addItem(req.body.name, count);
// Attempt confirming
res.send("success");
} else {
res.send("failure");
}
});
/**
* POST endpoint to remove items from DB when client orders items.
*
* @memberof clusterioMaster
* @instance
* @alias api/remove
* @param {itemStack} itemStack the name of and the number of items to remove (see typedef)
* @param {string} [itemStack.instanceID="unknown"]
* the unique/instanceID which is a numerical value for an instance
* @param {string} [itemStack.instanceName="unknown"]
* the name of an instance for identification in statistics, as
* provided when launching it. ex node client start [name]
* @returns {itemStack}
* the number of items actually removed, may be lower than what was
* asked for due to shortages.
*/
app.post("/api/remove", authenticate.middleware, function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// save items we get
var object = req.body;
if(!object.instanceID) {
object.instanceID = "unknown"
}
if(!object.instanceName) {
object.instanceName = "unknown";
}
if (db.slaves.has(String(object.instanceID))) {
object.instanceName = db.slaves.get(String(object.instanceID)).instanceName;
}
let itemCount = items.getItemCount(object.name);
if (
(config.disableImportsOfEverythingExceptElectricity === true || config.disableImportsOfEverythingExceptElectricity === "true" )
&& object.name != "electricity"
) {
if(config.logItemTransfers){
console.log('failure could not find ' + object.name);
}
res.send({name:object.name, count:0});
} else if(config.disableFairItemDistribution){
// Give out as much items as possible until there are 0 left. This might lead to one slave getting all the items and the rest nothing.
let numberToRemove = Math.min(Math.abs(Number(object.count)),itemCount);
items.removeItem(object.name, numberToRemove);
res.send({count: numberToRemove, name: object.name});
// track statistics and do graphing things
prometheusImportGauge.labels(object.instanceID, object.name).inc(Number(numberToRemove) || 0);
} else if(config.useNeuralNetDoleDivider){
// use fancy neural net to calculate a "fair" dole division rate.
neuralDole.divider({
res,
object,
config,
prometheusImportGauge
})
} else {
// Use dole division. Makes it really slow to drain out the last little bit.
dole.doleDivider({
itemCount,
object,
db,
config,
prometheusDoleFactorGauge,
prometheusImportGauge,
req,res,
})
}
});
/**
* GET endpoint to read the masters current inventory of items.
*
* @memberof clusterioMaster
* @instance
* @alias api/inventory
* @returns {object[]} JSON [{name:"iron-plate", count:100},{name:"copper-plate",count:5}]
*/
app.get("/api/inventory", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
var inventory = [];
for (let [name, count] of items._items) {
inventory.push({ name, count });
}
res.send(JSON.stringify(inventory));
});
/**
* GET endpoint to read the masters inventory as an object with key:value pairs
*
* @memberof clusterioMaster
* @instance
* @alias api/inventoryAsObject
* @returns {object} JSON {"iron-plate":100, "copper-plate":5}
*/
app.get("/api/inventoryAsObject", function(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
// Check it and send it
res.send(JSON.stringify(items.serialise()));
});
}
module.exports = {
addWebRoutes,
addApiRoutes,
}
@@ -1,208 +0,0 @@
// function to draw data we recieve from ajax requests
_lastData = [];
function drawcontents(data) {
data = data || _lastData; //Cache data so we can drawcontents without waiting for the server, for the search box.
_lastData = data;
// allow searching for multiple criteria separated by space
var searchArgs = document.querySelector("#search").value;
searchArgs = searchArgs.trim();
searchArgs = '(' + searchArgs.replace(/ +/g,")|(") + ')';
const search = new RegExp(searchArgs, 'i');
data = data.filter(function(item) {
return search.test(item.name);
})
sortByKey(data, "count");
const table = document.querySelector("#contents tbody"); //tables have tbody inserted automatically
const rows = table.children;
//update existing rows or create new ones
data.forEach(function(item, i) {
// format count to have .s in it for large number readability (1,000,000)
item.count = item.count.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
let row = rows[i];
if(!row) {
row = document.createElement('tr');
row.innerHTML = "<td><img width=32 height=32></td><td class=name></td><td class=count></td>";
table.appendChild(row);
}
const img = row.querySelector('img');
const imgName = getImageFromName(item.name);
if(img.getAttribute('src') !== imgName) {
img.setAttribute('src',imgName);
}
const name = row.querySelector('.name');
if(name.textContent !== item.name) {
name.textContent = item.name;
row.onclick = ()=>{
makeItemGraphs(item.name);
}
}
const count = row.querySelector('.count');
if(count.textContent !== ''+item.count) {
count.textContent = item.count;
}
});
//remove excess rows, for example, after filtering
while (data.length < rows.length) {
table.removeChild(rows[data.length]);
}
}
// get cluster inventory from master
function updateInventory() {
var xmlhttp = new XMLHttpRequest();
xmlhttp.onreadystatechange = function() {
if(xmlhttp.readyState == 4 && xmlhttp.status == 200) {
let data = JSON.parse(xmlhttp.responseText);
drawcontents(data);
}
}
xmlhttp.open("GET", "/api/inventory", true);
xmlhttp.send();
}
if(JSON.parse(localStorage.settings)["Periodically update storage screen"]) {
setInterval(updateInventory, 500);
} else {
updateInventory();
}
// function to sort arrays of objects after a keys value
function sortByKey(array, key) {
array.sort(function(a, b) {
return b[key] - a[key];
});
}
document.querySelector("#search").addEventListener('input', function() {
drawcontents();
});
// Make graph of cluster IO items
/*getJSON("/api/getStats", function(err, data){
}*/
function getItemStatData(item, callback){
var xhr = new XMLHttpRequest();
var url = "/api/getStats";
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-type", "application/json");
xhr.onreadystatechange = function () {
if (xhr.readyState === 4 && xhr.status === 200) {
var json = JSON.parse(xhr.responseText);
console.log(json.email + ", " + json.password);
callback(json);
}
};
var requestData = JSON.stringify({
"statistic": "place",
"itemName": "shit",
});
xhr.send(data);
}
function makeItemGraphs(itemName){
getJSON("/api/slaves", function(err, slaves){
if(err) throw err;
if(typeof slaves == "string"){
slaves = JSON.parse(slaves);
}
let instanceIDs = Object.keys(slaves);
let HTML = ""
// Draw in/out chart for unregistered slaves as well
instanceIDs.unshift("unknown");
instanceIDs.forEach(function(instanceID){
if(instanceID == "unknown" || Date.now() - slaves[instanceID].time < 120000 || JSON.parse(localStorage.settings)["Display offline slaves"]){
HTML += '<div id="contentGraph'+instanceID+'" class="storageGraph" style="width: 100%;"></div>';
["place", "remove"].forEach(statistic => {
console.log("Making chart for "+instanceID+"'s "+statistic+" statistic");
setTimeout(function(){
let xmlhttp = new XMLHttpRequest(); // new HttpRequest instance
xmlhttp.open("POST", "/api/getStats");
xmlhttp.setRequestHeader("Content-Type", "application/json");
xmlhttp.onreadystatechange = function(){
console.log(xmlhttp.status);
if(xmlhttp.readyState === XMLHttpRequest.DONE && xmlhttp.status === 200) {
// console.log(xmlhttp.responseText);
if(isJSON(xmlhttp.responseText) && JSON.parse(xmlhttp.responseText).statusForDebugging === undefined){
let slaveDisplayName = "unknown";
try {
slaveDisplayName = slaves[instanceID].instanceName;
} catch(e) {}
let whatGraphIsIt = "Item IO from ";
// if(statistic == "remove") whatGraphIsIt = "Items taken from cluster by";
let chartData = JSON.parse(xmlhttp.response).data;
switch(statistic){
case "place":
chartData.name = "Items exported:";
break;
case "remove":
chartData.name = "Items imported:";
break;
}
drawChartWhenCalledTwice(chartData, whatGraphIsIt + " " + slaveDisplayName);
//drawChart(statistic+"contentGraph"+instanceID, [JSON.parse(xmlhttp.response).data], `${whatGraphIsIt} ${slaveDisplayName} (${instanceID}) ${itemName}`);
// console.log(JSON.parse(xmlhttp.response).data);
}
}
}
xmlhttp.send(JSON.stringify({statistic:statistic, itemName:itemName, instanceID:instanceID}));
},1);
});
let chartLines = [];
function drawChartWhenCalledTwice(data, slaveDisplayName){
chartLines.push(data);
if(chartLines.length >= 2){
drawChart("contentGraph"+instanceID, chartLines, `${slaveDisplayName} (${instanceID}) ${itemName}`);
}
}
}
});
document.querySelector("#graphDisplay > #storageGraphContainer").innerHTML = HTML;
});
}
let chartsByID = {};
function drawChart(selector, chartData, title) {
// selector is ID of element, ex "chartContainer" or "-123199123"
console.log(chartData)
chartsByID[selector] = new CanvasJS.Chart(selector, {
title:{
text: title || "Production graph"
},
toolTip:{
shared: true,
// content: "{name}: {y}",
},
zoomEnabled: true,
axisY:{
includeZero: true,
},
legend: {
cursor: "pointer",
itemclick: function (e) {
//console.log("legend click: " + e.dataPointIndex);
//console.log(e);
if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
e.dataSeries.visible = false;
} else {
e.dataSeries.visible = true;
}
e.chart.render();
}
},
data: chartData
});
chart = chartsByID[selector];
// console.log(chart)
chart.render();
return chart;
}
-109
View File
@@ -1,109 +0,0 @@
const assert = require("assert").strict;
const express = require("express");
const bodyParser = require("body-parser");
const request = require("supertest");
const Prometheus = require('prom-client');
const jwt = require("jsonwebtoken");
const authenticate = require('lib/authenticate');
const database = require("lib/database");
const routes = require("../routes");
const testSecret = "TestSecretDoNotUse";
authenticate.setAuthSecret(testSecret);
const token = jwt.sign({ id: "api" }, testSecret, { expiresIn: 600 });
describe("clusterioMod endpoints", function() {
const app = express();
app.use(bodyParser.json({ limit: '10mb', }));
app.use(bodyParser.urlencoded({ parameterLimit: 100000, limit: '10mb', extended: true }));
const config = {
disableFairItemDistribution: true,
};
const items = new database.ItemDatabase();
const gauge = new Prometheus.Gauge({ name: 'test_gauge', help: "test", labelNames: ['route'] });
routes.addApiRoutes(app, config, { slaves: new Map() }, items, Prometheus, "test_", gauge);
let persistentMaster = request(app);
describe("#POST /api/place", function() {
it("adds an itemStack to the masters inventory", function() {
return persistentMaster.post("/api/place").send({
name:"steel-plate",
count:20,
instanceName:"unitTest"
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, "success", "something went wrong with the request")
});
});
});
describe("#GET /api/inventory", function() {
it("returns the masters current inventory", function() {
return persistentMaster.get("/api/inventory").expect(200).then(function(res) {
let inventory = JSON.parse(res.text);
assert.equal(typeof inventory, "object", "Inventory should be an object");
assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
let contains20SteelPlate = false;
inventory.forEach(itemStack => {
if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
});
assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
});
});
});
describe("#POST /api/remove", function() {
it("returns an itemStack of how many items were removed", function() {
return persistentMaster.post("/api/remove").send({
name:"steel-plate",
count:10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.count, 10, `Expected 10 steel got ${res.text}`);
});
});
it("works correctly for items named addItem or removeItem", async function() {
function add(item) {
return persistentMaster.post("/api/place").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.text, 'success', `/api/place ${item} failed with ${res.text}`);
})
}
function remove(item) {
return persistentMaster.post("/api/remove").send({
name: item,
count: 10
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.deepEqual(res.body, {"name":item,"count":10});
});
}
await Promise.all([add("addItem"), add("removeItem")]);
await Promise.all([remove("addItem"), remove("removeItem")]);
});
it("returns an empty itemStack if you don't have any of the item you request", function() {
return persistentMaster.post("/api/remove").send({
name:"imaginaryItem",
count:999999
}).set(
"X-Access-Token", token
).expect(200).then(function(res) {
assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
});
});
});
});
-12
View File
@@ -1,12 +0,0 @@
module.exports = {
// Name of package. For display somewhere I guess.
name: "researchSync",
version: "1.0.0",
arguments: ["index.js"],
description: "syncs research across servers",
// We'll send everything in this file to your stdin. Beware.
scriptOutputFileSubscription: "researchSync.txt",
fileReadDelay: 50
}
@@ -1,24 +0,0 @@
local data = {};
local force = game.forces["player"];
for _, tech in pairs(force.technologies) do
local progress;
if tech == force.current_research then
progress = force.research_progress
else
progress = force.get_saved_technology_progress(tech.name);
end
local infinite = tech.enabled == true and tech.prototype.max_level == 4294967295;
local tech_data = {
tech.name,
tostring(tech.researched),
tostring(tech.level),
tostring(progress),
tostring(infinite),
}
tech_data = table.concat(tech_data, ':')
table.insert(data, tech_data);
end
game.write_file("researchSync.txt", table.concat(data, "\n"), false, 0)
@@ -1,33 +0,0 @@
local function updateTech()
script.raise_event(defines.events.on_research_finished, {research=game.forces['player'].technologies['{tech_name}'], by_script=true})
if {notify} then
if 1 == {tech_infinite} then
game.print("Infinite technology {tech_name} level {tech_level} unlocked")
else
game.print("Technology {tech_name} researched")
end
game.play_sound({path="utility/research_completed"})
end
end
local force = game.forces['player']
local tech = force.technologies['{tech_name}']
if tech then
if 1 == {tech_infinite} then
if (tech.level ~= level) then
tech.level = {tech_level}
if tech == force.current_research then
force.research_progress = {tech_progress}
else
force.set_saved_technology_progress(tech.name, {tech_progress});
end
updateTech()
end
else
if not tech.researched then
tech.researched = true
updateTech()
end
end
end
-385
View File
@@ -1,385 +0,0 @@
const fs = require('fs');
const path = require('path')
const needle = require("needle");
function time() {
let d = new Date()
return `${d.getUTCHours()}:${d.getUTCMinutes()}:${d.getUTCSeconds()}.${d.getUTCMilliseconds()}`
}
class ResearchSync {
constructor(slaveConfig, messageInterface, extras = {}){
this.config = slaveConfig
this.messageInterface = messageInterface
this.functions = {
dumpResearch: this.loadFunc("dumpResearch.lua"),
enableResearch: this.loadFunc("enableResearch.lua"),
updateProgress: this.loadFunc("updateProgress.lua")
}
this.log_folder = './logs'
if (!fs.existsSync(this.log_folder))
fs.mkdirSync(this.log_folder);
this.log_file = path.join(this.log_folder, `${this.config.instanceName}-research.log`)
this.research = {}
this.prev_research = {}
this.initial_request_own_data(
() => this.setup_sync_task(extras)
)
}
log(data) {
try {
console.log(`researchSync: ${data}`)
fs.appendFileSync(this.log_file, `${time()}: ${data}\n`);
} catch (e) {
console.error(e)
}
}
error(data) {
try {
console.error(`researchSync: ${data}`)
fs.appendFileSync(this.log_file, `${time()}: ${data}\n`);
} catch (e) {
console.error(e)
}
}
initial_request_own_data(callback) {
const url = `${this.config.masterURL}/api/getSlaveMeta`
const data = {
instanceID: this.config.unique,
password: this.config.clientPassword,
}
const options = {
headers: {'x-access-token': this.config.masterAuthToken},
json: true,
compressed: true
}
needle.post(url, data, options, (err, res, techs) => {
if (err) {
this.log(`Can't get own slave data:`)
this.error(err)
return
}
if (res.statusCode === 404) {
this.log('slave is not registered yet. Delaying for 5 secs')
setTimeout(
() => this.initial_request_own_data(callback),
5000
)
return
}
if (res.statusCode !== 200) {
this.log(`Can't get own slave data:`)
this.error(`status code ${res.statusCode}, ${res.body}`)
return
}
if(techs.toString()!="") {
techs = JSON.parse(techs)
if (typeof techs.research === 'object')
this.research = techs.research
this.log('techs imported from master')
} else {
this.log('no techs imported from master, since there are none yet')
}
callback()
})
}
setup_sync_task(extras) {
const timeout = extras.researchSyncPollInterval || 5000
setInterval(() => this.sync_task(), timeout);
}
sync_task() {
this.messageInterface(this.functions.dumpResearch);
setTimeout(this.request_cluster_data.bind(this), 2000);
}
request_cluster_data() {
const slaves_data_url = `${this.config.masterURL}/api/slaves`
needle.get(slaves_data_url, {compressed:true}, this.sync_researches.bind(this))
}
sync_researches(err, resp, slaves_data) {
if (err) {
this.messageInterface("Unable to post JSON master/api/slaves, master might be unreachable");
return false;
}
if (resp.statusCode !== 200) {
this.messageInterface("got error when calling slaves", resp.statusCode, resp.body);
return;
}
slaves_data = Object.values(slaves_data)
slaves_data = slaves_data.filter(
slave_data => slave_data.unique !== this.config.unique.toString()
&& slave_data.meta && slave_data.meta.research
)
this.clear_contribution_to_researched_techs()
let cluster_techs = this.get_cluster_techs(slaves_data)
this.recount_cluster_research_progress(slaves_data, cluster_techs)
let to_research = this.filter_researched_techs(cluster_techs)
let to_update_progress = this.filter_updated_techs(cluster_techs, to_research)
this.research_technologies(to_research)
this.update_technologies_progress(to_update_progress)
this.print_own_contribution()
needle.post(this.config.masterURL + '/api/editSlaveMeta', {
instanceID: this.config.unique,
password: this.config.clientPassword,
meta: {research: this.research}
}, {headers: {'x-access-token': this.config.masterAuthToken}, json: true, compressed:true}, (err, resp) => {
if (err)
this.error(err)
})
}
clear_contribution_to_researched_techs() {
for (let [name, tech] of Object.entries(this.research)) {
if (!Object.hasOwnProperty.call(this.prev_research, name)) {
continue;
}
let researched;
if (tech.infinite) {
researched = this.prev_research[name].level < tech.level;
} else {
researched = this.prev_research[name].researched < tech.researched;
}
if (researched) {
tech.contribution = 0;
}
}
}
get_cluster_techs(slavesData) {
let cluster_techs = {}
for (let slave_data of slavesData) {
let node_researches = slave_data.meta.research
for (let [name, node_tech] of Object.entries(node_researches)) {
if (isNaN(node_tech.researched) || isNaN(node_tech.level) || isNaN(node_tech.infinite))
continue
if (Object.hasOwnProperty.call(cluster_techs, name)) {
if (cluster_techs[name].infinite === 1 && cluster_techs[name].level < node_tech.level) {
cluster_techs[name].level = node_tech.level
} else if (node_tech.researched > cluster_techs[name].researched) {
cluster_techs[name].researched = 1
}
} else {
cluster_techs[name] = node_tech
}
}
}
return cluster_techs
}
recount_cluster_research_progress(slaves_data, cluster_researches) {
for (let [name, tech] of Object.entries(cluster_researches)) {
if (Object.hasOwnProperty.call(this.research, name)) {
tech.progress = this.research[name].contribution
} else {
tech.progress = 0;
}
}
for (let slave_data of slaves_data) {
for (let [name, tech] of Object.entries(slave_data.meta.research)) {
if (!Object.hasOwnProperty.call(cluster_researches, name)) {
continue;
}
if (
isNaN(cluster_researches[name].progress)
|| isNaN(tech.contribution)
|| isNaN(tech.level)
|| isNaN(cluster_researches[name].level)
) {
continue;
}
if (cluster_researches[name].level === tech.level) {
cluster_researches[name].progress += tech.contribution;
}
}
}
for (let [name, tech] of Object.entries(cluster_researches)) {
if (Object.hasOwnProperty.call(this.research, name) && tech.progress > 1) {
tech.progress = null
tech.researched = 1
this.research[name].contribution = 0
tech.contribution = 0
if (this.research[name].level >= tech.level)
tech.level = this.research[name].level + 1
}
}
}
filter_researched_techs(cluster_researches) {
let result = {};
for (let [name, tech] of Object.entries(this.research)) {
if (!Object.hasOwnProperty.call(cluster_researches, name)) {
continue;
}
if (isNaN(cluster_researches[name].researched) || isNaN(cluster_researches[name].level))
continue
let researched
if (tech.infinite) {
researched = tech.level < cluster_researches[name].level;
} else {
researched = tech.researched < cluster_researches[name].researched;
}
if (researched)
result[name] = cluster_researches[name]
}
return result;
}
filter_updated_techs(cluster_techs, to_research) {
let result = {}
for (let [name, tech] of Object.entries(this.research)) {
if (Object.hasOwnProperty.call(to_research, name) || !Object.hasOwnProperty.call(cluster_techs, name)) {
continue;
}
if (isNaN(cluster_techs[name].progress))
continue
// Ignore researched techs
if (!tech.infinite && tech.researched) {
continue;
}
// Do not update progress for infinite techs with a local level that's higher.
if (tech.infinite && tech.level > cluster_techs[name].level) {
continue;
}
if (tech.progress < cluster_techs[name].progress)
result[name] = cluster_techs[name]
}
return result
}
research_technologies(to_research) {
for (let name of Object.keys(to_research))
if (!Object.hasOwnProperty.call(this.research, name)) {
delete to_research[name];
}
const notify = Object.keys(to_research).length === 1
for (let [name, tech] of Object.entries(to_research)) {
this.research[name].contribution = 0
this.research[name].progress = null
let command = this.functions.enableResearch;
command = command.replace(/{tech_name}/g, name);
command = command.replace(/{tech_level}/g, tech.level);
command = command.replace(/{tech_infinite}/g, tech.infinite);
command = command.replace(/{tech_progress}/g, tech.progress)
command = command.replace(/{notify}/g, notify);
this.messageInterface(command);
let log_message = tech.infinite
? `Unlocking infinite research ${name} at level ${this.research[name].level}`
: `Unlocking research ${name}`
this.log(log_message);
this.messageInterface(log_message);
this.research[name] = tech;
}
}
update_technologies_progress(to_update) {
for (let [name, tech] of Object.entries(to_update)) {
if (!Object.hasOwnProperty.call(this.research, name)) {
continue;
}
let progress = this.research[name].progress
if (progress === null)
progress = 'nil'
let command = this.functions.updateProgress
command = command.replace(/{tech_name}/g, name)
command = command.replace(/{last_check_progress}/g, progress)
command = command.replace(/{new_progress}/g, tech.progress)
this.messageInterface(command);
this.log(
`Updating ${name}: ${this.research[name].progress} += ${tech.progress - this.research[name].progress}`
);
this.research[name].progress = tech.progress
}
}
print_own_contribution() {
for (let [name, tech] of Object.entries(this.research)) {
if (!Object.hasOwnProperty.call(this.prev_research, name)) {
continue;
}
let diff = this.research[name].contribution - this.prev_research[name].contribution
if (Math.abs(diff) > Number.EPSILON * 1000)
this.log(`Own research ${name}: ${this.research[name].progress} += ${diff}`)
}
}
loadFunc(path, silent=true) {
let command = fs.readFileSync("sharedPlugins/researchSync/" + path,'utf-8')
command = command.replace(/\r?\n|\r/g,' ')
command = (silent ? '/silent-command ' : '/c ') + command
return command;
}
scriptOutput(data) {
let [name, researched, level, progress, infinite] = data.split(":")
researched = +(researched === 'true')
infinite = +(infinite === 'true')
level = parseInt(level);
if (progress === 'nil')
progress = null
else
progress = parseFloat(progress)
if (isNaN(level) || isNaN(researched))
return
if (Object.hasOwnProperty.call(this.research, name)) {
this.prev_research[name] = this.research[name];
} else {
this.prev_research[name] = {
researched: null,
level: null,
progress: null,
contribution: 0,
infinite,
}
}
this.research[name] = {
researched,
level,
progress,
contribution: this.prev_research[name].contribution,
infinite
}
if (this.prev_research[name].progress && this.research[name].progress) {
// this.prev_research[name].progress gets updated to overall cluster progress
// therefore contribution should be own research progress change over sync interval
let contribution = this.research[name].progress - this.prev_research[name].progress
this.research[name].contribution += contribution
}
if (Math.abs(this.research[name].contribution) < Number.EPSILON * 1000) {
// if contribution should be 0 but because of floating-point precision is e.g. 2.2564e-18
this.research[name].contribution = 0
}
}
}
module.exports = ResearchSync;
-128
View File
@@ -1,128 +0,0 @@
const assert = require("assert").strict;
const nock = require("nock");
const isFactorioCommand = require("lib/isFactorioCommand");
const researchSync = require("../index");
describe("researchSync/index.js", function() {
it("exports a single class (or at least a function)", function() {
assert.equal(typeof researchSync, "function");
});
const config = {
instanceName: "test",
masterURL: "http://localhost:8080",
unique: 99,
clientPassword: "password",
masterAuthToken: "masterToken",
};
const scope = nock('http://localhost:8080')
.post('/api/getSlaveMeta')
.reply(200, '{}')
.persist()
;
describe("class researchSync()", function() {
it(".filter_researched_techs(researces) returns research that needs enabling on the local instance", function() {
let reSync = new researchSync(config, function(){});
let local = {
a:{infinite: true, researched: true, level: 0},
b:{infinite: true, researched: true, level: 0},
c:{infinite: true, researched: false, level: 0},
d:{infinite: true, researched: false, level: 10},
};
let remote = {
a:{infinite: true, researched: true, level: 0},
b:{infinite: true, researched: true, level: 1},
c:{infinite: true, researched: false, level: 0},
e:{infinite: true, researched: false, level: 50},
};
reSync.research = local;
let diffResult = reSync.filter_researched_techs(remote);
console.log(diffResult);
assert.equal(diffResult.a, undefined);
assert.notEqual(diffResult.b, undefined);
assert.equal(diffResult.b.researched, true);
assert.equal(diffResult.b.level, 1);
assert.equal(diffResult.d, undefined);
// filter_researched_techs is broken
this.skip();
assert.notEqual(diffResult.e, undefined);
assert.equal(diffResult.e.researched, false);
assert.equal(diffResult.e.level, 50);
});
it(".pollResearch() dumps a long Lua command", function() {
// pollResearch doesn't exist
this.skip();
let reSync = new researchSync(config, io);
let ioRecieved;
function io(str){
ioRecieved = true;
assert(isFactorioCommand(str), "pollResearch should run a command");
}
assert(!ioRecieved);
reSync.pollResearch();
assert(ioRecieved);
});
describe(".scriptOutput(data) handles file writes from factorio", function() {
it("wants a key:value pair of a single research, parses and saves it", function() {
let reSync = new researchSync(config, console.log);
let researches = [
'automation:false:0:nil',
'automation-2:false:0:nil',
'automation-3:false:0:nil',
'electronics:false:0:nil',
'advanced-electronics:false:0:nil',
'advanced-electronics-2:false:0:nil',
'circuit-network:false:0:nil',
'explosives:false:0:nil',
'logistics:false:0:nil',
'logistics-2:false:0:nil',
'logistics-3:false:0:nil',
'optics:false:0:nil',
'laser:false:0:nil',
'solar-energy:false:0:nil',
'turrets:false:0:nil',
'laser-turrets:false:0:nil',
'stone-walls:false:0:nil',
'gates:false:0:nil',
'engine:false:0:nil',
]
researches.forEach(research => {
reSync.scriptOutput(research);
assert(!isFactorioCommand("/c x = "+research));
});
assert.equal(Object.keys(reSync.research).length, researches.length);
});
});
it("regularily polls and syncs research (configurable delay)", function(done) {
let reSync = new researchSync(config, io, {
researchSyncPollInterval: 100,
});
let ioRecieved;
function io(str){
ioRecieved = true;
assert(isFactorioCommand(str), "pollResearch should run a command");
}
function waitForIo(){
if(!ioRecieved){
setTimeout(waitForIo,50);
} else {
done();
}
}
waitForIo();
});
});
});
@@ -1,33 +0,0 @@
local function get_tech_progress(force, tech)
if tech == force.current_research then
return force.research_progress
else
return force.get_saved_technology_progress(tech.name);
end
end
local function set_tech_progress(force, tech, progress)
if tech == force.current_research then
force.research_progress = progress
else
force.set_saved_technology_progress(tech.name, progress);
end
end
local force = game.forces['player']
local tech_name = '{tech_name}'
local tech = force.technologies[tech_name]
local last_check_progress = {last_check_progress}
local new_progress = {new_progress}
local progress_delta = get_tech_progress(force, tech)
if last_check_progress ~= nil then
progress_delta = progress_delta - last_check_progress
end
if progress_delta ~= nil then
new_progress = new_progress + progress_delta
end
if new_progress > 1 then
new_progress = 1
end
set_tech_progress(force, tech, new_progress)
+4
View File
@@ -86,6 +86,10 @@ describe("Integration of Clusterio", function() {
it("creates the instance", async function() {
await clusterctl._commands.get("create-instance").run({id: 44, name: "test"}, testControl);
assert(master._db.instances.has(44), "Instance was not created");
// XXX breaks tests.
master._db.instances.get(44).set("subspace_storage.enabled", false);
master._db.instances.get(44).set("research_sync.enabled", false);
});
});
+4 -1
View File
@@ -72,7 +72,10 @@ class MockInstance extends link.Link {
this.server = new MockServer();
this.name = "test";
this.config = {
id: 7357,
get: (name) => {
if (name === "instance.id") { return 7357 }
throw Error("Not implemented");
}
}
}
}
+7 -9
View File
@@ -8,17 +8,15 @@
<div class="nav-item mr-1">
<a class="nav-link align-middle" href="/nodes">Slaves</a>
</div><%
for(let o in locals.masterPlugins) {
let plug = locals.masterPlugins[o];
for (let plug of locals.masterPlugins.values()) {
if (
plug.main
&& plug.main.ui
&& typeof plug.main.ui.sidebar == "object"
&& Array.isArray(plug.main.ui.sidebar)
plug
&& plug.ui
&& typeof plug.ui.sidebar == "object"
&& Array.isArray(plug.ui.sidebar)
) {
let sidebarEntries = plug.main.ui.sidebar;
for (let i in sidebarEntries) {
let button = sidebarEntries[i];
let sidebarEntries = plug.ui.sidebar;
for (let button of sidebarEntries) {
if (button.getHtml && typeof button.getHtml == "function") { %>
<%- button.getHtml() %><%
}
+8 -14
View File
@@ -17,27 +17,21 @@ for (let [id, slave] of locals.slaves) { %>
<div id="<%= id %>" class="slaveBox card mb-3">
<div class="card-body">
<h5 class="card-title"><a href='nodeDetails?instanceID=<%=id%>'>
<%= slave.instanceName %>
<%= slave.name %>
</a></h5>
</div>
<ul class="list-group list-group-flush">
<li class="list-group-item">ID: <%= id %></li>
<li class="list-group-item">Last seen: <%= moment(Number(slave.time)).format("DD.MM.YY, hh:mm") %></li>
<li class="list-group-item">Online players: <%= slave.playerCount %></li>
<li class="list-group-item">IP: <%= slave.publicIP %>:<%= slave.serverPort %></li>
<li class="list-group-item">Host: <%= slave.mac %></li>
</ul><%
for (let o in locals.masterPlugins) {
let plug = locals.masterPlugins[o];
for (let plug of locals.masterPlugins.values()) {
if (
plug.main
&& plug.main.ui
&& typeof plug.main.ui.slaveListing == "object"
&& Array.isArray(plug.main.ui.slaveListing)
plug
&& plug.ui
&& typeof plug.ui.slaveListing == "object"
&& Array.isArray(plug.ui.slaveListing)
) {
let slaveListingEntries = plug.main.ui.slaveListing;
for (let i in slaveListingEntries) {
let button = slaveListingEntries[i];
let slaveListingEntries = plug.ui.slaveListing;
for (let button of slaveListingEntries) {
if (button.getHtml && typeof button.getHtml == "function") { %>
<div class="plugin-link"><%- button.getHtml({slave}) %></div><%
}