Merge pull request #340 from Hornwitser/rcon-performance

Improvements from investigating RCON performance
This commit is contained in:
Hornwitser authored and GitHub committed 2020-11-20 22:27:19 +01:00
commit 8f176c969b
22 files changed
+1847 -196

No files matched your search

+1 -1
View File
@@ -214,7 +214,7 @@ module.exports = {
"no-useless-rename": "error", "no-useless-rename": "error",
"no-useless-return": "error", "no-useless-return": "error",
"no-var": "warn", "no-var": "warn",
"no-void": "error", "no-void": ["error", { "allowAsStatement": true }],
"no-warning-comments": "off", "no-warning-comments": "off",
"no-whitespace-before-property": "error", "no-whitespace-before-property": "error",
"nonblock-statement-body-position": "error", "nonblock-statement-body-position": "error",
+2
View File
@@ -47,6 +47,8 @@ Version 2.0.0
- Added synchronization of in-game adminlist, banlist and whitelist to - Added synchronization of in-game adminlist, banlist and whitelist to
Clusterio core. Previously this was handled by the playerManager plugin. Clusterio core. Previously this was handled by the playerManager plugin.
- Added stripping of long paths in the Factorio server log. - Added stripping of long paths in the Factorio server log.
- Added option to configure maximum number of commands to send in parallel.
- Added option to auto start instances on slave startup.
### Changes ### Changes
+42
View File
@@ -218,6 +218,15 @@ directory for the instance is created on the slave.
Defaults to null meaning not assigned. Defaults to null meaning not assigned.
### instance.auto_start
If enabled start this instance when the slave it is assigned to is
started up. Does not affect start or stop of the instance while the
slave is running.
Defaults to false
### factorio.version ### factorio.version
Version of Factorio to use for this instance. Can be a version like Version of Factorio to use for this instance. Can be a version like
@@ -342,6 +351,39 @@ banned status while the instance is running.
Defaults to true Defaults to true
### factorio.max_concurrent_commands
Maximum number of commands being transmitted into the game in parallel.
Since the rate at which long commands are streamed into the game is by
default limeted to 100 bytes/tick and scales down to 25 bytes/tick when
there's 20 players connected, long commands can hold up the command
interface and cause large latencies in getting data pushed into the
game. (The rates are controlled by the `segment_size` options in the
server settings.) To counteract this Clusterio sends up to this
configured value number of commands in parallel over the RCON interface.
This causes the game updates sent to clients to have this many command
streams in parallel contained in them, drastically increasing the
maxiumum size they can become. Since commands are only processed so
ofter the maximum rate of commands that can be sustained is roughly 30
times this configured value per second, though note that large commands
will lower this value.
The game updates sent by the server is split into roughly 500 byte
packets, and if any one of those parts are lost the entire update is
resent. This means a high number of concurrent commands can amplify
resends due to packet loss and degrade the player experience up to the
point of becomming completely unplayable.
Prior to 0.18.7 the maximum practical size of game update sent where
arround 8 kB due to the UDP receive buffer on windows defaulting to this
value, larger updates would not get through the receive buffer and cause
the connection to drop. To stay below the 8 kB limit the maximum safe
value for this option would be around 7000 / (3 * game_speed *
maximum_segment_size), which is around 20 in normal circumstances.
Defaults to 5
### <plugin_name>.enabled ### <plugin_name>.enabled
Whether to enable the given plugin on the instance. Note that plugins Whether to enable the given plugin on the instance. Note that plugins
+2 -2
View File
@@ -230,11 +230,11 @@ Communicating with Factorio
For pushing data into Factorio there's RCON, which lets you send For pushing data into Factorio there's RCON, which lets you send
arbitrary Lua commands to invoke whatever code you want in the game. arbitrary Lua commands to invoke whatever code you want in the game.
This is done by calling the `sendRcon` method on the instance's server object. This is done by calling the `sendRcon` method on the plugin object.
For example: For example:
async onStart() { async onStart() {
let response = await this.instance.server.sendRcon( let response = await this.sendRcon(
"/sc rcon.print('data')" "/sc rcon.print('data')"
); );
+3 -3
View File
@@ -203,13 +203,13 @@ instanceConfigCommands.add(new libCommand.Command({
instanceCommands.add(instanceConfigCommands); instanceCommands.add(instanceConfigCommands);
instanceCommands.add(new libCommand.Command({ instanceCommands.add(new libCommand.Command({
definition: ["assign <instance> <slave>", "Assign instance to a slave", (yargs) => { definition: ["assign <instance> [slave]", "Assign instance to a slave", (yargs) => {
yargs.positional("instance", { describe: "Instance to assign", type: "string" }); yargs.positional("instance", { describe: "Instance to assign", type: "string" });
yargs.positional("slave", { describe: "Slave to assign to", type: "string" }); yargs.positional("slave", { describe: "Slave to assign to or unassign if none", type: "string" });
}], }],
handler: async function(args, control) { handler: async function(args, control) {
let instanceId = await libCommand.resolveInstance(control, args.instance); let instanceId = await libCommand.resolveInstance(control, args.instance);
let slaveId = await libCommand.resolveSlave(control, args.slave); let slaveId = args.slave ? await libCommand.resolveSlave(control, args.slave) : null;
await libLink.messages.assignInstanceCommand.send(control, { await libLink.messages.assignInstanceCommand.send(control, {
instance_id: instanceId, instance_id: instanceId,
slave_id: slaveId, slave_id: slaveId,
+12
View File
@@ -196,6 +196,12 @@ InstanceGroup.define({
type: "number", type: "number",
optional: true, optional: true,
}); });
InstanceGroup.define({
name: "auto_start",
description: "Automatically start this instance when the slave hosting it is started up",
type: "boolean",
initial_value: false,
});
InstanceGroup.finalize(); InstanceGroup.finalize();
/** /**
@@ -280,6 +286,12 @@ FactorioGroup.define({
type: "boolean", type: "boolean",
initial_value: true, initial_value: true,
}); });
FactorioGroup.define({
name: "max_concurrent_commands",
description: "Maximum number of RCON commands trasmitted in parallel",
type: "number",
initial_value: 5,
});
FactorioGroup.finalize(); FactorioGroup.finalize();
/** /**
+28 -1
View File
@@ -518,6 +518,9 @@ class FactorioServer extends events.EventEmitter {
* @param {boolean} options.enableWhitelist - Turn on whitelisting. * @param {boolean} options.enableWhitelist - Turn on whitelisting.
* @param {boolean} options.verboseLogging - Enable verbose logging. * @param {boolean} options.verboseLogging - Enable verbose logging.
* @param {boolean} options.stripPaths - Strip paths in the console. * @param {boolean} options.stripPaths - Strip paths in the console.
* @param {number} options.maxConcurrentCommands -
* Maximum number of RCON commands transmitted in parallel on the
* RCON connection.
*/ */
constructor(factorioDir, writeDir, options) { constructor(factorioDir, writeDir, options) {
super(); super();
@@ -540,6 +543,8 @@ class FactorioServer extends events.EventEmitter {
this.enableWhitelist = options.enableWhitelist; this.enableWhitelist = options.enableWhitelist;
/** Enable verbose logging */ /** Enable verbose logging */
this.verboseLogging = options.verboseLogging; this.verboseLogging = options.verboseLogging;
/** Maximum number of RCON commands transmitted in parallel on the RCON connection */
this.maxConcurrentCommands = options.maxConcurrentCommands || 5;
this._state = "new"; this._state = "new";
this._server = null; this._server = null;
this._rconClient = null; this._rconClient = null;
@@ -689,7 +694,7 @@ class FactorioServer extends events.EventEmitter {
port: this.rconPort, port: this.rconPort,
password: this.rconPassword, password: this.rconPassword,
timeout: 200000, // 200s, should allow for commands up to 1250kB in length timeout: 200000, // 200s, should allow for commands up to 1250kB in length
maxPending: 5, maxPending: this.maxConcurrentCommands,
}; };
this._rconClient = new rconClient.Rcon(config); this._rconClient = new rconClient.Rcon(config);
@@ -700,9 +705,31 @@ class FactorioServer extends events.EventEmitter {
this._rconReady = false; this._rconReady = false;
this._startRcon().catch(() => { /* Ignore */ }); this._startRcon().catch(() => { /* Ignore */ });
}); });
// XXX: Workaround to suppress bogus event listener warning
if (this._rconClient.emitter instanceof events.EventEmitter) {
this._rconClient.emitter.setMaxListeners(this.maxConcurrentCommands + 5);
}
await this._rconClient.connect(); await this._rconClient.connect();
} }
get maxConcurrentCommands() {
return this._maxConcurrentCommands;
}
set maxConcurrentCommands(value) {
this._maxConcurrentCommands = value;
this.setMaxListeners(value + 5);
if (this._rconClient) {
if (this._rconClient.sendQueue) {
this._rconClient.sendQueue.maxConcurrent = value;
}
// XXX: Workaround to suppress bogus event listener warning
if (this._rconClient.emitter instanceof events.EventEmitter) {
this._rconClient.emitter.setMaxListeners(value + 5);
}
}
}
/** /**
* Initialize class instance * Initialize class instance
* *
+35
View File
@@ -19,6 +19,41 @@ function basicType(value) {
return typeof value; return typeof value;
} }
/**
* Asynchronously wait for the given duration
*
* @param {number} duration - Time to wait for in milliseconds.
*/
async function wait(duration) {
await new Promise(resolve => { setTimeout(resolve, duration); });
}
/**
* Resolve a promise with a timeout.
*
* @param {Promise} promise - Promise to wait for.
* @param {number} time - Maximum time im milliseconds to wait for.
* @param {*=} timeoutResult - Value to return if the operation timed out.
*/
async function timeout(promise, time, timeoutResult) {
let timer;
try {
return await Promise.race([
promise,
new Promise(resolve => {
timer = setTimeout(() => resolve(timeoutResult), time);
}),
]);
} finally {
clearTimeout(timer);
}
}
module.exports = { module.exports = {
basicType, basicType,
wait,
timeout,
}; };
+39 -31
View File
@@ -411,19 +411,10 @@ messages.assignInstanceCommand = new Request({
permission: "core.instance.assign", permission: "core.instance.assign",
requestProperties: { requestProperties: {
"instance_id": { type: "integer" }, "instance_id": { type: "integer" },
"slave_id": { type: "integer" }, "slave_id": { type: ["integer", "null"] },
}, },
}); });
messages.assignInstance = new Request({
type: "assign_instance",
links: ["master-slave"],
requestProperties: {
"serialized_config": { type: "object" },
},
forwardTo: "instance",
});
messages.startInstance = new Request({ messages.startInstance = new Request({
type: "start_instance", type: "start_instance",
links: ["control-master", "master-slave", "slave-instance"], links: ["control-master", "master-slave", "slave-instance"],
@@ -655,6 +646,44 @@ messages.deleteUser = new Request({
}, },
}); });
// Internal requests
messages.updateInstances = new Request({
type: "update_instances",
links: ["slave-master"],
requestProperties: {
"instances": {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["serialized_config", "status"],
properties: {
"serialized_config": { type: "object" },
"status": { enum: [
"stopped", "starting", "running", "creating_save", "exporting_data",
]},
},
},
},
},
});
messages.assignInstance = new Request({
type: "assign_instance",
links: ["master-slave"],
requestProperties: {
"serialized_config": { type: "object" },
},
forwardTo: "instance",
});
messages.unassignInstance = new Request({
type: "unassigne_instance",
links: ["master-slave"],
forwardTo: "instance",
});
messages.getMetrics = new Request({ messages.getMetrics = new Request({
type: "get_metrics", type: "get_metrics",
links: ["master-slave", "slave-instance"], links: ["master-slave", "slave-instance"],
@@ -846,27 +875,6 @@ messages.instanceStatusChanged = new Event({
}, },
}); });
messages.updateInstances = new Event({
type: "update_instances",
links: ["slave-master"],
eventProperties: {
"instances": {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["serialized_config", "status"],
properties: {
"serialized_config": { type: "object" },
"status": { enum: [
"stopped", "starting", "running", "creating_save", "exporting_data",
]},
},
},
},
},
});
messages.instanceOutput = new Event({ messages.instanceOutput = new Event({
type: "instance_output", type: "instance_output",
links: ["instance-slave", "slave-master", "master-control"], links: ["instance-slave", "slave-master", "master-control"],
+4
View File
@@ -192,6 +192,10 @@ class BaseInstancePlugin {
* @param {string} event.name - Name of the player that joined/left. * @param {string} event.name - Name of the player that joined/left.
*/ */
async onPlayerEvent(event) { } async onPlayerEvent(event) { }
async sendRcon(message, expectEmpty) {
return await this.instance.sendRcon(message, expectEmpty, this.info.name);
}
} }
/** /**
+1128 -47
View File
File diff suppressed because it is too large. Load diff
+71 -22
View File
@@ -54,6 +54,7 @@ const libPrometheus = require("@clusterio/lib/prometheus");
const libConfig = require("@clusterio/lib/config"); const libConfig = require("@clusterio/lib/config");
const libUsers = require("@clusterio/lib/users"); const libUsers = require("@clusterio/lib/users");
const libSharedCommands = require("@clusterio/lib/shared_commands"); const libSharedCommands = require("@clusterio/lib/shared_commands");
const libHelpers = require("@clusterio/lib/helpers");
const express = require("express"); const express = require("express");
const compression = require("compression"); const compression = require("compression");
@@ -147,8 +148,7 @@ async function getMetrics(req, res, next) {
let requests = []; let requests = [];
for (let slaveConnection of slaveConnections.values()) { for (let slaveConnection of slaveConnections.values()) {
// XXX this needs a timeout requests.push(libHelpers.timeout(libLink.messages.getMetrics.send(slaveConnection), 8e3, null));
requests.push(libLink.messages.getMetrics.send(slaveConnection));
} }
for await (let result of await libPrometheus.defaultRegistry.collect()) { for await (let result of await libPrometheus.defaultRegistry.collect()) {
@@ -157,14 +157,28 @@ async function getMetrics(req, res, next) {
let resultMap = new Map(); let resultMap = new Map();
for (let response of await Promise.all(requests)) { for (let response of await Promise.all(requests)) {
if (!response) {
// TODO: Log timeout occured?
continue;
}
for (let result of response.results) { for (let result of response.results) {
if (!resultMap.has(result.metric.name)) { if (!resultMap.has(result.metric.name)) {
resultMap.set(result.metric.name, result); resultMap.set(result.metric.name, result);
} else { } else {
// Merge metrics received by multiple slaves // Merge metrics received by multiple slaves
let stored = resultMap.get(result.metric.name); let receivedSamples = new Map(result.samples);
stored.samples.push(...result.samples); for (let [suffix, suffixSamples] of resultMap.get(result.metric.name).samples) {
if (receivedSamples.has(suffix)) {
suffixSamples.push(...receivedSamples.get(suffix));
receivedSamples.delete(suffix);
}
}
for (let entry of receivedSamples) {
result.samples.push(entry);
}
} }
} }
} }
@@ -712,29 +726,43 @@ class ControlConnection extends BaseConnection {
} }
async assignInstanceCommandRequestHandler(message, request) { async assignInstanceCommandRequestHandler(message, request) {
let instance = db.instances.get(message.data.instance_id); let { slave_id, instance_id } = message.data;
let instance = db.instances.get(instance_id);
if (!instance) { if (!instance) {
throw new libErrors.RequestError(`Instance with ID ${message.data.instance_id} does not exist`); throw new libErrors.RequestError(`Instance with ID ${instance_id} does not exist`);
} }
// XXX: Should the instance be stopped if it's running on another slave? // Check if target slave is connected
let newSlaveConnection;
let connection = slaveConnections.get(message.data.slave_id); if (slave_id !== null) {
if (!connection) { newSlaveConnection = slaveConnections.get(slave_id);
// The case of the slave not getting the assign instance message if (!newSlaveConnection) {
// still have to be handled, so it's not a requirement that the // The case of the slave not getting the assign instance message
// target slave be connected to the master while doing the // still have to be handled, so it's not a requirement that the
// assignment, but it is IMHO a better user experience if this // target slave be connected to the master while doing the
// is the case. // assignment, but it is IMHO a better user experience if this
throw new libErrors.RequestError("Target slave is not connected to the master server"); // is the case.
throw new libErrors.RequestError("Target slave is not connected to the master server");
}
} }
instance.config.set("instance.assigned_slave", message.data.slave_id); // Unassign from currently assigned slave if it is connected.
let currentAssignedSlave = instance.config.get("instance.assigned_slave");
if (currentAssignedSlave !== null && slave_id !== currentAssignedSlave) {
let oldSlaveConnection = slaveConnections.get(currentAssignedSlave);
if (oldSlaveConnection && !oldSlaveConnection.connector.closing) {
await libLink.messages.unassignInstance.send(oldSlaveConnection, { instance_id });
}
}
return await libLink.messages.assignInstance.send(connection, { // Assign to target
instance_id: instance.config.get("instance.id"), instance.config.set("instance.assigned_slave", slave_id);
serialized_config: instance.config.serialize(), if (slave_id !== null) {
}); await libLink.messages.assignInstance.send(newSlaveConnection, {
instance_id,
serialized_config: instance.config.serialize(),
});
}
} }
async setInstanceOutputSubscriptionsRequestHandler(message) { async setInstanceOutputSubscriptionsRequestHandler(message) {
@@ -967,13 +995,23 @@ class SlaveConnection extends BaseConnection {
async instanceStatusChangedEventHandler(message, event) { async instanceStatusChangedEventHandler(message, event) {
let instance = db.instances.get(message.data.instance_id); let instance = db.instances.get(message.data.instance_id);
// It's possible to get updates from an instance that does not exist
// or is not assigned to the slave it originated from if it was
// reassigned or deleted while the connection to the slave it was
// originally on was down at the time.
if (!instance || instance.config.get("instance.assigned_slave") !== this._id) {
console.log(`Clusterio | Got bogus update for instance id ${message.data.instance_id}`);
return;
}
let prev = instance.status; let prev = instance.status;
instance.status = message.data.status; instance.status = message.data.status;
console.log(`Clusterio | Instance ${instance.config.get("instance.name")} State: ${instance.status}`); console.log(`Clusterio | Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
await libPlugin.invokeHook(masterPlugins, "onInstanceStatusChanged", instance, prev); await libPlugin.invokeHook(masterPlugins, "onInstanceStatusChanged", instance, prev);
} }
async updateInstancesEventHandler(message) { async updateInstancesRequestHandler(message) {
// Push updated instance configs // Push updated instance configs
for (let instance of db.instances.values()) { for (let instance of db.instances.values()) {
if (instance.config.get("instance.assigned_slave") === this._id) { if (instance.config.get("instance.assigned_slave") === this._id) {
@@ -991,10 +1029,21 @@ class SlaveConnection extends BaseConnection {
let masterInstance = db.instances.get(instanceConfig.get("instance.id")); let masterInstance = db.instances.get(instanceConfig.get("instance.id"));
if (masterInstance) { if (masterInstance) {
// Check if this instance is assigned somewhere else.
if (masterInstance.config.get("instance.assigned_slave") !== this._id) {
await libLink.messages.unassignInstance.send(this, {
instance_id: masterInstance.config.get("instance.id"),
});
continue;
}
// Already have this instance, update state instead // Already have this instance, update state instead
if (masterInstance.status !== instance.status) { if (masterInstance.status !== instance.status) {
let prev = masterInstance.status; let prev = masterInstance.status;
masterInstance.status = instance.status; masterInstance.status = instance.status;
console.log(
`Clusterio | Instance ${instanceConfig.get("instance.name")} State: ${instance.status}`
);
await libPlugin.invokeHook(masterPlugins, "onInstanceStatusChanged", instance, prev); await libPlugin.invokeHook(masterPlugins, "onInstanceStatusChanged", instance, prev);
} }
continue; continue;
+110 -50
View File
@@ -36,12 +36,21 @@ const libConfig = require("@clusterio/lib/config");
const libSharedCommands = require("@clusterio/lib/shared_commands"); const libSharedCommands = require("@clusterio/lib/shared_commands");
const instanceRconCommandsCounter = new libPrometheus.Counter( const instanceRconCommandDuration = new libPrometheus.Histogram(
"clusterio_instance_rcon_commands_total", "clusterio_instance_rcon_command_duration_seconds",
"How many commands have been sent to the instance", "Histogram of the RCON command duration from request to response.",
{ labels: ["instance_id"] } { labels: ["instance_id"] }
); );
const instanceRconCommandSize = new libPrometheus.Histogram(
"clusterio_instance_rcon_command_size_bytes",
"Histogram of the RCON command sizes that are sent.",
{
labels: ["instance_id", "plugin"],
buckets: libPrometheus.Histogram.exponential(16, 2, 12),
}
);
const instanceFactorioCpuTime = new libPrometheus.Gauge( const instanceFactorioCpuTime = new libPrometheus.Gauge(
"clusterio_instance_factorio_cpu_time_total", "clusterio_instance_factorio_cpu_time_total",
"Factorio CPU time spent in seconds.", "Factorio CPU time spent in seconds.",
@@ -61,7 +70,7 @@ const instanceFactorioAutosaveSize = new libPrometheus.Gauge(
); );
function applyAsConfig(name) { function applyAsConfig(name) {
return async function action(server, value) { return async function action(instance, value) {
if (name === "tags" && value instanceof Array) { if (name === "tags" && value instanceof Array) {
// Replace spaces with non-break spaces and delimit by spaces. // Replace spaces with non-break spaces and delimit by spaces.
// This does change the defined tags, but there doesn't seem to // This does change the defined tags, but there doesn't seem to
@@ -69,7 +78,7 @@ function applyAsConfig(name) {
value = value.map(tag => tag.replace(/ /g, "\u00a0")).join(" "); value = value.map(tag => tag.replace(/ /g, "\u00a0")).join(" ");
} }
try { try {
await server.sendRcon(`/config set ${name} ${value}`); await instance.sendRcon(`/config set ${name} ${value}`);
} catch (err) { } catch (err) {
console.error(`Error applying server setting ${name}`, err); console.error(`Error applying server setting ${name}`, err);
} }
@@ -91,11 +100,11 @@ const serverSettingsActions = {
"game_password": applyAsConfig("password"), "game_password": applyAsConfig("password"),
"require_user_verification": applyAsConfig("require-user-verification"), "require_user_verification": applyAsConfig("require-user-verification"),
"tags": applyAsConfig("tags"), "tags": applyAsConfig("tags"),
"visibility": async (server, value) => { "visibility": async (instance, value) => {
for (let scope of ["lan", "public", "steam"]) { for (let scope of ["lan", "public", "steam"]) {
try { try {
let enabled = Boolean(value[scope]); let enabled = Boolean(value[scope]);
await server.sendRcon(`/config set visibility-${scope} ${enabled}`); await instance.sendRcon(`/config set visibility-${scope} ${enabled}`);
} catch (err) { } catch (err) {
console.error(`Error applying visibility ${scope}`, err); console.error(`Error applying visibility ${scope}`, err);
} }
@@ -124,6 +133,9 @@ class Instance extends libLink.Link{
} else if (group.name === "factorio" && field === "enable_whitelist") { } else if (group.name === "factorio" && field === "enable_whitelist") {
this.updateFactorioWhitelist(group.get(field)).finally(hook); this.updateFactorioWhitelist(group.get(field)).finally(hook);
} else { } else {
if (group.name === "factorio" && field === "max_concurrent_commands") {
this.server.maxConcurrentCommands = group.get(field);
}
hook(); hook();
} }
}; };
@@ -137,6 +149,7 @@ class Instance extends libLink.Link{
enableWhitelist: this.config.get("factorio.enable_whitelist"), enableWhitelist: this.config.get("factorio.enable_whitelist"),
verboseLogging: this.config.get("factorio.verbose_logging"), verboseLogging: this.config.get("factorio.verbose_logging"),
stripPaths: this.config.get("factorio.strip_paths"), stripPaths: this.config.get("factorio.strip_paths"),
maxConcurrentCommands: this.config.get("factorio.max_concurrent_commands"),
}; };
this._status = "stopped"; this._status = "stopped";
@@ -145,12 +158,6 @@ class Instance extends libLink.Link{
factorioDir, this._dir, serverOptions factorioDir, this._dir, serverOptions
); );
let originalSendRcon = this.server.sendRcon;
this.server.sendRcon = (...args) => {
instanceRconCommandsCounter.labels(String(this.config.get("instance.id"))).inc();
return originalSendRcon.call(this.server, ...args);
};
this.server.on("output", (output) => { this.server.on("output", (output) => {
libLink.messages.instanceOutput.send(this, { instance_id: this.config.get("instance.id"), output }); libLink.messages.instanceOutput.send(this, { instance_id: this.config.get("instance.id"), output });
@@ -176,6 +183,17 @@ class Instance extends libLink.Link{
}); });
} }
async sendRcon(message, expectEmpty, plugin = "") {
let instanceId = String(this.config.get("instance.id"));
let observeDuration = instanceRconCommandDuration.labels(instanceId).startTimer();
try {
return await this.server.sendRcon(message, expectEmpty);
} finally {
observeDuration();
instanceRconCommandSize.labels(instanceId, plugin).observe(Buffer.byteLength(message, "utf8"));
}
}
async _autosave(name) { async _autosave(name) {
let stat = await fs.stat(this.path("saves", `${name}.zip`)); let stat = await fs.stat(this.path("saves", `${name}.zip`));
instanceFactorioAutosaveSize.labels(String(this.config.get("instance.id"))).set(stat.size); instanceFactorioAutosaveSize.labels(String(this.config.get("instance.id"))).set(stat.size);
@@ -515,7 +533,7 @@ class Instance extends libLink.Link{
async updateInstanceData() { async updateInstanceData() {
let name = libLuaTools.escapeString(this.config.get("instance.name")); let name = libLuaTools.escapeString(this.config.get("instance.name"));
let id = this.config.get("instance.id"); let id = this.config.get("instance.id");
await this.server.sendRcon(`/sc clusterio_private.update_instance(${id}, "${name}")`, true); await this.sendRcon(`/sc clusterio_private.update_instance(${id}, "${name}")`, true);
} }
async updateFactorioSettings(current, previous) { async updateFactorioSettings(current, previous) {
@@ -524,7 +542,7 @@ class Instance extends libLink.Link{
for (let [key, action] of Object.entries(serverSettingsActions)) { for (let [key, action] of Object.entries(serverSettingsActions)) {
if (current[key] !== undefined && !util.isDeepStrictEqual(current[key], previous[key])) { if (current[key] !== undefined && !util.isDeepStrictEqual(current[key], previous[key])) {
await action(this.server, current[key]); await action(this, current[key]);
} }
} }
} }
@@ -537,18 +555,18 @@ class Instance extends libLink.Link{
*/ */
async updateFactorioWhitelist(enable) { async updateFactorioWhitelist(enable) {
if (!enable) { if (!enable) {
await this.server.sendRcon("/whitelist disable"); await this.sendRcon("/whitelist disable");
} }
if (this.config.get("factorio.sync_whitelist")) { if (this.config.get("factorio.sync_whitelist")) {
await this.server.sendRcon("/whitelist clear"); await this.sendRcon("/whitelist clear");
for (let player of this._slave.whitelist) { for (let player of this._slave.whitelist) {
await this.server.sendRcon(`/whitelist ${player}`); await this.sendRcon(`/whitelist ${player}`);
} }
} }
if (enable) { if (enable) {
await this.server.sendRcon("/whitelist enable"); await this.sendRcon("/whitelist enable");
} }
} }
@@ -559,7 +577,7 @@ class Instance extends libLink.Link{
let { name, admin } = message.data; let { name, admin } = message.data;
let command = admin ? `/promote ${name}` : `/demote ${name}`; let command = admin ? `/promote ${name}` : `/demote ${name}`;
await this.server.sendRcon(command); await this.sendRcon(command);
} }
async banlistUpdateEventHandler(message) { async banlistUpdateEventHandler(message) {
@@ -569,7 +587,7 @@ class Instance extends libLink.Link{
let { name, banned, reason } = message.data; let { name, banned, reason } = message.data;
let command = banned ? `/ban ${name} ${reason}` : `/unban ${name}`; let command = banned ? `/ban ${name} ${reason}` : `/unban ${name}`;
await this.server.sendRcon(command); await this.sendRcon(command);
} }
async whitelistUpdateEventHandler(message) { async whitelistUpdateEventHandler(message) {
@@ -579,7 +597,7 @@ class Instance extends libLink.Link{
let { name, whitelisted } = message.data; let { name, whitelisted } = message.data;
let command = whitelisted ? `/whitelist add ${name}` : `/whiteliste remove ${name}`; let command = whitelisted ? `/whitelist add ${name}` : `/whiteliste remove ${name}`;
await this.server.sendRcon(command); await this.sendRcon(command);
} }
@@ -719,7 +737,7 @@ class Instance extends libLink.Link{
} }
async sendRconRequestHandler(message) { async sendRconRequestHandler(message) {
let result = await this.server.sendRcon(message.data.command); let result = await this.sendRcon(message.data.command);
return { result }; return { result };
} }
@@ -751,15 +769,16 @@ class Instance extends libLink.Link{
* Searches for instances in the provided directory * Searches for instances in the provided directory
* *
* Looks through all sub-dirs of the provided directory for valid * Looks through all sub-dirs of the provided directory for valid
* instance definitions and updates the mapping of instance id to * instance definitions and return a mapping of instance id to
* instanceInfo objects. * instanceInfo objects.
* *
* @param {Map<integer, Object>} instanceInfos -
* mapping between instance id and information about this instance.
* @param {string} instancesDir - Directory containing instances * @param {string} instancesDir - Directory containing instances
* @param {Object} logger - console like logging interface. * @param {Object} logger - console like logging interface.
* @returns {Map<integer, Object>}
* mapping between instance id and information about this instance.
*/ */
async function discoverInstances(instanceInfos, instancesDir, logger) { async function discoverInstances(instancesDir, logger) {
let instanceInfos = new Map();
for (let entry of await fs.readdir(instancesDir, { withFileTypes: true })) { for (let entry of await fs.readdir(instancesDir, { withFileTypes: true })) {
if (entry.isDirectory()) { if (entry.isDirectory()) {
let instanceConfig = new libConfig.InstanceConfig(); let instanceConfig = new libConfig.InstanceConfig();
@@ -777,11 +796,6 @@ async function discoverInstances(instanceInfos, instancesDir, logger) {
continue; continue;
} }
// Ignore instances we have already discovered before.
if (instanceInfos.has(instanceConfig.get("instance.id"))) {
continue;
}
let instancePath = path.join(instancesDir, entry.name); let instancePath = path.join(instancesDir, entry.name);
logger.log(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`); logger.log(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`);
instanceInfos.set(instanceConfig.get("instance.id"), { instanceInfos.set(instanceConfig.get("instance.id"), {
@@ -790,6 +804,7 @@ async function discoverInstances(instanceInfos, instancesDir, logger) {
}); });
} }
} }
return instanceInfos;
} }
class InstanceConnection extends libLink.Link { class InstanceConnection extends libLink.Link {
@@ -815,7 +830,7 @@ class InstanceConnection extends libLink.Link {
let instanceConnection = this.slave.instanceConnections.get(instanceId); let instanceConnection = this.slave.instanceConnections.get(instanceId);
if (!instanceConnection) { if (!instanceConnection) {
// Instance is probably on another slave // Instance is probably on another slave
return await request.send(this.slave, message.data); return await this.forwardRequestToMaster(message, request);
} }
if (request.plugin && !instanceConnection.plugins.has(request.plugin)) { if (request.plugin && !instanceConnection.plugins.has(request.plugin)) {
@@ -831,7 +846,7 @@ class InstanceConnection extends libLink.Link {
let instanceConnection = this.slave.instanceConnections.get(instanceId); let instanceConnection = this.slave.instanceConnections.get(instanceId);
if (!instanceConnection) { if (!instanceConnection) {
// Instance is probably on another slave // Instance is probably on another slave
await this.slave.forwardEventToMaster(message, event); await this.forwardEventToMaster(message, event);
return; return;
} }
if (event.plugin && !instanceConnection.plugins.has(event.plugin)) { return; } if (event.plugin && !instanceConnection.plugins.has(event.plugin)) { return; }
@@ -911,6 +926,7 @@ class Slave extends libLink.Link {
this.config = slaveConfig; this.config = slaveConfig;
this.instanceConnections = new Map(); this.instanceConnections = new Map();
this.discoveredInstanceInfos = new Map();
this.instanceInfos = new Map(); this.instanceInfos = new Map();
this.adminlist = new Set(); this.adminlist = new Set();
@@ -922,6 +938,7 @@ class Slave extends libLink.Link {
this.serverPlugins = new Map(Object.entries(data.plugins)); this.serverPlugins = new Map(Object.entries(data.plugins));
}); });
this._startup = true;
this._disconnecting = false; this._disconnecting = false;
this._shuttingDown = false; this._shuttingDown = false;
@@ -1120,20 +1137,28 @@ class Slave extends libLink.Link {
console.log(`Clusterio | Updated config for ${instanceInfo.path}`); console.log(`Clusterio | Updated config for ${instanceInfo.path}`);
} else { } else {
let instanceConfig = new libConfig.InstanceConfig(); instanceInfo = this.discoveredInstanceInfos.get(instance_id);
await instanceConfig.load(serialized_config); if (instanceInfo) {
instanceInfo.config.update(serialized_config, true);
// XXX: race condition on multiple simultanious calls } else {
let instanceDir = await this._findNewInstanceDir(instanceConfig.get("instance.name")); let instanceConfig = new libConfig.InstanceConfig();
await instanceConfig.load(serialized_config);
// XXX: race condition on multiple simultanious calls
let instanceDir = await this._findNewInstanceDir(instanceConfig.get("instance.name"));
await Instance.create(instanceDir, this.config.get("slave.factorio_directory"));
instanceInfo = {
path: instanceDir,
config: instanceConfig,
};
this.discoveredInstanceInfos.set(instance_id, instanceInfo);
}
await Instance.create(instanceDir, this.config.get("slave.factorio_directory"));
instanceInfo = {
path: instanceDir,
config: instanceConfig,
};
this.instanceInfos.set(instance_id, instanceInfo); this.instanceInfos.set(instance_id, instanceInfo);
console.log(`Clusterio | assigned instance ${instanceConfig.get("instance.name")}`); console.log(`Clusterio | assigned instance ${instanceInfo.config.get("instance.name")}`);
} }
// Somewhat hacky, but in the event of a lost session the status is // Somewhat hacky, but in the event of a lost session the status is
@@ -1156,6 +1181,20 @@ class Slave extends libLink.Link {
); );
} }
async unassignInstanceRequestHandler(message) {
let instanceId = message.data.instance_id;
let instanceInfo = this.instanceInfos.get(instanceId);
if (instanceInfo) {
let instanceConnection = this.instanceConnections.get(instanceId);
if (instanceConnection && ["starting", "running"].includes(instanceConnection.status)) {
await libLink.messages.stopInstance.send(instanceConnection, { instance_id: instanceId });
}
this.instanceInfos.delete(instanceId);
console.log(`Clusterio | unassigned instance ${instanceInfo.config.get("instance.name")}`);
}
}
/** /**
* Initialize and connect an unloaded instance * Initialize and connect an unloaded instance
* *
@@ -1245,13 +1284,14 @@ class Slave extends libLink.Link {
throw new libErrors.RequestError(`Instance with ID ${instanceId} is running`); throw new libErrors.RequestError(`Instance with ID ${instanceId} is running`);
} }
let instanceInfo = this.instanceInfos.get(instanceId); let instanceInfo = this.discoveredInstanceInfos.get(instanceId);
if (!instanceInfo) { if (!instanceInfo) {
throw new libErrors.RequestError(`Instance with ID ${instanceId} does not exist`); throw new libErrors.RequestError(`Instance with ID ${instanceId} does not exist`);
} }
await fs.remove(instanceInfo.path); this.discoveredInstanceInfos.delete(instanceId);
this.instanceInfos.delete(instanceId); this.instanceInfos.delete(instanceId);
await fs.remove(instanceInfo.path);
} }
/** /**
@@ -1261,16 +1301,36 @@ class Slave extends libLink.Link {
* the slave and master server with the new list of instances. * the slave and master server with the new list of instances.
*/ */
async updateInstances() { async updateInstances() {
await discoverInstances(this.instanceInfos, this.config.get("slave.instances_directory"), console); this.discoveredInstanceInfos = await discoverInstances(this.config.get("slave.instances_directory"), console);
let list = []; let list = [];
for (let [instanceId, instanceInfo] of this.instanceInfos) { for (let [instanceId, instanceInfo] of this.discoveredInstanceInfos) {
let instanceConnection = this.instanceConnections.get(instanceId); let instanceConnection = this.instanceConnections.get(instanceId);
list.push({ list.push({
serialized_config: instanceInfo.config.serialize(), serialized_config: instanceInfo.config.serialize(),
status: instanceConnection ? instanceConnection.status : "stopped", status: instanceConnection ? instanceConnection.status : "stopped",
}); });
} }
libLink.messages.updateInstances.send(this, { instances: list }); await libLink.messages.updateInstances.send(this, { instances: list });
// Handle configured auto startup instances
if (this._startup) {
this._startup = false;
for (let [instanceId, instanceInfo] of this.instanceInfos) {
if (instanceInfo.config.get("instance.auto_start")) {
try {
let instanceConnection = await this._connectInstance(instanceId);
await libLink.messages.startInstance.send(instanceConnection, {
instance_id: instanceId,
save: null,
});
} catch (err) {
console.error(`Error during auto startup for ${instanceInfo.config.get("instance.name")}:`);
console.error(err);
}
}
}
}
} }
async prepareDisconnectRequestHandler(message, request) { async prepareDisconnectRequestHandler(message, request) {
+4
View File
@@ -49,6 +49,10 @@ export class Control extends libLink.Link {
} }
this.instanceOutputHandlers = new Map(); this.instanceOutputHandlers = new Map();
this.connector.on("connect", () => {
this.updateInstanceOutputSubscriptions().catch(console.error);
});
} }
async instanceOutputEventHandler(message) { async instanceOutputEventHandler(message) {
+1 -1
View File
@@ -33,7 +33,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
async chatEventHandler(message) { async chatEventHandler(message) {
// TODO check if cross server chat is enabled // TODO check if cross server chat is enabled
let content = `[${message.data.instance_name}] ${removeTags(message.data.content)}`; let content = `[${message.data.instance_name}] ${removeTags(message.data.content)}`;
await this.instance.server.sendRcon(`/sc game.print('${libLuaTools.escapeString(content)}')`, true); await this.sendRcon(`/sc game.print('${libLuaTools.escapeString(content)}')`, true);
} }
sendChat(message) { sendChat(message) {
+4 -4
View File
@@ -29,7 +29,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
async progressEventHandler(message) { async progressEventHandler(message) {
let techsJson = libLuaTools.escapeString(JSON.stringify(message.data.technologies)); let techsJson = libLuaTools.escapeString(JSON.stringify(message.data.technologies));
await this.instance.server.sendRcon(`/sc research_sync.update_progress("${techsJson}")`, true); await this.sendRcon(`/sc research_sync.update_progress("${techsJson}")`, true);
} }
async researchFinished(tech) { async researchFinished(tech) {
@@ -38,13 +38,13 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
async finishedEventHandler(message) { async finishedEventHandler(message) {
let { name, level, researched } = message.data; let { name, level, researched } = message.data;
await this.instance.server.sendRcon( await this.sendRcon(
`/sc research_sync.research_technology("${libLuaTools.escapeString(name)}", ${level})`, true `/sc research_sync.research_technology("${libLuaTools.escapeString(name)}", ${level})`, true
); );
} }
async onStart() { async onStart() {
let dumpJson = await this.instance.server.sendRcon("/sc research_sync.dump_technologies()"); let dumpJson = await this.sendRcon("/sc research_sync.dump_technologies()");
let techsToSend = []; let techsToSend = [];
let instanceTechs = new Map(); let instanceTechs = new Map();
for (let tech of JSON.parse(dumpJson)) { for (let tech of JSON.parse(dumpJson)) {
@@ -74,7 +74,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
if (techsToSync.length) { if (techsToSync.length) {
let syncJson = libLuaTools.escapeString(JSON.stringify(techsToSync)); let syncJson = libLuaTools.escapeString(JSON.stringify(techsToSync));
await this.instance.server.sendRcon(`/sc research_sync.sync_technologies("${syncJson}")`, true); await this.sendRcon(`/sc research_sync.sync_technologies("${syncJson}")`, true);
} }
} }
} }
+11 -2
View File
@@ -1,6 +1,7 @@
"use strict"; "use strict";
const libPlugin = require("@clusterio/lib/plugin"); const libPlugin = require("@clusterio/lib/plugin");
const { Gauge } = require("@clusterio/lib/prometheus"); const { Gauge } = require("@clusterio/lib/prometheus");
const libHelpers = require("@clusterio/lib/helpers");
const instancePlayerCount = new Gauge( const instancePlayerCount = new Gauge(
@@ -28,8 +29,8 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
} }
} }
async onMetrics() { async gatherMetrics() {
let string = await this.instance.server.sendRcon("/sc statistics_exporter.export()"); let string = await this.sendRcon("/sc statistics_exporter.export()");
let stats; let stats;
try { try {
stats = JSON.parse(string); stats = JSON.parse(string);
@@ -61,6 +62,14 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
} }
} }
} }
async onMetrics() {
// Wait up to 5 seconds for the metrics to be collected. It may
// take a long time for the command to go through if the command
// stream is overloaded. Should this take more than 5 seconds the
// previous values for the metrics will end up being sent to master.
await libHelpers.timeout(this.gatherMetrics(), 5e3);
}
} }
module.exports = { module.exports = {
+4 -4
View File
@@ -22,7 +22,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
async onStart() { async onStart() {
this.pingId = setInterval(() => { this.pingId = setInterval(() => {
this.instance.server.sendRcon( this.sendRcon(
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true "/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true
).catch(unexpectedError); ).catch(unexpectedError);
}, 5000); }, 5000);
@@ -30,7 +30,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
let response = await this.info.messages.getStorage.send(this.instance); let response = await this.info.messages.getStorage.send(this.instance);
// TODO Diff with dump of invdata produce minimal command to sync // TODO Diff with dump of invdata produce minimal command to sync
let itemsJson = libLuaTools.escapeString(JSON.stringify(response.items)); let itemsJson = libLuaTools.escapeString(JSON.stringify(response.items));
await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true); await this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true);
} }
onExit() { onExit() {
@@ -68,7 +68,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
} }
let itemsJson = libLuaTools.escapeString(JSON.stringify(response.items)); let itemsJson = libLuaTools.escapeString(JSON.stringify(response.items));
await this.instance.server.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`, true); await this.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`, true);
} }
// combinator signals --------------------------------------------------------- // combinator signals ---------------------------------------------------------
@@ -87,7 +87,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
items.push(["signal-unixtime", Math.floor(Date.now()/1000)]); items.push(["signal-unixtime", Math.floor(Date.now()/1000)]);
let itemsJson = libLuaTools.escapeString(JSON.stringify(items)); let itemsJson = libLuaTools.escapeString(JSON.stringify(items));
await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true); await this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true);
} }
} }
+27
View File
@@ -1,5 +1,6 @@
"use strict"; "use strict";
const assert = require("assert").strict; const assert = require("assert").strict;
const hrtime = process.hrtime.bigint;
const libHelpers = require("@clusterio/lib/helpers"); const libHelpers = require("@clusterio/lib/helpers");
@@ -15,4 +16,30 @@ describe("lib/helpers", function() {
assert.equal(libHelpers.basicType({ a: 1 }), "object"); assert.equal(libHelpers.basicType({ a: 1 }), "object");
}); });
}); });
describe("wait", function() {
it("should wait the approximate amount of time given", async function() {
let startNs = hrtime();
await libHelpers.wait(100);
let duration = Number(hrtime() - startNs) / 1e6;
assert(duration > 90 && duration < 110, `duration waited (${duration}ms) 10% of expected (100ms)`);
});
});
describe("timeout", function() {
it("should return the result of an already resolved promise", async function() {
assert.equal(await libHelpers.timeout(Promise.resolve("value"), 10, "timeout"), "value");
});
it("should return the result of the promise if received before timeout", async function() {
assert.equal(await libHelpers.timeout(
(async () => { await libHelpers.wait(10); return "value"; })(), 20, "timeout"
), "value");
});
it("should return the timeoutResult if not received before timeout", async function() {
assert.equal(await libHelpers.timeout(
(async () => { await libHelpers.wait(20); return "value"; })(), 10, "timeout"
), "timeout");
});
});
}); });
+314 -26
View File
@@ -200,7 +200,194 @@ describe("lib/prometheus", function() {
}); });
}); });
describe("class Histogram", function() {
let histogram;
let labeledHistogram;
beforeEach(function() {
histogram = new prometheus.Histogram(
"test", "Help", { register: false, buckets: [1, 2, 3, 4, Infinity] }
);
labeledHistogram = new prometheus.Histogram(
"test", "Help", { register: false, buckets: [1, Infinity], labels: ["a", "b"] }
);
});
describe("constructor", function() {
it("should have buckets, sum and count initialized to 0", function() {
assert.deepEqual(
histogram.buckets,
new Map([[1, 0], [2, 0], [3, 0], [4, 0], [Infinity, 0]])
);
assert.equal(histogram.sum, 0);
assert.equal(histogram.count, 0);
});
it("should provide a default for buckets", function() {
histogram = new prometheus.Histogram(
"test", "help", { register: false }
);
assert(histogram._buckets.length, "no default buckets were made");
});
it("should add inifinity bucket if not present", function() {
histogram = new prometheus.Histogram(
"test", "help", { buckets: [1], register: false }
);
assert.deepEqual(histogram._buckets, [1, Infinity]);
});
it("should not mutate bucket option passed to it", function() {
let buckets = [1];
histogram = new prometheus.Histogram(
"test", "help", { buckets, register: false }
);
assert.deepEqual(buckets, [1]);
});
it("should throw on unrecoginzed option", function() {
assert.throws(
() => new prometheus.Histogram("test", "help", { invalid: 1 }),
new Error("Unrecognized option 'invalid'")
);
});
it("should throw on using le as label", function() {
assert.throws(() => {
let test = new prometheus.Histogram(
"test", "Help", { register: false, labels: ["le", "test"] }
);
}, new Error("Reserved label 'le'"));
});
});
describe(".observe()", function() {
it("should increment the appropriate bucket", function() {
histogram.observe(2);
assert.deepEqual(
histogram.buckets,
new Map([[1, 0], [2, 1], [3, 1], [4, 1], [Infinity, 1]])
);
assert.equal(histogram.sum, 2);
assert.equal(histogram.count, 1);
histogram.observe(4);
assert.deepEqual(
histogram.buckets,
new Map([[1, 0], [2, 1], [3, 1], [4, 2], [Infinity, 2]])
);
assert.equal(histogram.sum, 6);
assert.equal(histogram.count, 2);
});
});
describe(".startTimer()", function() {
it("should add observations when closure is called", function() {
let observeDuration = histogram.startTimer();
assert.equal(histogram.count, 0);
observeDuration();
assert.equal(histogram.count, 1);
observeDuration();
assert.equal(histogram.count, 2);
});
});
describe("static .linear()", function() {
it("should give back the appropriate buckets", function() {
let linear = prometheus.Histogram.linear;
assert.deepEqual(linear(1, 2, 3), [1, 3, 5]);
assert.deepEqual(linear(-1, 4, 3), [-1, 3, 7]);
assert.deepEqual(linear(1, 1, 4), [1, 2, 3, 4]);
});
});
describe("static .exponential()", function() {
it("should give back the appropriate buckets", function() {
let exponential = prometheus.Histogram.exponential;
assert.deepEqual(exponential(1, 2, 3), [1, 2, 4]);
assert.deepEqual(exponential(2, 10, 3), [2, 20, 200]);
assert.deepEqual(exponential(1, 1.5, 4), [1, 1.5, 2.25, 3.375]);
});
});
describe(".labels()", function() {
it("should initialize a value for the label set", function() {
labeledHistogram.labels({ a: "1", b: "2" });
assert(labeledHistogram._bucketValues.has('a="1",b="2",le="1"'), "bucket value not set");
assert(labeledHistogram._bucketValues.has('a="1",b="2",le="+Inf"'), "bucket value not set");
assert(labeledHistogram._sumValues.has('a="1",b="2"'), "sum value not set");
assert(labeledHistogram._countValues.has('a="1",b="2"'), "count value not set");
labeledHistogram.labels("4", "5");
assert(labeledHistogram._bucketValues.has('a="4",b="5",le="1"'), "bucket value not set");
assert(labeledHistogram._bucketValues.has('a="4",b="5",le="+Inf"'), "bucket value not set");
assert(labeledHistogram._sumValues.has('a="4",b="5"'), "sum value not set");
assert(labeledHistogram._countValues.has('a="4",b="5"'), "count value not set");
});
it("should return a child supporting the histogram methods", function() {
let child = labeledHistogram.labels({ a: "1", b: "2" });
void child.buckets;
void child.sum;
void child.count;
child.observe(2);
let observeDuration = child.startTimer();
observeDuration();
child = labeledHistogram.labels("7", "8");
void child.buckets;
void child.sum;
void child.count;
child.observe(2);
observeDuration = child.startTimer();
observeDuration();
});
it("should return the same child for the same labels", function() {
let child = labeledHistogram.labels("1", "2");
assert.equal(labeledHistogram.labels("1", "2"), child);
assert.equal(labeledHistogram.labels({ a: "1", b: "2" }), child);
});
});
describe(".remove()", function() {
it("should remove a label set created with .labels()", function() {
labeledHistogram.labels("1", "2");
labeledHistogram.remove("1", "2");
assert(!labeledHistogram._bucketValues.has('a="1",b="2",le="1"'), "bucket value not removed");
assert(!labeledHistogram._bucketValues.has('a="1",b="2",le="+Inf"'), "bucket value not removed");
assert(!labeledHistogram._sumValues.has('a="1",b="2"'), "sum value not removed");
assert(!labeledHistogram._countValues.has('a="1",b="2"'), "count value not removed");
});
});
describe(".removeAll()", function() {
it("should remove a label set created with .labels()", function() {
labeledHistogram.labels("1", "2");
labeledHistogram.labels("1", "3");
labeledHistogram.labels("4", "6");
labeledHistogram.removeAll({ a: "1" });
assert.deepEqual(
labeledHistogram._bucketValues,
new Map([['a="4",b="6",le="1"', 0], ['a="4",b="6",le="+Inf"', 0]])
);
assert.deepEqual(labeledHistogram._sumValues, new Map([['a="4",b="6"', 0]]));
assert.deepEqual(labeledHistogram._countValues, new Map([['a="4",b="6"', 0]]));
});
});
describe(".clear()", function() {
it("should remove all label sets created with .labels()", function() {
labeledHistogram.labels("1", "2");
labeledHistogram.labels("1", "3");
labeledHistogram.labels("4", "6");
labeledHistogram.clear();
assert(labeledHistogram._bucketValues.size === 0, "buckets were not cleared");
assert(labeledHistogram._sumValues.size === 0, "sums were not cleared");
assert(labeledHistogram._countValues.size === 0, "counts were not cleared");
});
});
});
describe("class ValueCollector", function() { describe("class ValueCollector", function() {
describe(".get()", function() {
it("should return the current value", function() {
let gauge = new prometheus.Gauge("test", "Help", { register: false });
gauge.set(11);
assert.equal(gauge.get(), 11);
});
});
});
describe("class LabeledCollector", function() {
describe("constructor", function() { describe("constructor", function() {
it("should throw on invalid metric name", function() { it("should throw on invalid metric name", function() {
for (let name of ["", " ", "0a", "$a"]) { for (let name of ["", " ", "0a", "$a"]) {
@@ -242,14 +429,6 @@ describe("lib/prometheus", function() {
}); });
}); });
describe(".get()", function() {
it("should return the current value", function() {
let gauge = new prometheus.Gauge("test", "Help", { register: false });
gauge.set(11);
assert.equal(gauge.get(), 11);
});
});
describe(".labels()", function() { describe(".labels()", function() {
let gauge; let gauge;
beforeEach(function() { beforeEach(function() {
@@ -310,6 +489,7 @@ describe("lib/prometheus", function() {
it("should return the same child for the same labels", function() { it("should return the same child for the same labels", function() {
let child = gauge.labels("1", "2"); let child = gauge.labels("1", "2");
assert.equal(gauge.labels("1", "2"), child); assert.equal(gauge.labels("1", "2"), child);
assert.equal(gauge.labels({ a: "1", b: "2" }), child);
}); });
}); });
describe(".remove()", function() { describe(".remove()", function() {
@@ -393,8 +573,9 @@ describe("lib/prometheus", function() {
"collector did not give exactly one result" "collector did not give exactly one result"
); );
let value = results[0].samples.get("").get("");
assert( assert(
Math.abs(results[0].samples.get("") - Date.now() / 1000) < 1, Math.abs(value - Date.now() / 1000) < 1,
"value set is not close to the current unix time" "value set is not close to the current unix time"
); );
}); });
@@ -413,7 +594,7 @@ describe("lib/prometheus", function() {
"collector did not give exactly one result" "collector did not give exactly one result"
); );
let value = results[0].samples.get(""); let value = results[0].samples.get("").get("");
assert( assert(
value < 5 && value > 0, value < 5 && value > 0,
`process time is not between 0 and 5 seconds (${value})` `process time is not between 0 and 5 seconds (${value})`
@@ -434,7 +615,7 @@ describe("lib/prometheus", function() {
"collector did not give exactly one result" "collector did not give exactly one result"
); );
let value = results[0].samples.get(""); let value = results[0].samples.get("").get("");
assert( assert(
value < 200e6 && value > 0, value < 200e6 && value > 0,
`resident memory is not between 0 and 200 MB (${value})` `resident memory is not between 0 and 200 MB (${value})`
@@ -455,7 +636,7 @@ describe("lib/prometheus", function() {
"collector did not give exactly one result" "collector did not give exactly one result"
); );
let value = results[0].samples.get(""); let value = results[0].samples.get("").get("");
assert( assert(
value < 200e6 && value > 0, value < 200e6 && value > 0,
`heap is not between 0 and 200 MB (${value})` `heap is not between 0 and 200 MB (${value})`
@@ -500,6 +681,59 @@ describe("lib/prometheus", function() {
'test{a="x",b="y"} 0\n' 'test{a="x",b="y"} 0\n'
); );
}); });
it("should format histogram metrics", async function() {
let registry = new prometheus.CollectorRegistry();
let histogram = new prometheus.Histogram(
"test", "Help", { register: false, buckets: [1, 2, 3, 4] }
);
for (let value of [2, 2, 5, 2]) {
histogram.observe(value);
}
registry.register(histogram);
let exposition = await prometheus.exposition(registry.collect());
assert.equal(
exposition,
"# HELP test Help\n" +
"# TYPE test histogram\n" +
'test_bucket{le="1"} 0\n' +
'test_bucket{le="2"} 3\n' +
'test_bucket{le="3"} 3\n' +
'test_bucket{le="4"} 3\n' +
'test_bucket{le="+Inf"} 4\n' +
"test_sum 11\n" +
"test_count 4\n"
);
});
it("should format histogram metrics with labels", async function() {
let registry = new prometheus.CollectorRegistry();
let histogram = new prometheus.Histogram(
"test", "Help", { register: false, buckets: [1], labels: ["l"] }
);
for (let [value, label] of [[1, "a"], [2, "b"], [3, "b"]]) {
histogram.labels(label).observe(value);
}
histogram.labels("c");
registry.register(histogram);
let exposition = await prometheus.exposition(registry.collect());
assert.equal(
exposition,
"# HELP test Help\n" +
"# TYPE test histogram\n" +
'test_bucket{l="a",le="1"} 1\n' +
'test_bucket{l="a",le="+Inf"} 1\n' +
'test_bucket{l="b",le="1"} 0\n' +
'test_bucket{l="b",le="+Inf"} 2\n' +
'test_bucket{l="c",le="1"} 0\n' +
'test_bucket{l="c",le="+Inf"} 0\n' +
'test_sum{l="a"} 1\n' +
'test_sum{l="b"} 5\n' +
'test_sum{l="c"} 0\n' +
'test_count{l="a"} 1\n' +
'test_count{l="b"} 2\n' +
'test_count{l="c"} 0\n'
);
});
it("should format Infinity and NaN", async function() { it("should format Infinity and NaN", async function() {
let registry = new prometheus.CollectorRegistry(); let registry = new prometheus.CollectorRegistry();
let gauge = new prometheus.Gauge( let gauge = new prometheus.Gauge(
@@ -523,38 +757,64 @@ describe("lib/prometheus", function() {
let serializedResult = { let serializedResult = {
metric: { type: "type", name: "name", help: "Help", labels: ["a", "b"] }, metric: { type: "type", name: "name", help: "Help", labels: ["a", "b"] },
samples: [['a="1",b="2"', 1], ['a="3",b="4"', 5]], samples: [["", [['a="1",b="2"', 1], ['a="3",b="4"', 5]]]],
}; };
let deserializedResult = { let deserializedResult = {
metric: new prometheus.Metric("type", "name", "Help", ["a", "b"]), metric: new prometheus.Metric("type", "name", "Help", ["a", "b"]),
samples: new Map([['a="1",b="2"', 1], ['a="3",b="4"', 5]]), samples: new Map([["", new Map([['a="1",b="2"', 1], ['a="3",b="4"', 5]])]]),
}; };
describe("serializeResult()", function() { describe("serializeResult()", function() {
it("should convert a result into plain object", function() { it("should convert real results into the reference serialization", async function() {
let counter = new prometheus.Counter("a_counter", "Counter", { register: false });
let gauge = new prometheus.Gauge("a_gauge", "Gauge", { register: false, labels: ["a", "b"] });
let registry = new prometheus.CollectorRegistry();
registry.register(counter);
registry.register(gauge);
counter.inc(3);
gauge.labels("1", "2").set(1);
gauge.labels("3", "4").set(5);
let results = [];
for await (let result of registry.collect()) {
results.push(result);
}
let referenceSerialization = [
{
metric: { type: "counter", name: "a_counter", help: "Counter", labels: [] },
samples: [["", [["", 3]]]],
},
{
metric: { type: "gauge", name: "a_gauge", help: "Gauge", labels: ["a", "b"] },
samples: [["", [['a="1",b="2"', 1], ['a="3",b="4"', 5]]]],
},
];
assert.deepEqual( assert.deepEqual(
prometheus.serializeResult(deserializedResult), results.map(result => prometheus.serializeResult(result)),
serializedResult referenceSerialization
); );
}); });
it("should support adding labels", function() { it("should support adding labels", function() {
assert.deepEqual( assert.deepEqual(
prometheus.serializeResult({ prometheus.serializeResult({
metric: new prometheus.Metric("type", "name", "Help"), metric: new prometheus.Metric("type", "name", "Help"),
samples: new Map([["", 2]]), samples: new Map([["name", new Map([["", 2]])]]),
}, { addLabels: { c: "8" }}), }, { addLabels: { c: "8" }}),
{ {
metric: { type: "type", name: "name", help: "Help", labels: ["c"] }, metric: { type: "type", name: "name", help: "Help", labels: ["c"] },
samples: [['c="8"', 2]], samples: [["name", [['c="8"', 2]]]],
} }
); );
assert.deepEqual( assert.deepEqual(
prometheus.serializeResult({ prometheus.serializeResult({
metric: new prometheus.Metric("type", "name", "Help", ["a"]), metric: new prometheus.Metric("type", "name", "Help", ["a"]),
samples: new Map([['a="4"', 3]]), samples: new Map([["", new Map([['a="4"', 3]])]]),
}, { addLabels: { c: "8" }}), }, { addLabels: { c: "8" }}),
{ {
metric: { type: "type", name: "name", help: "Help", labels: ["a", "c"] }, metric: { type: "type", name: "name", help: "Help", labels: ["a", "c"] },
samples: [['a="4",c="8"', 3]], samples: [["", [['a="4",c="8"', 3]]]],
} }
); );
}); });
@@ -562,21 +822,21 @@ describe("lib/prometheus", function() {
assert.deepEqual( assert.deepEqual(
prometheus.serializeResult({ prometheus.serializeResult({
metric: new prometheus.Metric("type", "name", "Help"), metric: new prometheus.Metric("type", "name", "Help"),
samples: new Map([["", 2]]), samples: new Map([["", new Map([["", 2]])]]),
}, { metricName: "new" }), }, { metricName: "new" }),
{ {
metric: { type: "type", name: "new", help: "Help", labels: [] }, metric: { type: "type", name: "new", help: "Help", labels: [] },
samples: [["", 2]], samples: [["", [["", 2]]]],
} }
); );
assert.deepEqual( assert.deepEqual(
prometheus.serializeResult({ prometheus.serializeResult({
metric: new prometheus.Metric("type", "name", "Help"), metric: new prometheus.Metric("type", "name", "Help"),
samples: new Map([["", 2]]), samples: new Map([["", new Map([["", 2]])]]),
}, { metricHelp: "old" }), }, { metricHelp: "old" }),
{ {
metric: { type: "type", name: "name", help: "old", labels: [] }, metric: { type: "type", name: "name", help: "old", labels: [] },
samples: [["", 2]], samples: [["", [["", 2]]]],
} }
); );
}); });
@@ -584,7 +844,7 @@ describe("lib/prometheus", function() {
assert.throws( assert.throws(
() => prometheus.serializeResult({ () => prometheus.serializeResult({
metric: new prometheus.Metric("type", "name", "Help"), metric: new prometheus.Metric("type", "name", "Help"),
samples: new Map([["", 3]]), samples: new Map([["", new Map([["", 3]])]]),
}, { invalid: true }), }, { invalid: true }),
new Error("Unrecognized option 'invalid'") new Error("Unrecognized option 'invalid'")
); );
@@ -600,6 +860,34 @@ describe("lib/prometheus", function() {
}); });
}); });
describe("integration", function() {
it("should round trip serialize histogram results", async function() {
let registry = new prometheus.CollectorRegistry();
let histogram = new prometheus.Histogram(
"test", "Help", { register: false, buckets: [1], labels: ["l"] }
);
for (let [value, label] of [[1, "a"], [2, "b"], [3, "b"]]) {
histogram.labels(label).observe(value);
}
histogram.labels("c");
registry.register(histogram);
let results = [];
for await (let result of registry.collect()) {
results.push(result);
}
let exposition = await prometheus.exposition(results);
let serialized = results.map(res => prometheus.serializeResult(res));
let jsonRoundtrip = JSON.parse(JSON.stringify(serialized));
let deserialized = jsonRoundtrip.map(res => prometheus.deserializeResult(res));
assert.deepEqual(deserialized, results);
let roundTripExposition = await prometheus.exposition(deserialized);
assert.equal(roundTripExposition, exposition);
});
});
describe("post test checks", function() { describe("post test checks", function() {
it("should not have any collectors in the default registry", function() { it("should not have any collectors in the default registry", function() {
this.skip(); // XXX Doesn't work in Clusterio's shared testing env this.skip(); // XXX Doesn't work in Clusterio's shared testing env
+4
View File
@@ -92,6 +92,10 @@ class MockInstance extends libLink.Link {
}, },
}; };
} }
async sendRcon(command, expectEmpty, plugin) {
return await this.server.sendRcon(command);
}
} }
class MockSlave extends libLink.Link { class MockSlave extends libLink.Link {
+1 -2
View File
@@ -152,8 +152,7 @@ describe("Slave testing", function() {
it("should discover test instance", async function() { it("should discover test instance", async function() {
let logger = { log: () => {}, error: () => {} }; let logger = { log: () => {}, error: () => {} };
let instancePath = path.join("test", "file", "instances"); let instancePath = path.join("test", "file", "instances");
let instanceInfos = new Map(); let instanceInfos = await slave._discoverInstances(instancePath, logger);
await slave._discoverInstances(instanceInfos, instancePath, logger);
let referenceConfig = new libConfig.InstanceConfig(); let referenceConfig = new libConfig.InstanceConfig();
await referenceConfig.init(); await referenceConfig.init();