Merge pull request #338 from Hornwitser/assorted-improvements

Assorted improvements
This commit is contained in:
Hornwitser authored and GitHub committed 2020-11-15 20:03:35 +01:00
commit a06781ebc3
22 files changed
+297 -35

No files matched your search

+1
View File
@@ -46,6 +46,7 @@ Version 2.0.0
player data.
- Added synchronization of in-game adminlist, banlist and whitelist to
Clusterio core. Previously this was handled by the playerManager plugin.
- Added stripping of long paths in the Factorio server log.
### Changes
+30 -2
View File
@@ -84,7 +84,7 @@ heartbeat interval it will be considered stale and closed. A lower
value means less time between connections going stale and them being
closed.
Defaults to 60
Defaults to 45
### master.connector_shutdown_timeout
@@ -129,7 +129,7 @@ Defaults to "New Slave".
### slave.id
Imutable numeric id of the slave which uniquely identifies this slave in
Immutable numeric id of the slave which uniquely identifies this slave in
the cluster.
Defaults to a random 31 bit number.
@@ -280,6 +280,34 @@ here overrides the properties there.
Defaults to {"tags":["clusterio"],"auto_pause":false}.
### factorio.verbose_logging
Pass the `--verbose` flag to Factorio when starting the server. This
prints more messages to the log.
Defaults to false
### factorio.strip_paths
Strip down absolute paths in the server log going to files inside the
instance directory such that they are relative to the instance
directory. This improves the signal to noise ratio of the log.
Sample output when disabled:
0.487 Loading map /srv/clusterio/instances/test/saves/world.zip: 2615888 bytes.
0.614 Checksum for script /srv/clusterio/instances/test/temp/currently-playing/control.lua: 2390553941
1.277 Script @/srv/clusterio/instances/test/temp/currently-playing/modules/example/test.lua:7: Example log line.
Sample output when enabled:
0.487 Loading map saves/world.zip: 2615888 bytes.
0.614 Checksum for script temp/currently-playing/control.lua: 2390553941
1.277 Script @modules/example/test.lua:7: Example log line.
Defaults to true
### factorio.sync_adminlist
Synchronize in-game admin list with admin status of the users in
+1 -1
View File
@@ -27,4 +27,4 @@ that log into instances under its control. These accounts are used to
store per player data and authorize management of the cluster. By
default users accounts are assigned a Player role that gives a limited
read access to the cluster resources, and admins of the cluster can be
assigned an Admin role to get access to manage it.
assigned a Cluster Admin role to get access to manage it.
+1 -1
View File
@@ -107,7 +107,7 @@ Replaces the roles the user has in the cluster with a new list of roles.
Calling with an empty roles argument will remove all roles from the
user.
By default there's a roled named Admin which grants access to
By default there's a roled named Cluster Admin which grants access to
everything and a role named Player which grants a limited read access to
the cluster, see [the section on roles](#roles) for more information
about setting up roles and permissions.
+5 -1
View File
@@ -43,7 +43,11 @@ function formatOutputColored(output) {
let info = "";
if (output.type === "log") {
let level = output.level;
if (level === "Info") {
if (level === "Script") {
level = chalk.bold.greenBright(level);
} else if (level === "Verbose") {
level = chalk.bold.gray(level);
} else if (level === "Info") {
level = chalk.bold.blueBright(level);
} else if (output.level === "Warning") {
level = chalk.bold.yellowBright(level);
+1 -1
View File
@@ -5,4 +5,4 @@ This package contains functionality shared between master, slave, ctl
and/or plugins of Clusterio.
See [the Clusterio repository](https://github.com/clusterio/factorioClusterio)
for more inforation on Clusterio.
for more information on Clusterio.
+13 -1
View File
@@ -78,7 +78,7 @@ MasterGroup.define({
title: "Heartbeat Interval",
description: "Interval heartbeats are sent out on WebSocket connections",
type: "number",
initial_value: 60,
initial_value: 45,
});
MasterGroup.define({
name: "connector_shutdown_timeout",
@@ -250,6 +250,18 @@ FactorioGroup.define({
"auto_pause": false,
},
});
FactorioGroup.define({
name: "verbose_logging",
description: "Enable verbose logging on the Factorio server",
type: "boolean",
initial_value: false,
});
FactorioGroup.define({
name: "strip_paths",
description: "Strip down instance paths in the log",
type: "boolean",
initial_value: true,
});
FactorioGroup.define({
name: "sync_adminlist",
description: "Synchronize adminlist with master server",
+48 -3
View File
@@ -12,6 +12,22 @@ const rconClient = require("rcon-client");
const libErrors = require("@clusterio/lib/errors");
/**
* Escapes text for inclusion in a RegExp
*
* Adds \ character in front of special meta characters in the passsed in
* text so that it can be embedded into a RegExp and only match the text.
*
* See https://stackoverflow.com/a/9310752
*
* @param {string} text - Text to escape RegExp meta chars in.
* @returns {string} escaped text.
* @inner
*/
function escapeRegExp(text) {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
}
/**
* Determines the version of Factorio the datadir is pointing to by
* reading the changelog.txt in it.
@@ -302,13 +318,15 @@ function parseOutput(line, source) {
// A seconds output has two different kinds of formats: The first is a
// log level and source location and has a format of
// "Level File.cpp:123: message"
const secLogRegex = /^(\w+) (\w+\.cpp:\d+): (.*)$/;
// "Script @/path/to/file.lua:123: message"
// "Script =(command):123: message"
const secLogRegex = /^(\w+) ((\w+\.cpp:\d+)|([@=].*?:\d+)): (.*)$/;
let secLogMatch = secLogRegex.exec(secContent);
if (secLogMatch) {
output.type = "log";
output.level = secLogMatch[1];
output.file = secLogMatch[2];
output.message = secLogMatch[3];
output.message = secLogMatch[5];
// The other possibility is that the content is a generic message
} else {
@@ -497,6 +515,9 @@ class FactorioServer extends events.EventEmitter {
* @param {number} options.gamePort - UDP port to host game on.
* @param {number} options.rconPort - TCP port to use for RCON.
* @param {string} options.rconPassword - Password use for RCON.
* @param {boolean} options.enableWhitelist - Turn on whitelisting.
* @param {boolean} options.verboseLogging - Enable verbose logging.
* @param {boolean} options.stripPaths - Strip paths in the console.
*/
constructor(factorioDir, writeDir, options) {
super();
@@ -517,12 +538,27 @@ class FactorioServer extends events.EventEmitter {
this.rconPassword = options.rconPassword;
/** Enable player whitelist */
this.enableWhitelist = options.enableWhitelist;
/** Enable verbose logging */
this.verboseLogging = options.verboseLogging;
this._state = "new";
this._server = null;
this._rconClient = null;
this._rconReady = false;
this._gameReady = false;
if (options.stripPaths) {
let chars = new Set(this.writePath("temp"));
chars.delete(":"); // Having a colon could lead to matching the line number
chars = [...chars].join("");
this._stripRegExp = new RegExp(
`(${escapeRegExp(this.writePath("temp", "currently-playing", path.sep))})|` +
`(${escapeRegExp(this.writePath(path.sep))})|` +
`(\\.\\.\\.[${escapeRegExp(chars)}]*?currently-playing\\${path.sep})`,
"g"
);
}
// Array of possible causes for an unexpected shutdown
this._unexpected = [];
@@ -606,7 +642,13 @@ class FactorioServer extends events.EventEmitter {
this.emit(source, line);
let output = parseOutput(line.toString("utf-8"), source);
line = line.toString("utf-8");
if (this._stripRegExp) {
line = line.replace(this._stripRegExp, "");
}
let output = parseOutput(line, source);
heuristicLoop: for (let heuristic of outputHeuristics) {
for (let [name, expected] of Object.entries(heuristic.filter)) {
if (!Object.hasOwnProperty.call(output, name)) {
@@ -774,6 +816,7 @@ class FactorioServer extends events.EventEmitter {
[
"--config", this.writePath("config.ini"),
"--create", this.writePath("saves", name),
...(this.verboseLogging ? ["--verbose"] : []),
],
{
detached: true,
@@ -816,6 +859,7 @@ class FactorioServer extends events.EventEmitter {
"--rcon-port", this.rconPort,
"--rcon-password", this.rconPassword,
...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
...(this.verboseLogging ? ["--verbose"] : []),
],
{
detached: true,
@@ -850,6 +894,7 @@ class FactorioServer extends events.EventEmitter {
"--rcon-port", this.rconPort,
"--rcon-password", this.rconPassword,
...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
...(this.verboseLogging ? ["--verbose"] : []),
],
{
detached: true,
+1 -1
View File
@@ -315,7 +315,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
this._socket.onerror = event => {
// It's assumed that close is always called by ws
console.error("SOCKET | Error:", event);
console.error("SOCKET | Error:", event.message || "unknown error");
};
this._socket.onopen = () => {
+12
View File
@@ -39,6 +39,18 @@ class Link {
if (err.errors) {
console.error(err.errors);
}
// Send back an error response if this was a request.
if (
typeof payload.type === "string"
&& payload.type.endsWith("_request")
&& Number.isInteger(payload.seq)
) {
this.connector.send(
`${payload.type.slice(0, -8)}_response`,
{ seq: payload.seq, error: err.message }
);
}
} else {
this.connector.emit("error", err);
}
+31 -4
View File
@@ -3,6 +3,25 @@
const libSchema = require("@clusterio/lib/schema");
const libErrors = require("@clusterio/lib/errors");
class MissingLinkHandlerError extends Error {
constructor(type, source, target) {
super();
this.code = "MISSING_LINK_HANDLER";
this.type = type;
this.source = source;
this.target = target;
this.handler = null;
this.plugin = null;
}
get message() {
let handler = this.handler ? `${this.handler}` : "handler";
let plugin = this.plugin ? ` on plugin ${this.plugin}` : "";
return `Missing ${handler}${plugin} for ${this.type} on ${this.source}-${this.target} link`;
}
}
/**
* Represents a message that can be sent over the link
*
@@ -168,7 +187,7 @@ class Request extends Message {
}
if (!handler) {
throw new Error(`Missing handler for ${this.requestType} on ${link.source}-${link.target} link`);
throw new MissingLinkHandlerError(this.requestType, link.source, link.target);
}
// Check permission if this is a handler for a control connection on the master
@@ -637,7 +656,7 @@ messages.deleteUser = new Request({
});
messages.getMetrics = new Request({
type: "statistics_exporter:get_metrics",
type: "get_metrics",
links: ["master-slave", "slave-instance"],
responseProperties: {
"results": { type: "array" },
@@ -766,7 +785,7 @@ class Event extends Message {
}
if (!handler) {
throw new Error(`Missing handler for ${this.eventType} on ${link.source}-${link.target} link`);
throw new MissingLinkHandlerError(this.eventType, link.source, link.target);
}
link.setHandler(this.eventType, message => {
@@ -959,7 +978,15 @@ messages.playerEvent = new Event({
*/
function attachAllMessages(link) {
for (let [name, message] of Object.entries(messages)) {
message.attach(link, link[name + message.handlerSuffix]);
let handler = `${name}${message.handlerSuffix}`;
try {
message.attach(link, link[handler]);
} catch (err) {
if (err.code === "MISSING_LINK_HANDLER") {
err.handler = handler;
}
throw err;
}
}
}
+16 -6
View File
@@ -402,13 +402,23 @@ function attachPluginMessages(link, pluginInfo, plugin) {
}
let handler = name + messageFormat.handlerSuffix;
if (plugin === null || !plugin[handler]) {
messageFormat.attach(link);
try {
if (plugin === null || !plugin[handler]) {
messageFormat.attach(link);
} else {
messageFormat.attach(link, async function(message, format) {
return await plugin[handler](message, format, this);
});
} else {
messageFormat.attach(link, async function(message, format) {
return await plugin[handler](message, format, this);
});
}
} catch (err) {
if (err.code === "MISSING_LINK_HANDLER") {
err.plugin = pluginInfo.name;
err.handler = handler;
}
throw err;
}
}
}
+1 -1
View File
@@ -102,7 +102,7 @@ class Role {
function ensureDefaultAdminRole(roles) {
let admin = roles.get(0);
if (!admin) {
admin = new Role({ id: 0, name: "Admin", description: "Cluster wide administrator." });
admin = new Role({ id: 0, name: "Cluster Admin", description: "Cluster wide administrator." });
roles.set(0, admin);
}
admin.permissions.add("core.admin");
+43 -6
View File
@@ -135,8 +135,11 @@ class Instance extends libLink.Link{
rconPort: this.config.get("factorio.rcon_port"),
rconPassword: this.config.get("factorio.rcon_password"),
enableWhitelist: this.config.get("factorio.enable_whitelist"),
verboseLogging: this.config.get("factorio.verbose_logging"),
stripPaths: this.config.get("factorio.strip_paths"),
};
this._status = "stopped";
this._running = false;
this.server = new libFactorio.FactorioServer(
factorioDir, this._dir, serverOptions
@@ -179,11 +182,23 @@ class Instance extends libLink.Link{
}
notifyStatus(status) {
this._status = status;
libLink.messages.instanceStatusChanged.send(this, {
instance_id: this.config.get("instance.id"), status,
});
}
/**
* Current state of the instance
*
* One of stopped, starting, running, creating_save and exporting_data
*
* @returns {string} instance status.
*/
get status() {
return this._status;
}
notifyExit() {
this._running = false;
this.notifyStatus("stopped");
@@ -218,6 +233,7 @@ class Instance extends libLink.Link{
}
async init(pluginInfos) {
this.notifyStatus("starting");
await this.server.init();
// load plugins
@@ -794,6 +810,22 @@ class InstanceConnection extends libLink.Link {
return await request.send(this.slave, message.data);
}
async forwardRequestToInstance(message, request) {
let instanceId = message.data.instance_id;
let instanceConnection = this.slave.instanceConnections.get(instanceId);
if (!instanceConnection) {
// Instance is probably on another slave
return await request.send(this.slave, message.data);
}
if (request.plugin && !instanceConnection.plugins.has(request.plugin)) {
throw new libErrors.RequestError(`Instance ID ${instanceId} does not have ${request.plugin} plugin loaded`);
}
return await request.send(instanceConnection, message.data);
}
async forwardEventToInstance(message, event) {
let instanceId = message.data.instance_id;
let instanceConnection = this.slave.instanceConnections.get(instanceId);
@@ -1086,7 +1118,6 @@ class Slave extends libLink.Link {
if (instanceInfo) {
instanceInfo.config.update(serialized_config, true);
console.log(`Clusterio | Updated config for ${instanceInfo.path}`);
// TODO: Notify of update if instance is running
} else {
let instanceConfig = new libConfig.InstanceConfig();
@@ -1102,8 +1133,17 @@ class Slave extends libLink.Link {
};
this.instanceInfos.set(instance_id, instanceInfo);
console.log(`Clusterio | assigned instance ${instanceConfig.get("instance.name")}`);
}
// Somewhat hacky, but in the event of a lost session the status is
// resent on assigment since the master server sends an assigment
// request for all the instances it knows should be on this slave.
let instanceConnection = this.instanceConnections.get(instance_id);
libLink.messages.instanceStatusChanged.send(this, {
instance_id,
status: instanceConnection ? instanceConnection.status : "stopped",
});
// save a copy of the instance config
let warnedOutput = {
@@ -1114,9 +1154,6 @@ class Slave extends libLink.Link {
path.join(instanceInfo.path, "instance.json"),
JSON.stringify(warnedOutput, null, 4)
);
// Notify master the instance is no longer usassigned
libLink.messages.instanceStatusChanged.send(this, { instance_id, status: "stopped" });
}
/**
@@ -1140,10 +1177,10 @@ class Slave extends libLink.Link {
let instance = new Instance(
this, connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config
);
this.instanceConnections.set(instanceId, instanceConnection);
await instance.init(this.pluginInfos);
// XXX: race condition on multiple simultanious calls
this.instanceConnections.set(instanceId, instanceConnection);
return instanceConnection;
}
@@ -83,7 +83,11 @@ function formatOutputColored(output, key) {
let info = "";
if (output.type === "log") {
let level = output.level;
if (level === "Info") {
if (level === "Script") {
level = <span className="factorio-script">{level}</span>;
} else if (level === "Verbose") {
level = <span className="factorio-verbose">{level}</span>;
} else if (level === "Info") {
level = <span className="factorio-info">{level}</span>;
} else if (output.level === "Warning") {
level = <span className="factorio-warning">{level}</span>;
@@ -102,11 +106,13 @@ function formatOutputColored(output, key) {
function InstanceConsole(props) {
let control = useContext(ControlContext);
let [lines, setLines] = useState([{ message: "<previous messages are not shown>" }]);
let [lines, setLines] = useState([<span key={0}>{"<previous messages are not shown>"}<br/></span>]);
useEffect(() => {
function outputHandler(output) {
setLines(currentLines => currentLines.concat([output]));
setLines(currentLines => currentLines.concat(
[formatOutputColored(output, currentLines.length)]
));
}
control.onInstanceOutput(props.id, outputHandler);
@@ -117,7 +123,7 @@ function InstanceConsole(props) {
return <>
<Paragraph code className="instance-console">
{lines.map(formatOutputColored)}
{lines}
<div className="scroll-anchor" />
</Paragraph>
</>;
+10
View File
@@ -118,6 +118,16 @@ body {
color: #808080;
}
.instance-console .factorio-script {
font-weight: bold;
color: #49c33c;
}
.instance-console .factorio-verbose {
font-weight: bold;
color: #888888;
}
.instance-console .factorio-info {
font-weight: bold;
color: #4646ea;
+1 -1
View File
@@ -2,7 +2,7 @@ Clusterio Statistics Exporter Plugin
====================================
Plugin exporting item production, fluid production, kill count, entity
build and poluttion statistics to the Prometheus integration to
build and pollution statistics to the Prometheus integration to
Clusterio.
+5 -1
View File
@@ -185,6 +185,10 @@ describe("Integration of Clusterio", function() {
assert.equal(await sendRcon(44, `/config get ${configName}`), `${expectedResult}\n`);
}
});
it("should not change the instance status", async function() {
slowTest(this);
await checkInstanceStatus(44, "running");
});
});
describe("user set-admin/whitelisted/banned", function() {
@@ -312,7 +316,7 @@ describe("Integration of Clusterio", function() {
describe("user set-roles", function() {
it("should set the roles on the user", async function() {
await execCtl("user set-roles temp Admin");
await execCtl('user set-roles temp "Cluster Admin"');
let result = await libLink.messages.listUsers.send(getControl());
let tempUser = result.list.find(user => user.name === "temp");
assert.deepEqual(tempUser.roles, [0]);
+45
View File
@@ -143,6 +143,51 @@ let testLines = new Map([
message: "Saving finished",
},
],
[
" 1.277 Script @/srv/clusterio/instances/test/temp/currently-playing/modules/example/test.lua:7: " +
"Example log line.",
{
format: "seconds",
time: "1.277",
type: "log",
level: "Script",
file: "@/srv/clusterio/instances/test/temp/currently-playing/modules/example/test.lua:7",
message: "Example log line.",
},
],
[
" 1.277 Script @C:\\My Cluster\\temp\\current-save\\file.lua:7: Contrived :12: example",
{
format: "seconds",
time: "1.277",
type: "log",
level: "Script",
file: "@C:\\My Cluster\\temp\\current-save\\file.lua:7",
message: "Contrived :12: example",
},
],
[
" 344.683 Script =(command):1: test",
{
format: "seconds",
time: "344.683",
type: "log",
level: "Script",
file: "=(command):1",
message: "test",
},
],
[
" 0.751 Verbose ModManager.cpp:554: Time to load mods: 0.746762",
{
format: "seconds",
time: "0.751",
type: "log",
level: "Verbose",
file: "ModManager.cpp:554",
message: "Time to load mods: 0.746762",
},
],
]);
module.exports = {
+11
View File
@@ -170,6 +170,17 @@ describe("lib/factorio/server", function() {
let writePath = path.join("temp", "test", "server");
let server = new libFactorio.FactorioServer(path.join("test", "file", "factorio"), writePath, {});
describe("constructor()", function() {
it("should handle dashes in write path with strapPaths enabled", function() {
// eslint-disable-next-line no-new
new libFactorio.FactorioServer(
path.join("test", "file", "factorio"),
path.join("temp", "test", "server-1"),
{ stripPaths: true }
);
});
});
describe(".init()", function() {
it("should not throw on first call", async function() {
await server.init();
+10
View File
@@ -35,6 +35,16 @@ describe("lib/link/link", function() {
await assert.rejects(result, new Error("Handler error"));
});
it("should give an error response back on an unhandled request", function() {
testConnector.sentMessages = [];
testConnector.emit("message", { seq: 1, type: "unhandled_request", data: {}});
assert.deepEqual(testConnector.sentMessages, [{
type: "unhandled_response",
seq: testConnector._seq - 1,
data: { seq: 1, error: "No validator for unhandled_request on source-target" },
}]);
});
describe(".setValidator()", function() {
it("should set the validator", function() {
testLink.setValidator("test", () => true);
+1 -1
View File
@@ -55,7 +55,7 @@ describe("lib/plugin", function() {
it("should throw if missing handler for the given message", function() {
assert.throws(
() => libPlugin.attachPluginMessages(mockLink, { name: "test", messages: { mockEvent }}, {}),
new Error("Missing handler for test:test_event on source-target link")
new Error("Missing mockEventEventHandler on plugin test for test:test_event on source-target link")
);
});
it("should throw if message starts with the wrong prefix", function() {