From 9bcbc1e7f3a5185eed2146f4f3502542b0bebd01 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Thu, 18 Jan 2024 15:49:01 +0000 Subject: [PATCH 01/68] Add http logging for express --- packages/controller/package.json | 1 + packages/controller/src/Controller.ts | 8 ++++++++ packages/lib/src/logging.ts | 7 +++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/controller/package.json b/packages/controller/package.json index 042a53ad..1a1e6365 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -30,6 +30,7 @@ "set-blocking": "^2.0.0", "webpack-cli": "^5.1.4", "winston": "^3.10.0", + "express-winston": "^4.2.0", "winston-daily-rotate-file": "^4.7.1", "ws": "^8.13.0", "yargs": "^17.7.2" diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 89214e0f..2b5b67f2 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -3,6 +3,7 @@ import { BlockList, type AddressInfo, isIPv4, isIPv6 } from "net"; import type { ControllerArgs } from "../controller"; import express, { type Request, type Response, type NextFunction, type Application } from "express"; import type { Static } from "@sinclair/typebox"; +import * as expressWinston from "express-winston"; import compression from "compression"; import events, { EventEmitter } from "events"; @@ -164,6 +165,10 @@ export default class Controller { this.app = express(); this.app.locals.controller = this; this.app.locals.streams = new Map(); + this.app.use(expressWinston.logger({ + winstonInstance: this.clusterLogger, + level: "http", + })); this.trustedProxies = this.parseTrustedProxies(); this.wsServer = new WsServer(this); @@ -283,6 +288,9 @@ export default class Controller { // Load plugins await this.loadPlugins(); + this.app.use(expressWinston.errorLogger({ + winstonInstance: this.clusterLogger, + })); this.wsServer = new WsServer(this); diff --git a/packages/lib/src/logging.ts b/packages/lib/src/logging.ts index c673f98e..d8502c2d 100644 --- a/packages/lib/src/logging.ts +++ b/packages/lib/src/logging.ts @@ -9,7 +9,8 @@ export const levels = Object.freeze({ audit: 3, info: 4, server: 5, - verbose: 6, + http: 6, + verbose: 7, }); export const colors = { @@ -19,6 +20,7 @@ export const colors = { audit: "bold brightGreen", info: "bold brightBlue", server: "bold", + http: "bold grey", verbose: "bold grey", } as const; winston.addColors(colors); @@ -163,7 +165,7 @@ export const logger = winston.createLogger({ // Who in their right mind thought that log levels should be hard coded // into the type when you can define custom levels you want to support? export type Logger = Omit & { @@ -173,6 +175,7 @@ export type Logger = Omit Date: Thu, 18 Jan 2024 18:05:27 +0000 Subject: [PATCH 02/68] Remove insecure and bloated module --- packages/controller/package.json | 1 - packages/controller/src/Controller.ts | 51 ++++++++++++++++++++++----- packages/lib/src/logging.ts | 8 ++--- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/packages/controller/package.json b/packages/controller/package.json index 1a1e6365..042a53ad 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -30,7 +30,6 @@ "set-blocking": "^2.0.0", "webpack-cli": "^5.1.4", "winston": "^3.10.0", - "express-winston": "^4.2.0", "winston-daily-rotate-file": "^4.7.1", "ws": "^8.13.0", "yargs": "^17.7.2" diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 2b5b67f2..42c93d2d 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -3,7 +3,6 @@ import { BlockList, type AddressInfo, isIPv4, isIPv6 } from "net"; import type { ControllerArgs } from "../controller"; import express, { type Request, type Response, type NextFunction, type Application } from "express"; import type { Static } from "@sinclair/typebox"; -import * as expressWinston from "express-winston"; import compression from "compression"; import events, { EventEmitter } from "events"; @@ -165,10 +164,31 @@ export default class Controller { this.app = express(); this.app.locals.controller = this; this.app.locals.streams = new Map(); - this.app.use(expressWinston.logger({ - winstonInstance: this.clusterLogger, - level: "http", - })); + + // Log all requests made to express + this.app.use((req, res, next) => { + const startNs = process.hrtime.bigint(); + res.once("finish", () => { + const endNs = process.hrtime.bigint(); + const durationMs = (Number(endNs - startNs) / 1e6); + logger.log({ + level: "http", + message: `HTTP ${req.httpVersion} ${req.method} ${req.originalUrl}` + + ` (Status ${res.statusCode} in ${durationMs}ms)`, + meta: { + method: req.method, + url: req.originalUrl, + statusCode: res.statusCode, + responseTime: durationMs, + httpVersion: req.httpVersion, + headers: req.headers, + query: req.query, + }, + }); + }); + + next(); + }); this.trustedProxies = this.parseTrustedProxies(); this.wsServer = new WsServer(this); @@ -288,9 +308,24 @@ export default class Controller { // Load plugins await this.loadPlugins(); - this.app.use(expressWinston.errorLogger({ - winstonInstance: this.clusterLogger, - })); + + // Log all express errors from middleware or routes (app.use is missing the type overload) + this.app.use((err: Error, req: express.Request, res: express.Response, next: express.NextFunction): void => { + logger.log({ + level: "error", + message: "Error while handling HTTP" + + `${req.httpVersion} ${req.method} ${req.originalUrl}:\n${err.stack}`, + meta: { + method: req.method, + url: req.originalUrl, + httpVersion: req.httpVersion, + headers: req.headers, + query: req.query, + }, + }); + + next(); + }); this.wsServer = new WsServer(this); diff --git a/packages/lib/src/logging.ts b/packages/lib/src/logging.ts index d8502c2d..c78dea30 100644 --- a/packages/lib/src/logging.ts +++ b/packages/lib/src/logging.ts @@ -8,8 +8,8 @@ export const levels = Object.freeze({ warn: 2, audit: 3, info: 4, - server: 5, - http: 6, + http: 5, + server: 6, verbose: 7, }); @@ -19,8 +19,8 @@ export const colors = { warn: "bold brightYellow", audit: "bold brightGreen", info: "bold brightBlue", + http: "bold", server: "bold", - http: "bold grey", verbose: "bold grey", } as const; winston.addColors(colors); @@ -174,8 +174,8 @@ export type Logger = Omit Date: Thu, 18 Jan 2024 18:45:01 +0000 Subject: [PATCH 03/68] Clean up the request correctly --- packages/controller/package.json | 2 ++ packages/controller/src/Controller.ts | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/controller/package.json b/packages/controller/package.json index 042a53ad..4af76e17 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -24,6 +24,7 @@ "busboy": "^1.6.0", "compression": "^1.7.4", "express": "^4.18.2", + "finalhandler": "^1.2.0", "fs-extra": "^11.1.1", "jsonwebtoken": "^9.0.1", "jszip": "^3.10.1", @@ -39,6 +40,7 @@ "@types/busboy": "^1.5.0", "@types/compression": "^1.7.2", "@types/express": "^4.17.17", + "@types/finalhandler": "^1.2.0", "@types/fs-extra": "^11.0.1", "@types/jsonwebtoken": "^9.0.2", "@types/node": "^20.4.5", diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 42c93d2d..f89753c3 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -3,6 +3,7 @@ import { BlockList, type AddressInfo, isIPv4, isIPv6 } from "net"; import type { ControllerArgs } from "../controller"; import express, { type Request, type Response, type NextFunction, type Application } from "express"; import type { Static } from "@sinclair/typebox"; +import finalhandler from "finalhandler"; import compression from "compression"; import events, { EventEmitter } from "events"; @@ -173,7 +174,7 @@ export default class Controller { const durationMs = (Number(endNs - startNs) / 1e6); logger.log({ level: "http", - message: `HTTP ${req.httpVersion} ${req.method} ${req.originalUrl}` + + message: `HTTP${req.httpVersion} ${req.method} ${req.originalUrl}` + ` (Status ${res.statusCode} in ${durationMs}ms)`, meta: { method: req.method, @@ -324,7 +325,9 @@ export default class Controller { }, }); - next(); + // Create and call the final handler, this is the same method used by express + // This is used instead of next(err) to stop express from logging the error to the console + finalhandler(req, res, { env: this.app.get("env") })(err); }); this.wsServer = new WsServer(this); From d7edae99c5932b8f14c5fc210810efdc42138168 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 22 Jan 2024 19:29:36 +0100 Subject: [PATCH 04/68] Add metrics for pending request Track the number of requests currently awaiting a response from a request handler both from interally issued requests and requests forwarded from other parties. --- CHANGELOG.md | 3 ++- packages/controller/controller.ts | 18 ++++++++++++++++++ packages/host/host.ts | 31 ++++++++++++++++++++++++++----- packages/lib/src/link/link.ts | 7 +++++++ 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca5d4e2f..4dd4be7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ### Features -- Added support for `color-settings` Mod settings (#568)[https://github.com/clusterio/clusterio/issues/568]. +- Added support for `color-settings` Mod settings [#568](https://github.com/clusterio/clusterio/issues/568). +- Added `clusterio_controller_pending_requests` and `clusterio_host_pending_requests` metrics. ### Changes diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 14b18e46..b4161f65 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -37,6 +37,24 @@ import { version } from "./package.json"; // globals let controller: Controller; +void new lib.Gauge( + "clusterio_controller_pending_requests", + "Count of pending link requests currently waiting in memory on the controller.", + { + callback: (gauge) => { + let count = 0; + if (controller) { + for (const host of controller.wsServer.hostConnections.values()) { + count += host.pendingRequestCount; + } + for (const control of controller.wsServer.controlConnections.values()) { + count += control.pendingRequestCount; + } + } + gauge.set(count); + }, + } +); void new lib.Gauge( "clusterio_controller_host_mapping", diff --git a/packages/host/host.ts b/packages/host/host.ts index 46827d85..fa195675 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -26,6 +26,27 @@ import { ConsoleTransport, levels, logger } from "@clusterio/lib"; import Host from "./src/Host"; +let host: Host | undefined; + +void new lib.Gauge( + "clusterio_host_pending_requests", + "Count of pending link requests currently waiting in memory on the host.", + { + labels: ["host_id"], + callback: (gauge) => { + if (!host) { + return; + } + + let count = host.pendingRequestCount; + for (const connection of host.instanceConnections.values()) { + count += connection.pendingRequestCount; + count += connection.instance.pendingRequestCount; + } + gauge.labels(String(host.config.get("host.id"))).set(count); + }, + } +); export class HostConnector extends lib.WebSocketClientConnector { constructor( @@ -205,7 +226,7 @@ async function startHost() { } let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos); - let host = new Host( + host = new Host( hostConnector, args.config, hostConfig, @@ -227,7 +248,7 @@ async function startHost() { secondSigint = true; logger.info("Caught interrupt signal, shutting down"); - host.shutdown(); + host!.shutdown(); }); let secondSigterm = false; process.on("SIGTERM", () => { @@ -240,11 +261,11 @@ async function startHost() { secondSigterm = true; logger.info("Caught termination signal, shutting down"); - host.shutdown(); + host!.shutdown(); }); process.on("SIGHUP", () => { logger.info("Terminal closed, shutting down"); - host.shutdown(); + host!.shutdown(); }); try { @@ -255,7 +276,7 @@ async function startHost() { } hostConnector.once("connect", () => { - logger.add(new lib.LinkTransport({ link: host })); + logger.add(new lib.LinkTransport({ link: host! })); }); await hostConnector.connect(); diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts index deabd028..c7365190 100644 --- a/packages/lib/src/link/link.ts +++ b/packages/lib/src/link/link.ts @@ -141,6 +141,13 @@ export class Link { }); } + /** + * Count of requests currently waiting for a response on this link + */ + get pendingRequestCount() { + return this._pendingRequests.size + this._forwardedRequests.size; + } + _clearPendingRequests(err: Error & { code?: string }) { for (let pending of this._pendingRequests.values()) { pending.reject(err); From 123148e2777ad90dca275676e58438f84497d62e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 22 Jan 2024 19:54:41 +0100 Subject: [PATCH 05/68] Fix pending request not deleted on response Fix memory leak due to the recorded _pendingRequests entry for a request not being deleted when the response for the request has been received. --- CHANGELOG.md | 1 + packages/lib/src/link/link.ts | 2 ++ test/lib/link/link.js | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd4be7c..ca5d9a34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Fixed ctl not exporting types used for adding commands to ctl plugins. - Fixed controller only plugins throwing an error about missing a config field on startup out when added to hosts. - Fixed installer incorrectly claiming the options --factorio-dir and --no-download-headless conflicts with each other. +- Fixed a memory leak on every request sent over the WebSocket. ## Version 2.0.0-alpha.14 diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts index c7365190..4793008b 100644 --- a/packages/lib/src/link/link.ts +++ b/packages/lib/src/link/link.ts @@ -428,6 +428,7 @@ export class Link { throw new libErrors.InvalidMessage(`Received reply from ${message.src} for message sent to ${pending.dst}`); } + this._pendingRequests.delete(message.dst.requestId!); try { pending.resolve(pending.request.responseFromJSON(message.data)); } catch (err: any) { @@ -444,6 +445,7 @@ export class Link { ); } + this._pendingRequests.delete(message.dst.requestId!); pending.reject(new libErrors.RequestError(message.data.message, message.data.code, message.data.stack)); } diff --git a/test/lib/link/link.js b/test/lib/link/link.js index 0c991031..c8bd06c4 100644 --- a/test/lib/link/link.js +++ b/test/lib/link/link.js @@ -16,6 +16,7 @@ describe("lib/link/link", function() { describe("class Link", function() { let testConnector; + /** @type {lib.Link} */ let testLink; let src = addr({ controlId: 1 }); let dst = addr("controller"); @@ -281,6 +282,7 @@ describe("lib/link/link", function() { let srcReq = new lib.Address(lib.Address.control, 1, 1); testConnector.emit("message", new lib.MessageResponse(1, dst, srcReq)); await request; + assert.deepEqual(testLink._pendingRequests, new Map()); }); it("should send request and reject with error when error response is received", async function() { let message = events.once(testConnector, "send"); @@ -297,6 +299,7 @@ describe("lib/link/link", function() { request, { message: "Error" } ); + assert.deepEqual(testLink._pendingRequests, new Map()); }); it("should send request and resolve with data when response is received", async function() { let message = events.once(testConnector, "send"); @@ -305,6 +308,7 @@ describe("lib/link/link", function() { let srcReq = new lib.Address(lib.Address.control, 1, 1); testConnector.emit("message", new lib.MessageResponse(1, dst, srcReq, 44)); assert.deepEqual(await request, new NumberRequest.Response(44)); + assert.deepEqual(testLink._pendingRequests, new Map()); }); it("should send event to the other side of the link", async function() { let message = events.once(testConnector, "send"); From 96bd5fb1c2879f50bfa0371b3c196f68a1266b27 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 22 Jan 2024 20:33:46 +0100 Subject: [PATCH 06/68] Handle virtual links in Link._clearPendingRequests When a request is forwarded from a virtual link to a WebSocket link as is the case when an instance sends a request to the controller the origin for the request is a virtual link. Handle this case in _clearPendingRequests instead of throwing an assertion error. --- CHANGELOG.md | 1 + packages/lib/src/link/link.ts | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca5d9a34..bfa1c742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Fixed controller only plugins throwing an error about missing a config field on startup out when added to hosts. - Fixed installer incorrectly claiming the options --factorio-dir and --no-download-headless conflicts with each other. - Fixed a memory leak on every request sent over the WebSocket. +- Fixed assertion error on closing or invalidating a link with messages forwarded from a virtual link. ## Version 2.0.0-alpha.14 diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts index 4793008b..14b7d1ec 100644 --- a/packages/lib/src/link/link.ts +++ b/packages/lib/src/link/link.ts @@ -153,12 +153,18 @@ export class Link { pending.reject(err); } for (let pending of this._forwardedRequests.values()) { - assert(pending.origin.connector instanceof WebSocketBaseConnector); - if (pending.origin.connector.hasSession) { - pending.origin.connector.sendResponseError( - new libData.ResponseError(err.message, err.code), pending.src, pending.dst - ); + // Drop response if the origin is a WebSocket and the session + // has expired. The other end of the link will have sent an + // error response to the request in this case. + if ( + pending.origin.connector instanceof WebSocketBaseConnector + && !pending.origin.connector.hasSession + ) { + continue; } + pending.origin.connector.sendResponseError( + new libData.ResponseError(err.message, err.code), pending.src, pending.dst + ); } this._pendingRequests.clear(); this._forwardedRequests.clear(); From 256b455315bba73bc3f0f2b0f8234813dd15505c Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Jan 2024 16:07:55 +0100 Subject: [PATCH 07/68] Fix host hang on stopping during create save If during the startup of an instance the Factorio server is started up to create a new save and the host shutdown in initiated then the instance will be commanded to stop while the FactorioServer state is "create". This was was not handled by Instance.stop leaving the child_process running after the host is done shutting down. Kill the FactorioServer on stop if the current state of it is stopping or create to make sure the server actually stops. --- CHANGELOG.md | 1 + packages/host/src/Instance.ts | 5 +++++ packages/host/src/server.ts | 6 ++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfa1c742..1bab77a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Fixed installer incorrectly claiming the options --factorio-dir and --no-download-headless conflicts with each other. - Fixed a memory leak on every request sent over the WebSocket. - Fixed assertion error on closing or invalidating a link with messages forwarded from a virtual link. +- Fixed host hanging if stopped while an instance is in the process of creating a save during startup. ## Version 2.0.0-alpha.14 diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index c4e3d473..98d4d0eb 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -955,6 +955,11 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await lib.invokeHook(this.plugins, "onStop"); await this.server.stop(); await this.sendSaveListUpdate(); + } else if ( + this.server._state === "stopping" + || this.server._state === "create" + ) { + await this.server.kill(); } } diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index b34af3b9..60089036 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -865,7 +865,9 @@ export class FactorioServer extends events.EventEmitter { try { let [code, signal] = await events.once(this._server, "exit"); - void signal; + if (signal) { + throw new Error(`Factorio exited with signal ${signal}`); + } if (code !== 0) { throw new Error(`Factorio exited with status ${code}`); } @@ -1092,7 +1094,7 @@ export class FactorioServer extends events.EventEmitter { * process. */ async kill(unexpected = false) { - this._check(["running", "stopping"]); + this._check(["running", "stopping", "create"]); this._killed = true; if (!unexpected) { this._state = "stopping"; From fc5701f24313000c37e71043decd893562e1fb18 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Jan 2024 16:23:09 +0100 Subject: [PATCH 08/68] Fix instance auto start Bypass the Link layer and invoke the handleInstanceStartRequest method directly on the Instance object during the host's instance auto start phase. --- .eslintrc.js | 1 + CHANGELOG.md | 1 + README.md | 1 - packages/host/src/Host.ts | 2 +- test/integration/clusterio.js | 39 ++++++++++++++++++++++++++++++++++- 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 3cf36d34..1487649a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -35,6 +35,7 @@ module.exports = { "rules": { "prefer-arrow-callback": "off", "no-invalid-this": "off", + "valid-jsdoc": "off", "node/no-unpublished-import": "off", "node/no-unpublished-require": "off", }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bab77a0..dc84398d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Fixed a memory leak on every request sent over the WebSocket. - Fixed assertion error on closing or invalidating a link with messages forwarded from a virtual link. - Fixed host hanging if stopped while an instance is in the process of creating a save during startup. +- Fixed `instance.auto_start` option. ## Version 2.0.0-alpha.14 diff --git a/README.md b/README.md index 10be24e6..0857fc07 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,6 @@ If you want to use a different user for the command line interface, you can gene ## Known Issues -- The `instance.auto_start` option is currently is broken and will not work, turn it off and do not use it for now. - When the `server_select` plugin is installed you may see `[error] Unhandled event server_select:UpdateInstancesEvent` logged when starting an instance. This should be ignored as it does not indicate an error occured. - After migrating from Alpha 13 to Alpha 14 running an instance data export fails with the error `Upload failed: 401 Unauthorized` due to the host token being issued to a slave. diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 7381ad36..06a23809 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -1056,7 +1056,7 @@ export default class Host extends lib.Link { if (instanceInfo.config.get("instance.auto_start")) { try { let instanceConnection = await this._connectInstance(instanceId); - await instanceConnection.send(new lib.InstanceStartRequest()); + await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest()); } catch (err: any) { logger.error( `Error during auto startup for ${instanceInfo.config.get("instance.name")}:\n${err.stack}`, diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index dcb3c36d..d34b8a0d 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -17,6 +17,7 @@ const { } = require("./index"); +/** @returns {Promise>} */ async function getInstances() { let instances = await getControl().send(new lib.InstanceDetailsListRequest()); return new Map(instances.map(instance => [instance.id, instance])); @@ -27,6 +28,10 @@ async function checkInstanceStatus(id, status) { assert.equal(instances.get(44).status, status, "incorrect instance status"); } +async function spawnAltHost(config = "alt-host-config.json") { + return await spawn("alt-host:", `node ../../packages/host run --config ${config}`, /Started host/); +} + async function startAltHost() { let config = "alt-host-config.json"; let configPath = path.join("temp", "test", config); @@ -38,7 +43,7 @@ async function startAltHost() { await exec(`node ../../packages/host --config ${config} config set host.instances_directory alt-instances`); await exec(`node ../../packages/host --config ${config} config set host.factorio_directory ../../factorio`); await exec(`node ../../packages/host --config ${config} config set host.mods_directory alt-mods`); - return await spawn("alt-host:", `node ../../packages/host run --config ${config}`, /Started host/); + return await spawnAltHost(config); } async function stopAltHost(hostProcess) { @@ -200,6 +205,38 @@ describe("Integration of Clusterio", function() { assert(await fs.pathExists(clusterioLib), "mod was not downloaded by the test"); }); }); + it("should auto start instances with auto_start enabled", async function() { + slowTest(this); + + let hostProcess; + try { + hostProcess = await startAltHost(); + await execCtl("instance create alt-start --id 97"); + await execCtl("instance assign alt-start 5"); + await execCtl("instance config set alt-start instance.auto_start true"); + await stopAltHost(hostProcess); + hostProcess = await spawnAltHost(); + // Stop the host immediatly to test the handling of stopping + // instances while automatically started and a save is being created + await stopAltHost(hostProcess); + hostProcess = await spawnAltHost(); + let status; + // Wait for instance status to become running. + for (let i = 0; i < 100; i++) { + const instances = await getInstances(); + status = instances.get(97)?.status ?? "not_present"; + if (status === "running") { + break; + } + await wait(100); // Try again 10 times per second. + } + assert.equal(status, "running"); + await execCtl("instance stop alt-start"); + await execCtl("instance delete alt-start"); + } finally { + await stopAltHost(hostProcess); + } + }); }); describe("clusterioctl", function() { From 847945cfdd5df5354e3677a13f96283ce311b043 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 24 Jan 2024 01:45:45 +0100 Subject: [PATCH 09/68] Fix plugin broadcasts not being filtered When a plugin broadcasts an event to instances there may be hosts and instances in the cluster which does not have the plugin loaded. This would cause an error to be logged for every instance which does not have the plugin loaded. Filter out broadcast events sent to hosts and instances such that instances and hosts that does not have the event do not received them. --- CHANGELOG.md | 1 + packages/controller/index.ts | 2 + packages/controller/src/BaseConnection.ts | 3 +- packages/controller/src/Controller.ts | 6 + packages/controller/src/ControllerRouter.ts | 16 +- packages/host/index.ts | 3 + packages/host/src/Host.ts | 35 +- packages/lib/src/link/link.ts | 9 +- test/integration/routing.js | 416 ++++++++++++++++++++ 9 files changed, 470 insertions(+), 21 deletions(-) create mode 100644 test/integration/routing.js diff --git a/CHANGELOG.md b/CHANGELOG.md index dc84398d..0264def0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Fixed assertion error on closing or invalidating a link with messages forwarded from a virtual link. - Fixed host hanging if stopped while an instance is in the process of creating a save during startup. - Fixed `instance.auto_start` option. +- Fixed host log spammed with errors when plugins broadcast messages to instances with the plugin disabled. ## Version 2.0.0-alpha.14 diff --git a/packages/controller/index.ts b/packages/controller/index.ts index 5218034f..caebb912 100644 --- a/packages/controller/index.ts +++ b/packages/controller/index.ts @@ -1,7 +1,9 @@ import { bootstrap } from "./controller"; export { default as Controller } from "./src/Controller"; export { default as ControllerUser } from "./src/ControllerUser"; +export { default as ControllerRouter } from "./src/ControllerRouter"; export { default as ControlConnection } from "./src/ControlConnection"; +export { default as HostConnection } from "./src/HostConnection"; export { default as InstanceInfo } from "./src/InstanceInfo"; export { default as BaseControllerPlugin } from "./src/BaseControllerPlugin"; export { default as UserManager } from "./src/UserManager"; diff --git a/packages/controller/src/BaseConnection.ts b/packages/controller/src/BaseConnection.ts index 493ebbde..3ebac51e 100644 --- a/packages/controller/src/BaseConnection.ts +++ b/packages/controller/src/BaseConnection.ts @@ -6,7 +6,6 @@ import type WsServerConnector from "./WsServerConnector"; import * as lib from "@clusterio/lib"; const { logger } = lib; -import ControllerRouter from "./ControllerRouter"; import * as routes from "./routes"; @@ -24,7 +23,7 @@ export default class BaseConnection extends lib.Link { public _controller: Controller ) { super(connector); - this.router = new ControllerRouter(this._controller); + this.router = _controller.router; for (let [Request, handler] of this._controller._registeredRequests) { this.handle(Request, handler); } for (let [Request, handler] of this._controller._fallbackedRequests) { this.fallbackRequest(Request, handler); } for (let [Event, handler] of this._controller._registeredEvents) { this.handle(Event, handler); } diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 8930f854..3e81c0c1 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -26,6 +26,7 @@ import WsServer from "./WsServer"; import HostConnection from "./HostConnection"; import HostInfo from "./HostInfo"; import BaseControllerPlugin from "./BaseControllerPlugin"; +import ControllerRouter from "./ControllerRouter"; const endpointDurationSummary = new Summary( "clusterio_controller_http_endpoint_duration_seconds", @@ -74,6 +75,7 @@ export default class Controller { /** WebSocket server */ wsServer: WsServer; + router = new ControllerRouter(this); trustedProxies: BlockList; debugEvents: events.EventEmitter = new events.EventEmitter(); private _events: events.EventEmitter = new events.EventEmitter(); @@ -1374,7 +1376,11 @@ export default class Controller { } } else if (dst.id === lib.Address.instance || dst.id === lib.Address.host) { + const plugin = event.constructor.plugin; for (let hostConnection of this.wsServer.hostConnections.values()) { + if (plugin && !hostConnection.plugins.has(plugin)) { + continue; + } hostConnection.sendEvent(event, dst); } diff --git a/packages/controller/src/ControllerRouter.ts b/packages/controller/src/ControllerRouter.ts index 84420595..4679312d 100644 --- a/packages/controller/src/ControllerRouter.ts +++ b/packages/controller/src/ControllerRouter.ts @@ -7,8 +7,6 @@ import ControlConnection from "./ControlConnection"; const { logger } = lib; export default class ControllerRouter { - pendingRequests: Map = new Map(); - constructor( public controller: Controller ) {} @@ -16,12 +14,14 @@ export default class ControllerRouter { /** * @param origin - Source link of the message. * @param message - Link the message originated from. + * @param entry - message entry if message is a request or an event. * @param hasFallback - true if fallback handling is available. * @returns true if the message was handled */ forwardMessage( origin: ControlConnection | HostConnection, message: lib.MessageRoutable, + entry: lib.RequestEntry | lib.EventEntry | undefined, hasFallback: boolean ): boolean { if (!["request", "response", "responseError", "event"].includes(message.type)) { @@ -32,7 +32,7 @@ export default class ControllerRouter { let nextHop: ControlConnection | HostConnection | undefined; let msg: string = ""; if (dst.type === lib.Address.broadcast) { - this.broadcastMessage(origin, message); + this.broadcastMessage(origin, message, entry as lib.EventEntry); return true; } else if (dst.type === lib.Address.host) { nextHop = this.controller.wsServer.hostConnections.get(dst.id); @@ -96,13 +96,17 @@ export default class ControllerRouter { broadcastMessage( origin: lib.Link, message: lib.MessageRoutable, + entry: lib.EventEntry, ) { let dst = message.dst; if (message.type !== "event") { this.warnUnrouted(message, `Unexpected broadcast of ${message.type}`); - } else if (dst.id === lib.Address.host || dst.id === lib.Address.instance) { + return; + } + const plugin = entry.Event.plugin; + if (dst.id === lib.Address.host || dst.id === lib.Address.instance) { for (let hostConnection of this.controller.wsServer.hostConnections.values()) { - if (hostConnection !== origin) { + if (hostConnection !== origin && (!plugin || hostConnection.plugins.has(plugin))) { hostConnection.connector.send(message); } } @@ -126,5 +130,3 @@ export default class ControllerRouter { logger.warn(msg ? `${baseMsg}: ${msg}.` : `${baseMsg}.`); } } - -module.exports = ControllerRouter; diff --git a/packages/host/index.ts b/packages/host/index.ts index 2b01bc21..f39ce536 100644 --- a/packages/host/index.ts +++ b/packages/host/index.ts @@ -1,6 +1,9 @@ import { bootstrap } from "./host"; export { default as BaseHostPlugin } from "./src/BaseHostPlugin"; export { default as BaseInstancePlugin } from "./src/BaseInstancePlugin"; +export { default as Host, HostRouter } from "./src/Host"; +export { default as Instance } from "./src/Instance"; +export { default as InstanceConnection } from "./src/InstanceConnection"; if (module === require.main) { bootstrap(); diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 06a23809..b1b9dd41 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -82,7 +82,7 @@ const instanceStartingMessages = new Set([ lib.InstanceExportDataRequest.name, ]); -class HostRouter { +export class HostRouter { constructor( public host: Host ) { } @@ -93,16 +93,22 @@ class HostRouter { * @param origin - * Link the message originated from. * @param message - Message to process. + * @param entry - message entry if message is a request or an event. * @param hasFallback - is fallback available * @returns true if the message was handled, false if fallback * is requested */ - forwardMessage(origin: lib.Link, message: lib.MessageRoutable, hasFallback: boolean) { + forwardMessage( + origin: lib.Link, + message: lib.MessageRoutable, + entry: lib.RequestEntry | lib.EventEntry | undefined, + hasFallback: boolean, + ) { let dst = message.dst; let nextHop: lib.Link | undefined; let msg: string | undefined; if (dst.type === lib.Address.broadcast) { - this.broadcastMessage(origin, message); + this.broadcastMessage(origin, message, entry as lib.EventEntry); return true; } else if ( dst.type === lib.Address.controller @@ -153,27 +159,36 @@ class HostRouter { return true; } - broadcastMessage(origin: lib.Link, message: lib.MessageRoutable) { + broadcastMessage( + origin: lib.Link, + message: lib.MessageRoutable, + entry: lib.EventEntry, + ) { let dst = message.dst; if (message.type !== "event") { this.warnUnrouted(message, `Unexpected broadcast of ${message.type}`); - } else if (dst.id === lib.Address.host || dst.id === lib.Address.instance) { + return; + } + const plugin = entry.Event.plugin; + if (dst.id === lib.Address.instance) { for (let instanceConnection of this.host.instanceConnections.values()) { - if (instanceConnection !== origin) { + if (instanceConnection !== origin && (!plugin || instanceConnection.plugins.has(plugin))) { instanceConnection.connector.send(message); } } - if (this.host !== origin) { + if (this.host !== origin && (!plugin || this.host.serverPlugins.has(plugin))) { this.host.connector.send(message); } } else if (dst.id === lib.Address.control) { if (this.host !== origin) { - this.host.connector.send(message); + if (!plugin || this.host.serverPlugins.has(plugin)) { + this.host.connector.send(message); + } } else { - logger.warn(`Received control broadcast of ${(message as lib.MessageEvent).name} from master.`); + logger.warn(`Received control broadcast of ${(message as lib.MessageEvent).name} from controller.`); } } else { - this.warnUnrouted(message, `Unexpected broacdast target ${dst.id}`); + this.warnUnrouted(message, `Unexpected broadcast target ${dst.id}`); } } diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts index 14b7d1ec..520e5fe4 100644 --- a/packages/lib/src/link/link.ts +++ b/packages/lib/src/link/link.ts @@ -72,7 +72,12 @@ export type RequestHandler = (request: Req, src: libData.Address, dst: export type EventHandler = (event: T, src: libData.Address, dst: libData.Address) => Promise; interface Router { - forwardMessage(origin: Link, message: MessageRoutable, fallback: boolean): boolean, + forwardMessage( + origin: Link, + message: MessageRoutable, + entry: RequestEntry | EventEntry | undefined, + fallback: boolean, + ): boolean, } // Some definitions for the terminology used here: @@ -343,7 +348,7 @@ export class Link { let fallback = message.type === "request" ? this._requestFallbacks.get((entry as RequestEntry).Request) : undefined ; - if (this.router.forwardMessage(this, message, Boolean(fallback))) { + if (this.router.forwardMessage(this, message, entry, Boolean(fallback))) { return; } if (!fallback) { diff --git a/test/integration/routing.js b/test/integration/routing.js new file mode 100644 index 00000000..ed31de35 --- /dev/null +++ b/test/integration/routing.js @@ -0,0 +1,416 @@ +"use strict"; +const assert = require("assert").strict; + +const lib = require("@clusterio/lib"); +const { Host, Instance, InstanceConnection } = require("@clusterio/host"); +const { ControlConnection, Controller, HostConnection, InstanceInfo } = require("@clusterio/controller"); + +const addr = lib.Address.fromShorthand; + +class Control extends lib.Link { } + +/** + * @param {Controller} controller + * @param {number} hostId + */ +function connectHost(controller, hostId, plugins) { + const [controllerSide, hostSide] = lib.VirtualConnector.makePair(addr("controller"), addr({ hostId })); + const registerData = new lib.RegisterHostData("", "0.0.0", hostId, plugins); + const hostConnection = new HostConnection(registerData, controllerSide, controller, "host-a.test"); + controller.wsServer.hostConnections.set(hostId, hostConnection); + const hostConfig = new lib.HostConfig("host"); + hostConfig.set("host.id", hostId); + return new Host(hostSide, "invalid", hostConfig, undefined, []); +} + +/** + * @param {Controller} controller + * @param {Host} host + * @param {number} instanceId + */ +function connectInstance(controller, host, instanceId) { + const hostAddress = addr({ hostId: host.config.get("host.id")}); + const instanceAddress = addr({ instanceId }); + const [instanceSide, hostSide] = lib.VirtualConnector.makePair(instanceAddress, hostAddress); + const instanceConfig = new lib.InstanceConfig("host"); + instanceConfig.set("instance.id", instanceId); + instanceConfig.set("instance.assigned_host", host.config.get("host.id")); + host.instanceInfos.set(instanceId, { path: "invalid", config: instanceConfig }); + controller.instances.set(instanceId, new InstanceInfo(instanceConfig, "stopped")); + const instance = new Instance(host, instanceSide, "invalid", "invalid", instanceConfig); + const instanceConnection = new InstanceConnection(hostSide, host, instance); + host.instanceConnections.set(instanceId, instanceConnection); + return instance; +} + +/** + * @param {Controller} controller + * @param {number} hostId + */ +function connectControl(controller, controlId) { + const [controllerSide, controlSide] = lib.VirtualConnector.makePair(addr("controller"), addr({ controlId })); + const registerData = new lib.RegisterControlData("", "0.0.0"); + let user = controller.userManager.users.get("test"); + if (!user) { + user = controller.userManager.createUser("test"); + } + const controlConnection = new ControlConnection(registerData, controllerSide, controller, user, controlId); + controller.wsServer.controlConnections.set(controlId, controlConnection); + return new Control(controlSide); +} + +class TestRequest { + static type = "request"; + static src = ["controller", "host", "instance", "control"]; + static dst = ["controller", "host", "instance", "control"]; + static permission = null; +} + +class TestEvent { + static type = "event"; + static src = ["controller", "host", "instance", "control"]; + static dst = ["controller", "host", "instance", "control"]; + static permission = null; +} + +class PluginEvent { + static type = "event"; + static src = ["controller", "host", "instance", "control"]; + static dst = ["controller", "host", "instance", "control"]; + static permission = null; + static plugin = "test"; +} + +lib.Link.register(TestRequest); +lib.Link.register(TestEvent); +lib.Link.register(PluginEvent); + +describe("Integration of link routing", function() { + /** @type {Controller} */ + let controller; + /** @type {Host} */ + let hostA; + /** @type {Host} */ + let hostB; + /** @type {Instance} */ + let instanceA1; + /** @type {Instance} */ + let instanceA2; + /** @type {Instance} */ + let instanceB1; + /** @type {Control} */ + let controlA; + /** @type {Control} */ + let controlB; + + beforeEach(function() { + controller = new Controller( + {}, + [], + "invalid", + new lib.ControllerConfig("controller") + ); + hostA = connectHost(controller, 10, []); + hostB = connectHost(controller, 20, []); + instanceA1 = connectInstance(controller, hostA, 11); + instanceA2 = connectInstance(controller, hostA, 12); + instanceB1 = connectInstance(controller, hostB, 21); + controlA = connectControl(controller, 100); + controlB = connectControl(controller, 101); + }); + + /** @returns {Controller | Host | Instance} */ + function get(name) { + return { + controller, + hostA, + hostB, + instanceA1, + instanceA2, + instanceB1, + controlA, + controlB, + }[name]; + } + + // Combinations to try. Equivalent paths have been removed. + const parties = [ + ["controller", ["hostA", "instanceA1", "controlA"]], + ["hostA", ["controller", "hostB", "instanceA1", "instanceB1", "controlA"]], + ["instanceA1", ["controller", "hostA", "hostB", "instanceA2", "instanceB1", "controlA"]], + ["controlA", ["controller", "hostA", "instanceA1", "controlB"]], + ]; + + function handle(dst, Message, handler) { + if (dst instanceof Controller) { + for (const host of dst.wsServer.hostConnections.values()) { + host.handle(Message, handler); + } + for (const control of dst.wsServer.controlConnections.values()) { + control.handle(Message, handler); + } + } else if (dst instanceof Host) { + for (const instance of dst.instanceConnections.values()) { + instance.handle(Message, handler); + } + dst.handle(Message, handler); + } else { + dst.handle(Message, handler); + } + } + + /** + * @param {Controller | Host | Instance} link + * @param {Address} dst + */ + function send(link, dst, message) { + if (link instanceof Host) { + const instance = link.instanceConnections.get(dst.id); + if (instance) { + return instance.sendTo(dst, message); + } + } + return link.sendTo(dst, message); + } + + describe("request routing", function() { + for (const [srcName, dstNames] of parties) { + it(`should route from ${srcName}`, async function() { + for (const dstName of dstNames) { + let called; + const dst = get(dstName); + handle(dst, TestRequest, async () => { called = true; }); + const dstAddr = dst instanceof Controller ? addr("controller") : dst.connector.src; + await send(get(srcName), dstAddr, new TestRequest()); + assert(called, `${dstName} handler was not called`); + } + }); + } + }); + + describe("event routing", function() { + for (const [srcName, dstNames] of parties) { + it(`should route from ${srcName}`, async function() { + for (const dstName of dstNames) { + let called; + const dst = get(dstName); + handle(dst, TestEvent, async () => { called = true; }); + const dstAddr = dst instanceof Controller ? addr("controller") : dst.connector.src; + send(get(srcName), dstAddr, new TestEvent()); + assert(called, `${dstName} handler was not called`); + } + }); + } + }); + + /** + * @param {Controller | Host | Instance | Control} link + * @param {"allHosts" | "allInstances" | "allControls"} dst + */ + function bcast(link, dst, message) { + if (link instanceof Host) { + const plugin = message.constructor.plugin; + if (dst === "allInstances") { + for (const instance of link.instanceConnections.values()) { + if (plugin && !instance.plugins.has(plugin)) { + continue; + } + instance.sendTo(dst, message); + } + } + if (plugin && !link.serverPlugins.has(plugin)) { + return; + } + } + link.sendTo(dst, message); + } + + function bhandle(srcName, dst, Event, canReach = (dstB) => true) { + const asserts = []; + const dsts = { + allHosts: ["hostA", "hostB"], + allInstances: ["instanceA1", "instanceA2", "instanceB1"], + allControls: ["controlA", "controlB"], + }; + for (const dstName of dsts[dst]) { + if (srcName === dstName) { continue; } + // TODO remove when broadcasting from instance to hosts is fixed, see #575 + if (srcName === "instanceA1" && dstName === "hostB") { continue; } + let called; + handle(get(dstName), Event, async () => { called = true; }); + const should = canReach(dstName); + asserts.push(() => assert( + !should ^ called, + `${dstName} handler was ${should ? "not " : ""}called from ${srcName}`, + )); + } + return asserts; + } + + describe("event broadcast", function() { + for (const srcName of ["controller", "hostA", "instanceA1", "controlA"]) { + it(`should broadcast from ${srcName}`, async function() { + for (const dstName of ["allHosts", "allInstances", "allControls"]) { + const dstAsserts = bhandle(srcName, dstName, TestEvent); + bcast(get(srcName), dstName, new TestEvent()); + for (const dstAssert of dstAsserts) { + dstAssert(); + } + } + }); + } + }); + + const graph = { + controller: ["hostA", "hostB", "controlA", "controlB"], + hostA: ["controller", "instanceA1", "instanceA2"], + hostB: ["controller", "instanceB1"], + instanceA1: ["hostA"], + instanceA2: ["hostA"], + instanceB1: ["hostB"], + controlA: ["controller"], + controlB: ["controller"], + }; + + // Returns true if the path from src to dst goes through via + function goesVia(src, dst, via) { + // Places we've been to + const checked = new Set(); + // Places left to check + const paths = new Set([src]); + while (paths.size) { + const [candidate] = paths; + paths.delete(candidate); + if (candidate === via) { + continue; + } + if (checked.has(candidate)) { + continue; + } + if (candidate === dst) { + return false; + } + checked.add(candidate); + for (const newCandidate of graph[candidate]) { + paths.add(newCandidate); + } + } + return true; + } + assert(goesVia("instanceA1", "instanceB1", "controller")); + assert(!goesVia("instanceA1", "instanceA2", "controller")); + assert(goesVia("instanceA1", "hostB", "hostA")); + assert(!goesVia("instanceA1", "hostB", "controlA")); + assert(goesVia("controlA", "controlB", "controller")); + assert(goesVia("controlA", "hostA", "controller")); + + function addPluginExceptFor(exName) { + const ex = get(exName); + const exAddr = ex instanceof Controller ? addr("controller") : ex.connector.src; + for (const name of [ + "controller", "hostA", "hostB", "instanceA1", "instanceA2", "instanceB1", "controlA", "controlB", + ]) { + const party = get(name); + if (party instanceof Controller) { + for (const host of party.wsServer.hostConnections.values()) { + if (!host.connector.dst.equals(exAddr)) { + host.plugins.set("test", "0.0.0"); + } + } + for (const control of party.wsServer.controlConnections.values()) { + if (!control.connector.dst.equals(exAddr)) { + // No way to set plugins + } + } + } else if (party instanceof Host) { + for (const instance of party.instanceConnections.values()) { + if (!instance.connector.dst.equals(exAddr)) { + instance.plugins.set("test", "0.0.0"); + } + } + if (!party.connector.dst.equals(exAddr)) { + party.serverPlugins.set("test", "0.0.0"); + } + } else if (party instanceof Instance) { + // Currently does not handle plugins + } else if (party instanceof Control) { + // Currently does not handle plugins + } + } + } + + // When plugin events are broadcast the party sending the event should + // filter it out if the next hop it is forwarded to does not have plugin + // the event was defined in loaded. + describe("event broadcast with controller missing plugin", function() { + for (const srcName of ["hostA", "instanceA1", "controlA"]) { + it(`should broadcast from ${srcName}`, async function() { + if (srcName === "controlA") { + this.skip(); // Not implemented + } + addPluginExceptFor("controller"); + for (const dstName of ["allHosts", "allInstances", "allControls"]) { + const dstAsserts = bhandle( + srcName, dstName, PluginEvent, dstB => !goesVia(srcName, dstB, "controller"), + ); + bcast(get(srcName), dstName, new PluginEvent()); + for (const dstAssert of dstAsserts) { + dstAssert(); + } + } + }); + } + }); + + describe("event broadcast with host missing plugin", function() { + for (const srcName of ["controller", "hostA", "instanceA1", "controlA"]) { + it(`should broadcast from ${srcName}`, async function() { + addPluginExceptFor("hostB"); + for (const dstName of ["allHosts", "allInstances", "allControls"]) { + const dstAsserts = bhandle( + srcName, dstName, PluginEvent, dstB => !goesVia(srcName, dstB, "hostB"), + ); + bcast(get(srcName), dstName, new PluginEvent()); + for (const dstAssert of dstAsserts) { + dstAssert(); + } + } + }); + } + }); + + describe("event broadcast with instance missing plugin", function() { + for (const srcName of ["controller", "hostA", "instanceA1", "controlA"]) { + it(`should broadcast from ${srcName}`, async function() { + addPluginExceptFor("instanceA2"); + for (const dstName of ["allHosts", "allInstances", "allControls"]) { + const dstAsserts = bhandle( + srcName, dstName, PluginEvent, dstB => !goesVia(srcName, dstB, "instanceA2"), + ); + bcast(get(srcName), dstName, new PluginEvent()); + for (const dstAssert of dstAsserts) { + dstAssert(); + } + } + }); + } + }); + + describe("event broadcast with control missing plugin", function() { + for (const srcName of ["controller", "hostA", "instanceA1", "controlA"]) { + it(`should broadcast from ${srcName}`, async function() { + this.skip(); // Currently not implemented. + addPluginExceptFor("controlB"); + for (const dstName of ["allHosts", "allInstances", "allControls"]) { + const dstAsserts = bhandle( + srcName, dstName, PluginEvent, dstB => !goesVia(srcName, dstB, "controlB"), + ); + bcast(get(srcName), dstName, new PluginEvent()); + for (const dstAssert of dstAsserts) { + dstAssert(); + } + } + }); + } + }); +}); From 2a57ddb02e938d63a89c388428956a11f1b12663 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 24 Jan 2024 03:08:24 +0100 Subject: [PATCH 10/68] Add note about send and bcast test functions --- test/integration/routing.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/routing.js b/test/integration/routing.js index ed31de35..6881f079 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -164,6 +164,9 @@ describe("Integration of link routing", function() { * @param {Address} dst */ function send(link, dst, message) { + // This logic exists becasue .sendTo doesn't forward all kinds of + // messages to the correct location from all places. Idealy this + // would not be neccessary and .sendTo would do the right thing. if (link instanceof Host) { const instance = link.instanceConnections.get(dst.id); if (instance) { @@ -208,6 +211,9 @@ describe("Integration of link routing", function() { * @param {"allHosts" | "allInstances" | "allControls"} dst */ function bcast(link, dst, message) { + // This logic exists becasue .sendTo doesn't forward all kinds of + // messages to the correct location from all places. Idealy this + // would not be neccessary and .sendTo would do the right thing. if (link instanceof Host) { const plugin = message.constructor.plugin; if (dst === "allInstances") { From 38dcff222570d5f21524a95bc514882995ecc0d7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 22 Jan 2024 01:23:40 +0100 Subject: [PATCH 11/68] Use h: as short form of host in log lines Before the rename from hosts were called slave which is why s: made sense. Use h: instead. --- packages/lib/src/logging.ts | 2 +- packages/lib/src/logging_utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lib/src/logging.ts b/packages/lib/src/logging.ts index c673f98e..d9ce0c56 100644 --- a/packages/lib/src/logging.ts +++ b/packages/lib/src/logging.ts @@ -73,7 +73,7 @@ export class WebConsoleFormat { transform(info: any, _options: object) { let src = " "; if (info.host_id !== undefined) { - src += `s:${info.host_name} - `; + src += `h:${info.host_name} - `; } if (info.instance_id !== undefined) { src += `i:${info.instance_name}`; diff --git a/packages/lib/src/logging_utils.ts b/packages/lib/src/logging_utils.ts index 6471c3f7..b3b34d35 100644 --- a/packages/lib/src/logging_utils.ts +++ b/packages/lib/src/logging_utils.ts @@ -163,7 +163,7 @@ export class TerminalFormat { src += `i:${info.instance_name} `; } else if (info.host_name) { - src += `s:${info.host_name} `; + src += `h:${info.host_name} `; } if (info.plugin) { src += `${info.plugin}: `; From e3cfc090d37c6bde174df1f837510c7bba7a7c68 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Jan 2024 16:36:12 +0100 Subject: [PATCH 12/68] Fix string format of Address When host and instance id in Address was reordered in 1e66f11 the toString method Address was updated to account for this. Fix toString showing host addresses as being instance addresses and vice versa. --- packages/lib/src/data/messages_core.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/lib/src/data/messages_core.ts b/packages/lib/src/data/messages_core.ts index 67ea64a5..b4b3f91e 100644 --- a/packages/lib/src/data/messages_core.ts +++ b/packages/lib/src/data/messages_core.ts @@ -87,7 +87,7 @@ export class Address { } toString() { - const typeMap = ["controller", "instance", "host", "control"]; + const typeMap = ["controller", "host", "instance", "control"]; const type = typeMap[this.type] || this.type; if (this.type === Address.broadcast) { return `[Address ${type}:${typeMap[this.id] || this.id}]`; From 499e973db918e8e8cf32efeb46e55671a51d0b20 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 24 Jan 2024 02:27:14 +0100 Subject: [PATCH 13/68] Fix InstanceConfigSetPropRequest schema Fix the value incorrectly being set as required in the schema and thus preventing props on config field objects of the instance from being removed. --- CHANGELOG.md | 1 + packages/lib/src/config/classes.ts | 2 +- packages/lib/src/data/messages_instance.ts | 2 +- test/integration/clusterio.js | 9 +++++++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0264def0..6ece0937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Fixed host hanging if stopped while an instance is in the process of creating a save during startup. - Fixed `instance.auto_start` option. - Fixed host log spammed with errors when plugins broadcast messages to instances with the plugin disabled. +- Fixed properties of object config fields not being possible to remove. ## Version 2.0.0-alpha.14 diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index 3e26b2c8..dae4fc8a 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -459,7 +459,7 @@ export class Config< setProp( name: Field, prop: string, - value: unknown, + value?: unknown, location = this.location, ) { let def = this.constructor.fieldDefinitions[name]; diff --git a/packages/lib/src/data/messages_instance.ts b/packages/lib/src/data/messages_instance.ts index ff29d886..f71b2a13 100644 --- a/packages/lib/src/data/messages_instance.ts +++ b/packages/lib/src/data/messages_instance.ts @@ -208,7 +208,7 @@ export class InstanceConfigSetPropRequest { "instanceId": Type.Integer(), "field": Type.String(), "prop": Type.String(), - "value": Type.Unknown(), + "value": Type.Optional(Type.Unknown()), }); static fromJSON(json: Static) { diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index d34b8a0d..4fe1f626 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -557,6 +557,15 @@ describe("Integration of Clusterio", function() { slowTest(this); await checkInstanceStatus(44, "running"); }); + it("should allow creating and removing props", async function() { + slowTest(this); + await execCtl("instance config set-prop test factorio.settings new_property new-string-value"); + let config = await getControl().send(new lib.InstanceConfigGetRequest(44)); + assert.equal(config["factorio.settings"]["new_property"], "new-string-value"); + await execCtl("instance config set-prop test factorio.settings new_property"); + config = await getControl().send(new lib.InstanceConfigGetRequest(44)); + assert.equal(config["factorio.settings"]["new_property"], undefined); + }); }); describe("user set-admin/whitelisted/banned", function() { From 6732397061d8b16519eaba67eacf8fc7b091d469 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 24 Jan 2024 04:35:24 +0100 Subject: [PATCH 14/68] Fix Web UI not clearing config props (#578) --- packages/web_ui/src/components/ControllerConfigTree.tsx | 2 ++ packages/web_ui/src/components/HostConfigTree.tsx | 2 ++ packages/web_ui/src/components/InstanceConfigTree.tsx | 2 ++ 3 files changed, 6 insertions(+) diff --git a/packages/web_ui/src/components/ControllerConfigTree.tsx b/packages/web_ui/src/components/ControllerConfigTree.tsx index 71acc44e..a369fab7 100644 --- a/packages/web_ui/src/components/ControllerConfigTree.tsx +++ b/packages/web_ui/src/components/ControllerConfigTree.tsx @@ -27,6 +27,8 @@ export default function ControllerConfigTree() { } catch (err) { return; } + } else { + value = undefined; } await control.send(new lib.ControllerConfigSetPropRequest(field, prop, value)); } diff --git a/packages/web_ui/src/components/HostConfigTree.tsx b/packages/web_ui/src/components/HostConfigTree.tsx index 6f075ac3..aef520f0 100644 --- a/packages/web_ui/src/components/HostConfigTree.tsx +++ b/packages/web_ui/src/components/HostConfigTree.tsx @@ -24,6 +24,8 @@ export default function HostConfigTree(props: { id: number, available: boolean } } catch (err) { return; } + } else { + value = undefined; } await control.sendTo({ hostId: props.id }, new lib.HostConfigSetPropRequest(field, prop, value)); } diff --git a/packages/web_ui/src/components/InstanceConfigTree.tsx b/packages/web_ui/src/components/InstanceConfigTree.tsx index 51b0d4d2..c9a45530 100644 --- a/packages/web_ui/src/components/InstanceConfigTree.tsx +++ b/packages/web_ui/src/components/InstanceConfigTree.tsx @@ -28,6 +28,8 @@ export default function InstanceConfigTree(props: InstanceConfigTreeProps) { } catch (err) { return; } + } else { + value = undefined; } await control.send(new lib.InstanceConfigSetPropRequest(props.id, field, prop, value)); } From 7d6eac65582497f30365ba02180149c4d3581930 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 24 Jan 2024 04:40:21 +0100 Subject: [PATCH 15/68] v2.0.0-alpha.15 --- packages/controller/package.json | 6 +++--- packages/create/package.json | 2 +- packages/ctl/package.json | 4 ++-- packages/host/package.json | 4 ++-- packages/lib/package.json | 2 +- packages/web_ui/package.json | 4 ++-- plugins/global_chat/package.json | 6 +++--- plugins/inventory_sync/package.json | 6 +++--- plugins/player_auth/package.json | 6 +++--- plugins/research_sync/package.json | 6 +++--- plugins/statistics_exporter/package.json | 6 +++--- plugins/subspace_storage/package.json | 6 +++--- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/controller/package.json b/packages/controller/package.json index 6f967415..87cff858 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio controller", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "main": "dist/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", "@sinclair/typebox": "^0.30.4", "busboy": "^1.6.0", "compression": "^1.7.4", @@ -35,7 +35,7 @@ "yargs": "^17.7.2" }, "devDependencies": { - "@clusterio/web_ui": "^2.0.0-alpha.14", + "@clusterio/web_ui": "^2.0.0-alpha.15", "@types/busboy": "^1.5.0", "@types/compression": "^1.7.2", "@types/express": "^4.17.17", diff --git a/packages/create/package.json b/packages/create/package.json index 6f320a27..5cd7688d 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -3,7 +3,7 @@ "description": "Installer for clusterio", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "main": "create.js", "bin": { "clusteriocreate": "./create.js" diff --git a/packages/ctl/package.json b/packages/ctl/package.json index 18c95a2e..c4414789 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -3,7 +3,7 @@ "description": "Clusterio command line management tool", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "main": "dist/index.js", "scripts": { "prepare": "tsc --build", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", "as-table": "^1.0.55", "fs-extra": "^11.1.1", "phin": "^3.7.0", diff --git a/packages/host/package.json b/packages/host/package.json index 54e963e2..2f4a4297 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio host server", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "main": "dist/index.js", "scripts": { "prepare": "tsc --build", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", "@sinclair/typebox": "^0.30.4", "fs-extra": "^11.1.1", "jimp": "^0.22.8", diff --git a/packages/lib/package.json b/packages/lib/package.json index 6cd1aebc..1b305045 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -1,7 +1,7 @@ { "name": "@clusterio/lib", "description": "Shared library for Clusterio", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "repository": "https://github.com/clusterio/clusterio", "license": "MIT", "scripts": { diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 6608a14c..93ac9b45 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -3,7 +3,7 @@ "description": "Clusterio web interface implementation", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "browser": "src/index.ts", "main": "dist/src/index.js", "scripts": { @@ -19,7 +19,7 @@ "@ant-design/icons": "^5.1.4", "@babel/core": "^7.22.9", "@babel/preset-react": "^7.22.5", - "@clusterio/lib": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", "antd": "^5.13.0", "assert": "^2.0.0", "babel-loader": "^9.1.3", diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json index b438bec3..6e1ffcf0 100644 --- a/plugins/global_chat/package.json +++ b/plugins/global_chat/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-global_chat", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -21,8 +21,8 @@ "@clusterio/lib": "^2.0.0-alpha.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", - "@clusterio/web_ui": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/web_ui": "^2.0.0-alpha.15", "@types/node": "^20.4.5", "@types/react": "^18.2.21", "typescript": "^5.1.6", diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index 5664be56..f5530566 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-inventory_sync", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Danielv ", "homepage": "https://github.com/clusterio/factorioClusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", - "@clusterio/web_ui": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/web_ui": "^2.0.0-alpha.15", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index a7f78e0a..3a9bd9d5 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-player_auth", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "description": "Clusterio plugin authenticating logged in players to the web interface", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -26,8 +26,8 @@ "jsonwebtoken": "^9.0.1" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", - "@clusterio/web_ui": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/web_ui": "^2.0.0-alpha.15", "@types/express": "^4.17.17", "@types/jsonwebtoken": "^9.0.2", "@types/node": "^20.4.5", diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json index 19ab6b70..c2184c13 100644 --- a/plugins/research_sync/package.json +++ b/plugins/research_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-research_sync", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "description": "Clusterio plugin for synchronizing research between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", - "@clusterio/web_ui": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/web_ui": "^2.0.0-alpha.15", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json index a98facb8..a0d8fed8 100644 --- a/plugins/statistics_exporter/package.json +++ b/plugins/statistics_exporter/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-statistics_exporter", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "description": "Clusterio plugin for gathering statistics from Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -21,8 +21,8 @@ "@clusterio/lib": "^2.0.0-alpha.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", - "@clusterio/web_ui": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/web_ui": "^2.0.0-alpha.15", "@types/node": "^20.4.5", "@types/react": "^18.2.21", "typescript": "^5.1.6", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index 7cd98cf9..e76630bd 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-subspace_storage", - "version": "2.0.0-alpha.14", + "version": "2.0.0-alpha.15", "description": "Clusterio plugin for sharing storage between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.14", - "@clusterio/web_ui": "^2.0.0-alpha.14", + "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/web_ui": "^2.0.0-alpha.15", "@types/express": "^4.17.17", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", From f5255d6f765aee79e00a4eac10ade8fd998163db Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 26 Jan 2024 13:02:26 +0100 Subject: [PATCH 16/68] Fix message seq reuse on forwarding (#579) When messages were forwarded they were sent using the low level connector .send method which accepts a message and sends it over the link. This method sends the message as is with no modification which means that at no point during the forwarding logic was the message sequence number that originated from Link A renumbered to the correct sequence number to send out on Link B which it was forwarded on. This in turn caused the logic which drops acked messages from the send buffer to break which in turn meant the resume could cause duplication of message and/or messages to be lost. With enough duplicated messages the resume of one connection could cause other connections to drop and also initiate resume. Fix by renumbering the sequence on forwarded messages. In the future the sequences sent and received should also be checked to make sure this kind of problem does not happen again, this is not done here because partially updating a running cluster with sequence problems to one that checks and enforces sequences numbers will be problematic. --- CHANGELOG.md | 7 +++++ packages/controller/src/ControllerRouter.ts | 6 ++-- packages/host/src/Host.ts | 10 +++--- packages/lib/src/link/connectors.ts | 34 +++++++++++++++++++-- packages/lib/src/link/link.ts | 5 +-- test/mock.js | 4 +++ 6 files changed, 54 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ece0937..7e03d59a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Version 2.0.0-alpha.16 + +## Changes + +- Fixed message sequences not being renumbered when forwarded between links causing reconnects to send duplicated messages. + + ## Version 2.0.0-alpha.15 ### Features diff --git a/packages/controller/src/ControllerRouter.ts b/packages/controller/src/ControllerRouter.ts index 4679312d..4072c941 100644 --- a/packages/controller/src/ControllerRouter.ts +++ b/packages/controller/src/ControllerRouter.ts @@ -84,7 +84,7 @@ export default class ControllerRouter { if (message.type === "request") { nextHop.forwardRequest(message as lib.MessageRequest, origin); } else { - nextHop.connector.send(message); + nextHop.connector.forward(message); } } else { this.warnUnrouted(message, msg); @@ -107,13 +107,13 @@ export default class ControllerRouter { if (dst.id === lib.Address.host || dst.id === lib.Address.instance) { for (let hostConnection of this.controller.wsServer.hostConnections.values()) { if (hostConnection !== origin && (!plugin || hostConnection.plugins.has(plugin))) { - hostConnection.connector.send(message); + hostConnection.connector.forward(message); } } } else if (dst.id === lib.Address.control) { for (let controlConnection of this.controller.wsServer.controlConnections.values()) { if (controlConnection !== origin) { - controlConnection.connector.send(message); + controlConnection.connector.forward(message); } } } else { diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index b1b9dd41..c2fc5d67 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -173,16 +173,16 @@ export class HostRouter { if (dst.id === lib.Address.instance) { for (let instanceConnection of this.host.instanceConnections.values()) { if (instanceConnection !== origin && (!plugin || instanceConnection.plugins.has(plugin))) { - instanceConnection.connector.send(message); + instanceConnection.connector.forward(message); } } if (this.host !== origin && (!plugin || this.host.serverPlugins.has(plugin))) { - this.host.connector.send(message); + this.host.connector.forward(message); } } else if (dst.id === lib.Address.control) { if (this.host !== origin) { if (!plugin || this.host.serverPlugins.has(plugin)) { - this.host.connector.send(message); + this.host.connector.forward(message); } } else { logger.warn(`Received control broadcast of ${(message as lib.MessageEvent).name} from controller.`); @@ -208,7 +208,7 @@ export class HostRouter { } this.host._connectInstance(dst.id).then(instanceConnection => { - instanceConnection.connector.send(message); + instanceConnection.connector.forward(message); }).catch(err => { logger.error(`Error starting instance:\n${err.stack}`, this.host.instanceLogMeta(dst.id)); origin.connector.sendResponseError( @@ -222,7 +222,7 @@ export class HostRouter { if (message.type === "request") { nextHop.forwardRequest(message as lib.MessageRequest, origin); } else { - nextHop.connector.send(message); + nextHop.connector.forward(message); } } catch (err: any) { if (message.type === "request") { diff --git a/packages/lib/src/link/connectors.ts b/packages/lib/src/link/connectors.ts index 2bc15f19..999800ba 100644 --- a/packages/lib/src/link/connectors.ts +++ b/packages/lib/src/link/connectors.ts @@ -33,7 +33,30 @@ export abstract class BaseConnector extends events.EventEmitter { this._seq = 1; } - abstract send(message: libData.Message): void; + protected abstract send(message: libData.MessageRoutable): void; + + forward(message: libData.MessageRoutable) { + const seq = this._seq; + this._seq += 1; + // The message type currently can't be inferred like a tagged union + // TODO fix typing of message so it can be deduced from .type + if (message.type === "request") { + const request = message as libData.MessageRequest; + this.send(new libData.MessageRequest(seq, request.src, request.dst, request.name, request.data)); + } else if (message.type === "response") { + const response = message as libData.MessageResponse; + this.send(new libData.MessageResponse(seq, response.src, response.dst, response.data)); + } else if (message.type === "responseError") { + const error = message as libData.MessageResponseError; + this.send(new libData.MessageResponseError(seq, error.src, error.dst, error.data)); + } else if (message.type === "event") { + const event = message as libData.MessageEvent; + this.send(new libData.MessageEvent(seq, event.src, event.dst, event.name, event.data)); + } else { + throw new Error(`Cannot forward message type ${(message as any).type}`); + } + return seq; + } sendRequest( request: Request, @@ -242,7 +265,7 @@ export abstract class WebSocketBaseConnector extends BaseConnector { * * @param message - Message to send. */ - send(message: libData.MessageRoutable) { + protected send(message: libData.MessageRoutable) { if (!["connected", "resuming"].includes(this._state)) { throw new libErrors.SessionLost("No session"); } @@ -295,6 +318,13 @@ export abstract class WebSocketBaseConnector extends BaseConnector { await this.close(1000, "Disconnect"); } + /** + * Notify other end that the link is ready to disconnect + */ + sendDisconnectReady() { + this._sendInternal(new libData.MessageDisconnect("ready")); + } + /** * True if the connection is established and active */ diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts index 520e5fe4..e3a746c0 100644 --- a/packages/lib/src/link/link.ts +++ b/packages/lib/src/link/link.ts @@ -132,7 +132,8 @@ export class Link { } ).finally( () => { - this.connector.send(new libData.MessageDisconnect("ready")); + // Only WebSocket connectors issue disconnection events. + (this.connector as WebSocketBaseConnector).sendDisconnectReady(); }, ); }); @@ -561,7 +562,7 @@ export class Link { dst: message.dst, }; this._forwardedRequests.set(message.src.index(), pending); - this.connector.send(message); + this.connector.forward(message); } sendEvent(event: Event, dst: libData.Address) { diff --git a/test/mock.js b/test/mock.js index a00729c9..171c5923 100644 --- a/test/mock.js +++ b/test/mock.js @@ -80,6 +80,10 @@ class MockConnector extends lib.BaseConnector { this.sentMessages.push(message); setImmediate(() => this.emit("send", message)); } + + sendDisconnectReady() { + this.send(new lib.MessageDisconnect("ready")); + } } class MockServer extends events.EventEmitter { From c2b6a6e1c09cc05ca1f7cce56d968d79b4ad1625 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 26 Jan 2024 13:04:07 +0100 Subject: [PATCH 17/68] v2.0.0-alpha.16 --- packages/controller/package.json | 6 +++--- packages/create/package.json | 2 +- packages/ctl/package.json | 4 ++-- packages/host/package.json | 4 ++-- packages/lib/package.json | 2 +- packages/web_ui/package.json | 4 ++-- plugins/global_chat/package.json | 6 +++--- plugins/inventory_sync/package.json | 6 +++--- plugins/player_auth/package.json | 6 +++--- plugins/research_sync/package.json | 6 +++--- plugins/statistics_exporter/package.json | 6 +++--- plugins/subspace_storage/package.json | 6 +++--- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/controller/package.json b/packages/controller/package.json index 87cff858..a3da2405 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio controller", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "main": "dist/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", "@sinclair/typebox": "^0.30.4", "busboy": "^1.6.0", "compression": "^1.7.4", @@ -35,7 +35,7 @@ "yargs": "^17.7.2" }, "devDependencies": { - "@clusterio/web_ui": "^2.0.0-alpha.15", + "@clusterio/web_ui": "^2.0.0-alpha.16", "@types/busboy": "^1.5.0", "@types/compression": "^1.7.2", "@types/express": "^4.17.17", diff --git a/packages/create/package.json b/packages/create/package.json index 5cd7688d..40aa21ba 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -3,7 +3,7 @@ "description": "Installer for clusterio", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "main": "create.js", "bin": { "clusteriocreate": "./create.js" diff --git a/packages/ctl/package.json b/packages/ctl/package.json index c4414789..8793f1bb 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -3,7 +3,7 @@ "description": "Clusterio command line management tool", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "main": "dist/index.js", "scripts": { "prepare": "tsc --build", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", "as-table": "^1.0.55", "fs-extra": "^11.1.1", "phin": "^3.7.0", diff --git a/packages/host/package.json b/packages/host/package.json index 2f4a4297..ce801590 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio host server", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "main": "dist/index.js", "scripts": { "prepare": "tsc --build", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", "@sinclair/typebox": "^0.30.4", "fs-extra": "^11.1.1", "jimp": "^0.22.8", diff --git a/packages/lib/package.json b/packages/lib/package.json index 1b305045..538f5d75 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -1,7 +1,7 @@ { "name": "@clusterio/lib", "description": "Shared library for Clusterio", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "repository": "https://github.com/clusterio/clusterio", "license": "MIT", "scripts": { diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 93ac9b45..796c3242 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -3,7 +3,7 @@ "description": "Clusterio web interface implementation", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "browser": "src/index.ts", "main": "dist/src/index.js", "scripts": { @@ -19,7 +19,7 @@ "@ant-design/icons": "^5.1.4", "@babel/core": "^7.22.9", "@babel/preset-react": "^7.22.5", - "@clusterio/lib": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", "antd": "^5.13.0", "assert": "^2.0.0", "babel-loader": "^9.1.3", diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json index 6e1ffcf0..09d9adb6 100644 --- a/plugins/global_chat/package.json +++ b/plugins/global_chat/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-global_chat", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -21,8 +21,8 @@ "@clusterio/lib": "^2.0.0-alpha.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", - "@clusterio/web_ui": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/web_ui": "^2.0.0-alpha.16", "@types/node": "^20.4.5", "@types/react": "^18.2.21", "typescript": "^5.1.6", diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index f5530566..384a64dd 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-inventory_sync", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Danielv ", "homepage": "https://github.com/clusterio/factorioClusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", - "@clusterio/web_ui": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/web_ui": "^2.0.0-alpha.16", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index 3a9bd9d5..8cf5b5c9 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-player_auth", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "description": "Clusterio plugin authenticating logged in players to the web interface", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -26,8 +26,8 @@ "jsonwebtoken": "^9.0.1" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", - "@clusterio/web_ui": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/web_ui": "^2.0.0-alpha.16", "@types/express": "^4.17.17", "@types/jsonwebtoken": "^9.0.2", "@types/node": "^20.4.5", diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json index c2184c13..fb9d4424 100644 --- a/plugins/research_sync/package.json +++ b/plugins/research_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-research_sync", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "description": "Clusterio plugin for synchronizing research between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", - "@clusterio/web_ui": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/web_ui": "^2.0.0-alpha.16", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json index a0d8fed8..c8e5a12f 100644 --- a/plugins/statistics_exporter/package.json +++ b/plugins/statistics_exporter/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-statistics_exporter", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "description": "Clusterio plugin for gathering statistics from Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -21,8 +21,8 @@ "@clusterio/lib": "^2.0.0-alpha.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", - "@clusterio/web_ui": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/web_ui": "^2.0.0-alpha.16", "@types/node": "^20.4.5", "@types/react": "^18.2.21", "typescript": "^5.1.6", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index e76630bd..892e9ca9 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-subspace_storage", - "version": "2.0.0-alpha.15", + "version": "2.0.0-alpha.16", "description": "Clusterio plugin for sharing storage between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.15", - "@clusterio/web_ui": "^2.0.0-alpha.15", + "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/web_ui": "^2.0.0-alpha.16", "@types/express": "^4.17.17", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", From 520007c8d1907421ac9eb3d67fbc423a3dfa002c Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 15:51:15 +0100 Subject: [PATCH 18/68] Fix unhandled rejection on RCON lost (#580) When using .finally on a promise a new promise is created which calls the handler passed to it when the the original promise is resolved or rejected, and resolves or rejects with same value as the original promise. This means if the rejection of the new promise that was created by finally is not handled then an unhandled promise rejection will be raised. Fix by making sure the promise rejection from the finally promise created in subspace_storage's handleUpdateStorageEvent is properly handled. --- CHANGELOG.md | 7 +++++++ plugins/subspace_storage/instance.ts | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e03d59a..4275137c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Version 2.0.0-alpha.17 + +## Changes + +- Fixed an unhandled error in Subspace Storage crashing the host. + + ## Version 2.0.0-alpha.16 ## Changes diff --git a/plugins/subspace_storage/instance.ts b/plugins/subspace_storage/instance.ts index 9bc594a8..9771eb26 100644 --- a/plugins/subspace_storage/instance.ts +++ b/plugins/subspace_storage/instance.ts @@ -116,7 +116,6 @@ export class InstancePlugin extends BaseInstancePlugin { let itemsJson = lib.escapeString(JSON.stringify(items)); let task = this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true); this.pendingTasks.add(task); - task.finally(() => { this.pendingTasks.delete(task); }); - await task; + await task.finally(() => { this.pendingTasks.delete(task); }); } } From 48c31b4825797df5e8840ee699c5b96b280aec14 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 23:36:43 +0100 Subject: [PATCH 19/68] Fix unhandled rejection on disconnect (#582) If a link is closed while the prepareDisconnect action is in progress the link will still try to signal the disconnect ready over the socket when the prepareDisconnect action completes, which throws due to the socket no longer being present. This error is not caught due to the catch happening before the finally on the promise and triggers an unhandled rejection. Fix by rearranging the finally to come before the catch on the promise so that errors are always handled, and checking that the link still has a session before attempting to send the disconnect ready signal. --- CHANGELOG.md | 1 + packages/lib/src/link/link.ts | 17 ++++++++--------- test/mock.js | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4275137c..68e772fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Changes - Fixed an unhandled error in Subspace Storage crashing the host. +- Fixed an unhandled rejection in disconnect logic crashing the host. ## Version 2.0.0-alpha.16 diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts index e3a746c0..0f8f2d32 100644 --- a/packages/lib/src/link/link.ts +++ b/packages/lib/src/link/link.ts @@ -126,16 +126,15 @@ export class Link { }); connector.on("disconnectPrepare", () => { - this.prepareDisconnect().catch( - err => { - logger.error(`Unexpected error preparing disconnect:\n${err.stack}`); + this.prepareDisconnect().finally(() => { + // Only WebSocket connectors issue disconnection events. + const webSocketConnector = this.connector as WebSocketBaseConnector; + if (webSocketConnector.hasSession) { + webSocketConnector.sendDisconnectReady(); } - ).finally( - () => { - // Only WebSocket connectors issue disconnection events. - (this.connector as WebSocketBaseConnector).sendDisconnectReady(); - }, - ); + }).catch(err => { + logger.error(`Unexpected error preparing disconnect:\n${err.stack}`); + }); }); connector.on("invalidate", () => { diff --git a/test/mock.js b/test/mock.js index 171c5923..4d2a8564 100644 --- a/test/mock.js +++ b/test/mock.js @@ -73,6 +73,7 @@ class MockConnector extends lib.BaseConnector { super(src, dst); this.connected = true; + this.hasSession = true; this.sentMessages = []; } From 40f5a6ab7cd1a0a30dc1742580d2492435952226 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 18:02:21 +0100 Subject: [PATCH 20/68] Fix timestamp not set on clearing instance saves Set the save's updatedAtMs timestamp when clearing the save list of a reassigned or deleted instance. This fixes the Web UI not updating the saves list to remove the saves when an instance is reassigned. --- packages/controller/src/Controller.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 3e81c0c1..d9a1332b 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -876,9 +876,11 @@ export default class Controller { */ clearSavesOfInstance(instanceId: number) { const updates = []; + const now = Date.now(); for (const [id, save] of this.saves) { if (save.instanceId === instanceId) { save.isDeleted = true; + save.updatedAtMs = now; updates.push(save); this.saves.delete(id); } From 7ba14f74330dbf13602b8b061629f9ee6db991ac Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 18:12:30 +0100 Subject: [PATCH 21/68] Fix instance details not updating on reassigmnent Always send the instance details update when an instance is reassigned, not just when it is unassigned. The idea for not sending an update when assigned to a host was that the host would send an update on the status that in turn updates the instance details. However this is not actually the case; when a stopped instance is reassigned its status does not change so the update is ignored. --- CHANGELOG.md | 1 + packages/controller/src/Controller.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68e772fe..9a507824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Fixed an unhandled error in Subspace Storage crashing the host. - Fixed an unhandled rejection in disconnect logic crashing the host. +- Fixed instance screen in Web UI not updating when an instance was reassigned. ## Version 2.0.0-alpha.16 diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index d9a1332b..5422fedf 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -941,9 +941,9 @@ export default class Controller { ); } else { instance.status = "unassigned"; - instance.updatedAtMs = Date.now(); - this.instanceDetailsUpdated([instance]); } + instance.updatedAtMs = Date.now(); + this.instanceDetailsUpdated([instance]); } /** From 021af793fcece14d4e96a4eba279f2d663c0210f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 18:13:57 +0100 Subject: [PATCH 22/68] Handle instance status updates with port change If an instance is reassigned its port may change without the status changing, these updates should not be ignored. --- packages/controller/src/HostConnection.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 564b5b96..ab566a40 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -178,7 +178,10 @@ export default class HostConnection extends BaseConnection { // changed. Hosts also send status updates on assignInstance which // for hacky reason is also used to push config changes and // restablish status after a connection loss. - if (instance.status === request.status) { + if ( + instance.status === request.status + && instance.gamePort === request.gamePort + ) { return; } From 2e25831b4df2ce9bd2e8dc5cd5bbe3c04cf4c91b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 18:16:15 +0100 Subject: [PATCH 23/68] Warn on subscription updates with same timestamp If the value in a subscribed list is updated while the updatedAtMs timestamp is kept the same then the caching logic of the subscription system will cause updates to be silently dropped. Log a warning if a subscriber catches this happening to help tracking down such cases. --- packages/lib/src/subscriptions.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/lib/src/subscriptions.ts b/packages/lib/src/subscriptions.ts index de155090..120cf2fe 100644 --- a/packages/lib/src/subscriptions.ts +++ b/packages/lib/src/subscriptions.ts @@ -2,6 +2,7 @@ import { Type, Static } from "@sinclair/typebox"; import { Link, Event, EventClass, RequestHandler, WebSocketClientConnector, WebSocketBaseConnector } from "./link"; import { logger } from "./logging"; import { Address, MessageRequest, IControllerUser } from "./data"; +import isDeepStrictEqual from "./is_deep_strict_equal"; export type SubscriptionRequestHandler = RequestHandler | null>; export type EventSubscriberCallback = (updates: T[], synced: boolean) => void @@ -204,6 +205,14 @@ export class EventSubscriber< async _handle(event: EventSubscribable) { for (const value of event.updates) { this.lastResponseTimeMs = Math.max(this.lastResponseTimeMs, value.updatedAtMs); + // Warn about updates which changes content but don't update updatedAtMs + const existing = this.values.get(value.id as K); + if (existing && existing.updatedAtMs === value.updatedAtMs && !isDeepStrictEqual(existing, value)) { + logger.warn( + `${this.Event.name} contains update for value with id ${value.id} that has an identical ` + + "updatedAtMs timestamp but differing content" + ); + } if (value.isDeleted) { this.values.delete(value.id as K); } else { From 6b85567bc63135e5de105d7d83234dbba40e04a4 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 18:45:06 +0100 Subject: [PATCH 24/68] Provide theme to top level ErrorBoundary If the theme is not provided then the App's ErrorBoundary will render white text on white background, and vice versa for black. --- packages/web_ui/src/components/App.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/web_ui/src/components/App.tsx b/packages/web_ui/src/components/App.tsx index 02e9c726..e4dcfdc6 100644 --- a/packages/web_ui/src/components/App.tsx +++ b/packages/web_ui/src/components/App.tsx @@ -104,16 +104,16 @@ export default function App(props: AppProps) { } return ( - - + + {page} - - + + ); } From 2df67e32daf18676c11d1b18032cd6f1f8614399 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 18:46:55 +0100 Subject: [PATCH 25/68] Set key for top level page fragment Without a unique per page key set react will try to reuse the existing state and components of the previous page and update them to match the new page. This causes unexpected behaviour as components get turned into into a Frankenstein abomination of the previous page and the current one fused into one. Fix by setting a key on the top level fragment of the page which in turn forces React to re-render the whole page whenever the user switches to another page. --- CHANGELOG.md | 1 + packages/web_ui/src/components/SiteLayout.tsx | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a507824..fec0adec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Fixed an unhandled error in Subspace Storage crashing the host. - Fixed an unhandled rejection in disconnect logic crashing the host. - Fixed instance screen in Web UI not updating when an instance was reassigned. +- Fixed an error thrown in the rendering of one page in the Web UI causing all pages to become inaccessible. ## Version 2.0.0-alpha.16 diff --git a/packages/web_ui/src/components/SiteLayout.tsx b/packages/web_ui/src/components/SiteLayout.tsx index 2bf69d47..07a995d1 100644 --- a/packages/web_ui/src/components/SiteLayout.tsx +++ b/packages/web_ui/src/components/SiteLayout.tsx @@ -1,4 +1,4 @@ -import React, { useContext, useEffect, useState } from "react"; +import React, { Fragment, useContext, useEffect, useState } from "react"; import { Route, Routes, useNavigate } from "react-router-dom"; import { Dropdown, Layout, Menu, MenuProps } from "antd"; import UserOutlined from "@ant-design/icons/UserOutlined"; @@ -115,12 +115,12 @@ export default function SiteLayout() { {combinedPages.map(({path, sidebarPath, content}) => + element={ {content} - } + } />)} } /> From 535e43b5278b9e5c1229bc798e9132e71c60b6f7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 19:20:33 +0100 Subject: [PATCH 26/68] Rename save.list.subscribe permission Follow the same pattern as all the other subscribable lists by having a separate .list and .subscribe permissions for saves. This also fixes the save list not being visible due to a typo in the permission in the Web UI. --- CHANGELOG.md | 1 + packages/lib/src/data/Role.ts | 10 +++++++++- packages/lib/src/data/messages_instance.ts | 2 +- packages/lib/src/permissions.ts | 6 +++--- packages/web_ui/src/components/InstanceViewPage.tsx | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fec0adec..b5a6b7b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Fixed an unhandled rejection in disconnect logic crashing the host. - Fixed instance screen in Web UI not updating when an instance was reassigned. - Fixed an error thrown in the rendering of one page in the Web UI causing all pages to become inaccessible. +- Fixed Web UI not showing instance saves when the permission to list and subscribe the saves is granted but the user is not a cluster admin. ## Version 2.0.0-alpha.16 diff --git a/packages/lib/src/data/Role.ts b/packages/lib/src/data/Role.ts index 6356def8..56f104bc 100644 --- a/packages/lib/src/data/Role.ts +++ b/packages/lib/src/data/Role.ts @@ -19,12 +19,20 @@ export default class Role { permissions: Type.Array(Type.String()), }); + static remapPermission(permission: string) { + // migrate: core.instance.save.list.subscribe was renamed in alpha 17 + if (permission === "core.instance.save.list.subscribe") { + return "core.instance.save.subscribe"; + } + return permission; + } + static fromJSON(json: Static) { return new this( json.id, json.name, json.description, - new Set(json.permissions), + new Set(json.permissions.map(this.remapPermission)), ); } diff --git a/packages/lib/src/data/messages_instance.ts b/packages/lib/src/data/messages_instance.ts index f71b2a13..2a37d086 100644 --- a/packages/lib/src/data/messages_instance.ts +++ b/packages/lib/src/data/messages_instance.ts @@ -361,7 +361,7 @@ export class InstanceSaveDetailsUpdatesEvent { static type = "event" as const; static src = ["instance", "host", "controller"] as const; static dst = ["controller", "control"] as const; - static permission = "core.instance.save.list.subscribe" as const; + static permission = "core.instance.save.subscribe" as const; constructor( public updates: SaveDetails[], diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index cbaf9a93..306047de 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -239,11 +239,11 @@ definePermission({ definePermission({ name: "core.instance.save.list", title: "List saves", - description: "List the saves currently on the instance.", + description: "List the saves currently on instances.", }); definePermission({ - name: "core.instance.save.list.subscribe", - title: "Subscribe to save list updatess", + name: "core.instance.save.subscribe", + title: "Subscribe to save updates", description: "Subscribe to be notifed on updates to the list of saves of instances.", }); definePermission({ diff --git a/packages/web_ui/src/components/InstanceViewPage.tsx b/packages/web_ui/src/components/InstanceViewPage.tsx index 13258761..f5b7281c 100644 --- a/packages/web_ui/src/components/InstanceViewPage.tsx +++ b/packages/web_ui/src/components/InstanceViewPage.tsx @@ -204,7 +204,7 @@ export default function InstanceViewPage() { { - account.hasAllPermission("core.instance.save.list", "core.instance.save.list_subscribe") + account.hasAllPermission("core.instance.save.list", "core.instance.save.subscribe") && } { From d3f8c993b16e97829272ea2f21f4a4d2497cf4c0 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 19:26:23 +0100 Subject: [PATCH 27/68] Fix bad permission checks in Web UI Fix attempts to check for permissions that don't exist in the Web UI that has gone unnoticed because the check still succeeds when the account has the cluster admin permission. --- CHANGELOG.md | 1 + packages/web_ui/src/components/HostViewPage.tsx | 2 +- packages/web_ui/src/components/ModPackViewPage.tsx | 2 +- packages/web_ui/src/components/ModsPage.tsx | 4 ++-- packages/web_ui/src/pages.tsx | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5a6b7b5..65cbac3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Fixed instance screen in Web UI not updating when an instance was reassigned. - Fixed an error thrown in the rendering of one page in the Web UI causing all pages to become inaccessible. - Fixed Web UI not showing instance saves when the permission to list and subscribe the saves is granted but the user is not a cluster admin. +- Similarly fixed Web UI not showing the mod pack create, delete, list and host revoke token actions if the user is not a cluster admin. ## Version 2.0.0-alpha.16 diff --git a/packages/web_ui/src/components/HostViewPage.tsx b/packages/web_ui/src/components/HostViewPage.tsx index fc71311d..f70d9e2c 100644 --- a/packages/web_ui/src/components/HostViewPage.tsx +++ b/packages/web_ui/src/components/HostViewPage.tsx @@ -48,7 +48,7 @@ export default function HostViewPage() { let hostButtons = { - account.hasPermission("core.host.revoke_access") + account.hasPermission("core.host.revoke_token") && - {account.hasPermission("core.mod-pack.delete") && - {account.hasPermission("core.mod-pack.create") && } - {account.hasPermission("core.mod-pack.create") && } + {account.hasPermission("core.mod_pack.create") && } + {account.hasPermission("core.mod_pack.create") && } } /> account.hasAnyPermission("core.mod.list", "core.mod-pack.list"), + permission: (account: UserAccount) => account.hasAnyPermission("core.mod.list", "core.mod_pack.list"), content: , }, { From 44b0a8ff834434c6e3cd75ed232a3e24526bfb5a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 19:28:21 +0100 Subject: [PATCH 28/68] Throw error on bad permission check in Wed UI Throw an error if an attempt is made to check for a permission that does not exist in the Web UI to prevent future bad permission checks going unnoticed. --- packages/web_ui/src/model/account.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/web_ui/src/model/account.ts b/packages/web_ui/src/model/account.ts index 8086bf3a..dfce6832 100644 --- a/packages/web_ui/src/model/account.ts +++ b/packages/web_ui/src/model/account.ts @@ -22,6 +22,12 @@ export function useAccount(): UserAccount { }; }, [control]); + function checkPermissionExists(permission: string) { + if (!lib.permissions.has(permission)) { + throw new Error(`permission ${permission} does not exist`); + } + } + function hasPermission(permission: string): boolean { for (let role of roles) { if (role.permissions.includes("core.admin") || role.permissions.includes(permission)) { @@ -38,12 +44,16 @@ export function useAccount(): UserAccount { if (!roles) { return null; } + checkPermissionExists(permission); return hasPermission(permission); }, hasAnyPermission(...permissions: string[]): boolean | null { if (!roles) { return null; } + for (let permission of permissions) { + checkPermissionExists(permission); + } for (let permission of permissions) { if (hasPermission(permission)) { return true; @@ -55,6 +65,9 @@ export function useAccount(): UserAccount { if (!roles) { return null; } + for (let permission of permissions) { + checkPermissionExists(permission); + } for (let permission of permissions) { if (!hasPermission(permission)) { return false; From 6bf2bf51f23560570ee9b0d93bfa5eced165fc4a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 5 Feb 2024 19:29:50 +0100 Subject: [PATCH 29/68] Remove deprecated showArrow prop Ant design has deprecated the Select's showArrow prop. It's now always shown by default. --- packages/web_ui/src/components/UserViewPage.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/web_ui/src/components/UserViewPage.tsx b/packages/web_ui/src/components/UserViewPage.tsx index b0d138a4..8770571f 100644 --- a/packages/web_ui/src/components/UserViewPage.tsx +++ b/packages/web_ui/src/components/UserViewPage.tsx @@ -88,7 +88,6 @@ export default function UserViewPage() { onChange={() => setRolesDirty(true)} loading={!roles} mode="multiple" - showArrow={true} filterOption={(inputValue:string, option: any): boolean => { let role = roles.get(option.value); return role?.name.toLowerCase().includes(inputValue.toLowerCase()) ?? false; From dcf216ca09ba8c7c37da75053aa82a7ff58845b9 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 6 Feb 2024 20:50:19 +0100 Subject: [PATCH 30/68] Fix research sync applying progress to wrong level When the level of a technology changes either by setting level directly or by setting researched to true, the saved progress of the research is reset only if the technology is not the current research. If during the initial synchronisation with the controller an instance has progressed on a multi level research, it's the current research, and the controller's level of that research is higher, then the level is set to the higher level and the game keeps the progress. This in turn results in all of the progress made on the previous level being added to the higher level. Fix by setting the progress to zero when changing the level or researched status of a technology that's the active research. --- CHANGELOG.md | 1 + plugins/research_sync/module/sync.lua | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65cbac3b..77598f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Fixed an error thrown in the rendering of one page in the Web UI causing all pages to become inaccessible. - Fixed Web UI not showing instance saves when the permission to list and subscribe the saves is granted but the user is not a cluster admin. - Similarly fixed Web UI not showing the mod pack create, delete, list and host revoke token actions if the user is not a cluster admin. +- Fixed Research Sync applying progress of a previous techology level to the level on the controller when it's the active research during instance startup. ## Version 2.0.0-alpha.16 diff --git a/plugins/research_sync/module/sync.lua b/plugins/research_sync/module/sync.lua index 03a1bb9e..7abe268a 100644 --- a/plugins/research_sync/module/sync.lua +++ b/plugins/research_sync/module/sync.lua @@ -137,10 +137,21 @@ function research_sync.sync_technologies(data) for _, tech_data in pairs(game.json_to_table(data)) do local tech = force.technologies[tech_data[nameIndex]] if tech and tech.level <= tech_data[levelIndex] then - tech.level = math.min(tech_data[levelIndex], tech.prototype.max_level) + local new_level = math.min(tech_data[levelIndex], tech.prototype.max_level) + if new_level ~= tech.level then + -- when the level of the current research changes the + -- progress is not automatically reset. + if force.current_research == tech then + force.research_progress = 0 + end + tech.level = new_level + end local progress if tech_data[researchedIndex] then + if force.current_research == tech then + force.research_progress = 0 + end tech.researched = true progress = nil elseif tech_data[progressIndex] then From 99990c7a0d57ddae9c656f635c63bd1f80db9cd4 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Feb 2024 00:05:13 +0100 Subject: [PATCH 31/68] Fix incorrect header level in CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77598f4d..cac58b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Version 2.0.0-alpha.17 -## Changes +### Changes - Fixed an unhandled error in Subspace Storage crashing the host. - Fixed an unhandled rejection in disconnect logic crashing the host. @@ -15,7 +15,7 @@ ## Version 2.0.0-alpha.16 -## Changes +### Changes - Fixed message sequences not being renumbered when forwarded between links causing reconnects to send duplicated messages. From 8e5d581b9bd126b80753e86871c4132de24dc56d Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Feb 2024 17:25:42 +0100 Subject: [PATCH 32/68] Move tsconfig.web.json to tsconfig.browser.json Rename the config files for the browser build of modules to better communicate that it's purpose is to compile code for a browser environment rather than the Web UI in particular. This also aligns with the "browser" option in package.json, the existing browser.ts entrypoint in lib, and how the browser environment is typically referred to. --- ...tsconfig.web.json => tsconfig.browser.json} | 6 +++--- ...tsconfig.web.json => tsconfig.browser.json} | 2 +- ...tsconfig.web.json => tsconfig.browser.json} | 4 ++-- packages/web_ui/webpack.common.js | 2 +- plugins/global_chat/tsconfig.browser.json | 11 +++++++++++ plugins/global_chat/tsconfig.web.json | 11 ----------- plugins/inventory_sync/tsconfig.browser.json | 11 +++++++++++ plugins/inventory_sync/tsconfig.web.json | 11 ----------- plugins/player_auth/tsconfig.browser.json | 11 +++++++++++ plugins/player_auth/tsconfig.web.json | 11 ----------- plugins/research_sync/tsconfig.browser.json | 11 +++++++++++ plugins/research_sync/tsconfig.web.json | 11 ----------- .../statistics_exporter/tsconfig.browser.json | 11 +++++++++++ plugins/statistics_exporter/tsconfig.web.json | 11 ----------- plugins/subspace_storage/tsconfig.browser.json | 11 +++++++++++ plugins/subspace_storage/tsconfig.web.json | 11 ----------- tsconfig.web.json => tsconfig.browser.json | 0 tsconfig.json | 18 +++++++++--------- 18 files changed, 82 insertions(+), 82 deletions(-) rename packages/controller/{tsconfig.web.json => tsconfig.browser.json} (54%) rename packages/lib/{tsconfig.web.json => tsconfig.browser.json} (93%) rename packages/web_ui/{tsconfig.web.json => tsconfig.browser.json} (62%) create mode 100644 plugins/global_chat/tsconfig.browser.json delete mode 100644 plugins/global_chat/tsconfig.web.json create mode 100644 plugins/inventory_sync/tsconfig.browser.json delete mode 100644 plugins/inventory_sync/tsconfig.web.json create mode 100644 plugins/player_auth/tsconfig.browser.json delete mode 100644 plugins/player_auth/tsconfig.web.json create mode 100644 plugins/research_sync/tsconfig.browser.json delete mode 100644 plugins/research_sync/tsconfig.web.json create mode 100644 plugins/statistics_exporter/tsconfig.browser.json delete mode 100644 plugins/statistics_exporter/tsconfig.web.json create mode 100644 plugins/subspace_storage/tsconfig.browser.json delete mode 100644 plugins/subspace_storage/tsconfig.web.json rename tsconfig.web.json => tsconfig.browser.json (100%) diff --git a/packages/controller/tsconfig.web.json b/packages/controller/tsconfig.browser.json similarity index 54% rename from packages/controller/tsconfig.web.json rename to packages/controller/tsconfig.browser.json index 6685a163..3fff0691 100644 --- a/packages/controller/tsconfig.web.json +++ b/packages/controller/tsconfig.browser.json @@ -1,8 +1,8 @@ { - "extends": "../../tsconfig.web.json", + "extends": "../../tsconfig.browser.json", "references": [ - { "path": "../lib/tsconfig.web.json" }, - { "path": "../web_ui/tsconfig.web.json" }, + { "path": "../lib/tsconfig.browser.json" }, + { "path": "../web_ui/tsconfig.browser.json" }, ], "include": [ "web/**/*.tsx", "web/**/*.ts", "package.json" ], "compilerOptions": { diff --git a/packages/lib/tsconfig.web.json b/packages/lib/tsconfig.browser.json similarity index 93% rename from packages/lib/tsconfig.web.json rename to packages/lib/tsconfig.browser.json index a4349659..36ebfbd9 100644 --- a/packages/lib/tsconfig.web.json +++ b/packages/lib/tsconfig.browser.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.web.json", + "extends": "../../tsconfig.browser.json", "include": [ "browser.ts", "package.json", diff --git a/packages/web_ui/tsconfig.web.json b/packages/web_ui/tsconfig.browser.json similarity index 62% rename from packages/web_ui/tsconfig.web.json rename to packages/web_ui/tsconfig.browser.json index 65754630..8d0236e7 100644 --- a/packages/web_ui/tsconfig.web.json +++ b/packages/web_ui/tsconfig.browser.json @@ -1,7 +1,7 @@ { - "extends": "../../tsconfig.web.json", + "extends": "../../tsconfig.browser.json", "references": [ - { "path": "../lib/tsconfig.web.json" }, + { "path": "../lib/tsconfig.browser.json" }, ], "include": [ "src/**/*.tsx", "src/**/*.ts", "package.json", "global.d.ts" ], "compilerOptions": { diff --git a/packages/web_ui/webpack.common.js b/packages/web_ui/webpack.common.js index e8a5d8b7..99825c4b 100644 --- a/packages/web_ui/webpack.common.js +++ b/packages/web_ui/webpack.common.js @@ -79,7 +79,7 @@ module.exports = (env = {}) => ({ use: { loader: require.resolve("ts-loader"), options: { - configFile: "tsconfig.web.json", + configFile: "tsconfig.browser.json", projectReferences: true, }, }, diff --git a/plugins/global_chat/tsconfig.browser.json b/plugins/global_chat/tsconfig.browser.json new file mode 100644 index 00000000..a3109220 --- /dev/null +++ b/plugins/global_chat/tsconfig.browser.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.browser.json", + "references": [ + { "path": "../../packages/lib/tsconfig.browser.json" }, + { "path": "../../packages/web_ui/tsconfig.browser.json" }, + ], + "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], + "compilerOptions": { + "outDir": "dist/web/build", + } +} diff --git a/plugins/global_chat/tsconfig.web.json b/plugins/global_chat/tsconfig.web.json deleted file mode 100644 index c37f9141..00000000 --- a/plugins/global_chat/tsconfig.web.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.web.json", - "references": [ - { "path": "../../packages/lib/tsconfig.web.json" }, - { "path": "../../packages/web_ui/tsconfig.web.json" }, - ], - "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], - "compilerOptions": { - "outDir": "dist/web/build", - } -} diff --git a/plugins/inventory_sync/tsconfig.browser.json b/plugins/inventory_sync/tsconfig.browser.json new file mode 100644 index 00000000..a3109220 --- /dev/null +++ b/plugins/inventory_sync/tsconfig.browser.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.browser.json", + "references": [ + { "path": "../../packages/lib/tsconfig.browser.json" }, + { "path": "../../packages/web_ui/tsconfig.browser.json" }, + ], + "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], + "compilerOptions": { + "outDir": "dist/web/build", + } +} diff --git a/plugins/inventory_sync/tsconfig.web.json b/plugins/inventory_sync/tsconfig.web.json deleted file mode 100644 index c37f9141..00000000 --- a/plugins/inventory_sync/tsconfig.web.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.web.json", - "references": [ - { "path": "../../packages/lib/tsconfig.web.json" }, - { "path": "../../packages/web_ui/tsconfig.web.json" }, - ], - "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], - "compilerOptions": { - "outDir": "dist/web/build", - } -} diff --git a/plugins/player_auth/tsconfig.browser.json b/plugins/player_auth/tsconfig.browser.json new file mode 100644 index 00000000..a3109220 --- /dev/null +++ b/plugins/player_auth/tsconfig.browser.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.browser.json", + "references": [ + { "path": "../../packages/lib/tsconfig.browser.json" }, + { "path": "../../packages/web_ui/tsconfig.browser.json" }, + ], + "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], + "compilerOptions": { + "outDir": "dist/web/build", + } +} diff --git a/plugins/player_auth/tsconfig.web.json b/plugins/player_auth/tsconfig.web.json deleted file mode 100644 index c37f9141..00000000 --- a/plugins/player_auth/tsconfig.web.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.web.json", - "references": [ - { "path": "../../packages/lib/tsconfig.web.json" }, - { "path": "../../packages/web_ui/tsconfig.web.json" }, - ], - "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], - "compilerOptions": { - "outDir": "dist/web/build", - } -} diff --git a/plugins/research_sync/tsconfig.browser.json b/plugins/research_sync/tsconfig.browser.json new file mode 100644 index 00000000..a3109220 --- /dev/null +++ b/plugins/research_sync/tsconfig.browser.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.browser.json", + "references": [ + { "path": "../../packages/lib/tsconfig.browser.json" }, + { "path": "../../packages/web_ui/tsconfig.browser.json" }, + ], + "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], + "compilerOptions": { + "outDir": "dist/web/build", + } +} diff --git a/plugins/research_sync/tsconfig.web.json b/plugins/research_sync/tsconfig.web.json deleted file mode 100644 index c37f9141..00000000 --- a/plugins/research_sync/tsconfig.web.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.web.json", - "references": [ - { "path": "../../packages/lib/tsconfig.web.json" }, - { "path": "../../packages/web_ui/tsconfig.web.json" }, - ], - "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], - "compilerOptions": { - "outDir": "dist/web/build", - } -} diff --git a/plugins/statistics_exporter/tsconfig.browser.json b/plugins/statistics_exporter/tsconfig.browser.json new file mode 100644 index 00000000..0454f6c4 --- /dev/null +++ b/plugins/statistics_exporter/tsconfig.browser.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.browser.json", + "references": [ + { "path": "../../packages/lib/tsconfig.browser.json" }, + { "path": "../../packages/web_ui/tsconfig.browser.json" }, + ], + "include": [ "web/**/*.tsx", "web/**/*.ts", "package.json" ], + "compilerOptions": { + "outDir": "dist/web/build", + } +} diff --git a/plugins/statistics_exporter/tsconfig.web.json b/plugins/statistics_exporter/tsconfig.web.json deleted file mode 100644 index decec50c..00000000 --- a/plugins/statistics_exporter/tsconfig.web.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.web.json", - "references": [ - { "path": "../../packages/lib/tsconfig.web.json" }, - { "path": "../../packages/web_ui/tsconfig.web.json" }, - ], - "include": [ "web/**/*.tsx", "web/**/*.ts", "package.json" ], - "compilerOptions": { - "outDir": "dist/web/build", - } -} diff --git a/plugins/subspace_storage/tsconfig.browser.json b/plugins/subspace_storage/tsconfig.browser.json new file mode 100644 index 00000000..a3109220 --- /dev/null +++ b/plugins/subspace_storage/tsconfig.browser.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.browser.json", + "references": [ + { "path": "../../packages/lib/tsconfig.browser.json" }, + { "path": "../../packages/web_ui/tsconfig.browser.json" }, + ], + "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], + "compilerOptions": { + "outDir": "dist/web/build", + } +} diff --git a/plugins/subspace_storage/tsconfig.web.json b/plugins/subspace_storage/tsconfig.web.json deleted file mode 100644 index c37f9141..00000000 --- a/plugins/subspace_storage/tsconfig.web.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.web.json", - "references": [ - { "path": "../../packages/lib/tsconfig.web.json" }, - { "path": "../../packages/web_ui/tsconfig.web.json" }, - ], - "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], - "compilerOptions": { - "outDir": "dist/web/build", - } -} diff --git a/tsconfig.web.json b/tsconfig.browser.json similarity index 100% rename from tsconfig.web.json rename to tsconfig.browser.json diff --git a/tsconfig.json b/tsconfig.json index c9f896d7..fbb6a7ec 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,23 +2,23 @@ "files": [], "references": [ { "path": "./packages/lib" }, - { "path": "./packages/lib/tsconfig.web.json" }, { "path": "./packages/controller" }, - { "path": "./packages/controller/tsconfig.web.json" }, { "path": "./packages/host" }, { "path": "./packages/ctl" }, - { "path": "./packages/web_ui/tsconfig.web.json" }, { "path": "./plugins/global_chat" }, - { "path": "./plugins/global_chat/tsconfig.web.json" }, { "path": "./plugins/inventory_sync" }, - { "path": "./plugins/inventory_sync/tsconfig.web.json" }, { "path": "./plugins/player_auth" }, - { "path": "./plugins/player_auth/tsconfig.web.json" }, { "path": "./plugins/research_sync" }, - { "path": "./plugins/research_sync/tsconfig.web.json" }, { "path": "./plugins/statistics_exporter" }, - { "path": "./plugins/statistics_exporter/tsconfig.web.json" }, { "path": "./plugins/subspace_storage" }, - { "path": "./plugins/subspace_storage/tsconfig.web.json" }, + { "path": "./packages/lib/tsconfig.browser.json" }, + { "path": "./packages/controller/tsconfig.browser.json" }, + { "path": "./packages/web_ui/tsconfig.browser.json" }, + { "path": "./plugins/global_chat/tsconfig.browser.json" }, + { "path": "./plugins/inventory_sync/tsconfig.browser.json" }, + { "path": "./plugins/player_auth/tsconfig.browser.json" }, + { "path": "./plugins/research_sync/tsconfig.browser.json" }, + { "path": "./plugins/statistics_exporter/tsconfig.browser.json" }, + { "path": "./plugins/subspace_storage/tsconfig.browser.json" }, ], } From 152eb7d45433ae09bf38b3baad80345ba869b3d9 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Feb 2024 17:50:48 +0100 Subject: [PATCH 33/68] Move TypeScript browser build to dist/browser Use dist/browser as the output of the browser oriented TypeScript build of packages instead of dist/web/build. This distinguishes it better from the Web UI bundle which is put together with webpack. --- packages/controller/.npmignore | 2 +- packages/controller/tsconfig.browser.json | 2 +- packages/lib/.npmignore | 2 +- packages/lib/tsconfig.browser.json | 2 +- packages/web_ui/package.json | 2 +- packages/web_ui/tsconfig.browser.json | 2 +- plugins/global_chat/.npmignore | 2 +- plugins/global_chat/tsconfig.browser.json | 2 +- plugins/inventory_sync/.npmignore | 2 +- plugins/inventory_sync/tsconfig.browser.json | 2 +- plugins/player_auth/.npmignore | 2 +- plugins/player_auth/tsconfig.browser.json | 2 +- plugins/research_sync/.npmignore | 2 +- plugins/research_sync/tsconfig.browser.json | 2 +- plugins/statistics_exporter/.npmignore | 2 +- plugins/statistics_exporter/tsconfig.browser.json | 2 +- plugins/subspace_storage/.npmignore | 2 +- plugins/subspace_storage/tsconfig.browser.json | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/controller/.npmignore b/packages/controller/.npmignore index ebd99907..55622581 100644 --- a/packages/controller/.npmignore +++ b/packages/controller/.npmignore @@ -1,2 +1,2 @@ webpack.config.js -dist/web/build +dist/browser diff --git a/packages/controller/tsconfig.browser.json b/packages/controller/tsconfig.browser.json index 3fff0691..154bd176 100644 --- a/packages/controller/tsconfig.browser.json +++ b/packages/controller/tsconfig.browser.json @@ -7,6 +7,6 @@ "include": [ "web/**/*.tsx", "web/**/*.ts", "package.json" ], "compilerOptions": { "rootDir": "..", - "outDir": "dist/web/build", + "outDir": "dist/browser", } } diff --git a/packages/lib/.npmignore b/packages/lib/.npmignore index 5dfa68f1..55239636 100644 --- a/packages/lib/.npmignore +++ b/packages/lib/.npmignore @@ -1 +1 @@ -dist/web/build +dist/browser diff --git a/packages/lib/tsconfig.browser.json b/packages/lib/tsconfig.browser.json index 36ebfbd9..a53b11d7 100644 --- a/packages/lib/tsconfig.browser.json +++ b/packages/lib/tsconfig.browser.json @@ -25,6 +25,6 @@ ], "compilerOptions": { "rootDir": ".", - "outDir": "dist/web/build", + "outDir": "dist/browser", } } diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 796c3242..fb77d82d 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -5,7 +5,7 @@ "license": "MIT", "version": "2.0.0-alpha.16", "browser": "src/index.ts", - "main": "dist/src/index.js", + "main": "dist/browser/src/index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/packages/web_ui/tsconfig.browser.json b/packages/web_ui/tsconfig.browser.json index 8d0236e7..62dbe09d 100644 --- a/packages/web_ui/tsconfig.browser.json +++ b/packages/web_ui/tsconfig.browser.json @@ -5,6 +5,6 @@ ], "include": [ "src/**/*.tsx", "src/**/*.ts", "package.json", "global.d.ts" ], "compilerOptions": { - "outDir": "dist", + "outDir": "dist/browser", } } diff --git a/plugins/global_chat/.npmignore b/plugins/global_chat/.npmignore index ebd99907..55622581 100644 --- a/plugins/global_chat/.npmignore +++ b/plugins/global_chat/.npmignore @@ -1,2 +1,2 @@ webpack.config.js -dist/web/build +dist/browser diff --git a/plugins/global_chat/tsconfig.browser.json b/plugins/global_chat/tsconfig.browser.json index a3109220..e1b39d0f 100644 --- a/plugins/global_chat/tsconfig.browser.json +++ b/plugins/global_chat/tsconfig.browser.json @@ -6,6 +6,6 @@ ], "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], "compilerOptions": { - "outDir": "dist/web/build", + "outDir": "dist/browser", } } diff --git a/plugins/inventory_sync/.npmignore b/plugins/inventory_sync/.npmignore index ebd99907..55622581 100644 --- a/plugins/inventory_sync/.npmignore +++ b/plugins/inventory_sync/.npmignore @@ -1,2 +1,2 @@ webpack.config.js -dist/web/build +dist/browser diff --git a/plugins/inventory_sync/tsconfig.browser.json b/plugins/inventory_sync/tsconfig.browser.json index a3109220..e1b39d0f 100644 --- a/plugins/inventory_sync/tsconfig.browser.json +++ b/plugins/inventory_sync/tsconfig.browser.json @@ -6,6 +6,6 @@ ], "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], "compilerOptions": { - "outDir": "dist/web/build", + "outDir": "dist/browser", } } diff --git a/plugins/player_auth/.npmignore b/plugins/player_auth/.npmignore index ebd99907..55622581 100644 --- a/plugins/player_auth/.npmignore +++ b/plugins/player_auth/.npmignore @@ -1,2 +1,2 @@ webpack.config.js -dist/web/build +dist/browser diff --git a/plugins/player_auth/tsconfig.browser.json b/plugins/player_auth/tsconfig.browser.json index a3109220..e1b39d0f 100644 --- a/plugins/player_auth/tsconfig.browser.json +++ b/plugins/player_auth/tsconfig.browser.json @@ -6,6 +6,6 @@ ], "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], "compilerOptions": { - "outDir": "dist/web/build", + "outDir": "dist/browser", } } diff --git a/plugins/research_sync/.npmignore b/plugins/research_sync/.npmignore index ebd99907..55622581 100644 --- a/plugins/research_sync/.npmignore +++ b/plugins/research_sync/.npmignore @@ -1,2 +1,2 @@ webpack.config.js -dist/web/build +dist/browser diff --git a/plugins/research_sync/tsconfig.browser.json b/plugins/research_sync/tsconfig.browser.json index a3109220..e1b39d0f 100644 --- a/plugins/research_sync/tsconfig.browser.json +++ b/plugins/research_sync/tsconfig.browser.json @@ -6,6 +6,6 @@ ], "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], "compilerOptions": { - "outDir": "dist/web/build", + "outDir": "dist/browser", } } diff --git a/plugins/statistics_exporter/.npmignore b/plugins/statistics_exporter/.npmignore index ebd99907..55622581 100644 --- a/plugins/statistics_exporter/.npmignore +++ b/plugins/statistics_exporter/.npmignore @@ -1,2 +1,2 @@ webpack.config.js -dist/web/build +dist/browser diff --git a/plugins/statistics_exporter/tsconfig.browser.json b/plugins/statistics_exporter/tsconfig.browser.json index 0454f6c4..f2dc36da 100644 --- a/plugins/statistics_exporter/tsconfig.browser.json +++ b/plugins/statistics_exporter/tsconfig.browser.json @@ -6,6 +6,6 @@ ], "include": [ "web/**/*.tsx", "web/**/*.ts", "package.json" ], "compilerOptions": { - "outDir": "dist/web/build", + "outDir": "dist/browser", } } diff --git a/plugins/subspace_storage/.npmignore b/plugins/subspace_storage/.npmignore index ebd99907..55622581 100644 --- a/plugins/subspace_storage/.npmignore +++ b/plugins/subspace_storage/.npmignore @@ -1,2 +1,2 @@ webpack.config.js -dist/web/build +dist/browser diff --git a/plugins/subspace_storage/tsconfig.browser.json b/plugins/subspace_storage/tsconfig.browser.json index a3109220..e1b39d0f 100644 --- a/plugins/subspace_storage/tsconfig.browser.json +++ b/plugins/subspace_storage/tsconfig.browser.json @@ -6,6 +6,6 @@ ], "include": [ "web/**/*.tsx", "web/**/*.ts", "messages.ts", "package.json" ], "compilerOptions": { - "outDir": "dist/web/build", + "outDir": "dist/browser", } } From 8c4d2db19211cd3bcdad847b232945ee8f6231d1 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Feb 2024 18:27:46 +0100 Subject: [PATCH 34/68] Move TypeScript Node.js build to dist/node Use a separate folder for the TypeScript build for Node.js in the dist folder. This makes each type of build consistently have its own output folder instead of being interleaved with the Node.js build. --- .eslintrc.js | 2 +- docs/devs/plugin-migration-to-alpha-14.md | 2 +- packages/controller/package.json | 4 ++-- packages/controller/src/Controller.ts | 8 ++++---- packages/controller/tsconfig.json | 2 +- packages/ctl/package.json | 4 ++-- packages/ctl/tsconfig.json | 2 +- packages/host/package.json | 4 ++-- packages/host/src/Instance.ts | 2 +- packages/host/src/export.ts | 4 ++-- packages/host/tsconfig.json | 2 +- packages/lib/package.json | 2 +- packages/lib/scripts/compile_validator.js | 6 +++--- packages/lib/tsconfig.json | 2 +- plugins/global_chat/index.ts | 4 ++-- plugins/global_chat/package.json | 2 +- plugins/global_chat/test/plugin.js | 6 +++--- plugins/global_chat/tsconfig.json | 2 +- plugins/inventory_sync/index.ts | 4 ++-- plugins/inventory_sync/package.json | 2 +- plugins/inventory_sync/tsconfig.json | 2 +- plugins/player_auth/index.ts | 4 ++-- plugins/player_auth/package.json | 2 +- plugins/player_auth/test/plugin.js | 8 ++++---- plugins/player_auth/tsconfig.json | 2 +- plugins/research_sync/index.ts | 4 ++-- plugins/research_sync/package.json | 2 +- plugins/research_sync/test/missing.js | 4 ++-- plugins/research_sync/tsconfig.json | 2 +- plugins/statistics_exporter/index.ts | 2 +- plugins/statistics_exporter/package.json | 2 +- plugins/statistics_exporter/test/plugin.js | 4 ++-- plugins/statistics_exporter/tsconfig.json | 2 +- plugins/subspace_storage/index.ts | 4 ++-- plugins/subspace_storage/package.json | 2 +- plugins/subspace_storage/test/missing.js | 8 ++++---- plugins/subspace_storage/test/routes.js | 2 +- plugins/subspace_storage/tsconfig.json | 2 +- test/controller/HttpCloser.js | 2 +- test/controller/routes.js | 2 +- test/host.js | 4 ++-- test/host/Instance.js | 2 +- test/host/export.js | 4 ++-- test/host/patch.js | 2 +- test/host/server.js | 2 +- test/integration/export.js | 4 ++-- test/integration/index.js | 10 +++++----- test/integration/patch.js | 2 +- test/integration/server.js | 2 +- test/mock.js | 4 ++-- 50 files changed, 82 insertions(+), 82 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 1487649a..d4cbf768 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -391,7 +391,7 @@ module.exports = { "node/process-exit-as-throw": "error", "node/shebang": ["error", { "convertPath": { - "*.ts": ["^(.+)\\.ts$", "dist/$1.js"], + "*.ts": ["^(.+)\\.ts$", "dist/node/$1.js"], }, }], }, diff --git a/docs/devs/plugin-migration-to-alpha-14.md b/docs/devs/plugin-migration-to-alpha-14.md index 91962795..28266664 100644 --- a/docs/devs/plugin-migration-to-alpha-14.md +++ b/docs/devs/plugin-migration-to-alpha-14.md @@ -78,7 +78,7 @@ import { BaseInstancePlugin } from "@clusterio/host"; Previously the info of a plugin would be loaded by requiring `./plugin_dir/info`, this has been changed to require `./plugin_dir` which in turn means that `info.js` is no longer the default entrypoint. To solve this you can either add `"main": "info.js",` to package.json or rename `info.js` to `index.js` (the rest of this guide assumes the entrypoint was renamed to `index.js`). -The motivation behind this change was to allow TypeScript to output its build to a sub-folder of the plugin, if you're using TypeScript you'll likely have the entrypoint be located in `dist/index.js` instead. +The motivation behind this change was to allow TypeScript to output its build to a sub-folder of the plugin, if you're using TypeScript you'll likely have the entrypoint be located in `dist/node/index.js` instead. The expected place where the declaration of a plugin is exported has also changed from `module.exports` to `module.exports.plugin`: diff --git a/packages/controller/package.json b/packages/controller/package.json index a3da2405..7446edbd 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -4,13 +4,13 @@ "author": "Hornwitser", "license": "MIT", "version": "2.0.0-alpha.16", - "main": "dist/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: no test specified\" && exit 1" }, "bin": { - "clusteriocontroller": "./dist/controller.js" + "clusteriocontroller": "./dist/node/controller.js" }, "keywords": [ "factorio" diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 5422fedf..7bd67a52 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -274,7 +274,7 @@ export default class Controller { Controller.addAppRoutes(this.app, this.pluginInfos); if (!args.dev) { - let manifestPath = path.join(__dirname, "..", "web", "manifest.json"); + let manifestPath = path.join(__dirname, "..", "..", "web", "manifest.json"); let manifest = await Controller.loadJsonObject(manifestPath); if (!manifest["main.js"]) { @@ -333,7 +333,7 @@ export default class Controller { if (args.dev) { // eslint-disable-next-line node/no-missing-require - webpackConfigs.push(require("../../webpack.config")({})); // Path outside of build + webpackConfigs.push(require("../../../webpack.config")({})); // Path outside of build } if (args.devPlugin) { let devPlugins = new Map(); @@ -737,7 +737,7 @@ export default class Controller { // Set folder to serve static content from (the website) const staticOptions = { immutable: true, maxAge: 1000 * 86400 * 365 }; app.use("/static", - express.static(path.join(__dirname, "..", "..", "dist", "web", "static"), staticOptions) + express.static(path.join(__dirname, "..", "..", "web", "static"), staticOptions) ); app.use("/static", express.static("static", staticOptions)); // Used for data export files @@ -1226,7 +1226,7 @@ export default class Controller { mainBundle = stats.toJson().assetsByChunkName["main"]; } - fs.readFile(path.join(__dirname, "..", "..", "web", "index.html"), "utf8").then((content) => { + fs.readFile(path.join(__dirname, "..", "..", "..", "web", "index.html"), "utf8").then((content) => { res.type("text/html"); res.send(content .replace(/__CLUSTER_NAME__/g, res.app.locals.controller.config.get("controller.name")) diff --git a/packages/controller/tsconfig.json b/packages/controller/tsconfig.json index 335b8b16..8dcc9b8b 100644 --- a/packages/controller/tsconfig.json +++ b/packages/controller/tsconfig.json @@ -5,6 +5,6 @@ ], "include": ["index.ts", "controller.ts", "package.json", "src/**/*.ts"], "compilerOptions": { - "outDir": "dist", + "outDir": "dist/node", }, } diff --git a/packages/ctl/package.json b/packages/ctl/package.json index 8793f1bb..7d5f6707 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -4,13 +4,13 @@ "author": "Hornwitser", "license": "MIT", "version": "2.0.0-alpha.16", - "main": "dist/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", "test": "echo \"Error: no test specified\" && exit 1" }, "bin": { - "clusterioctl": "./dist/ctl.js" + "clusterioctl": "./dist/node/ctl.js" }, "keywords": [ "factorio" diff --git a/packages/ctl/tsconfig.json b/packages/ctl/tsconfig.json index 574dad9d..134019a5 100644 --- a/packages/ctl/tsconfig.json +++ b/packages/ctl/tsconfig.json @@ -5,6 +5,6 @@ ], "include": ["index.ts", "package.json", "ctl.ts", "src/**/*.ts"], "compilerOptions": { - "outDir": "dist", + "outDir": "dist/node", }, } diff --git a/packages/host/package.json b/packages/host/package.json index ce801590..c9f8446a 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -4,13 +4,13 @@ "author": "Hornwitser", "license": "MIT", "version": "2.0.0-alpha.16", - "main": "dist/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", "test": "echo \"Error: no test specified\" && exit 1" }, "bin": { - "clusteriohost": "./dist/host.js" + "clusteriohost": "./dist/node/host.js" }, "keywords": [ "factorio" diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 98d4d0eb..f6163611 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -800,7 +800,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); // Find stand alone modules to load // XXX for now only the included clusterio module is loaded - let modulesDirectory = path.join(__dirname, "..", "..", "modules"); + let modulesDirectory = path.join(__dirname, "..", "..", "..", "modules"); for (let entry of await fs.readdir(modulesDirectory, { withFileTypes: true })) { if (entry.isDirectory()) { if (modules.has(entry.name)) { diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index dcd2b01a..938836d4 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -5,7 +5,7 @@ import Jimp from "jimp"; import JSZip from "jszip"; import * as lib from "@clusterio/lib"; -import * as libBuildMod from "@clusterio/lib/dist/build_mod"; +import * as libBuildMod from "@clusterio/lib/dist/node/build_mod"; import type { FactorioServer } from "./server"; interface Prototype { @@ -62,7 +62,7 @@ async function generateExportMod(server: FactorioServer) { clean: false, build: true, pack: true, - sourceDir: path.join(__dirname, "..", "..", "lua", "export"), + sourceDir: path.join(__dirname, "..", "..", "..", "lua", "export"), outputDir: server.writePath("mods"), bumpPatch: false, factorioVersion: server.version.replace(/\.\d+$/, ""), diff --git a/packages/host/tsconfig.json b/packages/host/tsconfig.json index bc67208a..13df203e 100644 --- a/packages/host/tsconfig.json +++ b/packages/host/tsconfig.json @@ -5,6 +5,6 @@ ], "include": ["index.ts", "host.ts", "package.json", "src/**/*.ts", "src/**/*.js"], "compilerOptions": { - "outDir": "dist", + "outDir": "dist/node", }, } diff --git a/packages/lib/package.json b/packages/lib/package.json index 538f5d75..1c5cd2d6 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -8,7 +8,7 @@ "prepare": "tsc --build && pnpm run compile_validator", "compile_validator": "node scripts/compile_validator.js" }, - "main": "dist/index.js", + "main": "dist/node/index.js", "browser": "browser.ts", "keywords": [ "factorio" diff --git a/packages/lib/scripts/compile_validator.js b/packages/lib/scripts/compile_validator.js index e2eafeb3..9dd5fabd 100644 --- a/packages/lib/scripts/compile_validator.js +++ b/packages/lib/scripts/compile_validator.js @@ -7,9 +7,9 @@ const standaloneCode = require("ajv/dist/standalone").default; /* eslint-disable node/no-sync */ // Provide an empty file for message_core to load -fs.outputFileSync(path.join(__dirname, "../dist/src/data/message_validate.js"), ""); +fs.outputFileSync(path.join(__dirname, "../dist/node/src/data/message_validate.js"), ""); -const { Message } = require("../dist/src/data/messages_core"); +const { Message } = require("../dist/node/src/data/messages_core"); const ajv = new Ajv({ allowUnionTypes: true, @@ -28,5 +28,5 @@ function write(output) { `// generated by scripts/compile_validator.js\n// @ts-nocheck\n/* eslint-disable */\n${moduleCode}` ); } -write("../dist/src/data/message_validate.js"); +write("../dist/node/src/data/message_validate.js"); write("../src/data/message_validate.js"); diff --git a/packages/lib/tsconfig.json b/packages/lib/tsconfig.json index dbc005eb..d3dab4e1 100644 --- a/packages/lib/tsconfig.json +++ b/packages/lib/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../tsconfig.node.json", "include": ["index.ts", "browser.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"], "compilerOptions": { - "outDir": "dist", + "outDir": "dist/node", "allowJs": true, }, } diff --git a/plugins/global_chat/index.ts b/plugins/global_chat/index.ts index 620a8ecc..97c0985a 100644 --- a/plugins/global_chat/index.ts +++ b/plugins/global_chat/index.ts @@ -6,8 +6,8 @@ export const plugin: lib.PluginDeclaration = { name: "global_chat", title: "Global Chat", description: "Forwards chat between instances.", - instanceEntrypoint: "dist/plugin/instance", - ctlEntrypoint: "dist/plugin/control", + instanceEntrypoint: "dist/node/instance", + ctlEntrypoint: "dist/node/control", messages: [ ChatEvent, diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json index 09d9adb6..26518d2c 100644 --- a/plugins/global_chat/package.json +++ b/plugins/global_chat/package.json @@ -6,7 +6,7 @@ "homepage": "https://github.com/clusterio/clusterio#readme", "license": "MIT", "repository": "clusterio/clusterio", - "main": "dist/plugin/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/global_chat/test/plugin.js b/plugins/global_chat/test/plugin.js index d8b9b860..315df7eb 100644 --- a/plugins/global_chat/test/plugin.js +++ b/plugins/global_chat/test/plugin.js @@ -4,9 +4,9 @@ const lib = require("@clusterio/lib"); const mock = require("../../../test/mock"); const lines = require("../../../test/lib/factorio/lines"); -const instance = require("../dist/plugin/instance"); -const info = require("../dist/plugin/index").plugin; -const { ChatEvent } = require("../dist/plugin/messages"); +const instance = require("../dist/node/instance"); +const info = require("../dist/node/index").plugin; +const { ChatEvent } = require("../dist/node/messages"); describe("global_chat plugin", function() { diff --git a/plugins/global_chat/tsconfig.json b/plugins/global_chat/tsconfig.json index 087576d6..15815646 100644 --- a/plugins/global_chat/tsconfig.json +++ b/plugins/global_chat/tsconfig.json @@ -6,6 +6,6 @@ "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], "compilerOptions": { - "outDir": "dist/plugin", + "outDir": "dist/node", }, } diff --git a/plugins/inventory_sync/index.ts b/plugins/inventory_sync/index.ts index 029e3a46..a796ee97 100644 --- a/plugins/inventory_sync/index.ts +++ b/plugins/inventory_sync/index.ts @@ -22,7 +22,7 @@ export const plugin: lib.PluginDeclaration = { title: "Inventory sync", description: "Synchronizes players inventories between instances", - instanceEntrypoint: "dist/plugin/instance", + instanceEntrypoint: "dist/node/instance", instanceConfigFields: { "inventory_sync.rcon_chunk_size": { title: "Rcon inventory chunk size", @@ -33,7 +33,7 @@ export const plugin: lib.PluginDeclaration = { }, }, - controllerEntrypoint: "dist/plugin/controller", + controllerEntrypoint: "dist/node/controller", controllerConfigFields: { "inventory_sync.player_lock_timeout": { title: "Player Lock Timeout", diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index 384a64dd..2bf25a2a 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -6,7 +6,7 @@ "homepage": "https://github.com/clusterio/factorioClusterio#readme", "license": "MIT", "repository": "clusterio/factorioClusterio", - "main": "dist/plugin/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/inventory_sync/tsconfig.json b/plugins/inventory_sync/tsconfig.json index 087576d6..15815646 100644 --- a/plugins/inventory_sync/tsconfig.json +++ b/plugins/inventory_sync/tsconfig.json @@ -6,6 +6,6 @@ "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], "compilerOptions": { - "outDir": "dist/plugin", + "outDir": "dist/node", }, } diff --git a/plugins/player_auth/index.ts b/plugins/player_auth/index.ts index a181c508..62e7b109 100644 --- a/plugins/player_auth/index.ts +++ b/plugins/player_auth/index.ts @@ -15,8 +15,8 @@ export const plugin: lib.PluginDeclaration = { name: "player_auth", title: "Player Auth", description: "Provides authentication to the cluster via logging into a Factorio server.", - controllerEntrypoint: "dist/plugin/controller", - instanceEntrypoint: "dist/plugin/instance", + controllerEntrypoint: "dist/node/controller", + instanceEntrypoint: "dist/node/instance", webEntrypoint: "./web", controllerConfigFields: { "player_auth.code_length": { diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index 8cf5b5c9..733ebf7a 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -6,7 +6,7 @@ "homepage": "https://github.com/clusterio/clusterio#readme", "license": "MIT", "repository": "clusterio/clusterio", - "main": "dist/plugin/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/player_auth/test/plugin.js b/plugins/player_auth/test/plugin.js index 8dc59142..6b90348e 100644 --- a/plugins/player_auth/test/plugin.js +++ b/plugins/player_auth/test/plugin.js @@ -5,10 +5,10 @@ const jwt = require("jsonwebtoken"); const mock = require("../../../test/mock"); -const controller = require("../dist/plugin/controller"); -const instance = require("../dist/plugin/instance"); -const info = require("../dist/plugin/index").plugin; -const { FetchPlayerCodeRequest, SetVerifyCodeRequest } = require("../dist/plugin/messages"); +const controller = require("../dist/node/controller"); +const instance = require("../dist/node/instance"); +const info = require("../dist/node/index").plugin; +const { FetchPlayerCodeRequest, SetVerifyCodeRequest } = require("../dist/node/messages"); const lib = require("@clusterio/lib"); diff --git a/plugins/player_auth/tsconfig.json b/plugins/player_auth/tsconfig.json index 087576d6..15815646 100644 --- a/plugins/player_auth/tsconfig.json +++ b/plugins/player_auth/tsconfig.json @@ -6,6 +6,6 @@ "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], "compilerOptions": { - "outDir": "dist/plugin", + "outDir": "dist/node", }, } diff --git a/plugins/research_sync/index.ts b/plugins/research_sync/index.ts index af7c8364..42f162d5 100644 --- a/plugins/research_sync/index.ts +++ b/plugins/research_sync/index.ts @@ -6,8 +6,8 @@ export const plugin: lib.PluginDeclaration = { name: "research_sync", title: "Research Sync", description: "Synchronises technology research progress between instances.", - instanceEntrypoint: "dist/plugin/instance", - controllerEntrypoint: "dist/plugin/controller", + instanceEntrypoint: "dist/node/instance", + controllerEntrypoint: "dist/node/controller", messages: [ messages.ContributionEvent, diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json index fb9d4424..f5a74600 100644 --- a/plugins/research_sync/package.json +++ b/plugins/research_sync/package.json @@ -6,7 +6,7 @@ "homepage": "https://github.com/clusterio/clusterio#readme", "license": "MIT", "repository": "clusterio/clusterio", - "main": "dist/plugin/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/research_sync/test/missing.js b/plugins/research_sync/test/missing.js index 790baadb..ae16e8f1 100644 --- a/plugins/research_sync/test/missing.js +++ b/plugins/research_sync/test/missing.js @@ -2,5 +2,5 @@ // required here in order to track coverage for them. "use strict"; -require("../dist/plugin/instance"); -require("../dist/plugin/controller"); +require("../dist/node/instance"); +require("../dist/node/controller"); diff --git a/plugins/research_sync/tsconfig.json b/plugins/research_sync/tsconfig.json index 087576d6..15815646 100644 --- a/plugins/research_sync/tsconfig.json +++ b/plugins/research_sync/tsconfig.json @@ -6,6 +6,6 @@ "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], "compilerOptions": { - "outDir": "dist/plugin", + "outDir": "dist/node", }, } diff --git a/plugins/statistics_exporter/index.ts b/plugins/statistics_exporter/index.ts index 8748224c..5de5fc80 100644 --- a/plugins/statistics_exporter/index.ts +++ b/plugins/statistics_exporter/index.ts @@ -12,7 +12,7 @@ export const plugin: lib.PluginDeclaration = { description: "Provides in-game item/fluid production, builds, kills, and pollution "+ "statistics to the cluster's Prometheus endpoint.", - instanceEntrypoint: "dist/plugin/instance", + instanceEntrypoint: "dist/node/instance", instanceConfigFields: { "statistics_exporter.command_timeout": { title: "Command Timeout", diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json index c8e5a12f..21c9c87f 100644 --- a/plugins/statistics_exporter/package.json +++ b/plugins/statistics_exporter/package.json @@ -6,7 +6,7 @@ "homepage": "https://github.com/clusterio/clusterio#readme", "license": "MIT", "repository": "clusterio/clusterio", - "main": "dist/plugin/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/statistics_exporter/test/plugin.js b/plugins/statistics_exporter/test/plugin.js index e54184a4..18251e27 100644 --- a/plugins/statistics_exporter/test/plugin.js +++ b/plugins/statistics_exporter/test/plugin.js @@ -2,8 +2,8 @@ const assert = require("assert").strict; const mock = require("../../../test/mock"); -const instance = require("../dist/plugin/instance"); -const info = require("../dist/plugin/index").plugin; +const instance = require("../dist/node/instance"); +const info = require("../dist/node/index").plugin; describe("statistics_exporter plugin", function() { diff --git a/plugins/statistics_exporter/tsconfig.json b/plugins/statistics_exporter/tsconfig.json index 087576d6..15815646 100644 --- a/plugins/statistics_exporter/tsconfig.json +++ b/plugins/statistics_exporter/tsconfig.json @@ -6,6 +6,6 @@ "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], "compilerOptions": { - "outDir": "dist/plugin", + "outDir": "dist/node", }, } diff --git a/plugins/subspace_storage/index.ts b/plugins/subspace_storage/index.ts index 6543cd6c..1909ef29 100644 --- a/plugins/subspace_storage/index.ts +++ b/plugins/subspace_storage/index.ts @@ -22,7 +22,7 @@ export const plugin: lib.PluginDeclaration = { name: "subspace_storage", title: "Subspace Storage", description: "Provides shared storage across instances for the Subspace Storage mod", - instanceEntrypoint: "dist/plugin/instance", + instanceEntrypoint: "dist/node/instance", instanceConfigFields: { "subspace_storage.log_item_transfers": { title: "Log Item Transfers", @@ -32,7 +32,7 @@ export const plugin: lib.PluginDeclaration = { }, }, - controllerEntrypoint: "dist/plugin/controller", + controllerEntrypoint: "dist/node/controller", controllerConfigFields: { "subspace_storage.division_method": { title: "Division Method", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index 892e9ca9..5dce5e9b 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -6,7 +6,7 @@ "homepage": "https://github.com/clusterio/clusterio#readme", "license": "MIT", "repository": "clusterio/clusterio", - "main": "dist/plugin/index.js", + "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/subspace_storage/test/missing.js b/plugins/subspace_storage/test/missing.js index 6f7e6a1b..ab3009d7 100644 --- a/plugins/subspace_storage/test/missing.js +++ b/plugins/subspace_storage/test/missing.js @@ -2,7 +2,7 @@ // required here in order to track coverage for them. "use strict"; -require("../dist/plugin/instance"); -require("../dist/plugin/controller"); -require("../dist/plugin/dole"); -require("../dist/plugin/dole_nn_base"); +require("../dist/node/instance"); +require("../dist/node/controller"); +require("../dist/node/dole"); +require("../dist/node/dole_nn_base"); diff --git a/plugins/subspace_storage/test/routes.js b/plugins/subspace_storage/test/routes.js index 00de92b0..2753b41e 100644 --- a/plugins/subspace_storage/test/routes.js +++ b/plugins/subspace_storage/test/routes.js @@ -1,7 +1,7 @@ "use strict"; const assert = require("assert").strict; -const routes = require("../dist/plugin/routes"); +const routes = require("../dist/node/routes"); const { get } = require("../../../test/integration"); diff --git a/plugins/subspace_storage/tsconfig.json b/plugins/subspace_storage/tsconfig.json index 087576d6..15815646 100644 --- a/plugins/subspace_storage/tsconfig.json +++ b/plugins/subspace_storage/tsconfig.json @@ -6,6 +6,6 @@ "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], "compilerOptions": { - "outDir": "dist/plugin", + "outDir": "dist/node", }, } diff --git a/test/controller/HttpCloser.js b/test/controller/HttpCloser.js index 6c48ba2a..d630600b 100644 --- a/test/controller/HttpCloser.js +++ b/test/controller/HttpCloser.js @@ -7,7 +7,7 @@ const util = require("util"); const phin = require("phin"); -const HttpCloser = require("@clusterio/controller/dist/src/HttpCloser").default; +const HttpCloser = require("@clusterio/controller/dist/node/src/HttpCloser").default; // Time to wait during async operations to ensure they happened in order. const tick = 20; diff --git a/test/controller/routes.js b/test/controller/routes.js index a598f276..e69449ca 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -7,7 +7,7 @@ const http = require("http"); const phin = require("phin"); const { wait } = require("@clusterio/lib"); -const routes = require("@clusterio/controller/dist/src/routes"); +const routes = require("@clusterio/controller/dist/node/src/routes"); const mock = require("../mock"); diff --git a/test/host.js b/test/host.js index 3aa8162e..75244b9e 100644 --- a/test/host.js +++ b/test/host.js @@ -3,8 +3,8 @@ const assert = require("assert").strict; const path = require("path"); const lib = require("@clusterio/lib"); -const Host = require("@clusterio/host/dist/src/Host").default; -const { _discoverInstances } = require("@clusterio/host/dist/src/Host"); +const Host = require("@clusterio/host/dist/node/src/Host").default; +const { _discoverInstances } = require("@clusterio/host/dist/node/src/Host"); describe("Host testing", function() { describe("discoverInstances()", function() { diff --git a/test/host/Instance.js b/test/host/Instance.js index b899a335..9df6a598 100644 --- a/test/host/Instance.js +++ b/test/host/Instance.js @@ -4,7 +4,7 @@ const path = require("path"); const lib = require("@clusterio/lib"); const { PlayerStats, wait } = lib; -const Instance = require("@clusterio/host/dist/src/Instance").default; +const Instance = require("@clusterio/host/dist/node/src/Instance").default; const { MockConnector, MockServer } = require("../mock"); const addr = lib.Address.fromShorthand; diff --git a/test/host/export.js b/test/host/export.js index f99b53ba..b0de6a0a 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -2,8 +2,8 @@ const assert = require("assert").strict; const path = require("path"); -const { FactorioServer } = require("@clusterio/host/dist/src/server"); -const { _exportLocale } = require("@clusterio/host/dist/src/export"); +const { FactorioServer } = require("@clusterio/host/dist/node/src/server"); +const { _exportLocale } = require("@clusterio/host/dist/node/src/export"); describe("host/src/export", function() { diff --git a/test/host/patch.js b/test/host/patch.js index 9998bccc..8655aabd 100644 --- a/test/host/patch.js +++ b/test/host/patch.js @@ -5,7 +5,7 @@ const JSZip = require("jszip"); const path = require("path"); const lib = require("@clusterio/lib"); -const patch = require("@clusterio/host/dist/src/patch"); +const patch = require("@clusterio/host/dist/node/src/patch"); describe("host/patch", function() { diff --git a/test/host/server.js b/test/host/server.js index 12ade2e0..ea92a06c 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -4,7 +4,7 @@ const events = require("events"); const fs = require("fs-extra"); const path = require("path"); -const hostServer = require("@clusterio/host/dist/src/server"); +const hostServer = require("@clusterio/host/dist/node/src/server"); const lib = require("@clusterio/lib"); const { wait } = lib; const { testLines } = require("../lib/factorio/lines"); diff --git a/test/integration/export.js b/test/integration/export.js index 3ab962cc..a0d8345a 100644 --- a/test/integration/export.js +++ b/test/integration/export.js @@ -2,8 +2,8 @@ const assert = require("assert").strict; const path = require("path"); -const { _exportLocale } = require("@clusterio/host/dist/src/export"); -const { FactorioServer } = require("@clusterio/host/dist/src/server"); +const { _exportLocale } = require("@clusterio/host/dist/node/src/export"); +const { FactorioServer } = require("@clusterio/host/dist/node/src/server"); describe("Integration of lib/factorio/export", function() { diff --git a/test/integration/index.js b/test/integration/index.js index 6932e283..22450cbd 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -12,11 +12,11 @@ const lib = require("@clusterio/lib"); const { LineSplitter, ConsoleTransport, logger } = lib; // Make sure permissions from plugins are loaded -require("../../plugins/global_chat/dist/plugin/index"); -require("../../plugins/player_auth/dist/plugin/index"); -require("../../plugins/research_sync/dist/plugin/index"); -require("../../plugins/statistics_exporter/dist/plugin/index"); -require("../../plugins/subspace_storage/dist/plugin/index"); +require("../../plugins/global_chat/dist/node/index"); +require("../../plugins/player_auth/dist/node/index"); +require("../../plugins/research_sync/dist/node/index"); +require("../../plugins/statistics_exporter/dist/node/index"); +require("../../plugins/subspace_storage/dist/node/index"); const subspaceStorageVersion = "1.99.8"; const subspaceStorageGitHubTag = "1.99.6"; // Tag subspaceStorageVersion was released under on GitHub diff --git a/test/integration/patch.js b/test/integration/patch.js index bfc1f224..3b2cc72d 100644 --- a/test/integration/patch.js +++ b/test/integration/patch.js @@ -4,7 +4,7 @@ const fs = require("fs-extra"); const JSZip = require("jszip"); const path = require("path"); -const { patch, SaveModule } = require("@clusterio/host/dist/src/patch"); +const { patch, SaveModule } = require("@clusterio/host/dist/node/src/patch"); const lib = require("@clusterio/lib"); const { slowTest } = require("./index"); diff --git a/test/integration/server.js b/test/integration/server.js index 0aeac758..9073bbb8 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -5,7 +5,7 @@ const fs = require("fs-extra"); const path = require("path"); const lib = require("@clusterio/lib"); -const { FactorioServer, _getVersion } = require("@clusterio/host/dist/src/server"); +const { FactorioServer, _getVersion } = require("@clusterio/host/dist/node/src/server"); const { logger } = lib; const { slowTest } = require("./index"); diff --git a/test/mock.js b/test/mock.js index 4d2a8564..1c1fb84b 100644 --- a/test/mock.js +++ b/test/mock.js @@ -5,8 +5,8 @@ const express = require("express"); const lib = require("@clusterio/lib"); -const UserManager = require("@clusterio/controller/dist/src/UserManager").default; -const ControllerUser = require("@clusterio/controller/dist/src/ControllerUser").default; +const UserManager = require("@clusterio/controller/dist/node/src/UserManager").default; +const ControllerUser = require("@clusterio/controller/dist/node/src/ControllerUser").default; const addr = lib.Address.fromShorthand; From e4f90b348036b5b8039930bb49952327617c254b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Feb 2024 19:17:11 +0100 Subject: [PATCH 35/68] Move tsconfig.json to tsconfig.node.json Distinguish the Node.js build in the same way the browser build is with the .node suffix to the tsconfig files it uses. This means tsconfig.node outputs to dist/node and tsconfig.browser outputs to dist/browser. --- .../{tsconfig.json => tsconfig.node.json} | 2 +- .../ctl/{tsconfig.json => tsconfig.node.json} | 2 +- .../{tsconfig.json => tsconfig.node.json} | 2 +- .../lib/{tsconfig.json => tsconfig.node.json} | 0 .../tsconfig.node.json} | 2 +- .../tsconfig.node.json} | 2 +- .../tsconfig.node.json} | 2 +- .../tsconfig.node.json} | 2 +- plugins/statistics_exporter/tsconfig.json | 11 ---------- .../statistics_exporter/tsconfig.node.json | 11 ++++++++++ plugins/subspace_storage/tsconfig.json | 11 ---------- plugins/subspace_storage/tsconfig.node.json | 11 ++++++++++ tsconfig.json | 20 +++++++++---------- 13 files changed, 39 insertions(+), 39 deletions(-) rename packages/controller/{tsconfig.json => tsconfig.node.json} (81%) rename packages/ctl/{tsconfig.json => tsconfig.node.json} (80%) rename packages/host/{tsconfig.json => tsconfig.node.json} (82%) rename packages/lib/{tsconfig.json => tsconfig.node.json} (100%) rename plugins/{inventory_sync/tsconfig.json => global_chat/tsconfig.node.json} (76%) rename plugins/{player_auth/tsconfig.json => inventory_sync/tsconfig.node.json} (76%) rename plugins/{research_sync/tsconfig.json => player_auth/tsconfig.node.json} (76%) rename plugins/{global_chat/tsconfig.json => research_sync/tsconfig.node.json} (76%) delete mode 100644 plugins/statistics_exporter/tsconfig.json create mode 100644 plugins/statistics_exporter/tsconfig.node.json delete mode 100644 plugins/subspace_storage/tsconfig.json create mode 100644 plugins/subspace_storage/tsconfig.node.json diff --git a/packages/controller/tsconfig.json b/packages/controller/tsconfig.node.json similarity index 81% rename from packages/controller/tsconfig.json rename to packages/controller/tsconfig.node.json index 8dcc9b8b..9dcc91db 100644 --- a/packages/controller/tsconfig.json +++ b/packages/controller/tsconfig.node.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.node.json", "references": [ - { "path": "../lib" }, + { "path": "../lib/tsconfig.node.json" }, ], "include": ["index.ts", "controller.ts", "package.json", "src/**/*.ts"], "compilerOptions": { diff --git a/packages/ctl/tsconfig.json b/packages/ctl/tsconfig.node.json similarity index 80% rename from packages/ctl/tsconfig.json rename to packages/ctl/tsconfig.node.json index 134019a5..3a0fb105 100644 --- a/packages/ctl/tsconfig.json +++ b/packages/ctl/tsconfig.node.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.node.json", "references": [ - { "path": "../lib" }, + { "path": "../lib/tsconfig.node.json" }, ], "include": ["index.ts", "package.json", "ctl.ts", "src/**/*.ts"], "compilerOptions": { diff --git a/packages/host/tsconfig.json b/packages/host/tsconfig.node.json similarity index 82% rename from packages/host/tsconfig.json rename to packages/host/tsconfig.node.json index 13df203e..e7a488be 100644 --- a/packages/host/tsconfig.json +++ b/packages/host/tsconfig.node.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.node.json", "references": [ - { "path": "../lib" }, + { "path": "../lib/tsconfig.node.json" }, ], "include": ["index.ts", "host.ts", "package.json", "src/**/*.ts", "src/**/*.js"], "compilerOptions": { diff --git a/packages/lib/tsconfig.json b/packages/lib/tsconfig.node.json similarity index 100% rename from packages/lib/tsconfig.json rename to packages/lib/tsconfig.node.json diff --git a/plugins/inventory_sync/tsconfig.json b/plugins/global_chat/tsconfig.node.json similarity index 76% rename from plugins/inventory_sync/tsconfig.json rename to plugins/global_chat/tsconfig.node.json index 15815646..b6197500 100644 --- a/plugins/inventory_sync/tsconfig.json +++ b/plugins/global_chat/tsconfig.node.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.node.json", "references": [ - { "path": "../../packages/lib" }, + { "path": "../../packages/lib/tsconfig.node.json" }, ], "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], diff --git a/plugins/player_auth/tsconfig.json b/plugins/inventory_sync/tsconfig.node.json similarity index 76% rename from plugins/player_auth/tsconfig.json rename to plugins/inventory_sync/tsconfig.node.json index 15815646..b6197500 100644 --- a/plugins/player_auth/tsconfig.json +++ b/plugins/inventory_sync/tsconfig.node.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.node.json", "references": [ - { "path": "../../packages/lib" }, + { "path": "../../packages/lib/tsconfig.node.json" }, ], "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], diff --git a/plugins/research_sync/tsconfig.json b/plugins/player_auth/tsconfig.node.json similarity index 76% rename from plugins/research_sync/tsconfig.json rename to plugins/player_auth/tsconfig.node.json index 15815646..b6197500 100644 --- a/plugins/research_sync/tsconfig.json +++ b/plugins/player_auth/tsconfig.node.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.node.json", "references": [ - { "path": "../../packages/lib" }, + { "path": "../../packages/lib/tsconfig.node.json" }, ], "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], diff --git a/plugins/global_chat/tsconfig.json b/plugins/research_sync/tsconfig.node.json similarity index 76% rename from plugins/global_chat/tsconfig.json rename to plugins/research_sync/tsconfig.node.json index 15815646..b6197500 100644 --- a/plugins/global_chat/tsconfig.json +++ b/plugins/research_sync/tsconfig.node.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.node.json", "references": [ - { "path": "../../packages/lib" }, + { "path": "../../packages/lib/tsconfig.node.json" }, ], "include": ["./**/*.ts"], "exclude": ["test/*", "./dist/*"], diff --git a/plugins/statistics_exporter/tsconfig.json b/plugins/statistics_exporter/tsconfig.json deleted file mode 100644 index 15815646..00000000 --- a/plugins/statistics_exporter/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.node.json", - "references": [ - { "path": "../../packages/lib" }, - ], - "include": ["./**/*.ts"], - "exclude": ["test/*", "./dist/*"], - "compilerOptions": { - "outDir": "dist/node", - }, -} diff --git a/plugins/statistics_exporter/tsconfig.node.json b/plugins/statistics_exporter/tsconfig.node.json new file mode 100644 index 00000000..b6197500 --- /dev/null +++ b/plugins/statistics_exporter/tsconfig.node.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.node.json", + "references": [ + { "path": "../../packages/lib/tsconfig.node.json" }, + ], + "include": ["./**/*.ts"], + "exclude": ["test/*", "./dist/*"], + "compilerOptions": { + "outDir": "dist/node", + }, +} diff --git a/plugins/subspace_storage/tsconfig.json b/plugins/subspace_storage/tsconfig.json deleted file mode 100644 index 15815646..00000000 --- a/plugins/subspace_storage/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.node.json", - "references": [ - { "path": "../../packages/lib" }, - ], - "include": ["./**/*.ts"], - "exclude": ["test/*", "./dist/*"], - "compilerOptions": { - "outDir": "dist/node", - }, -} diff --git a/plugins/subspace_storage/tsconfig.node.json b/plugins/subspace_storage/tsconfig.node.json new file mode 100644 index 00000000..b6197500 --- /dev/null +++ b/plugins/subspace_storage/tsconfig.node.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.node.json", + "references": [ + { "path": "../../packages/lib/tsconfig.node.json" }, + ], + "include": ["./**/*.ts"], + "exclude": ["test/*", "./dist/*"], + "compilerOptions": { + "outDir": "dist/node", + }, +} diff --git a/tsconfig.json b/tsconfig.json index fbb6a7ec..e108f361 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,16 +1,16 @@ { "files": [], "references": [ - { "path": "./packages/lib" }, - { "path": "./packages/controller" }, - { "path": "./packages/host" }, - { "path": "./packages/ctl" }, - { "path": "./plugins/global_chat" }, - { "path": "./plugins/inventory_sync" }, - { "path": "./plugins/player_auth" }, - { "path": "./plugins/research_sync" }, - { "path": "./plugins/statistics_exporter" }, - { "path": "./plugins/subspace_storage" }, + { "path": "./packages/lib/tsconfig.node.json" }, + { "path": "./packages/controller/tsconfig.node.json" }, + { "path": "./packages/host/tsconfig.node.json" }, + { "path": "./packages/ctl/tsconfig.node.json" }, + { "path": "./plugins/global_chat/tsconfig.node.json" }, + { "path": "./plugins/inventory_sync/tsconfig.node.json" }, + { "path": "./plugins/player_auth/tsconfig.node.json" }, + { "path": "./plugins/research_sync/tsconfig.node.json" }, + { "path": "./plugins/statistics_exporter/tsconfig.node.json" }, + { "path": "./plugins/subspace_storage/tsconfig.node.json" }, { "path": "./packages/lib/tsconfig.browser.json" }, { "path": "./packages/controller/tsconfig.browser.json" }, { "path": "./packages/web_ui/tsconfig.browser.json" }, From 597d080d71577f9e3ff85e4fd0236afcbf1eff38 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Feb 2024 19:29:49 +0100 Subject: [PATCH 36/68] Add tsconfig.json that references both builds Add back unqualified tsconfig.json files to each package that references both the Node.js build and the browser build. This makes tsc --build compile both builds when the variant to do is not specified. --- packages/controller/tsconfig.json | 7 ++++++ packages/ctl/tsconfig.json | 6 +++++ packages/host/tsconfig.json | 6 +++++ packages/lib/tsconfig.json | 7 ++++++ packages/web_ui/tsconfig.json | 6 +++++ plugins/global_chat/tsconfig.json | 7 ++++++ plugins/inventory_sync/tsconfig.json | 7 ++++++ plugins/player_auth/tsconfig.json | 7 ++++++ plugins/research_sync/tsconfig.json | 7 ++++++ plugins/statistics_exporter/tsconfig.json | 7 ++++++ plugins/subspace_storage/tsconfig.json | 7 ++++++ tsconfig.json | 30 +++++++++-------------- 12 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 packages/controller/tsconfig.json create mode 100644 packages/ctl/tsconfig.json create mode 100644 packages/host/tsconfig.json create mode 100644 packages/lib/tsconfig.json create mode 100644 packages/web_ui/tsconfig.json create mode 100644 plugins/global_chat/tsconfig.json create mode 100644 plugins/inventory_sync/tsconfig.json create mode 100644 plugins/player_auth/tsconfig.json create mode 100644 plugins/research_sync/tsconfig.json create mode 100644 plugins/statistics_exporter/tsconfig.json create mode 100644 plugins/subspace_storage/tsconfig.json diff --git a/packages/controller/tsconfig.json b/packages/controller/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/packages/controller/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/packages/ctl/tsconfig.json b/packages/ctl/tsconfig.json new file mode 100644 index 00000000..20b10c53 --- /dev/null +++ b/packages/ctl/tsconfig.json @@ -0,0 +1,6 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + ], +} diff --git a/packages/host/tsconfig.json b/packages/host/tsconfig.json new file mode 100644 index 00000000..20b10c53 --- /dev/null +++ b/packages/host/tsconfig.json @@ -0,0 +1,6 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + ], +} diff --git a/packages/lib/tsconfig.json b/packages/lib/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/packages/lib/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/packages/web_ui/tsconfig.json b/packages/web_ui/tsconfig.json new file mode 100644 index 00000000..d96583bf --- /dev/null +++ b/packages/web_ui/tsconfig.json @@ -0,0 +1,6 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/plugins/global_chat/tsconfig.json b/plugins/global_chat/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/plugins/global_chat/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/plugins/inventory_sync/tsconfig.json b/plugins/inventory_sync/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/plugins/inventory_sync/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/plugins/player_auth/tsconfig.json b/plugins/player_auth/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/plugins/player_auth/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/plugins/research_sync/tsconfig.json b/plugins/research_sync/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/plugins/research_sync/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/plugins/statistics_exporter/tsconfig.json b/plugins/statistics_exporter/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/plugins/statistics_exporter/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/plugins/subspace_storage/tsconfig.json b/plugins/subspace_storage/tsconfig.json new file mode 100644 index 00000000..e074e85f --- /dev/null +++ b/plugins/subspace_storage/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.browser.json" }, + ], +} diff --git a/tsconfig.json b/tsconfig.json index e108f361..df2b7168 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,24 +1,16 @@ { "files": [], "references": [ - { "path": "./packages/lib/tsconfig.node.json" }, - { "path": "./packages/controller/tsconfig.node.json" }, - { "path": "./packages/host/tsconfig.node.json" }, - { "path": "./packages/ctl/tsconfig.node.json" }, - { "path": "./plugins/global_chat/tsconfig.node.json" }, - { "path": "./plugins/inventory_sync/tsconfig.node.json" }, - { "path": "./plugins/player_auth/tsconfig.node.json" }, - { "path": "./plugins/research_sync/tsconfig.node.json" }, - { "path": "./plugins/statistics_exporter/tsconfig.node.json" }, - { "path": "./plugins/subspace_storage/tsconfig.node.json" }, - { "path": "./packages/lib/tsconfig.browser.json" }, - { "path": "./packages/controller/tsconfig.browser.json" }, - { "path": "./packages/web_ui/tsconfig.browser.json" }, - { "path": "./plugins/global_chat/tsconfig.browser.json" }, - { "path": "./plugins/inventory_sync/tsconfig.browser.json" }, - { "path": "./plugins/player_auth/tsconfig.browser.json" }, - { "path": "./plugins/research_sync/tsconfig.browser.json" }, - { "path": "./plugins/statistics_exporter/tsconfig.browser.json" }, - { "path": "./plugins/subspace_storage/tsconfig.browser.json" }, + { "path": "./packages/lib/" }, + { "path": "./packages/controller/" }, + { "path": "./packages/host/" }, + { "path": "./packages/ctl/" }, + { "path": "./packages/web_ui/" }, + { "path": "./plugins/global_chat/" }, + { "path": "./plugins/inventory_sync/" }, + { "path": "./plugins/player_auth/" }, + { "path": "./plugins/research_sync/" }, + { "path": "./plugins/statistics_exporter/" }, + { "path": "./plugins/subspace_storage/" }, ], } From ebb93ada9453b08ff6f836cbf129bb8b41c1bdb7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 8 Feb 2024 03:02:35 +0100 Subject: [PATCH 37/68] Replace babel and ts-loader with swc Build and minify the Web UI using swc instead of babel, tsc and terser. This results in a significant speedup of the build process as it skips checking types and uses native code to do the minification. To prevent accidental type errors being introduced the TypeScript compiler is still used to perform type checking on the Web UI files. Due to peer dependency resolving shenanigans the swc/core package had to be added to the workspace, controller and web_ui to avoid pnpm creating a duplicate install of webpack that break the build. --- .npmrc | 1 + package.json | 1 + packages/controller/package.json | 1 + packages/web_ui/package.json | 6 ++--- packages/web_ui/webpack.common.js | 45 ++++++++++++++++++++----------- tsconfig.browser.json | 2 +- 6 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 .npmrc diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000..f301fedf --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +auto-install-peers=false diff --git a/package.json b/package.json index 336edd58..fbf50444 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@clusterio/ctl": "workspace:*", "@clusterio/host": "workspace:*", "@clusterio/lib": "workspace:*", + "@swc/core": "^1.4.0", "@typescript-eslint/eslint-plugin": "^6.4.1", "@typescript-eslint/parser": "^6.4.1", "eslint": "^8.45.0", diff --git a/packages/controller/package.json b/packages/controller/package.json index 7446edbd..97fd5f36 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -36,6 +36,7 @@ }, "devDependencies": { "@clusterio/web_ui": "^2.0.0-alpha.16", + "@swc/core": "^1.4.0", "@types/busboy": "^1.5.0", "@types/compression": "^1.7.2", "@types/express": "^4.17.17", diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index fb77d82d..a1f2575a 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -17,12 +17,10 @@ }, "dependencies": { "@ant-design/icons": "^5.1.4", - "@babel/core": "^7.22.9", - "@babel/preset-react": "^7.22.5", "@clusterio/lib": "^2.0.0-alpha.16", + "@swc/core": "^1.4.0", "antd": "^5.13.0", "assert": "^2.0.0", - "babel-loader": "^9.1.3", "browserify-zlib": "^0.2.0", "buffer": "^6.0.3", "clean-webpack-plugin": "^4.0.0", @@ -39,8 +37,8 @@ "react-router-dom": "^6.14.2", "stream-browserify": "^3.0.0", "style-loader": "^3.3.3", + "swc-loader": "^0.2.6", "terser-webpack-plugin": "^5.3.9", - "ts-loader": "^9.4.4", "typescript": "^5.1.6", "util": "^0.12.5", "webpack": "^5.88.2", diff --git a/packages/web_ui/webpack.common.js b/packages/web_ui/webpack.common.js index 99825c4b..73cae8e8 100644 --- a/packages/web_ui/webpack.common.js +++ b/packages/web_ui/webpack.common.js @@ -65,32 +65,47 @@ module.exports = (env = {}) => ({ }, ], }, - { - test: /\.js$/, - resolve: { - // XXX Required due to bug in babel-runtime - // see: https://github.com/babel/babel/issues/12058 - fullySpecified: false, - }, - }, { test: /\.tsx?$/, exclude: /node_modules/, use: { - loader: require.resolve("ts-loader"), + loader: require.resolve("swc-loader"), options: { - configFile: "tsconfig.browser.json", - projectReferences: true, + sourceMaps: true, + jsc: { + parser: { + syntax: "typescript", + tsx: true, + }, + target: "es2022", + transform: { + react: { + runtime: "automatic", + }, + }, + }, }, }, }, { - test: /\.jsx$/, + test: /\.jsx?$/, exclude: /node_modules/, use: { - loader: require.resolve("babel-loader"), + loader: require.resolve("swc-loader"), options: { - presets: [require.resolve("@babel/preset-react")], + sourceMaps: true, + jsc: { + parser: { + syntax: "ecmascript", + jsx: true, + }, + target: "es2022", + transform: { + react: { + runtime: "automatic", + }, + }, + }, }, }, }, @@ -123,10 +138,10 @@ module.exports = (env = {}) => ({ moduleIds: "deterministic", minimizer: [ new TerserPlugin({ + minify: TerserPlugin.swcMinify, terserOptions: { compress: { keep_classnames: true, - passes: 2, }, mangle: { keep_classnames: true, diff --git a/tsconfig.browser.json b/tsconfig.browser.json index 7edb5691..605c2c61 100644 --- a/tsconfig.browser.json +++ b/tsconfig.browser.json @@ -5,7 +5,7 @@ "module": "es2022", "moduleResolution": "bundler", "target": "es2022", - "jsx": "react", + "jsx": "react-jsx", "noEmitOnError": true, }, } From f70b4f74b160cdbd596ddf7bcac5fcb31bab23e8 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 8 Feb 2024 05:23:48 +0100 Subject: [PATCH 38/68] Add watch script to rebuild on changes --- docs/contributing.md | 3 +++ package.json | 1 + 2 files changed, 4 insertions(+) diff --git a/docs/contributing.md b/docs/contributing.md index 44582f4a..eb5b82d1 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -81,6 +81,9 @@ Note that to use `clusterioctl` you will have to create a control config first: Once you've set up the cluster you can use `node packages/controller run` to run the controller and `node packages/host run` to run the host which connects to the controller. +When editing TypeScript files runned on Node.js you need run `pnpm build` in order to compile the files to JavaScript before restarting the controller/host/ctl for changes take effect. +To have TypeScript watch for changes and rebuild immediadly you may run `pnpm watch` instead. + For web development on the controller there are also the following flags: `--dev` - Recompile the `web_ui` package on the fly providing live reloading of it. diff --git a/package.json b/package.json index fbf50444..a5f4a506 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "private": true, "scripts": { "build": "tsc --build", + "watch": "tsc --build --watch", "test": "pnpm run build && mocha --check-leaks -R spec --recursive test \"plugins/*/test/**\"", "fast-test": "FAST_TEST=y pnpm test", "cover": "nyc pnpm test", From 96082e32b7e25c0393057c7b78a7b2c71b5d1a31 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 7 Feb 2024 16:18:33 +0100 Subject: [PATCH 39/68] Replace deprecated usage of Select bordered --- packages/web_ui/src/components/ModPackViewPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web_ui/src/components/ModPackViewPage.tsx b/packages/web_ui/src/components/ModPackViewPage.tsx index 97f13d03..840155f2 100644 --- a/packages/web_ui/src/components/ModPackViewPage.tsx +++ b/packages/web_ui/src/components/ModPackViewPage.tsx @@ -202,7 +202,7 @@ function SearchModsTable(props: SearchModsTableProps) { align: "right", render: (_, result) => + @@ -97,7 +97,9 @@ export default function UsersPage() { title: "Roles", key: "roles", render: (_, user) => ( - [...user.roleIds].map(id => {(roles.get(id) || { name: id }).name}) + [...user.roleIds].map(id => e.stopPropagation()}> + {(roles.get(id) || { name: id }).name} + ) ), }, { From 89919f7a891aabcab682873d2bd708504cd40104 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 9 Feb 2024 06:49:17 +0300 Subject: [PATCH 46/68] Fix inconsistent indentation --- packages/web_ui/src/components/InstanceViewPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/web_ui/src/components/InstanceViewPage.tsx b/packages/web_ui/src/components/InstanceViewPage.tsx index 5597e206..c1c4e018 100644 --- a/packages/web_ui/src/components/InstanceViewPage.tsx +++ b/packages/web_ui/src/components/InstanceViewPage.tsx @@ -159,8 +159,8 @@ function InstanceButtons(props: { instance: lib.InstanceDetails }) { "core.instance.kill", "core.instance.delete", ) && - - } + + } ; } From 4aca5050b92d1ac89d5f65ded23b30007550368c Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Tue, 6 Feb 2024 09:25:19 +0300 Subject: [PATCH 47/68] Allow sending instance management messages from controller plugins --- packages/lib/src/data/messages_instance.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/lib/src/data/messages_instance.ts b/packages/lib/src/data/messages_instance.ts index 2a37d086..d14894d4 100644 --- a/packages/lib/src/data/messages_instance.ts +++ b/packages/lib/src/data/messages_instance.ts @@ -385,7 +385,7 @@ export class InstanceSaveDetailsUpdatesEvent { export class InstanceCreateSaveRequest { declare ["constructor"]: typeof InstanceCreateSaveRequest; static type = "request" as const; - static src = "control" as const; + static src = ["control", "controller"] as const; static dst = "instance" as const; static permission = "core.instance.save.create" as const; @@ -596,7 +596,7 @@ export class InstancePushSaveRequest { export class InstanceLoadScenarioRequest { declare ["constructor"]: typeof InstanceLoadScenarioRequest; static type = "request" as const; - static src = "control" as const; + static src = ["control", "controller"] as const; static dst = "instance" as const; static permission = "core.instance.load_scenario" as const; @@ -622,7 +622,7 @@ export class InstanceLoadScenarioRequest { export class InstanceExportDataRequest { declare ["constructor"]: typeof InstanceExportDataRequest; static type = "request" as const; - static src = "control" as const; + static src = ["control", "controller"] as const; static dst = "instance" as const; static permission = "core.instance.export_data" as const; } @@ -630,7 +630,7 @@ export class InstanceExportDataRequest { export class InstanceExtractPlayersRequest { declare ["constructor"]: typeof InstanceExtractPlayersRequest; static type = "request" as const; - static src = "control" as const; + static src = ["control", "controller"] as const; static dst = "instance" as const; static permission = "core.instance.extract_players" as const; } @@ -638,7 +638,7 @@ export class InstanceExtractPlayersRequest { export class InstanceStopRequest { declare ["constructor"]: typeof InstanceStopRequest; static type = "request" as const; - static src = "control" as const; + static src = ["control", "controller"] as const; static dst = "instance" as const; static permission = "core.instance.stop" as const; } @@ -646,7 +646,7 @@ export class InstanceStopRequest { export class InstanceKillRequest { declare ["constructor"]: typeof InstanceKillRequest; static type = "request" as const; - static src = "control" as const; + static src = ["control", "controller"] as const; static dst = "instance" as const; static permission = "core.instance.kill" as const; } @@ -693,7 +693,7 @@ export class InstanceDeleteInternalRequest { export class InstanceSendRconRequest { declare ["constructor"]: typeof InstanceSendRconRequest; static type = "request" as const; - static src = "control" as const; + static src = ["control", "controller"] as const; static dst = "instance" as const; static permission = "core.instance.send_rcon" as const; From 8879f8dd80b5dff31b77ea9df78e35629933a866 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Fri, 9 Feb 2024 09:11:53 +0300 Subject: [PATCH 48/68] Replaced react Link with custom component The custom component merges the behaviour of antd Typography.Link and the react-router.dom Link after the custom component prop was removed in v6 according to this guide: https://reactrouter.com/en/main/upgrading/v5#remove-link-component-prop --- .../web_ui/src/components/InstanceList.tsx | 7 ++- .../src/components/InstanceViewPage.tsx | 12 +++-- packages/web_ui/src/components/Link.tsx | 47 +++++++++++++++++++ packages/web_ui/src/components/PageLayout.tsx | 2 +- packages/web_ui/src/components/UsersPage.tsx | 3 +- 5 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 packages/web_ui/src/components/Link.tsx diff --git a/packages/web_ui/src/components/InstanceList.tsx b/packages/web_ui/src/components/InstanceList.tsx index d966ccad..c1aa76d0 100644 --- a/packages/web_ui/src/components/InstanceList.tsx +++ b/packages/web_ui/src/components/InstanceList.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { Link, useNavigate } from "react-router-dom"; +import { useNavigate } from "react-router-dom"; import { message, Button, Space, Table, Typography } from "antd"; import CopyOutlined from "@ant-design/icons/CopyOutlined"; import type { SizeType } from "antd/es/config-provider/SizeContext"; @@ -10,6 +10,7 @@ import { useHosts } from "../model/host"; import InstanceStatusTag from "./InstanceStatusTag"; import StartStopInstanceButton from "./StartStopInstanceButton"; import { InstanceDetails } from "@clusterio/lib"; +import Link from "./Link"; const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare; @@ -59,9 +60,7 @@ export default function InstanceList(props: InstanceListProps) { to={`/hosts/${instance.assignedHost}/view`} onClick={e => e.stopPropagation()} > - - {hostName(instance.assignedHost)} - + {hostName(instance.assignedHost)} , sorter: (a, b) => strcmp(hostName(a.assignedHost), hostName(b.assignedHost)), responsive: ["sm"], diff --git a/packages/web_ui/src/components/InstanceViewPage.tsx b/packages/web_ui/src/components/InstanceViewPage.tsx index c1c4e018..5973cdfa 100644 --- a/packages/web_ui/src/components/InstanceViewPage.tsx +++ b/packages/web_ui/src/components/InstanceViewPage.tsx @@ -1,5 +1,5 @@ import React, { useContext, useState } from "react"; -import { Link, useNavigate, useParams } from "react-router-dom"; +import { useNavigate, useParams } from "react-router-dom"; import { Alert, Button, Descriptions, Dropdown, Menu, MenuProps, Modal, Space, Spin, Typography } from "antd"; import type { ItemType } from "antd/es/menu/hooks/useItems"; import DeleteOutlined from "@ant-design/icons/DeleteOutlined"; @@ -23,6 +23,7 @@ import { notifyErrorHandler } from "../util/notify"; import { useInstance } from "../model/instance"; import { useHost } from "../model/host"; import InstanceStatusTag from "./InstanceStatusTag"; +import Link from "./Link"; const { Title } = Typography; @@ -43,7 +44,7 @@ function InstanceDescription(props: InstanceDescriptionProps) { {!assigned ? Unassigned : - {host?.name ?? instance.assignedHost} + {host?.name ?? instance.assignedHost} } {account.hasPermission("core.instance.assign") && - - } + ) + && + + } ; } diff --git a/packages/web_ui/src/components/Link.tsx b/packages/web_ui/src/components/Link.tsx new file mode 100644 index 00000000..224e651d --- /dev/null +++ b/packages/web_ui/src/components/Link.tsx @@ -0,0 +1,47 @@ +import React from "react"; +import { Typography } from "antd"; +import { + LinkProps, + useHref, + useLinkClickHandler, +} from "react-router-dom"; +import { BaseType } from "antd/es/typography/Base"; + +const Link = React.forwardRef( + ( + { + onClick, + replace = false, + state, + target, + to, + type, + ...rest + }: LinkProps, + ref + ) => { + let href = useHref(to); + let handleClick = useLinkClickHandler(to, { + replace, + state, + target, + }) as (event: React.MouseEvent) => void; + + return ( + { + onClick?.(event as React.MouseEvent); + if (!event.defaultPrevented) { + handleClick(event); + } + }} + ref={ref as React.RefObject | null} + target={target} + type={type as BaseType} + /> + ); + } +); +export default Link; diff --git a/packages/web_ui/src/components/PageLayout.tsx b/packages/web_ui/src/components/PageLayout.tsx index 8a9ce1e7..8361827b 100644 --- a/packages/web_ui/src/components/PageLayout.tsx +++ b/packages/web_ui/src/components/PageLayout.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { Link } from "react-router-dom"; import { Layout, Breadcrumb } from "antd"; +import Link from "./Link"; const { Content } = Layout; diff --git a/packages/web_ui/src/components/UsersPage.tsx b/packages/web_ui/src/components/UsersPage.tsx index c487adf5..83c605aa 100644 --- a/packages/web_ui/src/components/UsersPage.tsx +++ b/packages/web_ui/src/components/UsersPage.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useContext, useState } from "react"; -import { Link, useNavigate } from "react-router-dom"; +import { useNavigate } from "react-router-dom"; import { Button, Form, Input, Modal, Space, Table, Tag } from "antd"; import * as lib from "@clusterio/lib"; @@ -12,6 +12,7 @@ import PageHeader from "./PageHeader"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; import { formatFirstSeen, formatLastSeen, sortFirstSeen, sortLastSeen, useUsers } from "../model/user"; +import Link from "./Link"; const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare; From 3cddd2b9f828232f1ac6b6d9560949ef4f004425 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 10 Feb 2024 16:33:22 +0300 Subject: [PATCH 49/68] Export Link component for use by plugins --- packages/web_ui/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/web_ui/src/index.ts b/packages/web_ui/src/index.ts index 03e4e6cd..601e092c 100644 --- a/packages/web_ui/src/index.ts +++ b/packages/web_ui/src/index.ts @@ -28,3 +28,5 @@ export { default as PageHeader } from "./components/PageHeader"; export { default as PageLayout } from "./components/PageLayout"; export { default as SavesList } from "./components/SavesList"; export { default as SectionHeader } from "./components/SectionHeader"; + +export { default as Link } from "./components/Link"; From c9609259db644cb04b4c3cf29ca796361e8b0ef4 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 11 Feb 2024 15:48:45 +0100 Subject: [PATCH 50/68] Ignore error loading mod in fetchMod If a bad download or file transfer causes a mod in a host's mod folder to become corrupt the attempt to load the mod will throw an error that's not the file not existing. This in turn causes starting the instance to always fail as the mod is never attempted to be re-downloaded. Fix by ignoring errors attempting to load mods already present in the mods folder that aren't in the mod store so that a new download is attempted after that fails. --- CHANGELOG.md | 1 + packages/host/src/Host.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1ac02f..485f1130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Fixed data export with a mod using shorthand notation for the default value of a color-setting causing the ModPack data to be set to invalid data. - Fixed per user per instance player stats not being deleted when the instance it's for is deleted. - Fixed mods setting the item prototype icons property to an object mapping instead of an array breaking the data export. +- Fixed a bad mod in the host's mods folder causing instance start to fail instead of the host re-downloading the mod. ## Version 2.0.0-alpha.16 diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index c2fc5d67..75e80ffa 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -600,8 +600,9 @@ export default class Host extends lib.Link { try { modInfo = await this.modStore.loadMod(mod.name, mod.version); } catch (err: any) { + // A broken zip file may have been left in the mods folder if (err.code !== "ENOENT") { - throw err; + logger.error(`Error loading mod ${mod.name} ${mod.version}: ${err.message}`); } } if (modInfo && (!mod.sha1 || mod.sha1 === modInfo.sha1)) { From e990134b7045cf56947fe2e83dfdf36bb401954f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 11 Feb 2024 21:40:32 +0100 Subject: [PATCH 51/68] Use suggested wording in docs/contributing.md Co-authored-by: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> --- docs/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.md b/docs/contributing.md index eb5b82d1..8f1d73fe 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -81,7 +81,7 @@ Note that to use `clusterioctl` you will have to create a control config first: Once you've set up the cluster you can use `node packages/controller run` to run the controller and `node packages/host run` to run the host which connects to the controller. -When editing TypeScript files runned on Node.js you need run `pnpm build` in order to compile the files to JavaScript before restarting the controller/host/ctl for changes take effect. +When editing TypeScript files ran by Node.js you need to run `pnpm build` to compile the files to JavaScript before restarting the controller/host/ctl for changes take effect. To have TypeScript watch for changes and rebuild immediadly you may run `pnpm watch` instead. For web development on the controller there are also the following flags: From abcbf3fddce688ae8463155c8c37c7b67402ffaa Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:24:00 +0100 Subject: [PATCH 52/68] Add pagination to saves list (#592) Prevent the saves list from pushing the console and config off the screen by turning on the pagination for it. --- CHANGELOG.md | 4 ++++ packages/web_ui/src/components/SavesList.tsx | 2 +- packages/web_ui/src/style.css | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 485f1130..a23be684 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Version 2.0.0-alpha.17 +### Features + +- Added pagination to the saves list. + ### Changes - Fixed an unhandled error in Subspace Storage crashing the host. diff --git a/packages/web_ui/src/components/SavesList.tsx b/packages/web_ui/src/components/SavesList.tsx index 2cbe4c8e..92ce287c 100644 --- a/packages/web_ui/src/components/SavesList.tsx +++ b/packages/web_ui/src/components/SavesList.tsx @@ -207,6 +207,7 @@ export default function SavesList(props: { instance: lib.InstanceDetails }) { let hostOffline = ["unassigned", "unknown"].includes(props.instance.status!); const saveTable =
save.name} - pagination={false} expandable={{ columnWidth: 33, expandRowByClick: true, diff --git a/packages/web_ui/src/style.css b/packages/web_ui/src/style.css index 7fa95675..3cd12ed2 100644 --- a/packages/web_ui/src/style.css +++ b/packages/web_ui/src/style.css @@ -182,10 +182,19 @@ body { white-space: pre-wrap; } +.save-list-dragger .ant-upload.ant-upload-drag { + border: none; + background: none; +} + .save-list-dragger .ant-upload.ant-upload-btn { padding: 0; } +.save-list-table .ant-table-pagination.ant-pagination { + margin-bottom: 0; +} + .rcon-result code { display: block; background: #000; From d8f1d8a3e2f00684d45e36066c9a497c609d6525 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:25:10 +0100 Subject: [PATCH 53/68] Add known issue about Factorio migrations --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0857fc07..1873cf33 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,8 @@ If you want to use a different user for the command line interface, you can gene - When the `server_select` plugin is installed you may see `[error] Unhandled event server_select:UpdateInstancesEvent` logged when starting an instance. This should be ignored as it does not indicate an error occured. +- After updating Factorio you may get the error `Error: Expected empty response but got "Cannot execute command. Error: [string "clusterio_private.update_instance(..."]:1: attempt to index global 'clusterio_private' (a nil value)?` on instance start. + This is due to Factorio applying a migration replacing the patched content of the save with the original scenario, attempting to start the instance again should resolve the issue. - After migrating from Alpha 13 to Alpha 14 running an instance data export fails with the error `Upload failed: 401 Unauthorized` due to the host token being issued to a slave. To workaround this find the id of the host in the host config and generate a new host token with the same id. Stop the host and replace the value of the `host.controller_token` field in `config-host.json` with the new host token before starting it up again. From fac04adc6fd128068e3f6b6ec3f78c7657f5a1a0 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:29:51 +0100 Subject: [PATCH 54/68] Add known issue about the base mod --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1873cf33..c00b5ab5 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,8 @@ If you want to use a different user for the command line interface, you can gene This should be ignored as it does not indicate an error occured. - After updating Factorio you may get the error `Error: Expected empty response but got "Cannot execute command. Error: [string "clusterio_private.update_instance(..."]:1: attempt to index global 'clusterio_private' (a nil value)?` on instance start. This is due to Factorio applying a migration replacing the patched content of the save with the original scenario, attempting to start the instance again should resolve the issue. +- In the mod pack view of the web interface the "base" mod is displayed with an error complaining about the mod being missing from the controller storage. + This is due to the "base" mod currently not being connectly handled as a built-in mod to Factorio in the web interface, this error message should be ignored. - After migrating from Alpha 13 to Alpha 14 running an instance data export fails with the error `Upload failed: 401 Unauthorized` due to the host token being issued to a slave. To workaround this find the id of the host in the host config and generate a new host token with the same id. Stop the host and replace the value of the `host.controller_token` field in `config-host.json` with the new host token before starting it up again. From 8ae5e11021b5b84f155d6a90b7cb353168e81c01 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:33:13 +0100 Subject: [PATCH 55/68] Add changelog for #587 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a23be684..2c462e7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features +- Made the host an instance is assigned to link to the host page for that host in the Web UI [#587](https://github.com/clusterio/clusterio/pull/587). - Added pagination to the saves list. ### Changes From 8a580a66be09a4c4cadaded74ace5b03aaa6946d Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:37:27 +0100 Subject: [PATCH 56/68] Add breaking change note for 535e43b --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c462e7c..c7c8666a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ - Fixed mods setting the item prototype icons property to an object mapping instead of an array breaking the data export. - Fixed a bad mod in the host's mods folder causing instance start to fail instead of the host re-downloading the mod. +### Breaking Changes + +- Renamed the permission `core.instance.save.list.subscribe` to `core.instance.save.subscribe`. + ## Version 2.0.0-alpha.16 From a9c6430531e32f321ac068afd8189eb8b220d0ba Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:40:25 +0100 Subject: [PATCH 57/68] Document current publishing workflow Docement the adaptations I've made to the publishing procedure which uses a less dangerous publish command and removes the dist folders before publishing to avoid accidental upload of stale build files. --- docs/contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 8f1d73fe..9a4ef6a9 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -319,6 +319,6 @@ You will need commit access to the repository as well publish access to the @clu 4. Publish the packages. ```sh - # Warning: Double check your current working directory is the repository before running this - pnpm -r --filter='!./external_plugins/**' publish + rm -r ./{plugins,packages}/*/dist + pnpm -r --filter='./{plugins,packages}/*' publish ``` From cf2b3db20de0caf334d142285c87d2036dc812e3 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:50:33 +0100 Subject: [PATCH 58/68] Add thank you note to contributors of Alpha 17 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7c8666a..e213fd48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ - Renamed the permission `core.instance.save.list.subscribe` to `core.instance.save.subscribe`. +Many thanks to the following for contributing to this release: +[@Cooldude2606](https://github.com/Cooldude2606) +[@Danielv123](https://github.com/Danielv123) +[@Hornwitser](https://github.com/Hornwitser) + ## Version 2.0.0-alpha.16 From 5f85c0575e74e571c35ba91475dab5812cd55dde Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 17:54:48 +0100 Subject: [PATCH 59/68] v2.0.0-alpha.17 --- packages/controller/package.json | 6 +++--- packages/create/package.json | 2 +- packages/ctl/package.json | 4 ++-- packages/host/package.json | 4 ++-- packages/lib/package.json | 2 +- packages/web_ui/package.json | 4 ++-- plugins/global_chat/package.json | 6 +++--- plugins/inventory_sync/package.json | 6 +++--- plugins/player_auth/package.json | 6 +++--- plugins/research_sync/package.json | 6 +++--- plugins/statistics_exporter/package.json | 6 +++--- plugins/subspace_storage/package.json | 6 +++--- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/controller/package.json b/packages/controller/package.json index 97fd5f36..588b39c0 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio controller", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --env production", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", "@sinclair/typebox": "^0.30.4", "busboy": "^1.6.0", "compression": "^1.7.4", @@ -35,7 +35,7 @@ "yargs": "^17.7.2" }, "devDependencies": { - "@clusterio/web_ui": "^2.0.0-alpha.16", + "@clusterio/web_ui": "^2.0.0-alpha.17", "@swc/core": "^1.4.0", "@types/busboy": "^1.5.0", "@types/compression": "^1.7.2", diff --git a/packages/create/package.json b/packages/create/package.json index 40aa21ba..7929cbb1 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -3,7 +3,7 @@ "description": "Installer for clusterio", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "main": "create.js", "bin": { "clusteriocreate": "./create.js" diff --git a/packages/ctl/package.json b/packages/ctl/package.json index 7d5f6707..879afe37 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -3,7 +3,7 @@ "description": "Clusterio command line management tool", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", "as-table": "^1.0.55", "fs-extra": "^11.1.1", "phin": "^3.7.0", diff --git a/packages/host/package.json b/packages/host/package.json index c9f8446a..b339d0a0 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio host server", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", @@ -19,7 +19,7 @@ "node": ">=18" }, "dependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", "@sinclair/typebox": "^0.30.4", "fs-extra": "^11.1.1", "jimp": "^0.22.8", diff --git a/packages/lib/package.json b/packages/lib/package.json index 1c5cd2d6..82afa633 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -1,7 +1,7 @@ { "name": "@clusterio/lib", "description": "Shared library for Clusterio", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "repository": "https://github.com/clusterio/clusterio", "license": "MIT", "scripts": { diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index a1f2575a..0c7c4cc8 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -3,7 +3,7 @@ "description": "Clusterio web interface implementation", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "browser": "src/index.ts", "main": "dist/browser/src/index.js", "scripts": { @@ -17,7 +17,7 @@ }, "dependencies": { "@ant-design/icons": "^5.1.4", - "@clusterio/lib": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", "@swc/core": "^1.4.0", "antd": "^5.13.0", "assert": "^2.0.0", diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json index 26518d2c..8f82dc80 100644 --- a/plugins/global_chat/package.json +++ b/plugins/global_chat/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-global_chat", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -21,8 +21,8 @@ "@clusterio/lib": "^2.0.0-alpha.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", - "@clusterio/web_ui": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", + "@clusterio/web_ui": "^2.0.0-alpha.17", "@types/node": "^20.4.5", "@types/react": "^18.2.21", "typescript": "^5.1.6", diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index 2bf25a2a..566e4006 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-inventory_sync", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Danielv ", "homepage": "https://github.com/clusterio/factorioClusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", - "@clusterio/web_ui": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", + "@clusterio/web_ui": "^2.0.0-alpha.17", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index 733ebf7a..b65f9186 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-player_auth", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "description": "Clusterio plugin authenticating logged in players to the web interface", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -26,8 +26,8 @@ "jsonwebtoken": "^9.0.1" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", - "@clusterio/web_ui": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", + "@clusterio/web_ui": "^2.0.0-alpha.17", "@types/express": "^4.17.17", "@types/jsonwebtoken": "^9.0.2", "@types/node": "^20.4.5", diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json index f5a74600..ed6c298a 100644 --- a/plugins/research_sync/package.json +++ b/plugins/research_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-research_sync", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "description": "Clusterio plugin for synchronizing research between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", - "@clusterio/web_ui": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", + "@clusterio/web_ui": "^2.0.0-alpha.17", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", "@types/react": "^18.2.21", diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json index 21c9c87f..c1bb9490 100644 --- a/plugins/statistics_exporter/package.json +++ b/plugins/statistics_exporter/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-statistics_exporter", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "description": "Clusterio plugin for gathering statistics from Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -21,8 +21,8 @@ "@clusterio/lib": "^2.0.0-alpha.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", - "@clusterio/web_ui": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", + "@clusterio/web_ui": "^2.0.0-alpha.17", "@types/node": "^20.4.5", "@types/react": "^18.2.21", "typescript": "^5.1.6", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index 5dce5e9b..22fd77a8 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-subspace_storage", - "version": "2.0.0-alpha.16", + "version": "2.0.0-alpha.17", "description": "Clusterio plugin for sharing storage between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", @@ -25,8 +25,8 @@ "fs-extra": "^8.1.0" }, "devDependencies": { - "@clusterio/lib": "^2.0.0-alpha.16", - "@clusterio/web_ui": "^2.0.0-alpha.16", + "@clusterio/lib": "^2.0.0-alpha.17", + "@clusterio/web_ui": "^2.0.0-alpha.17", "@types/express": "^4.17.17", "@types/fs-extra": "^11.0.1", "@types/node": "^20.4.5", From b3fc278b2c4c434a53c2982a2f43d670278f421f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 12 Feb 2024 18:21:46 +0100 Subject: [PATCH 60/68] Add template to CHANGELOG.md Document the format and expected sections of entries as a template in the changelog so that it's easier to add new log entries. --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e213fd48..dce4cadc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog + + ## Version 2.0.0-alpha.17 ### Features From d2775e4db9dc245768969c2237214a9dae03eb1c Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Thu, 8 Feb 2024 14:37:42 +0300 Subject: [PATCH 61/68] Update message section of writing-plugins document --- docs/writing-plugins.md | 235 ++++++++++++++++------------------------ 1 file changed, 96 insertions(+), 139 deletions(-) diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 2b7e6566..5d6a70b4 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -12,8 +12,6 @@ The plugin classes have pre-defined hooks that are called during various stages - [Plugin Configuration](#plugin-configuration) - [Communicating with Factorio](#communicating-with-factorio) - [Defining Link Messages](#defining-link-messages) - - [Defining Events](#defining-events) - - [Defining Requests](#defining-requests) - [Sending Link Messages](#sending-link-messages) - [Handling connection events](#handling-connection-events) - [Collecting Statistics](#collecting-statistics) @@ -325,175 +323,134 @@ Events are simple one-way notifications that invoke a handler on the target it's Requests are pairs of request and response messages where the request is sent to the target and the response is the reply back from the target. The requests are similar to HTTP requests, except both parties of a link may initiate one. - -### Defining Events - -Events are defined as properties of the messages object exported on `plugin` that map to instances of the `Event` class from `lib/link`. -The name of the property correspond to the handler invoked on the plugin class. -The Event constructor takes an object of properties that define the event, for example the following could be defined in `plugin`: - -```js -messages: { - startFrobnication: new lib.Event({ - type: "foo_frobber:start_frobnication", - links: ["controller-host", "host-instance"], - forwardTo: "instance", - eventRequired: ["frobnication_type"], - eventProperties: { - "frobnication_type": { type: "string" }, - "urgent": { type: "boolean" }, - }, - }), -}, -``` - -This specifies an event that can be sent from the controller to a host, and from a host to an instance. -It also specifies that the event must contain the property `frobnication_type`, with a string value in the data payload and that it may optionally contain a boolean `urgent` property. -It will also be forwarded by hosts to a specific instance. - -The following properties are recognized by the Event constructor: - -#### type - -The message type sent over the wire. -This must start with the name of the plugin followed by colon and and be unique for the plugin. -The type of the message sent over the socket will have the suffix `_event` appended to it. - -#### links - -An array of strings describing which links this event can be sent over. -Direction matters, `"controller-host"` means the event can be sent from the controller to the host, but can't be sent back the other way, unless `"host-controller"` is also present in the links array. - -The available endpoints are `controller`, `host`, `instance`, and `control`. -Controller talks with host and control, and host talks to instance. -The full chain must be specified as the individual links in order for a message to travers multiple hops, (i.e., for a message to go from controller to instance it must have both `"controller-host"` and `"host-instance"` in the links array). -See `forwardTo` and `broadcastTo` for ways to forward an event to the next link in a chain. - -#### forwardTo - -Target to forward an event to. -Can either be `"controller"`, to indicate a host should forward it to the controller, or `"instance"`, to indicate it should be forwarded to the instances specified by the `instance_id` event property. -This works by using a default handler for the event at the links that forward it. - -#### broadcastTo - -Target to broadcast this message towards. -A value of "instance" means the event will be broadcast to all instances downstream of the target it's sent to, but not back from where it came from. -Currently, only "instance" is supported. -This means that sending the event to a host from an instance will cause it to be broadcast to all instances of that host except for the instance it came from. - -#### eventRequired - -By default all properties defined in `eventProperties` are required to be present in the payload for the event. -This can be overriden by specifying an array of properties which are required here. -This is equivalent to using the `required` keyword in JSON schema. - -#### eventProperties - -Object with properties mapping to a JSON schema of that property that specifies what's valid to send in the event. -This is equivalent to using the `properties` keyword in JSON schema, except that the properties specified are by default required and additional properties are not allowed. -See [this guide][guide] for an introduction to writing JSON schemas. - -The forwardTo and broadcastTo can be combined such that specifying `"controller"` as the forwardTo value and `"instance"` as the broadcastTo value will cause the event to be broadcast to all instances in the cluster. -For this to work, you will need to specify `instance-host`, `host-controller`, `controller-host`, and `host-instance` as the links. - -Keep in mind when forwarding events that if the target an event is being forwarded to is not online, the event will be dropped. -Use a request if you need a confirmation that the message was received. - -### Definining Requests - -Requests are defined as properties of the messages object exported by `plugin` that map to instances of the `Request` class from `lib/link`. -The name of the property corresponds to the handler invoked on the plugin class. -The Request constructor takes an object of properties that define the event. +Messages are defined as properties of the `plugin.messages` array exported by `index.js`. For example, the following could be defined in `plugin`: ```js -messages: { - reportFrobnication: new lib.Request({ - type: "foo_frobber:report_frobnication", - links: ["controller-host", "host-instance"], - forwardTo: "instance", - requestRequired: ["verbosity"], - requestProperties: { - "verbosity": { type: "integer" }, - "special": { type: "boolean" }, - }, - responseProperties: { - "report": { - type: "array", - items: { type: "string" }, +messages: [ + class Frobnicate { + static plugin = "foo_frobber"; // Plugin name + static type = "request"; + static src = "controller"; + static dst = "instance"; + static jsonSchema = { + type: "object", + properties: { + "verbosity": { type: "integer" }, + "special": { type: "boolean" }, }, - }, - }), -}, + required: ["verbosity"], + additionalProperties: false, + }; + constructor(json) { + this.verbosity = json.verbosity; + this.special = json.special; + } + static fromJSON(json) { + return new Frobnicate(json); + } + toJSON() { + return { + verbosity: this.verbosity, + special: this.special, + }; + } + static Response = { + jsonSchema: { + type: "object", + properties: { + "report": { + type: "array", + items: { type: "string" }, + }, + }, + }, + fromJSON(json) { + return json + } + } + } +] ``` -This specifies a request that can be sent from the controller to a host, and from a host to an instance. -The request data must contain the property `verbosity` with an integer number as the value, as well as the `instance_id` property (implied by `forwardTo: "instance"`) and it may also contain a boolean `special` property. -It also defines that the response sent must contain a `report` property mapping to an array of strings. -When received by a host, it will also be forwarded to the instance specified by `instance_id`. +This specifies a request that can be sent from the controller to an instance, being routed through the correct host behind the scenes. +The request data must contain the property `verbosity` with an integer number as the value, and it may also contain a boolean `special` property. +It also defines that the response sent may contain a `report` property mapping to an array of strings. -The following properties are recognized by the Request constructor: +The following properties are recognized as part of a message class: + +#### plugin + +Must contain the name of the plugin this message is defined by. +This property is not used in core messages. #### type The message type sent over the wire. -This must start with the name of the plugin followed by colon and and be unique for the plugin. -The type of the message sent over the socket will have the suffix `_request` appended to it for the request and `_response` appended to it for the response. +This can be either `request` or `event`. The primary difference is whether they allow for a response. -#### links +#### src -An array of strings describing which links this request can be sent over. -Direction matters; `"controller-host"` means the request can be sent from the controller to the host and the host can reply to the controller, but the host can't send a request to the controller unless `"host-controller"` is also present in the links array. +The allowed source of the message. +The source can be specified as `controller`, `host`, `instance`, or `control`. +This property also accepts an array of strings to allow multiple sources. +Messages sent from a different source will be rejected. -The available endpoints are `controller`, `host`, `instance`, and `control`. -Controller talks with host and control, and host talks to instance. -The full chain must be specified as the individual links in order for a message to travers multiple hops, (i.e., for a message to go from controller to instance it must have both `"controller-host"` and `"host-instance"` in the links array). -See `forwardTo` for ways to forward a request to the next link in a chain. +#### dst -#### forwardTo +The allowed destination of the message. +The destination can be specified as `controller`, `host`, `instance`, or `control`. +This property also accepts an array of strings to allow multiple destinations. +Messages sent to a different destination will be rejected. -Target to forward the request to. -Can either be `"controller"` to indicate a host should forward it to the controller when receiving it from an instance, or `"instance"` to indicate it should be forwarded to the instances specified by the `instance_id` request property. -This works by using a default handler for the request by the links that forward it. +#### jsonSchema -#### requestRequired +A JSON schema that specifies what's valid to send in the message. -By default all properties defined in `requestProperties` are required to be present in the payload for the request. -This can be overriden by specifying an array of properties which are required here. -This is equivalent to using the `required` keyword in JSON schema. +#### toJSON -#### requestProperties +Optional function to serialize the class to a JSON object, bypassing the default serialization provided by `JSON.stringify()` -Object with properties mapping to a JSON schema of that property that specifies what's valid to send in the request. -This is equivalent to using the `properties` keyword in JSON schema, except that the properties specified are by default required and additional properties are not allowed. -See [this guide][guide] for an introduction to writing JSON schemas +#### fromJSON -#### responseRequired +Required function to restore the class from a JSON object. +This function is called when the message reaches its destination and is expected to return an instance of the class. -Same as the requestRequired only for the response sent back by the target. - -#### responseProperties - -Same as the requestProperties only for the response sent back by the target. +#### Response +Optional class that specifies the response to the request. +Required properties are `jsonSchema` and `fromJSON`. ## Sending Link Messages -Link messages are sent by calling the `.send()` method on the Event/Request instance with the link you want to send it over and the data to send. -For `InstancePlugin` code the link to the host is the `instance` itself, which is accessible through the `.instance` property of the `InstancePlugin`. -The `.info` property of the plugin class exposes the data exported from the plugin's `plugin` export. -In other words: +Link messages are sent by calling the `.sendTo()` method on a connection object with the destination and the data you want to send, where the data is an instance of one of the defined message classes in plugin.messages. +For example: ```js // In an InstancePlugin class async frobnicate() { - this.info.messages.exampleEvent.send(this.instance, { foo: "bar" }); + const response = await this.instance.sendTo("controller", new messages.Frobnicate({ + verbosity: 2, + special: false, + })); + console.log(response); // { report: [...] } } ``` -For the Request class the send method is async and returns the response data received from the target it was sent to, or throws an error if the request failed. +For classes with `static type = "request"` the send method is async and returns a promise that resolves to the response data received from the target it was sent to, or rejects with an error if the request failed. +The destination specification can either be the ID of a particular control, host or instance, or one of the keywords used to send to multiple targets at once. +Example destinations: + +```js +sendTo("controller", message); +sendTo("allHosts", message); +sendTo("allControls", message); +sendTo("allInstances", message); +sendTo({ hostId: 123 }, message); +sendTo({ instanceId: 123 }, message); +sendTo({ controlId: 123 }, message); +``` ### Handling connection events @@ -508,7 +465,7 @@ For example the sending of an event from an `InstancePlugin` class can be stoppe ```js if (this.host.connected) { - this.info.messages.frobnicate.send(this.instance, { foo: "bar" }); + this.instance.sendTo("controller", new messages.Frobnicate({ foo: "bar" })); } ``` @@ -617,4 +574,4 @@ class CtlPlugin extends BaseCtlPlugin { module.exports = { CtlPlugin, } -``` \ No newline at end of file +``` From 735faf4b2cfcc943b0447fe7cb9aab539412c904 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 14 Feb 2024 09:25:18 +0100 Subject: [PATCH 62/68] Apply suggestions from review Co-authored-by: Hornwitser --- docs/writing-plugins.md | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 5d6a70b4..c87b3b40 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -323,7 +323,7 @@ Events are simple one-way notifications that invoke a handler on the target it's Requests are pairs of request and response messages where the request is sent to the target and the response is the reply back from the target. The requests are similar to HTTP requests, except both parties of a link may initiate one. -Messages are defined as properties of the `plugin.messages` array exported by `index.js`. +Messages are defined as items of the `plugin.messages` array exported by `index.js`. For example, the following could be defined in `plugin`: ```js @@ -379,46 +379,49 @@ It also defines that the response sent may contain a `report` property mapping t The following properties are recognized as part of a message class: -#### plugin +#### static plugin Must contain the name of the plugin this message is defined by. This property is not used in core messages. -#### type +#### static type -The message type sent over the wire. -This can be either `request` or `event`. The primary difference is whether they allow for a response. +A string describing the message type sent over the wire. +This can be either `"request"` or `"event"`. +The primary difference is whether they allow for a response. -#### src +#### static src -The allowed source of the message. -The source can be specified as `controller`, `host`, `instance`, or `control`. -This property also accepts an array of strings to allow multiple sources. +A string or an array of strings describing where this message is allowed to originate from. +The available endpoints are `"controller"`, `"host"`, `"instance"`, or `"control"`. Messages sent from a different source will be rejected. -#### dst +#### static dst -The allowed destination of the message. -The destination can be specified as `controller`, `host`, `instance`, or `control`. -This property also accepts an array of strings to allow multiple destinations. +A string or array of strings describing where this message may be addressed to. +The available endpoints are `"controller"`, `"host"`, `"instance"`, or `"control"`. +If the target is multiple hops away it the message will be forwarded towards its destination by the intermediaries. +For example a message sent from a control connection an instance will be forwarded by the controller to the relevant host and from the host to the correct instance. Messages sent to a different destination will be rejected. -#### jsonSchema +#### static jsonSchema A JSON schema that specifies what's valid to send in the message. #### toJSON -Optional function to serialize the class to a JSON object, bypassing the default serialization provided by `JSON.stringify()` +Optional method to serialize an instance of the class to a JSON object, see the [documentation for `JSON.stringify()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#description) for the semantics of this method. +This should return an object that's valid according to `jsonSchema` and accepted by `fromJSON`, and may be omitted if the default serialisation by `JSON.stringify()` already does this. -#### fromJSON +#### static fromJSON Required function to restore the class from a JSON object. -This function is called when the message reaches its destination and is expected to return an instance of the class. +This function is called when the message reaches its destination and is expected to return an identical instance of the class. +I.e. the result of `fromJSON(JSON.parse(JSON.stringify(instance)))` should be a complete copy of the message. -#### Response +#### static Response -Optional class that specifies the response to the request. +Optional [JSON sterilisable class](/docs/devs/json-serialisable-classes.md) that specifies the response to the request. Required properties are `jsonSchema` and `fromJSON`. ## Sending Link Messages From 0b2fcdcec471a5ba029a0f2b017ba8e35d44174b Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:37:48 +0000 Subject: [PATCH 63/68] Merged redundant middleware --- packages/controller/src/Controller.ts | 46 +++++++++++---------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index f89753c3..78f42436 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -166,31 +166,6 @@ export default class Controller { this.app.locals.controller = this; this.app.locals.streams = new Map(); - // Log all requests made to express - this.app.use((req, res, next) => { - const startNs = process.hrtime.bigint(); - res.once("finish", () => { - const endNs = process.hrtime.bigint(); - const durationMs = (Number(endNs - startNs) / 1e6); - logger.log({ - level: "http", - message: `HTTP${req.httpVersion} ${req.method} ${req.originalUrl}` + - ` (Status ${res.statusCode} in ${durationMs}ms)`, - meta: { - method: req.method, - url: req.originalUrl, - statusCode: res.statusCode, - responseTime: durationMs, - httpVersion: req.httpVersion, - headers: req.headers, - query: req.query, - }, - }); - }); - - next(); - }); - this.trustedProxies = this.parseTrustedProxies(); this.wsServer = new WsServer(this); @@ -765,14 +740,29 @@ export default class Controller { static addAppRoutes(app: Application, pluginInfos: any[]) { app.use((req: Request, res: Response, next) => { - let start = process.hrtime.bigint(); + const startNs = process.hrtime.bigint(); stream.finished(res, () => { let routePath = "static"; if (req.route && req.route.path) { routePath = req.route.path; } - let end = process.hrtime.bigint(); - endpointDurationSummary.labels(routePath).observe(Number(end - start) / 1e9); + const endNs = process.hrtime.bigint(); + const durationMs = (Number(endNs - startNs) / 1e6); + endpointDurationSummary.labels(routePath).observe(durationMs / 1e3); + logger.log({ + level: "http", + message: `HTTP${req.httpVersion} ${req.method} ${req.originalUrl}` + + ` (Status ${res.statusCode} in ${durationMs}ms)`, + meta: { + method: req.method, + url: req.originalUrl, + statusCode: res.statusCode, + responseTime: durationMs, + httpVersion: req.httpVersion, + headers: req.headers, + query: req.query, + }, + }); }); next(); }); From a70490648f4c5fa0f8e16509af9e56aa34cf5f4d Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:53:15 +0000 Subject: [PATCH 64/68] Fix total db size calculation for inv sync (#598) --- plugins/inventory_sync/controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inventory_sync/controller.ts b/plugins/inventory_sync/controller.ts index 6ce19850..0b4051fe 100644 --- a/plugins/inventory_sync/controller.ts +++ b/plugins/inventory_sync/controller.ts @@ -190,7 +190,7 @@ export class ControllerPlugin extends BaseControllerPlugin { })) .sort((a, b) => b.length - a.length); return new msg.DatabaseStatsRequest.Response( - playerDatastore.map(x => x.length).reduce((a, b) => b - a, 0), + playerDatastore.map(x => x.length).reduce((acc, val) => acc + val, 0), playerDatastore.length, { name: playerDatastore[0] && playerDatastore[0].name || "-", From f07ee22969f91e25f8aae61a0c3da7c2d9f8d406 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 23 Feb 2024 18:06:02 +0100 Subject: [PATCH 65/68] Add changelog for #556 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dce4cadc..b03acc7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,16 @@ Many thanks to the following for contributing to this release: [@username](https://github.com/username) --> +## Version 2.0.0-alpha.18 + +### Features + +- Added logging of HTTP requests and errors on the controller [#556](https://github.com/clusterio/clusterio/pull/556). + +Many thanks to the following for contributing to this release: +[@Cooldude2606](https://github.com/Cooldude2606) + + ## Version 2.0.0-alpha.17 ### Features From 2075814ed12bd203b3f443efd0f5c37871990870 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 23 Feb 2024 18:09:09 +0100 Subject: [PATCH 66/68] Add changelog for #598 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b03acc7d..a15e75b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,10 @@ Many thanks to the following for contributing to this release: - Added logging of HTTP requests and errors on the controller [#556](https://github.com/clusterio/clusterio/pull/556). +### Changes + +- Fixed Inventory Sync reporting a negative database size [ยค598](https://github.com/clusterio/clusterio/pull/598). + Many thanks to the following for contributing to this release: [@Cooldude2606](https://github.com/Cooldude2606) From a68bb9c0a8a07b464b65af8adb9c59bf0ef86360 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 25 Feb 2024 16:50:14 -0600 Subject: [PATCH 67/68] Increase libLink event emitter leak warning limit (#600) --- packages/lib/src/link/link.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/lib/src/link/link.ts b/packages/lib/src/link/link.ts index 0f8f2d32..1d3d3381 100644 --- a/packages/lib/src/link/link.ts +++ b/packages/lib/src/link/link.ts @@ -107,6 +107,9 @@ export class Link { ) { this.handle(libData.PingRequest, async () => {}); + // Prevent warnings about possible memory leak due to large number of event listeners + connector.setMaxListeners(20); + // Process messages received by the connector connector.on("message", payload => { try { From e5d82dcfdedd6f9e708eeb3e8e8faa36733d866f Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 26 Feb 2024 22:34:41 -0600 Subject: [PATCH 68/68] Don't fail CI due to project level codecov regressions --- codecov.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..8f46d2de --- /dev/null +++ b/codecov.yml @@ -0,0 +1,5 @@ +coverage: + status: + project: + default: + if_ci_failed: success