diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c374ea3e..c91ff2ee 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,10 +1,15 @@ -Describe why and what has been changed by this pullrequest, feel free to also ask for input on decisions you are unsure about etc. -The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferrentially link to the associated issues. If there are no relevant issues, link to the pull request itself. + + +### Changelog ``` ### Features -- Added doodad to web interface. [#999](https://github.com/clusterio/clusterio/issues/999) +- Added doodad to web interface. #999 ### Changes - ... @@ -12,6 +17,6 @@ The changelog lines below will be copied directly from merged pull requests to t ### Fixes - ... -### Breaking changes +### Breaking Changes - ... ``` diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 3ba681ab..e029119d 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -40,6 +40,12 @@ jobs: with: files: ./coverage/lcov.info token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload test logs for debugging + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.node-version }}-${{ matrix.factorio-version }} + path: temp/test/logs lint: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 854a9358..0ff3ffa5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,8 +13,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - with: - version: 8 - uses: actions/setup-node@v4 with: node-version: 22.x @@ -33,8 +31,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - with: - version: 8 - uses: actions/setup-node@v4 with: node-version: 22.x diff --git a/CHANGELOG.md b/CHANGELOG.md index fbbc9cba..22515dcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,66 @@ Many thanks to the following for contributing to this release: [@username](https://github.com/username) --> +## Version 2.0.0-alpha.23 + +### Major Features + +- Added automatic headless download and update on linux. [#330](https://github.com/clusterio/clusterio/issues/330) +- Added recipe notification sync to inventory sync. [#506](https://github.com/clusterio/clusterio/issues/506) + +### Features + +- Added config lock files to prevent process duplication. [#815](https://github.com/clusterio/clusterio/issues/815) +- Added process and system start times to system metrics. [#826](https://github.com/clusterio/clusterio/issues/826) +- Added icons to status tags for host and instances. [#827](https://github.com/clusterio/clusterio/issues/827) +- Extended export-data to produce icons for recipes, signals, technologies, planets, qualities and entities. [#838](https://github.com/clusterio/clusterio/pull/838) +- Added Icon Reference page to web interface at `/icon-reference` for browsing all exported icons. [#838](https://github.com/clusterio/clusterio/pull/838) +- Added visual flash to RCON result box to confirm command execution, even when the output is identical to the previous result. [#793](https://github.com/clusterio/clusterio/issues/793) + +### Changes + +- Login screen now displays the cluster name. [#818](https://github.com/clusterio/clusterio/issues/818) +- Instance versions can now be set using Major.Minor without specifying a patch version. [#820](https://github.com/clusterio/clusterio/issues/820) +- Versions for factorio and mod packs can now be selected using a drop down menu. [#820](https://github.com/clusterio/clusterio/issues/820) +- The controller will now refuse to start if no users were loaded from file. [#814](https://github.com/clusterio/clusterio/issues/814) +- Improved wording and display of the generate host token modal. [#828](https://github.com/clusterio/clusterio/issues/828) +- Installations where multiple conflicting clusterio/lib packages have been installed will now fail with an error [#863](https://github.com/clusterio/clusterio/issues/863). +- Export completion now logs per-category icon counts. [#838](https://github.com/clusterio/clusterio/pull/838) + +### Fixes + +- Removed debug log statement from equipment grid serialization. [#807](https://github.com/clusterio/clusterio/pull/807) +- Fix that the documentation had a dead link to details of the message passing protocol [#812](https://github.com/clusterio/clusterio/issues/812) +- Fixed case where achievements were not disabled. [#817](https://github.com/clusterio/clusterio/pull/817) +- Fix headless download on linux install. [#816](https://github.com/clusterio/clusterio/issues/816) +- Fix option `--no-download-headless` being ignored. [#816](https://github.com/clusterio/clusterio/issues/816) +- Fixed action buttons overflowing on narrow screens across the web interface. [#810](https://github.com/clusterio/clusterio/issues/810) +- Fixed player logistics failing to load on servers without unlocked logistics networks. [#806](https://github.com/clusterio/clusterio/issues/806) +- Fixed mod dependency parser breaking with inconsistent spaces. [#834](https://github.com/clusterio/clusterio/issues/834) +- No longer attempts to sync inventory in single player. [#835](https://github.com/clusterio/clusterio/issues/835) +- Fixed tests failing when the configured factorio_directory contains stuff other than Factorio installs [#540](https://github.com/clusterio/clusterio/issues/540). +- Always create a whitelist file so it can be watched for changes. [#865](https://github.com/clusterio/clusterio/issues/865) +- Added missing await statements which could cause an instance to start with an outdated admin/ban/whitelist. [#869](https://github.com/clusterio/clusterio/issues/869) +- Fixed metrics breaking when labels contain comma or equal signs [#861](https://github.com/clusterio/clusterio/issues/861) +- Fixed Host page show repeated errors when viewing an offline local host [#848](https://github.com/clusterio/clusterio/issues/848) +- Fixed bad mod paths in icon specifications crashing the export. [#841](https://github.com/clusterio/clusterio/issues/841) +- Fixed prototypes with no icon data causing errors during export. [#841](https://github.com/clusterio/clusterio/issues/841) + +### Breaking Changes + +- Custom permission methods should use `IUser` rather than `IControllerUser` [#837](https://github.com/clusterio/clusterio/pull/837) +- `controller.userManager` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837) +- `controller.instances` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837) +- `User/Instance/HostInfo` renamed to `User/Instance/HostRecord` [#837](https://github.com/clusterio/clusterio/pull/837) +- Removed `host.tls_ca` and `control.tls_ca` config options, see [section on self signed certificates](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate) for the replacement for this option [#857](https://github.com/clusterio/clusterio/issues/857) + +Many thanks to the following for contributing to this release: +[@Cooldude2606](https://github.com/Cooldude2606) +[@Daniel](https://github.com/Danielv123) +[@Double](https://github.com/DoubleThoughtTheProgrammer) +[@Hornwitser](https://github.com/Hornwitser) +[@Ryan Walker](https://github.com/solarcloud7) + ## Version 2.0.0-alpha.22 ### Major Features diff --git a/docs/configuration.md b/docs/configuration.md index 8d8e73b7..2285e319 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -182,14 +182,6 @@ You can generate an access token with `clusterioctl host generate-token --id { - if (err instanceof lib.StartupError) { + if (err instanceof lib.StartupError || err.code === "InstallationError") { logger.fatal(` +-------------------------------+ | Unable to to start controller | diff --git a/packages/controller/package.json b/packages/controller/package.json index 9f350bd6..81872cb8 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio controller", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --mode production", diff --git a/packages/controller/src/BaseConnection.ts b/packages/controller/src/BaseConnection.ts index 8efbf14c..07b753f8 100644 --- a/packages/controller/src/BaseConnection.ts +++ b/packages/controller/src/BaseConnection.ts @@ -32,6 +32,7 @@ export default class BaseConnection extends lib.Link { this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this)); this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this)); this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this)); + this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this)); this.handle(lib.SubscriptionRequest, this.handleSubscriptionRequest.bind(this)); this.connector.on("close", () => { @@ -106,4 +107,8 @@ export default class BaseConnection extends lib.Link { async handleSubscriptionRequest(request: lib.SubscriptionRequest, src: lib.Address, dst: lib.Address) { return await this._controller.subscriptions.handleRequest(this, request, src, dst); } + + async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) { + return await this._controller.factorioVersions.get(request.maxAgeMs); + } } diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index aa2350f3..93ca7c68 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -129,7 +129,6 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this)); this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this)); this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this)); - this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this)); } validateIngress(message: lib.MessageRequest | lib.MessageEvent) { @@ -1288,10 +1287,6 @@ export default class ControlConnection extends BaseConnection { this.connector._socket!.clusterio_ignore_dump = true; this._controller.debugEvents.on("message", this.ws_dumper); } - - async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) { - return await this._controller.factorioVersions.get(request.maxAgeMs); - } } module.exports = ControlConnection; diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 628073cc..122faa93 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -112,7 +112,14 @@ export default class Controller { // Cache for mod portal requests modPortalCache = new Map(); // Cache for factorio versions - factorioVersions = new lib.ValueCache(lib.fetchFactorioVersions); + factorioVersions = new lib.ValueCache(async () => { + try { + return await lib.fetchFactorioVersions(); + } catch (err: any) { + logger.warn(`Failed to retrieve Factorio versions ${err.message}`); + return []; + } + }); static async bootstrap(config: lib.ControllerConfig) { let databaseDirectory = config.get("controller.database_directory"); @@ -1070,6 +1077,9 @@ export default class Controller { this.plugins.set(pluginInfo.name, controllerPlugin); } catch (err: any) { + if (err.code === "InstallationError") { + throw err; + } throw new lib.PluginError(pluginInfo.name, err); } diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 66601e20..e7132925 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -81,7 +81,6 @@ export default class HostConnection extends BaseConnection { this.handle(lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsUpdatesEvent.bind(this)); this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this)); this.handle(lib.InstancePlayerUpdateEvent, this.handleInstancePlayerUpdateEvent.bind(this)); - this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this)); this.snoopEvent(lib.InstanceAdminlistUpdateEvent, this.snoopInstanceAdminlistUpdateEvent.bind(this)); this.snoopEvent(lib.InstanceBanlistUpdateEvent, this.snoopInstanceBanlistUpdateEvent.bind(this)); this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.snoopInstanceWhitelistUpdateEvent.bind(this)); @@ -359,10 +358,6 @@ export default class HostConnection extends BaseConnection { }); } - async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) { - return await this._controller.factorioVersions.get(request.maxAgeMs); - } - async snoopInstanceAdminlistUpdateEvent(event: lib.InstanceAdminlistUpdateEvent, src: lib.Address) { if (src.type !== lib.Address.instance) { return; diff --git a/packages/create/create.js b/packages/create/create.js index 69f17e7d..9e1181c2 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -335,7 +335,7 @@ async function migrateRename(args) { let mode = "standalone"; if (!hasRunSlave) { mode = "controller"; } if (!hasRunMaster) { mode = "host"; } - await writeScripts(mode); + await writeScripts(mode, ""); } logger.info( @@ -420,7 +420,7 @@ async function groupIdToName(gid) { } } -async function writeScripts(mode) { +async function writeScripts(mode, selfSignedCert) { if (["standalone", "controller"].includes(mode)) { if (process.platform === "win32") { await safeOutputFile( @@ -474,12 +474,13 @@ WantedBy=multi-user.target if (["standalone", "host"].includes(mode)) { if (process.platform === "win32") { + let extraCaCertsLine = selfSignedCert ? `set "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : ""; await safeOutputFile( "run-host.cmd", `\ @echo off set "NODE_OPTIONS=--enable-source-maps %NODE_OPTIONS%" -:restart +${extraCaCertsLine}:restart call .\\node_modules\\.bin\\clusteriohost.cmd run --can-restart if %errorlevel% equ 0 exit /b if %errorlevel% equ 8 exit /b @@ -487,18 +488,20 @@ goto restart ` ); } else { + let extraCaCertsLine = selfSignedCert ? `export "NODE_EXTRA_CA_CERTS=${selfSignedCert}"\n` : ""; await safeOutputFile( "run-host.sh", `\ #!/bin/bash export "NODE_OPTIONS=--enable-source-maps $NODE_OPTIONS" -while true; do +${extraCaCertsLine}while true; do ./node_modules/.bin/clusteriohost run --can-restart if [[ $? -eq 0 || $? -eq 8 ]]; then exit $?; fi done `, { mode: 0o755 }, ); + extraCaCertsLine = selfSignedCert ? `Environment=NODE_EXTRA_CA_CERTS=${selfSignedCert}\n` : ""; await safeOutputFile( "systemd/clusteriohost.service", `\ @@ -512,7 +515,7 @@ WorkingDirectory=${process.cwd()} KillMode=mixed KillSignal=SIGINT Environment=NODE_OPTIONS=--enable-source-maps -ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart +${extraCaCertsLine}ExecStart=${process.cwd()}/node_modules/.bin/clusteriohost run --log-level=warn --can-restart Restart=on-failure RestartPreventExitStatus=8 @@ -839,6 +842,9 @@ async function main() { .option("factorio-dir", { nargs: 1, describe: "Path to Factorio installation [standalone/host]", type: "string", }) + .option("self-signed-cert", { + nargs: 1, describe: "Path to self-signed certificate of the controller [standalone/host]", type: "string", + }) .option("remote-npm", { nargs: 0, description: "Allow remote updates via npm [standalone/controller/host]", type: "boolean", }) @@ -947,7 +953,7 @@ async function main() { if (!dev && ["standalone", "controller", "host"].includes(answers.mode)) { logger.info("Writing run scripts"); - await writeScripts(answers.mode); + await writeScripts(answers.mode, args.selfSignedCert); } if (answers.mode === "ctl") { diff --git a/packages/create/package.json b/packages/create/package.json index b26a0e45..4ea788f6 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -3,7 +3,7 @@ "description": "Installer for clusterio", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "main": "create.js", "bin": { "clusteriocreate": "./create.js" diff --git a/packages/ctl/ctl.ts b/packages/ctl/ctl.ts index 793dd1ab..a65b6c9e 100755 --- a/packages/ctl/ctl.ts +++ b/packages/ctl/ctl.ts @@ -27,8 +27,8 @@ import BaseCtlPlugin from "./src/BaseCtlPlugin"; class ControlConnector extends lib.WebSocketClientConnector { private _token: string; - constructor(url: string, maxReconnectDelay: number, tlsCa: string | undefined, token: string) { - super(url, maxReconnectDelay, tlsCa); + constructor(url: string, maxReconnectDelay: number, token: string) { + super(url, maxReconnectDelay); this._token = token; } @@ -54,8 +54,6 @@ class ControlConnector extends lib.WebSocketClientConnector { export class Control extends lib.Link { /** Control config used for connecting to the controller. */ config: lib.ControlConfig; - /** Certificate authority used to validate TLS connections to the controller. */ - tlsCa?: string; /** Mapping of plugin names to their instance for loaded plugins. */ plugins: Map; /** Keep the control connection alive after the command completes. */ @@ -64,12 +62,10 @@ export class Control extends lib.Link { constructor( connector: ControlConnector, controlConfig: lib.ControlConfig, - tlsCa: string | undefined, ctlPlugins: Map ) { super(connector); this.config = controlConfig; - this.tlsCa = tlsCa; this.plugins = ctlPlugins; this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this)); @@ -299,20 +295,13 @@ async function startControl() { return; } - let tlsCa: string | undefined; - let tlsCaPath = controlConfig.get("control.tls_ca"); - if (tlsCaPath) { - tlsCa = await fs.readFile(tlsCaPath, "utf8"); - } - let controlConnector = new ControlConnector( controlConfig.get("control.controller_url")!, controlConfig.get("control.max_reconnect_delay"), - tlsCa, controlConfig.get("control.controller_token")!, ); - let control = new Control(controlConnector, controlConfig, tlsCa, ctlPlugins); + let control = new Control(controlConnector, controlConfig, ctlPlugins); try { await controlConnector.connect(); } catch (err) { diff --git a/packages/ctl/package.json b/packages/ctl/package.json index af4819ed..07da9b6b 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -3,7 +3,7 @@ "description": "Clusterio command line management tool", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22", + "version": "2.0.0-alpha.23", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", @@ -24,7 +24,6 @@ "set-blocking": "catalog:", "fs-extra": "catalog:", "winston": "catalog:", - "phin": "catalog:", "as-table": "^1.0.55", "yargs": "^17.7.2" }, diff --git a/packages/ctl/src/commands_controller.ts b/packages/ctl/src/commands_controller.ts index 9b6481d5..bb17cbc8 100644 --- a/packages/ctl/src/commands_controller.ts +++ b/packages/ctl/src/commands_controller.ts @@ -1,7 +1,6 @@ import fs from "fs-extra"; import asTableModule from "as-table"; import events from "events"; -import phin from "phin"; import os from "os"; import child_process from "child_process"; @@ -166,12 +165,8 @@ controllerPluginCommands.add(new lib.Command({ handler: async function(args: object, control: Control) { let url = new URL(control.config.get("control.controller_url")!); url.pathname += "api/plugins"; - let response = await phin<[]>({ - url, - parse: "json", - core: { ca: control.tlsCa } as object, - }); - print(asTable(response.body)); + let response = await fetch(url); + print(asTable(await response.json() as [])); }, })); controllerPluginCommands.add(new lib.Command({ diff --git a/packages/ctl/src/commands_instance.ts b/packages/ctl/src/commands_instance.ts index 28a4efa4..d7bfacda 100644 --- a/packages/ctl/src/commands_instance.ts +++ b/packages/ctl/src/commands_instance.ts @@ -2,7 +2,6 @@ import fs from "fs-extra"; import asTableModule from "as-table"; import events from "events"; import path from "path"; -import phin from "phin"; import os from "os"; import child_process from "child_process"; import stream from "stream"; @@ -330,8 +329,6 @@ instanceSaveCommands.add(new lib.Command({ if (!filename.endsWith(".zip")) { throw new lib.CommandError("Save name must end with .zip"); } - // phin doesn't support streaming requests :( - let content = await fs.readFile(args.filepath); let instanceId = await lib.resolveInstance(control, args.instance); let url = new URL(control.config.get("control.controller_url")!); @@ -339,30 +336,32 @@ instanceSaveCommands.add(new lib.Command({ url.searchParams.append("instance_id", String(instanceId)); url.searchParams.append("filename", filename); - let result = await phin<{ errors?: string[], request_errors?: string[], saves?: string[] }>({ - url, method: "POST", + const response = await fetch(url, { + method: "POST", + body: await fs.createReadStream(args.filepath), + duplex: "half", headers: { - "X-Access-Token": control.config.get("control.controller_token"), + "X-Access-Token": control.config.get("control.controller_token")!, "Content-Type": "application/zip", }, - core: { ca: control.tlsCa } as object, - data: content, - parse: "json", }); - for (let error of result.body.errors || []) { + type ResponseType = { errors?: string[], request_errors?: string[], saves?: string[] }; + const result = await response.json() as ResponseType; + + for (let error of result.errors || []) { logger.error(error); } - for (let requestError of result.body.request_errors || []) { + for (let requestError of result.request_errors || []) { logger.error(requestError); } - if (result.body.saves && result.body.saves.length) { - logger.info(`Successfully uploaded as ${result.body.saves[0]}`); + if (result.saves && result.saves.length) { + logger.info(`Successfully uploaded as ${result.saves[0]}`); } - if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + if ((result.errors || []).length || (result.request_errors || []).length) { throw new lib.CommandError("Uploading save failed"); } }, @@ -418,11 +417,10 @@ instanceSaveCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - core: { ca: control.tlsCa } as object, - stream: true, - }); + let response = await fetch(url); + if (!response.body) { + throw new lib.CommandError("Got empty response from server."); + } let writeStream; let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip"); @@ -439,7 +437,7 @@ instanceSaveCommands.add(new lib.Command({ } } } - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); let filename = await lib.findUnusedName(".", args.save, ".zip"); diff --git a/packages/ctl/src/commands_mod.ts b/packages/ctl/src/commands_mod.ts index d4f9a3ec..a7e9f11e 100644 --- a/packages/ctl/src/commands_mod.ts +++ b/packages/ctl/src/commands_mod.ts @@ -2,7 +2,6 @@ import fs from "fs-extra"; import asTableModule from "as-table"; import events from "events"; import path from "path"; -import phin from "phin"; import stream from "stream"; import util from "util"; @@ -145,40 +144,38 @@ modCommands.add(new lib.Command({ if (!filename.endsWith(".zip")) { throw new lib.CommandError("Mod filename must end with .zip"); } - // phin doesn't support streaming requests :( - let content = await fs.readFile(args.file); let url = new URL(control.config.get("control.controller_url")!); url.pathname += "api/upload-mod"; url.searchParams.append("filename", filename); - let result = await phin< - { errors?: [], request_errors?: [], mods: Parameters[0][]} - >({ - url, method: "POST", + let response = await fetch(url, { + method: "POST", + body: fs.createReadStream(args.file), + duplex: "half", headers: { - "X-Access-Token": control.config.get("control.controller_token"), + "X-Access-Token": control.config.get("control.controller_token")!, "Content-Type": "application/zip", }, - core: { ca: control.tlsCa } as object, - data: content, - parse: "json", }); - for (let error of result.body.errors || []) { + type ResponseType = { errors?: [], request_errors?: [], mods: Parameters[0][]}; + const result = await response.json() as ResponseType; + + for (let error of result.errors || []) { logger.error(error); } - for (let requestError of result.body.request_errors || []) { + for (let requestError of result.request_errors || []) { logger.error(requestError); } - if (result.body.mods && result.body.mods.length) { - const mod = lib.ModInfo.fromJSON(result.body.mods[0]); + if (result.mods && result.mods.length) { + const mod = lib.ModInfo.fromJSON(result.mods[0]); logger.info(`Successfully uploaded ${mod.filename}`); } - if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + if ((result.errors || []).length || (result.request_errors || []).length) { throw new lib.CommandError("Uploading mod failed"); } }, @@ -197,11 +194,10 @@ modCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - core: { ca: control.tlsCa } as object, - stream: true, - }); + let response = await fetch(url); + if (!response.body) { + throw new lib.CommandError("Got empty response from server"); + } let writeStream; let filename = `${args.name}_${args.modVersion}.zip`; @@ -219,7 +215,7 @@ modCommands.add(new lib.Command({ } } } - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); await fs.rename(tempFilename, filename); diff --git a/packages/host/host.ts b/packages/host/host.ts index 19127713..485f2926 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -51,13 +51,11 @@ void new lib.Gauge( export class HostConnector extends lib.WebSocketClientConnector { constructor( public hostConfig: lib.HostConfig, - tlsCa: string | undefined, public pluginInfos: lib.PluginNodeEnvInfo[] ) { super( hostConfig.get("host.controller_url"), hostConfig.get("host.max_reconnect_delay"), - tlsCa ); } @@ -230,17 +228,10 @@ async function startHost() { return; } - let tlsCa: string | undefined; - let tlsCaPath = hostConfig.get("host.tls_ca"); - if (tlsCaPath) { - tlsCa = await fs.readFile(tlsCaPath, "utf8"); - } - - let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos); + let hostConnector = new HostConnector(hostConfig, pluginInfos); host = new Host( hostConnector, hostConfig, - tlsCa, pluginInfos, Boolean(args.canRestart), Boolean(args.recovery), @@ -308,7 +299,7 @@ I version of clusterio. Expect things to break. I if (err instanceof lib.WebSocketError) { // Already handled by connector.on("error") within Host - } else if (err instanceof lib.StartupError) { + } else if (err instanceof lib.StartupError || err.code === "InstallationError") { logger.fatal(` +----------------------------------+ | Unable to to start clusteriohost | diff --git a/packages/host/package.json b/packages/host/package.json index 638f1501..ee8bbfd6 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio host server", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", @@ -26,7 +26,6 @@ "set-blocking": "catalog:", "fs-extra": "catalog:", "jszip": "catalog:", - "phin": "catalog:", "winston": "catalog:", "winston-daily-rotate-file": "catalog:", "jimp": "^0.22.8", diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index f33bb604..57f0b911 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -3,7 +3,6 @@ import path from "path"; import events from "events"; import pidusage from "pidusage"; import setBlocking from "set-blocking"; -import phin from "phin"; import stream from "stream"; import util from "util"; @@ -75,6 +74,10 @@ async function discoverInstances(instancesDir: string) { return instances; } +// Requests that wake up instances are responsible for making sure the +// instance status is correctly updated even if they fail with an error. This +// means they should typically use try catch on anything that can throw +// and call notifyExit upon any errors. const instanceStartingMessages = new Set([ lib.InstanceStartRequest.name, lib.InstanceLoadScenarioRequest.name, @@ -259,10 +262,6 @@ export default class Host extends lib.Link { declare "connector": HostConnector; router = new HostRouter(this); - /** - * Certificate authority used to validate TLS connections to the controller. - */ - tlsCa?: string; pluginInfos: lib.PluginNodeEnvInfo[]; config: lib.HostConfig; @@ -304,7 +303,6 @@ export default class Host extends lib.Link { constructor( connector: HostConnector, hostConfig: lib.HostConfig, - tlsCa: string | undefined, pluginInfos: lib.PluginNodeEnvInfo[], /** * If true indicates that there is a process monitor present that @@ -319,7 +317,6 @@ export default class Host extends lib.Link { public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()), ) { super(connector); - this.tlsCa = tlsCa; this.pluginInfos = pluginInfos; this.config = hostConfig; @@ -448,6 +445,9 @@ export default class Host extends lib.Link { this.plugins.set(pluginInfo.name, hostPlugin); } catch (err: any) { + if (err.code === "InstallationError") { + throw err; + } throw new lib.PluginError(pluginInfo.name, err); } @@ -615,18 +615,17 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - core: { ca: this.tlsCa } as object, - stream: true, - }); + let response = await fetch(url); + if (!response.body) { + throw new Error(`Empty body downloading stream ${streamId} from controller`); + } const file = lib.ModInfo.filename(mod.name, mod.version); const filePath = path.join(this.modStore.modsDirectory, file); const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip"); const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" }); await events.once(writeStream, "open"); - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); await fs.rename(tempFilePath, filePath); @@ -1049,15 +1048,11 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - core: { ca: this.tlsCa } as object, - stream: true, - }); + let response = await fetch(url); - if (response.statusCode !== 200) { - let content = await lib.readStream(response); - throw new lib.RequestError(`Stream returned ${response.statusCode}: ${content.toString()}`); + if (response.status !== 200 || !response.body) { + let body = Buffer.from(await response.arrayBuffer()); + throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`); } let savesDir = path.join(instance.path, "saves"); @@ -1076,7 +1071,7 @@ export default class Host extends lib.Link { } } } - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); name = await lib.findUnusedName(savesDir, name, ".zip"); @@ -1091,10 +1086,11 @@ export default class Host extends lib.Link { checkRequestSaveName(name); let instance = this.getRequestInstance(instanceId); - let content: Buffer; + let content = fs.createReadStream(path.join(instance.path, "saves", name)); try { - // phin doesn't support streaming requests :( - content = await fs.readFile(path.join(instance.path, "saves", name)); + // TODO: Due to a bug in fs-extra the ready event is not emitted here. Replace + // "open" with "ready" when fs-extra is replaced with node's fs module. + await events.once(content, "open"); } catch (err: any) { if (err.code === "ENOENT") { throw new lib.RequestError(`${name} does not exist`); @@ -1104,10 +1100,10 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - phin({ - url, method: "PUT", - core: { ca: this.tlsCa } as object, - data: content, + fetch(url, { + method: "PUT", + body: content, + duplex: "half", }).catch(err => { logger.error(`Error pushing save to controller:\n${err.stack}`, this.instanceLogMeta(instanceId)); }); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index af5b9ef4..fa228ee8 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -1,7 +1,6 @@ import fs from "fs-extra"; import path from "path"; import pidusage from "pidusage"; -import phin from "phin"; import util from "util"; import type { Static } from "@sinclair/typebox"; import { exec } from "child_process"; @@ -638,9 +637,10 @@ end`.replace(/\r?\n/g, " "); } async init(pluginInfos: lib.PluginNodeEnvInfo[]) { + // After changing the status this function is responsible for calling notifyExit if it throws. this.notifyStatus("starting"); - const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest()); try { + const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest()); await this._loadStats(); await this.server.checkForUpdates(factorioVersions); await this.server.init(); @@ -845,7 +845,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_adminlist") !== "disabled") { this.logger.verbose("Writing server-adminlist.json"); - lib.safeOutputFile( + await lib.safeOutputFile( this.server.writePath("server-adminlist.json"), JSON.stringify([...this._host.adminlist], null, "\t") ); @@ -853,7 +853,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_banlist") !== "disabled") { this.logger.verbose("Writing server-banlist.json"); - lib.safeOutputFile( + await lib.safeOutputFile( this.server.writePath("server-banlist.json"), JSON.stringify([...this._host.banlist].map( ([username, reason]) => ({ username, reason }) @@ -863,10 +863,21 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_whitelist") !== "disabled") { this.logger.verbose("Writing server-whitelist.json"); - lib.safeOutputFile( + await lib.safeOutputFile( this.server.writePath("server-whitelist.json"), JSON.stringify([...this._host.whitelist], null, "\t") ); + } else { + // Always create the whitelist even if unused + // This allows it to be watched for changes without a server restart + try { + // wx will throw an error if the file already exists to prevent overwrite + await fs.writeFile(this.server.writePath("server-whitelist.json"), "\n[]\n", { flag: "wx" }); + } catch (err: any) { + if (err.code !== "EEXIST") { + throw err; + } + } } await this.syncMods(); @@ -1292,17 +1303,18 @@ end`.replace(/\r?\n/g, " "); let url = new URL(this._host.config.get("host.controller_url")); url.pathname += "api/upload-export"; url.searchParams.set("mod_pack_id", String(this.activeModPack.id)); - let response = await phin({ - url, method: "PUT", - data: content, - core: { ca: this._host.tlsCa } as object, + let response = await fetch(url, { + method: "PUT", + body: content, + duplex: "half", headers: { "Content-Type": "application/zip", "x-access-token": this._host.config.get("host.controller_token"), }, }); - if (response.statusCode !== 200) { - throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`); + if (response.status !== 200) { + const body = Buffer.from(await response.arrayBuffer()); + throw Error(`Upload failed: ${response.status} ${response.statusText}: ${body.toString()}`); } } finally { diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index 6152fee6..227d2319 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -887,7 +887,7 @@ export class FactorioServer extends events.EventEmitter { try { this._whitelistWatcher = fs.watch(filePath); } catch (err: any) { - this._logger.error(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`); + this._logger.warn(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`); return; } @@ -978,6 +978,10 @@ export class FactorioServer extends events.EventEmitter { .sort((v1, v2) => versionOrder(v1.version, v2.version)) [0]; + if (!latestVersion) { + return; + } + const installedVersions = await listFactorioVersions(this._factorioDir); if (!installedVersions.versions.has(latestVersion.version)) { const v = latestVersion.version; diff --git a/packages/lib/browser.ts b/packages/lib/browser.ts index 58c0debf..66b3d5c1 100644 --- a/packages/lib/browser.ts +++ b/packages/lib/browser.ts @@ -16,3 +16,6 @@ export * from "./src/subscriptions"; export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as RateLimiter } from "./src/RateLimiter"; export { default as ValueCache } from "./src/ValueCache"; + +import { checkSingletonImport } from "./src/check_singleton_import"; +checkSingletonImport(import.meta.url); diff --git a/packages/lib/index.ts b/packages/lib/index.ts index 96a4da80..d8240ffe 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -38,3 +38,6 @@ export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as ModStore } from "./src/ModStore"; export { default as RateLimiter } from "./src/RateLimiter"; export { default as ValueCache } from "./src/ValueCache"; + +import { checkSingletonImport } from "./src/check_singleton_import"; +checkSingletonImport(__filename); diff --git a/packages/lib/package.json b/packages/lib/package.json index 74e4f560..27c475b7 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -1,7 +1,7 @@ { "name": "@clusterio/lib", "description": "Shared library for Clusterio", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "repository": "https://github.com/clusterio/clusterio", "license": "MIT", "scripts": { diff --git a/packages/lib/src/check_singleton_import.ts b/packages/lib/src/check_singleton_import.ts new file mode 100644 index 00000000..6a734128 --- /dev/null +++ b/packages/lib/src/check_singleton_import.ts @@ -0,0 +1,21 @@ +import { InstallationError } from "./errors"; +declare global { + // eslint-disable-next-line vars-on-top + var _clusterioLibImportedFrom: string; +} + +export function checkSingletonImport(location: string) { + if (!global._clusterioLibImportedFrom) { + global._clusterioLibImportedFrom = location; + } else { + throw new InstallationError(`Attempt to import duplicate copy of @clusterio/lib + +Importing more than one copy of @clusterio/lib into the same runtime +will break Clusterio. This typically happens when the package +manager has installed two different versions of @clusterio/lib +and assigned different ones to different components of Clusterio. + +Previously imported @clusterio/lib was located at ${_clusterioLibImportedFrom} +This copy of @clusterio/lib is located at ${location}`); + } +} diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 498c018a..2d3fa645 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -285,7 +285,6 @@ export interface HostConfigFields { "host.instances_directory": string; "host.controller_url": string; "host.controller_token": string; - "host.tls_ca": string | null; "host.public_address": string; "host.factorio_port_range": string; "host.factorio_username": string | null, @@ -358,13 +357,6 @@ export class HostConfig extends classes.Config { initialValue: "enter token here", // TODO extendedValidation, valid JWT with matching host id }, - "host.tls_ca": { - description: "Path to Certificate Authority to validate TLS connection to controller against.", - restartRequired: true, - type: "string", - optional: true, - validator: validators.optional(validators.filePath), - }, "host.public_address": { description: "Public facing address players should connect to in order to join instances on this host", type: "string", @@ -679,7 +671,6 @@ export class InstanceConfig extends classes.Config { export interface ControlConfigFields { "control.controller_url": string | null; "control.controller_token": string | null; - "control.tls_ca": string | null; "control.max_reconnect_delay": number; } @@ -702,12 +693,6 @@ export class ControlConfig extends classes.Config { type: "string", optional: true, }, - "control.tls_ca": { - description: "Path to Certificate Authority to validate TLS connection to controller against.", - type: "string", - optional: true, - validator: validators.optional(validators.filePath), - }, "control.max_reconnect_delay": { title: "Max Reconnect Delay", description: "Maximum delay to wait before attempting to reconnect WebSocket", diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 0dae1a57..48b9d15b 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -290,18 +290,50 @@ export class KeyValueDatastore< export class SubscribableDatastore< V extends SubscribableValue, > extends Datastore { + /** Last time an update occurred */ + lastUpdatedAtMs: number = 0; + + private static maxUpdatedAtMs(values: Iterable) { + let max = 0; + for (const value of values) { + // Prevent one NaN, undefined, +Inf or similar from breaking the datastore. + if (Number.isFinite(value.updatedAtMs)) { + max = Math.max(max, value.updatedAtMs); + } + } + return max; + } + + private incrementLastUpdatedMs() { + this.lastUpdatedAtMs = Math.max(Date.now(), this.lastUpdatedAtMs + 1); + } + + constructor( + provider: DatastoreProvider = new MemoryDatastoreProvider(), + data = new Map(), + ) { + super(provider, data); + this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values()); + } + + async load() { + await super.load(); + this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values()); + } + // Set the value in the datastore, be careful of race conditions if you await any functions before calling set set(value: V) { + this.incrementLastUpdatedMs(); this.data.set(value.id, value); - value.updatedAtMs = Date.now(); + value.updatedAtMs = this.lastUpdatedAtMs; this.touch([value]); } // Set many values at once from an array of key value pairs setMany(values: V[]) { - const nowMs = Date.now(); + this.incrementLastUpdatedMs(); for (const value of values) { - value.updatedAtMs = nowMs; + value.updatedAtMs = this.lastUpdatedAtMs; this.data.set(value.id, value); } this.touch(values); @@ -309,18 +341,19 @@ export class SubscribableDatastore< // Delete a value in the datastore delete(value: V) { + this.incrementLastUpdatedMs(); this.data.delete(value.id); - value.updatedAtMs = Date.now(); + value.updatedAtMs = this.lastUpdatedAtMs; value.isDeleted = true; this.touch([value]); } // Delete many values at once from an array of values deleteMany(values: V[]) { - const nowMs = Date.now(); + this.incrementLastUpdatedMs(); for (const value of values) { value.isDeleted = true; - value.updatedAtMs = nowMs; + value.updatedAtMs = this.lastUpdatedAtMs; this.data.delete(value.id); } this.touch(values); diff --git a/packages/lib/src/errors.ts b/packages/lib/src/errors.ts index 00071895..e163ef60 100644 --- a/packages/lib/src/errors.ts +++ b/packages/lib/src/errors.ts @@ -10,6 +10,13 @@ export class CommandError extends Error { code = "CommandError"; } +/** + * Thrown if the installation is broken, such as when multiple copies of lib is imported + */ +export class InstallationError extends Error { + code = "InstallationError"; +} + /** * Thrown from requests sent when an error occured handling it */ diff --git a/packages/lib/src/external/FactorioVersions.ts b/packages/lib/src/external/FactorioVersions.ts index 45125433..86360b39 100644 --- a/packages/lib/src/external/FactorioVersions.ts +++ b/packages/lib/src/external/FactorioVersions.ts @@ -1,7 +1,7 @@ import { Static, Type } from "@sinclair/typebox"; -import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data"; +import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data/version"; -const ARCHIVE_URL = "https://factorio.com/download/archive"; +const ARCHIVE_URL = "https://factorio.com/download/archive/"; /** * Represents the parsed result of a factorio version diff --git a/packages/lib/src/link/connectors.ts b/packages/lib/src/link/connectors.ts index dd7df5d4..ebed9978 100644 --- a/packages/lib/src/link/connectors.ts +++ b/packages/lib/src/link/connectors.ts @@ -431,7 +431,6 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector) { pluginInfo = require(pluginPath).plugin; pluginPackage = require(path.posix.join(pluginPath, "package.json")); } catch (err: any) { + if (err.code === "InstallationError") { + throw err; + } throw new libErrors.PluginError(pluginName, err); } diff --git a/packages/lib/src/prometheus.ts b/packages/lib/src/prometheus.ts index 2a55ddd7..84999093 100644 --- a/packages/lib/src/prometheus.ts +++ b/packages/lib/src/prometheus.ts @@ -272,18 +272,31 @@ function labelsToKey(labels: LabelValues, metricLabels: string[]) { function keyToLabels(key: string) { let labels = new Map(); - if (key !== "") { - for (let pair of key.split(",")) { - let [name, value] = pair.split("=", 2); - if (!name || value === undefined) { - continue; + let pos = 0; + let matchLabel = /([^=]+)="(([^\\"]|\\["n\\])+)"/y; + while (pos < key.length) { + if (pos !== 0) { + /* istanbul ignore if */ + if (key[pos] !== ",") { + // Should be unreachable + throw Error(`Expected , at pos ${pos} in "${key}"`); } + pos += 1; + } + matchLabel.lastIndex = pos; + const match = matchLabel.exec(key); + /* istanbul ignore else */ + if (match) { + let [, name, value] = match; labels.set(name, value - .slice(1, -1) .replace(/\\"/g, "\"") .replace(/\\n/g, "\n") .replace(/\\\\/g, "\\") ); + pos += match[0].length; + } else { + // Should be unreachable + throw new Error(`Failed to parse label at pos ${pos} in "${key}"`); } } diff --git a/packages/lib/tsconfig.browser.json b/packages/lib/tsconfig.browser.json index 05554ac8..97bcd427 100644 --- a/packages/lib/tsconfig.browser.json +++ b/packages/lib/tsconfig.browser.json @@ -7,6 +7,7 @@ "./src/config/*.ts", "./src/data/*.ts", "./src/external/*.ts", + "./src/check_singleton_import.ts", "./src/datastore.ts", "./src/errors.ts", "./src/factorio/exchange_string.ts", diff --git a/packages/lib/tsconfig.node.json b/packages/lib/tsconfig.node.json index a963c8a0..ef6ccd30 100644 --- a/packages/lib/tsconfig.node.json +++ b/packages/lib/tsconfig.node.json @@ -1,6 +1,6 @@ { "extends": "../../tsconfig.node.json", - "include": ["index.ts", "browser.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"], + "include": ["index.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"], "compilerOptions": { "allowJs": true, }, diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 7cfe8161..8a63bdc5 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -3,7 +3,7 @@ "description": "Clusterio web interface implementation", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "browser": "src/index.ts", "main": "dist/browser/src/index.js", "scripts": { diff --git a/packages/web_ui/src/bootstrap.tsx b/packages/web_ui/src/bootstrap.tsx index 3b64bbd5..e1b6b1c3 100644 --- a/packages/web_ui/src/bootstrap.tsx +++ b/packages/web_ui/src/bootstrap.tsx @@ -137,7 +137,7 @@ export default async function bootstrap() { lib.addPluginConfigFields(pluginInfos); let wsUrl = new URL(webRoot, document.location.href); - let controlConnector = new ControlConnector(wsUrl.href, 120, undefined); + let controlConnector = new ControlConnector(wsUrl.href, 120); let control = new Control(controlConnector, new Map(pluginInfos.map(p => [p.name, p]))); control.plugins = await loadPlugins(pluginInfos, control); control.inputComponents = inputComponentsFromPlugins(control.plugins); diff --git a/packages/web_ui/src/components/InstanceRcon.tsx b/packages/web_ui/src/components/InstanceRcon.tsx index 75245f88..62702787 100644 --- a/packages/web_ui/src/components/InstanceRcon.tsx +++ b/packages/web_ui/src/components/InstanceRcon.tsx @@ -1,4 +1,4 @@ -import React, { useContext, useState } from "react"; +import React, { useContext, useRef, useState } from "react"; import { Input, Typography } from "antd"; import * as lib from "@clusterio/lib"; @@ -17,6 +17,7 @@ export default function InstanceRcon(props: InstanceRconProps) { let control = useContext(ControlContext); let [output, setOutput] = useState(null); let [running, setRunning] = useState(false); + let resultRef = useRef(null); async function sendCommand(command: string) { if (!command) { @@ -31,6 +32,8 @@ export default function InstanceRcon(props: InstanceRconProps) { new lib.InstanceSendRconRequest(command), ); setOutput(result); + const animation = resultRef.current?.children[0]?.getAnimations()[0]; + if (animation) { animation.currentTime = 0; } } finally { setRunning(false); } @@ -39,7 +42,7 @@ export default function InstanceRcon(props: InstanceRconProps) { return <> {output && <> Rcon result - {output} + {output} } ; } -export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: boolean, disabled?: boolean }) { +export function NpmButton(props: { + target: "controller" | { hostId: number }, + canRestart?: boolean, + disabled?: boolean +}) { const control = useContext(ControlContext); const account = useAccount(); const [open, setOpen] = useState(false); @@ -93,6 +97,7 @@ export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: bo remoteUpdates: true, pluginUpdates: true, pluginInstall: true, }); + const hostId = props.target === "controller" ? undefined : props.target.hostId; useEffect(() => { if (props.target === "controller") { if (!account.hasPermission("core.controller.get_config")) { @@ -121,7 +126,7 @@ export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: bo }); }).catch(notifyErrorHandler("Failed to fetch host config for remote updates")); } - }, [props.target]); + }, [hostId]); function onValuesChange({ action } : { action?: string }) { if (action) { diff --git a/packages/web_ui/src/style.css b/packages/web_ui/src/style.css index b082d447..937c4cd9 100644 --- a/packages/web_ui/src/style.css +++ b/packages/web_ui/src/style.css @@ -204,6 +204,12 @@ body { max-height: 200px; overflow-y: auto; white-space: pre-wrap; + animation: rcon-highlight 1s ease-out forwards; +} + +@keyframes rcon-highlight { + 0% { outline: 3px solid #1668dc; } + 100% { outline: 3px solid transparent; } } /* see https://blog.eqrion.net/pin-to-bottom/ for how scroll anchors works */ diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json index 14f0f38d..11119db4 100644 --- a/plugins/global_chat/package.json +++ b/plugins/global_chat/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-global_chat", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/inventory_sync/messages.ts b/plugins/inventory_sync/messages.ts index 532dfe2f..53178b56 100644 --- a/plugins/inventory_sync/messages.ts +++ b/plugins/inventory_sync/messages.ts @@ -18,6 +18,7 @@ export type IpcPlayerData = { hotbar?: string[], personal_logistic_slots?: {name:string, min:number, max:number}[], crafting_queue?: any, + recipe_notifications?: string, } const jsonPlayerData = Type.Object({ diff --git a/plugins/inventory_sync/module/get_script_data.lua b/plugins/inventory_sync/module/get_script_data.lua index eb92ad16..c51a198f 100644 --- a/plugins/inventory_sync/module/get_script_data.lua +++ b/plugins/inventory_sync/module/get_script_data.lua @@ -11,6 +11,7 @@ return function(no_early_return) inventory_sync.players = inventory_sync.players or {} inventory_sync.players_waiting_for_acquire = inventory_sync.players_waiting_for_acquire or {} inventory_sync.players_in_cutscene_to_sync = inventory_sync.players_in_cutscene_to_sync or {} + inventory_sync.failed_deserialization = inventory_sync.failed_deserialization or {} inventory_sync.active_downloads = inventory_sync.active_downloads or {} inventory_sync.finished_downloads = inventory_sync.finished_downloads or {} diff --git a/plugins/inventory_sync/module/inventory_sync.lua b/plugins/inventory_sync/module/inventory_sync.lua index 55a2ce3e..9805601c 100644 --- a/plugins/inventory_sync/module/inventory_sync.lua +++ b/plugins/inventory_sync/module/inventory_sync.lua @@ -36,6 +36,7 @@ local function remove_player(player) local script_data = get_script_data() script_data.players_waiting_for_acquire[player.name] = nil script_data.players_in_cutscene_to_sync[player.name] = nil + script_data.failed_deserialization[player.name] = nil script_data.active_downloads[player.name] = nil script_data.finished_downloads[player.name] = nil script_data.active_uploads[player.name] = nil @@ -73,7 +74,8 @@ function inventory_sync.serialize_player(player, player_record) end end - local serialized_player = serialize.serialize_player(player) + local failed_deserialization = get_script_data().failed_deserialization[player.name] or {} + local serialized_player = serialize.serialize_player(player, failed_deserialization) serialized_player.generation = player_record.generation return serialized_player @@ -106,7 +108,8 @@ function inventory_sync.deserialize_player(player, finished_record) -- Deserialize downloaded player data local serialized_player = compat.json_to_table(finished_record.data) - serialize.deserialize_player(player, serialized_player) + assert(type(serialized_player) == "table", "wrong type for serialized_player") + script_data.failed_deserialization[player.name] = serialize.deserialize_player(player, serialized_player) -- Restore player position and driving state restore_position(player, finished_record) diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index 989315e9..28f4fa06 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -6,6 +6,7 @@ local serialize = {} local v2_logistic_api = compat.version_ge("2.0.0") local v2_storage_api = compat.version_ge("2.0.0") +local recipe_notifications_api = compat.version_ge("2.0.67") function serialize.serialize_inventories(source, inventories) local serialized = {} @@ -342,28 +343,112 @@ function serialize.deserialize_crafting_queue(player, serialized) player.crafting_queue_progress = serialized.crafting_queue_progress end +---@param player LuaPlayer +---@param failed string[]? +---@return string? +function serialize.serialize_crafting_notifications(player, failed) + -- Get all unlocked recipes + local recipes = {} + for _, recipe in pairs(player.force.recipes) do + if recipe.enabled and not recipe.hidden then + recipes[recipe.name] = true + end + end + + -- Remove those with notifications + for _, recipe in pairs(player.get_recipe_notifications()) do + recipes[recipe.name] = nil + end + + -- Convert to a list of names + local recipe_names = failed or {} -- Includes previous recipes if any + local index = #recipe_names + 1 + for name in pairs(recipes) do + recipe_names[index] = name + index = index + 1 + end + + return helpers.encode_string(helpers.table_to_json(recipe_names)) +end + +--- @param player LuaPlayer +--- @param serialized string +--- @return string[]? failed Recipe names that do not exist +function serialize.deserialize_crafting_notifications(player, serialized) + -- Decode the recipe name strings + local recipe_names = helpers.json_to_table(assert(helpers.decode_string(serialized))) + assert(type(recipe_names) == "table", "wrong type decoded from json_to_table") + local clear_notification = player.clear_recipe_notification + local add_notification = player.add_recipe_notification + local notifications = {} + local failed = {} + + -- Assume all recipes need a notification + for _, recipe in pairs(player.force.recipes) do + if recipe.enabled and not recipe.hidden then + notifications[recipe.name] = true + end + end + + -- Remove recipes the player has cleared + for _, recipe_name in pairs(recipe_names) do + if notifications[recipe_name] then + notifications[recipe_name] = false + else + -- Recipe does not exist on this server + failed[#failed+1] = recipe_name + end + end + + -- Clear ones which are ones which are not required + for _, recipe in pairs(player.get_recipe_notifications()) do + if notifications[recipe.name] == false then + clear_notification(recipe.name) + end + notifications[recipe.name] = nil -- Prevents double add below + end + + -- Add any which the player did not have + for recipe_name, should_add in pairs(notifications) do + if should_add then + add_notification(recipe_name) + end + end + + -- Return failed recipes if any + return next(failed) and failed or nil +end + local controller_to_name = {} for name, value in pairs(defines.controllers) do controller_to_name[value] = name end --- Players are serialized into a table with the following fields: --- name --- controller: may not be cutscene or editor --- color --- chat_color --- tag --- force --- cheat_mod --- flashlight --- ticks_to_respawn (optional) --- character (optional) --- inventories: non-character inventories --- hotbar: table of string indexes to names (optional) --- personal_logistic_slots (optional) --- crafting_queue (optional) -function serialize.serialize_player(player) +--- @class SerializedPlayerData +--- @field generation number +--- @field name string +--- @field controller string +--- @field color Color +--- @field chat_color Color +--- @field tag string +--- @field force string +--- @field cheat_mode boolean +--- @field flashlight boolean +--- @field ticks_to_respawn number +--- @field character table? +--- @field inventories table? +--- @field hotbar table? +--- @field personal_logistic_slots table? +--- @field crafting_queue table? +--- @field recipe_notifications string? + +--- @param player LuaPlayer +--- @param failed_deserialization FailedDeserializationPlayerData +--- @return SerializedPlayerData +function serialize.serialize_player(player, failed_deserialization) + --- @type SerializedPlayerData local serialized = { + generation = 0, -- Gets replaced later controller = controller_to_name[player.controller_type], name = player.name, color = player.color, @@ -385,7 +470,9 @@ function serialize.serialize_player(player) -- Go back to waiting for respawn local character = player.character player.ticks_to_respawn = serialized.ticks_to_respawn - character.destroy() + if character and character.valid then + character.destroy() + end end -- Serialize character @@ -406,7 +493,7 @@ function serialize.serialize_player(player) if not serialized.hotbar then serialized.hotbar = {} end - serialized.hotbar[tostring(i)] = slot.name + serialized.hotbar[tostring(i)] = slot.name --[[@as string]] end end @@ -415,10 +502,23 @@ function serialize.serialize_player(player) serialized.crafting_queue = serialize.serialize_crafting_queue(player) end + -- Serialize recipe notifications + if recipe_notifications_api then + serialized.recipe_notifications = serialize.serialize_crafting_notifications(player, failed_deserialization.recipe_notifications) + end + return serialized end +--- @class FailedDeserializationPlayerData +--- @field recipe_notifications string[]? + +--- @param player LuaPlayer +--- @param serialized SerializedPlayerData +--- @return FailedDeserializationPlayerData? function serialize.deserialize_player(player, serialized) + local failed_deserialization = {} + if player.controller_type ~= defines.controllers[serialized.controller] or serialized.controller == "ghost" then -- If targeting the character or ghost controller then create a character if serialized.controller == "character" or serialized.controller == "ghost" then @@ -443,7 +543,9 @@ function serialize.deserialize_player(player, serialized) player.set_controller({ type = defines.controllers.god }) player.set_controller({ type = defines.controllers.ghost }) end - character.destroy() + if character and character.valid then + character.destroy() + end end else @@ -496,6 +598,13 @@ function serialize.deserialize_player(player, serialized) serialize.deserialize_crafting_queue(player, serialized.crafting_queue) end + -- Deserialize recipe notifications + if recipe_notifications_api and serialized.recipe_notifications then + failed_deserialization.recipe_notifications = + serialize.deserialize_crafting_notifications(player, serialized.recipe_notifications) + end + + return next(failed_deserialization) and failed_deserialization or nil end return serialize diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index 73e69188..d8a165ad 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-inventory_sync", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Danielv ", "homepage": "https://github.com/clusterio/factorioClusterio#readme", diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index 358f04c4..edc2f632 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-player_auth", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin authenticating logged in players to the web interface", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json index 29704b02..426c17eb 100644 --- a/plugins/research_sync/package.json +++ b/plugins/research_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-research_sync", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin for synchronizing research between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json index 683cf207..ba8df382 100644 --- a/plugins/statistics_exporter/package.json +++ b/plugins/statistics_exporter/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-statistics_exporter", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin for gathering statistics from Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index 63187fe0..9177e837 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-subspace_storage", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin for sharing storage between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/subspace_storage/test/routes.js b/plugins/subspace_storage/test/routes.js index 2753b41e..fc4aac5b 100644 --- a/plugins/subspace_storage/test/routes.js +++ b/plugins/subspace_storage/test/routes.js @@ -9,7 +9,7 @@ describe("subspace_storage endpoints", function() { describe("GET /api/inventory", function() { it("should return the controllers current inventory", async function() { let res = await get("/api/inventory"); - let inventory = res.body; + let inventory = await res.json(); assert.equal(typeof inventory, "object", "Inventory should be an object"); return; // XXX Disabled until a suitable test interface is added. diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 29bc79e4..fc8d08c6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,35 +6,35 @@ packages: - "test/" catalog: - "@types/node": ^20.17.19 - "@types/fs-extra": ^11.0.1 - "@types/express": ^4.17.17 - "@types/finalhandler": ^1.2.0 - "@types/jsonwebtoken": ^9.0.9 - "@types/set-blocking": ^2.0.0 - "@types/react": ^18.3.18 - "@types/ws": ^8.5.14 "@sinclair/typebox": ^0.30.4 "@swc/core": ^1.13.3 + "@types/express": ^4.17.17 + "@types/finalhandler": ^1.2.0 + "@types/fs-extra": ^11.0.1 + "@types/jsonwebtoken": ^9.0.9 + "@types/node": ^20.17.19 + "@types/react": ^18.3.18 + "@types/set-blocking": ^2.0.0 + "@types/ws": ^8.5.14 "antd": ^5.24.2 + "chalk": ^4.1.2 + "express": ^5.1.0 + "fs-extra": ^8.1.0 + "jsonwebtoken": ^9.0.2 + "jszip": ^3.10.1 "react": ^18.2.0 "react-dom": ^18.2.0 "react-router-dom": ^6.29.0 + "set-blocking": ^2.0.0 "typescript": ^5.7.3 "webpack": ^5.98.0 "webpack-cli": ^5.1.4 "webpack-merge": ^5.9.0 - "fs-extra": ^8.1.0 - "express": ^5.1.0 "winston": ^3.17.0 - "winston-transport": ^4.9.0 "winston-daily-rotate-file": ^4.7.1 - "jsonwebtoken": ^9.0.2 - "set-blocking": ^2.0.0 - "jszip": ^3.10.1 - "chalk": ^4.1.2 - "phin": ^3.7.0 + "winston-transport": ^4.9.0 "ws": ^8.18.1 catalogMode: strict linkWorkspacePackages: true +shellEmulator: true diff --git a/scripts/github_api.mts b/scripts/github_api.mts new file mode 100644 index 00000000..3b4f16ea --- /dev/null +++ b/scripts/github_api.mts @@ -0,0 +1,136 @@ +export async function githubFetch( + path: string, + query: Record= {}, + init: RequestInit & { headers?: Record } = {}, +) { + const url = new URL("https://api.github.com/"); + url.pathname = path; + for (const [name, value] of Object.entries(query)) { + url.searchParams.set(name, value); + } + init = { + ...init, + headers: { + ...init.headers as Record ?? {}, + "Accept": "application/vnd.github+json", + } + }; + if (process.env.GH_TOKEN) { + init.headers!["Authorization"] = `Bearer ${process.env.GH_TOKEN}`; + } + console.log("Fetching", url.href); + const response = await fetch(url, init); + if (!response.ok) { + console.log(response.headers); + throw new Error(`GitHub replied with ${response.status} ${response.statusText}: ${await response.text()}`); + } + if (!response.headers.get("Content-Type")?.startsWith("application/json")) { + throw new Error( + `GitHub replied with ${response.status} ${response.statusText}: ${response.headers.get("Content-Type")}` + ); + } + return response; +} + +export async function githubFetchJson( + path: string, + query: Record = {}, + init: RequestInit & { headers?: Record } = {}, +): Promise { + const response = await githubFetch(path, query, init); + return await response.json(); +} + +export async function githubFetchJsonPaginated( + path: string, + query: Record = {}, + init: RequestInit & { headers?: Record } = {}, + shouldContinue: (result: T[]) => boolean = () => true, +): Promise { + const data: T[] = []; + let pagesRemaining = true; + query = { + ...query, + per_page: "100", + }; + while (pagesRemaining) { + const response = await githubFetch(path, query, init); + const responseItems = await response.json(); + data.push(...responseItems); + const linkHeader = response.headers.get("Link"); + pagesRemaining = linkHeader?.includes(`rel="next"`) ?? false; + if (pagesRemaining) { + pagesRemaining = shouldContinue(responseItems); + } + if (pagesRemaining) { + query = { + ...query, + page: String((query.page ? (Number(query.page)) : 1) + 1), + } + } + } + return data; +} + +// Minimal types that include only what is used and useful for our scripts. See +// https://docs.github.com/en/rest for complete reference. +export interface PullRequest { + number: number, + url: string, + html_url: string, + state: string, + title: string, + body: string | null, + created_at: string, + updated_at: string, + closed_at: string | null, + merged_at: string | null, + merge_commit_sha: string | null, +} + +export interface Issue { + number: number, + title: string, + url: string, + html_url: string, + state: string, + body?: string | null, + pull_request?: { + url: string, + html_url: string, + merged_at?: string | null, + }, +} + +export interface Release { + tag_name: string, + target_commitish: string, + name: string | null, + body: string | null, + created_at: string, + published_at: string | null, + updated_at?: string | null, +} + +export interface Commit { + sha: string, + commit: { + message: string, + }, + author: null | {} | { + name?: null | string, + login: string, + type: string, + }, + parents: { + sha: string, + }[], +} + +export interface Reference { + ref: string, + object: { + type: string, + sha: string, + }, +} diff --git a/scripts/pr_to_changelog.mts b/scripts/pr_to_changelog.mts new file mode 100644 index 00000000..d974937f --- /dev/null +++ b/scripts/pr_to_changelog.mts @@ -0,0 +1,233 @@ +import { + type Release, + type Issue, + githubFetchJson, + githubFetchJsonPaginated, +} from "./github_api.mts"; + +type Changelog = Record; + +const changelogSections = [ + "Major Features", + "Features", + "Fixes", + "Changes", + "Breaking Changes", + "Meta", +]; + +// Generic result codes +const success = 0; +const failure = 1; + +type ParserState = { + pos: number, + lines: string[], +}; + +function createParser(lines: string[]): ParserState { + return { pos: 0, lines }; +} + +function atEnd(parser: ParserState) { + return parser.pos >= parser.lines.length; +} + +function currentLine(parser: ParserState) { + if (atEnd(parser)) { + throw new Error("Attempt to access line past end"); + } + return parser.lines[parser.pos]; +} + +function skipToLine( + parser: ParserState, + pattern: RegExp +) { + for (; parser.pos < parser.lines.length; parser.pos++) { + if (pattern.test(currentLine(parser))) { + return success; + } + } + return failure; +} + +function skipEmptyLines(parser: ParserState) { + while (parser.pos < parser.lines.length && /^ *$/.test(currentLine(parser))) { + parser.pos += 1; + } +} + +function extractPullRequestChangelog( + parser: ParserState, +): + | [typeof success, string[]] + | [typeof failure, undefined, string] +{ + const headerFail = skipToLine(parser, /^###? Changelog *$/); + if (headerFail) { + return [failure, , "Header not found"]; + } + parser.pos += 1; + skipEmptyLines(parser); + if (!atEnd(parser) && /^None/i.test(currentLine(parser))) { + parser.pos += 1; + return [success, []]; + } + const blockStartFail = skipToLine(parser, /```/); + if (blockStartFail) { + return [failure, , "Code block not found after header"]; + } + parser.pos += 1 + const blockStartPos = parser.pos; + const blockEndFail = skipToLine(parser, /```/); + if (blockEndFail) { + return [failure, , "End of code block not found after header"]; + } + return [success, parser.lines.slice(blockStartPos, parser.pos)]; +} + +function parseChangelog(parser: ParserState, pr: Issue, issues: Issue[]): [Changelog, string[]] { + let section: string | undefined; + let changelog: Changelog = Object.create(null); + let warnings: string[] = []; + while (!atEnd(parser)) { + const line = currentLine(parser); + parser.pos += 1; + if (line.trim() === "") { + continue; + } + if (line.startsWith("###")) { + section = line.slice(4).trim(); + if (!changelogSections.includes(section)) { + warnings.push( + `Changelog section ${section} on line ${parser.pos} is not one of ${changelogSections.join(", ")}` + ); + } + changelog[section] = changelog[section] ?? []; + continue; + } + if (!section) { + warnings.push(`Unexpected content "${line}" outside of a changelog section on line ${parser.pos}`); + continue; + } + if (line.startsWith("- ")) { + const entry = line + .slice(2) + .trim() + .replace(/\[#(\d+)\]\(https:\/\/github.com\/clusterio\/clusterio\/(issues|pull)\/\d+\)/g, "#$1"); + let hasRef = false + for (const ref of entry.matchAll(/#(\d+)/g)) { + hasRef = true; + const id = Number.parseInt(ref[1], 10); + const refIssue = issues.find(issue => issue.number === id); + if (!refIssue) { + warnings.push( + `Issue reference ${ref[0]} on line ${parser.pos} does not exist ` + + "or has not been updated since the last release" + ); + } else if (refIssue.pull_request && refIssue.number !== pr.number) { + warnings.push( + `Pull request reference ${ref[0]} on line ${parser.pos} is to a ` + + "different Pull request than this one." + ); + } + } + if (!hasRef) { + warnings.push(`Changelog entry on line ${parser.pos} does not reference an issue`); + } + changelog[section].push(entry); + continue; + } + warnings.push(`Unexpected content "${line}" on line ${parser.pos}`); + } + return [changelog, warnings]; +} + +function changelogFromPullRequests(pullRequests: Issue[], issues: Issue[]) { + const mergedChangelog: Changelog = Object.create(null); + for (const pr of pullRequests) { + const [ + changelogFail, + changelogLines, + changelogFailReason + ] = extractPullRequestChangelog(createParser(pr.body!.split(/\r?\n/))); + if (changelogFail) { + console.error(`Failed to extract changelog in ${pr.html_url}: ${changelogFailReason}`) + continue; + } + const [changelog, warnings] = parseChangelog(createParser(changelogLines), pr, issues); + for (const warning of warnings) { + console.warn(`Warning in changelog for ${pr.html_url}: ${warning}`) + } + for (const [section, items] of Object.entries(changelog)) { + mergedChangelog[section] = [...mergedChangelog[section] ?? [], ...items]; + } + } + return mergedChangelog; +} + +async function fetchLastRelease() { + // Assumes releases are ordered by their creation or tag time in descending order. + const releases = await githubFetchJson("/repos/clusterio/clusterio/releases"); + const latest = releases.find(r => r.target_commitish === "master"); + if (!latest) { + throw new Error("Unable to find latest release on master"); + } + return latest; +} + +async function fetchIssuesUpdatedSince(since: string) { + return await githubFetchJsonPaginated( + "/repos/clusterio/clusterio/issues", + { state: "all", since, sort: "updated", direction: "asc" }, + ); +} + +function printMarkdown(changelog: Changelog, issues: Issue[], refText: (issue: Issue) => string) { + const sections = new Set([...changelogSections, ...Object.keys(changelog)]); + for (const section of sections) { + if (section in changelog) { + console.log(`### ${section}`); + const entries = changelog[section]; + console.log(entries.map(text => ( + `- ${text.replaceAll(/#(\d+)/g, (ref, idAsText) => { + const id = Number.parseInt(idAsText, 10); + if (!Number.isFinite(id)) { + return ref; + } + const issue = issues.find(issue => issue.number === id); + if (!issue) { + return ref; + } + return refText(issue); + })}` + )).join("\n")); + console.log(); + } + } +} + +async function main() { + const lastRelease = await fetchLastRelease(); + const issues = await fetchIssuesUpdatedSince(lastRelease.created_at); + + const pullRequests = issues.filter(issue => ( + issue.pull_request + && issue.pull_request.merged_at + && issue.pull_request.merged_at > lastRelease.created_at + )); + + const changelog = changelogFromPullRequests(pullRequests, issues); + + console.log(); + console.log("=== Github markdown ==="); + printMarkdown(changelog, issues, issue => `clusterio/clusterio#${issue.number}`); + console.log(); + console.log("=== Discord markdown ==="); + printMarkdown(changelog, issues, issue => `[#${issue.number}](<${issue.html_url}>`); +} + +if (import.meta.main) { + main().catch(console.error); +} diff --git a/test/controller/Controller.js b/test/controller/Controller.js index ae45a11b..7b0d4127 100644 --- a/test/controller/Controller.js +++ b/test/controller/Controller.js @@ -252,6 +252,28 @@ describe("controller/src/Controller", function() { assert.equal(controller.config.restartRequired, true); }); }); + + describe(".factorioVersions", function() { + const _fetch = global.fetch; + const fetchCalls = []; + before(function() { + global.fetch = async (url) => { + fetchCalls.push(url); + return { ok: false, status: 503, statusText: "Service Unavailable" }; + }; + }); + it("should handle fetch failure", async function() { + await assert.deepEqual( + await controller.factorioVersions.get(0), + [], + ); + assert(fetchCalls.length === 1, "expected one fetch call"); + assert(fetchCalls[0].indexOf("/download") !== -1, "fetch call to be to a URL with /download in it"); + }); + after(function() { + global.fetch = _fetch; + }); + }); }); describe("finalise", function() { describe(".finaliseHosts()", function() { diff --git a/test/controller/HttpCloser.js b/test/controller/HttpCloser.js index d630600b..d5e34122 100644 --- a/test/controller/HttpCloser.js +++ b/test/controller/HttpCloser.js @@ -4,8 +4,7 @@ const fs = require("fs-extra"); const http = require("http"); const https = require("https"); const util = require("util"); - -const phin = require("phin"); +const events = require("node:events"); const HttpCloser = require("@clusterio/controller/dist/node/src/HttpCloser").default; @@ -38,8 +37,16 @@ function serverSuite(proto) { } async function get(url, options = {}) { - let core = { ...clientOptions, ...(options.core || {}) }; - return await phin({ ...options, url, core }); + const request = api.request(url, { ...clientOptions, ...options }); + request.end(); + const [response] = await events.once(request, "response"); + response.body = await new Promise((resolve, reject) => { + const chunks = []; + response.on("data", chunk => { chunks.push(chunk); }); + response.on("error", err => { reject(err); }); + response.on("end", () => { resolve(Buffer.concat(chunks)); }); + }); + return response; } before(async function() { @@ -101,7 +108,7 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true }); - let request = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); await waitForStage1; stage2(); await closer.close(); @@ -121,7 +128,7 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true }); - let request = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); await waitForStage1; await closer.close(); @@ -145,7 +152,7 @@ function serverSuite(proto) { let close = catched(closer.close()); let agent = new api.Agent({ keepAlive: true }); - await assert.rejects(get(addr, { core: { agent }}), { code: "ECONNREFUSED" }); + await assert.rejects(get(addr, { agent }), { code: "ECONNREFUSED" }); let response = await request; assert.equal(response.statusCode, 200, "non-200 response status code"); @@ -165,7 +172,7 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true }); - let request = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); await waitForStage1; stage2(); await closer.close(); @@ -186,8 +193,8 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true, maxSockets: 1 }); - let request = catched(get(addr, { core: { agent }})); - let droppedRequest = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); + let droppedRequest = catched(get(addr, { agent })); await waitForStage1; stage2(); await closer.close(); @@ -212,10 +219,7 @@ function serverSuite(proto) { await closer.close(tick*2); stage2(); - // Due to a bug in phin [https://github.com/ethanent/phin/issues/59] - // the request Promise never settles. - // TODO: uncomment when bug is fixed in phin. - // await assert.rejects(request); + await assert.rejects(request); }); } diff --git a/test/controller/User.test.js b/test/controller/User.test.js index 3e5ea4ab..cf605280 100644 --- a/test/controller/User.test.js +++ b/test/controller/User.test.js @@ -37,6 +37,12 @@ describe("controller/User", function () { user.saveRecord(); assert(user.updatedAtMs > prev); }); + it("should always increment updatedAtMs even if called in quick succession", function () { + user.saveRecord(); + let prev = user.updatedAtMs; + user.saveRecord(); + assert(user.updatedAtMs > prev, "updatedAtMs did not increase"); + }); }); describe(".set()", function () { diff --git a/test/host.js b/test/host.js index 63aa61bc..ad7a16aa 100644 --- a/test/host.js +++ b/test/host.js @@ -127,8 +127,8 @@ describe("Host testing", function() { version: hostVersion, }]; const hostConfig = new lib.HostConfig("host", { "host.version": hostVersion }); - const hostConnector = new HostConnector(hostConfig, undefined, pluginInfos); - host = new Host(hostConnector, hostConfig, undefined, pluginInfos); + const hostConnector = new HostConnector(hostConfig, pluginInfos); + host = new Host(hostConnector, hostConfig, pluginInfos); }); it("returns false when no changes are present", async function() { host.pluginInfos[0].hostEntrypoint = true; diff --git a/test/host/server.js b/test/host/server.js index 28d21054..13de4e75 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -342,264 +342,124 @@ describe("host/server", function() { }); }); - describe("full version", function() { - it("should do nothing when there is no newer version", async function() { - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); + for (const [suite, target] of [ + ["full version", "0.1.1"], + ["partial version", "0.1"], + ["latest version", "latest"], + ]) { + /* eslint-disable no-loop-func */ + describe(suite, function() { + it("should do nothing when there are no versions", async function() { + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target; + await server.checkForUpdates([]); - assert.equal(fetchCalledWith, null); + assert.equal(fetchCalledWith, null); + }); + if (target !== "latest") { + it("should do nothing when no version matches", async function() { + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target; + await server.checkForUpdates([{ + stable: true, + version: "0.2.1", + headlessUrl: "test1", + }, { + stable: false, + version: "0.2.0", + headlessUrl: "test2", + }]); + + assert.equal(fetchCalledWith, null); + }); + } + it("should do nothing when there is no newer version", async function() { + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target; + await server.checkForUpdates([{ + stable: true, + version: "0.1.1", + headlessUrl: "test1", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test2", + }]); + + assert.equal(fetchCalledWith, null); + }); + it("should do nothing for direct installs", async function() { + server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); + server._targetVersion = target === "0.1.1" ? "0.1.5" : target; + await server.checkForUpdates([{ + stable: true, + version: "0.1.5", + headlessUrl: "test1", + }, { + stable: true, + version: "0.1.1", + headlessUrl: "test1", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test2", + }]); + + assert.equal(fetchCalledWith, null); + }); + it("should do nothing when on windows", async function() { + let logLine = null; + server._logger = { info: line => { logLine = line; } }; + Object.defineProperty(process, "platform", { value: "win32" }); + + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target === "0.1.1" ? "0.1.5" : target; + await server.checkForUpdates([{ + stable: true, + version: "0.1.5", + headlessUrl: "test1", + }, { + stable: true, + version: "0.1.1", + headlessUrl: "test2", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test3", + }]); + + assert.equal(fetchCalledWith, null); + assert.ok(logLine !== null); + assert.ok(logLine.endsWith("but must be manually downloaded")); + }); + it("should do attempt to download on linux", async function() { + let logLine = null; + server._logger = { info: line => { logLine = line; } }; + Object.defineProperty(process, "platform", { value: "linux" }); + + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target === "0.1.1" ? "0.1.5" : target; + await assert.rejects(server.checkForUpdates([{ + stable: true, + version: "0.1.5", + headlessUrl: "test1", + }, { + stable: true, + version: "0.1.1", + headlessUrl: "test2", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test3", + }]), new Error("Failed to fetch test1: -1 Fetch called")); + + assert.equal(fetchCalledWith, "test1"); + assert.ok(logLine !== null); + assert.ok(logLine.endsWith("starting download...")); + }); }); - it("should do nothing for direct installs", async function() { - server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); - server._targetVersion = "0.1.5"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing when on windows", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "win32" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1.5"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]); - - assert.equal(fetchCalledWith, null); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("but must be manually downloaded")); - }); - it("should do attempt to download on linux", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "linux" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1.5"; - await assert.rejects(server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]), new Error("Failed to fetch test1: -1 Fetch called")); - - assert.equal(fetchCalledWith, "test1"); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("starting download...")); - }); - }); - describe("partial version", function() { - it("should do nothing when there is no newer version", async function() { - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing for direct installs", async function() { - server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); - server._targetVersion = "0.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing when on windows", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "win32" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]); - - assert.equal(fetchCalledWith, null); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("but must be manually downloaded")); - }); - it("should do attempt to download on linux", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "linux" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1"; - await assert.rejects(server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]), new Error("Failed to fetch test1: -1 Fetch called")); - - assert.equal(fetchCalledWith, "test1"); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("starting download...")); - }); - }); - describe("latest version", function() { - it("should do nothing when there is no newer version", async function() { - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "latest"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing for direct installs", async function() { - server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); - server._targetVersion = "latest"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing when on windows", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "win32" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "latest"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]); - - assert.equal(fetchCalledWith, null); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("but must be manually downloaded")); - }); - it("should do attempt to download on linux", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "linux" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "latest"; - await assert.rejects(server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]), new Error("Failed to fetch test1: -1 Fetch called")); - - assert.equal(fetchCalledWith, "test1"); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("starting download...")); - }); - }); + /* eslint-enable no-loop-func */ + } it("should download a version correctly (live api)", async function() { slowTest(this); if (_platform !== "linux") { diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 6052973f..be234991 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -4,7 +4,6 @@ const fs = require("fs-extra"); const jwt = require("jsonwebtoken"); const path = require("path"); const events = require("events"); -const phin = require("phin"); const lib = require("@clusterio/lib"); const libBuildMod = require("@clusterio/lib/build_mod"); @@ -41,7 +40,6 @@ async function startAltHost() { await fs.remove(path.join("temp", "test", "alt-instances")); await fs.remove(path.join("temp", "test", "alt-mods")); await execCtl(`host create-config --id 5 --name alt-host --generate-token --output ${config}`); - await execHost(`--config ${config} config set host.tls_ca ../../test/file/tls/cert.pem`); await execHost(`--config ${config} config set host.instances_directory alt-instances`); const dir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); await execHost(`--config ${config} config set host.factorio_directory ${dir}`); @@ -80,16 +78,15 @@ async function runWithAltHost(callback) { } async function uploadSave(instanceId, name, content) { - return await phin({ - url: `https://localhost:4443/api/upload-save?instance_id=${instanceId}&filename=${name}`, + const uploadUrl = `https://localhost:4443/api/upload-save?instance_id=${instanceId}&filename=${name}`; + return await fetch(uploadUrl, { method: "POST", - core: { rejectUnauthorized: false }, headers: { "X-Access-Token": controlToken, "Content-Type": "application/zip", "Content-Length": String(content.length), }, - data: content, + body: content, }); } @@ -168,8 +165,7 @@ describe("Integration of Clusterio", function() { describe("run", function() { it("should handle resume of an active connection", async function() { slowTest(this); - let tlsCa = await fs.readFile("test/file/tls/cert.pem"); - let connectorA = new TestControlConnector(url, 2, tlsCa); + let connectorA = new TestControlConnector(url, 2); connectorA.token = controlToken; let controlA = new TestControl(connectorA); await connectorA.connect(); @@ -177,7 +173,7 @@ describe("Integration of Clusterio", function() { connectorA.stopHeartbeat(); connectorA.on("error", () => {}); - let connectorB = new TestControlConnector(url, 2, tlsCa); + let connectorB = new TestControlConnector(url, 2); connectorB.token = controlToken; connectorB.src = connectorA.src; let controlB = new TestControl(connectorB); @@ -1679,8 +1675,7 @@ describe("Integration of Clusterio", function() { it("should kick existing sessions for the user", async function() { slowTest(this); await getControl().send(new lib.UserCreateRequest("revokee")); - let tlsCa = await fs.readFile("test/file/tls/cert.pem"); - let connector = new TestControlConnector(url, 2, tlsCa); + let connector = new TestControlConnector(url, 2); connector.token = jwt.sign( { aud: "user", user: "revokee" }, Buffer.from("TestSecretDoNotUse", "base64") ); @@ -1709,8 +1704,7 @@ describe("Integration of Clusterio", function() { await execCtl("user set-roles temp Player"); let tempControl; try { - let tlsCa = await fs.readFile("test/file/tls/cert.pem"); - let connector = new TestControlConnector(url, 2, tlsCa); + let connector = new TestControlConnector(url, 2); connector.token = jwt.sign( { aud: "user", user: "temp" }, Buffer.from("TestSecretDoNotUse", "base64") ); diff --git a/test/integration/index.js b/test/integration/index.js index aa6f24ca..3a11620d 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -4,7 +4,6 @@ const path = require("path"); const fs = require("fs-extra"); const child_process = require("child_process"); const jwt = require("jsonwebtoken"); -const phin = require("phin"); const util = require("util"); const events = require("events"); @@ -148,13 +147,12 @@ function hasFactorio() { } async function get(urlPath) { - let res = await phin({ - method: "GET", - url: `https://localhost:4443${urlPath}`, - core: { rejectUnauthorized: false }, - }); - if (res.statusCode !== 200) { - throw new Error(`Got response code ${res.statusCode}, content: ${res.body}`); + const url = new URL("https://localhost:4443"); + url.pathname = urlPath; + let res = await fetch(url); + if (res.status !== 200) { + const body = Buffer.from(await res.arrayBuffer()); + throw new Error(`Got response code ${res.status}, content: ${body.toString()}`); } return res; } @@ -194,11 +192,18 @@ let controllerConfigPath = path.join("temp", "test", "config-controller.json"); let hostConfigPath = path.join("temp", "test", "config-host.json"); let controlConfigPath = path.join("temp", "test", "config-control.json"); +function childOptions(options) { + return { + cwd: path.join("temp", "test"), + env: { ...process.env }, + ...options, + }; +} + function exec(command, options = {}) { // Uncomment to show commands run in tests // console.log(command); - options = { cwd: path.join("temp", "test"), ...options }; - return util.promisify(child_process.exec)(command, options); + return util.promisify(child_process.exec)(command, childOptions(options)); } function execController(...args) { @@ -264,7 +269,7 @@ function spawn(name, cmd, waitFor, options = {}) { return new Promise((resolve, reject) => { log(cmd); let parts = cmd.split(" "); - let process = child_process.spawn(parts[0], parts.slice(1), { cwd: path.join("temp", "test"), ...options }); + let process = child_process.spawn(parts[0], parts.slice(1), childOptions(options)); let stdout = new LineSplitter({ readableObjectMode: true }); let stderr = new LineSplitter({ readableObjectMode: true }); let onDataOut = line => { @@ -354,7 +359,6 @@ before(async function() { console.log("Bootstrapping"); await execController("bootstrap create-admin test"); await execController("bootstrap create-ctl-config test"); - await execCtlProcess("control-config set control.tls_ca ../../test/file/tls/cert.pem"); controllerProcess = await spawnNode("controller:", "../../packages/controller run", /Started controller/); @@ -362,12 +366,10 @@ before(async function() { const relativeFactorioDir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); await execCtlProcess("host create-config --id 4 --name host --generate-token"); await execHost(`config set host.factorio_directory ${relativeFactorioDir}`); - await execHost("config set host.tls_ca ../../test/file/tls/cert.pem"); hostProcess = await spawnNode("host:", "../../packages/host run", /Started host/); - const tlsCa = await fs.readFile("test/file/tls/cert.pem"); - const controlConnector = new TestControlConnector(url, 2, tlsCa); + const controlConnector = new TestControlConnector(url, 2); controlConnector.token = controlToken; control = new TestControl(controlConnector); await controlConnector.connect(); @@ -375,7 +377,6 @@ before(async function() { const initArgs = await initializeCtl(`--plugin-list ${pluginListPath} control-config list`, undefined, true); control.config = initArgs.controlConfig; control.plugins = initArgs.ctlPlugins; - control.tlsCa = tlsCa; const testPack = lib.ModPack.fromJSON({}); testPack.id = 12; diff --git a/test/integration/instance.js b/test/integration/instance.js index 2955e001..7cba7de8 100644 --- a/test/integration/instance.js +++ b/test/integration/instance.js @@ -1,11 +1,13 @@ "use strict"; +const path = require("path"); const assert = require("assert").strict; const lib = require("@clusterio/lib"); +const fs = require("fs-extra"); const { testMatrix } = require("../common"); const { slowTest, exec, execCtl, execCtlProcess, sendRcon, getControl, - requiresFactorio, hasFactorio, + requiresFactorio, hasFactorio, instancesDir, } = require("./index"); const instId = 48; @@ -67,6 +69,32 @@ describe("Clusterio Instance", function() { await execCtl(`instance stop ${instAltName}`); await execCtl(`instance delete ${instAltName}`); }); + describe("whitelist creation", function() { + const whitelistPath = path.join(instancesDir, instName, "server-whitelist.json"); + before(async function() { + this.timeout(20000); + await execCtl(`${instSetConfig} factorio.sync_whitelist disabled`); + await execCtl(`${instSetConfig} factorio.enable_whitelist false`); + }); + afterEach(async function() { + this.timeout(20000); + await execCtl(`instance stop ${instName}`); + }); + it("should always create a whitelist json even if unused", async function() { + slowTest(this); + await fs.remove(whitelistPath); + await execCtl(`instance start ${instName}`); + const whitelist = await fs.readJSON(whitelistPath); + assert.deepEqual(whitelist, []); + }); + it("should not overwrite the existing whitelist if sync is disabled", async function() { + slowTest(this); + await fs.writeJSON(whitelistPath, ["test_user"]); + await execCtl(`instance start ${instName}`); + const whitelist = await fs.readJSON(whitelistPath); + assert.deepEqual(whitelist, ["test_user"]); + }); + }); for (const [savePatchingEnabled, scriptCommandsEnabled] of testMatrix([true, false], [true, false])) { const strSavePatching = `save patching: ${savePatchingEnabled ? "Enabled" : "Disabled"}`; const strScriptCommands = `script commands: ${scriptCommandsEnabled ? "Enabled" : "Disabled"}`; @@ -81,6 +109,7 @@ describe("Clusterio Instance", function() { before(async function() { this.timeout(20000); // Create and start a patched save + await execCtl(`${instSetConfig} factorio.enable_whitelist true`); await execCtl(`${instSetConfig} factorio.enable_save_patching ${savePatchingEnabled}`); await execCtl(`${instSetConfig} factorio.enable_script_commands ${scriptCommandsEnabled}`); await execCtl(`instance save create ${instName}`); diff --git a/test/integration/link.js b/test/integration/link.js index be19acd6..3d75fbfd 100644 --- a/test/integration/link.js +++ b/test/integration/link.js @@ -11,18 +11,14 @@ let token = jwt.sign({ aud: "user", user: "test" }, Buffer.from("TestSecretDoNot let tokenHost = jwt.sign({ aud: "host", host: 0 }, Buffer.from("TestSecretDoNotUse", "base64")); describe("Integration of lib/link", function() { - let tlsCa; let control; let controlConnector; let hostConnector; - before(async function() { - tlsCa = await fs.readFile("test/file/tls/cert.pem"); - }); beforeEach(async function() { - controlConnector = new TestControlConnector(url, 0.2, tlsCa); + controlConnector = new TestControlConnector(url, 0.2); controlConnector.token = token; control = new TestControl(controlConnector, false); - hostConnector = new TestHostConnector(url, 0.2, tlsCa); + hostConnector = new TestHostConnector(url, 0.2); hostConnector.token = tokenHost; hostConnector.hostId = 0; }); @@ -123,7 +119,7 @@ describe("Integration of lib/link", function() { it("should close old connections from the same address", async function() { slowTest(this); - const hostConnectorFirst = new TestHostConnector(url, 0.2, tlsCa); + const hostConnectorFirst = new TestHostConnector(url, 0.2); hostConnectorFirst.token = hostConnector.token; hostConnectorFirst.hostId = hostConnector.hostId; const onceClose = events.once(hostConnectorFirst, "close"); diff --git a/test/integration/routing.js b/test/integration/routing.js index 02c5fcff..a4672305 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -23,7 +23,7 @@ function connectHost(controller, hostId, plugins) { controller.wsServer.hostConnections.set(hostId, hostConnection); const hostConfig = new lib.HostConfig("host"); hostConfig.set("host.id", hostId); - return new Host(hostSide, hostConfig, undefined, []); + return new Host(hostSide, hostConfig, []); } /** diff --git a/test/lib/check_singleton_import.test.js b/test/lib/check_singleton_import.test.js new file mode 100644 index 00000000..c4af4938 --- /dev/null +++ b/test/lib/check_singleton_import.test.js @@ -0,0 +1,16 @@ +const assert = require("assert").strict; +const { checkSingletonImport } = require("@clusterio/lib/dist/node/src/check_singleton_import"); +// Import lib so it sets the global indicating lib has been imported. +require("@clusterio/lib"); + +describe("lib/check_singleton_import", function() { + it("Should throw an installation error if called twice", function() { + assert.throws( + () => checkSingletonImport(__filename), + { + message: /Attempt to import duplicate copy of @clusterio\/lib/, + code: "InstallationError", + } + ); + }); +}); diff --git a/test/lib/datastore.js b/test/lib/datastore.js index 93b53277..ba2cf116 100644 --- a/test/lib/datastore.js +++ b/test/lib/datastore.js @@ -5,13 +5,6 @@ const lib = require("@clusterio/lib"); const fs = require("fs-extra"); const path = require("path"); -async function incrementDateMs() { - const prev = Date.now(); - do { - await lib.wait(1); - } while (prev >= Date.now()); -} - class MockDatastoreProvider extends lib.DatastoreProvider { constructor() { super(); @@ -571,12 +564,31 @@ describe("lib/datastore", function() { eventsRaised = []; datastoreProvider = new MockDatastoreProvider(); datastoreProvider.value = new Map([ - ["foo", {id: "foo"}], ["bar", {id: "bar"}], ["baz", {id: "baz"}], + ["foo", {id: "foo", updatedAtMs: 10}], ["bar", {id: "bar"}], ["baz", {id: "baz"}], ]); datastore = new lib.SubscribableDatastore(datastoreProvider, datastoreProvider.value); datastore.on("update", updates => eventsRaised.push(updates)); }); + function forceFuture() { + const future = {id: "future"}; + for (let i = 1; i < 100; i++) { + datastore.set(future); + } + return future.updatedAtMs; + } + + describe("lastUpdateAtMs", function() { + it("is set to the highest value after constructing a new instance", function() { + assert(datastore.lastUpdatedAtMs === 10, `${datastore.lastUpdatedAtMs} !== 10`); + }); + it("is set to the highest value after loading from a provider", async function() { + const future = Date.now() + 2000; + datastore.provider.value = ["future", {id: "future", updatedAtMs: future}]; + await datastore.load(); + assert(datastore.lastUpdatedAtMs === future, `${datastore.lastUpdatedAtMs} !== ${future}`); + }); + }); describe("set", function() { it("can set a value", function() { const value = { id: "test" }; @@ -598,12 +610,21 @@ describe("lib/datastore", function() { const value = { id: "test" }; datastore.set(value); assert.notEqual(value.updatedAtMs, undefined); - assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(value.updatedAtMs > 0, "updatedAtMs not greater than zero"); + }); + it("increments the updatedAtMs property", async function() { + const value = { id: "test" }; + datastore.set(value); const prev = value.updatedAtMs; - await incrementDateMs(); datastore.set(value); assert(value.updatedAtMs > prev, "updatedAtMs must increase"); }); + it("increments the updatedAtMs property past existing values", async function() { + const future = forceFuture(); + const value = { id: "test" }; + datastore.set(value); + assert(value.updatedAtMs > future, "updatedAtMs must increase"); + }); }); describe("setMany", function() { @@ -632,11 +653,22 @@ describe("lib/datastore", function() { assert.notEqual(updates[1].updatedAtMs, undefined); assert(updates[0].updatedAtMs > 0, "updatedAtMs greater than zero"); assert(updates[1].updatedAtMs > 0, "updatedAtMs greater than zero"); - const prev = [updates[0].updatedAtMs, updates[1].updatedAtMs]; - await incrementDateMs(); + }); + it("increments the updatedAtMs property", async function() { + const value = { id: "test1" }; + datastore.set(value); + const prev = value.updatedAtMs; + const updates = [{ id: "test1" }, { id: "test2" }]; datastore.setMany(updates); - assert(updates[0].updatedAtMs > prev[0], "updatedAtMs must increase"); - assert(updates[1].updatedAtMs > prev[1], "updatedAtMs must increase"); + assert(updates[0].updatedAtMs > prev, "updatedAtMs must increase"); + assert(updates[1].updatedAtMs > prev, "updatedAtMs must increase"); + }); + it("increments the updatedAtMs property past existing values", async function() { + const future = forceFuture(); + const updates = [{ id: "test1" }, { id: "test2" }]; + datastore.setMany(updates); + assert(updates[0].updatedAtMs > future, "updatedAtMs must increase"); + assert(updates[1].updatedAtMs > future, "updatedAtMs must increase"); }); }); @@ -667,7 +699,20 @@ describe("lib/datastore", function() { const value = { id: "test" }; datastore.delete(value); assert.notEqual(value.updatedAtMs, undefined); - assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(value.updatedAtMs > 0, "updatedAtMs not greater than zero"); + }); + it("increments the updatedAtMs property", function() { + const value = { id: "test" }; + datastore.set(value); + const prev = value.updatedAtMs; + datastore.delete(value); + assert(value.updatedAtMs > prev, "updatedAtMs must increase"); + }); + it("increments the updatedAtMs property past existing values", function() { + const future = forceFuture(); + const value = { id: "test" }; + datastore.delete(value); + assert(value.updatedAtMs > future, "updatedAtMs must increase"); }); it("sets the isDeleted property", function() { const value = { id: "test" }; @@ -704,6 +749,22 @@ describe("lib/datastore", function() { assert(values[0].updatedAtMs > 0, "updatedAtMs greater than zero"); assert(values[1].updatedAtMs > 0, "updatedAtMs greater than zero"); }); + it("increments the updatedAtMs property", function() { + const value = { id: "foo" }; + datastore.set(value); + const prev = value.updatedAtMs; + const values = [{id: "foo", updatedAtMs: prev}, {id: "bar", updatedAtMs: prev}]; + datastore.deleteMany(values); + assert(values[0].updatedAtMs > prev, "updatedAtMs must increase"); + assert(values[1].updatedAtMs > prev, "updatedAtMs must increase"); + }); + it("increments the updatedAtMs property past existing values", function() { + const future = forceFuture(); + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert(values[0].updatedAtMs > future, "updatedAtMs must increase"); + assert(values[1].updatedAtMs > future, "updatedAtMs must increase"); + }); it("sets the isDeleted property", function() { const values = [{id: "foo"}, {id: "bar"}]; datastore.deleteMany(values); diff --git a/test/lib/external/FactorioVersions.test.js b/test/lib/external/FactorioVersions.test.js index 9058e6e0..c2e4f49a 100644 --- a/test/lib/external/FactorioVersions.test.js +++ b/test/lib/external/FactorioVersions.test.js @@ -121,6 +121,12 @@ describe("FactorioVersions", function() { assert.equal(versions.filter(v => v.stable).length, 48); assert.equal(versions.filter(v => !v.stable).length, 57); }); + it("parses versions from empty html file", async function() { + mockFetch(""); + + const versions = await fetchFactorioVersions(); + assert.equal(versions.length, 0); + }); it("parses versions from live api", async function() { slowTest(this); global.fetch = originalFetch; diff --git a/test/lib/prometheus.js b/test/lib/prometheus.js index b0f24ca3..8ef24147 100644 --- a/test/lib/prometheus.js +++ b/test/lib/prometheus.js @@ -651,8 +651,20 @@ describe("lib/prometheus", function() { let gauge = new prometheus.Gauge( "test", "Help", { register: false, labels: ["a", "b"] } ); - gauge.labels('x\\\\\n\"\"\n\\', "2"); - gauge.removeAll({ a: 'x\\\\\n\"\"\n\\' }); + + let labelValue = ""; + // Include every ASCII codepoint + for (let i = 0; i < 128; i++) { + labelValue += String.fromCodePoint(i); + } + // Include Unicode codepoints from the basic and supplementary planes. + labelValue += "©💣"; + + gauge.labels(labelValue, "2"); + gauge.removeAll({ a: labelValue }); + assert(gauge._values.size === 0, "Matching labels remained"); + gauge.labels("1", labelValue); + gauge.removeAll({ a: "1" }); assert(gauge._values.size === 0, "Matching labels remained"); }); }); diff --git a/test/messages/host.test.js b/test/messages/host.test.js index faf3093d..56b1d121 100644 --- a/test/messages/host.test.js +++ b/test/messages/host.test.js @@ -14,7 +14,7 @@ describe("messages/host", function() { lib.Address.fromShorthand({ hostId: 1 }), lib.Address.fromShorthand("controller"), ); - host = new Host(hostConnector, hostConfig, undefined, []); + host = new Host(hostConnector, hostConfig, []); }); describe("HostUpdateRequest", function() { diff --git a/test/messages/plugin.test.js b/test/messages/plugin.test.js index 5b83bc8f..e2c7d9d8 100644 --- a/test/messages/plugin.test.js +++ b/test/messages/plugin.test.js @@ -28,7 +28,7 @@ describe("messages/plugin", function() { lib.Address.fromShorthand({ hostId: 1 }), lib.Address.fromShorthand("controller"), ); - host = new Host(hostConnector, hostConfig, undefined, []); + host = new Host(hostConnector, hostConfig, []); }); describe("PluginDetails", function() {