From 94d67cef71253d591512f547e4b265df4c9e79ea Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 3 Dec 2023 00:01:23 +0100 Subject: [PATCH 1/7] Add AsyncSerialCallback helper Add helper to serialise the calls made to an async function such that no overlapping invocations happen to the async function by queue up calls made while it's running to continue after it's running. --- packages/lib/src/helpers.ts | 48 ++++++++++++++++++++ test/lib/helpers.js | 88 +++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/packages/lib/src/helpers.ts b/packages/lib/src/helpers.ts index 38c804b3..c9589d92 100644 --- a/packages/lib/src/helpers.ts +++ b/packages/lib/src/helpers.ts @@ -107,6 +107,54 @@ export class AsyncSerialMergingCallback { } } +/** + * Helper to serialise calls to an async callback. + * + * Provides a convenient interface to serialise the calls made to an async + * callback such that no overlapping invocations of the callback is made, + * and successive calls are served on a first in first out basis. If the + * callback is already running when another invocation is requested the + * request will be queued until the callback returns and then the callback + * called again. + */ +export class AsyncSerialCallback< + Callback extends (...args: Parameters) => Promise>>, +> { + private _currentlyRunning = false; + private _currentlyWaiting: (() => void)[] = []; + + /** + * @param callback - Async function to serialise access to + */ + constructor( + public callback: Callback, + ) { } + + /** + * Invoke the assosiated callback. + * + * If the callback is currently running then this will wait until the + * existing invocation finishes and then invoke it again. If called + * multiple times while an invocation is running the calls will be + * merged into one call. + */ + async invoke(...args: Parameters) { + if (this._currentlyRunning) { + await new Promise(resolve => { + this._currentlyWaiting.push(resolve); + }); + } + + this._currentlyRunning = true; + try { + return await this.callback(...args); + } finally { + this._currentlyRunning = false; + this._currentlyWaiting.shift()?.(); + } + } +} + /** * Read stream to the end and return its content diff --git a/test/lib/helpers.js b/test/lib/helpers.js index f9466d75..25b0270c 100644 --- a/test/lib/helpers.js +++ b/test/lib/helpers.js @@ -104,6 +104,94 @@ describe("lib/helpers", function() { }); }); + describe("class AsyncSerialCallback", function() { + let started; + let ended; + let serialiser; + beforeEach(function() { + started = 0; + ended = 0; + serialiser = new lib.AsyncSerialCallback( + (input) => new Promise((resolve, reject) => { + started += 1; + process.nextTick(() => { + ended += 1; + if (input !== 99) { + resolve(input); + } else { + reject(new Error("99")); + } + }); + }) + ); + }); + it("should await the callback when invoked", async function() { + await serialiser.invoke(); + assert.equal(started, 1, "incorrect started count"); + assert.equal(ended, 1, "incorrect ended count"); + }); + it("should return the result of the callback", async function() { + const id = await serialiser.invoke(42); + assert.equal(id, 42, "returned result not passed"); + }); + it("should immediately call the callback", async function() { + let promise = serialiser.invoke(); + assert.equal(started, 1, "incorrect started count"); + assert.equal(ended, 0, "incorrect ended count"); + await promise; + assert.equal(started, 1, "incorrect started count"); + assert.equal(ended, 1, "incorrect ended count"); + }); + it("should call the callback twice if invoked in twice in paralell", async function() { + await Promise.all([ + serialiser.invoke(), + serialiser.invoke(), + ]); + assert.equal(started, 2, "incorrect started count"); + assert.equal(ended, 2, "incorrect ended count"); + }); + it("should call the callback many if invoked many times in paralell", async function() { + await Promise.all([ + serialiser.invoke(), + serialiser.invoke(), + serialiser.invoke(), + serialiser.invoke(), + ]); + assert.equal(started, 4, "incorrect started count"); + assert.equal(ended, 4, "incorrect ended count"); + }); + it("should serialise calls", async function() { + let events = []; + let calls = await Promise.all([ + serialiser.invoke(11).then((res) => { events.push(1); return res; }), + serialiser.invoke(12).then((res) => { events.push(2); return res; }), + serialiser.invoke(13).then((res) => { events.push(3); return res; }), + serialiser.invoke(14).then((res) => { events.push(4); return res; }), + ]); + assert.deepEqual(events, [1, 2, 3, 4]); + assert.deepEqual(calls, [11, 12, 13, 14]); + }); + it("should forward rejections", async function() { + await assert.rejects(serialiser.invoke(99), new Error("99")); + }); + it("should handle queued rejections", async function() { + let events = []; + let calls = await Promise.allSettled([ + serialiser.invoke(11).then((res) => { events.push(1); return res; }), + serialiser.invoke(99).then((res) => { events.push(2); return res; }), + serialiser.invoke(13).then((res) => { events.push(3); return res; }), + serialiser.invoke(14).then((res) => { events.push(4); return res; }), + ]); + assert.deepEqual(events, [1, 3, 4]); + assert.deepEqual(calls, [ + { status: "fulfilled", value: 11 }, + { status: "rejected", reason: new Error("99") }, + { status: "fulfilled", value: 13 }, + { status: "fulfilled", value: 14 }, + ]); + }); + }); + function parse(input, attributes) { const result = lib.parseSearchString(input, attributes); if (!result.issues.length) { From 8b0d7a67b5e0869c6dbac666696a8fe15cb3a465 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 3 Dec 2023 00:05:44 +0100 Subject: [PATCH 2/7] Automatically obtain required mods in test Build the clusterio_lib mod from source and download and copy the subspace_storage mod used for testing if it's not present in the mods directory. This removes a manual step needed to run tests on a new clone of the repository. --- .github/workflows/node.js.yml | 2 -- test/integration/index.js | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 6098895e..eb2ece36 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -26,8 +26,6 @@ jobs: node-version: ${{ matrix.node-version }} - run: pnpm i --no-optional --no-frozen-lockfile - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz - - run: node packages/lib/build_mod --source-dir packages/host/lua/clusterio_lib --output-dir temp/test/mods - - run: wget -q -O temp/test/mods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip - name: Run tests if: ${{ matrix.node-version != '18.x' }} run: pnpm test diff --git a/test/integration/index.js b/test/integration/index.js index 360f0af9..8fd2585e 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -18,6 +18,8 @@ require("../../plugins/research_sync/dist/plugin/info"); require("../../plugins/statistics_exporter/dist/plugin/info"); require("../../plugins/subspace_storage/dist/plugin/info"); +const subspaceStorageVersion = "1.99.8"; +const subspaceStorageGitHubTag = "1.99.6"; // Tag subspaceStorageVersion was released under on GitHub class TestControl extends lib.Link { constructor(connector, subscribe = true) { @@ -131,6 +133,7 @@ let control; let url = "https://localhost:4443/"; let controlToken = jwt.sign({ aud: "user", user: "test" }, Buffer.from("TestSecretDoNotUse", "base64")); let instancesDir = path.join("temp", "test", "instances"); +let modsDir = path.join("temp", "test", "mods"); let databaseDir = path.join("temp", "test", "database"); let pluginListPath = path.join("temp", "test", "plugin-list.json"); let controllerConfigPath = path.join("temp", "test", "config-controller.json"); @@ -186,8 +189,9 @@ before(async function() { format: new lib.TerminalFormat(), })); - await fs.remove(databaseDir); await fs.remove(instancesDir); + await fs.remove(modsDir); + await fs.remove(databaseDir); await fs.remove(pluginListPath); await fs.remove(controllerConfigPath); @@ -196,6 +200,32 @@ before(async function() { await fs.ensureDir(path.join("temp", "test")); + await fs.ensureDir(modsDir); + await lib.build({ + build: true, + pack: true, + sourceDir: "packages/host/lua/clusterio_lib", + outputDir: modsDir, + }); + const ssv = subspaceStorageVersion; + const ssTag = subspaceStorageGitHubTag; + const ssZip = `subspace_storage_${ssv}.zip`; + if (!await fs.pathExists(`mods/${ssZip}`)) { + console.log(`Downloading ${ssZip}`); + const ssUrl = `https://github.com/clusterio/subspace_storage/releases/download/${ssTag}/${ssZip}`; + let response = await phin({ + url: ssUrl, + method: "GET", + stream: false, // phin leaks open sockets when stream and followRedirects is combined + followRedirects: true, + }); + if (response.statusCode !== 200) { + throw new Error(`${ssUrl} returned ${response.statusCode} ${response.statusMessage}`); + } + await fs.outputFile(`mods/${ssZip}`, response.body); + } + await fs.copyFile(`mods/${ssZip}`, `${modsDir}/${ssZip}`); + await exec("node ../../packages/controller config set controller.auth_secret TestSecretDoNotUse"); await exec("node ../../packages/controller config set controller.http_port 8880"); await exec("node ../../packages/controller config set controller.https_port 4443"); @@ -233,7 +263,7 @@ before(async function() { testPack.name = "subspace_storage-pack"; testPack.factorioVersion = "1.1.0"; testPack.mods.set("clusterio_lib", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); - testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); + testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: subspaceStorageVersion }); await control.sendTo("controller", new lib.ModPackCreateRequest(testPack)); await control.sendTo( "controller", @@ -279,6 +309,7 @@ module.exports = { url, controlToken, instancesDir, + modsDir, databaseDir, controllerConfigPath, hostConfigPath, From 94cb60793040372977bd7dfd851efa5bb9a0aa93 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 3 Dec 2023 00:14:35 +0100 Subject: [PATCH 3/7] Improve typing of various controller entities --- packages/controller/src/BaseConnection.ts | 2 +- packages/controller/src/ControlConnection.ts | 8 +-- packages/controller/src/Controller.ts | 16 +++-- packages/controller/src/HostConnection.ts | 1 - packages/controller/src/routes.ts | 75 +++++++++++--------- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/packages/controller/src/BaseConnection.ts b/packages/controller/src/BaseConnection.ts index b12c7e3c..cf8b4052 100644 --- a/packages/controller/src/BaseConnection.ts +++ b/packages/controller/src/BaseConnection.ts @@ -13,7 +13,7 @@ import ControllerRouter from "./ControllerRouter"; * @alias module:controller/src/BaseConnection */ export default class BaseConnection extends lib.Link { - declare connector: lib.WebSocketBaseConnector; + declare ["connector"]: WsServerConnector; constructor( connector: lib.WebSocketBaseConnector, diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 940bdec3..32463aef 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -88,9 +88,9 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.InstanceConfigSetFieldRequest, this.handleInstanceConfigSetFieldRequest.bind(this)); this.handle(lib.InstanceConfigSetPropRequest, this.handleInstanceConfigSetPropRequest.bind(this)); this.handle(lib.InstanceAssignRequest, this.handleInstanceAssignRequest.bind(this)); - this.handle(lib.InstanceRenameSaveRequest, this._controller.sendToHostByInstanceId.bind(this._controller)); - this.handle(lib.InstanceCopySaveRequest, this._controller.sendToHostByInstanceId.bind(this._controller)); - this.handle(lib.InstanceDeleteSaveRequest, this._controller.sendToHostByInstanceId.bind(this._controller)); + this.handle(lib.InstanceRenameSaveRequest, controller.sendRequestToHostByInstanceId.bind(controller)); + this.handle(lib.InstanceCopySaveRequest, controller.sendRequestToHostByInstanceId.bind(controller)); + this.handle(lib.InstanceDeleteSaveRequest, controller.sendRequestToHostByInstanceId.bind(controller)); this.handle(lib.InstanceDownloadSaveRequest, this.handleInstanceDownloadSaveRequest.bind(this)); this.handle(lib.InstanceTransferSaveRequest, this.handleInstanceTransferSaveRequest.bind(this)); this.handle(lib.ModPackListRequest, this.handleModPackListRequest.bind(this)); @@ -576,7 +576,7 @@ export default class ControlConnection extends BaseConnection { stream.filename = mod.filename; stream.source = fs.createReadStream(modPath); stream.mime = "application/zip"; - stream.size = mod.size; + stream.size = String(mod.size); return stream.id; } diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index e21bfe68..4639898f 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -36,6 +36,8 @@ const logSizeGauge = new Gauge( "Size of all log files currently stored on the controller." ); +type InstanceId = { instanceId: number }; + /** * Manages all controller related operations * @alias module:controller/src/Controller @@ -1207,17 +1209,19 @@ export default class Controller { } } - async sendToHostByInstanceId(requestOrEvent: any) { + async sendToHostByInstanceId(request: Req & lib.Request): Promise; + sendToHostByInstanceId(event: T & lib.Event): void; + async sendToHostByInstanceId(requestOrEvent: lib.Request | lib.Event) { if (requestOrEvent.constructor.type === "request") { - return this.sendRequestToHostByInstanceId(requestOrEvent); + return this.sendRequestToHostByInstanceId(requestOrEvent as any); } if (requestOrEvent.constructor.type === "event") { - return this.sendEventToHostByInstanceId(requestOrEvent); + return this.sendEventToHostByInstanceId(requestOrEvent as any); } - throw Error(`Unknown type ${requestOrEvent.constructor.type}.`); + throw Error(`Unknown type ${(requestOrEvent.constructor as any).type}.`); } - async sendRequestToHostByInstanceId(request: any) { + async sendRequestToHostByInstanceId(request: Req & lib.Request) { let instance = this.getRequestInstance(request.instanceId); let hostId = instance.config.get("instance.assigned_host"); if (hostId === null) { @@ -1230,7 +1234,7 @@ export default class Controller { return await connection.send(request); } - sendEventToHostByInstanceId(event: any) { + sendEventToHostByInstanceId(event: T & lib.Event) { let instance = this.getRequestInstance(event.instanceId); let hostId = instance.config.get("instance.assigned_host"); if (hostId === null) { diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 71ffafc2..140bd124 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -25,7 +25,6 @@ export interface HostInfo { * @alias module:controller/src/HostConnection */ export default class HostConnection extends BaseConnection { - declare connector: WsServerConnector; private _agent: any; private _id: any; private _name: any; diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 719584ba..8abc77ec 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -16,6 +16,17 @@ const { logger } = lib; const finished = util.promisify(nodeStream.finished); +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Express { + export interface Locals { + controller: Controller, + mainBundle: string, + devPlugins: Map, + streams: Map + } + } +} // Merges samples from sourceResult to destinationResult function mergeSamples(destinationResult: lib.CollectorResultSerialized, sourceResult: lib.CollectorResultSerialized) { @@ -95,14 +106,14 @@ async function getMetrics(req: Request, res: Response, next: any) { function getPlugins(req: Request, res: Response) { let plugins: lib.PluginWebApi[] = []; - for (let pluginInfo of (req.app.locals.controller as Controller).pluginInfos) { + for (let pluginInfo of req.app.locals.controller.pluginInfos) { let name = pluginInfo.name; let loaded = req.app.locals.controller.plugins.has(name); - let enabled = loaded && req.app.locals.controller.config.group(pluginInfo.name).get("load_plugin"); + let enabled = loaded && req.app.locals.controller.config.group(pluginInfo.name).get("load_plugin") as boolean; let web: { main:any, error:string|undefined } = { main: undefined, error: undefined}; let devPlugins = req.app.locals.devPlugins; if (devPlugins && devPlugins.has(name)) { - let stats = res.locals.webpack.devMiddleware.stats.stats[devPlugins.get(name)]; + let stats = res.locals.webpack.devMiddleware.stats.stats[devPlugins.get(name)!]; web.main = stats.toJson().assetsByChunkName[name]; } else if (pluginInfo.manifest) { web.main = pluginInfo.manifest[`${pluginInfo.name}.js`]; @@ -137,7 +148,7 @@ function validateHostToken(req: Request, res: Response, next: any) { if (typeof tokenPayload === "string") { throw new Error("unexpected JsonWebToken type"); } - let host = (req.app.locals.controller as Controller).hosts.get(tokenPayload.host); + let host = req.app.locals.controller.hosts.get(tokenPayload.host); if (!host) { throw new Error("invalid host"); } @@ -251,12 +262,12 @@ async function uploadExport(req: Request, res: Response) { } -interface ProxyStream { +export interface ProxyStream { id: string; source: nodeStream.Readable | null; flowing: boolean; - size: number | null; - mime: string | null; + size?: string; + mime?: string; filename: string | null; events: events.EventEmitter; timeout: ReturnType; @@ -269,8 +280,8 @@ export async function createProxyStream(app: Application): Promise id, source: null, flowing: false, - size: null, - mime: null, + size: undefined, + mime: undefined, filename: null, events: new events.EventEmitter(), timeout: setTimeout(() => { @@ -289,7 +300,7 @@ export async function createProxyStream(app: Application): Promise } async function putStream(req: Request, res: Response) { - let stream = req.app.locals.streams.get(req.params.id); + const stream = req.app.locals.streams.get(req.params.id); if (!stream || stream.source) { res.sendStatus(404); return; @@ -309,35 +320,35 @@ async function putStream(req: Request, res: Response) { }); } +function startStream(res: Response, stream: ProxyStream) { + res.append("Content-Type", stream.mime); + if (stream.size) { + res.append("Content-Length", stream.size); + } + if (stream.filename) { + res.append("Content-Disposition", `attachment; filename="${stream.filename}"`); + } else { + res.append("Content-Disposition", "attachment"); + } + stream.source!.pipe(res); + stream.flowing = true; + res.on("close", () => { + stream.events.emit("close"); + }); + clearTimeout(stream.timeout); +} + async function getStream(req: Request, res: Response) { - let stream = req.app.locals.streams.get(req.params.id); + const stream = req.app.locals.streams.get(req.params.id); if (!stream || stream.flowing) { res.sendStatus(404); return; } - function startStream() { - res.append("Content-Type", stream.mime); - if (stream.size) { - res.append("Content-Length", stream.size); - } - if (stream.filename) { - res.append("Content-Disposition", `attachment; filename="${stream.filename}"`); - } else { - res.append("Content-Disposition", "attachment"); - } - stream.source.pipe(res); - stream.flowing = true; - res.on("close", () => { - stream.events.emit("close"); - }); - clearTimeout(stream.timeout); - } - if (stream.source) { - startStream(); + startStream(res, stream); } else { - stream.events.on("source", startStream); + stream.events.on("source", () => startStream(res, stream)); stream.events.on("timeout", () => { res.sendStatus(500); }); @@ -377,7 +388,7 @@ async function uploadSave(req: Request, res: Response) { let proxyStream = await createProxyStream(req.app); proxyStream.source = stream; proxyStream.mime = streamMime; - let timeout = new Promise((_, reject) => { + let timeout = new Promise((_, reject) => { proxyStream.events.on("timeout", () => { stream.resume(); reject(new Error("Timed out establishing stream to host")); From 1095d2466d80ec00737cb69beb49eda777697e88 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 3 Dec 2023 00:17:36 +0100 Subject: [PATCH 4/7] Fix path traversal on mod upload Fix possibility of a maliciously crafted mod being written outside of the mods directory and thus possibily overwriting unrelated files. --- packages/controller/src/routes.ts | 1 + packages/lib/build_mod.js | 4 ++-- test/file/path_traversing_mod/info.json | 9 +++++++++ test/integration/clusterio.js | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 test/file/path_traversing_mod/info.json diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 8abc77ec..8875bcd3 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -559,6 +559,7 @@ async function uploadMod(req: Request, res: Response) { await finished(writeStream); const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); + checkModName(modInfo.filename); await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename)); req.app.locals.controller.mods.set(modInfo.filename, modInfo); req.app.locals.controller.modUpdated(modInfo); diff --git a/packages/lib/build_mod.js b/packages/lib/build_mod.js index 29dc39cf..eee00d8d 100644 --- a/packages/lib/build_mod.js +++ b/packages/lib/build_mod.js @@ -12,10 +12,10 @@ const yargs = require("yargs"); const finished = util.promisify(stream.finished); -async function buildMod(args, info) { +async function buildMod(args, info, modName) { if (args.build) { await fs.ensureDir(args.outputDir); - let modName = `${info.name}_${info.version}`; + modName = args.modName ?? `${info.name}_${info.version}`; if (args.pack) { let zip = new JSZip(); diff --git a/test/file/path_traversing_mod/info.json b/test/file/path_traversing_mod/info.json new file mode 100644 index 00000000..941e2ef7 --- /dev/null +++ b/test/file/path_traversing_mod/info.json @@ -0,0 +1,9 @@ +{ + "name": "../bad_path_mod", + "version": "1.0.0", + "title": "Path Traversing Mod", + "author": "Me", + "description": "A mod who's name traverses the filesystem", + "factorio_version": "1.1", + "dependencies": [] +} diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index af982cea..b076b439 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -934,6 +934,21 @@ describe("Integration of Clusterio", function() { "mod not present in mods directory" ); }); + it("rejects path traversing mods", async function() { + await libBuildMod.build({ + build: true, + pack: true, + sourceDir: path.join("test", "file", "path_traversing_mod"), + outputDir: path.join("temp", "test"), + modName: "path_traversing_mod_1.0.0", + }); + await fs.remove(path.join("temp", "test", "bad_path_mod_1.0.0.zip")); + await assert.rejects(execCtl("mod upload path_traversing_mod_1.0.0.zip")); + assert( + !await fs.pathExists(path.join("temp", "test", "bad_path_mod_1.0.0.zip")), + "mod executed a path traversal attack", + ); + }); }); describe("mod show", function() { From 448cfbbb5118b5cfc36b41dcc3e834d84a8fce82 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 3 Dec 2023 00:21:26 +0100 Subject: [PATCH 5/7] Log errors on more instance failures conditions Log errors messages to the log when setting up an instance fails in the preparation phase, creating save phase, starting up phase, and so on. Previously these were only sent back as a an error response to the request that iniatied the wakeup of the instance. --- packages/host/src/Instance.ts | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 6be4638e..f68bdd24 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -634,6 +634,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); * Windows hard links are used instead due to symlinks being privileged. */ async syncMods() { + this.logger.info("Syncing mods"); const modPackId = this.config.get("factorio.mod_pack"); let modPack; if (modPackId === null) { @@ -994,7 +995,8 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); try { await this.prepare(); saveName = await this.prepareSave(saveName); - } catch (err) { + } catch (err: any) { + this.logger.error(`Error preparing instance: ${err.message}`); this.notifyExit(); await this.sendSaveListUpdate(); throw err; @@ -1002,7 +1004,8 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); try { await this.start(saveName); - } catch (err) { + } catch (err: any) { + this.logger.error(`Error starting ${saveName}: ${err.message}`); await this.stop(); throw err; } @@ -1016,7 +1019,8 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); try { await this.prepare(); - } catch (err) { + } catch (err: any) { + this.logger.error(`Error preparing instance: ${err.message}`); this.notifyExit(); await this.sendSaveListUpdate(); throw err; @@ -1025,7 +1029,8 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); let { scenario, seed, mapGenSettings, mapSettings } = request; try { await this.startScenario(scenario, seed, mapGenSettings, mapSettings); - } catch (err) { + } catch (err: any) { + this.logger.error(`Error starting scenario ${scenario}: ${err.message}`); await this.stop(); throw err; } @@ -1040,11 +1045,10 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); try { this.logger.verbose("Writing server-settings.json"); await this.writeServerSettings(); - - this.logger.verbose("Creating save ....."); await this.syncMods(); - } catch (err) { + } catch (err: any) { + this.logger.error(`Error preparing instance: ${err.message}`); this.notifyExit(); await this.sendSaveListUpdate(); throw err; @@ -1052,7 +1056,14 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); this.server.on("exit", () => this.notifyExit()); let { name, seed, mapGenSettings, mapSettings } = request; - await this.server.create(name, seed, mapGenSettings, mapSettings); + + try { + this.logger.info("Creating save"); + await this.server.create(name, seed, mapGenSettings, mapSettings); + } catch (err: any) { + this.logger.error(`Error creating save ${name}: ${err.message}`); + throw err; + } await this.sendSaveListUpdate(); this.logger.info("Successfully created save"); } From 853866492b0ae1fb563d0795c5b26ceeeb14e92d Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 3 Dec 2023 00:49:30 +0100 Subject: [PATCH 6/7] Refactor controller mod storage into ModStore Structure the storage of mods on the controller into into a reusable component in preparation for the host using the same kind of mod storage to retrieve and cache the mods it needs to run mod packs on instances. --- packages/controller/src/BaseConnection.ts | 30 +++++ packages/controller/src/ControlConnection.ts | 35 +----- packages/controller/src/Controller.ts | 60 ++-------- packages/controller/src/routes.ts | 3 +- packages/lib/index.ts | 1 + packages/lib/src/ModStore.ts | 110 +++++++++++++++++++ packages/lib/src/data/ModInfo.ts | 12 +- packages/lib/src/data/messages_mod.ts | 10 +- packages/web_ui/src/components/ModsPage.tsx | 4 +- 9 files changed, 174 insertions(+), 91 deletions(-) create mode 100644 packages/lib/src/ModStore.ts diff --git a/packages/controller/src/BaseConnection.ts b/packages/controller/src/BaseConnection.ts index cf8b4052..91f2a5ab 100644 --- a/packages/controller/src/BaseConnection.ts +++ b/packages/controller/src/BaseConnection.ts @@ -1,9 +1,13 @@ +import path from "path"; +import fs from "fs-extra"; + import type Controller from "./Controller"; import type WsServerConnector from "./WsServerConnector"; import * as lib from "@clusterio/lib"; const { logger } = lib; import ControllerRouter from "./ControllerRouter"; +import * as routes from "./routes"; /** @@ -28,6 +32,7 @@ export default class BaseConnection extends lib.Link { this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this)); this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this)); + this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this)); } async disconnect(code: number, reason: string) { @@ -68,4 +73,29 @@ export default class BaseConnection extends lib.Link { } return modPack; } + + getMod(mod: { name: string, version: string, sha1?: string }) { + let filename = lib.ModInfo.filename(mod.name, mod.version); + let modInfo = this._controller.modStore.files.get(filename); + if (!modInfo) { + throw new lib.RequestError(`Mod ${filename} does not exist on controller`); + } + if (mod.sha1 && mod.sha1 !== modInfo.sha1) { + throw new lib.RequestError(`Mod ${filename} checksum does not match controller's checksum`); + } + return modInfo; + } + + async handleModDownloadRequest(request: lib.ModDownloadRequest) { + let mod = this.getMod(request); + let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename); + + let stream = await routes.createProxyStream(this._controller.app); + stream.filename = mod.filename; + stream.source = fs.createReadStream(modPath); + stream.mime = "application/zip"; + stream.size = String(mod.size); + + return stream.id; + } } diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 32463aef..abd955b7 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -2,8 +2,6 @@ import type WsServerConnector from "./WsServerConnector"; import type { HostInfo } from "./HostConnection"; import events from "events"; -import fs from "fs-extra"; -import path from "path"; import * as lib from "@clusterio/lib"; const { logFilter, logger } = lib; @@ -100,7 +98,6 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this)); this.handle(lib.ModListRequest, this.handleModListRequest.bind(this)); this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this)); - this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this)); this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this)); this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this)); this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this)); @@ -461,17 +458,11 @@ export default class ControlConnection extends BaseConnection { } async handleModGetRequest(request: lib.ModGetRequest) { - let { name, version } = request; - let filename = `${name}_${version}.zip`; - let mod = this._controller.mods.get(filename); - if (!mod) { - throw new lib.RequestError(`Mod ${filename} does not exist`); - } - return mod; + return this.getMod(request); } async handleModListRequest() { - return [...this._controller.mods.values()]; + return [...this._controller.modStore.mods()]; } static termsMatchesMod(terms: lib.ParsedTerm[], mod: lib.ModInfo) { @@ -511,7 +502,7 @@ export default class ControlConnection extends BaseConnection { type ModVersions = { name: string, versions: lib.ModInfo[] }; let results: Map = new Map(); - for (let mod of this._controller.mods.values()) { + for (let mod of this._controller.modStore.mods()) { if ( mod.factorioVersion !== factorioVersion || !ControlConnection.termsMatchesMod(query.terms, mod) @@ -563,26 +554,8 @@ export default class ControlConnection extends BaseConnection { }; } - async handleModDownloadRequest(request: lib.ModDownloadRequest) { - let { name, version } = request; - let filename = `${name}_${version}.zip`; - let mod = this._controller.mods.get(filename); - if (!mod) { - throw new lib.RequestError(`Mod ${filename} does not exist`); - } - let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename); - - let stream = await routes.createProxyStream(this._controller.app); - stream.filename = mod.filename; - stream.source = fs.createReadStream(modPath); - stream.mime = "application/zip"; - stream.size = String(mod.size); - - return stream.id; - } - async handleModDeleteRequest(request: lib.ModDeleteRequest) { - await this._controller.deleteMod(request.name, request.version); + await this._controller.modStore.deleteMod(request.name, request.version); } async handleLogSetSubscriptionsRequest(request: lib.LogSetSubscriptionsRequest) { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 4639898f..be26d4b0 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -102,13 +102,13 @@ export default class Controller { let modsDirectory = config.get("controller.mods_directory"); await fs.ensureDir(modsDirectory); - const mods = await Controller.loadModInfos(modsDirectory); + const modStore = await lib.ModStore.fromDirectory(modsDirectory); return [ hosts, instances, modPacks, - mods, + modStore, userManager, ] as const; } @@ -125,8 +125,8 @@ export default class Controller { public instances = new Map(), /** Mapping of mod pack id to mod pack */ public modPacks = new Map(), - /** Mapping of mod names to mod infos */ - public mods = new Map(), + /** Mods stored on the controller */ + public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()), /** User and roles manager for the cluster */ public userManager = new UserManager(config), ) { @@ -142,6 +142,10 @@ export default class Controller { this.wsServer = new WsServer(this); this.subscriptions = new lib.SubscriptionController(this); + this.modStore.on("change", mod => { + this.modUpdated(mod); + }); + // Handle subscriptions for all internal properties this.subscriptions.handle(lib.HostUpdateEvent); this.subscriptions.handle(lib.InstanceDetailsUpdateEvent); @@ -511,40 +515,6 @@ export default class Controller { await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t")); } - static async loadModInfos(modsDirectory: string): Promise> { - let mods = new Map(); - for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) { - if (entry.isDirectory()) { - logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`); - continue; - } - - if (entry.name.toLowerCase().endsWith(".tmp.zip")) { - continue; - } - - logger.info(`Loading info for Mod ${entry.name}`); - let modInfo; - try { - modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, entry.name)); - } catch (err: any) { - logger.error(`Error loading mod ${entry.name}: ${err.message}`); - continue; - } - - if (modInfo.filename !== entry.name) { - logger.error( - `Mod ${entry.name}'s filename does not match the expected ` + - `name ${modInfo.filename}, it will be ignored.` - ); - continue; - } - - mods.set(modInfo.filename, modInfo); - } - return mods; - } - static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise { let manifest = {}; try { @@ -557,20 +527,6 @@ export default class Controller { return manifest; } - async deleteMod(name: string, version: string): Promise { - let filename = `${name}_${version}.zip`; - let mod = this.mods.get(filename); - if (!mod) { - throw new Error(`Mod ${filename} does not exist`); - } - - let modsDirectory = this.config.get("controller.mods_directory"); - await fs.unlink(path.join(modsDirectory, filename)); - this.mods.delete(filename); - mod.isDeleted = true; - this.modUpdated(mod); - } - /** * Query controller log * diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 8875bcd3..99446069 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -561,8 +561,7 @@ async function uploadMod(req: Request, res: Response) { const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); checkModName(modInfo.filename); await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename)); - req.app.locals.controller.mods.set(modInfo.filename, modInfo); - req.app.locals.controller.modUpdated(modInfo); + req.app.locals.controller.modStore.addMod(modInfo); mods.push(modInfo.toJSON()); } catch (err: any) { diff --git a/packages/lib/index.ts b/packages/lib/index.ts index 8f6e6872..d6385d7c 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -29,4 +29,5 @@ export * from "./src/zip_ops"; export * from "./src/subscriptions"; export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; +export { default as ModStore } from "./src/ModStore"; export { default as RateLimiter } from "./src/RateLimiter"; diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts new file mode 100644 index 00000000..54eda548 --- /dev/null +++ b/packages/lib/src/ModStore.ts @@ -0,0 +1,110 @@ +import fs from "fs-extra"; +import path from "path"; +import { logger } from "./logging"; +import { ModInfo } from "./data"; +import EventEmitter from "events"; + +abstract class TypedEventEmitter< + Keys extends string, + Events extends Record void>, +> extends EventEmitter { + declare ["on"]: (event: E, listener: Events[E]) => this; + declare ["off"]: (event: E, listener: Events[E]) => this; + declare ["once"]: (event: E, listener: Events[E]) => this; + declare ["emit"]: (event: E, ...args: Parameters) => boolean; +} + +export interface ModStoreEvents { + /** A stored mod was created, updated or deleted */ + change: (mod: ModInfo) => void; +} + +export default class ModStore extends TypedEventEmitter { + constructor( + public modsDirectory: string, + public files: Map, + ) { + super(); + } + + getMod(name: string, version: string, sha1?: string) { + const mod = this.files.get(ModInfo.filename(name, version)); + if (!mod || sha1 && sha1 !== mod.sha1) { + return undefined; + } + return mod; + } + + hasMod(name: string, version: string, sha1?: string): boolean { + const mod = this.files.get(ModInfo.filename(name, version)); + return Boolean(mod && (!sha1 || sha1 !== mod.sha1)); + } + + mods() { return this.files.values(); } + + static async loadFile(modsDirectory: string, file: string) { + let modInfo = await ModInfo.fromModFile(path.join(modsDirectory, file)); + if (modInfo.filename !== file) { + throw new Error(`filename does not match the expected name ${modInfo.filename}.`); + } + return modInfo; + } + + async loadFile(file: string) { + let modInfo = await ModStore.loadFile(this.modsDirectory, file); + this.files.set(modInfo.filename, modInfo); + this.emit("change", modInfo); + return modInfo; + } + + async deleteFile(file: string) { + let modInfo = this.files.get(file); + if (!modInfo) { + throw new Error(`Mod ${file} does not exist`); + } + + await fs.unlink(path.join(this.modsDirectory, file)); + this.files.delete(file); + modInfo.isDeleted = true; + this.emit("change", modInfo); + } + + addMod(modInfo: ModInfo) { + this.files.set(modInfo.filename, modInfo); + this.emit("change", modInfo); + } + + async loadMod(name: string, version: string) { + return await this.loadFile(ModInfo.filename(name, version)); + } + + async deleteMod(name: string, version: string) { + return await this.deleteFile(ModInfo.filename(name, version)); + } + + static async fromDirectory(modsDirectory: string) { + const files = new Map(); + for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) { + if (entry.isDirectory()) { + logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`); + continue; + } + + if (entry.name.toLowerCase().endsWith(".tmp.zip")) { + continue; + } + + logger.info(`Loading info for Mod ${entry.name}`); + let modInfo: ModInfo; + try { + modInfo = await this.loadFile(modsDirectory, entry.name); + } catch (err: any) { + logger.error(`Error loading mod ${entry.name}: ${err.message}`); + continue; + } + files.set(modInfo.filename, modInfo); + } + + return new this(modsDirectory, files); + } +} diff --git a/packages/lib/src/data/ModInfo.ts b/packages/lib/src/data/ModInfo.ts index f642a9f7..09aa35e0 100644 --- a/packages/lib/src/data/ModInfo.ts +++ b/packages/lib/src/data/ModInfo.ts @@ -90,7 +90,17 @@ export default class ModInfo { * Expected name of zip file containing this mod. */ get filename() { - return `${this.name}_${this.version}.zip`; + return ModInfo.filename(this.name, this.version); + } + + /** + * Expected name of zip file containing mod with given name and version. + * @param name - Mod's name + * @param version - Mod's version + * @returns string containing {name}_{version}.zip + */ + static filename(name: string, version: string) { + return `${name}_${version}.zip`; } /** diff --git a/packages/lib/src/data/messages_mod.ts b/packages/lib/src/data/messages_mod.ts index d9225205..b64c86cf 100644 --- a/packages/lib/src/data/messages_mod.ts +++ b/packages/lib/src/data/messages_mod.ts @@ -114,15 +114,17 @@ export class ModGetRequest { constructor( public name: string, public version: string, + public sha1?: string, ) { } static jsonSchema = Type.Object({ "name": Type.String(), "version": Type.String(), + "sha1": Type.Optional(Type.String()), }); static fromJSON(json: Static) { - return new this(json.name, json.version); + return new this(json.name, json.version, json.sha1); } static Response = ModInfo; @@ -198,22 +200,24 @@ export class ModSearchRequest { export class ModDownloadRequest { declare ["constructor"]: typeof ModDownloadRequest; static type = "request" as const; - static src = "control" as const; + static src = ["host", "control"] as const; static dst = "controller" as const; static permission = "core.mod.download" as const; constructor( public name: string, public version: string, + public sha1?: string, ) { } static jsonSchema = Type.Object({ "name": Type.String(), "version": Type.String(), + "sha1": Type.Optional(Type.String()), }); static fromJSON(json: Static) { - return new this(json.name, json.version); + return new this(json.name, json.version, json.sha1); } static Response = JsonString; diff --git a/packages/web_ui/src/components/ModsPage.tsx b/packages/web_ui/src/components/ModsPage.tsx index fb3133c4..3bc03e8a 100644 --- a/packages/web_ui/src/components/ModsPage.tsx +++ b/packages/web_ui/src/components/ModsPage.tsx @@ -132,7 +132,7 @@ export default function ModsPage() { let [modList] = useModList(); let [modPackList] = useModPackList(); - function actions(mod: lib.ModInfo|lib.ModRecord) { + function actions(mod: lib.ModInfo) { return {account.hasPermission("core.mod.download") && { control.send( new lib.ModDeleteRequest(mod.name, mod.version) From a2eb98ac937747e85e38b261c3718613b1518489 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 3 Dec 2023 00:56:40 +0100 Subject: [PATCH 7/7] Implement hosts downloading needed mods Download and cache needed mods to run instance mod packs on instance startup. To avoid opening too many download streams at once only one mod is downloaded at a time. --- packages/host/host.ts | 9 ++- packages/host/src/Host.ts | 82 +++++++++++++++++++++++++- packages/host/src/Instance.ts | 8 +-- packages/lib/src/config/definitions.ts | 10 +--- test/integration/clusterio.js | 20 +++++++ 5 files changed, 114 insertions(+), 15 deletions(-) diff --git a/packages/host/host.ts b/packages/host/host.ts index 41609cab..5b3daf19 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -203,7 +203,14 @@ async function startHost() { } let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos); - let host = new Host(hostConnector, args.config, hostConfig, tlsCa, pluginInfos); + let host = new Host( + hostConnector, + args.config, + hostConfig, + tlsCa, + pluginInfos, + ...await Host.bootstrap(hostConfig) + ); // Handle interrupts let secondSigint = false; diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index c170e1ad..c9395724 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -263,12 +263,21 @@ export default class Host extends lib.Link { _disconnecting = false; _shuttingDown = false; + static async bootstrap(hostConfig: lib.HostConfig) { + const modsDirectory = hostConfig.get("host.mods_directory"); + await fs.ensureDir(modsDirectory); + return [ + await lib.ModStore.fromDirectory(modsDirectory), + ] as const; + } + constructor( connector: HostConnector, hostConfigPath: string, hostConfig: lib.HostConfig, tlsCa: string | undefined, - pluginInfos: lib.PluginNodeEnvInfo[] + pluginInfos: lib.PluginNodeEnvInfo[], + public modStore: lib.ModStore, ) { super(connector); this.tlsCa = tlsCa; @@ -478,6 +487,77 @@ export default class Host extends lib.Link { } } + async downloadMod(mod: lib.ModRecord) { + let streamId = await this.send(new lib.ModDownloadRequest(mod.name, mod.version, mod.sha1)); + + let url = new URL(this.config.get("host.controller_url")); + url.pathname += `api/stream/${streamId}`; + let response = await phin({ + url, method: "GET", + core: { ca: this.tlsCa } as object, + stream: true, + }); + + const file = lib.ModInfo.filename(mod.name, mod.version); + const filePath = path.join(this.modStore.modsDirectory, file); + const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip"); + const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" }); + await events.once(writeStream, "open"); + response.pipe(writeStream); + await finished(writeStream); + await fs.rename(tempFilePath, filePath); + + const modInfo = await this.modStore.loadFile(file); + if (mod.sha1 && mod.sha1 !== modInfo.sha1) { + throw new Error(`Checksum mismatch downloading ${file} from controller`); + } + logger.info(`Downloaded ${file} from controller`); + return modInfo; + } + + private _fetchModQueue = new lib.AsyncSerialCallback(this._fetchModInternal.bind(this)); + + async fetchMod(mod: lib.ModRecord) { + return await this._fetchModQueue.invoke(mod); + } + + private async _fetchModInternal(mod: lib.ModRecord) { + // The mod may have already been downloaded in a previous call + let modInfo = this.modStore.getMod(mod.name, mod.version); + if (modInfo && (!mod.sha1 || mod.sha1 === modInfo.sha1)) { + return modInfo; + } + + // The mods folder may be shared with the controller and thus + // already have this mod, try loading it. + try { + modInfo = await this.modStore.loadMod(mod.name, mod.version); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + } + if (modInfo && (!mod.sha1 || mod.sha1 === modInfo.sha1)) { + return modInfo; + } + + // Fetch the mod from the controller. + return await this.downloadMod(mod); + } + + async fetchMods(mods: Iterable) { + const modInfos: Promise[] = []; + for (const mod of mods) { + // XXX These mods ship with Factorio, but the list of mods may + // change and should not be hardcoded here. + if (["core", "base"].includes(mod.name)) { + continue; + } + modInfos.push(this.fetchMod(mod)); + } + return await Promise.all(modInfos); + } + async handleInstanceAssignInternalRequest(request: lib.InstanceAssignInternalRequest) { let { instanceId, config } = request; let instanceInfo = this.instanceInfos.get(instanceId); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index f68bdd24..aab54cd1 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -646,9 +646,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); // TODO validate factorioVersion - if (!this._host.config.get("host.mods_directory_is_shared")) { - throw new Error("Fetching mods is not implemented"); - } + const mods = await this._host.fetchMods(modPack.mods.values()); await fs.ensureDir(this.path("mods")); @@ -668,8 +666,8 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); // Add mods from mod the pack const modsDir = this._host.config.get("host.mods_directory"); - for (let mod of this.activeModPack.mods.values()) { - const modFile = `${mod.name}_${mod.version}.zip`; + for (let mod of mods) { + const modFile = mod.filename; const target = path.join(modsDir, modFile); const link = this.path("mods", modFile); diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 282e5b60..098e8522 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -21,6 +21,7 @@ ControllerGroup.define({ name: "mods_directory", title: "Mods Directory", description: "Path to directory where mods shared with the cluster are stored.", + restartRequired: true, type: "string", initial_value: "mods", }); @@ -202,16 +203,10 @@ HostGroup.define({ name: "mods_directory", title: "Mods Directory", description: "Path to directory where mods for instances are cached.", + restartRequired: true, type: "string", initial_value: "mods", }); -HostGroup.define({ - name: "mods_directory_is_shared", - title: "Mods Directory is Shared", - description: "True if the mods directory is shared with the controller.", - type: "boolean", - initial_value: true, -}); HostGroup.define({ name: "instances_directory", description: "Path to directory to store instances in.", @@ -264,7 +259,6 @@ export class HostConfig extends classes.Config { get(name: "host.id"): number; get(name: "host.factorio_directory"): string; get(name: "host.mods_directory"): string; - get(name: "host.mods_directory_is_shared"): boolean; get(name: "host.instances_directory"): string; get(name: "host.controller_url"): string; get(name: "host.controller_token"): string; diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index b076b439..b9931847 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -32,9 +32,12 @@ async function startAltHost() { let configPath = path.join("temp", "test", config); await fs.remove(configPath); await fs.remove(path.join("temp", "test", "alt-instances")); + await fs.remove(path.join("temp", "test", "alt-mods")); await execCtl(`host create-config --id 5 --name alt-host --generate-token --output ${config}`); await exec(`node ../../packages/host --config ${config} config set host.tls_ca ../../test/file/tls/cert.pem`); await exec(`node ../../packages/host --config ${config} config set host.instances_directory alt-instances`); + await exec(`node ../../packages/host --config ${config} config set host.factorio_directory ../../factorio`); + await exec(`node ../../packages/host --config ${config} config set host.mods_directory alt-mods`); return await spawn("alt-host:", `node ../../packages/host run --config ${config}`, /Started host/); } @@ -176,6 +179,23 @@ describe("Integration of Clusterio", function() { assert(hosts.has(5), "Host list was not updated"); }); }); + it("should download mods from controller", async function() { + // On windows there's currently no way to automate graceful shutdown of the host + // process as CTRL+C is some weird terminal thing and SIGINT isn't a thing. + if (process.platform === "win32") { + this.skip(); + } + + slowTest(this); + await runWithAltHost(async () => { + const clusterioLib = path.join("temp", "test", "alt-mods", "clusterio_lib_0.1.2.zip"); + assert(!await fs.pathExists(clusterioLib), "mod was downloaded before the test"); + await execCtl("instance create alt-mod --id 98"); + await execCtl("instance assign alt-mod 5"); + await execCtl("instance save create alt-mod"); + assert(await fs.pathExists(clusterioLib), "mod was not downloaded by the test"); + }); + }); }); describe("clusterioctl", function() {