From 14934a0d3aed5d03890f1db786b64eab7f0a8b1e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Aug 2022 18:18:25 +0200 Subject: [PATCH 01/40] Move checkFilename and cleanFilename to libFileOps These are also useful for managing mod zip files on the master. --- packages/lib/file_ops.js | 85 ++++++++++++++++++++++++++++++++ packages/slave/src/Slave.js | 86 ++------------------------------ test/lib/file_ops.js | 97 +++++++++++++++++++++++++++++++++++++ test/slave.js | 96 ------------------------------------ 4 files changed, 185 insertions(+), 179 deletions(-) diff --git a/packages/lib/file_ops.js b/packages/lib/file_ops.js index c04077be..cbed8df5 100644 --- a/packages/lib/file_ops.js +++ b/packages/lib/file_ops.js @@ -108,9 +108,94 @@ async function safeOutputFile(file, data, options={}) { await fs.rename(temporary, file); } +/** + * Check if a string is a valid file name + * + * @param {string} name - Name to check + * @throws Error if the name is unsuitable. + */ +function checkFilename(name) { + // All of these are bad in Windows only, except for /, . and .. + // See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; + const badEnd = /[. ]$/; + + const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + const badNames = [ + // Relative path components + ".", "..", + + // Reserved filenames in Windows + "CON", "PRN", "AUX", "NUL", + ...oneToNine.map(n => `COM${n}`), + ...oneToNine.map(n => `LPT${n}`), + ]; + + if (typeof name !== "string") { + throw new Error("must be a string"); + } + + if (name === "") { + throw new Error("cannot be empty"); + } + + if (badChars.test(name)) { + throw new Error('cannot contain <>:"\\/|=* or control characters'); + } + + if (badNames.includes(name.toUpperCase())) { + throw new Error( + "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9" + ); + } + + if (badEnd.test(name)) { + throw new Error("cannot end with . or space"); + } +} + +/** + * Clean up string to be suitable for use as filename + * + * @param {string} name - Arbitrary name string + * @returns {string} Filename suitable to use in the filesystem + */ +function cleanFilename(name) { + // copied from checkFilename + const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; + const badEnd = /[. ]$/; + + const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + const badNames = [ + // Relative path components + ".", "..", + + // Reserved filenames in Windows + "CON", "PRN", "AUX", "NUL", + ...oneToNine.map(n => `COM${n}`), + ...oneToNine.map(n => `LPT${n}`), + ]; + + if (typeof name !== "string") { + throw new Error("name must be a string"); + } + + if (name === "" || badNames.includes(name.toUpperCase())) { + name += "_"; + } + + name = name.replace(badChars, "_"); + name = name.replace(badEnd, "_"); + + return name; +} + + module.exports = { getNewestFile, findUnusedName, getTempFile, safeOutputFile, + checkFilename, + cleanFilename, }; diff --git a/packages/slave/src/Slave.js b/packages/slave/src/Slave.js index 4c28b605..c8b85f57 100644 --- a/packages/slave/src/Slave.js +++ b/packages/slave/src/Slave.js @@ -25,86 +25,9 @@ const InstanceConnection = require("./InstanceConnection"); const finished = util.promisify(stream.finished); -function checkFilename(name) { - // All of these are bad in Windows only, except for /, . and .. - // See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file - const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; - const badEnd = /[. ]$/; - - const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - const badNames = [ - // Relative path components - ".", "..", - - // Reserved filenames in Windows - "CON", "PRN", "AUX", "NUL", - ...oneToNine.map(n => `COM${n}`), - ...oneToNine.map(n => `LPT${n}`), - ]; - - if (typeof name !== "string") { - throw new Error("must be a string"); - } - - if (name === "") { - throw new Error("cannot be empty"); - } - - if (badChars.test(name)) { - throw new Error('cannot contain <>:"\\/|=* or control characters'); - } - - if (badNames.includes(name.toUpperCase())) { - throw new Error( - "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9" - ); - } - - if (badEnd.test(name)) { - throw new Error("cannot end with . or space"); - } -} - -/** - * Clean up string to be suitable for use as filename - * - * @param {string} name - Arbitrary name string - * @returns {string} Filename suitable to use in the filesystem - * @private - */ -function cleanFilename(name) { - // copied from checkFilename - const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; - const badEnd = /[. ]$/; - - const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - const badNames = [ - // Relative path components - ".", "..", - - // Reserved filenames in Windows - "CON", "PRN", "AUX", "NUL", - ...oneToNine.map(n => `COM${n}`), - ...oneToNine.map(n => `LPT${n}`), - ]; - - if (typeof name !== "string") { - throw new Error("name must be a string"); - } - - if (name === "" || badNames.includes(name.toUpperCase())) { - name += "_"; - } - - name = name.replace(badChars, "_"); - name = name.replace(badEnd, "_"); - - return name; -} - function checkRequestSaveName(name) { try { - checkFilename(name); + libFileOps.checkFilename(name); } catch (err) { throw new libErrors.RequestError(`Save name ${err.message}`); } @@ -243,9 +166,9 @@ class Slave extends libLink.Link { } async _createNewInstanceDir(name) { - name = cleanFilename(name); + name = libFileOps.cleanFilename(name); try { - checkFilename(name); + libFileOps.checkFilename(name); } catch (err) { throw new Error(`Instance folder was unepectedly invalid: name ${err.message}`); } @@ -837,8 +760,5 @@ ${err.stack}` module.exports = Slave; // For testing only -module.exports._checkFilename = checkFilename; -module.exports._cleanFilename = cleanFilename; module.exports._discoverInstances = discoverInstances; -module.exports._Slave = Slave; diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index 21823231..416c90a0 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -84,4 +84,101 @@ describe("lib/file_ops", function() { assert.equal(await fs.readFile(target, "utf8"), "current"); }); }); + + describe("checkFilename()", function() { + it("should allow a basic name", function() { + libFileOps.checkFilename("file"); + }); + + function check(item, msg) { + assert.throws(() => libFileOps.checkFilename(item), new Error(msg)); + } + + it("should throw on non-string", function() { + check(undefined, "must be a string"); + check(null, "must be a string"); + check({}, "must be a string"); + check([], "must be a string"); + check(0, "must be a string"); + check(false, "must be a string"); + }); + + it("should throw on empty name", function() { + check("", "cannot be empty"); + }); + + it("should throw on <>:\"\\/|?* \\x00\\r\\n\\t", function() { + for (let char of '<>:"\\/|?*\x00\r\n\t') { + check(char, 'cannot contain <>:"\\/|=* or control characters'); + } + }); + + it("should throw on CON, PRN, AUX, NUL, COM1, LPT1", function() { + for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { + check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); + } + }); + + it("should throw on . and ..", function() { + for (let bad of [".", ".."]) { + check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); + } + }); + + it("should throw on names ending with . or space", function() { + check("a ", "cannot end with . or space"); + check("a.", "cannot end with . or space"); + }); + }); + + describe("cleanFilename()", function() { + function clean(item, expected) { + assert.equal(libFileOps.cleanFilename(item), expected); + } + + it("should allow a basic name", function() { + clean("file", "file"); + }); + + function check(item, msg) { + assert.throws(() => libFileOps.cleanFilename(item), new Error(msg)); + } + + it("should throw on non-string", function() { + check(undefined, "name must be a string"); + check(null, "name must be a string"); + check({}, "name must be a string"); + check([], "name must be a string"); + check(0, "name must be a string"); + check(false, "name must be a string"); + }); + + it("should clean empty name", function() { + clean("", "_"); + }); + + it("should clean <>:\"\\/|?* \\x00\\r\\n\\t", function() { + for (let char of '<>:"\\/|?*\x00\r\n\t') { + clean(char, "_"); + } + }); + + it("should clean CON, PRN, AUX, NUL, COM1, LPT1", function() { + for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { + clean(bad, `${bad}_`); + } + }); + + it("should clean . and ..", function() { + for (let bad of [".", ".."]) { + clean(bad, `${bad}_`); + } + }); + + it("should clean names ending with . or space", function() { + clean("a ", "a_"); + clean("a.", "a_"); + }); + }); + }); diff --git a/test/slave.js b/test/slave.js index 36772faa..0cb23498 100644 --- a/test/slave.js +++ b/test/slave.js @@ -13,102 +13,6 @@ describe("Slave testing", function() { libConfig.InstanceConfig.finalize(); }); - describe("checkFilename()", function() { - it("should allow a basic name", function() { - Slave._checkFilename("file"); - }); - - function check(item, msg) { - assert.throws(() => Slave._checkFilename(item), new Error(msg)); - } - - it("should throw on non-string", function() { - check(undefined, "must be a string"); - check(null, "must be a string"); - check({}, "must be a string"); - check([], "must be a string"); - check(0, "must be a string"); - check(false, "must be a string"); - }); - - it("should throw on empty name", function() { - check("", "cannot be empty"); - }); - - it("should throw on <>:\"\\/|?* \\x00\\r\\n\\t", function() { - for (let char of '<>:"\\/|?*\x00\r\n\t') { - check(char, 'cannot contain <>:"\\/|=* or control characters'); - } - }); - - it("should throw on CON, PRN, AUX, NUL, COM1, LPT1", function() { - for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { - check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); - } - }); - - it("should throw on . and ..", function() { - for (let bad of [".", ".."]) { - check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); - } - }); - - it("should throw on names ending with . or space", function() { - check("a ", "cannot end with . or space"); - check("a.", "cannot end with . or space"); - }); - }); - - describe("cleanFilename()", function() { - function clean(item, expected) { - assert.equal(Slave._cleanFilename(item), expected); - } - - it("should allow a basic name", function() { - clean("file", "file"); - }); - - function check(item, msg) { - assert.throws(() => Slave._cleanFilename(item), new Error(msg)); - } - - it("should throw on non-string", function() { - check(undefined, "name must be a string"); - check(null, "name must be a string"); - check({}, "name must be a string"); - check([], "name must be a string"); - check(0, "name must be a string"); - check(false, "name must be a string"); - }); - - it("should clean empty name", function() { - clean("", "_"); - }); - - it("should clean <>:\"\\/|?* \\x00\\r\\n\\t", function() { - for (let char of '<>:"\\/|?*\x00\r\n\t') { - clean(char, "_"); - } - }); - - it("should clean CON, PRN, AUX, NUL, COM1, LPT1", function() { - for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { - clean(bad, `${bad}_`); - } - }); - - it("should clean . and ..", function() { - for (let bad of [".", ".."]) { - clean(bad, `${bad}_`); - } - }); - - it("should clean names ending with . or space", function() { - clean("a ", "a_"); - clean("a.", "a_"); - }); - }); - describe("symlinkMods()", function() { let testDir = path.join("temp", "test", "symlink"); From 9a1c7b2ecdb6f2eb8ef055ccfb74557662a55236 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Aug 2022 18:35:41 +0200 Subject: [PATCH 02/40] Bad Windows names also apply with any extension A file name con.zip is also treated in the same way as the special CON redirection and may not be used as a normal filesystem path in Windows. This includes any extension, even with multiple dots. Fix checkFilename and cleaneFilename not handling these cases. --- packages/lib/file_ops.js | 50 ++++++++++++++++++++-------------------- test/lib/file_ops.js | 12 ++++++++-- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/packages/lib/file_ops.js b/packages/lib/file_ops.js index cbed8df5..be007912 100644 --- a/packages/lib/file_ops.js +++ b/packages/lib/file_ops.js @@ -108,6 +108,18 @@ async function safeOutputFile(file, data, options={}) { await fs.rename(temporary, file); } + +// Reserved names by allmost all filesystems +const badNames = [".", ".."]; + +// Reserved namespaces in Windows +const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; +const badWinNamespaces = [ + "CON", "PRN", "AUX", "NUL", + ...oneToNine.map(n => `COM${n}`), + ...oneToNine.map(n => `LPT${n}`), +]; + /** * Check if a string is a valid file name * @@ -120,17 +132,6 @@ function checkFilename(name) { const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; const badEnd = /[. ]$/; - const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - const badNames = [ - // Relative path components - ".", "..", - - // Reserved filenames in Windows - "CON", "PRN", "AUX", "NUL", - ...oneToNine.map(n => `COM${n}`), - ...oneToNine.map(n => `LPT${n}`), - ]; - if (typeof name !== "string") { throw new Error("must be a string"); } @@ -143,9 +144,15 @@ function checkFilename(name) { throw new Error('cannot contain <>:"\\/|=* or control characters'); } - if (badNames.includes(name.toUpperCase())) { + if (badNames.includes(name)) { throw new Error( - "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9" + `cannot be named ${name}` + ); + } + + if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) { + throw new Error( + "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9" ); } @@ -161,21 +168,10 @@ function checkFilename(name) { * @returns {string} Filename suitable to use in the filesystem */ function cleanFilename(name) { - // copied from checkFilename + // copied from checkFilename due to RegExp with global flag containing state. const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; const badEnd = /[. ]$/; - const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - const badNames = [ - // Relative path components - ".", "..", - - // Reserved filenames in Windows - "CON", "PRN", "AUX", "NUL", - ...oneToNine.map(n => `COM${n}`), - ...oneToNine.map(n => `LPT${n}`), - ]; - if (typeof name !== "string") { throw new Error("name must be a string"); } @@ -184,6 +180,10 @@ function cleanFilename(name) { name += "_"; } + if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) { + name = [`${name.split(".")[0]}_`, ...name.split(".").slice(1)].join("."); + } + name = name.replace(badChars, "_"); name = name.replace(badEnd, "_"); diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index 416c90a0..da256995 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -115,13 +115,17 @@ describe("lib/file_ops", function() { it("should throw on CON, PRN, AUX, NUL, COM1, LPT1", function() { for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { - check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); + check(bad, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}.zip`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}.anything.txt`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}....a`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}.`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); } }); it("should throw on . and ..", function() { for (let bad of [".", ".."]) { - check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); + check(bad, `cannot be named ${bad}`); } }); @@ -166,6 +170,10 @@ describe("lib/file_ops", function() { it("should clean CON, PRN, AUX, NUL, COM1, LPT1", function() { for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { clean(bad, `${bad}_`); + clean(`${bad}.zip`, `${bad}_.zip`); + clean(`${bad}.anything.txt`, `${bad}_.anything.txt`); + clean(`${bad}....a`, `${bad}_....a`); + clean(`${bad}.`, `${bad}__`); } }); From a40fefc8d7c17289df8a7f163260790e37ce8be5 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Aug 2022 18:42:42 +0200 Subject: [PATCH 03/40] Pull findRoot out into libZipOps The findRoot function useful for managing mod zip files on the master. --- packages/lib/factorio/patch.js | 38 +------------------------- packages/lib/index.js | 2 ++ packages/lib/zip_ops.js | 40 +++++++++++++++++++++++++++ test/lib/factorio/patch.js | 41 ---------------------------- test/lib/zip_ops.js | 49 ++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 78 deletions(-) create mode 100644 packages/lib/zip_ops.js create mode 100644 test/lib/zip_ops.js diff --git a/packages/lib/factorio/patch.js b/packages/lib/factorio/patch.js index f96e931a..7f92232f 100644 --- a/packages/lib/factorio/patch.js +++ b/packages/lib/factorio/patch.js @@ -8,6 +8,7 @@ const path = require("path"); const semver = require("semver"); const libHash = require("../hash"); +const { findRoot } = require("../zip_ops"); const knownScenarios = { @@ -21,42 +22,6 @@ const knownScenarios = { }, }; -/** - * Returns the root folder in the zip file - * - * Returns the name of the folder that all files in the zip file is - * contained in. Throws an error if there are multiple such folders. - * - * @param {JSZip} zip - Zip to search through. - * @returns {string} name of the root folder. - * @memberof module:lib/factorio - * @private - * @inner - */ -function findRoot(zip) { - let root = null; - zip.forEach((relativePath, file) => { - let index = relativePath.indexOf("/"); - if (index === -1) { - throw new Error(`Zip contains file '${relativePath}' in root dir`); - } - - let pathRoot = relativePath.slice(0, index); - if (root === null) { - root = pathRoot; - } else if (root !== pathRoot) { - throw new Error("Zip contains multiple root folders"); - } - }); - - if (root === null) { - throw new Error("Empty zip file"); - } - - return root; -} - - /** * Generates control.lua code for loading the Clusterio modules * @@ -295,7 +260,6 @@ module.exports = { patch, // For testing only - _findRoot: findRoot, _generateLoader: generateLoader, _reorderDependencies: reorderDependencies, }; diff --git a/packages/lib/index.js b/packages/lib/index.js index 0c89ef93..37777950 100644 --- a/packages/lib/index.js +++ b/packages/lib/index.js @@ -18,6 +18,7 @@ const libPrometheus = require("./prometheus"); const libSchema = require("./schema"); const libSharedCommands = require("./shared_commands"); const libUsers = require("./users"); +const libZipOps = require("./zip_ops"); const ExponentialBackoff = require("./ExponentialBackoff"); const RateLimiter = require("./RateLimiter"); @@ -43,6 +44,7 @@ module.exports = { libSchema, libSharedCommands, libUsers, + libZipOps, ExponentialBackoff, RateLimiter, diff --git a/packages/lib/zip_ops.js b/packages/lib/zip_ops.js new file mode 100644 index 00000000..f05c09b5 --- /dev/null +++ b/packages/lib/zip_ops.js @@ -0,0 +1,40 @@ +"use strict"; + +/** + * Returns the root folder in the zip file + * + * Returns the name of the folder that all files in the zip file is + * contained in. Throws an error if there are multiple such folders. + * + * @param {JSZip} zip - Zip to search through. + * @returns {string} name of the root folder. + * @memberof module:lib/factorio + * @private + * @inner + */ +function findRoot(zip) { + let root = null; + zip.forEach((relativePath, file) => { + let index = relativePath.indexOf("/"); + if (index === -1) { + throw new Error(`Zip contains file '${relativePath}' in root dir`); + } + + let pathRoot = relativePath.slice(0, index); + if (root === null) { + root = pathRoot; + } else if (root !== pathRoot) { + throw new Error("Zip contains multiple root folders"); + } + }); + + if (root === null) { + throw new Error("Empty zip file"); + } + + return root; +} + +module.exports = { + findRoot, +}; diff --git a/test/lib/factorio/patch.js b/test/lib/factorio/patch.js index 00850590..e301d583 100644 --- a/test/lib/factorio/patch.js +++ b/test/lib/factorio/patch.js @@ -8,47 +8,6 @@ const libFactorio = require("@clusterio/lib/factorio"); describe("lib/factorio", function() { - describe("findRoot()", function() { - it("should find the root of a zip", function() { - let zip = new JSZip(); - zip.file("root/foo.txt", ""); - zip.file("root/bar.txt", ""); - - assert.equal(libFactorio._findRoot(zip), "root"); - }); - - it("should throw if files are at the root of the zip", function() { - let zip = new JSZip(); - zip.file("root/foo.txt", ""); - zip.file("bar.txt", ""); - - assert.throws( - () => libFactorio._findRoot(zip), - new Error("Zip contains file 'bar.txt' in root dir") - ); - }); - - it("should throw if there are multiple root dirs", function() { - let zip = new JSZip(); - zip.file("root-1/foo.txt", ""); - zip.file("root-2/bar.txt", ""); - - assert.throws( - () => libFactorio._findRoot(zip), - new Error("Zip contains multiple root folders") - ); - }); - - it("should throw if given an empty zip file", function() { - let zip = new JSZip(); - - assert.throws( - () => libFactorio._findRoot(zip), - new Error("Empty zip file") - ); - }); - }); - describe("generateLoader()", function() { let reference = [ "-- Auto generated scenario module loader created by Clusterio", diff --git a/test/lib/zip_ops.js b/test/lib/zip_ops.js new file mode 100644 index 00000000..622c65ae --- /dev/null +++ b/test/lib/zip_ops.js @@ -0,0 +1,49 @@ +"use strict"; +const assert = require("assert").strict; +const JSZip = require("jszip"); + +const { findRoot } = require("@clusterio/lib/zip_ops"); + + +describe("lib/factorio", function() { + describe("findRoot()", function() { + it("should find the root of a zip", function() { + let zip = new JSZip(); + zip.file("root/foo.txt", ""); + zip.file("root/bar.txt", ""); + + assert.equal(findRoot(zip), "root"); + }); + + it("should throw if files are at the root of the zip", function() { + let zip = new JSZip(); + zip.file("root/foo.txt", ""); + zip.file("bar.txt", ""); + + assert.throws( + () => findRoot(zip), + new Error("Zip contains file 'bar.txt' in root dir") + ); + }); + + it("should throw if there are multiple root dirs", function() { + let zip = new JSZip(); + zip.file("root-1/foo.txt", ""); + zip.file("root-2/bar.txt", ""); + + assert.throws( + () => findRoot(zip), + new Error("Zip contains multiple root folders") + ); + }); + + it("should throw if given an empty zip file", function() { + let zip = new JSZip(); + + assert.throws( + () => findRoot(zip), + new Error("Empty zip file") + ); + }); + }); +}); From 921ebad8fb165c370a996b1dd9df7e256a292489 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Aug 2022 18:58:08 +0200 Subject: [PATCH 04/40] Allow importing libHash to succeed in browser Alsthough this does not make libHash work in the browser it allows other modules to require libHash while still being exported to the browser. --- packages/lib/hash.js | 2 +- packages/web_ui/webpack.common.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/lib/hash.js b/packages/lib/hash.js index df26114e..12359602 100644 --- a/packages/lib/hash.js +++ b/packages/lib/hash.js @@ -4,7 +4,7 @@ */ "use strict"; const crypto = require("crypto"); -const fs = require("fs"); +const fs = require("fs-extra"); /** diff --git a/packages/web_ui/webpack.common.js b/packages/web_ui/webpack.common.js index 8b871783..628eb217 100644 --- a/packages/web_ui/webpack.common.js +++ b/packages/web_ui/webpack.common.js @@ -95,6 +95,8 @@ module.exports = (env = {}) => ({ resolve: { extensions: [".js", ".json", ".jsx"], fallback: { + "crypto": false, + // Required for winston "util": require.resolve("util/"), "os": require.resolve("os-browserify/browser"), From 2ab915cc00e37b54264d845d13cbb889f3b6176a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 23 Aug 2022 19:01:56 +0200 Subject: [PATCH 05/40] Move formatBytes to libHelpers --- packages/lib/helpers.js | 19 +++++++++++++++++++ packages/web_ui/src/components/SavesList.jsx | 15 ++------------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/lib/helpers.js b/packages/lib/helpers.js index 17464ec5..c3fe23fb 100644 --- a/packages/lib/helpers.js +++ b/packages/lib/helpers.js @@ -90,6 +90,24 @@ function escapeRegExp(text) { return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); } +/** + * Format byte count for human readable display + * + * Shortens a large number of bytes using the kB/MB/GB/TB prefixes. + * @param {number} bytes - Count of bytes to format. + * @returns {string} formatted text. + */ +function formatBytes(bytes) { + if (bytes === 0) { + return "0 Bytes"; + } + + let units = [" Bytes", " kB", " MB", " GB", " TB"]; + let factor = 1000; + let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length); + return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; +} + module.exports = { basicType, @@ -97,4 +115,5 @@ module.exports = { timeout, readStream, escapeRegExp, + formatBytes, }; diff --git a/packages/web_ui/src/components/SavesList.jsx b/packages/web_ui/src/components/SavesList.jsx index 133cffac..686c567c 100644 --- a/packages/web_ui/src/components/SavesList.jsx +++ b/packages/web_ui/src/components/SavesList.jsx @@ -6,7 +6,7 @@ import { import CaretLeftOutlined from "@ant-design/icons/CaretLeftOutlined"; import LeftOutlined from "@ant-design/icons/LeftOutlined"; -import { libLink } from "@clusterio/lib"; +import { libLink, libHelpers } from "@clusterio/lib"; import { useAccount } from "../model/account"; import ControlContext from "./ControlContext"; @@ -179,17 +179,6 @@ function TransferModal(props) { ; } -function formatBytes(bytes) { - if (bytes === 0) { - return "0 Bytes"; - } - - let units = [" Bytes", " kB", " MB", " GB", " TB"]; - let factor = 1000; - let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length); - return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; -} - export default function SavesList(props) { let account = useAccount(); let control = useContext(ControlContext); @@ -214,7 +203,7 @@ export default function SavesList(props) { title: "Size", key: "size", responsive: ["sm"], - render: save => formatBytes(save.size), + render: save => libHelpers.formatBytes(save.size), align: "right", sorter: (a, b) => a.size - b.size, }, From 3b57c1e0ae5ec2f89f7db777b1b1959b836f5e4a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 24 Aug 2022 15:38:09 +0200 Subject: [PATCH 06/40] Add shared mods management Add the ability to manage a set of mods to be shared by the master server to the rest of the cluster. This implements storage, uploads, downloads, deletetion, and listing details of mods but does not actually do anything with the stored mods. --- .gitignore | 1 + packages/ctl/ctl.js | 2 + packages/ctl/src/commands.js | 146 +++++++++++++ packages/lib/browser.js | 1 + packages/lib/config/definitions.js | 7 + packages/lib/data/ModInfo.js | 230 ++++++++++++++++++++ packages/lib/data/index.js | 12 + packages/lib/index.js | 2 + packages/lib/link/messages.js | 72 +++++- packages/lib/plugin.js | 13 +- packages/lib/users.js | 35 +++ packages/master/src/ControlConnection.js | 60 ++++- packages/master/src/Master.js | 71 ++++++ packages/master/src/routes.js | 164 +++++++++++++- packages/web_ui/src/components/ModsPage.jsx | 165 ++++++++++++++ packages/web_ui/src/index.css | 9 + packages/web_ui/src/model/mods.jsx | 50 +++++ packages/web_ui/src/pages.jsx | 7 + packages/web_ui/src/util/websocket.jsx | 64 +++++- test/file/empty_mod/info.json | 9 + test/integration/clusterio.js | 87 ++++++++ test/integration/index.js | 8 + test/lib/data/ModInfo.js | 101 +++++++++ 23 files changed, 1311 insertions(+), 5 deletions(-) create mode 100644 packages/lib/data/ModInfo.js create mode 100644 packages/lib/data/index.js create mode 100644 packages/web_ui/src/components/ModsPage.jsx create mode 100644 packages/web_ui/src/model/mods.jsx create mode 100644 test/file/empty_mod/info.json create mode 100644 test/lib/data/ModInfo.js diff --git a/.gitignore b/.gitignore index 84213197..a769fb6c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ Factorio* /factorio /instances /sharedMods +/mods /database /plugin-list.json /config-master.json diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 9211543c..d054f89b 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -89,6 +89,8 @@ class Control extends libLink.Link { async saveListUpdateEventHandler() { } + async modUpdateEventHandler() { } + async userUpdateEventHandler() { } async setLogSubscriptions({ diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index ba5df6bd..2d09982c 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -860,6 +860,151 @@ instanceCommands.add(new libCommand.Command({ }, })); +const modCommands = new libCommand.CommandTree({ name: "mod", description: "Manage uploaded mods" }); +modCommands.add(new libCommand.Command({ + definition: ["show ", "Show details for a mod stored in the cluster", (yargs) => { + yargs.positional("name", { describe: "Mod name to show details for", type: "string" }); + yargs.positional("mod-version", { describe: "Version of the mod", type: "string" }); + }], + handler: async function(args, control) { + let response = await libLink.messages.getMod.send(control, { name: args.name, version: args.modVersion }); + for (let [field, value] of Object.entries(response.mod)) { + if (value instanceof Array) { + print(`${field}:`); + for (let entry of value) { + print(` ${entry}`); + } + } else { + print(`${field}: ${value}`); + } + } + }, +})); + +modCommands.add(new libCommand.Command({ + definition: [["list", "l"], "List mods stored in the cluster", (yargs) => { + yargs.options({ + "fields": { + describe: "Fields", + array: true, + type: "string", + default: ["name", "version", "title", "factorio_version"], + }, + }); + }], + handler: async function(args, control) { + let response = await libLink.messages.listMods.send(control); + if (!args.fields.includes("all")) { + for (let entry of response.list) { + for (let field of Object.keys(entry)) { + if (!args.fields.includes(field)) { + delete entry[field]; + } + } + } + } + print(asTable(response.list)); + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["upload ", "Upload mod to the cluster", (yargs) => { + yargs.positional("file", { describe: "File to upload", type: "string" }); + }], + handler: async function(args, control) { + let filename = path.basename(args.file); + if (!filename.endsWith(".zip")) { + throw new libErrors.CommandError("Mod filename must end with .zip"); + } + // phin doesn't support streaming requests :( + let content = await fs.readFile(args.file); + + let url = new URL(control.config.get("control.master_url")); + url.pathname += "api/upload-mod"; + url.searchParams.append("filename", filename); + + let result = await phin({ + url, method: "POST", + headers: { + "X-Access-Token": control.config.get("control.master_token"), + "Content-Type": "application/zip", + }, + core: { ca: control.tlsCa }, + data: content, + parse: "json", + }); + + for (let error of result.body.errors || []) { + logger.error(error); + } + + for (let requestError of result.body.request_errors || []) { + logger.error(requestError); + } + + if (result.body.mods && result.body.mods.length) { + logger.info(`Successfully uploaded ${result.body.mods[0].filename}`); + } + + if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + throw new libErrors.CommandError("Uploading mod failed"); + } + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["download ", "Download a mod from the cluster", (yargs) => { + yargs.positional("name", { describe: "Internal name of mod to download", type: "string" }); + yargs.positional("mod-version", { describe: "Version of mod to download", type: "string" }); + }], + handler: async function(args, control) { + let result = await libLink.messages.downloadMod.send(control, { + name: args.name, + version: args.modVersion, + }); + + let url = new URL(control.config.get("control.master_url")); + url.pathname += `api/stream/${result.stream_id}`; + let response = await phin({ + url, method: "GET", + core: { ca: control.tlsCa }, + stream: true, + }); + + let writeStream; + let filename = `${args.name}_${args.modVersion}.zip`; + let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + while (true) { + try { + writeStream = fs.createWriteStream(tempFilename, { flags: "wx" }); + await events.once(writeStream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(".", tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + response.pipe(writeStream); + await finished(writeStream); + await fs.rename(tempFilename, filename); + + logger.info(`Downloaded ${filename}`); + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["delete ", "Delete a mod stored in the cluster", (yargs) => { + yargs.positional("name", { describe: "Name of mod to delete", type: "string" }); + yargs.positional("mod-version", { describe: "Version of mod to delete", type: "string" }); + }], + handler: async function(args, control) { + await libLink.messages.deleteMod.send(control, { name: args.name, version: args.modVersion }); + }, +})); + const permissionCommands = new libCommand.CommandTree({ name: "permission", description: "Permission inspection" }); permissionCommands.add(new libCommand.Command({ definition: [["list", "l"], "List permissions in the cluster"], @@ -1194,6 +1339,7 @@ async function registerCommands(controlPlugins, yargs) { rootCommands.add(masterCommands); rootCommands.add(slaveCommands); rootCommands.add(instanceCommands); + rootCommands.add(modCommands); rootCommands.add(permissionCommands); rootCommands.add(roleCommands); rootCommands.add(userCommands); diff --git a/packages/lib/browser.js b/packages/lib/browser.js index 23c9850e..20866a72 100644 --- a/packages/lib/browser.js +++ b/packages/lib/browser.js @@ -1,4 +1,5 @@ export { default as libConfig } from "./config"; +export { default as libData } from "./data"; export { default as libErrors } from "./errors"; export { default as libFactorio } from "./factorio/exchange_string"; export { default as libHelpers } from "./helpers"; diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index e713f48f..01d74f78 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -19,6 +19,13 @@ MasterGroup.define({ type: "string", initial_value: "Your Cluster", }); +MasterGroup.define({ + name: "mods_directory", + title: "Mods Directory", + description: "Path to directory where mods shared with the cluster are stored.", + type: "string", + initial_value: "mods", +}); MasterGroup.define({ name: "database_directory", title: "Database directory", diff --git a/packages/lib/data/ModInfo.js b/packages/lib/data/ModInfo.js new file mode 100644 index 00000000..e1949d0c --- /dev/null +++ b/packages/lib/data/ModInfo.js @@ -0,0 +1,230 @@ +"use strict"; +const fs = require("fs-extra"); +const JSZip = require("jszip"); +const path = require("path"); + +const libHash = require("../hash"); +const libSchema = require("../schema"); +const { findRoot } = require("../zip_ops"); + + +/** + * Info about a mod available on the master server. + * + * See https://wiki.factorio.com/Tutorial:Mod_structure#info.json for + * details of the invidiual fields sourced from info.json. + * @alias module:lib/data/ModInfo + */ +class ModInfo { + /** + * Internal name of this mod. + * This is the name of the zip file as well as the name that appears in + * info.json. + * @type {string} + */ + name = ""; + + /** + * Version of the mod. + * Sourced from info.json. + * @type {string} + */ + version = ""; + + /** + * Integer representation of the version + * @type {number} + */ + get integerVersion() { + const [major, minor, sub] = this.version.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000 + sub; // Can't use bitwise here because this is 48-bits. + } + + /** + * Display name of the mod. + * Sourced from info.json. + * @type {string} + */ + title = ""; + + /** + * Author of the mod. + * Sourced from info.json. + * @type {string} + */ + author = ""; + + /** + * Contact field for mod. + * Sourced from info.json. + * @type {string} + */ + contact = ""; + + /** + * Homepage of the mod. + * Sourced from info.json. + * @type {string} + */ + homepage = ""; + + /** + * Description of the mod. + * Sourced from info.json. + * @type {string} + */ + description = ""; + + /** + * Major version of Factorio this mod supports. + * Sourced from info.json. + * @type {string} + */ + factorioVersion = "0.12"; + + /** + * Integer representation of the factorioVersion + * @type {number} + */ + get integerFactorioVersion() { + const [major, minor] = this.factorioVersion.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000; // Can't use bitwise here because this is 48-bits. + } + + /** + * Dependiences for this mod. + * Sourced from info.json. + * @type {Array} + */ + dependencies = ["base"]; + + /** + * Name of the file containing this mod. + * @type {number} + */ + filename = ""; + + /** + * Size of the mod in bytes + * @type {number} + */ + size = 0; + + /** + * Hash of this mod + * @type {string=} + */ + hash; + + /** + * True if this mod has been deleted + * @type {boolean} + */ + isDeleted = false; + + // Content of info.json found in mod files + static infoJsonSchema = { + type: "object", + required: ["name", "version", "title", "author"], + properties: { + "name": { type: "string" }, + "version": { type: "string" }, + "title": { type: "string" }, + "author": { type: "string" }, + "contact": { type: "string" }, + "homepage": { type: "string" }, + "description": { type: "string" }, + "factorio_version": { type: "string" }, + "dependencies": { type: "array", items: { type: "string" } }, + }, + }; + + static validateInfo = libSchema.compile(this.infoJsonSchema); + + static jsonSchema = { + type: "object", + properties: { + ...this.infoJsonSchema.properties, + "filename": { type: "string" }, + "size": { type: "integer" }, + "hash": { type: "string" }, + "is_deleted": { type: "boolean" }, + }, + }; + + static validate = libSchema.compile(this.jsonSchema); + + constructor(json = {}) { + // info.json fields + if (json.name) { this.name = json.name; } + if (json.version) { this.version = json.version; } + if (json.title) { this.title = json.title; } + if (json.author) { this.author = json.author; } + if (json.contact) { this.contact = json.contact; } + if (json.homepage) { this.homepage = json.homepage; } + if (json.description) { this.description = json.description; } + if (json.factorio_version) { this.factorioVersion = json.factorio_version; } + if (json.dependencies) { this.dependencies = json.dependencies; } + + // Additional data + if (json.filename) { this.filename = json.filename; } + if (json.size) { this.size = json.size; } + if (json.hash) { this.hash = json.hash; } + if (json.is_deleted) { this.isDeleted = json.is_deleted; } + } + + toJSON() { + let json = { + name: this.name, + version: this.version, + title: this.title, + author: this.author, + }; + if (this.contact) { json.contact = this.contact; } + if (this.homepage) { json.homepage = this.homepage; } + if (this.description) { json.description = this.description; } + if (this.factorio_version !== "0.12") { json.factorio_version = this.factorioVersion; } + if (this.dependencies.length !== 1 || this.dependencies[0] !== "base") { + json.dependencies = this.dependencies; + } + if (this.filename) { json.filename = this.filename; } + if (this.size) { json.size = this.size; } + if (this.hash) { json.hash = this.hash; } + if (this.isDeleted) { json.is_deleted = this.isDeleted; } + return json; + } + + static async fromModFile(modPath) { + let modInfo; + { + // XXX: JSZip needs the whole archive loaded in memory to work. + // This is clearly untenable and will be replaced later. + let zip = await JSZip.loadAsync(await fs.readFile(modPath)); + let root = zip.folder(findRoot(zip)); + + let infoFile = root.file("info.json"); + if (!infoFile) { + throw new Error("Mod contains no info.json file"); + } + + modInfo = JSON.parse(await infoFile.async("string")); + } + + let valid = this.validateInfo(modInfo); + if (!valid) { + throw new Error("Mod's info.json is not valid"); + } + + let hash = `sha1:${await libHash.hashFile(modPath)}`; + return new this({ + ...modInfo, + + filename: path.basename(modPath), + size: (await fs.stat(modPath)).size, + hash, + is_deleted: false, + }); + } +} + +module.exports = ModInfo; diff --git a/packages/lib/data/index.js b/packages/lib/data/index.js new file mode 100644 index 00000000..17402e53 --- /dev/null +++ b/packages/lib/data/index.js @@ -0,0 +1,12 @@ +/** + * Shared data types used in Clusterio + * @module lib/data + * @author Hornwitser + */ +"use strict"; +const ModInfo = require("./ModInfo"); + +module.exports = { + ModInfo, +}; + diff --git a/packages/lib/index.js b/packages/lib/index.js index 37777950..a642c629 100644 --- a/packages/lib/index.js +++ b/packages/lib/index.js @@ -2,6 +2,7 @@ const libBuildMod = require("./build_mod"); const libCommand = require("./command"); const libConfig = require("./config"); +const libData = require("./data"); const libDatabase = require("./database"); const libErrors = require("./errors"); const libFactorio = require("./factorio"); @@ -28,6 +29,7 @@ module.exports = { libBuildMod, libCommand, libConfig, + libData, libDatabase, libErrors, libFactorio, diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index 5b5361da..c066a0cb 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -1,8 +1,9 @@ // Message definitions for links "use strict"; -const libSchema = require("../schema"); +const libData = require("../data"); const libErrors = require("../errors"); const { logger } = require("../logging"); +const libSchema = require("../schema"); const PlayerStats = require("../PlayerStats"); class MissingLinkHandlerError extends Error { @@ -709,6 +710,67 @@ messages.sendRcon = new Request({ }, }); +messages.getMod = new Request({ + type: "get_mod", + links: ["control-master"], + permission: "core.mod.get", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, + responseProperties: { + "mod": libData.ModInfo.jsonSchema, + }, +}); + +messages.listMods = new Request({ + type: "list_mods", + links: ["control-master"], + permission: "core.mod.list", + responseProperties: { + "list": { + type: "array", + items: libData.ModInfo.jsonSchema, + }, + }, +}); + +messages.setModSubscriptions = new Request({ + type: "set_mod_subscriptions", + links: ["control-master"], + permission: "core.mod.subscribe", + requestProperties: { + "all": { type: "boolean" }, + "mod_names": { + type: "array", + items: { type: "string" }, + }, + }, +}); + +messages.downloadMod = new Request({ + type: "download_mod", + links: ["control-master"], + permission: "core.mod.download", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, + responseProperties: { + "stream_id": { type: "string" }, + }, +}); + +messages.deleteMod = new Request({ + type: "delete_mod", + links: ["control-master"], + permission: "core.mod.delete", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, +}); + messages.listPermissions = new Request({ type: "list_permissions", links: ["control-master"], @@ -1263,6 +1325,14 @@ messages.saveListUpdate = new Event({ }, }); +messages.modUpdate = new Event({ + type: "mod_update", + links: ["master-control"], + eventProperties: { + "mod": libData.ModInfo.jsonSchema, + }, +}); + messages.userUpdate = new Event({ type: "user_update", links: ["master-control"], diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index 9a1cc25f..22fb96e4 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -479,7 +479,6 @@ class BaseMasterPlugin { */ onControlConnectionEvent(connection, event) { } - /** * Called when a slave is preparing to disconnect from the master * @@ -494,6 +493,18 @@ class BaseMasterPlugin { */ async onPrepareSlaveDisconnect(connection) { } + /** + * Called when a mod stored on the master server is updated + * + * Invoked when a mod has been added, updated or deleted from the pool + * of shared mods stored on the cluster. + * + * If a mod has been deleted its `.isDeleted` property will be true. + * + * @param {module:lib/data/ModInfo} mod - Mod that updated. + */ + async onModUpdated(mod) { } + /** * Called when a player joins or leaves an instance * diff --git a/packages/lib/users.js b/packages/lib/users.js index a212c519..300393d3 100644 --- a/packages/lib/users.js +++ b/packages/lib/users.js @@ -511,6 +511,41 @@ definePermission({ description: "Send arbitrary RCON commands to instances.", }); +definePermission({ + name: "core.mod.get", + title: "Get mods", + description: "Get the details of a mod stored in the cluster.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.list", + title: "List mods", + description: "Get the full list of mods stored on the master", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.subscribe", + title: "Subscribe to mod updates", + description: "Subscribe to be notified on updates on the details of mods.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.upload", + title: "Upload mod", + description: "Upload mods to the master.", +}); +definePermission({ + name: "core.mod.download", + title: "Download mod", + description: "Download mods stored on the master.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.delete", + title: "Delete mod", + description: "Delete mods stored on the master.", +}); + definePermission({ name: "core.permission.list", title: "List permissions", diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index e60f1fd5..a090712b 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -1,6 +1,8 @@ "use strict"; -const jwt = require("jsonwebtoken"); const events = require("events"); +const fs = require("fs-extra"); +const jwt = require("jsonwebtoken"); +const path = require("path"); const libConfig = require("@clusterio/lib/config"); const libErrors = require("@clusterio/lib/errors"); @@ -53,6 +55,11 @@ class ControlConnection extends BaseConnection { instance_ids: [], }; + this.modSubscriptions = { + all: false, + mod_names: [], + }; + this.userSubscriptions = { all: false, names: [], @@ -458,11 +465,62 @@ class ControlConnection extends BaseConnection { return { save: result.save }; } + async getModRequestHandler(message) { + let { name, version } = message.data; + let key = `${name}_${version}`; + let mod = this._master.mods.get(key); + if (!mod) { + throw new libErrors.RequestError(`Mod ${key} does not exist`); + } + return { + mod: mod.toJSON(), + }; + } + + async listModsRequestHandler(message) { + return { list: [...this._master.mods.values()].map(mod => mod.toJSON()) }; + } + + async setModSubscriptionsRequestHandler(message) { + this.modSubscriptions = message.data; + } + + async downloadModRequestHandler(message) { + let { name, version } = message.data; + let key = `${name}_${version}`; + let mod = this._master.mods.get(key); + if (!mod) { + throw new libErrors.RequestError(`Mod ${key} does not exist`); + } + let modPath = path.join(this._master.config.get("master.mods_directory"), mod.filename); + + let stream = await routes.createProxyStream(this._master.app); + stream.filename = mod.filename; + stream.source = fs.createReadStream(modPath); + stream.mime = "application/zip"; + stream.size = mod.size; + + return { stream_id: stream.id }; + } + + async deleteModRequestHandler(message) { + await this._master.deleteMod(message.data.name, message.data.version); + } + async setLogSubscriptionsRequestHandler(message) { this.logSubscriptions = message.data; this.updateLogSubscriptions(); } + modUpdated(mod) { + if ( + this.modSubscriptions.all + || this.modSubscriptions.mod_names.includes(mod.name) + ) { + libLink.messages.modUpdate.send(this, { mod: mod.toJSON() }); + } + } + updateLogSubscriptions() { let { all, master, slave_ids, instance_ids } = this.logSubscriptions; if (all || master || slave_ids.length || instance_ids.length) { diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 8268021c..994916c1 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -9,6 +9,7 @@ const path = require("path"); const libConfig = require("@clusterio/lib/config"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); const libFileOps = require("@clusterio/lib/file_ops"); const libLink = require("@clusterio/lib/link"); @@ -65,6 +66,12 @@ class Master { */ this.instances = null; + /** + * Mapping of mod names to mod infos + * @type {Map} + */ + this.mods = null; + /** * User and roles manager for the cluster * @type {module:master/src/UserManager} @@ -165,6 +172,10 @@ class Master { await this.userManager.load(path.join(databaseDirectory, "users.json")); this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json")); + let modsDirectory = this.config.get("master.mods_directory"); + await fs.ensureDir(modsDirectory); + this.mods = await Master.loadModInfos(modsDirectory); + this.config.on("fieldChanged", (group, field, prev) => { libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev); }); @@ -360,6 +371,40 @@ class Master { await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4)); } + static async loadModInfos(modsDirectory) { + let mods = new Map(); + for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) { + if (entry.isDirectory()) { + logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`); + continue; + } + + if (entry.name.toLowerCase().endsWith("tmp.zip")) { + continue; + } + + logger.info(`Loading info for Mod ${entry.name}`); + let modInfo; + try { + modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, entry.name)); + } catch (err) { + logger.error(`Error loading mod ${entry.name}: ${err.message}`); + continue; + } + + let key = `${modInfo.name}_${modInfo.version}`; + if (mods.has(key)) { + logger.error( + `Ignoring Mod ${entry.name} as it contains the same name and ` + + `version as ${mods.get(key).filename}` + ); + continue; + } + mods.set(key, modInfo); + } + return mods; + } + static async loadJsonObject(filePath, throwOnMissing = false) { let manifest = {}; try { @@ -380,6 +425,20 @@ class Master { this.exportManifest = newManifest; } + async deleteMod(name, version) { + let key = `${name}_${version}`; + let mod = this.mods.get(key); + if (!mod) { + throw new Error(`Mod ${key} does not exist`); + } + + let modsDirectory = this.config.get("master.mods_directory"); + await fs.unlink(path.join(modsDirectory, mod.filename)); + this.mods.delete(key); + mod.isDeleted = true; + this.modUpdated(mod); + } + /** * Query master log * @@ -503,6 +562,18 @@ class Master { } } + modUpdated(mod) { + for (let controlConnection of this.wsServer.controlConnections) { + if (controlConnection.connector.closing) { + continue; + } + + controlConnection.modUpdated(mod); + } + + libPlugin.invokeHook(this.plugins, "onModUpdated", mod); + } + userUpdated(user) { for (let controlConnection of this.wsServer.controlConnections) { if (controlConnection.connector.closing) { diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 4052554a..a34a2259 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -2,11 +2,14 @@ const Busboy = require("busboy"); const crypto = require("crypto"); const events = require("events"); +const fs = require("fs-extra"); const JSZip = require("jszip"); const jwt = require("jsonwebtoken"); const path = require("path"); +const nodeStream = require("stream"); const util = require("util"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); const libFileOps = require("@clusterio/lib/file_ops"); const libHash = require("@clusterio/lib/hash"); @@ -18,6 +21,8 @@ const { logger } = require("@clusterio/lib/logging"); const { endpointHitCounter } = require("./metrics"); +const finished = util.promisify(nodeStream.finished); + // Merges samples from sourceResult to destinationResult function mergeSamples(destinationResult, sourceResult) { @@ -306,6 +311,8 @@ const zipMimes = [ "application/x-zip-compressed", ]; +const contentTypeRegExp = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/; + async function uploadSave(req, res) { try { res.locals.user.checkPermission("core.instance.save.upload"); @@ -313,9 +320,10 @@ async function uploadSave(req, res) { res.status(403).json({ request_errors: [err.message] }); return; } + endpointHitCounter.labels(req.route.path).inc(); let contentType = req.get("Content-Type"); - let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType); + let match = contentTypeRegExp.exec(contentType); if (!match) { res.status(415).json({ request_errors: ["invalid Content-Type"] }); return; @@ -436,6 +444,155 @@ async function uploadSave(req, res) { res.json({ saves }); } +function checkModName(name) { + try { + libFileOps.checkFilename(name); + } catch (err) { + throw new libErrors.RequestError(`Mod name ${err.message}`); + } +} + +async function uploadMod(req, res) { + try { + res.locals.user.checkPermission("core.mod.upload"); + } catch (err) { + res.status(403).json({ request_errors: [err.message] }); + return; + } + endpointHitCounter.labels(req.route.path).inc(); + + let contentType = req.get("Content-Type"); + let match = contentTypeRegExp.exec(contentType); + if (!match) { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + let mimeType = match[1].toLowerCase(); + + let tasks = []; + let errors = []; + let requestErrors = []; + let mods = []; + + async function handleFile(stream, filename) { + try { + checkModName(filename); + + } catch (err) { + logger.error(`Error uploading mod: ${err.message}`); + errors.push(err.message); + stream.resume(); + return; + } + + const modsDirectory = req.app.locals.master.config.get("master.mods_directory"); + let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + try { + + let writeStream; + while (true) { + try { + writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" }); + await events.once(writeStream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(modsDirectory, tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + stream.pipe(writeStream); + await finished(writeStream); + + const modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); + modInfo.filename = filename; + await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, filename)); + req.app.locals.master.mods.set(`${modInfo.name}_${modInfo.version}`, modInfo); + req.app.locals.master.modUpdated(modInfo); + mods.push(modInfo.toJSON()); + + } catch (err) { + logger.error(`Error uploading mod: ${err.message}`); + errors.push(err.message); + stream.resume(); + + // Attempt to clean up. + writeStream.destroy(); + try { + await fs.unlink(path.join(modsDirectory, tempFilename)); + } catch (unlinkErr) { + if (unlinkErr.code !== "ENOENT") { + logger.error(`Error removing ${tempFilename}: ${err.message}`); + } + } + } + } + + if (mimeType === "multipart/form-data") { + await new Promise(resolve => { + let busboy = new Busboy({ headers: req.headers }); + busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => { + if (!zipMimes.includes(fileMime)) { + requestErrors.push("invalid file Content-Type"); + } + + if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + + if (errors.length || requestErrors.length) { + stream.resume(); + return; + } + + tasks.push(handleFile(stream, filename)); + }); + busboy.on("finish", resolve); + busboy.on("error", (err) => { + logger.error(`Error parsing multipart request in upload-mod:\n${err.stack}`); + errors.push(err.message); + }); + req.pipe(busboy); + }); + + } else if (zipMimes.includes(mimeType)) { + let filename = req.query.filename; + if (typeof filename !== "string") { + requestErrors.push("Missing or invalid filename parameter"); + } else if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + + if (errors.length || requestErrors.length) { + req.resume(); + } else { + tasks.push(handleFile(req, filename)); + } + + } else { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + + await Promise.all(tasks); + + if (errors.length) { + res.status(500); + res.json({ errors, request_errors: requestErrors }); + return; + } + if (requestErrors.length) { + res.status(400); + res.json({ request_errors: requestErrors }); + return; + } + + res.json({ mods }); +} + + function addRouteHandlers(app) { app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next)); app.get("/api/plugins", getPlugins); @@ -450,6 +607,10 @@ function addRouteHandlers(app) { validateUserToken, (req, res, next) => uploadSave(req, res).catch(next) ); + app.post("/api/upload-mod", + validateUserToken, + (req, res, next) => uploadMod(req, res).catch(next) + ); } // Routes used in the web interface and served by the master server @@ -460,6 +621,7 @@ const webRoutes = [ "/slaves/:id/view", "/instances", "/instances/:id/view", + "/mods", "/users", "/users/:id/view", "/roles", diff --git a/packages/web_ui/src/components/ModsPage.jsx b/packages/web_ui/src/components/ModsPage.jsx new file mode 100644 index 00000000..41625b36 --- /dev/null +++ b/packages/web_ui/src/components/ModsPage.jsx @@ -0,0 +1,165 @@ +import React, { Fragment, useContext } from "react"; +import { Button, Descriptions, Grid, PageHeader, Popconfirm, Space, Table, Typography, Upload } from "antd"; +import UploadOutlined from "@ant-design/icons/UploadOutlined"; + +import { libHelpers, libLink } from "@clusterio/lib"; + +import { useAccount } from "../model/account"; +import { useModList } from "../model/mods"; +import { notifyErrorHandler } from "../util/notify"; +import ControlContext from "./ControlContext"; +import PageLayout from "./PageLayout"; +import PluginExtra from "./PluginExtra"; + +const { useBreakpoint } = Grid; +const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; + + +export default function ModsPage() { + let account = useAccount(); + let control = useContext(ControlContext); + let [modList] = useModList(); + let screens = useBreakpoint(); + + function actions(mod) { + return + {account.hasPermission("core.mod.download") + && { + libLink.messages.downloadMod.send( + control, { name: mod.name, version: mod.version } + ).then(response => { + let url = new URL(webRoot, document.location); + url.pathname += `api/stream/${response.stream_id}`; + document.location = url; + }).catch( + notifyErrorHandler("Error downloading save") + ); + }} + >download + } + {account.hasPermission("core.mod.delete") + && { + libLink.messages.deleteMod.send( + control, { name: mod.name, version: mod.version } + ).catch(notifyErrorHandler("Error deleting mod")); + }} + okText="Delete" + okButtonProps={{ danger: true }} + > + delete + + } + ; + } + + let uploadButton; + if (account.hasPermission("core.mod.upload")) { + uploadButton = + + ; + } + + return + + ( + strcmp(a.name, b.name) || a.integerVersion - b.integerVersion + ), + }, + { + title: "Version", + key: "version", + align: "right", + render: mod => <> + {`${mod.version} `} + {`/ ${mod.factorioVersion}`} + , + }, + { + title: "Filename", + dataIndex: "filename", + responsive: ["xl"], + sorter: (a, b) => strcmp(a.filename, b.filename), + }, + { + title: "Size", + key: "size", + responsive: ["lg"], + render: mod => libHelpers.formatBytes(mod.size), + align: "right", + sorter: (a, b) => a.size - b.size, + }, + { + title: "Action", + key: "action", + responsive: ["lg"], + render: mod => actions(mod), + }, + ]} + expandable={{ + expandedRowRender: mod => <> + + {!screens.lg + && {actions(mod)} + } + {mod.title} + {mod.version} + {mod.description + && {mod.description} + } + {mod.author} + + + {`https://mods.factorio.com/mod/${mod.name}`} + + + {mod.contact + && {mod.contact} + } + {mod.homepage + && {mod.homepage} + } + {mod.name} + {mod.factorioVersion} + {mod.filename} + {libHelpers.formatBytes(mod.size)} + {mod.hash} + {mod.dependencies.length + && + {mod.dependencies + .filter(e => !e.startsWith("(?)")) + .map((e, i) => {e}
) + } + + } + + , + expandedRowClassName: () => "no-expanded-padding", + }} + dataSource={modList} + pagination={false} + rowKey={mod => `${mod.name}_${mod.version}`} + /> + + ; +} diff --git a/packages/web_ui/src/index.css b/packages/web_ui/src/index.css index 37f3889b..9fd4b010 100644 --- a/packages/web_ui/src/index.css +++ b/packages/web_ui/src/index.css @@ -314,3 +314,12 @@ body { top: 8px; right: 8px; } + + +.ant-descriptions-bordered.borderless > .ant-descriptions-view { + border: none; +} + +tr.no-expanded-padding > td { + padding: 0; +} diff --git a/packages/web_ui/src/model/mods.jsx b/packages/web_ui/src/model/mods.jsx new file mode 100644 index 00000000..2ef9971c --- /dev/null +++ b/packages/web_ui/src/model/mods.jsx @@ -0,0 +1,50 @@ +import { useContext, useEffect, useState } from "react"; + +import { libData, libLink, libLogging } from "@clusterio/lib"; + +import ControlContext from "../components/ControlContext"; + +const { logger } = libLogging; + + +export function useModList() { + let control = useContext(ControlContext); + let [modList, setModList] = useState([]); + + function updateModList() { + libLink.messages.listMods.send(control).then(result => { + setModList(result.list.map(mod => new libData.ModInfo(mod))); + }).catch(err => { + logger.error(`Failed to list mods:\n${err}`); + }); + } + + useEffect(() => { + updateModList(); + + function updateHandler(newMod) { + setModList(oldList => { + let newList = oldList.concat(); + let index = newList.findIndex(mod => mod.name === newMod.name && mod.version === newMod.version); + if (!newMod.isDeleted) { + if (index !== -1) { + newList[index] = newMod; + } else { + newList.push(newMod); + } + } else if (index !== -1) { + newList.splice(index, 1); + } + return newList; + }); + } + + control.onModUpdate(null, updateHandler); + return () => { + control.offModUpdate(null, updateHandler); + }; + }, []); + + return [modList]; +} + diff --git a/packages/web_ui/src/pages.jsx b/packages/web_ui/src/pages.jsx index cec0de30..7f190564 100644 --- a/packages/web_ui/src/pages.jsx +++ b/packages/web_ui/src/pages.jsx @@ -5,6 +5,7 @@ import SlavesPage from "./components/SlavesPage"; import SlaveViewPage from "./components/SlaveViewPage"; import InstancesPage from "./components/InstancesPage"; import InstanceViewPage from "./components/InstanceViewPage"; +import ModsPage from "./components/ModsPage"; import UsersPage from "./components/UsersPage"; import UserViewPage from "./components/UserViewPage"; import RolesPage from "./components/RolesPage"; @@ -41,6 +42,12 @@ export const pages = [ sidebarPath: "/instances", content: , }, + { + path: "/mods", + sidebarName: "Mods", + permission: "core.mod.list", + content: , + }, { path: "/users", sidebarName: "Users", diff --git a/packages/web_ui/src/util/websocket.jsx b/packages/web_ui/src/util/websocket.jsx index 12d8eacd..7ac65b0f 100644 --- a/packages/web_ui/src/util/websocket.jsx +++ b/packages/web_ui/src/util/websocket.jsx @@ -1,4 +1,4 @@ -import { libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib"; +import { libData, libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib"; const { logger } = libLogging; import packageJson from "../../package.json"; @@ -86,6 +86,7 @@ export class Control extends libLink.Link { this.slaveUpdateHandlers = new Map(); this.instanceUpdateHandlers = new Map(); this.saveListUpdateHandlers = new Map(); + this.modUpdateHandlers = new Map(); this.userUpdateHandlers = new Map(); this.logHandlers = new Map(); @@ -102,6 +103,9 @@ export class Control extends libLink.Link { this.updateSaveListSubscriptions().catch(err => logger.error( `Unexpected error updating save list subscriptions:\n${err.stack}` )); + this.updateModSubscriptions().catch(err => logger.error( + `Unexpected error updating mod subscriptions:\n${err.stack}` + )); this.updateUserSubscriptions().catch(err => logger.error( `Unexpected error updating user subscriptions:\n${err.stack}` )); @@ -319,6 +323,64 @@ export class Control extends libLink.Link { }); } + async modUpdateEventHandler(message) { + let mod = new libData.ModInfo(message.data.mod); + let handlers = [].concat( + this.modUpdateHandlers.get(null) || [], + this.modUpdateHandlers.get(mod.name) || [] + ); + for (let handler of handlers) { + handler(mod); + } + } + + async onModUpdate(name, handler) { + if (name !== null && typeof name !== "string") { + throw new Error("Invalid mod name"); + } + + let handlers = this.modUpdateHandlers.get(name); + if (!handlers) { + handlers = []; + this.modUpdateHandlers.set(name, handlers); + } + + handlers.push(handler); + + if (handlers.length === 1) { + await this.updateModSubscriptions(); + } + } + + async offModUpdate(name, handler) { + let handlers = this.modUpdateHandlers.get(name); + if (!handlers || !handlers.length) { + throw new Error(`No handlers for mod pack ${name} exist`); + } + + let index = handlers.lastIndexOf(handler); + if (index === -1) { + throw new Error(`Given handler is not registered for mod pack ${name}`); + } + + handlers.splice(index, 1); + if (!handlers.length) { + this.modUpdateHandlers.delete(name); + await this.updateModSubscriptions(); + } + } + + async updateModSubscriptions() { + if (!this.connector.connected) { + return; + } + + await libLink.messages.setModSubscriptions.send(this, { + all: this.modUpdateHandlers.has(null), + mod_names: [...this.modUpdateHandlers.keys()].filter(k => k !== null), + }); + } + async userUpdateEventHandler(message) { let handlers = [].concat( this.userUpdateHandlers.get(null) || [], diff --git a/test/file/empty_mod/info.json b/test/file/empty_mod/info.json new file mode 100644 index 00000000..0537aa9a --- /dev/null +++ b/test/file/empty_mod/info.json @@ -0,0 +1,9 @@ +{ + "name": "empty_mod", + "version": "1.0.0", + "title": "An Empty Mod", + "author": "Me", + "description": "An empty mod for testing", + "factorio_version": "1.1", + "dependencies": [] +} diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index cb91a631..3c9a58f6 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -5,6 +5,8 @@ const path = require("path"); const events = require("events"); const phin = require("phin"); +const libBuildMod = require("@clusterio/lib/build_mod"); +const libHash = require("@clusterio/lib/hash"); const libLink = require("@clusterio/lib/link"); const libUsers = require("@clusterio/lib/users"); const { wait } = require("@clusterio/lib/helpers"); @@ -818,6 +820,91 @@ describe("Integration of Clusterio", function() { }); }); + describe("mod upload", function() { + it("uploads a mod", async function() { + await libBuildMod.build({ + build: true, + pack: true, + sourceDir: path.join("test", "file", "empty_mod"), + outputDir: path.join("temp", "test"), + }); + await execCtl("mod upload empty_mod_1.0.0.zip"); + assert( + await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")), + "mod not present in mods directory" + ); + }); + }); + + describe("mod show", function() { + it("gives details of a mod", async function() { + let result = await execCtl("mod show empty_mod 1.0.0"); + let hash = await libHash.hashFile(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")); + let stat = await fs.stat(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")); + assert.equal( + result.stdout, + "name: empty_mod\n" + + "version: 1.0.0\n" + + "title: An Empty Mod\n" + + "author: Me\n" + + "description: An empty mod for testing\n" + + "factorio_version: 1.1\n" + + "dependencies:\n" + + "filename: empty_mod_1.0.0.zip\n" + + `size: ${stat.size}\n` + + `hash: sha1:${hash}\n`, + ); + }); + }); + + describe("mod list", function() { + it("shows the list of mods", async function() { + let result = await execCtl("mod list"); + assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the list"); + }); + }); + + describe("mod download", function() { + it("downloads a mod", async function() { + await fs.unlink(path.join("temp", "test", "empty_mod_1.0.0.zip")); + await execCtl("mod download empty_mod 1.0.0"); + assert( + await fs.pathExists(path.join("temp", "test", "empty_mod_1.0.0.zip")), + "mod not downloaded to cwd" + ); + }); + }); + + describe("mod delete", function() { + it("deletes a mod", async function() { + await execCtl("mod delete empty_mod 1.0.0"); + assert( + !await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")), + "mod still present in mods dir" + ); + }); + }); + + describe("modUpdateEventHandler()", function() { + it("should have triggered for the previous mod updates", function() { + let eventsToCheck = new Set(["updated", "deleted"]); + let eventsNotSeen = new Set(eventsToCheck); + + for (let modUpdate of getControl().modUpdates) { + if (modUpdate.name !== "empty_mod" || modUpdate.version !== "1.0.0") { + continue; + } + if (modUpdate.is_deleted) { + eventsNotSeen.delete("deleted"); + } else { + eventsNotSeen.delete("updated"); + } + } + + assert(eventsNotSeen.size === 0, `Did not see the events ${[...eventsNotSeen]}`); + }); + }); + describe("permission list", function() { it("runs", async function() { await execCtl("permission list"); diff --git a/test/integration/index.js b/test/integration/index.js index 1084570e..0a4485af 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -28,6 +28,7 @@ class TestControl extends libLink.Link { this.slaveUpdates = []; this.instanceUpdates = []; this.saveListUpdates = []; + this.modUpdates = []; this.userUpdates = []; this.connector.on("connect", () => { @@ -40,6 +41,9 @@ class TestControl extends libLink.Link { libLink.messages.setSaveListSubscriptions.send( this, { all: true, instance_ids: [] } ).catch(err => logger.error(`Error setting save list subscriptions:\n${err.stack}`)); + libLink.messages.setModSubscriptions.send( + this, { all: true, mod_names: [] } + ).catch(err => logger.error(`Error setting mod subscriptions:\n${err.stack}`)); libLink.messages.setUserSubscriptions.send( this, { all: true, names: [] } ).catch(err => logger.error(`Error setting user subscriptions:\n${err.stack}`)); @@ -67,6 +71,10 @@ class TestControl extends libLink.Link { this.saveListUpdates.push(message.data); } + async modUpdateEventHandler(message) { + this.modUpdates.push(message.data.mod); + } + async userUpdateEventHandler(message) { this.userUpdates.push(message.data); } diff --git a/test/lib/data/ModInfo.js b/test/lib/data/ModInfo.js new file mode 100644 index 00000000..f3de1794 --- /dev/null +++ b/test/lib/data/ModInfo.js @@ -0,0 +1,101 @@ +"use strict"; +const assert = require("assert").strict; +const fs = require("fs-extra"); +const path = require("path"); + +const libSchema = require("@clusterio/lib/schema"); +const libHash = require("@clusterio/lib/hash"); +const libBuildMod = require("@clusterio/lib/build_mod"); +const ModInfo = require("@clusterio/lib/data/ModInfo"); + + +describe("lib/data/ModInfo", function() { + describe("class ModInfo", function() { + it("should round trip serialize", function() { + const validate = libSchema.compile(ModInfo.jsonSchema); + function check(pack) { + const json = JSON.parse(JSON.stringify(pack)); + if (!validate(json)) { + throw validate.errors; + } + assert.deepEqual(new ModInfo(json), pack); + } + + check(new ModInfo()); + check(new ModInfo({ name: "MyMod" })); + check(new ModInfo({ version: "1.0.0" })); + check(new ModInfo({ title: "My Extravagent Mod" })); + check(new ModInfo({ author: "Me" })); + check(new ModInfo({ contact: "Use my email" })); + check(new ModInfo({ homepage: "It's over there" })); + check(new ModInfo({ description: "My Description" })); + check(new ModInfo({ factorio_version: "2.0" })); + check(new ModInfo({ dependencies: [] })); + check(new ModInfo({ dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"] })); + check(new ModInfo({ filename: "MyMod_1.0.0.zip" })); + check(new ModInfo({ size: 1024 })); + check(new ModInfo({ hash: "sha5:verified-as-MyMod" })); + check(new ModInfo({ is_deleted: true })); + + // All at once + check(new ModInfo({ + name: "MyMod", + version: "1.0.0", + title: "My Extravagent Mod", + author: "Me", + contact: "Use my email", + homepage: "It's over there", + description: "My Description", + factorio_version: "2.0", + dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"], + filename: "MyMod_1.0.0.zip", + size: 1024, + hash: "sha5:verified-as-MyMod", + is_deleted: true, + })); + }); + + it("should sort integer mod versions lexicographically", function() { + let unsortedVersions = ["1.0.0", "1.1.0", "0.1.0", "3.0.0", "1.2.0", "0.3.1", "0.3.3", "2.1.1", "0.0.1"]; + let sortedVersions = ["0.0.1", "0.1.0", "0.3.1", "0.3.3", "1.0.0", "1.1.0", "1.2.0", "2.1.1", "3.0.0"]; + let mods = unsortedVersions.map(v => new ModInfo({ version: v })); + mods.sort((a, b) => a.integerVersion - b.integerVersion); + assert.deepEqual(mods.map(mod => mod.version), sortedVersions); + }); + it("should sort integer factorio versions lexicographically", function() { + let unsortedVersions = ["1.0", "1.1", "0.1", "3.0", "1.2", "0.3", "2.1"]; + let sortedVersions = ["0.1", "0.3", "1.0", "1.1", "1.2", "2.1", "3.0"]; + let factorioMods = unsortedVersions.map(v => new ModInfo({ factorio_version: v })); + factorioMods.sort((a, b) => a.integerFactorioVersion - b.integerFactorioVersion); + assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions); + }); + + it("should load from a mod zip file", async function() { + await libBuildMod.build({ + build: true, + pack: true, + sourceDir: path.join("test", "file", "empty_mod"), + outputDir: path.join("temp", "test"), + }); + const filePath = path.join("temp", "test", "empty_mod_1.0.0.zip"); + const hash = await libHash.hashFile(filePath); + const stat = await fs.stat(filePath); + const mod = await ModInfo.fromModFile(filePath); + assert.deepEqual( + mod, + new ModInfo({ + name: "empty_mod", + version: "1.0.0", + title: "An Empty Mod", + author: "Me", + description: "An empty mod for testing", + factorio_version: "1.1", + dependencies: [], + filename: "empty_mod_1.0.0.zip", + size: stat.size, + hash: `sha1:${hash}`, + }) + ); + }); + }); +}); From dfcbc559b85d6563242732ba8dfe0ac731b56dba Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 13 Sep 2022 20:33:52 +0200 Subject: [PATCH 07/40] Add parseSearchString to helpers Implement parsing of a search string into terms which can be used to match results with them. --- packages/lib/helpers.js | 171 ++++++++++++++++++++++++++++++++++++++++ test/lib/helpers.js | 109 +++++++++++++++++++++++++ 2 files changed, 280 insertions(+) diff --git a/packages/lib/helpers.js b/packages/lib/helpers.js index c3fe23fb..d2157d37 100644 --- a/packages/lib/helpers.js +++ b/packages/lib/helpers.js @@ -12,6 +12,7 @@ * * @param {*} value - value to return the type of. * @returns {string} basic type of the value passed. + * @static */ function basicType(value) { if (value === null) { return "null"; } @@ -24,6 +25,7 @@ function basicType(value) { * Asynchronously wait for the given duration * * @param {number} duration - Time to wait for in milliseconds. + * @static */ async function wait(duration) { await new Promise(resolve => { setTimeout(resolve, duration); }); @@ -36,6 +38,7 @@ async function wait(duration) { * @param {Promise} promise - Promise to wait for. * @param {number} time - Maximum time im milliseconds to wait for. * @param {*=} timeoutResult - Value to return if the operation timed out. + * @static */ async function timeout(promise, time, timeoutResult) { let timer; @@ -61,6 +64,7 @@ async function timeout(promise, time, timeoutResult) { * * @param {Readable} stream - byte stream to read to the end. * @returns {Buffer} content of the stream. + * @static */ async function readStream(stream) { let chunks = []; @@ -85,6 +89,7 @@ async function readStream(stream) { * * @param {string} text - Text to escape RegExp meta chars in. * @returns {string} escaped text. + * @static */ function escapeRegExp(text) { return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); @@ -96,6 +101,7 @@ function escapeRegExp(text) { * Shortens a large number of bytes using the kB/MB/GB/TB prefixes. * @param {number} bytes - Count of bytes to format. * @returns {string} formatted text. + * @static */ function formatBytes(bytes) { if (bytes === 0) { @@ -108,6 +114,169 @@ function formatBytes(bytes) { return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; } +function parseSearchIdentifier(pos, input) { + // character = ? all characters excluding space : " ? + // identifier = 1*character + let startPos = pos; + while (pos < input.length && ![" ", '"', ":"].includes(input.charAt(pos))) { + pos += 1; + } + return [pos, input.slice(startPos, pos)]; +} + +function parseSearchWord(pos, input) { + // non quote character = ? all characters excluding " ? + // word op = "-" + // word = [ word op ], ( identifier | '"', *non quote character, '"' ) + let word = { + type: "word", + }; + if (input.charAt(pos) === "-") { + word.exclude = true; + pos += 1; + } + if (input.charAt(pos) === '"') { + pos += 1; + let startPos = pos; + let endPos = pos; + while (pos < input.length && input.charAt(pos) !== '"') { + pos += 1; + endPos += 1; + } + if (input.charAt(pos) === '"') { + pos += 1; + } + word.value = input.slice(startPos, endPos); + } else { + ([pos, word.value] = parseSearchIdentifier(pos, input)); + } + return [pos, word]; +} + +function parseSearchTerm(pos, input, attributes, issues) { + // attribute = identifier, ':', word + // term = word | attribute + let term; + if (["-", '"'].includes(input.charAt(pos))) { + ([pos, term] = parseSearchWord(pos, input)); + } else { + let identifier; + ([pos, identifier] = parseSearchIdentifier(pos, input)); + if (input.charAt(pos) === ":") { + pos += 1; + identifier = identifier.toLowerCase(); + if (!Object.prototype.hasOwnProperty.call(attributes, identifier)) { + issues.push(`Unregonized attribute "${identifier}", use quotes to escape colons`); + } else if (attributes[identifier] === "word") { + let value; + ([pos, value] = parseSearchWord(pos, input)); + term = { type: "attribute", name: identifier, value }; + } else { + throw new Error(`Bad attribute format ${attributes[identifier]} for ${identifier}`); + } + } else { + term = { type: "word", value: identifier }; + } + } + return [pos, term]; +} + +/** + * Match a parsed word term with the given texts. + * + * Returns true if the passed word term in the search matches at least one + * of the passed text snippets. If the word mode is exclude then returns + * true if none of the passed text snippets match. + * + * @param {lib/helpers.ParsedTerm} word - Word to match. + * @param {...string} texts - Text to match word in. + * @returns {boolean} true if the word matches. + */ +function wordMatches(word, ...texts) { + if (word.type !== "word") { + throw Error("wordMatches: parameter is not a word"); + } + let matches = false; + for (let text of texts) { + if (text.includes(word.value)) { + matches = true; + break; + } + } + if (word.exclude) { + matches = !matches; + } + return matches; +} + +/** + * @typedef {object} module:lib/helpers~ParsedTerm + * @property {string} type - Type of term, either attribute or word. + * @property {string=} name - attribute only: Name of attribute. + * @property {boolean=} exclude - word only: exclude results with this word. + * @property {string | module:lib/helpers.ParsedTerm} value - + * Parsed value of this term. Is a string if type is word, and a + * ParsedTerm if type is attribute. + */ + +/** + * @typedef {object} module:lib/helpers~ParsedSearch + * @property {Array} terms - + * Parsed result of search terms. + * @property {Array} issues - + * Issues detected while parsing the seach string. + */ + +/** + * Parse a search string with optional attributes + * + * Parses the given input as a search expression consisting of space + * delimited words and attributes:word pairs. For aattributes to be + * recognized they need to be passed as name: "word" in the attributes + * parameter. + * + * Words in the search expression can optionally be prefixed by a - to + * search for results that does not contain that word. E.g. -author contains + * all results not matching author. + * + * If parsing fails an string describing the problem will be added to the + * issues array returned and parsing will resume at some arbitrary point. + * The issues should be shown to the end user so that they can correct their + * search. + * + * @param {string} input - Search expression to parse. + * @param {Object} attributes - + * Recognized attributes and their format. Currently only word is + * supported. + * @returns {module:lib/helpers~ParsedSearch} parsed terms of the search. + * @static + */ +function parseSearchString(input, attributes = {}) { + // whitespace = 1*" " + // search = [ term, *( [ whitespace ], term ) ] + input = input.trim(); + let parsed = { + terms: [], + issues: [], + }; + let pos = 0; + while (pos < input.length) { + let term; + ([pos, term] = parseSearchTerm(pos, input, attributes, parsed.issues)); + if (term) { + parsed.terms.push(term); + } + while (pos < input.length && input.charAt(pos) === " ") { + pos += 1; + } + } + // istanbul ignore if (should not be possible) + if (pos !== input.length) { + throw new Error(`parse search ended at ${pos} which is not the end of the input (${input.length})`); + } + return parsed; +} + module.exports = { basicType, @@ -116,4 +285,6 @@ module.exports = { readStream, escapeRegExp, formatBytes, + wordMatches, + parseSearchString, }; diff --git a/test/lib/helpers.js b/test/lib/helpers.js index 5b5159f7..3d4b89e3 100644 --- a/test/lib/helpers.js +++ b/test/lib/helpers.js @@ -43,4 +43,113 @@ describe("lib/helpers", function() { ), "timeout"); }); }); + + function parse(input, attributes) { + const result = libHelpers.parseSearchString(input, attributes); + if (!result.issues.length) { + return result.terms; + } + return result; + } + + describe("parseSearchString()", function() { + function word(value, opts = {}) { + return { type: "word", value, ...opts }; + } + function attr(name, value) { + return { type: "attribute", name, value: typeof value === "string" ? word(value) : value }; + } + it("should split words", function() { + assert.deepEqual(parse(""), [].map(word)); + assert.deepEqual(parse("word"), ["word"].map(word)); + assert.deepEqual(parse("some words here"), ["some", "words", "here"].map(word)); + }); + it("should trim whitespace", function() { + assert.deepEqual(parse(" "), [].map(word)); + assert.deepEqual(parse("word "), ["word"].map(word)); + assert.deepEqual(parse(" some words here"), ["some", "words", "here"].map(word)); + }); + it("should parse negated words", function() { + assert.deepEqual(parse("this -that"), [word("this"), word("that", { exclude: true })]); + assert.deepEqual(parse("not-negated"), [word("not-negated")]); + }); + it("should not split quoted words", function() { + assert.deepEqual(parse('"some quoted words" here'), ["some quoted words", "here"].map(word)); + assert.deepEqual(parse('"a quoted string"'), ["a quoted string"].map(word)); + assert.deepEqual(parse('words and "a quoted string"'), ["words", "and", "a quoted string"].map(word)); + }); + it("should split on quotation mark", function() { + assert.deepEqual(parse('"quoted"unquoted'), ["quoted", "unquoted"].map(word)); + assert.deepEqual(parse('"quoted""double"'), ["quoted", "double"].map(word)); + assert.deepEqual(parse('word"with"quote'), ["word", "with", "quote"].map(word)); + assert.deepEqual( + parse('word-"with"-negation'), + [word("word-"), word("with"), word("negation", { exclude: true })] + ); + }); + it("should allow colons in quotes", function() { + assert.deepEqual(parse('"word:value"'), ["word:value"].map(word)); + }); + it("should parse attribute:value", function() { + assert.deepEqual(parse("author:Me", { author: "word" }), [attr("author", "Me")]); + assert.deepEqual(parse("word author:Me", { author: "word" }), [word("word"), attr("author", "Me")]); + assert.deepEqual( + parse("author:Me and author:You", { author: "word" }), + [attr("author", "Me"), word("and"), attr("author", "You")] + ); + assert.deepEqual( + parse("author:Me"), + { + terms: [word("Me")], + issues: ['Unregonized attribute "author", use quotes to escape colons'], + } + ); + assert.deepEqual( + parse("constructor:value"), + { + terms: [word("value")], + issues: ['Unregonized attribute "constructor", use quotes to escape colons'], + } + ); + }); + it("should throw on bad attribute format", function() { + assert.throws( + () => parse("author:Me", { author: "bad" }), + new Error("Bad attribute format bad for author") + ); + }); + it("should ignore attribute case", function() { + assert.deepEqual(parse("Author:Me", { author: "word" }), [attr("author", "Me")]); + assert.deepEqual(parse("AUTHOR:Me", { author: "word" }), [attr("author", "Me")]); + assert.deepEqual(parse("auThOR:Me", { author: "word" }), [attr("author", "Me")]); + }); + it("should parse partial inputs", function() { + assert.deepEqual(parse("-"), [word("", { exclude: true })]); + assert.deepEqual(parse('"'), [""].map(word)); + assert.deepEqual(parse('"word'), ["word"].map(word)); + assert.deepEqual(parse("author:", { author: "word" }), [attr("author", "")]); + assert.deepEqual(parse("author:-", { author: "word" }), [attr("author", word("", { exclude: true }))]); + }); + }); + describe("wordMatches()", function() { + const match = libHelpers.wordMatches; + it("should match any inputs", function() { + assert(match(parse("word")[0], "a word in a string"), "one text"); + assert(!match(parse("word")[0], "a string without"), "one text no match"); + assert(match(parse("word")[0], "no match", "no things", "a word", "bad text", ""), "five text"); + assert(!match(parse("unword")[0], "no match", "no things", "a word", "bad text", ""), "five text no match"); + }); + it("should invert match if using exclude", function() { + assert(!match(parse("-word")[0], "a word in a string"), "one text"); + assert(match(parse("-word")[0], "a string without"), "one text no match"); + assert(!match(parse("-word")[0], "no match", "no things", "a word", "bad text", ""), "five text"); + assert(match(parse("-unword")[0], "no match", "no things", "a word", "bad text", ""), "five text no match"); + }); + it("should throw on bad input", function() { + assert.throws( + () => match(parse("attr:value", { attr: "word" })[0], "a string"), + new Error("wordMatches: parameter is not a word"), + ); + }); + }); }); From 9fb3d6d8db02bcfaca5a70c8834d671f05cac211 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 15 Sep 2022 23:32:53 +0200 Subject: [PATCH 08/40] Implement mod pack storage Add ModPack data type with corresponding storage on the master, messages to create, update and delete them, as well as commands to invoke them. The ModPack data type contains a list of mods with versions and optional sha1 hashes, and mod settings to be used. --- packages/ctl/ctl.js | 2 + packages/ctl/src/commands.js | 203 +++++++++++++++++++++ packages/lib/command.js | 34 ++++ packages/lib/data/ModPack.js | 215 +++++++++++++++++++++++ packages/lib/data/index.js | 3 +- packages/lib/link/messages.js | 73 ++++++++ packages/lib/plugin.js | 13 ++ packages/lib/users.js | 33 ++++ packages/master/src/ControlConnection.js | 62 +++++++ packages/master/src/Master.js | 39 ++++ packages/master/src/SlaveConnection.js | 5 + packages/web_ui/src/model/mod_pack.jsx | 78 ++++++++ packages/web_ui/src/util/websocket.jsx | 66 ++++++- test/integration/clusterio.js | 65 +++++++ test/integration/index.js | 5 + test/lib/data/ModPack.js | 94 ++++++++++ 16 files changed, 987 insertions(+), 3 deletions(-) create mode 100644 packages/lib/data/ModPack.js create mode 100644 packages/web_ui/src/model/mod_pack.jsx create mode 100644 test/lib/data/ModPack.js diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index d054f89b..b4cfaf90 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -89,6 +89,8 @@ class Control extends libLink.Link { async saveListUpdateEventHandler() { } + async modPackUpdateEventHandler() { } + async modUpdateEventHandler() { } async userUpdateEventHandler() { } diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index 2d09982c..9dc00bcb 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -14,6 +14,7 @@ const libLink = require("@clusterio/lib/link"); const libErrors = require("@clusterio/lib/errors"); const libConfig = require("@clusterio/lib/config"); const libCommand = require("@clusterio/lib/command"); +const libData = require("@clusterio/lib/data"); const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging"); const libLoggingUtils = require("@clusterio/lib/logging_utils"); const libHelpers = require("@clusterio/lib/helpers"); @@ -860,6 +861,207 @@ instanceCommands.add(new libCommand.Command({ }, })); +const modPackCommands = new libCommand.CommandTree({ name: "mod-pack", description: "Mod Pack" }); +modPackCommands.add(new libCommand.Command({ + definition: ["show ", "Show details of mod pack", (yargs) => { + yargs.positional("mod-pack", { describe: "Mod pack to show", type: "string" }); + }], + handler: async function(args, control) { + let response = await libLink.messages.getModPack.send(control, { + id: await libCommand.resolveModPack(control, args.modPack), + }); + + for (let [field, value] of Object.entries(response.mod_pack)) { + if (field === "mods") { + print(`${field}:`); + for (let entry of value) { + print(` ${entry.name} ${entry.version}${entry.hash ? ` (${entry.hash})` : ""}`); + } + } else if (field === "settings") { + print(`${field}:`); + for (let [scope, settings] of Object.entries(value)) { + print(` ${scope}:`); + for (let [setting, settingValue] of Object.entries(settings)) { + print(` ${setting}: ${JSON.stringify(settingValue.value)}`); + } + } + + } else { + print(`${field}: ${value}`); + } + } + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: [["list", "l"], "List mod packs in the cluster"], + handler: async function(args, control) { + let response = await libLink.messages.listModPacks.send(control); + let fields = ["id", "name", "factorio_version"]; + for (let entry of response.list) { + for (let field of Object.keys(entry)) { + if (!fields.includes(field)) { + delete entry[field]; + } + } + } + print(asTable(response.list)); + }, +})); + +function setModPackSettings(modPack, args) { + function doSettings(settings, cast) { + for (let i = 0; i + 2 < settings.length; i += 3) { + const [scope, name, value] = settings.slice(i, i + 3); + if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) { + throw new libErrors.CommandError( + `Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}` + ); + } + modPack.settings[scope].set(name, { value: cast(value) }); + } + } + + doSettings(args.boolSetting || [], value => { + if (!["true", "false"].includes(value)) { + throw new libErrors.CommandError(`boolean value must be one of true or false, not ${value}`); + } + return value === "true"; + }); + doSettings(args.intSetting || [], value => { + // eslint-disable-next-line radix + let number = Number.parseInt(value); + if (isNaN(number)) { + throw new libErrors.CommandError(`int value must be an integer, not ${value}`); + } + return number; + }); + doSettings(args.doubleSetting || [], value => { + let number = Number.parseFloat(value); + if (isNaN(number)) { + throw new libErrors.CommandError(`double value must be a number, not ${value}`); + } + return number; + }); + doSettings(args.stringSetting || [], value => value); +} + +function setModPackMods(modPack, mods) { + for (let mod of mods || []) { + const [name, version, sha1] = mod.split(":"); + if (!version) { + throw new libErrors.CommandError("Added mod must be formatted as name:version or name:version:sha1"); + } + if (!/^\d+\.\d+\.\d+$/.test(version)) { + throw new libErrors.CommandError("version must match the format digit.digit.digit"); + } + if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) { + throw new libErrors.CommandError("sha1 must be a 40 digit lower case hex string"); + } + modPack.mods.set(name, { name, version, hash: sha1 ? `sha1:${sha1}` : undefined }); + } +} + +modPackCommands.add(new libCommand.Command({ + definition: ["create ", "Create mod-pack", (yargs) => { + yargs.positional("name", { describe: "Name of mod pack to create", type: "string" }); + yargs.positional("factorio-version", { describe: "Version of factorio the mod pack is for", type: "string" }); + yargs.options({ + "description": { describe: "Description for mod pack", type: "string" }, + "mods": { describe: "Mods in the form of name:version[:sha1]", array: true, type: "string" }, + "bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" }, + "int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" }, + "double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" }, + "string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" }, + }); + }], + handler: async function(args, control) { + const modPack = new libData.ModPack(); + modPack.name = args.name; + if (args.description) { modPack.description = args.description; } + if (args.factorioVersion) { + if (!/^\d+\.\d+(\.\d+)?$/.test(args.factorioVersion)) { + throw new libErrors.CommandError("factorio-version must match the format digit.digit[.digit]"); + } + modPack.factorioVersion = args.factorioVersion; + } + setModPackMods(modPack, args.mods); + setModPackSettings(modPack, args); + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + print(`Created mod pack ${modPack.name} (${modPack.id})`); + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: ["edit ", "Edit mod-pack", (yargs) => { + yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" }); + yargs.options({ + "name": { describe: "New name for mod pack", type: "string" }, + "description": { describe: "New description for mod pack", type: "string" }, + "factorio-version": { describe: "Set version of factorio the mod pack is for", type: "string" }, + "add-mods": { describe: "Mods in the form of name:version[:sha1] to add", array: true, type: "string" }, + "remove-mods": { describe: "Name of mods to remove", array: true, type: "string" }, + "bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" }, + "int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" }, + "double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" }, + "string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" }, + "remove-setting": { describe: "Remove a setting", array: true, nargs: 2, type: "string" }, + }); + }], + handler: async function(args, control) { + let response = await libLink.messages.getModPack.send(control, { + id: await libCommand.resolveModPack(control, args.modPack), + }); + let modPack = new libData.ModPack(response.mod_pack); + + if (args.name) { modPack.name = args.name; } + if (args.description) { modPack.description = args.description; } + if (args.factorioVersion) { + if (!/^\d+\.\d+(\.\d+)?$/.test(args.factorioVersion)) { + throw new libErrors.CommandError("factorio-version must match the format digit.digit[.digit]"); + } + modPack.factorioVersion = args.factorioVersion; + } + + setModPackMods(modPack, args.addMods); + for (let name of args.removeMods || []) { + if (modPack.mods.has(name)) { + modPack.mods.delete(name); + } else { + logger.warn(`Mod ${name} did not exist on ${modPack.name}`); + } + } + + setModPackSettings(modPack, args); + if (args.removeSetting) { + for (let i = 0; i + 1 < args.removeSetting.length; i += 2) { + const [scope, name] = settings.slice(i, i + 2); + if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) { + throw new libErrors.CommandError( + `Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}` + ); + } + if (modPack.settings[scope].has(name)) { + modPack.settings[scope].delete(name); + } else { + logger.warn(`Mod setting ${scope} ${name} did not exist on ${modPack.name}`); + } + } + } + await libLink.messages.updateModPack.send(control, { mod_pack: modPack.toJSON() }); + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: ["delete ", "Delete mod-pack", (yargs) => { + yargs.positional("mod-pack", { describe: "Mod pack to delete", type: "string" }); + }], + handler: async function(args, control) { + const id = await libCommand.resolveModPack(control, args.modPack); + await libLink.messages.deleteModPack.send(control, { id }); + }, +})); + const modCommands = new libCommand.CommandTree({ name: "mod", description: "Manage uploaded mods" }); modCommands.add(new libCommand.Command({ definition: ["show ", "Show details for a mod stored in the cluster", (yargs) => { @@ -1339,6 +1541,7 @@ async function registerCommands(controlPlugins, yargs) { rootCommands.add(masterCommands); rootCommands.add(slaveCommands); rootCommands.add(instanceCommands); + rootCommands.add(modPackCommands); rootCommands.add(modCommands); rootCommands.add(permissionCommands); rootCommands.add(roleCommands); diff --git a/packages/lib/command.js b/packages/lib/command.js index 133a8805..f0738e8b 100644 --- a/packages/lib/command.js +++ b/packages/lib/command.js @@ -198,6 +198,39 @@ async function resolveInstance(client, instanceName) { return instanceId; } +/** + * Resolve a string to a mod pack ID + * + * Resolevs a string with either a mod pack name or an id into an integer + * with the mod pack ID. + * + * @param {module:lib/link.Link} client - + * link to master server to query mod pack on. + * @param {string} modPackName - string with name or id of mod pack. + * @returns {Promise} mod pack ID. + * @static + */ +async function resolveModPack(client, modPackName) { + let modPackId; + if (/^-?\d+/.test(modPackName)) { + modPackId = parseInt(modPackName, 10); + } else { + let response = await libLink.messages.listModPacks.send(client); + for (let mod_pack of response.list) { + if (mod_pack["name"] === modPackName) { + modPackId = mod_pack["id"]; + break; + } + } + + if (modPackId === undefined) { + throw new libErrors.CommandError(`No mod pack named ${modPackName}`); + } + } + + return modPackId; +} + /** * Retrieve role object from string * @@ -244,5 +277,6 @@ module.exports = { resolveSlave, resolveInstance, + resolveModPack, retrieveRole, }; diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js new file mode 100644 index 00000000..9dd09154 --- /dev/null +++ b/packages/lib/data/ModPack.js @@ -0,0 +1,215 @@ +"use strict"; +const zlib = require("zlib"); + +const libSchema = require("../schema"); + + +/** + * A setting for a mod. + * @typedef {object} module:lib/data/ModPack~ModSetting + * @property {boolean|number|string} value - Value of the given mod setting. + */ +const ModSettingJsonSchema = { + type: "object", + additionalProperties: false, + required: ["value"], + properties: { + "value": { type: ["boolean", "number", "string"] }, + }, +}; + +const ModSettingsJsonSchema = { + type: "object", + additionalProperties: ModSettingJsonSchema, +}; + +/** + * A mod in a mod pack + * @typedef {object} module:lib/data/ModPack~ModRecord + * @property {string} name - name of the mod. + * @property {string} version - version of the mod. + * @property {string} hash - hash of the zip file. + */ +const ModRecordJsonSchema = { + type: "object", + additionalProperties: false, + required: ["name", "version"], + properties: { + "name": { type: "string" }, + "version": { type: "string" }, + "hash": { type: "string" }, + }, +}; + +/** + * Factorio Mod Pack + * + * Tracks mods and settings for a collection of Factorio mods. + * @alias module:lib/data/ModPack + */ +class ModPack { + /** + * Id of this mod pack + * @type {number} + */ + id; + + /** + * Name of this mod pack. + * @type {string} + */ + name = ""; + + /** + * Description for this modpcak. + * @type {string} + */ + description = ""; + + /** + * Version of Factorio this mod pack is for + * @type {string} + */ + factorioVersion = "1.1.0"; + + /** + * Integer representation of the factorioVersion + * @type {number} + */ + get integerFactorioVersion() { + const [major, minor, sub] = this.factorioVersion.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000 + (sub || 0); // Can't use bitwise here because this is 48-bits. + } + + /** + * Mods included in this mod pack + * @type {Map} + */ + mods = new Map(); + + /** + * Mod settings for this mod pack + * @type {Object>=} + */ + settings = { + "startup": new Map(), + "runtime-global": new Map(), + "runtime-per-user": new Map(), + }; + + /** + * True if this mod pack has been deleted from the list of mod packs. + * @type {boolean} + */ + isDeleted = false; + + static jsonSchema = { + type: "object", + additionalProperties: false, + properties: { + "id": { type: "integer" }, + "name": { type: "string" }, + "description": { type: "string" }, + "factorio_version": { type: "string" }, + "mods": { + type: "array", + items: ModRecordJsonSchema, + }, + "settings": { + type: "object", + additionalProperties: false, + required: ["startup", "runtime-global", "runtime-per-user"], + properties: { + "startup": ModSettingsJsonSchema, + "runtime-global": ModSettingsJsonSchema, + "runtime-per-user": ModSettingsJsonSchema, + }, + }, + "is_deleted": { type: "boolean" }, + }, + }; + + static validate = libSchema.compile(this.jsonSchema); + + constructor(json = {}) { + if (json.id) { + this.id = json.id; + } else { + this.id = Math.random() * 2**31 | 0; + } + if (json.name) { this.name = json.name; } + if (json.description) { this.description = json.description; } + if (json.factorio_version) { this.factorioVersion = json.factorio_version; } + if (json.mods) { this.mods = new Map(json.mods.map(m => [m.name, m])); } + if (json.settings) { + this.settings = { + "startup": new Map(Object.entries(json.settings["startup"])), + "runtime-global": new Map(Object.entries(json.settings["runtime-global"])), + "runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])), + }; + } + if (json.is_deleted) { this.isDeleted = json.is_deleted; } + } + + toJSON() { + let json = { + id: this.id, + }; + if (this.name) { json.name = this.name; } + if (this.description) { json.description = this.description; } + json.factorio_version = this.factorioVersion; + if (this.mods.size) { json.mods = [...this.mods.values()]; } + if (this.settings) { + json.settings = { + "startup": Object.fromEntries(this.settings["startup"]), + "runtime-global": Object.fromEntries(this.settings["runtime-global"]), + "runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]), + }; + } + if (this.isDeleted) { json.is_deleted = this.isDeleted; } + return json; + } + + toModPackString(includeHash = false) { + const json = this.toJSON(); + delete json.id; + if (json.mods && !includeHash) { + json.mods = json.mods.map(m => { + m = { ...m }; + delete m.hash; + return m; + }); + } + + // eslint-disable-next-line node/no-sync + let buf = zlib.deflateSync(JSON.stringify(json)); + return buf.toString("base64"); + } + + static fromModPackString(modPackString) { + let buf = Buffer.from(modPackString, "base64"); + try { + // eslint-disable-next-line node/no-sync + buf = zlib.inflateSync(buf); + } catch (err) { + if (err.code.startsWith("Z_")) { + throw new Error("Malformed mod pack string: zlib inflate failed"); + } + } + + let json; + try { + json = JSON.parse(buf); + } catch (err) { + throw new Error(`Malformed mod pack string: ${err.message}`); + } + + if (!this.validate(json)) { + throw new Error("Malformed mod pack string: Schema validation failed"); + } + + return new this(json); + } +} + +module.exports = ModPack; diff --git a/packages/lib/data/index.js b/packages/lib/data/index.js index 17402e53..2b280cc2 100644 --- a/packages/lib/data/index.js +++ b/packages/lib/data/index.js @@ -5,8 +5,9 @@ */ "use strict"; const ModInfo = require("./ModInfo"); +const ModPack = require("./ModPack"); module.exports = { ModInfo, + ModPack, }; - diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index c066a0cb..61b00ce1 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -710,6 +710,71 @@ messages.sendRcon = new Request({ }, }); +messages.getModPack = new Request({ + type: "get_mod_pack", + links: ["control-master", "instance-slave", "slave-master"], + permission: "core.mod_pack.get", + forwardTo: "master", + requestProperties: { + "id": { type: "integer" }, + }, + responseProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.listModPacks = new Request({ + type: "list_mod_packs", + links: ["control-master"], + permission: "core.mod_pack.list", + responseProperties: { + "list": { + type: "array", + items: libData.ModPack.jsonSchema, + }, + }, +}); + +messages.createModPack = new Request({ + type: "create_mod_pack", + links: ["control-master"], + permission: "core.mod_pack.create", + requestProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.updateModPack = new Request({ + type: "update_mod_pack", + links: ["control-master"], + permission: "core.mod_pack.update", + requestProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.deleteModPack = new Request({ + type: "delete_mod_pack", + links: ["control-master"], + permission: "core.mod_pack.delete", + requestProperties: { + "id": { type: "integer" }, + }, +}); + +messages.setModPackSubscriptions = new Request({ + type: "set_mod_pack_subscriptions", + links: ["control-master"], + permission: "core.mod_pack.subscribe", + requestProperties: { + "all": { type: "boolean" }, + "mod_pack_ids": { + type: "array", + items: { type: "integer" }, + }, + }, +}); + messages.getMod = new Request({ type: "get_mod", links: ["control-master"], @@ -1325,6 +1390,14 @@ messages.saveListUpdate = new Event({ }, }); +messages.modPackUpdate = new Event({ + type: "mod_pack_update", + links: ["master-control"], + eventProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + messages.modUpdate = new Event({ type: "mod_update", links: ["master-control"], diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index 22fb96e4..38a7675d 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -493,6 +493,19 @@ class BaseMasterPlugin { */ async onPrepareSlaveDisconnect(connection) { } + /** + * Called when a mod pack is updated + * + * Invoked when a mod pack has been added, updated or deleted in the + * cluster. + * + * If the mod pack has been deleted its `.isDeleted` property will be + * true. + * + * @param {module:lib/data/ModPack} modPack - Mod pack that updated. + */ + async onModPackUpdated(modPack) { } + /** * Called when a mod stored on the master server is updated * diff --git a/packages/lib/users.js b/packages/lib/users.js index 300393d3..f6ef8558 100644 --- a/packages/lib/users.js +++ b/packages/lib/users.js @@ -511,6 +511,39 @@ definePermission({ description: "Send arbitrary RCON commands to instances.", }); +definePermission({ + name: "core.mod_pack.get", + title: "Retrieve mod pack", + description: "Get the details of a mod pack in the cluster.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod_pack.list", + title: "List mod packs", + description: "Get the full list of a mod packs in the cluster.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod_pack.subscribe", + title: "Subscribe to mod pack updates", + description: "Subscribe to be notified on updates on the details of mod packs.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod_pack.create", + title: "Create mod pack", + description: "Create new mod pack in the cluster.", +}); +definePermission({ + name: "core.mod_pack.update", + title: "Update mod pack", + description: "Edit existing mod pack definition in the cluster.", +}); +definePermission({ + name: "core.mod_pack.delete", + title: "Delete mod pack", + description: "Delete mod pack definition in the cluster.", +}); definePermission({ name: "core.mod.get", title: "Get mods", diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index a090712b..12472705 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -5,7 +5,9 @@ const jwt = require("jsonwebtoken"); const path = require("path"); const libConfig = require("@clusterio/lib/config"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); +const libHelpers = require("@clusterio/lib/helpers"); const libLink = require("@clusterio/lib/link"); const { logger } = require("@clusterio/lib/logging"); const libLoggingUtils = require("@clusterio/lib/logging_utils"); @@ -55,6 +57,11 @@ class ControlConnection extends BaseConnection { instance_ids: [], }; + this.modPackSubscriptions = { + all: false, + mod_pack_ids: [], + }; + this.modSubscriptions = { all: false, mod_names: [], @@ -465,6 +472,61 @@ class ControlConnection extends BaseConnection { return { save: result.save }; } + async getModPackRequestHandler(message) { + let { id } = message.data; + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } + + async listModPacksRequestHandler(message) { + return { list: [...this._master.modPacks.values()].map(pack => pack.toJSON()) }; + } + + async setModPackSubscriptionsRequestHandler(message) { + this.modPackSubscriptions = message.data; + } + + async createModPackRequestHandler(message) { + let modPack = new libData.ModPack(message.data.mod_pack); + if (this._master.modPacks.has(modPack.id)) { + throw new libErrors.RequestError(`Mod pack with ID ${modPack.id} already exist`); + } + this._master.modPacks.set(modPack.id, modPack); + this._master.modPackUpdated(modPack); + } + + async updateModPackRequestHandler(message) { + let modPack = new libData.ModPack(message.data.mod_pack); + if (!this._master.modPacks.has(modPack.id)) { + throw new libErrors.RequestError(`Mod pack with ID ${modPack.id} does not exist`); + } + this._master.modPacks.set(modPack.id, modPack); + this._master.modPackUpdated(modPack); + } + + async deleteModPackRequestHandler(message) { + let { id } = message.data; + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); + } + modPack.isDeleted = true; + this._master.modPacks.delete(id); + this._master.modPackUpdated(modPack); + } + + modPackUpdated(modPack) { + if ( + this.modPackSubscriptions.all + || this.modPackSubscriptions.mod_pack_ids.includes(modPack.id) + ) { + libLink.messages.modPackUpdate.send(this, { mod_pack: modPack.toJSON() }); + } + } + async getModRequestHandler(message) { let { name, version } = message.data; let key = `${name}_${version}`; diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 994916c1..abc67bcc 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -66,6 +66,12 @@ class Master { */ this.instances = null; + /** + * Mapping of mod pack id to mod pack + * @type {Map} + */ + this.modPacks = null; + /** * Mapping of mod names to mod infos * @type {Map} @@ -168,6 +174,7 @@ class Master { this.slaves = await Master.loadSlaves(path.join(databaseDirectory, "slaves.json")); this.instances = await Master.loadInstances(path.join(databaseDirectory, "instances.json")); + this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json")); this.userManager = new UserManager(this.config); await this.userManager.load(path.join(databaseDirectory, "users.json")); this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json")); @@ -295,6 +302,10 @@ class Master { await Master.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); } + if (this.modPacks) { + await Master.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); + } + if (this.userManager) { await this.userManager.save(path.join(databaseDirectory, "users.json")); } @@ -371,6 +382,23 @@ class Master { await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4)); } + static async loadModPacks(filePath) { + let json; + try { + json = JSON.parse(await fs.readFile(filePath)); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + return new Map(json.map(e => [e.id, new libData.ModPack(e)])); + } + + static async saveModPacks(filePath, modPacks) { + await libFileOps.safeOutputFile(filePath, JSON.stringify([...modPacks.values()])); + } + static async loadModInfos(modsDirectory) { let mods = new Map(); for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) { @@ -562,6 +590,17 @@ class Master { } } + modPackUpdated(modPack) { + for (let controlConnection of this.wsServer.controlConnections) { + if (controlConnection.connector.closing) { + continue; + } + controlConnection.modPackUpdated(modPack); + } + + libPlugin.invokeHook(this.plugins, "onModPackUpdated", modPack); + } + modUpdated(mod) { for (let controlConnection of this.wsServer.controlConnections) { if (controlConnection.connector.closing) { diff --git a/packages/master/src/SlaveConnection.js b/packages/master/src/SlaveConnection.js index 35564264..4f9203ed 100644 --- a/packages/master/src/SlaveConnection.js +++ b/packages/master/src/SlaveConnection.js @@ -3,6 +3,7 @@ const libConfig = require("@clusterio/lib/config"); const libLink = require("@clusterio/lib/link"); const { logger } = require("@clusterio/lib/logging"); const libPlugin = require("@clusterio/lib/plugin"); +const libErrors = require("@clusterio/lib/errors"); const PlayerStats = require("@clusterio/lib/PlayerStats"); const BaseConnection = require("./BaseConnection"); @@ -167,6 +168,10 @@ class SlaveConnection extends BaseConnection { this._master.saveListUpdate(message.data); } + async getModPackRequestHandler(message) { + throw new libErrors.RequestError("Not implemented"); + } + async logMessageEventHandler(message) { this._master.clusterLogger.log({ ...message.data.info, diff --git a/packages/web_ui/src/model/mod_pack.jsx b/packages/web_ui/src/model/mod_pack.jsx new file mode 100644 index 00000000..4983cf54 --- /dev/null +++ b/packages/web_ui/src/model/mod_pack.jsx @@ -0,0 +1,78 @@ +import { useContext, useEffect, useState } from "react"; + +import { libData, libLink, libLogging } from "@clusterio/lib"; + +import ControlContext from "../components/ControlContext"; + +const { logger } = libLogging; + +export function useModPack(id) { + let control = useContext(ControlContext); + let [modPack, setModPack] = useState({ loading: true }); + + function updateModPack() { + libLink.messages.getModPack.send(control, { id }).then(result => { + setModPack(new libData.ModPack(result.mod_pack)); + }).catch(err => { + logger.error(`Failed to get mod pack: ${err}`); + setModPack({ missing: true }); + }); + } + + useEffect(() => { + if (typeof id !== "number") { + setModPack({ missing: true }); + return undefined; + } + updateModPack(); + + control.onModPackUpdate(id, setModPack); + return () => { + control.offModPackUpdate(id, setModPack); + }; + }, [id]); + + return [modPack]; +} + +export function useModPackList() { + let control = useContext(ControlContext); + let [modPackList, setModPackList] = useState([]); + + function updateModPackList() { + libLink.messages.listModPacks.send(control).then(result => { + setModPackList(result.list.map(pack => new libData.ModPack(pack))); + }).catch(err => { + logger.error(`Failed to list mod packs:\n${err}`); + }); + } + + useEffect(() => { + updateModPackList(); + + function updateHandler(newModPack) { + setModPackList(oldList => { + let newList = oldList.concat(); + let index = newList.findIndex(u => u.id === newModPack.id); + if (!newModPack.isDeleted) { + if (index !== -1) { + newList[index] = newModPack; + } else { + newList.push(newModPack); + } + } else if (index !== -1) { + newList.splice(index, 1); + } + return newList; + }); + } + + control.onModPackUpdate(null, updateHandler); + return () => { + control.offModPackUpdate(null, updateHandler); + }; + }, []); + + return [modPackList]; + +} diff --git a/packages/web_ui/src/util/websocket.jsx b/packages/web_ui/src/util/websocket.jsx index 7ac65b0f..fd9b56a3 100644 --- a/packages/web_ui/src/util/websocket.jsx +++ b/packages/web_ui/src/util/websocket.jsx @@ -86,6 +86,7 @@ export class Control extends libLink.Link { this.slaveUpdateHandlers = new Map(); this.instanceUpdateHandlers = new Map(); this.saveListUpdateHandlers = new Map(); + this.modPackUpdateHandlers = new Map(); this.modUpdateHandlers = new Map(); this.userUpdateHandlers = new Map(); this.logHandlers = new Map(); @@ -103,6 +104,9 @@ export class Control extends libLink.Link { this.updateSaveListSubscriptions().catch(err => logger.error( `Unexpected error updating save list subscriptions:\n${err.stack}` )); + this.updateModPackSubscriptions().catch(err => logger.error( + `Unexpected error updating mod pack subscriptions:\n${err.stack}` + )); this.updateModSubscriptions().catch(err => logger.error( `Unexpected error updating mod subscriptions:\n${err.stack}` )); @@ -323,6 +327,64 @@ export class Control extends libLink.Link { }); } + async modPackUpdateEventHandler(message) { + let modPack = new libData.ModPack(message.data.mod_pack); + let handlers = [].concat( + this.modPackUpdateHandlers.get(null) || [], + this.modPackUpdateHandlers.get(modPack.id) || [], + ); + for (let handler of handlers) { + handler(modPack); + } + } + + async onModPackUpdate(id, handler) { + if (id !== null && typeof id !== "number") { + throw new Error("Invalid mod pack id"); + } + + let handlers = this.modPackUpdateHandlers.get(id); + if (!handlers) { + handlers = []; + this.modPackUpdateHandlers.set(id, handlers); + } + + handlers.push(handler); + + if (handlers.length === 1) { + await this.updateModPackSubscriptions(); + } + } + + async offModPackUpdate(id, handler) { + let handlers = this.modPackUpdateHandlers.get(id); + if (!handlers || !handlers.length) { + throw new Error(`No handlers for mod pack ${id} exist`); + } + + let index = handlers.lastIndexOf(handler); + if (index === -1) { + throw new Error(`Given handler is not registered for mod pack ${id}`); + } + + handlers.splice(index, 1); + if (!handlers.length) { + this.modPackUpdateHandlers.delete(id); + await this.updateModPackSubscriptions(); + } + } + + async updateModPackSubscriptions() { + if (!this.connector.connected) { + return; + } + + await libLink.messages.setModPackSubscriptions.send(this, { + all: this.modPackUpdateHandlers.has(null), + mod_pack_ids: [...this.modPackUpdateHandlers.keys()].filter(k => k !== null), + }); + } + async modUpdateEventHandler(message) { let mod = new libData.ModInfo(message.data.mod); let handlers = [].concat( @@ -355,12 +417,12 @@ export class Control extends libLink.Link { async offModUpdate(name, handler) { let handlers = this.modUpdateHandlers.get(name); if (!handlers || !handlers.length) { - throw new Error(`No handlers for mod pack ${name} exist`); + throw new Error(`No handlers for mod ${name} exist`); } let index = handlers.lastIndexOf(handler); if (index === -1) { - throw new Error(`Given handler is not registered for mod pack ${name}`); + throw new Error(`Given handler is not registered for mod ${name}`); } handlers.splice(index, 1); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 3c9a58f6..2cd230c8 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -6,6 +6,7 @@ const events = require("events"); const phin = require("phin"); const libBuildMod = require("@clusterio/lib/build_mod"); +const libData = require("@clusterio/lib/data"); const libHash = require("@clusterio/lib/hash"); const libLink = require("@clusterio/lib/link"); const libUsers = require("@clusterio/lib/users"); @@ -820,6 +821,70 @@ describe("Integration of Clusterio", function() { }); }); + describe("mod-pack create", function() { + it("should create a mod-pack", async function() { + await execCtl("mod-pack create empty-pack 1.1"); + let response = await libLink.messages.listModPacks.send(getControl()); + assert(response.list.some(modPack => modPack.name === "empty-pack"), "created pack is not in the list"); + }); + it("should allow setting all fields", async function() { + await execCtl( + "mod-pack create full-pack 0.17 " + + "--description Description " + + "--mods empty_mod:1.0.0 " + + "--bool-setting startup MyBool true " + + "--int-setting runtime-global MyInt 1235 " + + "--double-setting runtime-global MyDouble 12.25 " + + "--string-setting runtime-per-user MyString a-string" + ); + let response = await libLink.messages.listModPacks.send(getControl()); + let modPack = response.list.find(entry => entry.name === "full-pack"); + assert(modPack, "created mod pack not found"); + let reference = new libData.ModPack(); + reference.id = modPack.id; + reference.name = "full-pack"; + reference.description = "Description"; + reference.factorioVersion = "0.17"; + reference.mods.set("empty_mod", { name: "empty_mod", version: "1.0.0" }); + reference.settings["startup"].set("MyBool", { value: true }); + reference.settings["runtime-global"].set("MyInt", { value: 1235 }); + reference.settings["runtime-global"].set("MyDouble", { value: 12.25 }); + reference.settings["runtime-per-user"].set("MyString", { value: "a-string" }); + assert.deepEqual(new libData.ModPack(modPack), reference); + }); + }); + + describe("mod-pack list", function() { + it("runs", async function() { + await execCtl("mod-pack list"); + }); + }); + + describe("mod-pack show", function() { + it("runs", async function() { + await execCtl("mod-pack show empty-pack"); + }); + }); + + describe("mod-pack edit", function() { + it("runs", async function() { + await execCtl("mod-pack edit full-pack --factorio-version 1.2"); + let response = await libLink.messages.listModPacks.send(getControl()); + let modPack = response.list.find(entry => entry.name === "full-pack"); + assert(modPack, "created mod pack not found"); + assert.equal(modPack.factorio_version, "1.2"); + }); + }); + + describe("mod-pack delete", function() { + it("deletes the pack", async function() { + await execCtl("mod-pack delete full-pack"); + let response = await libLink.messages.listModPacks.send(getControl()); + let modPack = response.list.find(entry => entry.name === "full-pack"); + assert(!modPack, "mod pack not deleted"); + }); + }); + describe("mod upload", function() { it("uploads a mod", async function() { await libBuildMod.build({ diff --git a/test/integration/index.js b/test/integration/index.js index 0a4485af..2830e39f 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -29,6 +29,7 @@ class TestControl extends libLink.Link { this.instanceUpdates = []; this.saveListUpdates = []; this.modUpdates = []; + this.modPackUpdates = []; this.userUpdates = []; this.connector.on("connect", () => { @@ -75,6 +76,10 @@ class TestControl extends libLink.Link { this.modUpdates.push(message.data.mod); } + async modPackUpdateEventHandler(message) { + this.modPackUpdates.push(message.data.mod_pack); + } + async userUpdateEventHandler(message) { this.userUpdates.push(message.data); } diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js new file mode 100644 index 00000000..9c32a17e --- /dev/null +++ b/test/lib/data/ModPack.js @@ -0,0 +1,94 @@ +"use strict"; +const assert = require("assert").strict; +const zlib = require("zlib"); + +const libSchema = require("@clusterio/lib/schema"); +const ModPack = require("@clusterio/lib/data/ModPack"); + + +describe("lib/data/ModPack", function() { + describe("class ModPack", function() { + it("should round trip serialize", function() { + const validate = libSchema.compile(ModPack.jsonSchema); + function check(pack) { + const json = JSON.parse(JSON.stringify(pack)); + if (!validate(json)) { + throw validate.errors; + } + assert.deepEqual(new ModPack(json), pack); + const packStringed = ModPack.fromModPackString(pack.toModPackString(true)); + packStringed.id = pack.id; + assert.deepEqual(packStringed, pack); + } + + check(new ModPack()); + check(new ModPack({ name: "MyPack" })); + check(new ModPack({ description: "My Description" })); + check(new ModPack({ factorio_version: "2.0" })); + check(new ModPack({ mods: [ + { name: "subspace_storage", version: "1.99.8" }, + { name: "clusterio_lib", version: "0.1.2", hash: "sha1:012345abcd" }, + ]})); + check(new ModPack({ settings: { + "startup": { + "bool-setting": { "value": true }, + }, + "runtime-global": { + "number-setting": { "value": 123 }, + }, + "runtime-per-user": { + "string-setting": { "value": "a string" }, + }, + }})); + check(new ModPack({ deleted: true })); + check(new ModPack({ + name: "Super pack", + description: "Every option at once.", + factorio_version: "2.0", + mods: [ + { name: "subspace_storage", version: "1.99.8" }, + { name: "clusterio_lib", version: "0.1.2", hash: "sha1:012345abcd" }, + ], + settings: { + "startup": { + "bool-setting": { "value": true }, + }, + "runtime-global": { + "number-setting": { "value": 123 }, + }, + "runtime-per-user": { + "string-setting": { "value": "a string" }, + }, + }, + deleted: true, + })); + }); + + it("should sort integer factorio versions lexicographically", function() { + let unsortedVersions = ["1.0", "1.1.0", "0.1", "3.0.0", "1.2", "0.3.1", "0.3.3", "2.1.1", "0.0.1"]; + let sortedVersions = ["0.0.1", "0.1", "0.3.1", "0.3.3", "1.0", "1.1.0", "1.2", "2.1.1", "3.0.0"]; + let factorioMods = unsortedVersions.map(v => new ModPack({ factorio_version: v })); + factorioMods.sort((a, b) => a.integerFactorioVersion - b.integerFactorioVersion); + assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions); + }); + + describe(".fromModPackString()", function() { + it("should handle malformed strings", function() { + assert.throws( + () => ModPack.fromModPackString("AMalformedString"), + new Error("Malformed mod pack string: zlib inflate failed") + ); + assert.throws( + // eslint-disable-next-line node/no-sync + () => ModPack.fromModPackString(Buffer.from(zlib.deflateSync("Not Json")).toString("base64")), + new Error("Malformed mod pack string: Unexpected token N in JSON at position 0") + ); + assert.throws( + // eslint-disable-next-line node/no-sync + () => ModPack.fromModPackString(Buffer.from(zlib.deflateSync('{"i":1}')).toString("base64")), + new Error("Malformed mod pack string: Schema validation failed") + ); + }); + }); + }); +}); From 82c644814952c6f4d0c47c1c5d114983ad56c3ba Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 00:30:18 +0200 Subject: [PATCH 09/40] Use no-break space in formatBytes Prevent tables cells containing byte sizes from having a line break inserted between the number and the byte unit when they get squished by using a no-break space instead of a regular space. --- packages/lib/helpers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lib/helpers.js b/packages/lib/helpers.js index d2157d37..3738666e 100644 --- a/packages/lib/helpers.js +++ b/packages/lib/helpers.js @@ -105,10 +105,10 @@ function escapeRegExp(text) { */ function formatBytes(bytes) { if (bytes === 0) { - return "0 Bytes"; + return "0\u{A0}Bytes"; // No-break space } - let units = [" Bytes", " kB", " MB", " GB", " TB"]; + let units = ["\u{A0}Bytes", "\u{A0}kB", "\u{A0}MB", "\u{A0}GB", "\u{A0}TB"]; let factor = 1000; let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length); return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; From 40a0b929d05e9a29503a302b63819b20abea660b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 00:39:16 +0200 Subject: [PATCH 10/40] Break mod details out into a component --- packages/web_ui/src/components/ModDetails.jsx | 46 +++++++++++++++++++ packages/web_ui/src/components/ModsPage.jsx | 41 +---------------- 2 files changed, 48 insertions(+), 39 deletions(-) create mode 100644 packages/web_ui/src/components/ModDetails.jsx diff --git a/packages/web_ui/src/components/ModDetails.jsx b/packages/web_ui/src/components/ModDetails.jsx new file mode 100644 index 00000000..f6e1b1d0 --- /dev/null +++ b/packages/web_ui/src/components/ModDetails.jsx @@ -0,0 +1,46 @@ +import React, { Fragment } from "react"; +import { Descriptions, Grid, Typography } from "antd"; + +import { libHelpers } from "@clusterio/lib"; + +const { useBreakpoint } = Grid; + + +export default function ModDetails(props) { + let screens = useBreakpoint(); + const mod = props.mod; + + return + {props.actions && !screens.lg + && {props.actions(mod)} + } + {mod.title && {mod.title}} + {mod.version && {mod.version}} + {mod.description && {mod.description} } + {mod.author && {mod.author}} + + + {`https://mods.factorio.com/mod/${mod.name}`} + + + {mod.contact && {mod.contact}} + {mod.homepage && {mod.homepage}} + {mod.name} + {mod.factorioVersion + && {mod.factorioVersion} + } + {mod.filename && {mod.filename}} + {mod.size !== undefined + && {libHelpers.formatBytes(mod.size)} + } + {mod.hash && {mod.hash}} + {mod.dependencies && mod.dependencies.length + && + {mod.dependencies + .filter(e => !e.startsWith("(?)")) + .map((e, i) => {e}
) + } + + } + ; +} diff --git a/packages/web_ui/src/components/ModsPage.jsx b/packages/web_ui/src/components/ModsPage.jsx index 41625b36..bd97c786 100644 --- a/packages/web_ui/src/components/ModsPage.jsx +++ b/packages/web_ui/src/components/ModsPage.jsx @@ -10,8 +10,8 @@ import { notifyErrorHandler } from "../util/notify"; import ControlContext from "./ControlContext"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; +import ModDetails from "./ModDetails"; -const { useBreakpoint } = Grid; const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; @@ -19,7 +19,6 @@ export default function ModsPage() { let account = useAccount(); let control = useContext(ControlContext); let [modList] = useModList(); - let screens = useBreakpoint(); function actions(mod) { return @@ -117,43 +116,7 @@ export default function ModsPage() { }, ]} expandable={{ - expandedRowRender: mod => <> - - {!screens.lg - && {actions(mod)} - } - {mod.title} - {mod.version} - {mod.description - && {mod.description} - } - {mod.author} - - - {`https://mods.factorio.com/mod/${mod.name}`} - - - {mod.contact - && {mod.contact} - } - {mod.homepage - && {mod.homepage} - } - {mod.name} - {mod.factorioVersion} - {mod.filename} - {libHelpers.formatBytes(mod.size)} - {mod.hash} - {mod.dependencies.length - && - {mod.dependencies - .filter(e => !e.startsWith("(?)")) - .map((e, i) => {e}
) - } - - } - - , + expandedRowRender: mod => , expandedRowClassName: () => "no-expanded-padding", }} dataSource={modList} From 476f460f750127cd50e0341dc9220618c69bb6cc Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 00:44:13 +0200 Subject: [PATCH 11/40] Add react wrapper for navigator.clipboard Add a thin wrapper of the navigator.clipboard interface suitable for use with React. --- packages/web_ui/src/util/clipboard.jsx | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 packages/web_ui/src/util/clipboard.jsx diff --git a/packages/web_ui/src/util/clipboard.jsx b/packages/web_ui/src/util/clipboard.jsx new file mode 100644 index 00000000..00f51e21 --- /dev/null +++ b/packages/web_ui/src/util/clipboard.jsx @@ -0,0 +1,69 @@ +import { useEffect, useState } from "react"; +import { EventEmitter } from "events"; + +class Permission extends EventEmitter { + status; + + constructor(name) { + super(); + navigator.permissions.query({ name }).then( + status => { + this.status = status; + this.emit("change", this.status.state); + + if (status.addEventListener) { + status.addEventListener("change", () => { + this.emit("change", this.status.state); + }); + } + }, + // Pretend the clipboard will work if this query fails. + () => { + this.status = { state: "prompt" }; + this.emit("change", "prompt"); + } + ); + } + + get state() { + return this.status.state || "prompt"; + } +} + +let readPermission = new Permission("clipboard-read"); +let writePermission = new Permission("clipboard-write"); + +// Wrapper for navigator.clipboard suitable for use in React +export function useClipboard() { + let [readPermissionState, setReadPermissionState] = useState(readPermission.state); + let [writePermissionState, setWritePermissionState] = useState(writePermission.state); + + useEffect(() => { + readPermission.on("change", setReadPermissionState); + writePermission.on("change", setWritePermissionState); + return () => { + readPermission.off("change", setReadPermissionState); + writePermission.off("change", setWritePermissionState); + }; + }, []); + + return { + readPermissionState: readPermissionState, + read: async () => await navigator.clipboard.read(), + readText: async () => await navigator.clipboard.readText(), + writePermissionState: writePermissionState, + write: async (data) => await navigator.clipboard.write(data), + writeText: async (text) => await navigator.clipboard.writeText(text), + + // Give a possible reason for why the clipboard is not available + deniedReason: () => { + if (window.isSecureContext === false) { + if (window.location.protocol !== "https:") { + return "Clipboard interaction is blocked due this page not being served over https."; + } + return "Clipboard interaction is blocked due this page not being a secure context."; + } + return "Clipboard interaction is blocked by the browser."; + }, + }; +} From 85cec761e12d5434b463c680498c12a5e2364154 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 00:46:38 +0200 Subject: [PATCH 12/40] Use overflow clip in site layout Use overflow clip instead of hidden in the page area of the site layout to avoid creating a formatting context that would make sticky not work as expected inside the page area. --- packages/web_ui/src/index.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web_ui/src/index.css b/packages/web_ui/src/index.css index 9fd4b010..c15de513 100644 --- a/packages/web_ui/src/index.css +++ b/packages/web_ui/src/index.css @@ -27,7 +27,7 @@ body { } .site-layout { - overflow-x: hidden; + overflow-x: clip; } .site-layout-content-container { From 6039715a536b0c1fd45a320ca66fa61eb021cbc1 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 00:49:56 +0200 Subject: [PATCH 13/40] Wrap section header extra content under the header Make the extra content for section headers flow under the heading when there is not enough space to show both the heading and the extra content in a single line. --- packages/web_ui/src/index.css | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/web_ui/src/index.css b/packages/web_ui/src/index.css index c15de513..7fec4888 100644 --- a/packages/web_ui/src/index.css +++ b/packages/web_ui/src/index.css @@ -73,6 +73,7 @@ body { .section-header { display: flex; + flex-wrap: wrap; align-items: center; margin-top: 16px; margin-bottom: 8px; From 3eb63db3130bb0473832063f7ec6fc7d2aef5abf Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 21:10:38 +0200 Subject: [PATCH 14/40] Add search mods request Add paged search mods request suitable for showing a real time list of results for a user provided search expression. --- packages/ctl/src/commands.js | 61 +++++++++++++++- packages/lib/link/messages.js | 36 ++++++++++ packages/lib/users.js | 8 ++- packages/master/src/ControlConnection.js | 90 ++++++++++++++++++++++++ test/integration/clusterio.js | 7 ++ 5 files changed, 200 insertions(+), 2 deletions(-) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index 9dc00bcb..81400b8d 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -1087,7 +1087,7 @@ modCommands.add(new libCommand.Command({ definition: [["list", "l"], "List mods stored in the cluster", (yargs) => { yargs.options({ "fields": { - describe: "Fields", + describe: "Fields to show, supports 'all'.", array: true, type: "string", default: ["name", "version", "title", "factorio_version"], @@ -1109,6 +1109,65 @@ modCommands.add(new libCommand.Command({ }, })); +modCommands.add(new libCommand.Command({ + definition: ["search [query]", "Search mods stored in the cluster", (yargs) => { + yargs.positional("factorio-version", { describe: "Major version of Factorio to search for", type: "string" }); + yargs.positional("query", { describe: "Search query", type: "string", default: "" }); + yargs.options({ + "page": { + describe: "Result page to show", + type: "number", + default: 1, + }, + "page-size": { + describe: "Results per page to show", + type: "number", + default: 10, + }, + "sort": { + describe: "sort results by given field", + type: "string", + }, + "sort-order": { + describe: "order to sort results in (asc/desc)", + type: "string", + default: "asc", + }, + "fields": { + describe: "Fields to show, supports 'all'.", + array: true, + type: "string", + default: ["name", "version", "title", "factorio_version"], + }, + }); + }], + handler: async function(args, control) { + let response = await libLink.messages.searchMods.send(control, { + "query": args.query, + "factorio_version": args.factorioVersion, + "page_size": args.pageSize, + "page": args.page, + "sort": args.sort, + "sort_order": args.sortOrder, + }); + let results = response.results.flatMap(result => result.versions); + if (!args.fields.includes("all")) { + for (let entry of results) { + for (let field of Object.keys(entry)) { + if (!args.fields.includes(field)) { + delete entry[field]; + } + } + } + } + for (let issue of response.query_issues) { + print(issue); + } + print(`page ${args.page} of ${response.page_count} (${response.result_count} results)`); + print(asTable(results)); + }, +})); + modCommands.add(new libCommand.Command({ definition: ["upload ", "Upload mod to the cluster", (yargs) => { yargs.positional("file", { describe: "File to upload", type: "string" }); diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index 61b00ce1..4404a25d 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -800,6 +800,42 @@ messages.listMods = new Request({ }, }); +messages.searchMods = new Request({ + type: "search_mods", + links: ["control-master"], + permission: "core.mod.search", + requestRequired: ["query", "factorio_version", "page"], + requestProperties: { + "query": { type: "string" }, + "factorio_version": { type: "string" }, + "page_size": { type: "integer" }, + "page": { type: "integer" }, + "sort": { type: "string" }, + "sort_order": { type: "string" }, + }, + responseProperties: { + "query_issues": { + type: "array", + items: { type: "string" }, + }, + "page_count": { type: "integer" }, + "result_count": { type: "integer" }, + "results": { + type: "array", + items: { + type: "object", + properties: { + "name": { type: "string" }, + "versions": { + type: "array", + items: libData.ModInfo.jsonSchema, + }, + }, + }, + }, + }, +}); + messages.setModSubscriptions = new Request({ type: "set_mod_subscriptions", links: ["control-master"], diff --git a/packages/lib/users.js b/packages/lib/users.js index f6ef8558..4f3fa0e8 100644 --- a/packages/lib/users.js +++ b/packages/lib/users.js @@ -553,7 +553,13 @@ definePermission({ definePermission({ name: "core.mod.list", title: "List mods", - description: "Get the full list of mods stored on the master", + description: "Get the full list of mods stored on the master.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.search", + title: "Search mods", + description: "Search through the list of mods stored on the master.", grantByDefault: true, }); definePermission({ diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index 12472705..bcae4226 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -18,6 +18,8 @@ const libUsers = require("@clusterio/lib/users"); const BaseConnection = require("./BaseConnection"); const routes = require("./routes"); +const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; + const lastQueryLogTime = new libPrometheus.Gauge( "clusterio_master_last_query_log_duration_seconds", "Time in seconds the last log query took to execute." @@ -543,6 +545,94 @@ class ControlConnection extends BaseConnection { return { list: [...this._master.mods.values()].map(mod => mod.toJSON()) }; } + static termsMatchesMod(terms, mod) { + for (let term of terms) { + if (term.type === "word") { + if (!libHelpers.wordMatches(term, + mod.name, mod.version, mod.title, mod.author, mod.contact, + mod.homepage, mod.description, mod.filename + )) { + return false; + } + } else if (term.type === "attribute") { + if (!libHelpers.wordMatches(term.value, mod[term.name])) { + return false; + } + } + } + return true; + } + + async searchModsRequestHandler(message) { + let query = libHelpers.parseSearchString(message.data.query, { + name: "word", + // version + title: "word", + author: "word", + contact: "word", + homepage: "word", + description: "word", + // factorioVersion + // dependencies + filename: "word", + // size + hash: "word", + }); + let factorioVersion = message.data.factorio_version; + + let results = new Map(); + for (let mod of this._master.mods.values()) { + if ( + mod.factorioVersion !== factorioVersion + || !ControlConnection.termsMatchesMod(query.terms, mod) + ) { + continue; + } + let result = results.get(mod.name); + if (!result) { + result = { + name: mod.name, + versions: [], + }; + results.set(mod.name, result); + } + result.versions.push(mod); + } + for (let result of results.values()) { + result.versions.sort((a, b) => b.integerVersion - a.integerVersion); + result.versions.map(e => e.toJSON()); + } + let resultList = [...results.values()]; + + const sort = message.data.sort; + if (sort) { + const sorters = { + name: (a, b) => strcmp(a.versions[0].name, b.versions[0].name), + title: (a, b) => strcmp(a.versions[0].title, b.versions[0].title), + author: (a, b) => strcmp(a.versions[0].author, b.versions[0].author), + }; + if (!Object.prototype.hasOwnProperty.call(sorters, sort)) { + throw new libErrors.RequestError(`Invalid value for sort: ${sort}`); + } + resultList.sort(sorters[sort]); + let order = message.data.sort_order; + if (order === "desc") { + resultList.reverse(); + } + } + + const page = message.data.page; + const pageSize = message.data.page_size || 10; + resultList = resultList.slice((page - 1) * pageSize, page * pageSize); + + return { + query_issues: query.issues, + page_count: Math.ceil(results.size / pageSize), + result_count: results.size, + results: resultList, + }; + } + async setModSubscriptionsRequestHandler(message) { this.modSubscriptions = message.data; } diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 2cd230c8..8c88c244 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -929,6 +929,13 @@ describe("Integration of Clusterio", function() { }); }); + describe("mod search", function() { + it("searches the list of mods", async function() { + let result = await execCtl("mod search 1.1 name:empty_mod"); + assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the result"); + }); + }); + describe("mod download", function() { it("downloads a mod", async function() { await fs.unlink(path.join("temp", "test", "empty_mod_1.0.0.zip")); From bddde30be4d99a050fa27d59a556c3ba5e00c493 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 21:17:18 +0200 Subject: [PATCH 15/40] Add web UI for mod packs Implement create, import, show, export, update, and delete UI for mod packs on the master server. --- packages/master/src/routes.js | 1 + .../web_ui/src/components/ModPackViewPage.jsx | 651 ++++++++++++++++++ packages/web_ui/src/components/ModsPage.jsx | 157 ++++- packages/web_ui/src/components/SiteLayout.jsx | 8 +- packages/web_ui/src/index.css | 32 +- packages/web_ui/src/pages.jsx | 8 +- 6 files changed, 850 insertions(+), 7 deletions(-) create mode 100644 packages/web_ui/src/components/ModPackViewPage.jsx diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index a34a2259..0318bcea 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -622,6 +622,7 @@ const webRoutes = [ "/instances", "/instances/:id/view", "/mods", + "/mods/mod-packs/:id/view", "/users", "/users/:id/view", "/roles", diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx new file mode 100644 index 00000000..1d01ca5e --- /dev/null +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -0,0 +1,651 @@ +import React, { useEffect, useContext, useRef, useState } from "react"; +import { useHistory, useParams } from "react-router-dom"; +import { + Button, Card, Checkbox, Col, ConfigProvider, Descriptions, Form, Input, PageHeader, Pagination, + Popconfirm, Row, Table, Tag, Typography, Select, Skeleton, Space, Spin, Switch, Modal, Tooltip, +} from "antd"; +import InfoCircleOutlined from "@ant-design/icons/InfoCircleOutlined"; +import ExportOutlined from "@ant-design/icons/ExportOutlined"; +import CloseOutlined from "@ant-design/icons/CloseOutlined"; +import DeleteOutlined from "@ant-design/icons/DeleteOutlined"; +import PlusOutlined from "@ant-design/icons/PlusOutlined"; +import UploadOutlined from "@ant-design/icons/UploadOutlined"; + +import { libData, libLink, libLogging } from "@clusterio/lib"; +import ControlContext from "./ControlContext"; +import { useAccount } from "../model/account"; +import { useModPack } from "../model/mod_pack"; +import { useModList } from "../model/mods"; +import { useClipboard } from "../util/clipboard"; +import notify, { notifyErrorHandler } from "../util/notify"; +import PageLayout from "./PageLayout"; +import SectionHeader from "./SectionHeader"; +import ModDetails from "./ModDetails"; + +const { logger } = libLogging; +const { Paragraph, Text } = Typography; +const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; + +function SearchModsTable(props) { + const control = useContext(ControlContext); + const [searchText, setSearchText] = useState(""); + const [modResultSort, setModResultSort] = useState("title"); + const [modResultSortOrder, setModResultSortOrder] = useState("asc"); + const [modResults, setModResults] = useState([]); + const [modResultPage, setModResultPage] = useState(1); + const [modResultPageSize, setModResultPageSize] = useState(10); + const [modResultCount, setModResultCount] = useState(2); + const [modResultSelectedVersion, setModResultSelectedVersion] = useState(new Map()); + + const factorioVersion = props.modPack.factorioVersion.split(".").slice(0, 2).join("."); + + useEffect(() => { + let canceled = false; + libLink.messages.searchMods.send(control, { + query: searchText, + factorio_version: factorioVersion, + page_size: modResultPageSize, + page: modResultPage, + sort: modResultSort, + sort_order: modResultSortOrder, + }).then(response => { + if (canceled) { + return; + } + // In React < v18 this causes 3 renders and a partial state necessitating the use + // of || 0 on modResultSelectedVersion.get() calls. Remove when updating React. + setModResults(response.results.map( + ({ name, versions }) => ({ name, versions: versions.map(mod => new libData.ModInfo(mod)) }) + )); + setModResultSelectedVersion(new Map(response.results.map(({ name, versions }) => [name, 0]))); + setModResultCount(response.result_count); + }); + + return () => { + canceled = true; + }; + }, [searchText, factorioVersion, modResultPageSize, modResultPage, modResultSort, modResultSortOrder]); + + function modResultPageChanged(page, pageSize) { + setModResultPage(page); + setModResultPageSize(pageSize); + } + + function modResultsTableChanged(pagination, filters, sorters, extra) { + if (extra.action === "sort") { + if (sorters.order) { + const mapping = { + ascend: "asc", + descend: "desc", + }; + setModResultSort(sorters.columnKey); + setModResultSortOrder(mapping[sorters.order]); + } else { + setModResultSort(undefined); + setModResultSortOrder(undefined); + } + } + } + + function actions(mod) { + return + {(!props.modPack.mods.has(mod.name) || props.modPack.mods.get(mod.name).version !== mod.version) + && { + props.onChange({ + type: "mods.set", + name: mod.name, + value: { name: mod.name, version: mod.version, hash: mod.hash }, + }); + }} + >{props.modPack.mods.has(mod.name) ? "change" : "add"} + } + {props.modPack.mods.has(mod.name) + && { + props.onChange({ + type: "mods.delete", + name: mod.name, + value: { name: mod.name, version: mod.version, hash: mod.hash }, + }); + }} + >remove + } + ; + } + + return <> + { + let search = event.target.value.trim(); + setSearchText(search); + setModResultPage(1); + }} + /> + `${total} results`} + showSizeChanger={true} + size="small" + />} + /> + "No results"}> +
results.versions[0].title, + defaultSortOrder: "ascend", + sorter: true, + }, + { + title: "Author", + key: "author", + render: results => results.versions[0].author, + responsive: ["lg"], + sorter: true, + }, + { + title: "Version", + key: "version", + align: "right", + render: result =>
mod.title || mod.name, + defaultSortOrder: "ascend", + sorter: (a, b) => strcmp(a.name, b.name), + }, + { + title: "Version", + dataIndex: "version", + sorter: (a, b) => strcmp(a.version, b.version), + }, + { + title: "Action", + key: "action", + responsive: ["lg"], + render: mod => actions(mod), + }, + ]} + expandable={{ + expandedRowRender: mod => , + expandedRowClassName: () => "no-expanded-padding", + }} + dataSource={mods} + pagination={false} + rowKey={mod => mod.name} + rowClassName={mod => { + if (deletedMods.has(mod.name)) { + return "deleted"; + } + if (changedMods.has(mod.name)) { + return "changed"; + } + return ""; + }} + /> + ; +} + +// Returns true if a change record with the given fields exist +function hasChange(changes, field) { + return changes.some(change => Object.entries(field).every(([key, value]) => change[key] === value)); +} + +// picks out the value property from settings. +function pickValue(map) { + return [...map].map(([name, value]) => [name, value.value]); +} + +function SettingsTable(props) { + function controls(scope, fields) { + return fields.map(([fieldName, fieldValue]) => { + const isBoolean = typeof fieldValue === "boolean"; + const isNumber = typeof fieldValue === "number"; + + return + {isBoolean ? : } + ; + }); + } + + return <> + + Use clusterioctl to add and remove fields. + Startup + {controls("startup", pickValue(props.modPack.settings["startup"]))} + + Map + {controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))} + + Per Player + {controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))} + ; +} + +function CopyButton(props) { + let clipboard = useClipboard(); + let [copiedVisible, setCopiedVisible] = useState(false); + let copiedTimeout = useRef(); + + useEffect(() => () => { + clearTimeout(copiedTimeout.current); + setCopiedVisible(false); + }, []); + + let denied = clipboard.readPermissionState === "denied"; + return + + ; +} + +function ExportButton(props) { + let [visible, setVisible] = useState(false); + function close() { + setVisible(false); + } + + let exportString = props.modPack.toModPackString(); + + return <> + + + + + } + > + + {exportString} + + + ; +} + +export default function ModPackViewPage() { + let account = useAccount(); + let history = useHistory(); + let control = useContext(ControlContext); + let params = useParams(); + let modPackId = Number(params.id); + + const [form] = Form.useForm(); + let [modPack] = useModPack(modPackId); + let [changes, setChanges] = useState([]); + let [modifiedModPack, setModifiedModPack] = useState(modPack); + let syncTimeout = useRef(); + + useEffect(() => () => { clearTimeout(syncTimeout.current); }); + + function pushChange(change) { + setChanges(oldChanges => { + const newChanges = [...oldChanges]; + if ( + newChanges.length + && newChanges.slice(-1)[0].type === change.type + && newChanges.slice(-1)[0].name === change.name + && newChanges.slice(-1)[0].scope === change.scope + ) { + newChanges.splice(-1, 1, change); + } else { + newChanges.push(change); + } + return newChanges; + }); + } + + function revertChange(change) { + const newChanges = [...changes]; + const index = newChanges.findLastIndex(reference => ( + change.type === reference.type && change.name === reference.name + )); + if (index === -1) { + logger.error(`Unable to revert ${change.type} ${change.name}: change not found`); + return; + } + newChanges.splice(index, 1); + setChanges(newChanges); + } + + function syncPack(fields) { + for (let field of fields) { + if (!field.touched || field.validating || field.errors.length) { + continue; + } + if (["name", "description", "factorioVersion"].includes(field.name[0])) { + if (modifiedModPack[field.name[0]] !== field.value) { + pushChange({ type: field.name[0], value: field.value }); + } + + } else if (["startup", "runtime-global", "runtime-per-user"].includes(field.name[0])) { + let [scope, settingName] = field.name; + let value = field.value; + if (typeof modPack.settings[scope].get(settingName) === "number") { + value = Number(value); + if (Number.isNaN(value)) { + continue; + } + } + if (modifiedModPack.settings[scope].get(settingName) !== value) { + pushChange({ type: "settings.set", scope, name: settingName, value: { value }}); + } + } + } + } + + useEffect(() => { + // Loading or invalid id + if (!(modPack instanceof libData.ModPack)) { + setModifiedModPack(modPack); + return; + } + + const modified = new libData.ModPack(modPack.toJSON()); + for (let change of changes) { + if (change.type === "name") { + modified.name = change.value; + } else if (change.type === "description") { + modified.description = change.value; + } else if (change.type === "factorioVersion") { + modified.factorioVersion = change.value; + } else if (change.type === "mods.set") { + modified.mods.set(change.name, change.value); + } else if (change.type === "mods.delete") { + modified.mods.delete(change.name); + } else if (change.type === "settings.set") { + modified.settings[change.scope].set(change.name, change.value); + } else if (change.type === "settings.delete") { + modified.settings[change.scope].delete(change.name); + } else { + throw new Error(`Unknown change type ${change.type}`); + } + } + setModifiedModPack(modified); + }, [modPack, changes]); + + let nav = [{ name: "Mods", path: "/mods" }, { name: "Mod Packs" }, { name: modPack.name || modPackId }]; + if (modifiedModPack.loading) { + return + + + ; + } + + if (modifiedModPack.missing) { + return + +

Mod pack with id {modPackId} was not found on the master server.

+ ; + } + + return + + + {account.hasPermission("core.mod-pack.delete") && { + libLink.messages.deleteModPack.send( + control, { id: modPack.id } + ).then(() => { + history.push("/mods"); + }).catch(notifyErrorHandler("Error deleting mod pack")); + }} + > + + } + } + /> +
{ + clearTimeout(syncTimeout.current); + if (typeof changedFields[0].value === "boolean") { + syncPack(allFields); + } else { + // Defer sync if this is not a checkbox to make the UI responsive. + syncTimeout.current = setTimeout(syncPack.bind(undefined, allFields), 500); + } + }} + initialValues={{ + name: modPack.name, + description: modPack.description, + factorioVersion: modPack.factorioVersion, + "startup": Object.fromEntries(pickValue(modPack.settings["startup"])), + "runtime-global": Object.fromEntries(pickValue(modPack.settings["runtime-global"])), + "runtime-per-user": Object.fromEntries(pickValue(modPack.settings["runtime-per-user"])), + }} + > + + + + + + + + + + + + + {modifiedModPack.mods.size} + + + + + + + + + + + +
+
+ +
You have unsaved changes + + + + + + + + + + ; + +} diff --git a/packages/web_ui/src/components/ModsPage.jsx b/packages/web_ui/src/components/ModsPage.jsx index bd97c786..71a1cca6 100644 --- a/packages/web_ui/src/components/ModsPage.jsx +++ b/packages/web_ui/src/components/ModsPage.jsx @@ -1,24 +1,138 @@ -import React, { Fragment, useContext } from "react"; -import { Button, Descriptions, Grid, PageHeader, Popconfirm, Space, Table, Typography, Upload } from "antd"; +import React, { Fragment, useState, useContext } from "react"; +import { useHistory } from "react-router-dom"; +import { Button, Form, Input, Modal, PageHeader, Popconfirm, Space, Table, Typography, Upload } from "antd"; +import ImportOutlined from "@ant-design/icons/ImportOutlined"; +import PlusOutlined from "@ant-design/icons/PlusOutlined"; import UploadOutlined from "@ant-design/icons/UploadOutlined"; -import { libHelpers, libLink } from "@clusterio/lib"; +import { libData, libHelpers, libLink, libLogging } from "@clusterio/lib"; import { useAccount } from "../model/account"; import { useModList } from "../model/mods"; +import { useModPackList } from "../model/mod_pack"; import { notifyErrorHandler } from "../util/notify"; import ControlContext from "./ControlContext"; import PageLayout from "./PageLayout"; import PluginExtra from "./PluginExtra"; +import SectionHeader from "./SectionHeader"; import ModDetails from "./ModDetails"; const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; +const { logger } = libLogging; + + +function ImportModPackButton() { + let control = useContext(ControlContext); + let history = useHistory(); + let [visible, setVisible] = useState(false); + let [form] = Form.useForm(); + function close() { + setVisible(false); + } + return <> + + { setVisible(false); }} + onOk={() => { + (async () => { + let values; + try { + values = await form.validateFields(); + } catch { + return; // Validation failed + } + const modPack = libData.ModPack.fromModPackString(values.string); + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + history.push(`/mods/mod-packs/${modPack.id}/view`); + })().catch(notifyErrorHandler("Error creating mod pack")); + }} + > +
+ + + + + + ; +} + +function CreateModPackButton() { + let control = useContext(ControlContext); + let history = useHistory(); + let [visible, setVisible] = useState(false); + let [form] = Form.useForm(); + function close() { + setVisible(false); + } + return <> + + { setVisible(false); }} + onOk={() => { + (async () => { + let values; + try { + values = await form.validateFields(); + } catch { + return; // Validation failed + } + const modPack = new libData.ModPack(); + modPack.name = values.name; + modPack.factorioVersion = values.factorioVersion; + if (values.description) { modPack.description = values.description; } + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + history.push(`/mods/mod-packs/${modPack.id}/view`); + })().catch(notifyErrorHandler("Error creating mod pack")); + }} + > +
+ + + + + + + + + + + + ; +} export default function ModsPage() { let account = useAccount(); let control = useContext(ControlContext); + let history = useHistory(); let [modList] = useModList(); + let [modPackList] = useModPackList(); function actions(mod) { return @@ -73,8 +187,43 @@ export default function ModsPage() { + + {account.hasPermission("core.mod-pack.create") && } + {account.hasPermission("core.mod-pack.create") && } + } + /> +
strcmp(a.name, b.name), + }, + { + title: "Factorio Version", + dataIndex: "factorioVersion", + sorter: (a, b) => a.integerFactorioVersion - b.integerFactorioVersion, + }, + { + title: "Mods", + key: "roles", + render: modPack => modPack.mods.size, + }, + ]} + dataSource={modPackList} + pagination={false} + rowKey={modPack => modPack.id} + onRow={(modPack, rowIndex) => ({ + onClick: event => { + history.push(`/mods/mod-packs/${modPack.id}/view`); + }, + })} + /> +
code { + display: block; +} + /* * Render codeblock with options for syntax highlighting * https://user-images.githubusercontent.com/6792749/152066701-21420461-710b-4fae-a80a-89ef90a0b18f.png @@ -321,6 +325,32 @@ body { border: none; } -tr.no-expanded-padding > td { +tr.no-expanded-padding > td, +.ant-table.ant-table-small .ant-table-tbody > tr.no-expanded-padding > td { padding: 0; } + +.ant-form-item.changed, +input.ant-input.changed, +textarea.ant-input.changed, +tr.ant-table-row.changed { + background: #2a1912; +} + +.ant-table-tbody > tr.ant-table-row.changed:hover > td, +.ant-table-tbody > tr.changed > td.ant-table-cell-row-hover +tr.ant-table-row.deleted { + background: #3a2922; +} + +tr.ant-table-row.deleted { + color: rgba(255, 255, 255, 0.45); + background: #340c08; + text-decoration: line-through; +} + +.ant-table-tbody > tr.ant-table-row.deleted:hover > td, +.ant-table-tbody > tr.deleted > td.ant-table-cell-row-hover +tr.ant-table-row.deleted { + background: #441c18; +} diff --git a/packages/web_ui/src/pages.jsx b/packages/web_ui/src/pages.jsx index 7f190564..ed85e7a5 100644 --- a/packages/web_ui/src/pages.jsx +++ b/packages/web_ui/src/pages.jsx @@ -5,6 +5,7 @@ import SlavesPage from "./components/SlavesPage"; import SlaveViewPage from "./components/SlaveViewPage"; import InstancesPage from "./components/InstancesPage"; import InstanceViewPage from "./components/InstanceViewPage"; +import ModPackViewPage from "./components/ModPackViewPage"; import ModsPage from "./components/ModsPage"; import UsersPage from "./components/UsersPage"; import UserViewPage from "./components/UserViewPage"; @@ -45,9 +46,14 @@ export const pages = [ { path: "/mods", sidebarName: "Mods", - permission: "core.mod.list", + permission: account => account.hasAnyPermission("core.mod.list", "core.mod-pack.list"), content: , }, + { + path: "/mods/mod-packs/:id/view", + sidebarPath: "/mods", + content: , + }, { path: "/users", sidebarName: "Users", From 9b4ca86650b0e587fb8b570ee6c9c51ae9cfaa9e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 21:43:07 +0200 Subject: [PATCH 16/40] Rename mod hash field to sha1 The Factorio Mod Portal uses and provides SHA1 hashes of mods, so it makes sense to align the ModPack and ModInfo structures to sha1 by renaming the genericly named hash property to sha1. --- packages/ctl/src/commands.js | 4 ++-- packages/lib/data/ModInfo.js | 13 ++++++------- packages/lib/data/ModPack.js | 6 +++--- packages/master/src/ControlConnection.js | 2 +- packages/web_ui/src/components/ModDetails.jsx | 2 +- packages/web_ui/src/components/ModPackViewPage.jsx | 4 ++-- test/integration/clusterio.js | 2 +- test/lib/data/ModInfo.js | 6 +++--- test/lib/data/ModPack.js | 4 ++-- 9 files changed, 21 insertions(+), 22 deletions(-) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index 81400b8d..3ee812c7 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -875,7 +875,7 @@ modPackCommands.add(new libCommand.Command({ if (field === "mods") { print(`${field}:`); for (let entry of value) { - print(` ${entry.name} ${entry.version}${entry.hash ? ` (${entry.hash})` : ""}`); + print(` ${entry.name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`); } } else if (field === "settings") { print(`${field}:`); @@ -958,7 +958,7 @@ function setModPackMods(modPack, mods) { if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) { throw new libErrors.CommandError("sha1 must be a 40 digit lower case hex string"); } - modPack.mods.set(name, { name, version, hash: sha1 ? `sha1:${sha1}` : undefined }); + modPack.mods.set(name, { name, version, sha1 }); } } diff --git a/packages/lib/data/ModInfo.js b/packages/lib/data/ModInfo.js index e1949d0c..e3bcce46 100644 --- a/packages/lib/data/ModInfo.js +++ b/packages/lib/data/ModInfo.js @@ -111,10 +111,10 @@ class ModInfo { size = 0; /** - * Hash of this mod + * SHA1 hash of this mod * @type {string=} */ - hash; + sha1; /** * True if this mod has been deleted @@ -147,7 +147,7 @@ class ModInfo { ...this.infoJsonSchema.properties, "filename": { type: "string" }, "size": { type: "integer" }, - "hash": { type: "string" }, + "sha1": { type: "string" }, "is_deleted": { type: "boolean" }, }, }; @@ -169,7 +169,7 @@ class ModInfo { // Additional data if (json.filename) { this.filename = json.filename; } if (json.size) { this.size = json.size; } - if (json.hash) { this.hash = json.hash; } + if (json.sha1) { this.sha1 = json.sha1; } if (json.is_deleted) { this.isDeleted = json.is_deleted; } } @@ -189,7 +189,7 @@ class ModInfo { } if (this.filename) { json.filename = this.filename; } if (this.size) { json.size = this.size; } - if (this.hash) { json.hash = this.hash; } + if (this.sha1) { json.sha1 = this.sha1; } if (this.isDeleted) { json.is_deleted = this.isDeleted; } return json; } @@ -215,13 +215,12 @@ class ModInfo { throw new Error("Mod's info.json is not valid"); } - let hash = `sha1:${await libHash.hashFile(modPath)}`; return new this({ ...modInfo, filename: path.basename(modPath), size: (await fs.stat(modPath)).size, - hash, + sha1: await libHash.hashFile(modPath), is_deleted: false, }); } diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 9dd09154..c13e6bf0 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -28,7 +28,7 @@ const ModSettingsJsonSchema = { * @typedef {object} module:lib/data/ModPack~ModRecord * @property {string} name - name of the mod. * @property {string} version - version of the mod. - * @property {string} hash - hash of the zip file. + * @property {string} sha1 - SHA1 hash of the zip file. */ const ModRecordJsonSchema = { type: "object", @@ -37,7 +37,7 @@ const ModRecordJsonSchema = { properties: { "name": { type: "string" }, "version": { type: "string" }, - "hash": { type: "string" }, + "sha1": { type: "string" }, }, }; @@ -176,7 +176,7 @@ class ModPack { if (json.mods && !includeHash) { json.mods = json.mods.map(m => { m = { ...m }; - delete m.hash; + delete m.sha1; return m; }); } diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index bcae4226..6e9cd9a2 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -576,7 +576,7 @@ class ControlConnection extends BaseConnection { // dependencies filename: "word", // size - hash: "word", + sha1: "word", }); let factorioVersion = message.data.factorio_version; diff --git a/packages/web_ui/src/components/ModDetails.jsx b/packages/web_ui/src/components/ModDetails.jsx index f6e1b1d0..af5ebd78 100644 --- a/packages/web_ui/src/components/ModDetails.jsx +++ b/packages/web_ui/src/components/ModDetails.jsx @@ -33,7 +33,7 @@ export default function ModDetails(props) { {mod.size !== undefined && {libHelpers.formatBytes(mod.size)} } - {mod.hash && {mod.hash}} + {mod.sha1 && {mod.sha1}} {mod.dependencies && mod.dependencies.length && {mod.dependencies diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 1d01ca5e..17042616 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -95,7 +95,7 @@ function SearchModsTable(props) { props.onChange({ type: "mods.set", name: mod.name, - value: { name: mod.name, version: mod.version, hash: mod.hash }, + value: { name: mod.name, version: mod.version, sha1: mod.sha1 }, }); }} >{props.modPack.mods.has(mod.name) ? "change" : "add"} @@ -107,7 +107,7 @@ function SearchModsTable(props) { props.onChange({ type: "mods.delete", name: mod.name, - value: { name: mod.name, version: mod.version, hash: mod.hash }, + value: { name: mod.name, version: mod.version, sha1: mod.sha1 }, }); }} >remove diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 8c88c244..f1ecbb1c 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -917,7 +917,7 @@ describe("Integration of Clusterio", function() { "dependencies:\n" + "filename: empty_mod_1.0.0.zip\n" + `size: ${stat.size}\n` + - `hash: sha1:${hash}\n`, + `sha1: ${hash}\n`, ); }); }); diff --git a/test/lib/data/ModInfo.js b/test/lib/data/ModInfo.js index f3de1794..9e435e81 100644 --- a/test/lib/data/ModInfo.js +++ b/test/lib/data/ModInfo.js @@ -34,7 +34,7 @@ describe("lib/data/ModInfo", function() { check(new ModInfo({ dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"] })); check(new ModInfo({ filename: "MyMod_1.0.0.zip" })); check(new ModInfo({ size: 1024 })); - check(new ModInfo({ hash: "sha5:verified-as-MyMod" })); + check(new ModInfo({ sha1: "verified-as-MyMod" })); check(new ModInfo({ is_deleted: true })); // All at once @@ -50,7 +50,7 @@ describe("lib/data/ModInfo", function() { dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"], filename: "MyMod_1.0.0.zip", size: 1024, - hash: "sha5:verified-as-MyMod", + sha1: "verified-as-MyMod", is_deleted: true, })); }); @@ -93,7 +93,7 @@ describe("lib/data/ModInfo", function() { dependencies: [], filename: "empty_mod_1.0.0.zip", size: stat.size, - hash: `sha1:${hash}`, + sha1: hash, }) ); }); diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index 9c32a17e..9d08bdc5 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -27,7 +27,7 @@ describe("lib/data/ModPack", function() { check(new ModPack({ factorio_version: "2.0" })); check(new ModPack({ mods: [ { name: "subspace_storage", version: "1.99.8" }, - { name: "clusterio_lib", version: "0.1.2", hash: "sha1:012345abcd" }, + { name: "clusterio_lib", version: "0.1.2", sha1: "012345abcd" }, ]})); check(new ModPack({ settings: { "startup": { @@ -47,7 +47,7 @@ describe("lib/data/ModPack", function() { factorio_version: "2.0", mods: [ { name: "subspace_storage", version: "1.99.8" }, - { name: "clusterio_lib", version: "0.1.2", hash: "sha1:012345abcd" }, + { name: "clusterio_lib", version: "0.1.2", sha1: "012345abcd" }, ], settings: { "startup": { From af79e2abbdcc075a0e87f462a8e76d197cb28e01 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 24 Sep 2022 21:49:41 +0200 Subject: [PATCH 17/40] Remove includeHash option to toModPackString Since the Mod Portal provides SHA1 hashes it makes sense to always include the hashes with mod pack strings if they are available. --- packages/lib/data/ModPack.js | 9 +-------- test/lib/data/ModPack.js | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index c13e6bf0..ed9ac60d 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -170,16 +170,9 @@ class ModPack { return json; } - toModPackString(includeHash = false) { + toModPackString() { const json = this.toJSON(); delete json.id; - if (json.mods && !includeHash) { - json.mods = json.mods.map(m => { - m = { ...m }; - delete m.sha1; - return m; - }); - } // eslint-disable-next-line node/no-sync let buf = zlib.deflateSync(JSON.stringify(json)); diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index 9d08bdc5..fede2912 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -16,7 +16,7 @@ describe("lib/data/ModPack", function() { throw validate.errors; } assert.deepEqual(new ModPack(json), pack); - const packStringed = ModPack.fromModPackString(pack.toModPackString(true)); + const packStringed = ModPack.fromModPackString(pack.toModPackString()); packStringed.id = pack.id; assert.deepEqual(packStringed, pack); } From 7d34c60dd172a0e3f2c43b47485609c656aec5d6 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 25 Sep 2022 23:33:22 +0200 Subject: [PATCH 18/40] Break out version to integer functions Move the multiple implementation of version to integers calculation to dedicated functions in one place. --- packages/lib/data/ModInfo.js | 8 ++++---- packages/lib/data/ModPack.js | 5 +++-- packages/lib/data/version.js | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 packages/lib/data/version.js diff --git a/packages/lib/data/ModInfo.js b/packages/lib/data/ModInfo.js index e3bcce46..ab2a3a50 100644 --- a/packages/lib/data/ModInfo.js +++ b/packages/lib/data/ModInfo.js @@ -7,6 +7,8 @@ const libHash = require("../hash"); const libSchema = require("../schema"); const { findRoot } = require("../zip_ops"); +const { integerModVersion, integerFactorioVersion } = require("./version"); + /** * Info about a mod available on the master server. @@ -36,8 +38,7 @@ class ModInfo { * @type {number} */ get integerVersion() { - const [major, minor, sub] = this.version.split(".").map(n => Number.parseInt(n, 10)); - return major * 0x100000000 + minor * 0x10000 + sub; // Can't use bitwise here because this is 48-bits. + return integerModVersion(this.version); } /** @@ -87,8 +88,7 @@ class ModInfo { * @type {number} */ get integerFactorioVersion() { - const [major, minor] = this.factorioVersion.split(".").map(n => Number.parseInt(n, 10)); - return major * 0x100000000 + minor * 0x10000; // Can't use bitwise here because this is 48-bits. + return integerFactorioVersion(this.factorioVersion); } /** diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index ed9ac60d..73bd8aab 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -3,6 +3,8 @@ const zlib = require("zlib"); const libSchema = require("../schema"); +const { integerFactorioVersion } = require("./version"); + /** * A setting for a mod. @@ -77,8 +79,7 @@ class ModPack { * @type {number} */ get integerFactorioVersion() { - const [major, minor, sub] = this.factorioVersion.split(".").map(n => Number.parseInt(n, 10)); - return major * 0x100000000 + minor * 0x10000 + (sub || 0); // Can't use bitwise here because this is 48-bits. + return integerFactorioVersion(this.factorioVersion); } /** diff --git a/packages/lib/data/version.js b/packages/lib/data/version.js new file mode 100644 index 00000000..ff533ee8 --- /dev/null +++ b/packages/lib/data/version.js @@ -0,0 +1,34 @@ +// Helper functions for dealing with factorio version numbers +"use strict"; + + +/** + * Integer representation of a mod version + * + * Suitable for sorting and comparing version numbers which may have different + * number of leading zeros in them. + * @param {string} version - string representation of version. + * @returns {number} numeric representation. + */ +function integerModVersion(version) { + const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000 + sub; // Can't use bitwise here because this is 48-bits. +} + +/** + * Integer representation of a factorio version + * + * Suitable for sorting and comparing version numbers which may have different + * number of leading zeros in them. + * @param {string} version - string representation of version. + * @returns {number} numeric representation. + */ +function integerFactorioVersion(version) { + const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000 + (sub || 0); // Can't use bitwise here because this is 48-bits. +} + +module.exports = { + integerModVersion, + integerFactorioVersion, +}; From 7072b94355e1a8608a18079e402163fa41ce57d5 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 26 Sep 2022 00:34:25 +0200 Subject: [PATCH 19/40] Derive mod filename from name and version Enforce the filename of uploaded mods match {name}_{version}.zip with name and version taken from the zip file's info.json entry. This allows it to be used as a unique key for the mod and solves part of the issues with mods having different number of leading zeros in their version. --- packages/ctl/src/commands.js | 3 ++- packages/lib/data/ModInfo.js | 16 +++++++++------- packages/lib/data/version.js | 10 ++++++++++ packages/master/src/ControlConnection.js | 12 ++++++------ packages/master/src/Master.js | 20 ++++++++++---------- packages/master/src/routes.js | 5 ++--- test/integration/clusterio.js | 1 - 7 files changed, 39 insertions(+), 28 deletions(-) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index 3ee812c7..f4a31510 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -1204,7 +1204,8 @@ modCommands.add(new libCommand.Command({ } if (result.body.mods && result.body.mods.length) { - logger.info(`Successfully uploaded ${result.body.mods[0].filename}`); + const mod = new libData.ModInfo(result.body.mods[0]); + logger.info(`Successfully uploaded ${mod.filename}`); } if ((result.body.errors || []).length || (result.body.request_errors || []).length) { diff --git a/packages/lib/data/ModInfo.js b/packages/lib/data/ModInfo.js index ab2a3a50..4ad3873b 100644 --- a/packages/lib/data/ModInfo.js +++ b/packages/lib/data/ModInfo.js @@ -7,7 +7,7 @@ const libHash = require("../hash"); const libSchema = require("../schema"); const { findRoot } = require("../zip_ops"); -const { integerModVersion, integerFactorioVersion } = require("./version"); +const { integerModVersion, integerFactorioVersion, modVersionRegExp } = require("./version"); /** @@ -99,10 +99,12 @@ class ModInfo { dependencies = ["base"]; /** - * Name of the file containing this mod. + * Expected name of zip file containing this mod. * @type {number} */ - filename = ""; + get filename() { + return `${this.name}_${this.version}.zip`; + } /** * Size of the mod in bytes @@ -145,7 +147,6 @@ class ModInfo { type: "object", properties: { ...this.infoJsonSchema.properties, - "filename": { type: "string" }, "size": { type: "integer" }, "sha1": { type: "string" }, "is_deleted": { type: "boolean" }, @@ -167,7 +168,6 @@ class ModInfo { if (json.dependencies) { this.dependencies = json.dependencies; } // Additional data - if (json.filename) { this.filename = json.filename; } if (json.size) { this.size = json.size; } if (json.sha1) { this.sha1 = json.sha1; } if (json.is_deleted) { this.isDeleted = json.is_deleted; } @@ -187,7 +187,6 @@ class ModInfo { if (this.dependencies.length !== 1 || this.dependencies[0] !== "base") { json.dependencies = this.dependencies; } - if (this.filename) { json.filename = this.filename; } if (this.size) { json.size = this.size; } if (this.sha1) { json.sha1 = this.sha1; } if (this.isDeleted) { json.is_deleted = this.isDeleted; } @@ -215,10 +214,13 @@ class ModInfo { throw new Error("Mod's info.json is not valid"); } + if (!modVersionRegExp.test(modInfo.version)) { + throw new Error(`Mod's version (${modInfo.version}) is invalid`); + } + return new this({ ...modInfo, - filename: path.basename(modPath), size: (await fs.stat(modPath)).size, sha1: await libHash.hashFile(modPath), is_deleted: false, diff --git a/packages/lib/data/version.js b/packages/lib/data/version.js index ff533ee8..bda52be7 100644 --- a/packages/lib/data/version.js +++ b/packages/lib/data/version.js @@ -9,6 +9,7 @@ * number of leading zeros in them. * @param {string} version - string representation of version. * @returns {number} numeric representation. + * @private */ function integerModVersion(version) { const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); @@ -22,13 +23,22 @@ function integerModVersion(version) { * number of leading zeros in them. * @param {string} version - string representation of version. * @returns {number} numeric representation. + * @private */ function integerFactorioVersion(version) { const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); return major * 0x100000000 + minor * 0x10000 + (sub || 0); // Can't use bitwise here because this is 48-bits. } +/** + * Matches valid mod versions. + * @type {RegExp} + * @private + */ +const modVersionRegExp = /^\d+\.\d+\.\d+$/; + module.exports = { integerModVersion, integerFactorioVersion, + modVersionRegExp, }; diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index 6e9cd9a2..48834a90 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -531,10 +531,10 @@ class ControlConnection extends BaseConnection { async getModRequestHandler(message) { let { name, version } = message.data; - let key = `${name}_${version}`; - let mod = this._master.mods.get(key); + let filename = `${name}_${version}.zip`; + let mod = this._master.mods.get(filename); if (!mod) { - throw new libErrors.RequestError(`Mod ${key} does not exist`); + throw new libErrors.RequestError(`Mod ${filename} does not exist`); } return { mod: mod.toJSON(), @@ -639,10 +639,10 @@ class ControlConnection extends BaseConnection { async downloadModRequestHandler(message) { let { name, version } = message.data; - let key = `${name}_${version}`; - let mod = this._master.mods.get(key); + let filename = `${name}_${version}.zip`; + let mod = this._master.mods.get(filename); if (!mod) { - throw new libErrors.RequestError(`Mod ${key} does not exist`); + throw new libErrors.RequestError(`Mod ${filename} does not exist`); } let modPath = path.join(this._master.config.get("master.mods_directory"), mod.filename); diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index abc67bcc..6175ff46 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -420,15 +420,15 @@ class Master { continue; } - let key = `${modInfo.name}_${modInfo.version}`; - if (mods.has(key)) { + if (modInfo.filename !== entry.name) { logger.error( - `Ignoring Mod ${entry.name} as it contains the same name and ` + - `version as ${mods.get(key).filename}` + `Mod ${entry.name}'s filename does not match the expected ` + + `name ${modInfo.filename}, it will be ignored.` ); continue; } - mods.set(key, modInfo); + + mods.set(modInfo.filename, modInfo); } return mods; } @@ -454,15 +454,15 @@ class Master { } async deleteMod(name, version) { - let key = `${name}_${version}`; - let mod = this.mods.get(key); + let filename = `${name}_${version}.zip`; + let mod = this.mods.get(filename); if (!mod) { - throw new Error(`Mod ${key} does not exist`); + throw new Error(`Mod ${filename} does not exist`); } let modsDirectory = this.config.get("master.mods_directory"); - await fs.unlink(path.join(modsDirectory, mod.filename)); - this.mods.delete(key); + await fs.unlink(path.join(modsDirectory, filename)); + this.mods.delete(filename); mod.isDeleted = true; this.modUpdated(mod); } diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 0318bcea..e50d5caa 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -507,9 +507,8 @@ async function uploadMod(req, res) { await finished(writeStream); const modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); - modInfo.filename = filename; - await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, filename)); - req.app.locals.master.mods.set(`${modInfo.name}_${modInfo.version}`, modInfo); + await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename)); + req.app.locals.master.mods.set(modInfo.filename, modInfo); req.app.locals.master.modUpdated(modInfo); mods.push(modInfo.toJSON()); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index f1ecbb1c..2751ab87 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -915,7 +915,6 @@ describe("Integration of Clusterio", function() { "description: An empty mod for testing\n" + "factorio_version: 1.1\n" + "dependencies:\n" + - "filename: empty_mod_1.0.0.zip\n" + `size: ${stat.size}\n` + `sha1: ${hash}\n`, ); From 2384c48749de7c586d36771f35c8c80c929c9708 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 26 Sep 2022 16:54:19 +0200 Subject: [PATCH 20/40] Document and expose data members under lib/data Make members of lib/data exposed under lib/data instead of lib/data/. --- packages/lib/data/ModInfo.js | 2 +- packages/lib/data/ModPack.js | 12 ++++++------ packages/lib/data/index.js | 2 ++ packages/lib/data/version.js | 6 +++--- packages/lib/plugin.js | 4 ++-- packages/master/src/Master.js | 4 ++-- 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/packages/lib/data/ModInfo.js b/packages/lib/data/ModInfo.js index 4ad3873b..2c254935 100644 --- a/packages/lib/data/ModInfo.js +++ b/packages/lib/data/ModInfo.js @@ -15,7 +15,7 @@ const { integerModVersion, integerFactorioVersion, modVersionRegExp } = require( * * See https://wiki.factorio.com/Tutorial:Mod_structure#info.json for * details of the invidiual fields sourced from info.json. - * @alias module:lib/data/ModInfo + * @alias module:lib/data.ModInfo */ class ModInfo { /** diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 73bd8aab..50f21db1 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -8,7 +8,7 @@ const { integerFactorioVersion } = require("./version"); /** * A setting for a mod. - * @typedef {object} module:lib/data/ModPack~ModSetting + * @typedef {object} module:lib/data.ModPack~ModSetting * @property {boolean|number|string} value - Value of the given mod setting. */ const ModSettingJsonSchema = { @@ -27,10 +27,10 @@ const ModSettingsJsonSchema = { /** * A mod in a mod pack - * @typedef {object} module:lib/data/ModPack~ModRecord + * @typedef {object} module:lib/data.ModPack~ModRecord * @property {string} name - name of the mod. * @property {string} version - version of the mod. - * @property {string} sha1 - SHA1 hash of the zip file. + * @property {string=} sha1 - SHA1 hash of the zip file. */ const ModRecordJsonSchema = { type: "object", @@ -47,7 +47,7 @@ const ModRecordJsonSchema = { * Factorio Mod Pack * * Tracks mods and settings for a collection of Factorio mods. - * @alias module:lib/data/ModPack + * @alias module:lib/data.ModPack */ class ModPack { /** @@ -84,13 +84,13 @@ class ModPack { /** * Mods included in this mod pack - * @type {Map} + * @type {Map} */ mods = new Map(); /** * Mod settings for this mod pack - * @type {Object>=} + * @type {Object>=} */ settings = { "startup": new Map(), diff --git a/packages/lib/data/index.js b/packages/lib/data/index.js index 2b280cc2..cec20aed 100644 --- a/packages/lib/data/index.js +++ b/packages/lib/data/index.js @@ -6,8 +6,10 @@ "use strict"; const ModInfo = require("./ModInfo"); const ModPack = require("./ModPack"); +const version = require("./version"); module.exports = { ModInfo, ModPack, + ...version, }; diff --git a/packages/lib/data/version.js b/packages/lib/data/version.js index bda52be7..52588c26 100644 --- a/packages/lib/data/version.js +++ b/packages/lib/data/version.js @@ -9,7 +9,7 @@ * number of leading zeros in them. * @param {string} version - string representation of version. * @returns {number} numeric representation. - * @private + * @alias module:lib/data.integerModVersion */ function integerModVersion(version) { const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); @@ -23,7 +23,7 @@ function integerModVersion(version) { * number of leading zeros in them. * @param {string} version - string representation of version. * @returns {number} numeric representation. - * @private + * @alias module:lib/data.integerFactorioVersion */ function integerFactorioVersion(version) { const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); @@ -33,7 +33,7 @@ function integerFactorioVersion(version) { /** * Matches valid mod versions. * @type {RegExp} - * @private + * @alias module:lib/data.modVersionRegExp */ const modVersionRegExp = /^\d+\.\d+\.\d+$/; diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index 38a7675d..ac2d3ffb 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -502,7 +502,7 @@ class BaseMasterPlugin { * If the mod pack has been deleted its `.isDeleted` property will be * true. * - * @param {module:lib/data/ModPack} modPack - Mod pack that updated. + * @param {module:lib/data.ModPack} modPack - Mod pack that updated. */ async onModPackUpdated(modPack) { } @@ -514,7 +514,7 @@ class BaseMasterPlugin { * * If a mod has been deleted its `.isDeleted` property will be true. * - * @param {module:lib/data/ModInfo} mod - Mod that updated. + * @param {module:lib/data.ModInfo} mod - Mod that updated. */ async onModUpdated(mod) { } diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 6175ff46..2974f571 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -68,13 +68,13 @@ class Master { /** * Mapping of mod pack id to mod pack - * @type {Map} + * @type {Map} */ this.modPacks = null; /** * Mapping of mod names to mod infos - * @type {Map} + * @type {Map} */ this.mods = null; From f863aee3c28bc79b78215ab1aacf74e6cd61e6bc Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 9 Oct 2022 16:22:34 +0200 Subject: [PATCH 21/40] Add enabled field for mods in mod packs Make it possible to disable built in mods like base by providing an enabled field that can be set to false in mod packs. --- packages/ctl/src/commands.js | 20 ++++++++++++-- packages/lib/data/ModPack.js | 5 ++-- .../web_ui/src/components/ModPackViewPage.jsx | 26 ++++++++++++++++++- packages/web_ui/src/index.css | 4 +-- test/integration/clusterio.js | 2 +- test/lib/data/ModPack.js | 8 +++--- 6 files changed, 53 insertions(+), 12 deletions(-) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index f4a31510..6173e808 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -875,7 +875,8 @@ modPackCommands.add(new libCommand.Command({ if (field === "mods") { print(`${field}:`); for (let entry of value) { - print(` ${entry.name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`); + const enabled = entry.enabled ? "" : "(disabled) "; + print(` ${enabled}${entry.name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`); } } else if (field === "settings") { print(`${field}:`); @@ -958,7 +959,16 @@ function setModPackMods(modPack, mods) { if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) { throw new libErrors.CommandError("sha1 must be a 40 digit lower case hex string"); } - modPack.mods.set(name, { name, version, sha1 }); + modPack.mods.set(name, { name, enabled: true, version, sha1 }); + } +} + +function setModPackModsEnabled(modPack, mods, enabled) { + for (let mod of mods || []) { + if (!modPack.mods.has(mod)) { + throw new libErrors.CommandError(`Mod named ${mod} does not exist in the mod pack`); + } + modPack.mods.get(mod).enabled = enabled; } } @@ -969,6 +979,7 @@ modPackCommands.add(new libCommand.Command({ yargs.options({ "description": { describe: "Description for mod pack", type: "string" }, "mods": { describe: "Mods in the form of name:version[:sha1]", array: true, type: "string" }, + "disabled-mods": { describe: "Mods that are in the pack but not enabled", array: true, type: "string" }, "bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" }, "int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" }, "double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" }, @@ -986,6 +997,7 @@ modPackCommands.add(new libCommand.Command({ modPack.factorioVersion = args.factorioVersion; } setModPackMods(modPack, args.mods); + setModPackModsEnabled(modPack, args.disabledMods, false); setModPackSettings(modPack, args); await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); print(`Created mod pack ${modPack.name} (${modPack.id})`); @@ -1000,6 +1012,8 @@ modPackCommands.add(new libCommand.Command({ "description": { describe: "New description for mod pack", type: "string" }, "factorio-version": { describe: "Set version of factorio the mod pack is for", type: "string" }, "add-mods": { describe: "Mods in the form of name:version[:sha1] to add", array: true, type: "string" }, + "enable-mods": { describe: "Mods to set as enabled", array: true, type: "string" }, + "disable-mods": { describe: "Mods to set as disabled", array: true, type: "string" }, "remove-mods": { describe: "Name of mods to remove", array: true, type: "string" }, "bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" }, "int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" }, @@ -1031,6 +1045,8 @@ modPackCommands.add(new libCommand.Command({ logger.warn(`Mod ${name} did not exist on ${modPack.name}`); } } + setModPackModsEnabled(modPack, args.disableMods, false); + setModPackModsEnabled(modPack, args.enableMods, true); setModPackSettings(modPack, args); if (args.removeSetting) { diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 50f21db1..60d3c53c 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -29,15 +29,16 @@ const ModSettingsJsonSchema = { * A mod in a mod pack * @typedef {object} module:lib/data.ModPack~ModRecord * @property {string} name - name of the mod. + * @property {boolean} enabled - if mod is to be loaded. * @property {string} version - version of the mod. * @property {string=} sha1 - SHA1 hash of the zip file. */ const ModRecordJsonSchema = { type: "object", - additionalProperties: false, - required: ["name", "version"], + required: ["name", "enabled", "version"], properties: { "name": { type: "string" }, + "enabled": { type: "boolean" }, "version": { type: "string" }, "sha1": { type: "string" }, }, diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 17042616..39264fe1 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -95,7 +95,7 @@ function SearchModsTable(props) { props.onChange({ type: "mods.set", name: mod.name, - value: { name: mod.name, version: mod.version, sha1: mod.sha1 }, + value: { name: mod.name, enabled: true, version: mod.version, sha1: mod.sha1 }, }); }} >{props.modPack.mods.has(mod.name) ? "change" : "add"} @@ -255,6 +255,30 @@ function ModsTable(props) { }
mod.enabled).map(mod => mod.name), + getCheckboxProps(mod) { + return { disabled: deletedMods.has(mod.name) }; + }, + onChange(selectedRowKeys, selectedRows) { + function setEnabled(modNames, enabled) { + for (let name of modNames) { + props.onChange({ + type: "mods.set", + name, + value: { ...props.modPack.mods.get(name), enabled }, + }); + } + } + const added = selectedRowKeys.filter(key => !props.modPack.mods.get(key).enabled); + const removed = [...props.modPack.mods.values()].filter( + mod => mod.enabled && selectedRowKeys.indexOf(mod.name) === -1 + ).map(mod => mod.name); + setEnabled(added, true); + setEnabled(removed, false); + }, + }} columns={[ { title: "Name", diff --git a/packages/web_ui/src/index.css b/packages/web_ui/src/index.css index f4a3d45d..267f11bf 100644 --- a/packages/web_ui/src/index.css +++ b/packages/web_ui/src/index.css @@ -333,13 +333,13 @@ tr.no-expanded-padding > td, .ant-form-item.changed, input.ant-input.changed, textarea.ant-input.changed, -tr.ant-table-row.changed { +tr.ant-table-row.changed > td { background: #2a1912; } .ant-table-tbody > tr.ant-table-row.changed:hover > td, .ant-table-tbody > tr.changed > td.ant-table-cell-row-hover -tr.ant-table-row.deleted { +tr.ant-table-row.deleted > td { background: #3a2922; } diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 2751ab87..843248c3 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -845,7 +845,7 @@ describe("Integration of Clusterio", function() { reference.name = "full-pack"; reference.description = "Description"; reference.factorioVersion = "0.17"; - reference.mods.set("empty_mod", { name: "empty_mod", version: "1.0.0" }); + reference.mods.set("empty_mod", { name: "empty_mod", enabled: true, version: "1.0.0" }); reference.settings["startup"].set("MyBool", { value: true }); reference.settings["runtime-global"].set("MyInt", { value: 1235 }); reference.settings["runtime-global"].set("MyDouble", { value: 12.25 }); diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index fede2912..c26540dd 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -26,8 +26,8 @@ describe("lib/data/ModPack", function() { check(new ModPack({ description: "My Description" })); check(new ModPack({ factorio_version: "2.0" })); check(new ModPack({ mods: [ - { name: "subspace_storage", version: "1.99.8" }, - { name: "clusterio_lib", version: "0.1.2", sha1: "012345abcd" }, + { name: "subspace_storage", enabled: true, version: "1.99.8" }, + { name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" }, ]})); check(new ModPack({ settings: { "startup": { @@ -46,8 +46,8 @@ describe("lib/data/ModPack", function() { description: "Every option at once.", factorio_version: "2.0", mods: [ - { name: "subspace_storage", version: "1.99.8" }, - { name: "clusterio_lib", version: "0.1.2", sha1: "012345abcd" }, + { name: "subspace_storage", enabled: true, version: "1.99.8" }, + { name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" }, ], settings: { "startup": { From fafdac21fbf9189f15d895d3a627d8154975b1df Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sun, 9 Oct 2022 16:26:43 +0200 Subject: [PATCH 22/40] Follow mod pack string proposal Make all properties on the mod pack object except sha1 required and add the base mod if it does not exist in order to align the JSON representation with the proposed Mod Pack String format I published on the Factorio Forums (https://forums.factorio.com/103578) --- packages/lib/data/ModPack.js | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 60d3c53c..52c99d6a 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -108,6 +108,7 @@ class ModPack { static jsonSchema = { type: "object", additionalProperties: false, + required: ["name", "description", "factorio_version", "mods", "settings"], properties: { "id": { type: "integer" }, "name": { type: "string" }, @@ -151,23 +152,25 @@ class ModPack { }; } if (json.is_deleted) { this.isDeleted = json.is_deleted; } + + if (!this.mods.has("base")) { + this.mods.set("base", { name: "base", enabled: true, version: this.factorioVersion }); + } } toJSON() { let json = { id: this.id, - }; - if (this.name) { json.name = this.name; } - if (this.description) { json.description = this.description; } - json.factorio_version = this.factorioVersion; - if (this.mods.size) { json.mods = [...this.mods.values()]; } - if (this.settings) { - json.settings = { + name: this.name, + description: this.description, + factorio_version: this.factorioVersion, + mods: [...this.mods.values()], + settings: { "startup": Object.fromEntries(this.settings["startup"]), "runtime-global": Object.fromEntries(this.settings["runtime-global"]), "runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]), - }; - } + }, + }; if (this.isDeleted) { json.is_deleted = this.isDeleted; } return json; } From 53a5d7ea6ff1ae14e1ac861bdd3df32ac5790b2e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 11 Oct 2022 13:17:01 +0200 Subject: [PATCH 23/40] Add ModPack.toModSettingsDat Add utility function to convert the settings representation of a ModPack into the mod-settings.dat format which Factorio reads. --- packages/lib/data/ModPack.js | 97 ++++++++++++++++++++++++++++++++++++ test/lib/data/ModPack.js | 70 ++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 52c99d6a..0b58a0f2 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -208,6 +208,103 @@ class ModPack { return new this(json); } + + toModSettingsDat() { + function uint8Byte(int) { + return Buffer.from(Uint8Array.from([int]).buffer); + } + function int16Bytes(int) { + return Buffer.from(Int16Array.from([int]).buffer); + } + function uint32Bytes(int) { + return Buffer.from(Uint32Array.from([int]).buffer); + } + function uint32SpaceOptimizedBytes(int) { + if (0 <= int && int < 0xff) { + return uint8Byte(int); + } + return Buffer.concat([ + uint8Byte(0xff), + uint32Bytes(int), + ]); + } + function doubleBytes(double) { + return Buffer.from(Float64Array.from([double]).buffer); + } + function stringBytes(str) { + const buf = Buffer.from(str, "utf8"); + return Buffer.concat([ + uint32SpaceOptimizedBytes(buf.length), + buf, + ]); + } + function immutableStringBytes(str) { + if (str === undefined) { + return uint8Byte(1); // empty + } + return Buffer.concat([ + uint8Byte(0), // empty + stringBytes(str), + ]); + } + function versionBytes(version) { + const [main, major, minor] = version.split(".").map(n => Number.parseInt(n, 10)); + return Buffer.concat([ + int16Bytes(main), + int16Bytes(major), + int16Bytes(minor), + Buffer.alloc(2), // developer + Buffer.alloc(1), // reserved byte + ]); + } + function propertyTreeListBytes(entries) { + const sizeBytes = uint32Bytes(entries.length); + const itemBytes = entries.flatMap(([key, item]) => [ + immutableStringBytes(key), + // eslint-disable-next-line no-use-before-define + propertyTreeBytes(item), + ]); + return Buffer.concat([ + sizeBytes, + ...itemBytes, + ]); + } + function propertyTreeBytes(element) { + let type; + let dataBytes; + if (typeof element === null) { + type = 1; + dataBytes = Buffer.alloc(0); + } else if (typeof element === "boolean") { + type = 1; + dataBytes = uint8Byte(element); + } else if (typeof element === "number") { + type = 2; + dataBytes = doubleBytes(element); + } else if (typeof element === "string") { + type = 3; + dataBytes = immutableStringBytes(element); + } else if (element instanceof Array) { + type = 4; + dataBytes = propertyTreeListBytes(element.map(item => [undefined, item])); + } else if (typeof element === "object") { + type = 5; + dataBytes = propertyTreeListBytes(Object.entries(element)); + } else { + throw new Error("Bad element passed to propertyTreeBytes"); + } + return Buffer.concat([ + uint8Byte(type), + Buffer.alloc(1), // anyTypeFlag + dataBytes, + ]); + } + + return Buffer.concat([ + versionBytes(this.factorioVersion), + propertyTreeBytes(this.toJSON().settings), + ]); + } } module.exports = ModPack; diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index c26540dd..8a6fed3e 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -90,5 +90,75 @@ describe("lib/data/ModPack", function() { ); }); }); + + describe("toModSettingsDat()", function() { + it("should properly serialise settings", function() { + const pack = new ModPack({ settings: { + "startup": { + "bool-setting": { "value": true }, + }, + "runtime-global": { + "number-setting": { "value": 123 }, + }, + "runtime-per-user": { + "string-setting": { "value": "a string" }, + }, + }}); + + function istr(str) { + return [ + Uint8Array.from([0, str.length]), + Buffer.from(str), + ]; + } + /* eslint-disable indent */ + assert.deepEqual( + pack.toModSettingsDat(), + Buffer.concat([ + new Uint8Array(Uint16Array.from([1, 1, 0, 0]).buffer), // version + Uint8Array.from([0]), // reserved + + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([3]).buffer), // items + + ...istr("startup"), // name + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("bool-setting"), // name + Uint8Array.from([5, 0]), // dictinary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("value"), // name + Uint8Array.from([1, 0, 1]), // boolean + + ...istr("runtime-global"), // name + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("number-setting"), // name + Uint8Array.from([5, 0]), // dictinary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("value"), // name + Uint8Array.from([2, 0]), // number + new Uint8Array(Float64Array.from([123]).buffer), + + ...istr("runtime-per-user"), // name + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("string-setting"), // name + Uint8Array.from([5, 0]), // dictinary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("value"), // name + Uint8Array.from([3, 0, 0, "a string".length]), // string + Buffer.from("a string"), + ]) + ); + /* eslint-enable indent */ + }); + }); }); }); From 660d0f08a13eb82fe55e3206c67bcc9c9e529b17 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 11 Oct 2022 13:20:34 +0200 Subject: [PATCH 24/40] Fully specify mod pack Factorio version Remove support for major only versions of Factorio being specified in mod packs. --- packages/ctl/src/commands.js | 8 ++++---- packages/web_ui/src/components/ModPackViewPage.jsx | 4 ++-- test/integration/clusterio.js | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index 6173e808..354a6460 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -991,8 +991,8 @@ modPackCommands.add(new libCommand.Command({ modPack.name = args.name; if (args.description) { modPack.description = args.description; } if (args.factorioVersion) { - if (!/^\d+\.\d+(\.\d+)?$/.test(args.factorioVersion)) { - throw new libErrors.CommandError("factorio-version must match the format digit.digit[.digit]"); + if (!/^\d+\.\d+\.\d+?$/.test(args.factorioVersion)) { + throw new libErrors.CommandError("factorio-version must match the format digit.digit.digit"); } modPack.factorioVersion = args.factorioVersion; } @@ -1031,8 +1031,8 @@ modPackCommands.add(new libCommand.Command({ if (args.name) { modPack.name = args.name; } if (args.description) { modPack.description = args.description; } if (args.factorioVersion) { - if (!/^\d+\.\d+(\.\d+)?$/.test(args.factorioVersion)) { - throw new libErrors.CommandError("factorio-version must match the format digit.digit[.digit]"); + if (!/^\d+\.\d+\.\d+?$/.test(args.factorioVersion)) { + throw new libErrors.CommandError("factorio-version must match the format digit.digit.digit"); } modPack.factorioVersion = args.factorioVersion; } diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 39264fe1..115560a3 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -620,8 +620,8 @@ export default function ModPackViewPage() { name="factorioVersion" style={{ marginBottom: 0 }} rules={[{ - pattern: /^\d+\.\d+(\.\d+)?$/, - message: "Must be an a.b or a.b.c version number..", + pattern: /^\d+\.\d+\.\d+?$/, + message: "Must be an a.b.c version number.", }]} > modPack.name === "empty-pack"), "created pack is not in the list"); }); it("should allow setting all fields", async function() { await execCtl( - "mod-pack create full-pack 0.17 " + + "mod-pack create full-pack 0.17.59 " + "--description Description " + "--mods empty_mod:1.0.0 " + "--bool-setting startup MyBool true " + @@ -844,7 +844,7 @@ describe("Integration of Clusterio", function() { reference.id = modPack.id; reference.name = "full-pack"; reference.description = "Description"; - reference.factorioVersion = "0.17"; + reference.factorioVersion = "0.17.59"; reference.mods.set("empty_mod", { name: "empty_mod", enabled: true, version: "1.0.0" }); reference.settings["startup"].set("MyBool", { value: true }); reference.settings["runtime-global"].set("MyInt", { value: 1235 }); @@ -868,11 +868,11 @@ describe("Integration of Clusterio", function() { describe("mod-pack edit", function() { it("runs", async function() { - await execCtl("mod-pack edit full-pack --factorio-version 1.2"); + await execCtl("mod-pack edit full-pack --factorio-version 1.2.0"); let response = await libLink.messages.listModPacks.send(getControl()); let modPack = response.list.find(entry => entry.name === "full-pack"); assert(modPack, "created mod pack not found"); - assert.equal(modPack.factorio_version, "1.2"); + assert.equal(modPack.factorio_version, "1.2.0"); }); }); From 20e620993bdccee1a63ae0ddeb0dd922a062633c Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 11 Oct 2022 13:25:30 +0200 Subject: [PATCH 25/40] Use Factorio's version component names Minor nitpick: The coponent names of the Factorio version is called main, major, minor. Change integerFactorioVersion to match. --- packages/lib/data/version.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lib/data/version.js b/packages/lib/data/version.js index 52588c26..32706c04 100644 --- a/packages/lib/data/version.js +++ b/packages/lib/data/version.js @@ -26,8 +26,8 @@ function integerModVersion(version) { * @alias module:lib/data.integerFactorioVersion */ function integerFactorioVersion(version) { - const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); - return major * 0x100000000 + minor * 0x10000 + (sub || 0); // Can't use bitwise here because this is 48-bits. + const [main, major, minor] = version.split(".").map(n => Number.parseInt(n, 10)); + return main * 0x100000000 + major * 0x10000 + (minor || 0); // Can't use bitwise here because this is 48-bits. } /** From 2ae8270a1dc6b3027b41f2cbb8cab93430ce4495 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 15 Oct 2022 17:42:06 +0200 Subject: [PATCH 26/40] Don't hypenate mod-pack --- packages/ctl/src/commands.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index 354a6460..70644707 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -973,7 +973,7 @@ function setModPackModsEnabled(modPack, mods, enabled) { } modPackCommands.add(new libCommand.Command({ - definition: ["create ", "Create mod-pack", (yargs) => { + definition: ["create ", "Create mod pack", (yargs) => { yargs.positional("name", { describe: "Name of mod pack to create", type: "string" }); yargs.positional("factorio-version", { describe: "Version of factorio the mod pack is for", type: "string" }); yargs.options({ @@ -1005,7 +1005,7 @@ modPackCommands.add(new libCommand.Command({ })); modPackCommands.add(new libCommand.Command({ - definition: ["edit ", "Edit mod-pack", (yargs) => { + definition: ["edit ", "Edit mod pack", (yargs) => { yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" }); yargs.options({ "name": { describe: "New name for mod pack", type: "string" }, @@ -1069,7 +1069,7 @@ modPackCommands.add(new libCommand.Command({ })); modPackCommands.add(new libCommand.Command({ - definition: ["delete ", "Delete mod-pack", (yargs) => { + definition: ["delete ", "Delete mod pack", (yargs) => { yargs.positional("mod-pack", { describe: "Mod pack to delete", type: "string" }); }], handler: async function(args, control) { From 0ebb2f58583c986f9bd30b6cc2a4433f46576080 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 15 Oct 2022 17:50:12 +0200 Subject: [PATCH 27/40] Add import and export mod-pack commands --- packages/ctl/src/commands.js | 24 ++++++++++++++++++++++++ test/integration/clusterio.js | 19 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index 70644707..c5f3fe9d 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -1004,6 +1004,30 @@ modPackCommands.add(new libCommand.Command({ }, })); +modPackCommands.add(new libCommand.Command({ + definition: ["import ", "Import mod pack string", (yargs) => { + yargs.positional("string", { describe: "Mod pack string to import", type: "string" }); + }], + handler: async function(args, control) { + const modPack = libData.ModPack.fromModPackString(args.string); + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + print(`Created mod pack ${modPack.name} (${modPack.id})`); + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: ["export ", "Export mod pack string", (yargs) => { + yargs.positional("string", { describe: "Mod pack to export", type: "string" }); + }], + handler: async function(args, control) { + const response = await libLink.messages.getModPack.send(control, { + id: await libCommand.resolveModPack(control, args.modPack), + }); + const modPack = new libData.ModPack(response.mod_pack); + print(modPack.toModPackString()); + }, +})); + modPackCommands.add(new libCommand.Command({ definition: ["edit ", "Edit mod pack", (yargs) => { yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" }); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index b54d9390..e5da8157 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -866,6 +866,25 @@ describe("Integration of Clusterio", function() { }); }); + describe("mod-pack import/export", function() { + it("should should roundtrip a mod-pack", async function() { + let reference = new libData.ModPack(); + reference.name = "imported-pack"; + reference.description = "Description"; + reference.factorioVersion = "0.17.59"; + reference.mods.set("empty_mod", { name: "empty_mod", enabled: true, version: "1.0.0" }); + reference.settings["startup"].set("MyBool", { value: true }); + reference.settings["runtime-global"].set("MyInt", { value: 1235 }); + reference.settings["runtime-global"].set("MyDouble", { value: 12.25 }); + reference.settings["runtime-per-user"].set("MyString", { value: "a-string" }); + await execCtl(`mod-pack import ${reference.toModPackString()}`); + const result = await execCtl("mod-pack export imported-pack"); + const roundtrip = libData.ModPack.fromModPackString(result.stdout.trim()); + roundtrip.id = reference.id; + assert.deepEqual(roundtrip, reference); + }); + }); + describe("mod-pack edit", function() { it("runs", async function() { await execCtl("mod-pack edit full-pack --factorio-version 1.2.0"); From fb4ffaffed772c2983cb2d75b5d71c6f179f9a15 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 15 Oct 2022 17:53:22 +0200 Subject: [PATCH 28/40] Implement mod pack loading for shared mods Wipe and reconstruct the instances' mods/ folder using the configured ModPack for the case were the mods folder is shared with the master server. This assumes the master server will check the mod checksums before sending the start request. --- .dockerignore | 3 +- .github/workflows/node.js.yml | 4 +- .gitignore | 1 - Dockerfile | 8 +- packages/lib/config/definitions.js | 20 +++ packages/master/src/BaseConnection.js | 9 ++ packages/master/src/ControlConnection.js | 9 -- packages/master/src/SlaveConnection.js | 5 - packages/slave/slave.js | 2 +- packages/slave/src/Instance.js | 149 +++++++++--------- .../web_ui/src/components/ModPackViewPage.jsx | 2 + test/integration/clusterio.js | 1 + test/integration/index.js | 9 ++ test/slave.js | 67 -------- 14 files changed, 124 insertions(+), 165 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2ad21c71..74f895db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,8 +13,7 @@ factorio factorio.tar.gz # We build internal mods during build, mods from local test env should not be applied -sharedMods -sharedModsDisabled +mods # These files should never be required during the build .git diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index ee732740..c3a3923d 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -26,8 +26,8 @@ jobs: node-version: ${{ matrix.node-version }} - run: pnpm i --no-optional --no-frozen-lockfile - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz - - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods - - run: wget -q -O temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip + - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/mods + - run: wget -q -O temp/test/mods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip - name: Run tests if: ${{ matrix.node-version != '12.x' }} run: pnpm test diff --git a/.gitignore b/.gitignore index a769fb6c..37bf1a49 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,6 @@ node_modules Factorio* /factorio /instances -/sharedMods /mods /database /plugin-list.json diff --git a/Dockerfile b/Dockerfile index 71008b26..3eb36c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,15 +22,15 @@ RUN pnpm install #RUN pnpm install @clusterio/plugin-subspace_storage #RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage -COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/sharedMods/ +COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/mods/ # Build Lua library mod RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \ -&& mv dist/* sharedMods/ \ +&& mv dist/* mods/ \ && mkdir temp \ && mkdir temp/test \ -&& cp sharedMods/ temp/test/ -r \ -&& ls sharedMods +&& cp mods/ temp/test/ -r \ +&& ls mods # Remove node_modules RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \; diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index 01d74f78..b0926270 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -160,6 +160,20 @@ SlaveGroup.define({ type: "string", initial_value: "factorio", }); +SlaveGroup.define({ + name: "mods_directory", + title: "Mods Directory", + description: "Path to directory where mods for instances are cached.", + type: "string", + initial_value: "mods", +}); +SlaveGroup.define({ + name: "mods_directory_is_shared", + title: "Mods Directory is Shared", + description: "True if the mods directory is shared with the master server.", + type: "boolean", + initial_value: true, +}); SlaveGroup.define({ name: "instances_directory", description: "Path to directory to store instances in.", @@ -280,6 +294,12 @@ FactorioGroup.define({ type: "number", initial_value: 5, }); +FactorioGroup.define({ + name: "mod_pack", + description: "Mod pack to use on this server", + type: "number", + optional: true, +}); FactorioGroup.define({ name: "enable_save_patching", description: "Patch saves with Lua code. Required for Clusterio integrations, lua modules, and most plugins.", diff --git a/packages/master/src/BaseConnection.js b/packages/master/src/BaseConnection.js index e7bca6fa..f22094cf 100644 --- a/packages/master/src/BaseConnection.js +++ b/packages/master/src/BaseConnection.js @@ -84,6 +84,15 @@ class BaseConnection extends libLink.Link { get connected() { return !this._disconnecting && this.connector.connected; } + + async getModPackRequestHandler(message) { + let { id } = message.data; + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } } module.exports = BaseConnection; diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index 48834a90..9856b33b 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -474,15 +474,6 @@ class ControlConnection extends BaseConnection { return { save: result.save }; } - async getModPackRequestHandler(message) { - let { id } = message.data; - let modPack = this._master.modPacks.get(id); - if (!modPack) { - throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); - } - return { mod_pack: modPack.toJSON() }; - } - async listModPacksRequestHandler(message) { return { list: [...this._master.modPacks.values()].map(pack => pack.toJSON()) }; } diff --git a/packages/master/src/SlaveConnection.js b/packages/master/src/SlaveConnection.js index 4f9203ed..35564264 100644 --- a/packages/master/src/SlaveConnection.js +++ b/packages/master/src/SlaveConnection.js @@ -3,7 +3,6 @@ const libConfig = require("@clusterio/lib/config"); const libLink = require("@clusterio/lib/link"); const { logger } = require("@clusterio/lib/logging"); const libPlugin = require("@clusterio/lib/plugin"); -const libErrors = require("@clusterio/lib/errors"); const PlayerStats = require("@clusterio/lib/PlayerStats"); const BaseConnection = require("./BaseConnection"); @@ -168,10 +167,6 @@ class SlaveConnection extends BaseConnection { this._master.saveListUpdate(message.data); } - async getModPackRequestHandler(message) { - throw new libErrors.RequestError("Not implemented"); - } - async logMessageEventHandler(message) { this._master.clusterLogger.log({ ...message.data.info, diff --git a/packages/slave/slave.js b/packages/slave/slave.js index 9c4fb8c6..8f9f70e7 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -177,7 +177,7 @@ async function startSlave() { // If we get here the command was run await fs.ensureDir(slaveConfig.get("slave.instances_directory")); - await fs.ensureDir("sharedMods"); + await fs.ensureDir(slaveConfig.get("slave.mods_directory")); await fs.ensureDir("modules"); // Set the process title, shows up as the title of the CMD window on windows diff --git a/packages/slave/src/Instance.js b/packages/slave/src/Instance.js index a5a717a6..2148206e 100644 --- a/packages/slave/src/Instance.js +++ b/packages/slave/src/Instance.js @@ -6,6 +6,7 @@ const phin = require("phin"); const util = require("util"); // internal libraries +const libData = require("@clusterio/lib/data"); const libFileOps = require("@clusterio/lib/file_ops"); const libFactorio = require("@clusterio/lib/factorio"); const libLink = require("@clusterio/lib/link"); @@ -94,71 +95,6 @@ const serverSettingsActions = { }, }; -/** - * Create and update symlinks for shared mods in an instance - * - * Creates symlinks for .zip and .dat files that are not present in the - * instance mods directory but is present in the sharedMods directory, - * and removes any symlinks that don't point to a file in the instance - * mods directory. If the instance mods directory doesn't exist it will - * be created. - * - * Note that on Windows this creates hard links instead of symbolic - * links as the latter requires elevated privileges. This unfortunately - * means the removal of mods from the shared mods dir can't be detected. - * - * @param {Instance} instance - Instance to link mods for - * @param {string} sharedMods - Path to folder to link mods from. - * @private - */ -async function symlinkMods(instance, sharedMods) { - await fs.ensureDir(instance.path("mods")); - - // Remove broken symlinks in instance mods. - for (let entry of await fs.readdir(instance.path("mods"), { withFileTypes: true })) { - if (entry.isSymbolicLink()) { - if (!await fs.pathExists(instance.path("mods", entry.name))) { - instance.logger.verbose(`Removing broken symlink ${entry.name}`); - await fs.unlink(instance.path("mods", entry.name)); - } - } - } - - // Link entries that are in sharedMods but not in instance mods. - let instanceModsEntries = new Set(await fs.readdir(instance.path("mods"))); - for (let entry of await fs.readdir(sharedMods, { withFileTypes: true })) { - if (entry.isFile()) { - if ([".zip", ".dat", ".json"].includes(path.extname(entry.name))) { - if (!instanceModsEntries.has(entry.name)) { - instance.logger.verbose(`linking ${entry.name} from ${sharedMods}`); - let target = path.join(sharedMods, entry.name); - let link = instance.path("mods", entry.name); - - /* eslint-disable max-depth */ - if (process.platform !== "win32") { - await fs.symlink(path.relative(path.dirname(link), target), link); - - // On Windows symlinks require elevated privileges, which is - // not something we want to have. For this reason the mods - // are hard linked instead. This has the drawback of not - // being able to identify when mods are removed from the - // sharedMods directory, or which mods are linked. - } else { - await fs.link(target, link); - } - /* eslint-enable max-depth */ - } - - } else { - instance.logger.warn(`Warning: ignoring file '${entry.name}' in sharedMods`); - } - - } else { - instance.logger.warn(`Warning: ignoring non-file '${entry.name}' in sharedMods`); - } - } -} - /** * Keeps track of the runtime parameters of an instance @@ -643,6 +579,77 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await fs.ensureDir(path.join(instanceDir, "saves")); } + /** + * Sync instance mods directory with configured mod pack + * + * Adds, deletes, and updates all files in the instance mods folder to + * match up with the mod pack that's configured. If no mod pack is + * configured then an empty mod pack is used, esentially turning it into + * a vanilla mods folder. If the instance mods directory does't exist + * then it will be created. + * + * On Linux this creates symlinks to mods in the slave's mods folder, on + * Windows hard links are used instead due to symlinks being privileged. + */ + async syncMods() { + const modPackId = this.config.get("factorio.mod_pack"); + let modPack; + if (modPackId === null) { + modPack = new libData.ModPack(); + } else { + const response = await libLink.messages.getModPack.send(this, { id: modPackId }); + modPack = new libData.ModPack(response.mod_pack); + } + + // TODO validate factorioVersion + + if (!this._slave.config.get("slave.mods_directory_is_shared")) { + throw new Error("Fetching mods is not implemented"); + } + + await fs.ensureDir(this.path("mods")); + + // Remove all files + for (let entry of await fs.readdir(this.path("mods"), { withFileTypes: true })) { + if (entry.isDirectory()) { + this.logger.warn( + `Found unexpected directory ${entry.name} in mods folder, it may break Clusterio's mod syncing` + ); + continue; + } + + if (entry.isFile() || entry.isSymbolicLink()) { + await fs.unlink(this.path("mods", entry.name)); + } + } + + // Add mods from mod the pack + const modsDir = this._slave.config.get("slave.mods_directory"); + for (let mod of modPack.mods.values()) { + const modFile = `${mod.name}_${mod.version}.zip`; + const target = path.join(modsDir, modFile); + const link = this.path("mods", modFile); + + if (process.platform !== "win32") { + await fs.symlink(path.relative(path.dirname(link), target), link); + + // On Windows symlinks require elevated privileges, which is + // not something we want to have. For this reason the mods + // are hard linked instead. + } else { + await fs.link(target, link); + } + } + + // Write mod-list.json + await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({ + mods: [{ name: "base" }, ...modPack.mods.values()].map(m => ({ name: m.name, enabled: true })), + }, null, 2)); + + // Write mod-settings.dat + await fs.outputFile(this.path("mods", "mod-settings.dat"), modPack.toModSettingsDat()); + } + /** * Prepare instance for starting * @@ -678,8 +685,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); ); } - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } /** @@ -1030,8 +1036,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.verbose("Creating save ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } catch (err) { this.notifyExit(); @@ -1053,8 +1058,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.info("Exporting data ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); let zip = await libFactorio.exportData(this.server); let content = await zip.generateAsync({ type: "nodebuffer" }); @@ -1116,6 +1120,3 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); } module.exports = Instance; - -// For testing only -module.exports._symlinkMods = symlinkMods; diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 115560a3..dfce060d 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -368,9 +368,11 @@ function SettingsTable(props) { {controls("startup", pickValue(props.modPack.settings["startup"]))} Map + These only apply for new saves. {controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))} Per Player + Default settings for new players, these only apply for new saves. {controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))} ; } diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index e5da8157..6ab91fa9 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -265,6 +265,7 @@ describe("Integration of Clusterio", function() { ); await execCtl(`instance config set-prop test factorio.settings visibility "'${value}'"`); await execCtl("instance config set-prop test factorio.settings require_user_verification false"); + await execCtl("instance config set test factorio.mod_pack 12"); }); }); diff --git a/test/integration/index.js b/test/integration/index.js index 2830e39f..b604bb1f 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -8,6 +8,7 @@ const phin = require("phin"); const util = require("util"); const events = require("events"); +const libData = require("@clusterio/lib/data"); const libLink = require("@clusterio/lib/link"); const { LineSplitter } = require("@clusterio/lib/stream"); const { ConsoleTransport, logger } = require("@clusterio/lib/logging"); @@ -220,6 +221,14 @@ before(async function() { controlConnector.token = controlToken; control = new TestControl(controlConnector); await controlConnector.connect(); + + const testPack = new libData.ModPack(); + testPack.id = 12; + testPack.name = "subspace_storage-pack"; + testPack.factorioVersion = "1.1.0"; + testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); + testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); + libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); }); after(async function() { diff --git a/test/slave.js b/test/slave.js index 0cb23498..d8d34788 100644 --- a/test/slave.js +++ b/test/slave.js @@ -1,11 +1,8 @@ "use strict"; const assert = require("assert").strict; const path = require("path"); -const fs = require("fs-extra"); -const libLink = require("@clusterio/lib/link"); const libConfig = require("@clusterio/lib/config"); -const Instance = require("@clusterio/slave/src/Instance"); const Slave = require("@clusterio/slave/src/Slave"); describe("Slave testing", function() { @@ -13,70 +10,6 @@ describe("Slave testing", function() { libConfig.InstanceConfig.finalize(); }); - describe("symlinkMods()", function() { - let testDir = path.join("temp", "test", "symlink"); - - let instance; - before(async function() { - // Remove previous test output - await fs.remove(testDir); - - // Add some test mods - await fs.outputFile(path.join(testDir, "shared", "mod_a.zip"), "a"); - await fs.outputFile(path.join(testDir, "shared", "mod_b.zip"), "b"); - await fs.outputFile(path.join(testDir, "shared", "mod.dat"), "c"); - await fs.outputFile(path.join(testDir, "shared", "mod-list.json"), "d"); - - let instanceConfig = new libConfig.InstanceConfig("slave"); - await instanceConfig.init(); - instanceConfig.set("instance.name", "test"); - instance = new Instance( - {}, new libLink.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig - ); - await fs.outputFile(instance.path("mods", "mod_i.zip"), "i"); - }); - - it("should link mods and data files", async function() { - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert.equal(await fs.readFile(instance.path("mods", "mod_a.zip"), "utf-8"), "a"); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - - it("should ignore directories", async function() { - await fs.ensureDir(path.join(testDir, "shared", "dir")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "dir"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should ignore files", async function() { - await fs.outputFile(path.join(testDir, "shared", "file"), "f"); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "file"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should unlink removed mods", async function() { - // This does not work on Windows - if (process.platform === "win32") { - this.skip(); - } - - await fs.unlink(path.join(testDir, "shared", "mod_a.zip")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - await assert.rejects(fs.lstat(instance.path("mods", "mod_a.zip")), { code: "ENOENT" }); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - }); - describe("discoverInstances()", function() { it("should discover test instance", async function() { let instancePath = path.join("test", "file", "instances"); From dfadc627209750040c3dbba771258b875d48a276 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 15 Oct 2022 17:56:50 +0200 Subject: [PATCH 29/40] Prety print database/mod-packs.json --- packages/master/src/Master.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 2974f571..edb1288c 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -396,7 +396,7 @@ class Master { } static async saveModPacks(filePath, modPacks) { - await libFileOps.safeOutputFile(filePath, JSON.stringify([...modPacks.values()])); + await libFileOps.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, 4)); } static async loadModInfos(modsDirectory) { From 32cf9321ccd4a69500586c9a31e86ef897bdeb54 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 15 Oct 2022 17:57:28 +0200 Subject: [PATCH 30/40] Make sure checksum matches in ModsTable Validate sha1 sum matches for a mod used in ModsTable if it is present. This prevents showing details for a zip file where the version matches but the cheksum does not. --- packages/web_ui/src/components/ModPackViewPage.jsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index dfce060d..981b4881 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -211,7 +211,13 @@ function ModsTable(props) { const modListMap = new Map(modList.map(mod => [`${mod.name}_${mod.version}`, mod])); const mods = [...props.modPack.mods.values(), ...deletedMods.values()].map( - mod => modListMap.get(`${mod.name}_${mod.version}`) || mod + mod => { + const candidate = modListMap.get(`${mod.name}_${mod.version}`); + if (!candidate || mod.sha1 && candidate.sha1 !== mod.sha1) { + return mod; + } + return candidate; + } ); function actions(mod) { From e6add2f7e606842413322953f9f37f91e8e13df8 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 15 Oct 2022 17:59:38 +0200 Subject: [PATCH 31/40] Fix handling of numeric settings in ModPackViewPage Due to a mistake in the logic numeric settings would end up getting converted to string settings when edited. --- packages/web_ui/src/components/ModPackViewPage.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 981b4881..4507f534 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -497,13 +497,13 @@ export default function ModPackViewPage() { } else if (["startup", "runtime-global", "runtime-per-user"].includes(field.name[0])) { let [scope, settingName] = field.name; let value = field.value; - if (typeof modPack.settings[scope].get(settingName) === "number") { + if (typeof modPack.settings[scope].get(settingName).value === "number") { value = Number(value); if (Number.isNaN(value)) { continue; } } - if (modifiedModPack.settings[scope].get(settingName) !== value) { + if (modifiedModPack.settings[scope].get(settingName).value !== value) { pushChange({ type: "settings.set", scope, name: settingName, value: { value }}); } } From fb75e47eda341b2c723a922acb8d74b3a5118b3e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 10 Nov 2022 00:25:46 +0100 Subject: [PATCH 32/40] Mark missing or missmatched checksum for mods in mod packs Mark mods in the UI and the mod-pack show ctl command that are missing with a question mark and mark mods with a missmatched checksum with an exclamation mark. --- packages/ctl/src/commands.js | 9 ++++++- packages/web_ui/src/components/ModDetails.jsx | 7 ++++++ .../web_ui/src/components/ModPackViewPage.jsx | 24 ++++++++++++++++--- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index c5f3fe9d..a99e9eb7 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -867,6 +867,9 @@ modPackCommands.add(new libCommand.Command({ yargs.positional("mod-pack", { describe: "Mod pack to show", type: "string" }); }], handler: async function(args, control) { + let mods = new Map( + (await libLink.messages.listMods.send(control)).list.map(m => [`${m.name}_${m.version}`, m]) + ); let response = await libLink.messages.getModPack.send(control, { id: await libCommand.resolveModPack(control, args.modPack), }); @@ -875,8 +878,12 @@ modPackCommands.add(new libCommand.Command({ if (field === "mods") { print(`${field}:`); for (let entry of value) { + const mod = mods.get(`${entry.name}_${entry.version}`); + const missing = !mod && "? "; + const badChecksum = mod && entry.sha1 && entry.sha1 !== mod.sha1 && "! "; + const warn = missing || badChecksum || ""; const enabled = entry.enabled ? "" : "(disabled) "; - print(` ${enabled}${entry.name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`); + print(` ${warn}${enabled}${entry.name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`); } } else if (field === "settings") { print(`${field}:`); diff --git a/packages/web_ui/src/components/ModDetails.jsx b/packages/web_ui/src/components/ModDetails.jsx index af5ebd78..19487421 100644 --- a/packages/web_ui/src/components/ModDetails.jsx +++ b/packages/web_ui/src/components/ModDetails.jsx @@ -1,5 +1,6 @@ import React, { Fragment } from "react"; import { Descriptions, Grid, Typography } from "antd"; +import ExclamationCircleOutlined from "@ant-design/icons/ExclamationCircleOutlined"; import { libHelpers } from "@clusterio/lib"; @@ -11,6 +12,12 @@ export default function ModDetails(props) { const mod = props.mod; return + {mod.error && Error} span={2}> + {mod.error === "missing" && "The given version of this mod is missing from the master storage."} + {mod.error === "bad_checksum" + && "Checksum of the mod in the master storage does not match the checksum provided by this mod pack." + } + } {props.actions && !screens.lg && {props.actions(mod)} } diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 4507f534..0d2a426f 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -6,6 +6,8 @@ import { } from "antd"; import InfoCircleOutlined from "@ant-design/icons/InfoCircleOutlined"; import ExportOutlined from "@ant-design/icons/ExportOutlined"; +import FileUnknownOutlined from "@ant-design/icons/FileUnknownOutlined"; +import FileExclamationOutlined from "@ant-design/icons/FileExclamationOutlined"; import CloseOutlined from "@ant-design/icons/CloseOutlined"; import DeleteOutlined from "@ant-design/icons/DeleteOutlined"; import PlusOutlined from "@ant-design/icons/PlusOutlined"; @@ -213,8 +215,16 @@ function ModsTable(props) { const mods = [...props.modPack.mods.values(), ...deletedMods.values()].map( mod => { const candidate = modListMap.get(`${mod.name}_${mod.version}`); - if (!candidate || mod.sha1 && candidate.sha1 !== mod.sha1) { - return mod; + if (!candidate) { + return { + ...mod, + error: "missing", + }; + } else if (mod.sha1 && candidate.sha1 !== mod.sha1) { + return { + ...mod, + error: "bad_checksum", + }; } return candidate; } @@ -289,7 +299,15 @@ function ModsTable(props) { { title: "Name", key: "name", - render: mod => mod.title || mod.name, + render: mod => <> + {mod.error === "missing" && + {" "} + } + {mod.error === "bad_checksum" && + {" "} + } + {mod.title || mod.name} + , defaultSortOrder: "ascend", sorter: (a, b) => strcmp(a.name, b.name), }, From 7c6aaeb05de5980a62288805aa9df0fcead597a6 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 30 Jan 2023 04:27:49 +0100 Subject: [PATCH 33/40] Export settings and all prototypes Add settings and all prototypes to the export of data done. Settings are needed for a settings GUI to function, while prototype data would be useful to display any sort of information about things from the game. --- packages/lib/factorio/export.js | 66 ++++++++++- packages/master/src/routes.js | 2 + packages/slave/lua/export/clusterio_api.lua | 73 ++++++++++++ packages/slave/lua/export/control.lua | 3 + .../slave/lua/export/data-final-fixes.lua | 106 +----------------- .../slave/lua/export/settings-final-fixes.lua | 8 ++ test/integration/clusterio.js | 13 ++- 7 files changed, 161 insertions(+), 110 deletions(-) create mode 100644 packages/slave/lua/export/clusterio_api.lua create mode 100644 packages/slave/lua/export/settings-final-fixes.lua diff --git a/packages/lib/factorio/export.js b/packages/lib/factorio/export.js index 173f5cb3..b4eec039 100644 --- a/packages/lib/factorio/export.js +++ b/packages/lib/factorio/export.js @@ -207,6 +207,42 @@ async function loadLayeredIcon(server, modVersions, item, size, iconCache) { return icon; } +const itemTypes = new Set([ + "item", + "ammo", + "capsule", + "gun", + "item-with-entity-data", + "item-with-label", + "item-with-inventory", + "blueprint-book", + "item-with-tags", + "selection-tool", + "blueprint", + "copy-paste-tool", + "deconstruction-item", + "upgrade-item", + "module", + "rail-planner", + "spidertron-remote", + "tool", + "armor", + "mining-tool", + "repair-tool", + + // XXX Bad hack to get icons for fluids. These should be treated as a + // separate namespace from items as they may be named the same as items. + "fluid", + // TODO consider adding virtual signals and recipes +]); + +function filterItems(prototypes) { + return Object.entries(prototypes) + .filter(([type, _]) => itemTypes.has(type)) + .flatMap(([_, typePrototypes]) => Object.values(typePrototypes)) + ; +} + /** * Export item icons and data * @@ -348,11 +384,29 @@ async function exportLocale(server, modVersions, modOrder, languageCode) { async function exportData(server) { await generateExportMod(server); - let items = []; + let settings = {}; + let prototypes = {}; let modVersions = new Map(); let modOrder = []; - server.on("ipc-item_export", data => items.push(data)); + function add(obj, prototype) { + if (!Object.prototype.hasOwnProperty.call(obj, prototype.type)) { + obj[prototype.type] = {}; + } + obj[prototype.type][prototype.name] = prototype; + } + + server.on("ipc-prototype_export", data => add(prototypes, data)); + server.on("ipc-settings_export", data => add(settings, data)); + server.on("ipc-mod_setting_mod", ({ name, mod }) => { + for (let type of Object.values(settings)) { + if (Object.prototype.hasOwnProperty.call(type, name)) { + type[name].mod = mod; + return; + } + } + server._logger.error(`Unable to find ${name} in settings prototypes`); + }); server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); }); server.on("output", parsed => { if (parsed.format === "seconds" && parsed.type === "generic") { @@ -370,8 +424,8 @@ async function exportData(server) { await fs.unlink(server.writePath("mods", "export_0.0.0.zip")); } - if (!items.length) { - throw new Error("No items got exported"); + if (!Object.keys(prototypes).length) { + throw new Error("No prototypes got exported"); } // Some mod authors put leading zeros into the versions of their zip files. @@ -396,13 +450,15 @@ async function exportData(server) { } } - let { iconSheet, itemData } = await exportItems(server, modVersions, items); + let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes)); let locale = await exportLocale(server, modVersions, modOrder, "en"); // Free up the memory used by zip files loaded during the export. zipCache.clear(); let zip = new JSZip(); + zip.file("export/settings.json", JSON.stringify(settings)); + zip.file("export/prototypes.json", JSON.stringify(prototypes)); zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG)); zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()])); zip.file("export/locale.json", JSON.stringify([...locale.entries()])); diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index e50d5caa..2ed9cd25 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -197,6 +197,8 @@ async function uploadExport(req, res) { // This is hardcoded to prevent path expansion attacks let exportFiles = [ + "export/settings.json", + "export/prototypes.json", "export/item-spritesheet.png", "export/item-metadata.json", "export/locale.json", diff --git a/packages/slave/lua/export/clusterio_api.lua b/packages/slave/lua/export/clusterio_api.lua new file mode 100644 index 00000000..56f61124 --- /dev/null +++ b/packages/slave/lua/export/clusterio_api.lua @@ -0,0 +1,73 @@ +local function is_array(t) + local count = 0 + for k in pairs(t) do + if type(k) ~= "number" then + return false + end + count = count + 1 + end + + for i = 1, count do + if not t[i] then + return false + end + end + + return count ~= 0 +end + +-- no game.table_to_json here :( +local function table_to_json(data) + if type(data) == "number" then + if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it. + if data == -math.huge then return "-1e500" end + if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead. + return string.format("%.17g", data) + + elseif type(data) == "string" then + data = data:gsub('([\x00-\x1f\\"])', function(match) + return "\\u00" .. string.format("%02x", match:byte()) + end) + return '"' .. data .. '"' + + elseif data == nil then + return "null" + + elseif data == true then + return "true" + elseif data == false then + return "false" + + elseif type(data) == "table" then + local r + if is_array(data) then + r = {"["} + for i=1, #data do + r[#r+1] = table_to_json(data[i]) + r[#r+1] = "," + end + + -- overwrite last comma, empty table is an object + r[#r] = "]" + + else + r = {"{"} + for k, v in pairs(data) do + r[#r+1] = table_to_json(tostring(k)) + r[#r+1] = ":" + r[#r+1] = table_to_json(v) + r[#r+1] = "," + end + r[#r + (r[#r] == "," and 0 or 1)] = "}" + end + + return table.concat(r) + end +end + +local clusterio_api = {} +function clusterio_api.send_json(channel, data) + data = table_to_json(data) + print("\f$ipc:" .. channel .. "?j" .. data) +end +return clusterio_api diff --git a/packages/slave/lua/export/control.lua b/packages/slave/lua/export/control.lua index 4d27f895..b810f2f0 100644 --- a/packages/slave/lua/export/control.lua +++ b/packages/slave/lua/export/control.lua @@ -5,4 +5,7 @@ end script.on_init(function() send_json("mod_list", game.active_mods) + for name, mod_setting in pairs(game.mod_setting_prototypes) do + send_json("mod_setting_mod", { name = name, mod = mod_setting.mod }) + end end) diff --git a/packages/slave/lua/export/data-final-fixes.lua b/packages/slave/lua/export/data-final-fixes.lua index 991bb831..f3820c37 100644 --- a/packages/slave/lua/export/data-final-fixes.lua +++ b/packages/slave/lua/export/data-final-fixes.lua @@ -1,104 +1,8 @@ -local item_types = { - -- Item categories - "item", - "ammo", - "capsule", - "gun", - "item-with-entity-data", - "item-with-label", - "item-with-inventory", - "blueprint-book", - "item-with-tags", - "selection-tool", - "blueprint", - "deconstruction-item", - "copy-paste-tool", - "upgrade-item", - "module", - "rail-planner", - "tool", - "armor", - "mining-tool", - "repair-tool", +local clusterio_api = require("clusterio_api") - -- Fluids - "fluid", -} - -local function is_array(t) - local count = 0 - for k in pairs(t) do - if type(k) ~= "number" then - return false - end - count = count + 1 - end - - for i = 1, count do - if not t[i] then - return false - end - end - - return count ~= 0 -end - --- no game.table_to_json here :( -local function table_to_json(data) - if type(data) == "number" then - if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it. - if data == -math.huge then return "-1e500" end - if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead. - return string.format("%.17g", data) - - elseif type(data) == "string" then - data = data:gsub('([\x00-\x1f\\"])', function(match) - return "\\u00" .. string.format("%02x", match:byte()) - end) - return '"' .. data .. '"' - - elseif data == nil then - return "null" - - elseif data == true then - return "true" - elseif data == false then - return "false" - - elseif type(data) == "table" then - local r - if is_array(data) then - r = {"["} - for i=1, #data do - r[#r+1] = table_to_json(data[i]) - r[#r+1] = "," - end - - -- overwrite last comma, empty table is an object - r[#r] = "]" - - else - r = {"{"} - for k, v in pairs(data) do - r[#r+1] = table_to_json(tostring(k)) - r[#r+1] = ":" - r[#r+1] = table_to_json(v) - r[#r+1] = "," - end - r[#r + (r[#r] == "," and 0 or 1)] = "}" - end - - return table.concat(r) - end -end - -local function send_json(channel, data) - data = table_to_json(data) - print("\f$ipc:" .. channel .. "?j" .. data) -end - -for _, group in ipairs(item_types) do - for _, item in pairs(data.raw[group]) do - send_json("item_export", item) +log("Exporting prototypes") +for group, prototypes in pairs(data.raw) do + for name, prototype in pairs(prototypes) do + clusterio_api.send_json("prototype_export", prototype) end end diff --git a/packages/slave/lua/export/settings-final-fixes.lua b/packages/slave/lua/export/settings-final-fixes.lua new file mode 100644 index 00000000..fe0ea4e7 --- /dev/null +++ b/packages/slave/lua/export/settings-final-fixes.lua @@ -0,0 +1,8 @@ +local clusterio_api = require("clusterio_api") + +log("Exporting settings") +for group, prototypes in pairs(data.raw) do + for name, prototype in pairs(prototypes) do + clusterio_api.send_json("settings_export", prototype) + end +end diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 6ab91fa9..41bd159d 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -310,10 +310,15 @@ describe("Integration of Clusterio", function() { let response = await get("/api/export-manifest"); let manifest = JSON.parse(response.body); assert(Object.keys(manifest).length > 1, "Export manifest is empty"); - assert( - await fs.exists(path.join(exportPath, "..", manifest["item-metadata"])), - "Export was not written to filesystem" - ); + for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) { + assert(manifest[key], `Missing ${key} from manifest`); + assert( + await fs.exists(path.join(exportPath, "..", manifest[key])), + `Manifest entry for ${key} was not written to filesystem` + ); + } + let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, "..", manifest["prototypes"]))); + assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types"); await checkInstanceStatus(44, "stopped"); }); }); From 13e12f733592fdb5111001e481e0e6f60d61c6ea Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 31 Jan 2023 04:18:01 +0100 Subject: [PATCH 34/40] Tie data exports to mod packs Store the export manifest in ModPack to allow mulitple mod packs to have different data exported. This is necessary as the export contains the settings prototypes needed to create a proper UI for mod settings. For now the useLocale and useItemMetadata interfaces uses the manifest for a default mod pack set in the master config. --- packages/ctl/src/commands.js | 6 +++ packages/lib/config/definitions.js | 7 +++ packages/lib/data/ExportManifest.js | 46 +++++++++++++++++++ packages/lib/data/ModPack.js | 11 +++++ packages/lib/data/index.js | 2 + packages/lib/link/messages.js | 10 ++++ packages/master/src/BaseConnection.js | 12 +++++ packages/master/src/Master.js | 9 ---- packages/master/src/routes.js | 29 ++++++++---- packages/slave/src/Instance.js | 1 + packages/web_ui/src/model/export_manifest.jsx | 13 ++++-- packages/web_ui/src/model/item_metadata.jsx | 10 ++-- packages/web_ui/src/model/locale.jsx | 4 +- test/integration/clusterio.js | 13 +++--- test/integration/index.js | 3 +- test/lib/data/ModPack.js | 4 ++ 16 files changed, 146 insertions(+), 34 deletions(-) create mode 100644 packages/lib/data/ExportManifest.js diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index a99e9eb7..4f1c03ce 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -893,6 +893,12 @@ modPackCommands.add(new libCommand.Command({ print(` ${setting}: ${JSON.stringify(settingValue.value)}`); } } + } else if (field === "export_manifest") { + print(`${field}:`); + print(" assets:"); + for (let [name, fileName] of Object.entries(value.assets)) { + print(` ${name}: ${fileName}`); + } } else { print(`${field}: ${value}`); diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index b0926270..d6605c85 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -115,6 +115,13 @@ MasterGroup.define({ type: "number", initial_value: 15, }); +MasterGroup.define({ + name: "default_mod_pack_id", + title: "Default Mod Pack", + description: "Mod pack used by default for instances.", + type: "number", + optional: true, +}); MasterGroup.define({ name: "default_role_id", title: "Default role", diff --git a/packages/lib/data/ExportManifest.js b/packages/lib/data/ExportManifest.js new file mode 100644 index 00000000..a07d663e --- /dev/null +++ b/packages/lib/data/ExportManifest.js @@ -0,0 +1,46 @@ +"use strict"; + +/** + * Export manifest for a mod pack + * + * Tracks files stored on disk for data that extracted from Factorio using a + * given mod pack. + * @alias module:lib/data.ExportManifest + */ +class ExportManifest { + + /** + * Mapping between known assets and their file names. + * + * Currently exported assets are + * - settings: JSON of mod settings prototypes. + * - prototypes: JSON of all game prototypes except settings. + * - item-spritesheet: PNG containing icons from the game. + * - item-metadata: JSON of Map entries describing icons in item-spritesheet + * - locale: JSON of flattened Map entries of en locale strings from the game. + * @type {Object} + */ + assets = {}; + + static jsonSchema = { + type: "object", + additionalProperties: false, + required: ["assets"], + properties: { + "assets": { + type: "object", + additionalProperties: { type: "string" }, + }, + }, + }; + + constructor(json = {}) { + if (json.assets) { this.assets = json.assets; } + } + + toJSON() { + return { assets: this.assets }; + } +} + +module.exports = ExportManifest; diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 0b58a0f2..286dcea7 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -3,6 +3,7 @@ const zlib = require("zlib"); const libSchema = require("../schema"); +const ExportManifest = require("./ExportManifest"); const { integerFactorioVersion } = require("./version"); @@ -99,6 +100,12 @@ class ModPack { "runtime-per-user": new Map(), }; + /** + * Mapping to files containing exported data for this modpack + * @type {module:lib/data.ExportManifest|undefined} + */ + exportManifest; + /** * True if this mod pack has been deleted from the list of mod packs. * @type {boolean} @@ -128,6 +135,7 @@ class ModPack { "runtime-per-user": ModSettingsJsonSchema, }, }, + "export_manifest": ExportManifest.jsonSchema, "is_deleted": { type: "boolean" }, }, }; @@ -151,6 +159,7 @@ class ModPack { "runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])), }; } + if (json.export_manifest) { this.exportManifest = new ExportManifest(json.export_manifest); } if (json.is_deleted) { this.isDeleted = json.is_deleted; } if (!this.mods.has("base")) { @@ -171,6 +180,7 @@ class ModPack { "runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]), }, }; + if (this.exportManifest) { json.export_manifest = this.exportManifest.toJSON(); } if (this.isDeleted) { json.is_deleted = this.isDeleted; } return json; } @@ -178,6 +188,7 @@ class ModPack { toModPackString() { const json = this.toJSON(); delete json.id; + delete json.export_manifest; // eslint-disable-next-line node/no-sync let buf = zlib.deflateSync(JSON.stringify(json)); diff --git a/packages/lib/data/index.js b/packages/lib/data/index.js index cec20aed..f3856763 100644 --- a/packages/lib/data/index.js +++ b/packages/lib/data/index.js @@ -4,11 +4,13 @@ * @author Hornwitser */ "use strict"; +const ExportManifest = require("./ExportManifest"); const ModInfo = require("./ModInfo"); const ModPack = require("./ModPack"); const version = require("./version"); module.exports = { + ExportManifest, ModInfo, ModPack, ...version, diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index 4404a25d..3da48aec 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -723,6 +723,16 @@ messages.getModPack = new Request({ }, }); +messages.getDefaultModPack = new Request({ + type: "get_default_mod_pack", + links: ["control-master", "instance-slave", "slave-master"], + permission: "core.mod_pack.get", + forwardTo: "master", + responseProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + messages.listModPacks = new Request({ type: "list_mod_packs", links: ["control-master"], diff --git a/packages/master/src/BaseConnection.js b/packages/master/src/BaseConnection.js index f22094cf..92e25c1d 100644 --- a/packages/master/src/BaseConnection.js +++ b/packages/master/src/BaseConnection.js @@ -93,6 +93,18 @@ class BaseConnection extends libLink.Link { } return { mod_pack: modPack.toJSON() }; } + + async getDefaultModPackRequestHandler(message) { + let id = this._master.config.get("master.default_mod_pack_id"); + if (id === null) { + throw new libErrors.RequestError("Default mod pack not set on master"); + } + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Default mod pack configured (${id}) does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } } module.exports = BaseConnection; diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index edb1288c..2bb3ccb7 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -177,7 +177,6 @@ class Master { this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json")); this.userManager = new UserManager(this.config); await this.userManager.load(path.join(databaseDirectory, "users.json")); - this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json")); let modsDirectory = this.config.get("master.mods_directory"); await fs.ensureDir(modsDirectory); @@ -445,14 +444,6 @@ class Master { return manifest; } - async updateExportManifest(newManifest) { - let databaseDirectory = this.config.get("master.database_directory"); - await libFileOps.safeOutputFile( - path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4) - ); - this.exportManifest = newManifest; - } - async deleteMod(name, version) { let filename = `${name}_${version}.zip`; let mod = this.mods.get(filename); diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 2ed9cd25..4f14fda3 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -187,6 +187,21 @@ async function uploadExport(req, res) { return; } + if (typeof req.query.mod_pack_id !== "string") { + res.sendStatus(400); + return; + } + let modPackId = Number.parseInt(req.query.mod_pack_id, 10); + if (!Number.isInteger(modPackId)) { + res.sendStatus(400); + return; + } + let modPack = res.app.locals.master.modPacks.get(modPackId); + if (!modPack) { + res.sendStatus(400); + return; + } + let data = []; for await (let chunk of req) { data.push(chunk); @@ -204,7 +219,7 @@ async function uploadExport(req, res) { "export/locale.json", ]; - let manifest = {}; + let assets = {}; for (let filePath of exportFiles) { let file = zip.file(filePath); if (!file) { @@ -213,19 +228,16 @@ async function uploadExport(req, res) { let { name, ext } = path.posix.parse(filePath); let hash = await libHash.hashStream(file.nodeStream()); - manifest[name] = `static/${name}.${hash}${ext}`; + assets[name] = `${name}.${hash}${ext}`; await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer")); } - await res.app.locals.master.updateExportManifest(manifest); + + modPack.exportManifest = new libData.ExportManifest({ assets }); + res.app.locals.master.modPackUpdated(modPack); res.sendStatus(200); } -async function getExportManifest(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.json(res.app.locals.master.exportManifest); -} - async function createProxyStream(app) { let asyncRandomBytes = util.promisify(crypto.randomBytes); let id = (await asyncRandomBytes(8)).toString("hex"); @@ -601,7 +613,6 @@ function addRouteHandlers(app) { validateSlaveToken, (req, res, next) => uploadExport(req, res).catch(next) ); - app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next)); app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next)); app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next)); app.post("/api/upload-save", diff --git a/packages/slave/src/Instance.js b/packages/slave/src/Instance.js index 2148206e..a39287ef 100644 --- a/packages/slave/src/Instance.js +++ b/packages/slave/src/Instance.js @@ -1064,6 +1064,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); let content = await zip.generateAsync({ type: "nodebuffer" }); let url = new URL(this._slave.config.get("slave.master_url")); url.pathname += "api/upload-export"; + url.searchParams.set("mod_pack_id", this.config.get("factorio.mod_pack")); let response = await phin({ url, method: "PUT", data: content, diff --git a/packages/web_ui/src/model/export_manifest.jsx b/packages/web_ui/src/model/export_manifest.jsx index ebb436d9..50ef1467 100644 --- a/packages/web_ui/src/model/export_manifest.jsx +++ b/packages/web_ui/src/model/export_manifest.jsx @@ -1,15 +1,20 @@ -import { useEffect, useState } from "react"; +import { useEffect, useContext, useState } from "react"; + +import { libData, libLink } from "@clusterio/lib"; +import ControlContext from "../components/ControlContext"; let exportManifestCache = null; let emptyCache = {}; export function useExportManifest() { + const control = useContext(ControlContext); let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache); useEffect(() => { async function load() { - let response = await fetch(`${webRoot}api/export-manifest`); - if (response.ok) { - exportManifestCache = await response.json(); + let result = await libLink.messages.getDefaultModPack.send(control); + let modPack = new libData.ModPack(result.mod_pack); + if (modPack.exportManifest) { + exportManifestCache = modPack.exportManifest; setExportManifest(exportManifestCache); } } diff --git a/packages/web_ui/src/model/item_metadata.jsx b/packages/web_ui/src/model/item_metadata.jsx index 3858e296..8c1ae078 100644 --- a/packages/web_ui/src/model/item_metadata.jsx +++ b/packages/web_ui/src/model/item_metadata.jsx @@ -9,10 +9,14 @@ export function useItemMetadata() { let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map()); useEffect(() => { async function load() { - if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) { + if ( + !exportManifest + || !exportManifest.assets["item-metadata"] + || !exportManifest.assets["item-spritesheet"] + ) { return; } - let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`); + let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`); if (response.ok) { let data = await response.json(); itemMetadataCache = new Map(data); @@ -22,7 +26,7 @@ export function useItemMetadata() { for (let [name, meta] of itemMetadataCache) { style.sheet.insertRule( `.item-${CSS.escape(name)} { - background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}"); + background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}"); background-repeat: no-repeat; background-position: -${meta.x}px -${meta.y}px; height: ${meta.size}px; diff --git a/packages/web_ui/src/model/locale.jsx b/packages/web_ui/src/model/locale.jsx index 3a22e6d7..431833fc 100644 --- a/packages/web_ui/src/model/locale.jsx +++ b/packages/web_ui/src/model/locale.jsx @@ -9,10 +9,10 @@ export function useLocale() { let [locale, setLocale] = useState(localeCache || new Map()); useEffect(() => { async function load() { - if (!exportManifest["locale"]) { + if (!exportManifest || !exportManifest.assets["locale"]) { return; } - let response = await fetch(`${staticRoot}${exportManifest["locale"]}`); + let response = await fetch(`${staticRoot}static/${exportManifest.assets["locale"]}`); if (response.ok) { let data = await response.json(); localeCache = new Map(data); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 41bd159d..272d5064 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -307,17 +307,18 @@ describe("Integration of Clusterio", function() { let exportPath = path.join("temp", "test", "static"); await fs.remove(exportPath); await execCtl("instance export-data test"); - let response = await get("/api/export-manifest"); - let manifest = JSON.parse(response.body); - assert(Object.keys(manifest).length > 1, "Export manifest is empty"); + let result = await libLink.messages.getDefaultModPack.send(getControl()); + let modPack = new libData.ModPack(result.mod_pack); + let assets = modPack.exportManifest.assets; + assert(Object.keys(assets).length > 1, "Export assets is empty"); for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) { - assert(manifest[key], `Missing ${key} from manifest`); + assert(assets[key], `Missing ${key} from assets`); assert( - await fs.exists(path.join(exportPath, "..", manifest[key])), + await fs.exists(path.join(exportPath, assets[key])), `Manifest entry for ${key} was not written to filesystem` ); } - let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, "..", manifest["prototypes"]))); + let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, assets["prototypes"]))); assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types"); await checkInstanceStatus(44, "stopped"); }); diff --git a/test/integration/index.js b/test/integration/index.js index b604bb1f..54cf1c29 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -228,7 +228,8 @@ before(async function() { testPack.factorioVersion = "1.1.0"; testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); - libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); + await libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); + await libLink.messages.setMasterConfigField.send(control, { field: "master.default_mod_pack_id", value: "12" }); }); after(async function() { diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index 8a6fed3e..a5c105e5 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -3,6 +3,7 @@ const assert = require("assert").strict; const zlib = require("zlib"); const libSchema = require("@clusterio/lib/schema"); +const ExportManifest = require("@clusterio/lib/data/ExportManifest"); const ModPack = require("@clusterio/lib/data/ModPack"); @@ -18,6 +19,7 @@ describe("lib/data/ModPack", function() { assert.deepEqual(new ModPack(json), pack); const packStringed = ModPack.fromModPackString(pack.toModPackString()); packStringed.id = pack.id; + packStringed.exportManifest = pack.exportManifest; assert.deepEqual(packStringed, pack); } @@ -40,6 +42,7 @@ describe("lib/data/ModPack", function() { "string-setting": { "value": "a string" }, }, }})); + check(new ModPack({ export_manifest: { assets: { setting: "settings.json" }}})); check(new ModPack({ deleted: true })); check(new ModPack({ name: "Super pack", @@ -60,6 +63,7 @@ describe("lib/data/ModPack", function() { "string-setting": { "value": "a string" }, }, }, + export_manifest: { assets: { setting: "settings.json" }}, deleted: true, })); }); From 39ed098a648dce7200943b6cf861c87642f4a3c7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 1 Feb 2023 15:30:01 +0100 Subject: [PATCH 35/40] Fix error messages changing with Node.js version Updates to Node.js error reporting have caused tests to fail due to them being hardcoded to expect a certain format that has since changed. Fix by making Node.js create the error message for us. --- test/lib/data/ModPack.js | 8 +++++++- test/lib/plugin_loader.js | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index a5c105e5..15f94ee7 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -82,10 +82,16 @@ describe("lib/data/ModPack", function() { () => ModPack.fromModPackString("AMalformedString"), new Error("Malformed mod pack string: zlib inflate failed") ); + let badJsonMsg; + try { + JSON.parse("Not Json"); + } catch (err) { + badJsonMsg = err.message; + } assert.throws( // eslint-disable-next-line node/no-sync () => ModPack.fromModPackString(Buffer.from(zlib.deflateSync("Not Json")).toString("base64")), - new Error("Malformed mod pack string: Unexpected token N in JSON at position 0") + new Error(`Malformed mod pack string: ${badJsonMsg}`) ); assert.throws( // eslint-disable-next-line node/no-sync diff --git a/test/lib/plugin_loader.js b/test/lib/plugin_loader.js index b5098955..d1153c1b 100644 --- a/test/lib/plugin_loader.js +++ b/test/lib/plugin_loader.js @@ -45,9 +45,16 @@ describe("lib/plugin_loader", function() { ); }); it("should reject on broken plugin", async function() { + let brokenMessage; + try { + // eslint-disable-next-line node/global-require + require(path.resolve(brokenPlugin, "info.js")); + } catch (err) { + brokenMessage = err.message; + } await assert.rejects( libPluginLoader.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])), - { message: "PluginError: Unexpected identifier" } + { message: `PluginError: ${brokenMessage}` } ); }); it("should reject on invalid plugin", async function() { From ddc548f590f4291ff9a22acb5eecffc73fb30ca9 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 1 Feb 2023 23:32:14 +0100 Subject: [PATCH 36/40] Set default values for mod settings on export Add missing mod settings set to their default values to a mod pack when performing a data export based on the settings prototypes that were extracted from the game. This mirrors the behaviour of Factorio when new setting prototypes are added by a mod. --- packages/lib/data/ModPack.js | 34 +++++++++++++ packages/master/src/routes.js | 6 +++ test/lib/data/ModPack.js | 93 +++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+) diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 286dcea7..8279cf03 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -316,6 +316,40 @@ class ModPack { propertyTreeBytes(this.toJSON().settings), ]); } + + /** + * Fill in missing settings with their default values + * + * Uses the provided setting prototypes to add any missing mod settings + * in the mod pack with the default value from the prototype. + * + * @param {Object} settingPrototypes - + * Setting prototypes exported from the game. + * @param {Logger} logger - Logger used to report warnings on. + */ + fillDefaultSettings(settingPrototypes, logger) { + const knownTypes = ["bool-setting", "int-setting", "double-setting", "string-setting"]; + let prototypes = Object.entries(settingPrototypes) + .filter(([type, _]) => knownTypes.includes(type)) + .flatMap(([_, settings]) => Object.values(settings)) + ; + + for (let prototype of prototypes) { + if (!["startup", "runtime-global", "runtime-per-user"].includes(prototype.setting_type)) { + logger.warn(`Ignoring ${prototype.name} with unknown setting_type ${prototype.setting_type}`); + continue; + } + if (prototype.default_value === undefined) { + logger.warn(`Ignoring ${prototype.name} with missing default_value`); + continue; + } + if (this.settings[prototype.setting_type].get(prototype.name)) { + continue; + } + + this.settings[prototype.setting_type].set(prototype.name, { value: prototype.default_value }); + } + } } module.exports = ModPack; diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 4f14fda3..4e48f1f5 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -220,12 +220,17 @@ async function uploadExport(req, res) { ]; let assets = {}; + let settingPrototypes = {}; for (let filePath of exportFiles) { let file = zip.file(filePath); if (!file) { continue; } + if (filePath === "export/settings.json") { + settingPrototypes = JSON.parse(await file.async("text")); + } + let { name, ext } = path.posix.parse(filePath); let hash = await libHash.hashStream(file.nodeStream()); assets[name] = `${name}.${hash}${ext}`; @@ -233,6 +238,7 @@ async function uploadExport(req, res) { } modPack.exportManifest = new libData.ExportManifest({ assets }); + modPack.fillDefaultSettings(settingPrototypes, logger); res.app.locals.master.modPackUpdated(modPack); res.sendStatus(200); diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index 15f94ee7..43cdc856 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -76,6 +76,99 @@ describe("lib/data/ModPack", function() { assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions); }); + describe(".fillDefaultSettings()", function() { + const prototypes = { + "bool-setting": { + "bool": { + name: "bool", + type: "bool-setting", + setting_type: "startup", + default_value: true, + }, + }, + "double-setting": { + "number": { + name: "number", + type: "double-setting", + setting_type: "runtime-global", + default_value: 123, + }, + }, + "string-setting": { + "string": { + name: "string", + type: "string-setting", + setting_type: "runtime-per-user", + default_value: "a string", + }, + }, + }; + const mockLogger = { warn: () => {} }; + it("should fill in defaults for settings", function() { + const pack = new ModPack(); + pack.fillDefaultSettings(prototypes, mockLogger); + assert.deepEqual(pack.toJSON().settings, { + "startup": { + "bool": { "value": true }, + }, + "runtime-global": { + "number": { "value": 123 }, + }, + "runtime-per-user": { + "string": { "value": "a string" }, + }, + }); + }); + it("should not overwrite existing values", function() { + const pack = new ModPack({ settings: { + "startup": { + "bool": { "value": false }, + }, + "runtime-global": { + "number": { "value": 2 }, + }, + "runtime-per-user": { + "string": { "value": "spam" }, + }, + }}); + pack.fillDefaultSettings(prototypes, mockLogger); + assert.deepEqual(pack.toJSON().settings, { + "startup": { + "bool": { "value": false }, + }, + "runtime-global": { + "number": { "value": 2 }, + }, + "runtime-per-user": { + "string": { "value": "spam" }, + }, + }); + }); + it("should ignore unknown setting_type and missing default_value", function() { + const pack = new ModPack(); + pack.fillDefaultSettings({ + "string-setting": { + "foo": { + name: "foo", + type: "string-setting", + setting_type: "magic-that-does-not-exist", + default_value: "a string", + }, + "bar": { + name: "bar", + type: "string-setting", + setting_type: "startup", + }, + }, + }, mockLogger); + assert.deepEqual(pack.toJSON().settings, { + "startup": {}, + "runtime-global": {}, + "runtime-per-user": {}, + }); + }); + }); + describe(".fromModPackString()", function() { it("should handle malformed strings", function() { assert.throws( From 3f6a7e43361a4ea89ddc381397cf909c7c7bab68 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 2 Feb 2023 00:00:54 +0100 Subject: [PATCH 37/40] Format mod settings with prototypes and locale in web UI Use exported settings prototypes and locale to format the settings UI to with mod names, settings names, and select dropdowns. This is a crude approximation of how the game presents these settings to the user. --- .../web_ui/src/components/ModPackViewPage.jsx | 166 ++++++++++++++---- 1 file changed, 134 insertions(+), 32 deletions(-) diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx index 0d2a426f..69393489 100644 --- a/packages/web_ui/src/components/ModPackViewPage.jsx +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useContext, useRef, useState } from "react"; +import React, { Fragment, useEffect, useContext, useRef, useState } from "react"; import { useHistory, useParams } from "react-router-dom"; import { Button, Card, Checkbox, Col, ConfigProvider, Descriptions, Form, Input, PageHeader, Pagination, @@ -353,49 +353,115 @@ function pickValue(map) { return [...map].map(([name, value]) => [name, value.value]); } -function SettingsTable(props) { - function controls(scope, fields) { - return fields.map(([fieldName, fieldValue]) => { - const isBoolean = typeof fieldValue === "boolean"; - const isNumber = typeof fieldValue === "number"; +function groupToMap(array, fn) { + const map = new Map(); + for (let index = 0; index < array.length; index++) { + let key = fn(array[index], index, array); + if (!map.has(key)) { + map.set(key, []); + } + map.get(key).push(array[index]); + } + return map; +} - return `${mod.name}_${mod.version}`)); + const modTitles = new Map(modList + .filter(mod => modsInPack.has(`${mod.name}_${mod.version}`)) + .map(mod => [`${mod.name}`, mod.title]) + ); + + const types = ["bool-setting", "int-setting", "double-setting", "string-setting"]; + let prototypes = Object.entries(props.prototypes) + .filter(([type, _]) => types.includes(type)) + .flatMap(([_, settingPrototypes]) => Object.values(settingPrototypes)) + ; + + function controls(scope, storedFields) { + let fields = new Map(prototypes + .filter(p => p.setting_type === scope) + .sort((a, b) => strcmp(a.mod, b.mod) || strcmp(a.order || "", b.order || "") || strcmp(a.name, b.name)) + .map(p => [p.name, p]) + ); + for (let [name, value] of storedFields) { + if (!fields.has(name)) { + let type; + if (typeof value === "boolean") { + type = "bool-setting"; + } else if (typeof value === "number") { + type = "double-setting"; + } else if (typeof value === "string") { + type = "string-setting"; + } else { + throw Error(`Unhandled setting type ${typeof value}`); } - > - {isBoolean ? : } - ; - }); + fields.set(name, { mod: "Unknown Fields", name, setting_type: scope, type }); + } + } + let fieldsByMod = groupToMap([...fields.values()], prototype => prototype.mod); + return [...fieldsByMod].map(([mod, fieldsForMod]) => + + {props.locale.get(`mod-name.${mod}`) || modTitles.get(mod) || mod} + + {[...fieldsForMod].map(prototype => { + const name = prototype.name; + const isBoolean = prototype.type === "bool-setting"; + const isNumber = ["int-setting", "double-setting"].includes(prototype.type); + const isSelect = Boolean(prototype.allowed_values); + let input; + if (isBoolean) { + input = ; + } else if (isSelect) { + const options = Object.values(prototype.allowed_values).map(value => ({ + value, + label: props.locale.get(`string-mod-setting.${name}-${value}`) || value, + })); + input = ; + } + + return + {input} + ; + })} + ); } return <> - Use clusterioctl to add and remove fields. - Startup + Run export data on an instance to update available fields. + Startup {controls("startup", pickValue(props.modPack.settings["startup"]))} - Map + Map These only apply for new saves. {controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))} - Per Player + Per Player Default settings for new players, these only apply for new saves. {controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))} ; @@ -457,6 +523,33 @@ function ExportButton(props) { ; } +function useExportedAsset(modPack, asset) { + let [assetData, setAssetData] = useState(asset === "locale" ? new Map() : {}); + let assetFilename; + if (modPack instanceof libData.ModPack && modPack.exportManifest && modPack.exportManifest.assets[asset]) { + assetFilename = modPack.exportManifest.assets[asset]; + } + useEffect(() => { + async function load() { + if (!assetFilename) { + return; + } + + let response = await fetch(`${staticRoot}static/${assetFilename}`); + if (response.ok) { + let data = await response.json(); + setAssetData(asset === "locale" ? new Map(data) : data); + } else { + logger.error(`Error loading mod pack asset "${asset}" (${response.status}): ${await response.text()}`); + } + } + + load().catch(notifyErrorHandler(`Loading mod pack asset "${asset}" failed`)); + }, [assetFilename]); + + return assetData; +} + export default function ModPackViewPage() { let account = useAccount(); let history = useHistory(); @@ -466,6 +559,8 @@ export default function ModPackViewPage() { const [form] = Form.useForm(); let [modPack] = useModPack(modPackId); + let prototypes = useExportedAsset(modPack, "settings"); + let locale = useExportedAsset(modPack, "locale"); let [changes, setChanges] = useState([]); let [modifiedModPack, setModifiedModPack] = useState(modPack); let syncTimeout = useRef(); @@ -657,7 +752,14 @@ export default function ModPackViewPage() { - +
Date: Thu, 2 Feb 2023 00:04:27 +0100 Subject: [PATCH 38/40] Fix export for mods with uncommon root directory The directory which all files are placed under in a mod is commonly named _, but this is not a requirement as it can be named anything. Fix code incorrectly assuming the zipped mod's root folder follows this pattern during data export. --- packages/lib/factorio/export.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/lib/factorio/export.js b/packages/lib/factorio/export.js index b4eec039..22c74de9 100644 --- a/packages/lib/factorio/export.js +++ b/packages/lib/factorio/export.js @@ -7,6 +7,7 @@ const JSZip = require("jszip"); const libIni = require("../ini"); const libBuildMod = require("../build_mod"); +const libZipOps = require("../zip_ops"); /** * Generate the export mod needed for exportData @@ -56,7 +57,7 @@ async function loadZip(server, modVersions, mod) { zipCache.set(zipPath, zip); } - return zip.folder(`${mod}_${modVersion}`); + return zip.folder(libZipOps.findRoot(zip)); } /** From 91af66e2dbeb8fa601b7cc36621bb42dc92a44e2 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 2 Feb 2023 01:07:08 +0100 Subject: [PATCH 39/40] Fix export when default mod pack is used --- packages/slave/src/Instance.js | 22 ++++++++++++++-------- test/integration/clusterio.js | 1 - 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/slave/src/Instance.js b/packages/slave/src/Instance.js index a39287ef..2d33aad9 100644 --- a/packages/slave/src/Instance.js +++ b/packages/slave/src/Instance.js @@ -191,6 +191,12 @@ class Instance extends libLink.Link { } }); + /** + * Mod pack currently running on this instance + * @type {module:lib/data.ModPack>} + */ + this.activeModPack = undefined; + /** * Per player statistics recorded by this instance. * @type {Map} @@ -593,13 +599,13 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); */ async syncMods() { const modPackId = this.config.get("factorio.mod_pack"); - let modPack; + let response; if (modPackId === null) { - modPack = new libData.ModPack(); + response = await libLink.messages.getDefaultModPack.send(this); } else { - const response = await libLink.messages.getModPack.send(this, { id: modPackId }); - modPack = new libData.ModPack(response.mod_pack); + response = await libLink.messages.getModPack.send(this, { id: modPackId }); } + this.activeModPack = new libData.ModPack(response.mod_pack); // TODO validate factorioVersion @@ -625,7 +631,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); // Add mods from mod the pack const modsDir = this._slave.config.get("slave.mods_directory"); - for (let mod of modPack.mods.values()) { + for (let mod of this.activeModPack.mods.values()) { const modFile = `${mod.name}_${mod.version}.zip`; const target = path.join(modsDir, modFile); const link = this.path("mods", modFile); @@ -643,11 +649,11 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); // Write mod-list.json await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({ - mods: [{ name: "base" }, ...modPack.mods.values()].map(m => ({ name: m.name, enabled: true })), + mods: [{ name: "base" }, ...this.activeModPack.mods.values()].map(m => ({ name: m.name, enabled: true })), }, null, 2)); // Write mod-settings.dat - await fs.outputFile(this.path("mods", "mod-settings.dat"), modPack.toModSettingsDat()); + await fs.outputFile(this.path("mods", "mod-settings.dat"), this.activeModPack.toModSettingsDat()); } /** @@ -1064,7 +1070,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); let content = await zip.generateAsync({ type: "nodebuffer" }); let url = new URL(this._slave.config.get("slave.master_url")); url.pathname += "api/upload-export"; - url.searchParams.set("mod_pack_id", this.config.get("factorio.mod_pack")); + url.searchParams.set("mod_pack_id", this.activeModPack.id); let response = await phin({ url, method: "PUT", data: content, diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 272d5064..ab94dd89 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -265,7 +265,6 @@ describe("Integration of Clusterio", function() { ); await execCtl(`instance config set-prop test factorio.settings visibility "'${value}'"`); await execCtl("instance config set-prop test factorio.settings require_user_verification false"); - await execCtl("instance config set test factorio.mod_pack 12"); }); }); From 0eabdf011b1fb632d26142686dc21248353eb72b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 2 Feb 2023 01:07:33 +0100 Subject: [PATCH 40/40] Fix mod loading tests --- test/integration/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/index.js b/test/integration/index.js index 54cf1c29..41eabfc9 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -226,7 +226,7 @@ before(async function() { testPack.id = 12; testPack.name = "subspace_storage-pack"; testPack.factorioVersion = "1.1.0"; - testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); + testPack.mods.set("clusterio_lib", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); await libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); await libLink.messages.setMasterConfigField.send(control, { field: "master.default_mod_pack_id", value: "12" });