diff --git a/.dockerignore b/.dockerignore index 2ad21c71..74f895db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,8 +13,7 @@ factorio factorio.tar.gz # We build internal mods during build, mods from local test env should not be applied -sharedMods -sharedModsDisabled +mods # These files should never be required during the build .git diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index ee732740..c3a3923d 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -26,8 +26,8 @@ jobs: node-version: ${{ matrix.node-version }} - run: pnpm i --no-optional --no-frozen-lockfile - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz - - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods - - run: wget -q -O temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip + - run: node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/mods + - run: wget -q -O temp/test/mods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/subspace_storage/releases/download/1.99.6/subspace_storage_1.99.8.zip - name: Run tests if: ${{ matrix.node-version != '12.x' }} run: pnpm test diff --git a/.gitignore b/.gitignore index 84213197..37bf1a49 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,7 @@ node_modules Factorio* /factorio /instances -/sharedMods +/mods /database /plugin-list.json /config-master.json diff --git a/Dockerfile b/Dockerfile index 71008b26..3eb36c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,15 +22,15 @@ RUN pnpm install #RUN pnpm install @clusterio/plugin-subspace_storage #RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage -COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/sharedMods/ +COPY --from=subspace_storage_builder /subspace_storage/dist/ /clusterio/mods/ # Build Lua library mod RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \ -&& mv dist/* sharedMods/ \ +&& mv dist/* mods/ \ && mkdir temp \ && mkdir temp/test \ -&& cp sharedMods/ temp/test/ -r \ -&& ls sharedMods +&& cp mods/ temp/test/ -r \ +&& ls mods # Remove node_modules RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \; diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 9211543c..b4cfaf90 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -89,6 +89,10 @@ class Control extends libLink.Link { async saveListUpdateEventHandler() { } + async modPackUpdateEventHandler() { } + + async modUpdateEventHandler() { } + async userUpdateEventHandler() { } async setLogSubscriptions({ diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index ba5df6bd..4f1c03ce 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -14,6 +14,7 @@ const libLink = require("@clusterio/lib/link"); const libErrors = require("@clusterio/lib/errors"); const libConfig = require("@clusterio/lib/config"); const libCommand = require("@clusterio/lib/command"); +const libData = require("@clusterio/lib/data"); const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging"); const libLoggingUtils = require("@clusterio/lib/logging_utils"); const libHelpers = require("@clusterio/lib/helpers"); @@ -860,6 +861,465 @@ instanceCommands.add(new libCommand.Command({ }, })); +const modPackCommands = new libCommand.CommandTree({ name: "mod-pack", description: "Mod Pack" }); +modPackCommands.add(new libCommand.Command({ + definition: ["show ", "Show details of mod pack", (yargs) => { + yargs.positional("mod-pack", { describe: "Mod pack to show", type: "string" }); + }], + handler: async function(args, control) { + let mods = new Map( + (await libLink.messages.listMods.send(control)).list.map(m => [`${m.name}_${m.version}`, m]) + ); + let response = await libLink.messages.getModPack.send(control, { + id: await libCommand.resolveModPack(control, args.modPack), + }); + + for (let [field, value] of Object.entries(response.mod_pack)) { + if (field === "mods") { + print(`${field}:`); + for (let entry of value) { + const mod = mods.get(`${entry.name}_${entry.version}`); + const missing = !mod && "? "; + const badChecksum = mod && entry.sha1 && entry.sha1 !== mod.sha1 && "! "; + const warn = missing || badChecksum || ""; + const enabled = entry.enabled ? "" : "(disabled) "; + print(` ${warn}${enabled}${entry.name} ${entry.version}${entry.sha1 ? ` (${entry.sha1})` : ""}`); + } + } else if (field === "settings") { + print(`${field}:`); + for (let [scope, settings] of Object.entries(value)) { + print(` ${scope}:`); + for (let [setting, settingValue] of Object.entries(settings)) { + print(` ${setting}: ${JSON.stringify(settingValue.value)}`); + } + } + } else if (field === "export_manifest") { + print(`${field}:`); + print(" assets:"); + for (let [name, fileName] of Object.entries(value.assets)) { + print(` ${name}: ${fileName}`); + } + + } else { + print(`${field}: ${value}`); + } + } + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: [["list", "l"], "List mod packs in the cluster"], + handler: async function(args, control) { + let response = await libLink.messages.listModPacks.send(control); + let fields = ["id", "name", "factorio_version"]; + for (let entry of response.list) { + for (let field of Object.keys(entry)) { + if (!fields.includes(field)) { + delete entry[field]; + } + } + } + print(asTable(response.list)); + }, +})); + +function setModPackSettings(modPack, args) { + function doSettings(settings, cast) { + for (let i = 0; i + 2 < settings.length; i += 3) { + const [scope, name, value] = settings.slice(i, i + 3); + if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) { + throw new libErrors.CommandError( + `Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}` + ); + } + modPack.settings[scope].set(name, { value: cast(value) }); + } + } + + doSettings(args.boolSetting || [], value => { + if (!["true", "false"].includes(value)) { + throw new libErrors.CommandError(`boolean value must be one of true or false, not ${value}`); + } + return value === "true"; + }); + doSettings(args.intSetting || [], value => { + // eslint-disable-next-line radix + let number = Number.parseInt(value); + if (isNaN(number)) { + throw new libErrors.CommandError(`int value must be an integer, not ${value}`); + } + return number; + }); + doSettings(args.doubleSetting || [], value => { + let number = Number.parseFloat(value); + if (isNaN(number)) { + throw new libErrors.CommandError(`double value must be a number, not ${value}`); + } + return number; + }); + doSettings(args.stringSetting || [], value => value); +} + +function setModPackMods(modPack, mods) { + for (let mod of mods || []) { + const [name, version, sha1] = mod.split(":"); + if (!version) { + throw new libErrors.CommandError("Added mod must be formatted as name:version or name:version:sha1"); + } + if (!/^\d+\.\d+\.\d+$/.test(version)) { + throw new libErrors.CommandError("version must match the format digit.digit.digit"); + } + if (sha1 && !/^[0-9a-f]{40}$/.test(sha1)) { + throw new libErrors.CommandError("sha1 must be a 40 digit lower case hex string"); + } + modPack.mods.set(name, { name, enabled: true, version, sha1 }); + } +} + +function setModPackModsEnabled(modPack, mods, enabled) { + for (let mod of mods || []) { + if (!modPack.mods.has(mod)) { + throw new libErrors.CommandError(`Mod named ${mod} does not exist in the mod pack`); + } + modPack.mods.get(mod).enabled = enabled; + } +} + +modPackCommands.add(new libCommand.Command({ + definition: ["create ", "Create mod pack", (yargs) => { + yargs.positional("name", { describe: "Name of mod pack to create", type: "string" }); + yargs.positional("factorio-version", { describe: "Version of factorio the mod pack is for", type: "string" }); + yargs.options({ + "description": { describe: "Description for mod pack", type: "string" }, + "mods": { describe: "Mods in the form of name:version[:sha1]", array: true, type: "string" }, + "disabled-mods": { describe: "Mods that are in the pack but not enabled", array: true, type: "string" }, + "bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" }, + "int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" }, + "double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" }, + "string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" }, + }); + }], + handler: async function(args, control) { + const modPack = new libData.ModPack(); + modPack.name = args.name; + if (args.description) { modPack.description = args.description; } + if (args.factorioVersion) { + if (!/^\d+\.\d+\.\d+?$/.test(args.factorioVersion)) { + throw new libErrors.CommandError("factorio-version must match the format digit.digit.digit"); + } + modPack.factorioVersion = args.factorioVersion; + } + setModPackMods(modPack, args.mods); + setModPackModsEnabled(modPack, args.disabledMods, false); + setModPackSettings(modPack, args); + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + print(`Created mod pack ${modPack.name} (${modPack.id})`); + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: ["import ", "Import mod pack string", (yargs) => { + yargs.positional("string", { describe: "Mod pack string to import", type: "string" }); + }], + handler: async function(args, control) { + const modPack = libData.ModPack.fromModPackString(args.string); + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + print(`Created mod pack ${modPack.name} (${modPack.id})`); + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: ["export ", "Export mod pack string", (yargs) => { + yargs.positional("string", { describe: "Mod pack to export", type: "string" }); + }], + handler: async function(args, control) { + const response = await libLink.messages.getModPack.send(control, { + id: await libCommand.resolveModPack(control, args.modPack), + }); + const modPack = new libData.ModPack(response.mod_pack); + print(modPack.toModPackString()); + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: ["edit ", "Edit mod pack", (yargs) => { + yargs.positional("mod-pack", { describe: "Mod pack to remove from", type: "string" }); + yargs.options({ + "name": { describe: "New name for mod pack", type: "string" }, + "description": { describe: "New description for mod pack", type: "string" }, + "factorio-version": { describe: "Set version of factorio the mod pack is for", type: "string" }, + "add-mods": { describe: "Mods in the form of name:version[:sha1] to add", array: true, type: "string" }, + "enable-mods": { describe: "Mods to set as enabled", array: true, type: "string" }, + "disable-mods": { describe: "Mods to set as disabled", array: true, type: "string" }, + "remove-mods": { describe: "Name of mods to remove", array: true, type: "string" }, + "bool-setting": { describe: "Set boolean setting", array: true, nargs: 3, type: "string" }, + "int-setting": { describe: "Set int setting", array: true, nargs: 3, type: "string" }, + "double-setting": { describe: "Set double setting", array: true, nargs: 3, type: "string" }, + "string-setting": { describe: "Set string setting", array: true, nargs: 3, type: "string" }, + "remove-setting": { describe: "Remove a setting", array: true, nargs: 2, type: "string" }, + }); + }], + handler: async function(args, control) { + let response = await libLink.messages.getModPack.send(control, { + id: await libCommand.resolveModPack(control, args.modPack), + }); + let modPack = new libData.ModPack(response.mod_pack); + + if (args.name) { modPack.name = args.name; } + if (args.description) { modPack.description = args.description; } + if (args.factorioVersion) { + if (!/^\d+\.\d+\.\d+?$/.test(args.factorioVersion)) { + throw new libErrors.CommandError("factorio-version must match the format digit.digit.digit"); + } + modPack.factorioVersion = args.factorioVersion; + } + + setModPackMods(modPack, args.addMods); + for (let name of args.removeMods || []) { + if (modPack.mods.has(name)) { + modPack.mods.delete(name); + } else { + logger.warn(`Mod ${name} did not exist on ${modPack.name}`); + } + } + setModPackModsEnabled(modPack, args.disableMods, false); + setModPackModsEnabled(modPack, args.enableMods, true); + + setModPackSettings(modPack, args); + if (args.removeSetting) { + for (let i = 0; i + 1 < args.removeSetting.length; i += 2) { + const [scope, name] = settings.slice(i, i + 2); + if (!["startup", "runtime-global", "runtime-per-user"].includes(scope)) { + throw new libErrors.CommandError( + `Setting scope must be one of startup, runtime-gloabl, or runtime-per-user, not ${scope}` + ); + } + if (modPack.settings[scope].has(name)) { + modPack.settings[scope].delete(name); + } else { + logger.warn(`Mod setting ${scope} ${name} did not exist on ${modPack.name}`); + } + } + } + await libLink.messages.updateModPack.send(control, { mod_pack: modPack.toJSON() }); + }, +})); + +modPackCommands.add(new libCommand.Command({ + definition: ["delete ", "Delete mod pack", (yargs) => { + yargs.positional("mod-pack", { describe: "Mod pack to delete", type: "string" }); + }], + handler: async function(args, control) { + const id = await libCommand.resolveModPack(control, args.modPack); + await libLink.messages.deleteModPack.send(control, { id }); + }, +})); + +const modCommands = new libCommand.CommandTree({ name: "mod", description: "Manage uploaded mods" }); +modCommands.add(new libCommand.Command({ + definition: ["show ", "Show details for a mod stored in the cluster", (yargs) => { + yargs.positional("name", { describe: "Mod name to show details for", type: "string" }); + yargs.positional("mod-version", { describe: "Version of the mod", type: "string" }); + }], + handler: async function(args, control) { + let response = await libLink.messages.getMod.send(control, { name: args.name, version: args.modVersion }); + for (let [field, value] of Object.entries(response.mod)) { + if (value instanceof Array) { + print(`${field}:`); + for (let entry of value) { + print(` ${entry}`); + } + } else { + print(`${field}: ${value}`); + } + } + }, +})); + +modCommands.add(new libCommand.Command({ + definition: [["list", "l"], "List mods stored in the cluster", (yargs) => { + yargs.options({ + "fields": { + describe: "Fields to show, supports 'all'.", + array: true, + type: "string", + default: ["name", "version", "title", "factorio_version"], + }, + }); + }], + handler: async function(args, control) { + let response = await libLink.messages.listMods.send(control); + if (!args.fields.includes("all")) { + for (let entry of response.list) { + for (let field of Object.keys(entry)) { + if (!args.fields.includes(field)) { + delete entry[field]; + } + } + } + } + print(asTable(response.list)); + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["search [query]", "Search mods stored in the cluster", (yargs) => { + yargs.positional("factorio-version", { describe: "Major version of Factorio to search for", type: "string" }); + yargs.positional("query", { describe: "Search query", type: "string", default: "" }); + yargs.options({ + "page": { + describe: "Result page to show", + type: "number", + default: 1, + }, + "page-size": { + describe: "Results per page to show", + type: "number", + default: 10, + }, + "sort": { + describe: "sort results by given field", + type: "string", + }, + "sort-order": { + describe: "order to sort results in (asc/desc)", + type: "string", + default: "asc", + }, + "fields": { + describe: "Fields to show, supports 'all'.", + array: true, + type: "string", + default: ["name", "version", "title", "factorio_version"], + }, + }); + }], + handler: async function(args, control) { + let response = await libLink.messages.searchMods.send(control, { + "query": args.query, + "factorio_version": args.factorioVersion, + "page_size": args.pageSize, + "page": args.page, + "sort": args.sort, + "sort_order": args.sortOrder, + }); + let results = response.results.flatMap(result => result.versions); + if (!args.fields.includes("all")) { + for (let entry of results) { + for (let field of Object.keys(entry)) { + if (!args.fields.includes(field)) { + delete entry[field]; + } + } + } + } + for (let issue of response.query_issues) { + print(issue); + } + print(`page ${args.page} of ${response.page_count} (${response.result_count} results)`); + print(asTable(results)); + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["upload ", "Upload mod to the cluster", (yargs) => { + yargs.positional("file", { describe: "File to upload", type: "string" }); + }], + handler: async function(args, control) { + let filename = path.basename(args.file); + if (!filename.endsWith(".zip")) { + throw new libErrors.CommandError("Mod filename must end with .zip"); + } + // phin doesn't support streaming requests :( + let content = await fs.readFile(args.file); + + let url = new URL(control.config.get("control.master_url")); + url.pathname += "api/upload-mod"; + url.searchParams.append("filename", filename); + + let result = await phin({ + url, method: "POST", + headers: { + "X-Access-Token": control.config.get("control.master_token"), + "Content-Type": "application/zip", + }, + core: { ca: control.tlsCa }, + data: content, + parse: "json", + }); + + for (let error of result.body.errors || []) { + logger.error(error); + } + + for (let requestError of result.body.request_errors || []) { + logger.error(requestError); + } + + if (result.body.mods && result.body.mods.length) { + const mod = new libData.ModInfo(result.body.mods[0]); + logger.info(`Successfully uploaded ${mod.filename}`); + } + + if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + throw new libErrors.CommandError("Uploading mod failed"); + } + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["download ", "Download a mod from the cluster", (yargs) => { + yargs.positional("name", { describe: "Internal name of mod to download", type: "string" }); + yargs.positional("mod-version", { describe: "Version of mod to download", type: "string" }); + }], + handler: async function(args, control) { + let result = await libLink.messages.downloadMod.send(control, { + name: args.name, + version: args.modVersion, + }); + + let url = new URL(control.config.get("control.master_url")); + url.pathname += `api/stream/${result.stream_id}`; + let response = await phin({ + url, method: "GET", + core: { ca: control.tlsCa }, + stream: true, + }); + + let writeStream; + let filename = `${args.name}_${args.modVersion}.zip`; + let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + while (true) { + try { + writeStream = fs.createWriteStream(tempFilename, { flags: "wx" }); + await events.once(writeStream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(".", tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + response.pipe(writeStream); + await finished(writeStream); + await fs.rename(tempFilename, filename); + + logger.info(`Downloaded ${filename}`); + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["delete ", "Delete a mod stored in the cluster", (yargs) => { + yargs.positional("name", { describe: "Name of mod to delete", type: "string" }); + yargs.positional("mod-version", { describe: "Version of mod to delete", type: "string" }); + }], + handler: async function(args, control) { + await libLink.messages.deleteMod.send(control, { name: args.name, version: args.modVersion }); + }, +})); + const permissionCommands = new libCommand.CommandTree({ name: "permission", description: "Permission inspection" }); permissionCommands.add(new libCommand.Command({ definition: [["list", "l"], "List permissions in the cluster"], @@ -1194,6 +1654,8 @@ async function registerCommands(controlPlugins, yargs) { rootCommands.add(masterCommands); rootCommands.add(slaveCommands); rootCommands.add(instanceCommands); + rootCommands.add(modPackCommands); + rootCommands.add(modCommands); rootCommands.add(permissionCommands); rootCommands.add(roleCommands); rootCommands.add(userCommands); diff --git a/packages/lib/browser.js b/packages/lib/browser.js index 23c9850e..20866a72 100644 --- a/packages/lib/browser.js +++ b/packages/lib/browser.js @@ -1,4 +1,5 @@ export { default as libConfig } from "./config"; +export { default as libData } from "./data"; export { default as libErrors } from "./errors"; export { default as libFactorio } from "./factorio/exchange_string"; export { default as libHelpers } from "./helpers"; diff --git a/packages/lib/command.js b/packages/lib/command.js index 133a8805..f0738e8b 100644 --- a/packages/lib/command.js +++ b/packages/lib/command.js @@ -198,6 +198,39 @@ async function resolveInstance(client, instanceName) { return instanceId; } +/** + * Resolve a string to a mod pack ID + * + * Resolevs a string with either a mod pack name or an id into an integer + * with the mod pack ID. + * + * @param {module:lib/link.Link} client - + * link to master server to query mod pack on. + * @param {string} modPackName - string with name or id of mod pack. + * @returns {Promise} mod pack ID. + * @static + */ +async function resolveModPack(client, modPackName) { + let modPackId; + if (/^-?\d+/.test(modPackName)) { + modPackId = parseInt(modPackName, 10); + } else { + let response = await libLink.messages.listModPacks.send(client); + for (let mod_pack of response.list) { + if (mod_pack["name"] === modPackName) { + modPackId = mod_pack["id"]; + break; + } + } + + if (modPackId === undefined) { + throw new libErrors.CommandError(`No mod pack named ${modPackName}`); + } + } + + return modPackId; +} + /** * Retrieve role object from string * @@ -244,5 +277,6 @@ module.exports = { resolveSlave, resolveInstance, + resolveModPack, retrieveRole, }; diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index e713f48f..d6605c85 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -19,6 +19,13 @@ MasterGroup.define({ type: "string", initial_value: "Your Cluster", }); +MasterGroup.define({ + name: "mods_directory", + title: "Mods Directory", + description: "Path to directory where mods shared with the cluster are stored.", + type: "string", + initial_value: "mods", +}); MasterGroup.define({ name: "database_directory", title: "Database directory", @@ -108,6 +115,13 @@ MasterGroup.define({ type: "number", initial_value: 15, }); +MasterGroup.define({ + name: "default_mod_pack_id", + title: "Default Mod Pack", + description: "Mod pack used by default for instances.", + type: "number", + optional: true, +}); MasterGroup.define({ name: "default_role_id", title: "Default role", @@ -153,6 +167,20 @@ SlaveGroup.define({ type: "string", initial_value: "factorio", }); +SlaveGroup.define({ + name: "mods_directory", + title: "Mods Directory", + description: "Path to directory where mods for instances are cached.", + type: "string", + initial_value: "mods", +}); +SlaveGroup.define({ + name: "mods_directory_is_shared", + title: "Mods Directory is Shared", + description: "True if the mods directory is shared with the master server.", + type: "boolean", + initial_value: true, +}); SlaveGroup.define({ name: "instances_directory", description: "Path to directory to store instances in.", @@ -273,6 +301,12 @@ FactorioGroup.define({ type: "number", initial_value: 5, }); +FactorioGroup.define({ + name: "mod_pack", + description: "Mod pack to use on this server", + type: "number", + optional: true, +}); FactorioGroup.define({ name: "enable_save_patching", description: "Patch saves with Lua code. Required for Clusterio integrations, lua modules, and most plugins.", diff --git a/packages/lib/data/ExportManifest.js b/packages/lib/data/ExportManifest.js new file mode 100644 index 00000000..a07d663e --- /dev/null +++ b/packages/lib/data/ExportManifest.js @@ -0,0 +1,46 @@ +"use strict"; + +/** + * Export manifest for a mod pack + * + * Tracks files stored on disk for data that extracted from Factorio using a + * given mod pack. + * @alias module:lib/data.ExportManifest + */ +class ExportManifest { + + /** + * Mapping between known assets and their file names. + * + * Currently exported assets are + * - settings: JSON of mod settings prototypes. + * - prototypes: JSON of all game prototypes except settings. + * - item-spritesheet: PNG containing icons from the game. + * - item-metadata: JSON of Map entries describing icons in item-spritesheet + * - locale: JSON of flattened Map entries of en locale strings from the game. + * @type {Object} + */ + assets = {}; + + static jsonSchema = { + type: "object", + additionalProperties: false, + required: ["assets"], + properties: { + "assets": { + type: "object", + additionalProperties: { type: "string" }, + }, + }, + }; + + constructor(json = {}) { + if (json.assets) { this.assets = json.assets; } + } + + toJSON() { + return { assets: this.assets }; + } +} + +module.exports = ExportManifest; diff --git a/packages/lib/data/ModInfo.js b/packages/lib/data/ModInfo.js new file mode 100644 index 00000000..2c254935 --- /dev/null +++ b/packages/lib/data/ModInfo.js @@ -0,0 +1,231 @@ +"use strict"; +const fs = require("fs-extra"); +const JSZip = require("jszip"); +const path = require("path"); + +const libHash = require("../hash"); +const libSchema = require("../schema"); +const { findRoot } = require("../zip_ops"); + +const { integerModVersion, integerFactorioVersion, modVersionRegExp } = require("./version"); + + +/** + * Info about a mod available on the master server. + * + * See https://wiki.factorio.com/Tutorial:Mod_structure#info.json for + * details of the invidiual fields sourced from info.json. + * @alias module:lib/data.ModInfo + */ +class ModInfo { + /** + * Internal name of this mod. + * This is the name of the zip file as well as the name that appears in + * info.json. + * @type {string} + */ + name = ""; + + /** + * Version of the mod. + * Sourced from info.json. + * @type {string} + */ + version = ""; + + /** + * Integer representation of the version + * @type {number} + */ + get integerVersion() { + return integerModVersion(this.version); + } + + /** + * Display name of the mod. + * Sourced from info.json. + * @type {string} + */ + title = ""; + + /** + * Author of the mod. + * Sourced from info.json. + * @type {string} + */ + author = ""; + + /** + * Contact field for mod. + * Sourced from info.json. + * @type {string} + */ + contact = ""; + + /** + * Homepage of the mod. + * Sourced from info.json. + * @type {string} + */ + homepage = ""; + + /** + * Description of the mod. + * Sourced from info.json. + * @type {string} + */ + description = ""; + + /** + * Major version of Factorio this mod supports. + * Sourced from info.json. + * @type {string} + */ + factorioVersion = "0.12"; + + /** + * Integer representation of the factorioVersion + * @type {number} + */ + get integerFactorioVersion() { + return integerFactorioVersion(this.factorioVersion); + } + + /** + * Dependiences for this mod. + * Sourced from info.json. + * @type {Array} + */ + dependencies = ["base"]; + + /** + * Expected name of zip file containing this mod. + * @type {number} + */ + get filename() { + return `${this.name}_${this.version}.zip`; + } + + /** + * Size of the mod in bytes + * @type {number} + */ + size = 0; + + /** + * SHA1 hash of this mod + * @type {string=} + */ + sha1; + + /** + * True if this mod has been deleted + * @type {boolean} + */ + isDeleted = false; + + // Content of info.json found in mod files + static infoJsonSchema = { + type: "object", + required: ["name", "version", "title", "author"], + properties: { + "name": { type: "string" }, + "version": { type: "string" }, + "title": { type: "string" }, + "author": { type: "string" }, + "contact": { type: "string" }, + "homepage": { type: "string" }, + "description": { type: "string" }, + "factorio_version": { type: "string" }, + "dependencies": { type: "array", items: { type: "string" } }, + }, + }; + + static validateInfo = libSchema.compile(this.infoJsonSchema); + + static jsonSchema = { + type: "object", + properties: { + ...this.infoJsonSchema.properties, + "size": { type: "integer" }, + "sha1": { type: "string" }, + "is_deleted": { type: "boolean" }, + }, + }; + + static validate = libSchema.compile(this.jsonSchema); + + constructor(json = {}) { + // info.json fields + if (json.name) { this.name = json.name; } + if (json.version) { this.version = json.version; } + if (json.title) { this.title = json.title; } + if (json.author) { this.author = json.author; } + if (json.contact) { this.contact = json.contact; } + if (json.homepage) { this.homepage = json.homepage; } + if (json.description) { this.description = json.description; } + if (json.factorio_version) { this.factorioVersion = json.factorio_version; } + if (json.dependencies) { this.dependencies = json.dependencies; } + + // Additional data + if (json.size) { this.size = json.size; } + if (json.sha1) { this.sha1 = json.sha1; } + if (json.is_deleted) { this.isDeleted = json.is_deleted; } + } + + toJSON() { + let json = { + name: this.name, + version: this.version, + title: this.title, + author: this.author, + }; + if (this.contact) { json.contact = this.contact; } + if (this.homepage) { json.homepage = this.homepage; } + if (this.description) { json.description = this.description; } + if (this.factorio_version !== "0.12") { json.factorio_version = this.factorioVersion; } + if (this.dependencies.length !== 1 || this.dependencies[0] !== "base") { + json.dependencies = this.dependencies; + } + if (this.size) { json.size = this.size; } + if (this.sha1) { json.sha1 = this.sha1; } + if (this.isDeleted) { json.is_deleted = this.isDeleted; } + return json; + } + + static async fromModFile(modPath) { + let modInfo; + { + // XXX: JSZip needs the whole archive loaded in memory to work. + // This is clearly untenable and will be replaced later. + let zip = await JSZip.loadAsync(await fs.readFile(modPath)); + let root = zip.folder(findRoot(zip)); + + let infoFile = root.file("info.json"); + if (!infoFile) { + throw new Error("Mod contains no info.json file"); + } + + modInfo = JSON.parse(await infoFile.async("string")); + } + + let valid = this.validateInfo(modInfo); + if (!valid) { + throw new Error("Mod's info.json is not valid"); + } + + if (!modVersionRegExp.test(modInfo.version)) { + throw new Error(`Mod's version (${modInfo.version}) is invalid`); + } + + return new this({ + ...modInfo, + + size: (await fs.stat(modPath)).size, + sha1: await libHash.hashFile(modPath), + is_deleted: false, + }); + } +} + +module.exports = ModInfo; diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js new file mode 100644 index 00000000..8279cf03 --- /dev/null +++ b/packages/lib/data/ModPack.js @@ -0,0 +1,355 @@ +"use strict"; +const zlib = require("zlib"); + +const libSchema = require("../schema"); + +const ExportManifest = require("./ExportManifest"); +const { integerFactorioVersion } = require("./version"); + + +/** + * A setting for a mod. + * @typedef {object} module:lib/data.ModPack~ModSetting + * @property {boolean|number|string} value - Value of the given mod setting. + */ +const ModSettingJsonSchema = { + type: "object", + additionalProperties: false, + required: ["value"], + properties: { + "value": { type: ["boolean", "number", "string"] }, + }, +}; + +const ModSettingsJsonSchema = { + type: "object", + additionalProperties: ModSettingJsonSchema, +}; + +/** + * A mod in a mod pack + * @typedef {object} module:lib/data.ModPack~ModRecord + * @property {string} name - name of the mod. + * @property {boolean} enabled - if mod is to be loaded. + * @property {string} version - version of the mod. + * @property {string=} sha1 - SHA1 hash of the zip file. + */ +const ModRecordJsonSchema = { + type: "object", + required: ["name", "enabled", "version"], + properties: { + "name": { type: "string" }, + "enabled": { type: "boolean" }, + "version": { type: "string" }, + "sha1": { type: "string" }, + }, +}; + +/** + * Factorio Mod Pack + * + * Tracks mods and settings for a collection of Factorio mods. + * @alias module:lib/data.ModPack + */ +class ModPack { + /** + * Id of this mod pack + * @type {number} + */ + id; + + /** + * Name of this mod pack. + * @type {string} + */ + name = ""; + + /** + * Description for this modpcak. + * @type {string} + */ + description = ""; + + /** + * Version of Factorio this mod pack is for + * @type {string} + */ + factorioVersion = "1.1.0"; + + /** + * Integer representation of the factorioVersion + * @type {number} + */ + get integerFactorioVersion() { + return integerFactorioVersion(this.factorioVersion); + } + + /** + * Mods included in this mod pack + * @type {Map} + */ + mods = new Map(); + + /** + * Mod settings for this mod pack + * @type {Object>=} + */ + settings = { + "startup": new Map(), + "runtime-global": new Map(), + "runtime-per-user": new Map(), + }; + + /** + * Mapping to files containing exported data for this modpack + * @type {module:lib/data.ExportManifest|undefined} + */ + exportManifest; + + /** + * True if this mod pack has been deleted from the list of mod packs. + * @type {boolean} + */ + isDeleted = false; + + static jsonSchema = { + type: "object", + additionalProperties: false, + required: ["name", "description", "factorio_version", "mods", "settings"], + properties: { + "id": { type: "integer" }, + "name": { type: "string" }, + "description": { type: "string" }, + "factorio_version": { type: "string" }, + "mods": { + type: "array", + items: ModRecordJsonSchema, + }, + "settings": { + type: "object", + additionalProperties: false, + required: ["startup", "runtime-global", "runtime-per-user"], + properties: { + "startup": ModSettingsJsonSchema, + "runtime-global": ModSettingsJsonSchema, + "runtime-per-user": ModSettingsJsonSchema, + }, + }, + "export_manifest": ExportManifest.jsonSchema, + "is_deleted": { type: "boolean" }, + }, + }; + + static validate = libSchema.compile(this.jsonSchema); + + constructor(json = {}) { + if (json.id) { + this.id = json.id; + } else { + this.id = Math.random() * 2**31 | 0; + } + if (json.name) { this.name = json.name; } + if (json.description) { this.description = json.description; } + if (json.factorio_version) { this.factorioVersion = json.factorio_version; } + if (json.mods) { this.mods = new Map(json.mods.map(m => [m.name, m])); } + if (json.settings) { + this.settings = { + "startup": new Map(Object.entries(json.settings["startup"])), + "runtime-global": new Map(Object.entries(json.settings["runtime-global"])), + "runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])), + }; + } + if (json.export_manifest) { this.exportManifest = new ExportManifest(json.export_manifest); } + if (json.is_deleted) { this.isDeleted = json.is_deleted; } + + if (!this.mods.has("base")) { + this.mods.set("base", { name: "base", enabled: true, version: this.factorioVersion }); + } + } + + toJSON() { + let json = { + id: this.id, + name: this.name, + description: this.description, + factorio_version: this.factorioVersion, + mods: [...this.mods.values()], + settings: { + "startup": Object.fromEntries(this.settings["startup"]), + "runtime-global": Object.fromEntries(this.settings["runtime-global"]), + "runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]), + }, + }; + if (this.exportManifest) { json.export_manifest = this.exportManifest.toJSON(); } + if (this.isDeleted) { json.is_deleted = this.isDeleted; } + return json; + } + + toModPackString() { + const json = this.toJSON(); + delete json.id; + delete json.export_manifest; + + // eslint-disable-next-line node/no-sync + let buf = zlib.deflateSync(JSON.stringify(json)); + return buf.toString("base64"); + } + + static fromModPackString(modPackString) { + let buf = Buffer.from(modPackString, "base64"); + try { + // eslint-disable-next-line node/no-sync + buf = zlib.inflateSync(buf); + } catch (err) { + if (err.code.startsWith("Z_")) { + throw new Error("Malformed mod pack string: zlib inflate failed"); + } + } + + let json; + try { + json = JSON.parse(buf); + } catch (err) { + throw new Error(`Malformed mod pack string: ${err.message}`); + } + + if (!this.validate(json)) { + throw new Error("Malformed mod pack string: Schema validation failed"); + } + + return new this(json); + } + + toModSettingsDat() { + function uint8Byte(int) { + return Buffer.from(Uint8Array.from([int]).buffer); + } + function int16Bytes(int) { + return Buffer.from(Int16Array.from([int]).buffer); + } + function uint32Bytes(int) { + return Buffer.from(Uint32Array.from([int]).buffer); + } + function uint32SpaceOptimizedBytes(int) { + if (0 <= int && int < 0xff) { + return uint8Byte(int); + } + return Buffer.concat([ + uint8Byte(0xff), + uint32Bytes(int), + ]); + } + function doubleBytes(double) { + return Buffer.from(Float64Array.from([double]).buffer); + } + function stringBytes(str) { + const buf = Buffer.from(str, "utf8"); + return Buffer.concat([ + uint32SpaceOptimizedBytes(buf.length), + buf, + ]); + } + function immutableStringBytes(str) { + if (str === undefined) { + return uint8Byte(1); // empty + } + return Buffer.concat([ + uint8Byte(0), // empty + stringBytes(str), + ]); + } + function versionBytes(version) { + const [main, major, minor] = version.split(".").map(n => Number.parseInt(n, 10)); + return Buffer.concat([ + int16Bytes(main), + int16Bytes(major), + int16Bytes(minor), + Buffer.alloc(2), // developer + Buffer.alloc(1), // reserved byte + ]); + } + function propertyTreeListBytes(entries) { + const sizeBytes = uint32Bytes(entries.length); + const itemBytes = entries.flatMap(([key, item]) => [ + immutableStringBytes(key), + // eslint-disable-next-line no-use-before-define + propertyTreeBytes(item), + ]); + return Buffer.concat([ + sizeBytes, + ...itemBytes, + ]); + } + function propertyTreeBytes(element) { + let type; + let dataBytes; + if (typeof element === null) { + type = 1; + dataBytes = Buffer.alloc(0); + } else if (typeof element === "boolean") { + type = 1; + dataBytes = uint8Byte(element); + } else if (typeof element === "number") { + type = 2; + dataBytes = doubleBytes(element); + } else if (typeof element === "string") { + type = 3; + dataBytes = immutableStringBytes(element); + } else if (element instanceof Array) { + type = 4; + dataBytes = propertyTreeListBytes(element.map(item => [undefined, item])); + } else if (typeof element === "object") { + type = 5; + dataBytes = propertyTreeListBytes(Object.entries(element)); + } else { + throw new Error("Bad element passed to propertyTreeBytes"); + } + return Buffer.concat([ + uint8Byte(type), + Buffer.alloc(1), // anyTypeFlag + dataBytes, + ]); + } + + return Buffer.concat([ + versionBytes(this.factorioVersion), + propertyTreeBytes(this.toJSON().settings), + ]); + } + + /** + * Fill in missing settings with their default values + * + * Uses the provided setting prototypes to add any missing mod settings + * in the mod pack with the default value from the prototype. + * + * @param {Object} settingPrototypes - + * Setting prototypes exported from the game. + * @param {Logger} logger - Logger used to report warnings on. + */ + fillDefaultSettings(settingPrototypes, logger) { + const knownTypes = ["bool-setting", "int-setting", "double-setting", "string-setting"]; + let prototypes = Object.entries(settingPrototypes) + .filter(([type, _]) => knownTypes.includes(type)) + .flatMap(([_, settings]) => Object.values(settings)) + ; + + for (let prototype of prototypes) { + if (!["startup", "runtime-global", "runtime-per-user"].includes(prototype.setting_type)) { + logger.warn(`Ignoring ${prototype.name} with unknown setting_type ${prototype.setting_type}`); + continue; + } + if (prototype.default_value === undefined) { + logger.warn(`Ignoring ${prototype.name} with missing default_value`); + continue; + } + if (this.settings[prototype.setting_type].get(prototype.name)) { + continue; + } + + this.settings[prototype.setting_type].set(prototype.name, { value: prototype.default_value }); + } + } +} + +module.exports = ModPack; diff --git a/packages/lib/data/index.js b/packages/lib/data/index.js new file mode 100644 index 00000000..f3856763 --- /dev/null +++ b/packages/lib/data/index.js @@ -0,0 +1,17 @@ +/** + * Shared data types used in Clusterio + * @module lib/data + * @author Hornwitser + */ +"use strict"; +const ExportManifest = require("./ExportManifest"); +const ModInfo = require("./ModInfo"); +const ModPack = require("./ModPack"); +const version = require("./version"); + +module.exports = { + ExportManifest, + ModInfo, + ModPack, + ...version, +}; diff --git a/packages/lib/data/version.js b/packages/lib/data/version.js new file mode 100644 index 00000000..32706c04 --- /dev/null +++ b/packages/lib/data/version.js @@ -0,0 +1,44 @@ +// Helper functions for dealing with factorio version numbers +"use strict"; + + +/** + * Integer representation of a mod version + * + * Suitable for sorting and comparing version numbers which may have different + * number of leading zeros in them. + * @param {string} version - string representation of version. + * @returns {number} numeric representation. + * @alias module:lib/data.integerModVersion + */ +function integerModVersion(version) { + const [major, minor, sub] = version.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000 + sub; // Can't use bitwise here because this is 48-bits. +} + +/** + * Integer representation of a factorio version + * + * Suitable for sorting and comparing version numbers which may have different + * number of leading zeros in them. + * @param {string} version - string representation of version. + * @returns {number} numeric representation. + * @alias module:lib/data.integerFactorioVersion + */ +function integerFactorioVersion(version) { + const [main, major, minor] = version.split(".").map(n => Number.parseInt(n, 10)); + return main * 0x100000000 + major * 0x10000 + (minor || 0); // Can't use bitwise here because this is 48-bits. +} + +/** + * Matches valid mod versions. + * @type {RegExp} + * @alias module:lib/data.modVersionRegExp + */ +const modVersionRegExp = /^\d+\.\d+\.\d+$/; + +module.exports = { + integerModVersion, + integerFactorioVersion, + modVersionRegExp, +}; diff --git a/packages/lib/factorio/export.js b/packages/lib/factorio/export.js index 173f5cb3..22c74de9 100644 --- a/packages/lib/factorio/export.js +++ b/packages/lib/factorio/export.js @@ -7,6 +7,7 @@ const JSZip = require("jszip"); const libIni = require("../ini"); const libBuildMod = require("../build_mod"); +const libZipOps = require("../zip_ops"); /** * Generate the export mod needed for exportData @@ -56,7 +57,7 @@ async function loadZip(server, modVersions, mod) { zipCache.set(zipPath, zip); } - return zip.folder(`${mod}_${modVersion}`); + return zip.folder(libZipOps.findRoot(zip)); } /** @@ -207,6 +208,42 @@ async function loadLayeredIcon(server, modVersions, item, size, iconCache) { return icon; } +const itemTypes = new Set([ + "item", + "ammo", + "capsule", + "gun", + "item-with-entity-data", + "item-with-label", + "item-with-inventory", + "blueprint-book", + "item-with-tags", + "selection-tool", + "blueprint", + "copy-paste-tool", + "deconstruction-item", + "upgrade-item", + "module", + "rail-planner", + "spidertron-remote", + "tool", + "armor", + "mining-tool", + "repair-tool", + + // XXX Bad hack to get icons for fluids. These should be treated as a + // separate namespace from items as they may be named the same as items. + "fluid", + // TODO consider adding virtual signals and recipes +]); + +function filterItems(prototypes) { + return Object.entries(prototypes) + .filter(([type, _]) => itemTypes.has(type)) + .flatMap(([_, typePrototypes]) => Object.values(typePrototypes)) + ; +} + /** * Export item icons and data * @@ -348,11 +385,29 @@ async function exportLocale(server, modVersions, modOrder, languageCode) { async function exportData(server) { await generateExportMod(server); - let items = []; + let settings = {}; + let prototypes = {}; let modVersions = new Map(); let modOrder = []; - server.on("ipc-item_export", data => items.push(data)); + function add(obj, prototype) { + if (!Object.prototype.hasOwnProperty.call(obj, prototype.type)) { + obj[prototype.type] = {}; + } + obj[prototype.type][prototype.name] = prototype; + } + + server.on("ipc-prototype_export", data => add(prototypes, data)); + server.on("ipc-settings_export", data => add(settings, data)); + server.on("ipc-mod_setting_mod", ({ name, mod }) => { + for (let type of Object.values(settings)) { + if (Object.prototype.hasOwnProperty.call(type, name)) { + type[name].mod = mod; + return; + } + } + server._logger.error(`Unable to find ${name} in settings prototypes`); + }); server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); }); server.on("output", parsed => { if (parsed.format === "seconds" && parsed.type === "generic") { @@ -370,8 +425,8 @@ async function exportData(server) { await fs.unlink(server.writePath("mods", "export_0.0.0.zip")); } - if (!items.length) { - throw new Error("No items got exported"); + if (!Object.keys(prototypes).length) { + throw new Error("No prototypes got exported"); } // Some mod authors put leading zeros into the versions of their zip files. @@ -396,13 +451,15 @@ async function exportData(server) { } } - let { iconSheet, itemData } = await exportItems(server, modVersions, items); + let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes)); let locale = await exportLocale(server, modVersions, modOrder, "en"); // Free up the memory used by zip files loaded during the export. zipCache.clear(); let zip = new JSZip(); + zip.file("export/settings.json", JSON.stringify(settings)); + zip.file("export/prototypes.json", JSON.stringify(prototypes)); zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG)); zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()])); zip.file("export/locale.json", JSON.stringify([...locale.entries()])); diff --git a/packages/lib/factorio/patch.js b/packages/lib/factorio/patch.js index f96e931a..7f92232f 100644 --- a/packages/lib/factorio/patch.js +++ b/packages/lib/factorio/patch.js @@ -8,6 +8,7 @@ const path = require("path"); const semver = require("semver"); const libHash = require("../hash"); +const { findRoot } = require("../zip_ops"); const knownScenarios = { @@ -21,42 +22,6 @@ const knownScenarios = { }, }; -/** - * Returns the root folder in the zip file - * - * Returns the name of the folder that all files in the zip file is - * contained in. Throws an error if there are multiple such folders. - * - * @param {JSZip} zip - Zip to search through. - * @returns {string} name of the root folder. - * @memberof module:lib/factorio - * @private - * @inner - */ -function findRoot(zip) { - let root = null; - zip.forEach((relativePath, file) => { - let index = relativePath.indexOf("/"); - if (index === -1) { - throw new Error(`Zip contains file '${relativePath}' in root dir`); - } - - let pathRoot = relativePath.slice(0, index); - if (root === null) { - root = pathRoot; - } else if (root !== pathRoot) { - throw new Error("Zip contains multiple root folders"); - } - }); - - if (root === null) { - throw new Error("Empty zip file"); - } - - return root; -} - - /** * Generates control.lua code for loading the Clusterio modules * @@ -295,7 +260,6 @@ module.exports = { patch, // For testing only - _findRoot: findRoot, _generateLoader: generateLoader, _reorderDependencies: reorderDependencies, }; diff --git a/packages/lib/file_ops.js b/packages/lib/file_ops.js index c04077be..be007912 100644 --- a/packages/lib/file_ops.js +++ b/packages/lib/file_ops.js @@ -108,9 +108,94 @@ async function safeOutputFile(file, data, options={}) { await fs.rename(temporary, file); } + +// Reserved names by allmost all filesystems +const badNames = [".", ".."]; + +// Reserved namespaces in Windows +const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; +const badWinNamespaces = [ + "CON", "PRN", "AUX", "NUL", + ...oneToNine.map(n => `COM${n}`), + ...oneToNine.map(n => `LPT${n}`), +]; + +/** + * Check if a string is a valid file name + * + * @param {string} name - Name to check + * @throws Error if the name is unsuitable. + */ +function checkFilename(name) { + // All of these are bad in Windows only, except for /, . and .. + // See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; + const badEnd = /[. ]$/; + + if (typeof name !== "string") { + throw new Error("must be a string"); + } + + if (name === "") { + throw new Error("cannot be empty"); + } + + if (badChars.test(name)) { + throw new Error('cannot contain <>:"\\/|=* or control characters'); + } + + if (badNames.includes(name)) { + throw new Error( + `cannot be named ${name}` + ); + } + + if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) { + throw new Error( + "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9" + ); + } + + if (badEnd.test(name)) { + throw new Error("cannot end with . or space"); + } +} + +/** + * Clean up string to be suitable for use as filename + * + * @param {string} name - Arbitrary name string + * @returns {string} Filename suitable to use in the filesystem + */ +function cleanFilename(name) { + // copied from checkFilename due to RegExp with global flag containing state. + const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; + const badEnd = /[. ]$/; + + if (typeof name !== "string") { + throw new Error("name must be a string"); + } + + if (name === "" || badNames.includes(name.toUpperCase())) { + name += "_"; + } + + if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) { + name = [`${name.split(".")[0]}_`, ...name.split(".").slice(1)].join("."); + } + + name = name.replace(badChars, "_"); + name = name.replace(badEnd, "_"); + + return name; +} + + module.exports = { getNewestFile, findUnusedName, getTempFile, safeOutputFile, + checkFilename, + cleanFilename, }; diff --git a/packages/lib/hash.js b/packages/lib/hash.js index df26114e..12359602 100644 --- a/packages/lib/hash.js +++ b/packages/lib/hash.js @@ -4,7 +4,7 @@ */ "use strict"; const crypto = require("crypto"); -const fs = require("fs"); +const fs = require("fs-extra"); /** diff --git a/packages/lib/helpers.js b/packages/lib/helpers.js index 17464ec5..3738666e 100644 --- a/packages/lib/helpers.js +++ b/packages/lib/helpers.js @@ -12,6 +12,7 @@ * * @param {*} value - value to return the type of. * @returns {string} basic type of the value passed. + * @static */ function basicType(value) { if (value === null) { return "null"; } @@ -24,6 +25,7 @@ function basicType(value) { * Asynchronously wait for the given duration * * @param {number} duration - Time to wait for in milliseconds. + * @static */ async function wait(duration) { await new Promise(resolve => { setTimeout(resolve, duration); }); @@ -36,6 +38,7 @@ async function wait(duration) { * @param {Promise} promise - Promise to wait for. * @param {number} time - Maximum time im milliseconds to wait for. * @param {*=} timeoutResult - Value to return if the operation timed out. + * @static */ async function timeout(promise, time, timeoutResult) { let timer; @@ -61,6 +64,7 @@ async function timeout(promise, time, timeoutResult) { * * @param {Readable} stream - byte stream to read to the end. * @returns {Buffer} content of the stream. + * @static */ async function readStream(stream) { let chunks = []; @@ -85,11 +89,194 @@ async function readStream(stream) { * * @param {string} text - Text to escape RegExp meta chars in. * @returns {string} escaped text. + * @static */ function escapeRegExp(text) { return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); } +/** + * Format byte count for human readable display + * + * Shortens a large number of bytes using the kB/MB/GB/TB prefixes. + * @param {number} bytes - Count of bytes to format. + * @returns {string} formatted text. + * @static + */ +function formatBytes(bytes) { + if (bytes === 0) { + return "0\u{A0}Bytes"; // No-break space + } + + let units = ["\u{A0}Bytes", "\u{A0}kB", "\u{A0}MB", "\u{A0}GB", "\u{A0}TB"]; + let factor = 1000; + let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length); + return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; +} + +function parseSearchIdentifier(pos, input) { + // character = ? all characters excluding space : " ? + // identifier = 1*character + let startPos = pos; + while (pos < input.length && ![" ", '"', ":"].includes(input.charAt(pos))) { + pos += 1; + } + return [pos, input.slice(startPos, pos)]; +} + +function parseSearchWord(pos, input) { + // non quote character = ? all characters excluding " ? + // word op = "-" + // word = [ word op ], ( identifier | '"', *non quote character, '"' ) + let word = { + type: "word", + }; + if (input.charAt(pos) === "-") { + word.exclude = true; + pos += 1; + } + if (input.charAt(pos) === '"') { + pos += 1; + let startPos = pos; + let endPos = pos; + while (pos < input.length && input.charAt(pos) !== '"') { + pos += 1; + endPos += 1; + } + if (input.charAt(pos) === '"') { + pos += 1; + } + word.value = input.slice(startPos, endPos); + } else { + ([pos, word.value] = parseSearchIdentifier(pos, input)); + } + return [pos, word]; +} + +function parseSearchTerm(pos, input, attributes, issues) { + // attribute = identifier, ':', word + // term = word | attribute + let term; + if (["-", '"'].includes(input.charAt(pos))) { + ([pos, term] = parseSearchWord(pos, input)); + } else { + let identifier; + ([pos, identifier] = parseSearchIdentifier(pos, input)); + if (input.charAt(pos) === ":") { + pos += 1; + identifier = identifier.toLowerCase(); + if (!Object.prototype.hasOwnProperty.call(attributes, identifier)) { + issues.push(`Unregonized attribute "${identifier}", use quotes to escape colons`); + } else if (attributes[identifier] === "word") { + let value; + ([pos, value] = parseSearchWord(pos, input)); + term = { type: "attribute", name: identifier, value }; + } else { + throw new Error(`Bad attribute format ${attributes[identifier]} for ${identifier}`); + } + } else { + term = { type: "word", value: identifier }; + } + } + return [pos, term]; +} + +/** + * Match a parsed word term with the given texts. + * + * Returns true if the passed word term in the search matches at least one + * of the passed text snippets. If the word mode is exclude then returns + * true if none of the passed text snippets match. + * + * @param {lib/helpers.ParsedTerm} word - Word to match. + * @param {...string} texts - Text to match word in. + * @returns {boolean} true if the word matches. + */ +function wordMatches(word, ...texts) { + if (word.type !== "word") { + throw Error("wordMatches: parameter is not a word"); + } + let matches = false; + for (let text of texts) { + if (text.includes(word.value)) { + matches = true; + break; + } + } + if (word.exclude) { + matches = !matches; + } + return matches; +} + +/** + * @typedef {object} module:lib/helpers~ParsedTerm + * @property {string} type - Type of term, either attribute or word. + * @property {string=} name - attribute only: Name of attribute. + * @property {boolean=} exclude - word only: exclude results with this word. + * @property {string | module:lib/helpers.ParsedTerm} value - + * Parsed value of this term. Is a string if type is word, and a + * ParsedTerm if type is attribute. + */ + +/** + * @typedef {object} module:lib/helpers~ParsedSearch + * @property {Array} terms - + * Parsed result of search terms. + * @property {Array} issues - + * Issues detected while parsing the seach string. + */ + +/** + * Parse a search string with optional attributes + * + * Parses the given input as a search expression consisting of space + * delimited words and attributes:word pairs. For aattributes to be + * recognized they need to be passed as name: "word" in the attributes + * parameter. + * + * Words in the search expression can optionally be prefixed by a - to + * search for results that does not contain that word. E.g. -author contains + * all results not matching author. + * + * If parsing fails an string describing the problem will be added to the + * issues array returned and parsing will resume at some arbitrary point. + * The issues should be shown to the end user so that they can correct their + * search. + * + * @param {string} input - Search expression to parse. + * @param {Object} attributes - + * Recognized attributes and their format. Currently only word is + * supported. + * @returns {module:lib/helpers~ParsedSearch} parsed terms of the search. + * @static + */ +function parseSearchString(input, attributes = {}) { + // whitespace = 1*" " + // search = [ term, *( [ whitespace ], term ) ] + input = input.trim(); + let parsed = { + terms: [], + issues: [], + }; + let pos = 0; + while (pos < input.length) { + let term; + ([pos, term] = parseSearchTerm(pos, input, attributes, parsed.issues)); + if (term) { + parsed.terms.push(term); + } + while (pos < input.length && input.charAt(pos) === " ") { + pos += 1; + } + } + // istanbul ignore if (should not be possible) + if (pos !== input.length) { + throw new Error(`parse search ended at ${pos} which is not the end of the input (${input.length})`); + } + return parsed; +} + module.exports = { basicType, @@ -97,4 +284,7 @@ module.exports = { timeout, readStream, escapeRegExp, + formatBytes, + wordMatches, + parseSearchString, }; diff --git a/packages/lib/index.js b/packages/lib/index.js index 0c89ef93..a642c629 100644 --- a/packages/lib/index.js +++ b/packages/lib/index.js @@ -2,6 +2,7 @@ const libBuildMod = require("./build_mod"); const libCommand = require("./command"); const libConfig = require("./config"); +const libData = require("./data"); const libDatabase = require("./database"); const libErrors = require("./errors"); const libFactorio = require("./factorio"); @@ -18,6 +19,7 @@ const libPrometheus = require("./prometheus"); const libSchema = require("./schema"); const libSharedCommands = require("./shared_commands"); const libUsers = require("./users"); +const libZipOps = require("./zip_ops"); const ExponentialBackoff = require("./ExponentialBackoff"); const RateLimiter = require("./RateLimiter"); @@ -27,6 +29,7 @@ module.exports = { libBuildMod, libCommand, libConfig, + libData, libDatabase, libErrors, libFactorio, @@ -43,6 +46,7 @@ module.exports = { libSchema, libSharedCommands, libUsers, + libZipOps, ExponentialBackoff, RateLimiter, diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index 5b5361da..3da48aec 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -1,8 +1,9 @@ // Message definitions for links "use strict"; -const libSchema = require("../schema"); +const libData = require("../data"); const libErrors = require("../errors"); const { logger } = require("../logging"); +const libSchema = require("../schema"); const PlayerStats = require("../PlayerStats"); class MissingLinkHandlerError extends Error { @@ -709,6 +710,178 @@ messages.sendRcon = new Request({ }, }); +messages.getModPack = new Request({ + type: "get_mod_pack", + links: ["control-master", "instance-slave", "slave-master"], + permission: "core.mod_pack.get", + forwardTo: "master", + requestProperties: { + "id": { type: "integer" }, + }, + responseProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.getDefaultModPack = new Request({ + type: "get_default_mod_pack", + links: ["control-master", "instance-slave", "slave-master"], + permission: "core.mod_pack.get", + forwardTo: "master", + responseProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.listModPacks = new Request({ + type: "list_mod_packs", + links: ["control-master"], + permission: "core.mod_pack.list", + responseProperties: { + "list": { + type: "array", + items: libData.ModPack.jsonSchema, + }, + }, +}); + +messages.createModPack = new Request({ + type: "create_mod_pack", + links: ["control-master"], + permission: "core.mod_pack.create", + requestProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.updateModPack = new Request({ + type: "update_mod_pack", + links: ["control-master"], + permission: "core.mod_pack.update", + requestProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.deleteModPack = new Request({ + type: "delete_mod_pack", + links: ["control-master"], + permission: "core.mod_pack.delete", + requestProperties: { + "id": { type: "integer" }, + }, +}); + +messages.setModPackSubscriptions = new Request({ + type: "set_mod_pack_subscriptions", + links: ["control-master"], + permission: "core.mod_pack.subscribe", + requestProperties: { + "all": { type: "boolean" }, + "mod_pack_ids": { + type: "array", + items: { type: "integer" }, + }, + }, +}); + +messages.getMod = new Request({ + type: "get_mod", + links: ["control-master"], + permission: "core.mod.get", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, + responseProperties: { + "mod": libData.ModInfo.jsonSchema, + }, +}); + +messages.listMods = new Request({ + type: "list_mods", + links: ["control-master"], + permission: "core.mod.list", + responseProperties: { + "list": { + type: "array", + items: libData.ModInfo.jsonSchema, + }, + }, +}); + +messages.searchMods = new Request({ + type: "search_mods", + links: ["control-master"], + permission: "core.mod.search", + requestRequired: ["query", "factorio_version", "page"], + requestProperties: { + "query": { type: "string" }, + "factorio_version": { type: "string" }, + "page_size": { type: "integer" }, + "page": { type: "integer" }, + "sort": { type: "string" }, + "sort_order": { type: "string" }, + }, + responseProperties: { + "query_issues": { + type: "array", + items: { type: "string" }, + }, + "page_count": { type: "integer" }, + "result_count": { type: "integer" }, + "results": { + type: "array", + items: { + type: "object", + properties: { + "name": { type: "string" }, + "versions": { + type: "array", + items: libData.ModInfo.jsonSchema, + }, + }, + }, + }, + }, +}); + +messages.setModSubscriptions = new Request({ + type: "set_mod_subscriptions", + links: ["control-master"], + permission: "core.mod.subscribe", + requestProperties: { + "all": { type: "boolean" }, + "mod_names": { + type: "array", + items: { type: "string" }, + }, + }, +}); + +messages.downloadMod = new Request({ + type: "download_mod", + links: ["control-master"], + permission: "core.mod.download", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, + responseProperties: { + "stream_id": { type: "string" }, + }, +}); + +messages.deleteMod = new Request({ + type: "delete_mod", + links: ["control-master"], + permission: "core.mod.delete", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, +}); + messages.listPermissions = new Request({ type: "list_permissions", links: ["control-master"], @@ -1263,6 +1436,22 @@ messages.saveListUpdate = new Event({ }, }); +messages.modPackUpdate = new Event({ + type: "mod_pack_update", + links: ["master-control"], + eventProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + +messages.modUpdate = new Event({ + type: "mod_update", + links: ["master-control"], + eventProperties: { + "mod": libData.ModInfo.jsonSchema, + }, +}); + messages.userUpdate = new Event({ type: "user_update", links: ["master-control"], diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index 9a1cc25f..ac2d3ffb 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -479,7 +479,6 @@ class BaseMasterPlugin { */ onControlConnectionEvent(connection, event) { } - /** * Called when a slave is preparing to disconnect from the master * @@ -494,6 +493,31 @@ class BaseMasterPlugin { */ async onPrepareSlaveDisconnect(connection) { } + /** + * Called when a mod pack is updated + * + * Invoked when a mod pack has been added, updated or deleted in the + * cluster. + * + * If the mod pack has been deleted its `.isDeleted` property will be + * true. + * + * @param {module:lib/data.ModPack} modPack - Mod pack that updated. + */ + async onModPackUpdated(modPack) { } + + /** + * Called when a mod stored on the master server is updated + * + * Invoked when a mod has been added, updated or deleted from the pool + * of shared mods stored on the cluster. + * + * If a mod has been deleted its `.isDeleted` property will be true. + * + * @param {module:lib/data.ModInfo} mod - Mod that updated. + */ + async onModUpdated(mod) { } + /** * Called when a player joins or leaves an instance * diff --git a/packages/lib/users.js b/packages/lib/users.js index a212c519..4f3fa0e8 100644 --- a/packages/lib/users.js +++ b/packages/lib/users.js @@ -511,6 +511,80 @@ definePermission({ description: "Send arbitrary RCON commands to instances.", }); +definePermission({ + name: "core.mod_pack.get", + title: "Retrieve mod pack", + description: "Get the details of a mod pack in the cluster.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod_pack.list", + title: "List mod packs", + description: "Get the full list of a mod packs in the cluster.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod_pack.subscribe", + title: "Subscribe to mod pack updates", + description: "Subscribe to be notified on updates on the details of mod packs.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod_pack.create", + title: "Create mod pack", + description: "Create new mod pack in the cluster.", +}); +definePermission({ + name: "core.mod_pack.update", + title: "Update mod pack", + description: "Edit existing mod pack definition in the cluster.", +}); +definePermission({ + name: "core.mod_pack.delete", + title: "Delete mod pack", + description: "Delete mod pack definition in the cluster.", +}); +definePermission({ + name: "core.mod.get", + title: "Get mods", + description: "Get the details of a mod stored in the cluster.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.list", + title: "List mods", + description: "Get the full list of mods stored on the master.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.search", + title: "Search mods", + description: "Search through the list of mods stored on the master.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.subscribe", + title: "Subscribe to mod updates", + description: "Subscribe to be notified on updates on the details of mods.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.upload", + title: "Upload mod", + description: "Upload mods to the master.", +}); +definePermission({ + name: "core.mod.download", + title: "Download mod", + description: "Download mods stored on the master.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.delete", + title: "Delete mod", + description: "Delete mods stored on the master.", +}); + definePermission({ name: "core.permission.list", title: "List permissions", diff --git a/packages/lib/zip_ops.js b/packages/lib/zip_ops.js new file mode 100644 index 00000000..f05c09b5 --- /dev/null +++ b/packages/lib/zip_ops.js @@ -0,0 +1,40 @@ +"use strict"; + +/** + * Returns the root folder in the zip file + * + * Returns the name of the folder that all files in the zip file is + * contained in. Throws an error if there are multiple such folders. + * + * @param {JSZip} zip - Zip to search through. + * @returns {string} name of the root folder. + * @memberof module:lib/factorio + * @private + * @inner + */ +function findRoot(zip) { + let root = null; + zip.forEach((relativePath, file) => { + let index = relativePath.indexOf("/"); + if (index === -1) { + throw new Error(`Zip contains file '${relativePath}' in root dir`); + } + + let pathRoot = relativePath.slice(0, index); + if (root === null) { + root = pathRoot; + } else if (root !== pathRoot) { + throw new Error("Zip contains multiple root folders"); + } + }); + + if (root === null) { + throw new Error("Empty zip file"); + } + + return root; +} + +module.exports = { + findRoot, +}; diff --git a/packages/master/src/BaseConnection.js b/packages/master/src/BaseConnection.js index e7bca6fa..92e25c1d 100644 --- a/packages/master/src/BaseConnection.js +++ b/packages/master/src/BaseConnection.js @@ -84,6 +84,27 @@ class BaseConnection extends libLink.Link { get connected() { return !this._disconnecting && this.connector.connected; } + + async getModPackRequestHandler(message) { + let { id } = message.data; + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } + + async getDefaultModPackRequestHandler(message) { + let id = this._master.config.get("master.default_mod_pack_id"); + if (id === null) { + throw new libErrors.RequestError("Default mod pack not set on master"); + } + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Default mod pack configured (${id}) does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } } module.exports = BaseConnection; diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index e60f1fd5..9856b33b 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -1,9 +1,13 @@ "use strict"; -const jwt = require("jsonwebtoken"); const events = require("events"); +const fs = require("fs-extra"); +const jwt = require("jsonwebtoken"); +const path = require("path"); const libConfig = require("@clusterio/lib/config"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); +const libHelpers = require("@clusterio/lib/helpers"); const libLink = require("@clusterio/lib/link"); const { logger } = require("@clusterio/lib/logging"); const libLoggingUtils = require("@clusterio/lib/logging_utils"); @@ -14,6 +18,8 @@ const libUsers = require("@clusterio/lib/users"); const BaseConnection = require("./BaseConnection"); const routes = require("./routes"); +const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; + const lastQueryLogTime = new libPrometheus.Gauge( "clusterio_master_last_query_log_duration_seconds", "Time in seconds the last log query took to execute." @@ -53,6 +59,16 @@ class ControlConnection extends BaseConnection { instance_ids: [], }; + this.modPackSubscriptions = { + all: false, + mod_pack_ids: [], + }; + + this.modSubscriptions = { + all: false, + mod_names: [], + }; + this.userSubscriptions = { all: false, names: [], @@ -458,11 +474,196 @@ class ControlConnection extends BaseConnection { return { save: result.save }; } + async listModPacksRequestHandler(message) { + return { list: [...this._master.modPacks.values()].map(pack => pack.toJSON()) }; + } + + async setModPackSubscriptionsRequestHandler(message) { + this.modPackSubscriptions = message.data; + } + + async createModPackRequestHandler(message) { + let modPack = new libData.ModPack(message.data.mod_pack); + if (this._master.modPacks.has(modPack.id)) { + throw new libErrors.RequestError(`Mod pack with ID ${modPack.id} already exist`); + } + this._master.modPacks.set(modPack.id, modPack); + this._master.modPackUpdated(modPack); + } + + async updateModPackRequestHandler(message) { + let modPack = new libData.ModPack(message.data.mod_pack); + if (!this._master.modPacks.has(modPack.id)) { + throw new libErrors.RequestError(`Mod pack with ID ${modPack.id} does not exist`); + } + this._master.modPacks.set(modPack.id, modPack); + this._master.modPackUpdated(modPack); + } + + async deleteModPackRequestHandler(message) { + let { id } = message.data; + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Mod pack with ID ${id} does not exist`); + } + modPack.isDeleted = true; + this._master.modPacks.delete(id); + this._master.modPackUpdated(modPack); + } + + modPackUpdated(modPack) { + if ( + this.modPackSubscriptions.all + || this.modPackSubscriptions.mod_pack_ids.includes(modPack.id) + ) { + libLink.messages.modPackUpdate.send(this, { mod_pack: modPack.toJSON() }); + } + } + + async getModRequestHandler(message) { + let { name, version } = message.data; + let filename = `${name}_${version}.zip`; + let mod = this._master.mods.get(filename); + if (!mod) { + throw new libErrors.RequestError(`Mod ${filename} does not exist`); + } + return { + mod: mod.toJSON(), + }; + } + + async listModsRequestHandler(message) { + return { list: [...this._master.mods.values()].map(mod => mod.toJSON()) }; + } + + static termsMatchesMod(terms, mod) { + for (let term of terms) { + if (term.type === "word") { + if (!libHelpers.wordMatches(term, + mod.name, mod.version, mod.title, mod.author, mod.contact, + mod.homepage, mod.description, mod.filename + )) { + return false; + } + } else if (term.type === "attribute") { + if (!libHelpers.wordMatches(term.value, mod[term.name])) { + return false; + } + } + } + return true; + } + + async searchModsRequestHandler(message) { + let query = libHelpers.parseSearchString(message.data.query, { + name: "word", + // version + title: "word", + author: "word", + contact: "word", + homepage: "word", + description: "word", + // factorioVersion + // dependencies + filename: "word", + // size + sha1: "word", + }); + let factorioVersion = message.data.factorio_version; + + let results = new Map(); + for (let mod of this._master.mods.values()) { + if ( + mod.factorioVersion !== factorioVersion + || !ControlConnection.termsMatchesMod(query.terms, mod) + ) { + continue; + } + let result = results.get(mod.name); + if (!result) { + result = { + name: mod.name, + versions: [], + }; + results.set(mod.name, result); + } + result.versions.push(mod); + } + for (let result of results.values()) { + result.versions.sort((a, b) => b.integerVersion - a.integerVersion); + result.versions.map(e => e.toJSON()); + } + let resultList = [...results.values()]; + + const sort = message.data.sort; + if (sort) { + const sorters = { + name: (a, b) => strcmp(a.versions[0].name, b.versions[0].name), + title: (a, b) => strcmp(a.versions[0].title, b.versions[0].title), + author: (a, b) => strcmp(a.versions[0].author, b.versions[0].author), + }; + if (!Object.prototype.hasOwnProperty.call(sorters, sort)) { + throw new libErrors.RequestError(`Invalid value for sort: ${sort}`); + } + resultList.sort(sorters[sort]); + let order = message.data.sort_order; + if (order === "desc") { + resultList.reverse(); + } + } + + const page = message.data.page; + const pageSize = message.data.page_size || 10; + resultList = resultList.slice((page - 1) * pageSize, page * pageSize); + + return { + query_issues: query.issues, + page_count: Math.ceil(results.size / pageSize), + result_count: results.size, + results: resultList, + }; + } + + async setModSubscriptionsRequestHandler(message) { + this.modSubscriptions = message.data; + } + + async downloadModRequestHandler(message) { + let { name, version } = message.data; + let filename = `${name}_${version}.zip`; + let mod = this._master.mods.get(filename); + if (!mod) { + throw new libErrors.RequestError(`Mod ${filename} does not exist`); + } + let modPath = path.join(this._master.config.get("master.mods_directory"), mod.filename); + + let stream = await routes.createProxyStream(this._master.app); + stream.filename = mod.filename; + stream.source = fs.createReadStream(modPath); + stream.mime = "application/zip"; + stream.size = mod.size; + + return { stream_id: stream.id }; + } + + async deleteModRequestHandler(message) { + await this._master.deleteMod(message.data.name, message.data.version); + } + async setLogSubscriptionsRequestHandler(message) { this.logSubscriptions = message.data; this.updateLogSubscriptions(); } + modUpdated(mod) { + if ( + this.modSubscriptions.all + || this.modSubscriptions.mod_names.includes(mod.name) + ) { + libLink.messages.modUpdate.send(this, { mod: mod.toJSON() }); + } + } + updateLogSubscriptions() { let { all, master, slave_ids, instance_ids } = this.logSubscriptions; if (all || master || slave_ids.length || instance_ids.length) { diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 8268021c..2bb3ccb7 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -9,6 +9,7 @@ const path = require("path"); const libConfig = require("@clusterio/lib/config"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); const libFileOps = require("@clusterio/lib/file_ops"); const libLink = require("@clusterio/lib/link"); @@ -65,6 +66,18 @@ class Master { */ this.instances = null; + /** + * Mapping of mod pack id to mod pack + * @type {Map} + */ + this.modPacks = null; + + /** + * Mapping of mod names to mod infos + * @type {Map} + */ + this.mods = null; + /** * User and roles manager for the cluster * @type {module:master/src/UserManager} @@ -161,9 +174,13 @@ class Master { this.slaves = await Master.loadSlaves(path.join(databaseDirectory, "slaves.json")); this.instances = await Master.loadInstances(path.join(databaseDirectory, "instances.json")); + this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json")); this.userManager = new UserManager(this.config); await this.userManager.load(path.join(databaseDirectory, "users.json")); - this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json")); + + let modsDirectory = this.config.get("master.mods_directory"); + await fs.ensureDir(modsDirectory); + this.mods = await Master.loadModInfos(modsDirectory); this.config.on("fieldChanged", (group, field, prev) => { libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev); @@ -284,6 +301,10 @@ class Master { await Master.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); } + if (this.modPacks) { + await Master.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); + } + if (this.userManager) { await this.userManager.save(path.join(databaseDirectory, "users.json")); } @@ -360,6 +381,57 @@ class Master { await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4)); } + static async loadModPacks(filePath) { + let json; + try { + json = JSON.parse(await fs.readFile(filePath)); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + return new Map(json.map(e => [e.id, new libData.ModPack(e)])); + } + + static async saveModPacks(filePath, modPacks) { + await libFileOps.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, 4)); + } + + static async loadModInfos(modsDirectory) { + let mods = new Map(); + for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) { + if (entry.isDirectory()) { + logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`); + continue; + } + + if (entry.name.toLowerCase().endsWith("tmp.zip")) { + continue; + } + + logger.info(`Loading info for Mod ${entry.name}`); + let modInfo; + try { + modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, entry.name)); + } catch (err) { + logger.error(`Error loading mod ${entry.name}: ${err.message}`); + continue; + } + + if (modInfo.filename !== entry.name) { + logger.error( + `Mod ${entry.name}'s filename does not match the expected ` + + `name ${modInfo.filename}, it will be ignored.` + ); + continue; + } + + mods.set(modInfo.filename, modInfo); + } + return mods; + } + static async loadJsonObject(filePath, throwOnMissing = false) { let manifest = {}; try { @@ -372,12 +444,18 @@ class Master { return manifest; } - async updateExportManifest(newManifest) { - let databaseDirectory = this.config.get("master.database_directory"); - await libFileOps.safeOutputFile( - path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4) - ); - this.exportManifest = newManifest; + async deleteMod(name, version) { + let filename = `${name}_${version}.zip`; + let mod = this.mods.get(filename); + if (!mod) { + throw new Error(`Mod ${filename} does not exist`); + } + + let modsDirectory = this.config.get("master.mods_directory"); + await fs.unlink(path.join(modsDirectory, filename)); + this.mods.delete(filename); + mod.isDeleted = true; + this.modUpdated(mod); } /** @@ -503,6 +581,29 @@ class Master { } } + modPackUpdated(modPack) { + for (let controlConnection of this.wsServer.controlConnections) { + if (controlConnection.connector.closing) { + continue; + } + controlConnection.modPackUpdated(modPack); + } + + libPlugin.invokeHook(this.plugins, "onModPackUpdated", modPack); + } + + modUpdated(mod) { + for (let controlConnection of this.wsServer.controlConnections) { + if (controlConnection.connector.closing) { + continue; + } + + controlConnection.modUpdated(mod); + } + + libPlugin.invokeHook(this.plugins, "onModUpdated", mod); + } + userUpdated(user) { for (let controlConnection of this.wsServer.controlConnections) { if (controlConnection.connector.closing) { diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 4052554a..4e48f1f5 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -2,11 +2,14 @@ const Busboy = require("busboy"); const crypto = require("crypto"); const events = require("events"); +const fs = require("fs-extra"); const JSZip = require("jszip"); const jwt = require("jsonwebtoken"); const path = require("path"); +const nodeStream = require("stream"); const util = require("util"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); const libFileOps = require("@clusterio/lib/file_ops"); const libHash = require("@clusterio/lib/hash"); @@ -18,6 +21,8 @@ const { logger } = require("@clusterio/lib/logging"); const { endpointHitCounter } = require("./metrics"); +const finished = util.promisify(nodeStream.finished); + // Merges samples from sourceResult to destinationResult function mergeSamples(destinationResult, sourceResult) { @@ -182,6 +187,21 @@ async function uploadExport(req, res) { return; } + if (typeof req.query.mod_pack_id !== "string") { + res.sendStatus(400); + return; + } + let modPackId = Number.parseInt(req.query.mod_pack_id, 10); + if (!Number.isInteger(modPackId)) { + res.sendStatus(400); + return; + } + let modPack = res.app.locals.master.modPacks.get(modPackId); + if (!modPack) { + res.sendStatus(400); + return; + } + let data = []; for await (let chunk of req) { data.push(chunk); @@ -192,33 +212,38 @@ async function uploadExport(req, res) { // This is hardcoded to prevent path expansion attacks let exportFiles = [ + "export/settings.json", + "export/prototypes.json", "export/item-spritesheet.png", "export/item-metadata.json", "export/locale.json", ]; - let manifest = {}; + let assets = {}; + let settingPrototypes = {}; for (let filePath of exportFiles) { let file = zip.file(filePath); if (!file) { continue; } + if (filePath === "export/settings.json") { + settingPrototypes = JSON.parse(await file.async("text")); + } + let { name, ext } = path.posix.parse(filePath); let hash = await libHash.hashStream(file.nodeStream()); - manifest[name] = `static/${name}.${hash}${ext}`; + assets[name] = `${name}.${hash}${ext}`; await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer")); } - await res.app.locals.master.updateExportManifest(manifest); + + modPack.exportManifest = new libData.ExportManifest({ assets }); + modPack.fillDefaultSettings(settingPrototypes, logger); + res.app.locals.master.modPackUpdated(modPack); res.sendStatus(200); } -async function getExportManifest(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.json(res.app.locals.master.exportManifest); -} - async function createProxyStream(app) { let asyncRandomBytes = util.promisify(crypto.randomBytes); let id = (await asyncRandomBytes(8)).toString("hex"); @@ -306,6 +331,8 @@ const zipMimes = [ "application/x-zip-compressed", ]; +const contentTypeRegExp = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/; + async function uploadSave(req, res) { try { res.locals.user.checkPermission("core.instance.save.upload"); @@ -313,9 +340,10 @@ async function uploadSave(req, res) { res.status(403).json({ request_errors: [err.message] }); return; } + endpointHitCounter.labels(req.route.path).inc(); let contentType = req.get("Content-Type"); - let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType); + let match = contentTypeRegExp.exec(contentType); if (!match) { res.status(415).json({ request_errors: ["invalid Content-Type"] }); return; @@ -436,6 +464,154 @@ async function uploadSave(req, res) { res.json({ saves }); } +function checkModName(name) { + try { + libFileOps.checkFilename(name); + } catch (err) { + throw new libErrors.RequestError(`Mod name ${err.message}`); + } +} + +async function uploadMod(req, res) { + try { + res.locals.user.checkPermission("core.mod.upload"); + } catch (err) { + res.status(403).json({ request_errors: [err.message] }); + return; + } + endpointHitCounter.labels(req.route.path).inc(); + + let contentType = req.get("Content-Type"); + let match = contentTypeRegExp.exec(contentType); + if (!match) { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + let mimeType = match[1].toLowerCase(); + + let tasks = []; + let errors = []; + let requestErrors = []; + let mods = []; + + async function handleFile(stream, filename) { + try { + checkModName(filename); + + } catch (err) { + logger.error(`Error uploading mod: ${err.message}`); + errors.push(err.message); + stream.resume(); + return; + } + + const modsDirectory = req.app.locals.master.config.get("master.mods_directory"); + let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + try { + + let writeStream; + while (true) { + try { + writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" }); + await events.once(writeStream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(modsDirectory, tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + stream.pipe(writeStream); + await finished(writeStream); + + const modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); + await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename)); + req.app.locals.master.mods.set(modInfo.filename, modInfo); + req.app.locals.master.modUpdated(modInfo); + mods.push(modInfo.toJSON()); + + } catch (err) { + logger.error(`Error uploading mod: ${err.message}`); + errors.push(err.message); + stream.resume(); + + // Attempt to clean up. + writeStream.destroy(); + try { + await fs.unlink(path.join(modsDirectory, tempFilename)); + } catch (unlinkErr) { + if (unlinkErr.code !== "ENOENT") { + logger.error(`Error removing ${tempFilename}: ${err.message}`); + } + } + } + } + + if (mimeType === "multipart/form-data") { + await new Promise(resolve => { + let busboy = new Busboy({ headers: req.headers }); + busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => { + if (!zipMimes.includes(fileMime)) { + requestErrors.push("invalid file Content-Type"); + } + + if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + + if (errors.length || requestErrors.length) { + stream.resume(); + return; + } + + tasks.push(handleFile(stream, filename)); + }); + busboy.on("finish", resolve); + busboy.on("error", (err) => { + logger.error(`Error parsing multipart request in upload-mod:\n${err.stack}`); + errors.push(err.message); + }); + req.pipe(busboy); + }); + + } else if (zipMimes.includes(mimeType)) { + let filename = req.query.filename; + if (typeof filename !== "string") { + requestErrors.push("Missing or invalid filename parameter"); + } else if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + + if (errors.length || requestErrors.length) { + req.resume(); + } else { + tasks.push(handleFile(req, filename)); + } + + } else { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + + await Promise.all(tasks); + + if (errors.length) { + res.status(500); + res.json({ errors, request_errors: requestErrors }); + return; + } + if (requestErrors.length) { + res.status(400); + res.json({ request_errors: requestErrors }); + return; + } + + res.json({ mods }); +} + + function addRouteHandlers(app) { app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next)); app.get("/api/plugins", getPlugins); @@ -443,13 +619,16 @@ function addRouteHandlers(app) { validateSlaveToken, (req, res, next) => uploadExport(req, res).catch(next) ); - app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next)); app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next)); app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next)); app.post("/api/upload-save", validateUserToken, (req, res, next) => uploadSave(req, res).catch(next) ); + app.post("/api/upload-mod", + validateUserToken, + (req, res, next) => uploadMod(req, res).catch(next) + ); } // Routes used in the web interface and served by the master server @@ -460,6 +639,8 @@ const webRoutes = [ "/slaves/:id/view", "/instances", "/instances/:id/view", + "/mods", + "/mods/mod-packs/:id/view", "/users", "/users/:id/view", "/roles", diff --git a/packages/slave/lua/export/clusterio_api.lua b/packages/slave/lua/export/clusterio_api.lua new file mode 100644 index 00000000..56f61124 --- /dev/null +++ b/packages/slave/lua/export/clusterio_api.lua @@ -0,0 +1,73 @@ +local function is_array(t) + local count = 0 + for k in pairs(t) do + if type(k) ~= "number" then + return false + end + count = count + 1 + end + + for i = 1, count do + if not t[i] then + return false + end + end + + return count ~= 0 +end + +-- no game.table_to_json here :( +local function table_to_json(data) + if type(data) == "number" then + if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it. + if data == -math.huge then return "-1e500" end + if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead. + return string.format("%.17g", data) + + elseif type(data) == "string" then + data = data:gsub('([\x00-\x1f\\"])', function(match) + return "\\u00" .. string.format("%02x", match:byte()) + end) + return '"' .. data .. '"' + + elseif data == nil then + return "null" + + elseif data == true then + return "true" + elseif data == false then + return "false" + + elseif type(data) == "table" then + local r + if is_array(data) then + r = {"["} + for i=1, #data do + r[#r+1] = table_to_json(data[i]) + r[#r+1] = "," + end + + -- overwrite last comma, empty table is an object + r[#r] = "]" + + else + r = {"{"} + for k, v in pairs(data) do + r[#r+1] = table_to_json(tostring(k)) + r[#r+1] = ":" + r[#r+1] = table_to_json(v) + r[#r+1] = "," + end + r[#r + (r[#r] == "," and 0 or 1)] = "}" + end + + return table.concat(r) + end +end + +local clusterio_api = {} +function clusterio_api.send_json(channel, data) + data = table_to_json(data) + print("\f$ipc:" .. channel .. "?j" .. data) +end +return clusterio_api diff --git a/packages/slave/lua/export/control.lua b/packages/slave/lua/export/control.lua index 4d27f895..b810f2f0 100644 --- a/packages/slave/lua/export/control.lua +++ b/packages/slave/lua/export/control.lua @@ -5,4 +5,7 @@ end script.on_init(function() send_json("mod_list", game.active_mods) + for name, mod_setting in pairs(game.mod_setting_prototypes) do + send_json("mod_setting_mod", { name = name, mod = mod_setting.mod }) + end end) diff --git a/packages/slave/lua/export/data-final-fixes.lua b/packages/slave/lua/export/data-final-fixes.lua index 991bb831..f3820c37 100644 --- a/packages/slave/lua/export/data-final-fixes.lua +++ b/packages/slave/lua/export/data-final-fixes.lua @@ -1,104 +1,8 @@ -local item_types = { - -- Item categories - "item", - "ammo", - "capsule", - "gun", - "item-with-entity-data", - "item-with-label", - "item-with-inventory", - "blueprint-book", - "item-with-tags", - "selection-tool", - "blueprint", - "deconstruction-item", - "copy-paste-tool", - "upgrade-item", - "module", - "rail-planner", - "tool", - "armor", - "mining-tool", - "repair-tool", +local clusterio_api = require("clusterio_api") - -- Fluids - "fluid", -} - -local function is_array(t) - local count = 0 - for k in pairs(t) do - if type(k) ~= "number" then - return false - end - count = count + 1 - end - - for i = 1, count do - if not t[i] then - return false - end - end - - return count ~= 0 -end - --- no game.table_to_json here :( -local function table_to_json(data) - if type(data) == "number" then - if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it. - if data == -math.huge then return "-1e500" end - if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead. - return string.format("%.17g", data) - - elseif type(data) == "string" then - data = data:gsub('([\x00-\x1f\\"])', function(match) - return "\\u00" .. string.format("%02x", match:byte()) - end) - return '"' .. data .. '"' - - elseif data == nil then - return "null" - - elseif data == true then - return "true" - elseif data == false then - return "false" - - elseif type(data) == "table" then - local r - if is_array(data) then - r = {"["} - for i=1, #data do - r[#r+1] = table_to_json(data[i]) - r[#r+1] = "," - end - - -- overwrite last comma, empty table is an object - r[#r] = "]" - - else - r = {"{"} - for k, v in pairs(data) do - r[#r+1] = table_to_json(tostring(k)) - r[#r+1] = ":" - r[#r+1] = table_to_json(v) - r[#r+1] = "," - end - r[#r + (r[#r] == "," and 0 or 1)] = "}" - end - - return table.concat(r) - end -end - -local function send_json(channel, data) - data = table_to_json(data) - print("\f$ipc:" .. channel .. "?j" .. data) -end - -for _, group in ipairs(item_types) do - for _, item in pairs(data.raw[group]) do - send_json("item_export", item) +log("Exporting prototypes") +for group, prototypes in pairs(data.raw) do + for name, prototype in pairs(prototypes) do + clusterio_api.send_json("prototype_export", prototype) end end diff --git a/packages/slave/lua/export/settings-final-fixes.lua b/packages/slave/lua/export/settings-final-fixes.lua new file mode 100644 index 00000000..fe0ea4e7 --- /dev/null +++ b/packages/slave/lua/export/settings-final-fixes.lua @@ -0,0 +1,8 @@ +local clusterio_api = require("clusterio_api") + +log("Exporting settings") +for group, prototypes in pairs(data.raw) do + for name, prototype in pairs(prototypes) do + clusterio_api.send_json("settings_export", prototype) + end +end diff --git a/packages/slave/slave.js b/packages/slave/slave.js index 9c4fb8c6..8f9f70e7 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -177,7 +177,7 @@ async function startSlave() { // If we get here the command was run await fs.ensureDir(slaveConfig.get("slave.instances_directory")); - await fs.ensureDir("sharedMods"); + await fs.ensureDir(slaveConfig.get("slave.mods_directory")); await fs.ensureDir("modules"); // Set the process title, shows up as the title of the CMD window on windows diff --git a/packages/slave/src/Instance.js b/packages/slave/src/Instance.js index a5a717a6..2d33aad9 100644 --- a/packages/slave/src/Instance.js +++ b/packages/slave/src/Instance.js @@ -6,6 +6,7 @@ const phin = require("phin"); const util = require("util"); // internal libraries +const libData = require("@clusterio/lib/data"); const libFileOps = require("@clusterio/lib/file_ops"); const libFactorio = require("@clusterio/lib/factorio"); const libLink = require("@clusterio/lib/link"); @@ -94,71 +95,6 @@ const serverSettingsActions = { }, }; -/** - * Create and update symlinks for shared mods in an instance - * - * Creates symlinks for .zip and .dat files that are not present in the - * instance mods directory but is present in the sharedMods directory, - * and removes any symlinks that don't point to a file in the instance - * mods directory. If the instance mods directory doesn't exist it will - * be created. - * - * Note that on Windows this creates hard links instead of symbolic - * links as the latter requires elevated privileges. This unfortunately - * means the removal of mods from the shared mods dir can't be detected. - * - * @param {Instance} instance - Instance to link mods for - * @param {string} sharedMods - Path to folder to link mods from. - * @private - */ -async function symlinkMods(instance, sharedMods) { - await fs.ensureDir(instance.path("mods")); - - // Remove broken symlinks in instance mods. - for (let entry of await fs.readdir(instance.path("mods"), { withFileTypes: true })) { - if (entry.isSymbolicLink()) { - if (!await fs.pathExists(instance.path("mods", entry.name))) { - instance.logger.verbose(`Removing broken symlink ${entry.name}`); - await fs.unlink(instance.path("mods", entry.name)); - } - } - } - - // Link entries that are in sharedMods but not in instance mods. - let instanceModsEntries = new Set(await fs.readdir(instance.path("mods"))); - for (let entry of await fs.readdir(sharedMods, { withFileTypes: true })) { - if (entry.isFile()) { - if ([".zip", ".dat", ".json"].includes(path.extname(entry.name))) { - if (!instanceModsEntries.has(entry.name)) { - instance.logger.verbose(`linking ${entry.name} from ${sharedMods}`); - let target = path.join(sharedMods, entry.name); - let link = instance.path("mods", entry.name); - - /* eslint-disable max-depth */ - if (process.platform !== "win32") { - await fs.symlink(path.relative(path.dirname(link), target), link); - - // On Windows symlinks require elevated privileges, which is - // not something we want to have. For this reason the mods - // are hard linked instead. This has the drawback of not - // being able to identify when mods are removed from the - // sharedMods directory, or which mods are linked. - } else { - await fs.link(target, link); - } - /* eslint-enable max-depth */ - } - - } else { - instance.logger.warn(`Warning: ignoring file '${entry.name}' in sharedMods`); - } - - } else { - instance.logger.warn(`Warning: ignoring non-file '${entry.name}' in sharedMods`); - } - } -} - /** * Keeps track of the runtime parameters of an instance @@ -255,6 +191,12 @@ class Instance extends libLink.Link { } }); + /** + * Mod pack currently running on this instance + * @type {module:lib/data.ModPack>} + */ + this.activeModPack = undefined; + /** * Per player statistics recorded by this instance. * @type {Map} @@ -643,6 +585,77 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await fs.ensureDir(path.join(instanceDir, "saves")); } + /** + * Sync instance mods directory with configured mod pack + * + * Adds, deletes, and updates all files in the instance mods folder to + * match up with the mod pack that's configured. If no mod pack is + * configured then an empty mod pack is used, esentially turning it into + * a vanilla mods folder. If the instance mods directory does't exist + * then it will be created. + * + * On Linux this creates symlinks to mods in the slave's mods folder, on + * Windows hard links are used instead due to symlinks being privileged. + */ + async syncMods() { + const modPackId = this.config.get("factorio.mod_pack"); + let response; + if (modPackId === null) { + response = await libLink.messages.getDefaultModPack.send(this); + } else { + response = await libLink.messages.getModPack.send(this, { id: modPackId }); + } + this.activeModPack = new libData.ModPack(response.mod_pack); + + // TODO validate factorioVersion + + if (!this._slave.config.get("slave.mods_directory_is_shared")) { + throw new Error("Fetching mods is not implemented"); + } + + await fs.ensureDir(this.path("mods")); + + // Remove all files + for (let entry of await fs.readdir(this.path("mods"), { withFileTypes: true })) { + if (entry.isDirectory()) { + this.logger.warn( + `Found unexpected directory ${entry.name} in mods folder, it may break Clusterio's mod syncing` + ); + continue; + } + + if (entry.isFile() || entry.isSymbolicLink()) { + await fs.unlink(this.path("mods", entry.name)); + } + } + + // Add mods from mod the pack + const modsDir = this._slave.config.get("slave.mods_directory"); + for (let mod of this.activeModPack.mods.values()) { + const modFile = `${mod.name}_${mod.version}.zip`; + const target = path.join(modsDir, modFile); + const link = this.path("mods", modFile); + + if (process.platform !== "win32") { + await fs.symlink(path.relative(path.dirname(link), target), link); + + // On Windows symlinks require elevated privileges, which is + // not something we want to have. For this reason the mods + // are hard linked instead. + } else { + await fs.link(target, link); + } + } + + // Write mod-list.json + await fs.outputFile(this.path("mods", "mod-list.json"), JSON.stringify({ + mods: [{ name: "base" }, ...this.activeModPack.mods.values()].map(m => ({ name: m.name, enabled: true })), + }, null, 2)); + + // Write mod-settings.dat + await fs.outputFile(this.path("mods", "mod-settings.dat"), this.activeModPack.toModSettingsDat()); + } + /** * Prepare instance for starting * @@ -678,8 +691,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); ); } - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } /** @@ -1030,8 +1042,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.verbose("Creating save ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); } catch (err) { this.notifyExit(); @@ -1053,13 +1064,13 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); await this.writeServerSettings(); this.logger.info("Exporting data ....."); - // eslint-disable-next-line no-use-before-define - await symlinkMods(this, "sharedMods"); + await this.syncMods(); let zip = await libFactorio.exportData(this.server); let content = await zip.generateAsync({ type: "nodebuffer" }); let url = new URL(this._slave.config.get("slave.master_url")); url.pathname += "api/upload-export"; + url.searchParams.set("mod_pack_id", this.activeModPack.id); let response = await phin({ url, method: "PUT", data: content, @@ -1116,6 +1127,3 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); } module.exports = Instance; - -// For testing only -module.exports._symlinkMods = symlinkMods; diff --git a/packages/slave/src/Slave.js b/packages/slave/src/Slave.js index 4c28b605..c8b85f57 100644 --- a/packages/slave/src/Slave.js +++ b/packages/slave/src/Slave.js @@ -25,86 +25,9 @@ const InstanceConnection = require("./InstanceConnection"); const finished = util.promisify(stream.finished); -function checkFilename(name) { - // All of these are bad in Windows only, except for /, . and .. - // See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file - const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; - const badEnd = /[. ]$/; - - const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - const badNames = [ - // Relative path components - ".", "..", - - // Reserved filenames in Windows - "CON", "PRN", "AUX", "NUL", - ...oneToNine.map(n => `COM${n}`), - ...oneToNine.map(n => `LPT${n}`), - ]; - - if (typeof name !== "string") { - throw new Error("must be a string"); - } - - if (name === "") { - throw new Error("cannot be empty"); - } - - if (badChars.test(name)) { - throw new Error('cannot contain <>:"\\/|=* or control characters'); - } - - if (badNames.includes(name.toUpperCase())) { - throw new Error( - "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9" - ); - } - - if (badEnd.test(name)) { - throw new Error("cannot end with . or space"); - } -} - -/** - * Clean up string to be suitable for use as filename - * - * @param {string} name - Arbitrary name string - * @returns {string} Filename suitable to use in the filesystem - * @private - */ -function cleanFilename(name) { - // copied from checkFilename - const badChars = /[<>:"\/\\|?*\x00-\x1f]/g; - const badEnd = /[. ]$/; - - const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - const badNames = [ - // Relative path components - ".", "..", - - // Reserved filenames in Windows - "CON", "PRN", "AUX", "NUL", - ...oneToNine.map(n => `COM${n}`), - ...oneToNine.map(n => `LPT${n}`), - ]; - - if (typeof name !== "string") { - throw new Error("name must be a string"); - } - - if (name === "" || badNames.includes(name.toUpperCase())) { - name += "_"; - } - - name = name.replace(badChars, "_"); - name = name.replace(badEnd, "_"); - - return name; -} - function checkRequestSaveName(name) { try { - checkFilename(name); + libFileOps.checkFilename(name); } catch (err) { throw new libErrors.RequestError(`Save name ${err.message}`); } @@ -243,9 +166,9 @@ class Slave extends libLink.Link { } async _createNewInstanceDir(name) { - name = cleanFilename(name); + name = libFileOps.cleanFilename(name); try { - checkFilename(name); + libFileOps.checkFilename(name); } catch (err) { throw new Error(`Instance folder was unepectedly invalid: name ${err.message}`); } @@ -837,8 +760,5 @@ ${err.stack}` module.exports = Slave; // For testing only -module.exports._checkFilename = checkFilename; -module.exports._cleanFilename = cleanFilename; module.exports._discoverInstances = discoverInstances; -module.exports._Slave = Slave; diff --git a/packages/web_ui/src/components/ModDetails.jsx b/packages/web_ui/src/components/ModDetails.jsx new file mode 100644 index 00000000..19487421 --- /dev/null +++ b/packages/web_ui/src/components/ModDetails.jsx @@ -0,0 +1,53 @@ +import React, { Fragment } from "react"; +import { Descriptions, Grid, Typography } from "antd"; +import ExclamationCircleOutlined from "@ant-design/icons/ExclamationCircleOutlined"; + +import { libHelpers } from "@clusterio/lib"; + +const { useBreakpoint } = Grid; + + +export default function ModDetails(props) { + let screens = useBreakpoint(); + const mod = props.mod; + + return + {mod.error && Error} span={2}> + {mod.error === "missing" && "The given version of this mod is missing from the master storage."} + {mod.error === "bad_checksum" + && "Checksum of the mod in the master storage does not match the checksum provided by this mod pack." + } + } + {props.actions && !screens.lg + && {props.actions(mod)} + } + {mod.title && {mod.title}} + {mod.version && {mod.version}} + {mod.description && {mod.description} } + {mod.author && {mod.author}} + + + {`https://mods.factorio.com/mod/${mod.name}`} + + + {mod.contact && {mod.contact}} + {mod.homepage && {mod.homepage}} + {mod.name} + {mod.factorioVersion + && {mod.factorioVersion} + } + {mod.filename && {mod.filename}} + {mod.size !== undefined + && {libHelpers.formatBytes(mod.size)} + } + {mod.sha1 && {mod.sha1}} + {mod.dependencies && mod.dependencies.length + && + {mod.dependencies + .filter(e => !e.startsWith("(?)")) + .map((e, i) => {e}
) + } + + } + ; +} diff --git a/packages/web_ui/src/components/ModPackViewPage.jsx b/packages/web_ui/src/components/ModPackViewPage.jsx new file mode 100644 index 00000000..69393489 --- /dev/null +++ b/packages/web_ui/src/components/ModPackViewPage.jsx @@ -0,0 +1,803 @@ +import React, { Fragment, useEffect, useContext, useRef, useState } from "react"; +import { useHistory, useParams } from "react-router-dom"; +import { + Button, Card, Checkbox, Col, ConfigProvider, Descriptions, Form, Input, PageHeader, Pagination, + Popconfirm, Row, Table, Tag, Typography, Select, Skeleton, Space, Spin, Switch, Modal, Tooltip, +} from "antd"; +import InfoCircleOutlined from "@ant-design/icons/InfoCircleOutlined"; +import ExportOutlined from "@ant-design/icons/ExportOutlined"; +import FileUnknownOutlined from "@ant-design/icons/FileUnknownOutlined"; +import FileExclamationOutlined from "@ant-design/icons/FileExclamationOutlined"; +import CloseOutlined from "@ant-design/icons/CloseOutlined"; +import DeleteOutlined from "@ant-design/icons/DeleteOutlined"; +import PlusOutlined from "@ant-design/icons/PlusOutlined"; +import UploadOutlined from "@ant-design/icons/UploadOutlined"; + +import { libData, libLink, libLogging } from "@clusterio/lib"; +import ControlContext from "./ControlContext"; +import { useAccount } from "../model/account"; +import { useModPack } from "../model/mod_pack"; +import { useModList } from "../model/mods"; +import { useClipboard } from "../util/clipboard"; +import notify, { notifyErrorHandler } from "../util/notify"; +import PageLayout from "./PageLayout"; +import SectionHeader from "./SectionHeader"; +import ModDetails from "./ModDetails"; + +const { logger } = libLogging; +const { Paragraph, Text } = Typography; +const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; + +function SearchModsTable(props) { + const control = useContext(ControlContext); + const [searchText, setSearchText] = useState(""); + const [modResultSort, setModResultSort] = useState("title"); + const [modResultSortOrder, setModResultSortOrder] = useState("asc"); + const [modResults, setModResults] = useState([]); + const [modResultPage, setModResultPage] = useState(1); + const [modResultPageSize, setModResultPageSize] = useState(10); + const [modResultCount, setModResultCount] = useState(2); + const [modResultSelectedVersion, setModResultSelectedVersion] = useState(new Map()); + + const factorioVersion = props.modPack.factorioVersion.split(".").slice(0, 2).join("."); + + useEffect(() => { + let canceled = false; + libLink.messages.searchMods.send(control, { + query: searchText, + factorio_version: factorioVersion, + page_size: modResultPageSize, + page: modResultPage, + sort: modResultSort, + sort_order: modResultSortOrder, + }).then(response => { + if (canceled) { + return; + } + // In React < v18 this causes 3 renders and a partial state necessitating the use + // of || 0 on modResultSelectedVersion.get() calls. Remove when updating React. + setModResults(response.results.map( + ({ name, versions }) => ({ name, versions: versions.map(mod => new libData.ModInfo(mod)) }) + )); + setModResultSelectedVersion(new Map(response.results.map(({ name, versions }) => [name, 0]))); + setModResultCount(response.result_count); + }); + + return () => { + canceled = true; + }; + }, [searchText, factorioVersion, modResultPageSize, modResultPage, modResultSort, modResultSortOrder]); + + function modResultPageChanged(page, pageSize) { + setModResultPage(page); + setModResultPageSize(pageSize); + } + + function modResultsTableChanged(pagination, filters, sorters, extra) { + if (extra.action === "sort") { + if (sorters.order) { + const mapping = { + ascend: "asc", + descend: "desc", + }; + setModResultSort(sorters.columnKey); + setModResultSortOrder(mapping[sorters.order]); + } else { + setModResultSort(undefined); + setModResultSortOrder(undefined); + } + } + } + + function actions(mod) { + return + {(!props.modPack.mods.has(mod.name) || props.modPack.mods.get(mod.name).version !== mod.version) + && { + props.onChange({ + type: "mods.set", + name: mod.name, + value: { name: mod.name, enabled: true, version: mod.version, sha1: mod.sha1 }, + }); + }} + >{props.modPack.mods.has(mod.name) ? "change" : "add"} + } + {props.modPack.mods.has(mod.name) + && { + props.onChange({ + type: "mods.delete", + name: mod.name, + value: { name: mod.name, version: mod.version, sha1: mod.sha1 }, + }); + }} + >remove + } + ; + } + + return <> + { + let search = event.target.value.trim(); + setSearchText(search); + setModResultPage(1); + }} + /> + `${total} results`} + showSizeChanger={true} + size="small" + />} + /> + "No results"}> + results.versions[0].title, + defaultSortOrder: "ascend", + sorter: true, + }, + { + title: "Author", + key: "author", + render: results => results.versions[0].author, + responsive: ["lg"], + sorter: true, + }, + { + title: "Version", + key: "version", + align: "right", + render: result =>
mod.enabled).map(mod => mod.name), + getCheckboxProps(mod) { + return { disabled: deletedMods.has(mod.name) }; + }, + onChange(selectedRowKeys, selectedRows) { + function setEnabled(modNames, enabled) { + for (let name of modNames) { + props.onChange({ + type: "mods.set", + name, + value: { ...props.modPack.mods.get(name), enabled }, + }); + } + } + const added = selectedRowKeys.filter(key => !props.modPack.mods.get(key).enabled); + const removed = [...props.modPack.mods.values()].filter( + mod => mod.enabled && selectedRowKeys.indexOf(mod.name) === -1 + ).map(mod => mod.name); + setEnabled(added, true); + setEnabled(removed, false); + }, + }} + columns={[ + { + title: "Name", + key: "name", + render: mod => <> + {mod.error === "missing" && + {" "} + } + {mod.error === "bad_checksum" && + {" "} + } + {mod.title || mod.name} + , + defaultSortOrder: "ascend", + sorter: (a, b) => strcmp(a.name, b.name), + }, + { + title: "Version", + dataIndex: "version", + sorter: (a, b) => strcmp(a.version, b.version), + }, + { + title: "Action", + key: "action", + responsive: ["lg"], + render: mod => actions(mod), + }, + ]} + expandable={{ + expandedRowRender: mod => , + expandedRowClassName: () => "no-expanded-padding", + }} + dataSource={mods} + pagination={false} + rowKey={mod => mod.name} + rowClassName={mod => { + if (deletedMods.has(mod.name)) { + return "deleted"; + } + if (changedMods.has(mod.name)) { + return "changed"; + } + return ""; + }} + /> + ; +} + +// Returns true if a change record with the given fields exist +function hasChange(changes, field) { + return changes.some(change => Object.entries(field).every(([key, value]) => change[key] === value)); +} + +// picks out the value property from settings. +function pickValue(map) { + return [...map].map(([name, value]) => [name, value.value]); +} + +function groupToMap(array, fn) { + const map = new Map(); + for (let index = 0; index < array.length; index++) { + let key = fn(array[index], index, array); + if (!map.has(key)) { + map.set(key, []); + } + map.get(key).push(array[index]); + } + return map; +} + +function SettingsTable(props) { + const [modList] = useModList(); + const modsInPack = new Set([...props.modPack.mods.values()].map(mod => `${mod.name}_${mod.version}`)); + const modTitles = new Map(modList + .filter(mod => modsInPack.has(`${mod.name}_${mod.version}`)) + .map(mod => [`${mod.name}`, mod.title]) + ); + + const types = ["bool-setting", "int-setting", "double-setting", "string-setting"]; + let prototypes = Object.entries(props.prototypes) + .filter(([type, _]) => types.includes(type)) + .flatMap(([_, settingPrototypes]) => Object.values(settingPrototypes)) + ; + + function controls(scope, storedFields) { + let fields = new Map(prototypes + .filter(p => p.setting_type === scope) + .sort((a, b) => strcmp(a.mod, b.mod) || strcmp(a.order || "", b.order || "") || strcmp(a.name, b.name)) + .map(p => [p.name, p]) + ); + for (let [name, value] of storedFields) { + if (!fields.has(name)) { + let type; + if (typeof value === "boolean") { + type = "bool-setting"; + } else if (typeof value === "number") { + type = "double-setting"; + } else if (typeof value === "string") { + type = "string-setting"; + } else { + throw Error(`Unhandled setting type ${typeof value}`); + } + fields.set(name, { mod: "Unknown Fields", name, setting_type: scope, type }); + } + } + let fieldsByMod = groupToMap([...fields.values()], prototype => prototype.mod); + return [...fieldsByMod].map(([mod, fieldsForMod]) => + + {props.locale.get(`mod-name.${mod}`) || modTitles.get(mod) || mod} + + {[...fieldsForMod].map(prototype => { + const name = prototype.name; + const isBoolean = prototype.type === "bool-setting"; + const isNumber = ["int-setting", "double-setting"].includes(prototype.type); + const isSelect = Boolean(prototype.allowed_values); + let input; + if (isBoolean) { + input = ; + } else if (isSelect) { + const options = Object.values(prototype.allowed_values).map(value => ({ + value, + label: props.locale.get(`string-mod-setting.${name}-${value}`) || value, + })); + input = ; + } + + return + {input} + ; + })} + ); + } + + return <> + + Run export data on an instance to update available fields. + Startup + {controls("startup", pickValue(props.modPack.settings["startup"]))} + + Map + These only apply for new saves. + {controls("runtime-global", pickValue(props.modPack.settings["runtime-global"]))} + + Per Player + Default settings for new players, these only apply for new saves. + {controls("runtime-per-user", pickValue(props.modPack.settings["runtime-per-user"]))} + ; +} + +function CopyButton(props) { + let clipboard = useClipboard(); + let [copiedVisible, setCopiedVisible] = useState(false); + let copiedTimeout = useRef(); + + useEffect(() => () => { + clearTimeout(copiedTimeout.current); + setCopiedVisible(false); + }, []); + + let denied = clipboard.readPermissionState === "denied"; + return + + ; +} + +function ExportButton(props) { + let [visible, setVisible] = useState(false); + function close() { + setVisible(false); + } + + let exportString = props.modPack.toModPackString(); + + return <> + + + + + } + > + + {exportString} + + + ; +} + +function useExportedAsset(modPack, asset) { + let [assetData, setAssetData] = useState(asset === "locale" ? new Map() : {}); + let assetFilename; + if (modPack instanceof libData.ModPack && modPack.exportManifest && modPack.exportManifest.assets[asset]) { + assetFilename = modPack.exportManifest.assets[asset]; + } + useEffect(() => { + async function load() { + if (!assetFilename) { + return; + } + + let response = await fetch(`${staticRoot}static/${assetFilename}`); + if (response.ok) { + let data = await response.json(); + setAssetData(asset === "locale" ? new Map(data) : data); + } else { + logger.error(`Error loading mod pack asset "${asset}" (${response.status}): ${await response.text()}`); + } + } + + load().catch(notifyErrorHandler(`Loading mod pack asset "${asset}" failed`)); + }, [assetFilename]); + + return assetData; +} + +export default function ModPackViewPage() { + let account = useAccount(); + let history = useHistory(); + let control = useContext(ControlContext); + let params = useParams(); + let modPackId = Number(params.id); + + const [form] = Form.useForm(); + let [modPack] = useModPack(modPackId); + let prototypes = useExportedAsset(modPack, "settings"); + let locale = useExportedAsset(modPack, "locale"); + let [changes, setChanges] = useState([]); + let [modifiedModPack, setModifiedModPack] = useState(modPack); + let syncTimeout = useRef(); + + useEffect(() => () => { clearTimeout(syncTimeout.current); }); + + function pushChange(change) { + setChanges(oldChanges => { + const newChanges = [...oldChanges]; + if ( + newChanges.length + && newChanges.slice(-1)[0].type === change.type + && newChanges.slice(-1)[0].name === change.name + && newChanges.slice(-1)[0].scope === change.scope + ) { + newChanges.splice(-1, 1, change); + } else { + newChanges.push(change); + } + return newChanges; + }); + } + + function revertChange(change) { + const newChanges = [...changes]; + const index = newChanges.findLastIndex(reference => ( + change.type === reference.type && change.name === reference.name + )); + if (index === -1) { + logger.error(`Unable to revert ${change.type} ${change.name}: change not found`); + return; + } + newChanges.splice(index, 1); + setChanges(newChanges); + } + + function syncPack(fields) { + for (let field of fields) { + if (!field.touched || field.validating || field.errors.length) { + continue; + } + if (["name", "description", "factorioVersion"].includes(field.name[0])) { + if (modifiedModPack[field.name[0]] !== field.value) { + pushChange({ type: field.name[0], value: field.value }); + } + + } else if (["startup", "runtime-global", "runtime-per-user"].includes(field.name[0])) { + let [scope, settingName] = field.name; + let value = field.value; + if (typeof modPack.settings[scope].get(settingName).value === "number") { + value = Number(value); + if (Number.isNaN(value)) { + continue; + } + } + if (modifiedModPack.settings[scope].get(settingName).value !== value) { + pushChange({ type: "settings.set", scope, name: settingName, value: { value }}); + } + } + } + } + + useEffect(() => { + // Loading or invalid id + if (!(modPack instanceof libData.ModPack)) { + setModifiedModPack(modPack); + return; + } + + const modified = new libData.ModPack(modPack.toJSON()); + for (let change of changes) { + if (change.type === "name") { + modified.name = change.value; + } else if (change.type === "description") { + modified.description = change.value; + } else if (change.type === "factorioVersion") { + modified.factorioVersion = change.value; + } else if (change.type === "mods.set") { + modified.mods.set(change.name, change.value); + } else if (change.type === "mods.delete") { + modified.mods.delete(change.name); + } else if (change.type === "settings.set") { + modified.settings[change.scope].set(change.name, change.value); + } else if (change.type === "settings.delete") { + modified.settings[change.scope].delete(change.name); + } else { + throw new Error(`Unknown change type ${change.type}`); + } + } + setModifiedModPack(modified); + }, [modPack, changes]); + + let nav = [{ name: "Mods", path: "/mods" }, { name: "Mod Packs" }, { name: modPack.name || modPackId }]; + if (modifiedModPack.loading) { + return + + + ; + } + + if (modifiedModPack.missing) { + return + +

Mod pack with id {modPackId} was not found on the master server.

+ ; + } + + return + + + {account.hasPermission("core.mod-pack.delete") && { + libLink.messages.deleteModPack.send( + control, { id: modPack.id } + ).then(() => { + history.push("/mods"); + }).catch(notifyErrorHandler("Error deleting mod pack")); + }} + > + + } + } + /> +
{ + clearTimeout(syncTimeout.current); + if (typeof changedFields[0].value === "boolean") { + syncPack(allFields); + } else { + // Defer sync if this is not a checkbox to make the UI responsive. + syncTimeout.current = setTimeout(syncPack.bind(undefined, allFields), 500); + } + }} + initialValues={{ + name: modPack.name, + description: modPack.description, + factorioVersion: modPack.factorioVersion, + "startup": Object.fromEntries(pickValue(modPack.settings["startup"])), + "runtime-global": Object.fromEntries(pickValue(modPack.settings["runtime-global"])), + "runtime-per-user": Object.fromEntries(pickValue(modPack.settings["runtime-per-user"])), + }} + > + + + + + + + + + + + + + {modifiedModPack.mods.size} + + + + + + + + + + + +
+
+ +
You have unsaved changes + + + + + + + + + + ; + +} diff --git a/packages/web_ui/src/components/ModsPage.jsx b/packages/web_ui/src/components/ModsPage.jsx new file mode 100644 index 00000000..71a1cca6 --- /dev/null +++ b/packages/web_ui/src/components/ModsPage.jsx @@ -0,0 +1,277 @@ +import React, { Fragment, useState, useContext } from "react"; +import { useHistory } from "react-router-dom"; +import { Button, Form, Input, Modal, PageHeader, Popconfirm, Space, Table, Typography, Upload } from "antd"; +import ImportOutlined from "@ant-design/icons/ImportOutlined"; +import PlusOutlined from "@ant-design/icons/PlusOutlined"; +import UploadOutlined from "@ant-design/icons/UploadOutlined"; + +import { libData, libHelpers, libLink, libLogging } from "@clusterio/lib"; + +import { useAccount } from "../model/account"; +import { useModList } from "../model/mods"; +import { useModPackList } from "../model/mod_pack"; +import { notifyErrorHandler } from "../util/notify"; +import ControlContext from "./ControlContext"; +import PageLayout from "./PageLayout"; +import PluginExtra from "./PluginExtra"; +import SectionHeader from "./SectionHeader"; +import ModDetails from "./ModDetails"; + +const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; + +const { logger } = libLogging; + + +function ImportModPackButton() { + let control = useContext(ControlContext); + let history = useHistory(); + let [visible, setVisible] = useState(false); + let [form] = Form.useForm(); + function close() { + setVisible(false); + } + return <> + + { setVisible(false); }} + onOk={() => { + (async () => { + let values; + try { + values = await form.validateFields(); + } catch { + return; // Validation failed + } + const modPack = libData.ModPack.fromModPackString(values.string); + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + history.push(`/mods/mod-packs/${modPack.id}/view`); + })().catch(notifyErrorHandler("Error creating mod pack")); + }} + > +
+ + + + + + ; +} + +function CreateModPackButton() { + let control = useContext(ControlContext); + let history = useHistory(); + let [visible, setVisible] = useState(false); + let [form] = Form.useForm(); + function close() { + setVisible(false); + } + return <> + + { setVisible(false); }} + onOk={() => { + (async () => { + let values; + try { + values = await form.validateFields(); + } catch { + return; // Validation failed + } + const modPack = new libData.ModPack(); + modPack.name = values.name; + modPack.factorioVersion = values.factorioVersion; + if (values.description) { modPack.description = values.description; } + await libLink.messages.createModPack.send(control, { mod_pack: modPack.toJSON() }); + history.push(`/mods/mod-packs/${modPack.id}/view`); + })().catch(notifyErrorHandler("Error creating mod pack")); + }} + > +
+ + + + + + + + + + + + ; +} + +export default function ModsPage() { + let account = useAccount(); + let control = useContext(ControlContext); + let history = useHistory(); + let [modList] = useModList(); + let [modPackList] = useModPackList(); + + function actions(mod) { + return + {account.hasPermission("core.mod.download") + && { + libLink.messages.downloadMod.send( + control, { name: mod.name, version: mod.version } + ).then(response => { + let url = new URL(webRoot, document.location); + url.pathname += `api/stream/${response.stream_id}`; + document.location = url; + }).catch( + notifyErrorHandler("Error downloading save") + ); + }} + >download + } + {account.hasPermission("core.mod.delete") + && { + libLink.messages.deleteMod.send( + control, { name: mod.name, version: mod.version } + ).catch(notifyErrorHandler("Error deleting mod")); + }} + okText="Delete" + okButtonProps={{ danger: true }} + > + delete + + } + ; + } + + let uploadButton; + if (account.hasPermission("core.mod.upload")) { + uploadButton = + + ; + } + + return + + + {account.hasPermission("core.mod-pack.create") && } + {account.hasPermission("core.mod-pack.create") && } + } + /> +
strcmp(a.name, b.name), + }, + { + title: "Factorio Version", + dataIndex: "factorioVersion", + sorter: (a, b) => a.integerFactorioVersion - b.integerFactorioVersion, + }, + { + title: "Mods", + key: "roles", + render: modPack => modPack.mods.size, + }, + ]} + dataSource={modPackList} + pagination={false} + rowKey={modPack => modPack.id} + onRow={(modPack, rowIndex) => ({ + onClick: event => { + history.push(`/mods/mod-packs/${modPack.id}/view`); + }, + })} + /> + +
( + strcmp(a.name, b.name) || a.integerVersion - b.integerVersion + ), + }, + { + title: "Version", + key: "version", + align: "right", + render: mod => <> + {`${mod.version} `} + {`/ ${mod.factorioVersion}`} + , + }, + { + title: "Filename", + dataIndex: "filename", + responsive: ["xl"], + sorter: (a, b) => strcmp(a.filename, b.filename), + }, + { + title: "Size", + key: "size", + responsive: ["lg"], + render: mod => libHelpers.formatBytes(mod.size), + align: "right", + sorter: (a, b) => a.size - b.size, + }, + { + title: "Action", + key: "action", + responsive: ["lg"], + render: mod => actions(mod), + }, + ]} + expandable={{ + expandedRowRender: mod => , + expandedRowClassName: () => "no-expanded-padding", + }} + dataSource={modList} + pagination={false} + rowKey={mod => `${mod.name}_${mod.version}`} + /> + + ; +} diff --git a/packages/web_ui/src/components/SavesList.jsx b/packages/web_ui/src/components/SavesList.jsx index 133cffac..686c567c 100644 --- a/packages/web_ui/src/components/SavesList.jsx +++ b/packages/web_ui/src/components/SavesList.jsx @@ -6,7 +6,7 @@ import { import CaretLeftOutlined from "@ant-design/icons/CaretLeftOutlined"; import LeftOutlined from "@ant-design/icons/LeftOutlined"; -import { libLink } from "@clusterio/lib"; +import { libLink, libHelpers } from "@clusterio/lib"; import { useAccount } from "../model/account"; import ControlContext from "./ControlContext"; @@ -179,17 +179,6 @@ function TransferModal(props) { ; } -function formatBytes(bytes) { - if (bytes === 0) { - return "0 Bytes"; - } - - let units = [" Bytes", " kB", " MB", " GB", " TB"]; - let factor = 1000; - let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length); - return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; -} - export default function SavesList(props) { let account = useAccount(); let control = useContext(ControlContext); @@ -214,7 +203,7 @@ export default function SavesList(props) { title: "Size", key: "size", responsive: ["sm"], - render: save => formatBytes(save.size), + render: save => libHelpers.formatBytes(save.size), align: "right", sorter: (a, b) => a.size - b.size, }, diff --git a/packages/web_ui/src/components/SiteLayout.jsx b/packages/web_ui/src/components/SiteLayout.jsx index 61606449..2d83b3d5 100644 --- a/packages/web_ui/src/components/SiteLayout.jsx +++ b/packages/web_ui/src/components/SiteLayout.jsx @@ -49,7 +49,13 @@ export default function SiteLayout(props) { let menuItems = []; let menuGroups = new Map(); for (let { sidebarName, sidebarGroup, permission, path } of combinedPages) { - if (!sidebarName || permission && !account.hasPermission(permission)) { + if ( + !sidebarName || permission && ( + typeof permission === "function" + ? !permission(account) + : !account.hasPermission(permission) + ) + ) { continue; } if (sidebarGroup) { diff --git a/packages/web_ui/src/index.css b/packages/web_ui/src/index.css index 37f3889b..267f11bf 100644 --- a/packages/web_ui/src/index.css +++ b/packages/web_ui/src/index.css @@ -27,7 +27,7 @@ body { } .site-layout { - overflow-x: hidden; + overflow-x: clip; } .site-layout-content-container { @@ -73,6 +73,7 @@ body { .section-header { display: flex; + flex-wrap: wrap; align-items: center; margin-top: 16px; margin-bottom: 8px; @@ -295,6 +296,10 @@ body { width: 32px; } +.code > code { + display: block; +} + /* * Render codeblock with options for syntax highlighting * https://user-images.githubusercontent.com/6792749/152066701-21420461-710b-4fae-a80a-89ef90a0b18f.png @@ -314,3 +319,38 @@ body { top: 8px; right: 8px; } + + +.ant-descriptions-bordered.borderless > .ant-descriptions-view { + border: none; +} + +tr.no-expanded-padding > td, +.ant-table.ant-table-small .ant-table-tbody > tr.no-expanded-padding > td { + padding: 0; +} + +.ant-form-item.changed, +input.ant-input.changed, +textarea.ant-input.changed, +tr.ant-table-row.changed > td { + background: #2a1912; +} + +.ant-table-tbody > tr.ant-table-row.changed:hover > td, +.ant-table-tbody > tr.changed > td.ant-table-cell-row-hover +tr.ant-table-row.deleted > td { + background: #3a2922; +} + +tr.ant-table-row.deleted { + color: rgba(255, 255, 255, 0.45); + background: #340c08; + text-decoration: line-through; +} + +.ant-table-tbody > tr.ant-table-row.deleted:hover > td, +.ant-table-tbody > tr.deleted > td.ant-table-cell-row-hover +tr.ant-table-row.deleted { + background: #441c18; +} diff --git a/packages/web_ui/src/model/export_manifest.jsx b/packages/web_ui/src/model/export_manifest.jsx index ebb436d9..50ef1467 100644 --- a/packages/web_ui/src/model/export_manifest.jsx +++ b/packages/web_ui/src/model/export_manifest.jsx @@ -1,15 +1,20 @@ -import { useEffect, useState } from "react"; +import { useEffect, useContext, useState } from "react"; + +import { libData, libLink } from "@clusterio/lib"; +import ControlContext from "../components/ControlContext"; let exportManifestCache = null; let emptyCache = {}; export function useExportManifest() { + const control = useContext(ControlContext); let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache); useEffect(() => { async function load() { - let response = await fetch(`${webRoot}api/export-manifest`); - if (response.ok) { - exportManifestCache = await response.json(); + let result = await libLink.messages.getDefaultModPack.send(control); + let modPack = new libData.ModPack(result.mod_pack); + if (modPack.exportManifest) { + exportManifestCache = modPack.exportManifest; setExportManifest(exportManifestCache); } } diff --git a/packages/web_ui/src/model/item_metadata.jsx b/packages/web_ui/src/model/item_metadata.jsx index 3858e296..8c1ae078 100644 --- a/packages/web_ui/src/model/item_metadata.jsx +++ b/packages/web_ui/src/model/item_metadata.jsx @@ -9,10 +9,14 @@ export function useItemMetadata() { let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map()); useEffect(() => { async function load() { - if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) { + if ( + !exportManifest + || !exportManifest.assets["item-metadata"] + || !exportManifest.assets["item-spritesheet"] + ) { return; } - let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`); + let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`); if (response.ok) { let data = await response.json(); itemMetadataCache = new Map(data); @@ -22,7 +26,7 @@ export function useItemMetadata() { for (let [name, meta] of itemMetadataCache) { style.sheet.insertRule( `.item-${CSS.escape(name)} { - background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}"); + background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}"); background-repeat: no-repeat; background-position: -${meta.x}px -${meta.y}px; height: ${meta.size}px; diff --git a/packages/web_ui/src/model/locale.jsx b/packages/web_ui/src/model/locale.jsx index 3a22e6d7..431833fc 100644 --- a/packages/web_ui/src/model/locale.jsx +++ b/packages/web_ui/src/model/locale.jsx @@ -9,10 +9,10 @@ export function useLocale() { let [locale, setLocale] = useState(localeCache || new Map()); useEffect(() => { async function load() { - if (!exportManifest["locale"]) { + if (!exportManifest || !exportManifest.assets["locale"]) { return; } - let response = await fetch(`${staticRoot}${exportManifest["locale"]}`); + let response = await fetch(`${staticRoot}static/${exportManifest.assets["locale"]}`); if (response.ok) { let data = await response.json(); localeCache = new Map(data); diff --git a/packages/web_ui/src/model/mod_pack.jsx b/packages/web_ui/src/model/mod_pack.jsx new file mode 100644 index 00000000..4983cf54 --- /dev/null +++ b/packages/web_ui/src/model/mod_pack.jsx @@ -0,0 +1,78 @@ +import { useContext, useEffect, useState } from "react"; + +import { libData, libLink, libLogging } from "@clusterio/lib"; + +import ControlContext from "../components/ControlContext"; + +const { logger } = libLogging; + +export function useModPack(id) { + let control = useContext(ControlContext); + let [modPack, setModPack] = useState({ loading: true }); + + function updateModPack() { + libLink.messages.getModPack.send(control, { id }).then(result => { + setModPack(new libData.ModPack(result.mod_pack)); + }).catch(err => { + logger.error(`Failed to get mod pack: ${err}`); + setModPack({ missing: true }); + }); + } + + useEffect(() => { + if (typeof id !== "number") { + setModPack({ missing: true }); + return undefined; + } + updateModPack(); + + control.onModPackUpdate(id, setModPack); + return () => { + control.offModPackUpdate(id, setModPack); + }; + }, [id]); + + return [modPack]; +} + +export function useModPackList() { + let control = useContext(ControlContext); + let [modPackList, setModPackList] = useState([]); + + function updateModPackList() { + libLink.messages.listModPacks.send(control).then(result => { + setModPackList(result.list.map(pack => new libData.ModPack(pack))); + }).catch(err => { + logger.error(`Failed to list mod packs:\n${err}`); + }); + } + + useEffect(() => { + updateModPackList(); + + function updateHandler(newModPack) { + setModPackList(oldList => { + let newList = oldList.concat(); + let index = newList.findIndex(u => u.id === newModPack.id); + if (!newModPack.isDeleted) { + if (index !== -1) { + newList[index] = newModPack; + } else { + newList.push(newModPack); + } + } else if (index !== -1) { + newList.splice(index, 1); + } + return newList; + }); + } + + control.onModPackUpdate(null, updateHandler); + return () => { + control.offModPackUpdate(null, updateHandler); + }; + }, []); + + return [modPackList]; + +} diff --git a/packages/web_ui/src/model/mods.jsx b/packages/web_ui/src/model/mods.jsx new file mode 100644 index 00000000..2ef9971c --- /dev/null +++ b/packages/web_ui/src/model/mods.jsx @@ -0,0 +1,50 @@ +import { useContext, useEffect, useState } from "react"; + +import { libData, libLink, libLogging } from "@clusterio/lib"; + +import ControlContext from "../components/ControlContext"; + +const { logger } = libLogging; + + +export function useModList() { + let control = useContext(ControlContext); + let [modList, setModList] = useState([]); + + function updateModList() { + libLink.messages.listMods.send(control).then(result => { + setModList(result.list.map(mod => new libData.ModInfo(mod))); + }).catch(err => { + logger.error(`Failed to list mods:\n${err}`); + }); + } + + useEffect(() => { + updateModList(); + + function updateHandler(newMod) { + setModList(oldList => { + let newList = oldList.concat(); + let index = newList.findIndex(mod => mod.name === newMod.name && mod.version === newMod.version); + if (!newMod.isDeleted) { + if (index !== -1) { + newList[index] = newMod; + } else { + newList.push(newMod); + } + } else if (index !== -1) { + newList.splice(index, 1); + } + return newList; + }); + } + + control.onModUpdate(null, updateHandler); + return () => { + control.offModUpdate(null, updateHandler); + }; + }, []); + + return [modList]; +} + diff --git a/packages/web_ui/src/pages.jsx b/packages/web_ui/src/pages.jsx index cec0de30..ed85e7a5 100644 --- a/packages/web_ui/src/pages.jsx +++ b/packages/web_ui/src/pages.jsx @@ -5,6 +5,8 @@ import SlavesPage from "./components/SlavesPage"; import SlaveViewPage from "./components/SlaveViewPage"; import InstancesPage from "./components/InstancesPage"; import InstanceViewPage from "./components/InstanceViewPage"; +import ModPackViewPage from "./components/ModPackViewPage"; +import ModsPage from "./components/ModsPage"; import UsersPage from "./components/UsersPage"; import UserViewPage from "./components/UserViewPage"; import RolesPage from "./components/RolesPage"; @@ -41,6 +43,17 @@ export const pages = [ sidebarPath: "/instances", content: , }, + { + path: "/mods", + sidebarName: "Mods", + permission: account => account.hasAnyPermission("core.mod.list", "core.mod-pack.list"), + content: , + }, + { + path: "/mods/mod-packs/:id/view", + sidebarPath: "/mods", + content: , + }, { path: "/users", sidebarName: "Users", diff --git a/packages/web_ui/src/util/clipboard.jsx b/packages/web_ui/src/util/clipboard.jsx new file mode 100644 index 00000000..00f51e21 --- /dev/null +++ b/packages/web_ui/src/util/clipboard.jsx @@ -0,0 +1,69 @@ +import { useEffect, useState } from "react"; +import { EventEmitter } from "events"; + +class Permission extends EventEmitter { + status; + + constructor(name) { + super(); + navigator.permissions.query({ name }).then( + status => { + this.status = status; + this.emit("change", this.status.state); + + if (status.addEventListener) { + status.addEventListener("change", () => { + this.emit("change", this.status.state); + }); + } + }, + // Pretend the clipboard will work if this query fails. + () => { + this.status = { state: "prompt" }; + this.emit("change", "prompt"); + } + ); + } + + get state() { + return this.status.state || "prompt"; + } +} + +let readPermission = new Permission("clipboard-read"); +let writePermission = new Permission("clipboard-write"); + +// Wrapper for navigator.clipboard suitable for use in React +export function useClipboard() { + let [readPermissionState, setReadPermissionState] = useState(readPermission.state); + let [writePermissionState, setWritePermissionState] = useState(writePermission.state); + + useEffect(() => { + readPermission.on("change", setReadPermissionState); + writePermission.on("change", setWritePermissionState); + return () => { + readPermission.off("change", setReadPermissionState); + writePermission.off("change", setWritePermissionState); + }; + }, []); + + return { + readPermissionState: readPermissionState, + read: async () => await navigator.clipboard.read(), + readText: async () => await navigator.clipboard.readText(), + writePermissionState: writePermissionState, + write: async (data) => await navigator.clipboard.write(data), + writeText: async (text) => await navigator.clipboard.writeText(text), + + // Give a possible reason for why the clipboard is not available + deniedReason: () => { + if (window.isSecureContext === false) { + if (window.location.protocol !== "https:") { + return "Clipboard interaction is blocked due this page not being served over https."; + } + return "Clipboard interaction is blocked due this page not being a secure context."; + } + return "Clipboard interaction is blocked by the browser."; + }, + }; +} diff --git a/packages/web_ui/src/util/websocket.jsx b/packages/web_ui/src/util/websocket.jsx index 12d8eacd..fd9b56a3 100644 --- a/packages/web_ui/src/util/websocket.jsx +++ b/packages/web_ui/src/util/websocket.jsx @@ -1,4 +1,4 @@ -import { libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib"; +import { libData, libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib"; const { logger } = libLogging; import packageJson from "../../package.json"; @@ -86,6 +86,8 @@ export class Control extends libLink.Link { this.slaveUpdateHandlers = new Map(); this.instanceUpdateHandlers = new Map(); this.saveListUpdateHandlers = new Map(); + this.modPackUpdateHandlers = new Map(); + this.modUpdateHandlers = new Map(); this.userUpdateHandlers = new Map(); this.logHandlers = new Map(); @@ -102,6 +104,12 @@ export class Control extends libLink.Link { this.updateSaveListSubscriptions().catch(err => logger.error( `Unexpected error updating save list subscriptions:\n${err.stack}` )); + this.updateModPackSubscriptions().catch(err => logger.error( + `Unexpected error updating mod pack subscriptions:\n${err.stack}` + )); + this.updateModSubscriptions().catch(err => logger.error( + `Unexpected error updating mod subscriptions:\n${err.stack}` + )); this.updateUserSubscriptions().catch(err => logger.error( `Unexpected error updating user subscriptions:\n${err.stack}` )); @@ -319,6 +327,122 @@ export class Control extends libLink.Link { }); } + async modPackUpdateEventHandler(message) { + let modPack = new libData.ModPack(message.data.mod_pack); + let handlers = [].concat( + this.modPackUpdateHandlers.get(null) || [], + this.modPackUpdateHandlers.get(modPack.id) || [], + ); + for (let handler of handlers) { + handler(modPack); + } + } + + async onModPackUpdate(id, handler) { + if (id !== null && typeof id !== "number") { + throw new Error("Invalid mod pack id"); + } + + let handlers = this.modPackUpdateHandlers.get(id); + if (!handlers) { + handlers = []; + this.modPackUpdateHandlers.set(id, handlers); + } + + handlers.push(handler); + + if (handlers.length === 1) { + await this.updateModPackSubscriptions(); + } + } + + async offModPackUpdate(id, handler) { + let handlers = this.modPackUpdateHandlers.get(id); + if (!handlers || !handlers.length) { + throw new Error(`No handlers for mod pack ${id} exist`); + } + + let index = handlers.lastIndexOf(handler); + if (index === -1) { + throw new Error(`Given handler is not registered for mod pack ${id}`); + } + + handlers.splice(index, 1); + if (!handlers.length) { + this.modPackUpdateHandlers.delete(id); + await this.updateModPackSubscriptions(); + } + } + + async updateModPackSubscriptions() { + if (!this.connector.connected) { + return; + } + + await libLink.messages.setModPackSubscriptions.send(this, { + all: this.modPackUpdateHandlers.has(null), + mod_pack_ids: [...this.modPackUpdateHandlers.keys()].filter(k => k !== null), + }); + } + + async modUpdateEventHandler(message) { + let mod = new libData.ModInfo(message.data.mod); + let handlers = [].concat( + this.modUpdateHandlers.get(null) || [], + this.modUpdateHandlers.get(mod.name) || [] + ); + for (let handler of handlers) { + handler(mod); + } + } + + async onModUpdate(name, handler) { + if (name !== null && typeof name !== "string") { + throw new Error("Invalid mod name"); + } + + let handlers = this.modUpdateHandlers.get(name); + if (!handlers) { + handlers = []; + this.modUpdateHandlers.set(name, handlers); + } + + handlers.push(handler); + + if (handlers.length === 1) { + await this.updateModSubscriptions(); + } + } + + async offModUpdate(name, handler) { + let handlers = this.modUpdateHandlers.get(name); + if (!handlers || !handlers.length) { + throw new Error(`No handlers for mod ${name} exist`); + } + + let index = handlers.lastIndexOf(handler); + if (index === -1) { + throw new Error(`Given handler is not registered for mod ${name}`); + } + + handlers.splice(index, 1); + if (!handlers.length) { + this.modUpdateHandlers.delete(name); + await this.updateModSubscriptions(); + } + } + + async updateModSubscriptions() { + if (!this.connector.connected) { + return; + } + + await libLink.messages.setModSubscriptions.send(this, { + all: this.modUpdateHandlers.has(null), + mod_names: [...this.modUpdateHandlers.keys()].filter(k => k !== null), + }); + } + async userUpdateEventHandler(message) { let handlers = [].concat( this.userUpdateHandlers.get(null) || [], diff --git a/packages/web_ui/webpack.common.js b/packages/web_ui/webpack.common.js index 8b871783..628eb217 100644 --- a/packages/web_ui/webpack.common.js +++ b/packages/web_ui/webpack.common.js @@ -95,6 +95,8 @@ module.exports = (env = {}) => ({ resolve: { extensions: [".js", ".json", ".jsx"], fallback: { + "crypto": false, + // Required for winston "util": require.resolve("util/"), "os": require.resolve("os-browserify/browser"), diff --git a/test/file/empty_mod/info.json b/test/file/empty_mod/info.json new file mode 100644 index 00000000..0537aa9a --- /dev/null +++ b/test/file/empty_mod/info.json @@ -0,0 +1,9 @@ +{ + "name": "empty_mod", + "version": "1.0.0", + "title": "An Empty Mod", + "author": "Me", + "description": "An empty mod for testing", + "factorio_version": "1.1", + "dependencies": [] +} diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index cb91a631..ab94dd89 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -5,6 +5,9 @@ const path = require("path"); const events = require("events"); const phin = require("phin"); +const libBuildMod = require("@clusterio/lib/build_mod"); +const libData = require("@clusterio/lib/data"); +const libHash = require("@clusterio/lib/hash"); const libLink = require("@clusterio/lib/link"); const libUsers = require("@clusterio/lib/users"); const { wait } = require("@clusterio/lib/helpers"); @@ -303,13 +306,19 @@ describe("Integration of Clusterio", function() { let exportPath = path.join("temp", "test", "static"); await fs.remove(exportPath); await execCtl("instance export-data test"); - let response = await get("/api/export-manifest"); - let manifest = JSON.parse(response.body); - assert(Object.keys(manifest).length > 1, "Export manifest is empty"); - assert( - await fs.exists(path.join(exportPath, "..", manifest["item-metadata"])), - "Export was not written to filesystem" - ); + let result = await libLink.messages.getDefaultModPack.send(getControl()); + let modPack = new libData.ModPack(result.mod_pack); + let assets = modPack.exportManifest.assets; + assert(Object.keys(assets).length > 1, "Export assets is empty"); + for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) { + assert(assets[key], `Missing ${key} from assets`); + assert( + await fs.exists(path.join(exportPath, assets[key])), + `Manifest entry for ${key} was not written to filesystem` + ); + } + let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, assets["prototypes"]))); + assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types"); await checkInstanceStatus(44, "stopped"); }); }); @@ -818,6 +827,180 @@ describe("Integration of Clusterio", function() { }); }); + describe("mod-pack create", function() { + it("should create a mod-pack", async function() { + await execCtl("mod-pack create empty-pack 1.1.0"); + let response = await libLink.messages.listModPacks.send(getControl()); + assert(response.list.some(modPack => modPack.name === "empty-pack"), "created pack is not in the list"); + }); + it("should allow setting all fields", async function() { + await execCtl( + "mod-pack create full-pack 0.17.59 " + + "--description Description " + + "--mods empty_mod:1.0.0 " + + "--bool-setting startup MyBool true " + + "--int-setting runtime-global MyInt 1235 " + + "--double-setting runtime-global MyDouble 12.25 " + + "--string-setting runtime-per-user MyString a-string" + ); + let response = await libLink.messages.listModPacks.send(getControl()); + let modPack = response.list.find(entry => entry.name === "full-pack"); + assert(modPack, "created mod pack not found"); + let reference = new libData.ModPack(); + reference.id = modPack.id; + reference.name = "full-pack"; + reference.description = "Description"; + reference.factorioVersion = "0.17.59"; + reference.mods.set("empty_mod", { name: "empty_mod", enabled: true, version: "1.0.0" }); + reference.settings["startup"].set("MyBool", { value: true }); + reference.settings["runtime-global"].set("MyInt", { value: 1235 }); + reference.settings["runtime-global"].set("MyDouble", { value: 12.25 }); + reference.settings["runtime-per-user"].set("MyString", { value: "a-string" }); + assert.deepEqual(new libData.ModPack(modPack), reference); + }); + }); + + describe("mod-pack list", function() { + it("runs", async function() { + await execCtl("mod-pack list"); + }); + }); + + describe("mod-pack show", function() { + it("runs", async function() { + await execCtl("mod-pack show empty-pack"); + }); + }); + + describe("mod-pack import/export", function() { + it("should should roundtrip a mod-pack", async function() { + let reference = new libData.ModPack(); + reference.name = "imported-pack"; + reference.description = "Description"; + reference.factorioVersion = "0.17.59"; + reference.mods.set("empty_mod", { name: "empty_mod", enabled: true, version: "1.0.0" }); + reference.settings["startup"].set("MyBool", { value: true }); + reference.settings["runtime-global"].set("MyInt", { value: 1235 }); + reference.settings["runtime-global"].set("MyDouble", { value: 12.25 }); + reference.settings["runtime-per-user"].set("MyString", { value: "a-string" }); + await execCtl(`mod-pack import ${reference.toModPackString()}`); + const result = await execCtl("mod-pack export imported-pack"); + const roundtrip = libData.ModPack.fromModPackString(result.stdout.trim()); + roundtrip.id = reference.id; + assert.deepEqual(roundtrip, reference); + }); + }); + + describe("mod-pack edit", function() { + it("runs", async function() { + await execCtl("mod-pack edit full-pack --factorio-version 1.2.0"); + let response = await libLink.messages.listModPacks.send(getControl()); + let modPack = response.list.find(entry => entry.name === "full-pack"); + assert(modPack, "created mod pack not found"); + assert.equal(modPack.factorio_version, "1.2.0"); + }); + }); + + describe("mod-pack delete", function() { + it("deletes the pack", async function() { + await execCtl("mod-pack delete full-pack"); + let response = await libLink.messages.listModPacks.send(getControl()); + let modPack = response.list.find(entry => entry.name === "full-pack"); + assert(!modPack, "mod pack not deleted"); + }); + }); + + describe("mod upload", function() { + it("uploads a mod", async function() { + await libBuildMod.build({ + build: true, + pack: true, + sourceDir: path.join("test", "file", "empty_mod"), + outputDir: path.join("temp", "test"), + }); + await execCtl("mod upload empty_mod_1.0.0.zip"); + assert( + await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")), + "mod not present in mods directory" + ); + }); + }); + + describe("mod show", function() { + it("gives details of a mod", async function() { + let result = await execCtl("mod show empty_mod 1.0.0"); + let hash = await libHash.hashFile(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")); + let stat = await fs.stat(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")); + assert.equal( + result.stdout, + "name: empty_mod\n" + + "version: 1.0.0\n" + + "title: An Empty Mod\n" + + "author: Me\n" + + "description: An empty mod for testing\n" + + "factorio_version: 1.1\n" + + "dependencies:\n" + + `size: ${stat.size}\n` + + `sha1: ${hash}\n`, + ); + }); + }); + + describe("mod list", function() { + it("shows the list of mods", async function() { + let result = await execCtl("mod list"); + assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the list"); + }); + }); + + describe("mod search", function() { + it("searches the list of mods", async function() { + let result = await execCtl("mod search 1.1 name:empty_mod"); + assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the result"); + }); + }); + + describe("mod download", function() { + it("downloads a mod", async function() { + await fs.unlink(path.join("temp", "test", "empty_mod_1.0.0.zip")); + await execCtl("mod download empty_mod 1.0.0"); + assert( + await fs.pathExists(path.join("temp", "test", "empty_mod_1.0.0.zip")), + "mod not downloaded to cwd" + ); + }); + }); + + describe("mod delete", function() { + it("deletes a mod", async function() { + await execCtl("mod delete empty_mod 1.0.0"); + assert( + !await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")), + "mod still present in mods dir" + ); + }); + }); + + describe("modUpdateEventHandler()", function() { + it("should have triggered for the previous mod updates", function() { + let eventsToCheck = new Set(["updated", "deleted"]); + let eventsNotSeen = new Set(eventsToCheck); + + for (let modUpdate of getControl().modUpdates) { + if (modUpdate.name !== "empty_mod" || modUpdate.version !== "1.0.0") { + continue; + } + if (modUpdate.is_deleted) { + eventsNotSeen.delete("deleted"); + } else { + eventsNotSeen.delete("updated"); + } + } + + assert(eventsNotSeen.size === 0, `Did not see the events ${[...eventsNotSeen]}`); + }); + }); + describe("permission list", function() { it("runs", async function() { await execCtl("permission list"); diff --git a/test/integration/index.js b/test/integration/index.js index 1084570e..41eabfc9 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -8,6 +8,7 @@ const phin = require("phin"); const util = require("util"); const events = require("events"); +const libData = require("@clusterio/lib/data"); const libLink = require("@clusterio/lib/link"); const { LineSplitter } = require("@clusterio/lib/stream"); const { ConsoleTransport, logger } = require("@clusterio/lib/logging"); @@ -28,6 +29,8 @@ class TestControl extends libLink.Link { this.slaveUpdates = []; this.instanceUpdates = []; this.saveListUpdates = []; + this.modUpdates = []; + this.modPackUpdates = []; this.userUpdates = []; this.connector.on("connect", () => { @@ -40,6 +43,9 @@ class TestControl extends libLink.Link { libLink.messages.setSaveListSubscriptions.send( this, { all: true, instance_ids: [] } ).catch(err => logger.error(`Error setting save list subscriptions:\n${err.stack}`)); + libLink.messages.setModSubscriptions.send( + this, { all: true, mod_names: [] } + ).catch(err => logger.error(`Error setting mod subscriptions:\n${err.stack}`)); libLink.messages.setUserSubscriptions.send( this, { all: true, names: [] } ).catch(err => logger.error(`Error setting user subscriptions:\n${err.stack}`)); @@ -67,6 +73,14 @@ class TestControl extends libLink.Link { this.saveListUpdates.push(message.data); } + async modUpdateEventHandler(message) { + this.modUpdates.push(message.data.mod); + } + + async modPackUpdateEventHandler(message) { + this.modPackUpdates.push(message.data.mod_pack); + } + async userUpdateEventHandler(message) { this.userUpdates.push(message.data); } @@ -207,6 +221,15 @@ before(async function() { controlConnector.token = controlToken; control = new TestControl(controlConnector); await controlConnector.connect(); + + const testPack = new libData.ModPack(); + testPack.id = 12; + testPack.name = "subspace_storage-pack"; + testPack.factorioVersion = "1.1.0"; + testPack.mods.set("clusterio_lib", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); + testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); + await libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); + await libLink.messages.setMasterConfigField.send(control, { field: "master.default_mod_pack_id", value: "12" }); }); after(async function() { diff --git a/test/lib/data/ModInfo.js b/test/lib/data/ModInfo.js new file mode 100644 index 00000000..9e435e81 --- /dev/null +++ b/test/lib/data/ModInfo.js @@ -0,0 +1,101 @@ +"use strict"; +const assert = require("assert").strict; +const fs = require("fs-extra"); +const path = require("path"); + +const libSchema = require("@clusterio/lib/schema"); +const libHash = require("@clusterio/lib/hash"); +const libBuildMod = require("@clusterio/lib/build_mod"); +const ModInfo = require("@clusterio/lib/data/ModInfo"); + + +describe("lib/data/ModInfo", function() { + describe("class ModInfo", function() { + it("should round trip serialize", function() { + const validate = libSchema.compile(ModInfo.jsonSchema); + function check(pack) { + const json = JSON.parse(JSON.stringify(pack)); + if (!validate(json)) { + throw validate.errors; + } + assert.deepEqual(new ModInfo(json), pack); + } + + check(new ModInfo()); + check(new ModInfo({ name: "MyMod" })); + check(new ModInfo({ version: "1.0.0" })); + check(new ModInfo({ title: "My Extravagent Mod" })); + check(new ModInfo({ author: "Me" })); + check(new ModInfo({ contact: "Use my email" })); + check(new ModInfo({ homepage: "It's over there" })); + check(new ModInfo({ description: "My Description" })); + check(new ModInfo({ factorio_version: "2.0" })); + check(new ModInfo({ dependencies: [] })); + check(new ModInfo({ dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"] })); + check(new ModInfo({ filename: "MyMod_1.0.0.zip" })); + check(new ModInfo({ size: 1024 })); + check(new ModInfo({ sha1: "verified-as-MyMod" })); + check(new ModInfo({ is_deleted: true })); + + // All at once + check(new ModInfo({ + name: "MyMod", + version: "1.0.0", + title: "My Extravagent Mod", + author: "Me", + contact: "Use my email", + homepage: "It's over there", + description: "My Description", + factorio_version: "2.0", + dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"], + filename: "MyMod_1.0.0.zip", + size: 1024, + sha1: "verified-as-MyMod", + is_deleted: true, + })); + }); + + it("should sort integer mod versions lexicographically", function() { + let unsortedVersions = ["1.0.0", "1.1.0", "0.1.0", "3.0.0", "1.2.0", "0.3.1", "0.3.3", "2.1.1", "0.0.1"]; + let sortedVersions = ["0.0.1", "0.1.0", "0.3.1", "0.3.3", "1.0.0", "1.1.0", "1.2.0", "2.1.1", "3.0.0"]; + let mods = unsortedVersions.map(v => new ModInfo({ version: v })); + mods.sort((a, b) => a.integerVersion - b.integerVersion); + assert.deepEqual(mods.map(mod => mod.version), sortedVersions); + }); + it("should sort integer factorio versions lexicographically", function() { + let unsortedVersions = ["1.0", "1.1", "0.1", "3.0", "1.2", "0.3", "2.1"]; + let sortedVersions = ["0.1", "0.3", "1.0", "1.1", "1.2", "2.1", "3.0"]; + let factorioMods = unsortedVersions.map(v => new ModInfo({ factorio_version: v })); + factorioMods.sort((a, b) => a.integerFactorioVersion - b.integerFactorioVersion); + assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions); + }); + + it("should load from a mod zip file", async function() { + await libBuildMod.build({ + build: true, + pack: true, + sourceDir: path.join("test", "file", "empty_mod"), + outputDir: path.join("temp", "test"), + }); + const filePath = path.join("temp", "test", "empty_mod_1.0.0.zip"); + const hash = await libHash.hashFile(filePath); + const stat = await fs.stat(filePath); + const mod = await ModInfo.fromModFile(filePath); + assert.deepEqual( + mod, + new ModInfo({ + name: "empty_mod", + version: "1.0.0", + title: "An Empty Mod", + author: "Me", + description: "An empty mod for testing", + factorio_version: "1.1", + dependencies: [], + filename: "empty_mod_1.0.0.zip", + size: stat.size, + sha1: hash, + }) + ); + }); + }); +}); diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js new file mode 100644 index 00000000..43cdc856 --- /dev/null +++ b/test/lib/data/ModPack.js @@ -0,0 +1,267 @@ +"use strict"; +const assert = require("assert").strict; +const zlib = require("zlib"); + +const libSchema = require("@clusterio/lib/schema"); +const ExportManifest = require("@clusterio/lib/data/ExportManifest"); +const ModPack = require("@clusterio/lib/data/ModPack"); + + +describe("lib/data/ModPack", function() { + describe("class ModPack", function() { + it("should round trip serialize", function() { + const validate = libSchema.compile(ModPack.jsonSchema); + function check(pack) { + const json = JSON.parse(JSON.stringify(pack)); + if (!validate(json)) { + throw validate.errors; + } + assert.deepEqual(new ModPack(json), pack); + const packStringed = ModPack.fromModPackString(pack.toModPackString()); + packStringed.id = pack.id; + packStringed.exportManifest = pack.exportManifest; + assert.deepEqual(packStringed, pack); + } + + check(new ModPack()); + check(new ModPack({ name: "MyPack" })); + check(new ModPack({ description: "My Description" })); + check(new ModPack({ factorio_version: "2.0" })); + check(new ModPack({ mods: [ + { name: "subspace_storage", enabled: true, version: "1.99.8" }, + { name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" }, + ]})); + check(new ModPack({ settings: { + "startup": { + "bool-setting": { "value": true }, + }, + "runtime-global": { + "number-setting": { "value": 123 }, + }, + "runtime-per-user": { + "string-setting": { "value": "a string" }, + }, + }})); + check(new ModPack({ export_manifest: { assets: { setting: "settings.json" }}})); + check(new ModPack({ deleted: true })); + check(new ModPack({ + name: "Super pack", + description: "Every option at once.", + factorio_version: "2.0", + mods: [ + { name: "subspace_storage", enabled: true, version: "1.99.8" }, + { name: "clusterio_lib", enabled: true, version: "0.1.2", sha1: "012345abcd" }, + ], + settings: { + "startup": { + "bool-setting": { "value": true }, + }, + "runtime-global": { + "number-setting": { "value": 123 }, + }, + "runtime-per-user": { + "string-setting": { "value": "a string" }, + }, + }, + export_manifest: { assets: { setting: "settings.json" }}, + deleted: true, + })); + }); + + it("should sort integer factorio versions lexicographically", function() { + let unsortedVersions = ["1.0", "1.1.0", "0.1", "3.0.0", "1.2", "0.3.1", "0.3.3", "2.1.1", "0.0.1"]; + let sortedVersions = ["0.0.1", "0.1", "0.3.1", "0.3.3", "1.0", "1.1.0", "1.2", "2.1.1", "3.0.0"]; + let factorioMods = unsortedVersions.map(v => new ModPack({ factorio_version: v })); + factorioMods.sort((a, b) => a.integerFactorioVersion - b.integerFactorioVersion); + assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions); + }); + + describe(".fillDefaultSettings()", function() { + const prototypes = { + "bool-setting": { + "bool": { + name: "bool", + type: "bool-setting", + setting_type: "startup", + default_value: true, + }, + }, + "double-setting": { + "number": { + name: "number", + type: "double-setting", + setting_type: "runtime-global", + default_value: 123, + }, + }, + "string-setting": { + "string": { + name: "string", + type: "string-setting", + setting_type: "runtime-per-user", + default_value: "a string", + }, + }, + }; + const mockLogger = { warn: () => {} }; + it("should fill in defaults for settings", function() { + const pack = new ModPack(); + pack.fillDefaultSettings(prototypes, mockLogger); + assert.deepEqual(pack.toJSON().settings, { + "startup": { + "bool": { "value": true }, + }, + "runtime-global": { + "number": { "value": 123 }, + }, + "runtime-per-user": { + "string": { "value": "a string" }, + }, + }); + }); + it("should not overwrite existing values", function() { + const pack = new ModPack({ settings: { + "startup": { + "bool": { "value": false }, + }, + "runtime-global": { + "number": { "value": 2 }, + }, + "runtime-per-user": { + "string": { "value": "spam" }, + }, + }}); + pack.fillDefaultSettings(prototypes, mockLogger); + assert.deepEqual(pack.toJSON().settings, { + "startup": { + "bool": { "value": false }, + }, + "runtime-global": { + "number": { "value": 2 }, + }, + "runtime-per-user": { + "string": { "value": "spam" }, + }, + }); + }); + it("should ignore unknown setting_type and missing default_value", function() { + const pack = new ModPack(); + pack.fillDefaultSettings({ + "string-setting": { + "foo": { + name: "foo", + type: "string-setting", + setting_type: "magic-that-does-not-exist", + default_value: "a string", + }, + "bar": { + name: "bar", + type: "string-setting", + setting_type: "startup", + }, + }, + }, mockLogger); + assert.deepEqual(pack.toJSON().settings, { + "startup": {}, + "runtime-global": {}, + "runtime-per-user": {}, + }); + }); + }); + + describe(".fromModPackString()", function() { + it("should handle malformed strings", function() { + assert.throws( + () => ModPack.fromModPackString("AMalformedString"), + new Error("Malformed mod pack string: zlib inflate failed") + ); + let badJsonMsg; + try { + JSON.parse("Not Json"); + } catch (err) { + badJsonMsg = err.message; + } + assert.throws( + // eslint-disable-next-line node/no-sync + () => ModPack.fromModPackString(Buffer.from(zlib.deflateSync("Not Json")).toString("base64")), + new Error(`Malformed mod pack string: ${badJsonMsg}`) + ); + assert.throws( + // eslint-disable-next-line node/no-sync + () => ModPack.fromModPackString(Buffer.from(zlib.deflateSync('{"i":1}')).toString("base64")), + new Error("Malformed mod pack string: Schema validation failed") + ); + }); + }); + + describe("toModSettingsDat()", function() { + it("should properly serialise settings", function() { + const pack = new ModPack({ settings: { + "startup": { + "bool-setting": { "value": true }, + }, + "runtime-global": { + "number-setting": { "value": 123 }, + }, + "runtime-per-user": { + "string-setting": { "value": "a string" }, + }, + }}); + + function istr(str) { + return [ + Uint8Array.from([0, str.length]), + Buffer.from(str), + ]; + } + /* eslint-disable indent */ + assert.deepEqual( + pack.toModSettingsDat(), + Buffer.concat([ + new Uint8Array(Uint16Array.from([1, 1, 0, 0]).buffer), // version + Uint8Array.from([0]), // reserved + + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([3]).buffer), // items + + ...istr("startup"), // name + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("bool-setting"), // name + Uint8Array.from([5, 0]), // dictinary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("value"), // name + Uint8Array.from([1, 0, 1]), // boolean + + ...istr("runtime-global"), // name + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("number-setting"), // name + Uint8Array.from([5, 0]), // dictinary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("value"), // name + Uint8Array.from([2, 0]), // number + new Uint8Array(Float64Array.from([123]).buffer), + + ...istr("runtime-per-user"), // name + Uint8Array.from([5, 0]), // dictionary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("string-setting"), // name + Uint8Array.from([5, 0]), // dictinary + new Uint8Array(Uint32Array.from([1]).buffer), // items + + ...istr("value"), // name + Uint8Array.from([3, 0, 0, "a string".length]), // string + Buffer.from("a string"), + ]) + ); + /* eslint-enable indent */ + }); + }); + }); +}); diff --git a/test/lib/factorio/patch.js b/test/lib/factorio/patch.js index 00850590..e301d583 100644 --- a/test/lib/factorio/patch.js +++ b/test/lib/factorio/patch.js @@ -8,47 +8,6 @@ const libFactorio = require("@clusterio/lib/factorio"); describe("lib/factorio", function() { - describe("findRoot()", function() { - it("should find the root of a zip", function() { - let zip = new JSZip(); - zip.file("root/foo.txt", ""); - zip.file("root/bar.txt", ""); - - assert.equal(libFactorio._findRoot(zip), "root"); - }); - - it("should throw if files are at the root of the zip", function() { - let zip = new JSZip(); - zip.file("root/foo.txt", ""); - zip.file("bar.txt", ""); - - assert.throws( - () => libFactorio._findRoot(zip), - new Error("Zip contains file 'bar.txt' in root dir") - ); - }); - - it("should throw if there are multiple root dirs", function() { - let zip = new JSZip(); - zip.file("root-1/foo.txt", ""); - zip.file("root-2/bar.txt", ""); - - assert.throws( - () => libFactorio._findRoot(zip), - new Error("Zip contains multiple root folders") - ); - }); - - it("should throw if given an empty zip file", function() { - let zip = new JSZip(); - - assert.throws( - () => libFactorio._findRoot(zip), - new Error("Empty zip file") - ); - }); - }); - describe("generateLoader()", function() { let reference = [ "-- Auto generated scenario module loader created by Clusterio", diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index 21823231..da256995 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -84,4 +84,109 @@ describe("lib/file_ops", function() { assert.equal(await fs.readFile(target, "utf8"), "current"); }); }); + + describe("checkFilename()", function() { + it("should allow a basic name", function() { + libFileOps.checkFilename("file"); + }); + + function check(item, msg) { + assert.throws(() => libFileOps.checkFilename(item), new Error(msg)); + } + + it("should throw on non-string", function() { + check(undefined, "must be a string"); + check(null, "must be a string"); + check({}, "must be a string"); + check([], "must be a string"); + check(0, "must be a string"); + check(false, "must be a string"); + }); + + it("should throw on empty name", function() { + check("", "cannot be empty"); + }); + + it("should throw on <>:\"\\/|?* \\x00\\r\\n\\t", function() { + for (let char of '<>:"\\/|?*\x00\r\n\t') { + check(char, 'cannot contain <>:"\\/|=* or control characters'); + } + }); + + it("should throw on CON, PRN, AUX, NUL, COM1, LPT1", function() { + for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { + check(bad, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}.zip`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}.anything.txt`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}....a`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + check(`${bad}.`, "cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"); + } + }); + + it("should throw on . and ..", function() { + for (let bad of [".", ".."]) { + check(bad, `cannot be named ${bad}`); + } + }); + + it("should throw on names ending with . or space", function() { + check("a ", "cannot end with . or space"); + check("a.", "cannot end with . or space"); + }); + }); + + describe("cleanFilename()", function() { + function clean(item, expected) { + assert.equal(libFileOps.cleanFilename(item), expected); + } + + it("should allow a basic name", function() { + clean("file", "file"); + }); + + function check(item, msg) { + assert.throws(() => libFileOps.cleanFilename(item), new Error(msg)); + } + + it("should throw on non-string", function() { + check(undefined, "name must be a string"); + check(null, "name must be a string"); + check({}, "name must be a string"); + check([], "name must be a string"); + check(0, "name must be a string"); + check(false, "name must be a string"); + }); + + it("should clean empty name", function() { + clean("", "_"); + }); + + it("should clean <>:\"\\/|?* \\x00\\r\\n\\t", function() { + for (let char of '<>:"\\/|?*\x00\r\n\t') { + clean(char, "_"); + } + }); + + it("should clean CON, PRN, AUX, NUL, COM1, LPT1", function() { + for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { + clean(bad, `${bad}_`); + clean(`${bad}.zip`, `${bad}_.zip`); + clean(`${bad}.anything.txt`, `${bad}_.anything.txt`); + clean(`${bad}....a`, `${bad}_....a`); + clean(`${bad}.`, `${bad}__`); + } + }); + + it("should clean . and ..", function() { + for (let bad of [".", ".."]) { + clean(bad, `${bad}_`); + } + }); + + it("should clean names ending with . or space", function() { + clean("a ", "a_"); + clean("a.", "a_"); + }); + }); + }); diff --git a/test/lib/helpers.js b/test/lib/helpers.js index 5b5159f7..3d4b89e3 100644 --- a/test/lib/helpers.js +++ b/test/lib/helpers.js @@ -43,4 +43,113 @@ describe("lib/helpers", function() { ), "timeout"); }); }); + + function parse(input, attributes) { + const result = libHelpers.parseSearchString(input, attributes); + if (!result.issues.length) { + return result.terms; + } + return result; + } + + describe("parseSearchString()", function() { + function word(value, opts = {}) { + return { type: "word", value, ...opts }; + } + function attr(name, value) { + return { type: "attribute", name, value: typeof value === "string" ? word(value) : value }; + } + it("should split words", function() { + assert.deepEqual(parse(""), [].map(word)); + assert.deepEqual(parse("word"), ["word"].map(word)); + assert.deepEqual(parse("some words here"), ["some", "words", "here"].map(word)); + }); + it("should trim whitespace", function() { + assert.deepEqual(parse(" "), [].map(word)); + assert.deepEqual(parse("word "), ["word"].map(word)); + assert.deepEqual(parse(" some words here"), ["some", "words", "here"].map(word)); + }); + it("should parse negated words", function() { + assert.deepEqual(parse("this -that"), [word("this"), word("that", { exclude: true })]); + assert.deepEqual(parse("not-negated"), [word("not-negated")]); + }); + it("should not split quoted words", function() { + assert.deepEqual(parse('"some quoted words" here'), ["some quoted words", "here"].map(word)); + assert.deepEqual(parse('"a quoted string"'), ["a quoted string"].map(word)); + assert.deepEqual(parse('words and "a quoted string"'), ["words", "and", "a quoted string"].map(word)); + }); + it("should split on quotation mark", function() { + assert.deepEqual(parse('"quoted"unquoted'), ["quoted", "unquoted"].map(word)); + assert.deepEqual(parse('"quoted""double"'), ["quoted", "double"].map(word)); + assert.deepEqual(parse('word"with"quote'), ["word", "with", "quote"].map(word)); + assert.deepEqual( + parse('word-"with"-negation'), + [word("word-"), word("with"), word("negation", { exclude: true })] + ); + }); + it("should allow colons in quotes", function() { + assert.deepEqual(parse('"word:value"'), ["word:value"].map(word)); + }); + it("should parse attribute:value", function() { + assert.deepEqual(parse("author:Me", { author: "word" }), [attr("author", "Me")]); + assert.deepEqual(parse("word author:Me", { author: "word" }), [word("word"), attr("author", "Me")]); + assert.deepEqual( + parse("author:Me and author:You", { author: "word" }), + [attr("author", "Me"), word("and"), attr("author", "You")] + ); + assert.deepEqual( + parse("author:Me"), + { + terms: [word("Me")], + issues: ['Unregonized attribute "author", use quotes to escape colons'], + } + ); + assert.deepEqual( + parse("constructor:value"), + { + terms: [word("value")], + issues: ['Unregonized attribute "constructor", use quotes to escape colons'], + } + ); + }); + it("should throw on bad attribute format", function() { + assert.throws( + () => parse("author:Me", { author: "bad" }), + new Error("Bad attribute format bad for author") + ); + }); + it("should ignore attribute case", function() { + assert.deepEqual(parse("Author:Me", { author: "word" }), [attr("author", "Me")]); + assert.deepEqual(parse("AUTHOR:Me", { author: "word" }), [attr("author", "Me")]); + assert.deepEqual(parse("auThOR:Me", { author: "word" }), [attr("author", "Me")]); + }); + it("should parse partial inputs", function() { + assert.deepEqual(parse("-"), [word("", { exclude: true })]); + assert.deepEqual(parse('"'), [""].map(word)); + assert.deepEqual(parse('"word'), ["word"].map(word)); + assert.deepEqual(parse("author:", { author: "word" }), [attr("author", "")]); + assert.deepEqual(parse("author:-", { author: "word" }), [attr("author", word("", { exclude: true }))]); + }); + }); + describe("wordMatches()", function() { + const match = libHelpers.wordMatches; + it("should match any inputs", function() { + assert(match(parse("word")[0], "a word in a string"), "one text"); + assert(!match(parse("word")[0], "a string without"), "one text no match"); + assert(match(parse("word")[0], "no match", "no things", "a word", "bad text", ""), "five text"); + assert(!match(parse("unword")[0], "no match", "no things", "a word", "bad text", ""), "five text no match"); + }); + it("should invert match if using exclude", function() { + assert(!match(parse("-word")[0], "a word in a string"), "one text"); + assert(match(parse("-word")[0], "a string without"), "one text no match"); + assert(!match(parse("-word")[0], "no match", "no things", "a word", "bad text", ""), "five text"); + assert(match(parse("-unword")[0], "no match", "no things", "a word", "bad text", ""), "five text no match"); + }); + it("should throw on bad input", function() { + assert.throws( + () => match(parse("attr:value", { attr: "word" })[0], "a string"), + new Error("wordMatches: parameter is not a word"), + ); + }); + }); }); diff --git a/test/lib/plugin_loader.js b/test/lib/plugin_loader.js index b5098955..d1153c1b 100644 --- a/test/lib/plugin_loader.js +++ b/test/lib/plugin_loader.js @@ -45,9 +45,16 @@ describe("lib/plugin_loader", function() { ); }); it("should reject on broken plugin", async function() { + let brokenMessage; + try { + // eslint-disable-next-line node/global-require + require(path.resolve(brokenPlugin, "info.js")); + } catch (err) { + brokenMessage = err.message; + } await assert.rejects( libPluginLoader.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])), - { message: "PluginError: Unexpected identifier" } + { message: `PluginError: ${brokenMessage}` } ); }); it("should reject on invalid plugin", async function() { diff --git a/test/lib/zip_ops.js b/test/lib/zip_ops.js new file mode 100644 index 00000000..622c65ae --- /dev/null +++ b/test/lib/zip_ops.js @@ -0,0 +1,49 @@ +"use strict"; +const assert = require("assert").strict; +const JSZip = require("jszip"); + +const { findRoot } = require("@clusterio/lib/zip_ops"); + + +describe("lib/factorio", function() { + describe("findRoot()", function() { + it("should find the root of a zip", function() { + let zip = new JSZip(); + zip.file("root/foo.txt", ""); + zip.file("root/bar.txt", ""); + + assert.equal(findRoot(zip), "root"); + }); + + it("should throw if files are at the root of the zip", function() { + let zip = new JSZip(); + zip.file("root/foo.txt", ""); + zip.file("bar.txt", ""); + + assert.throws( + () => findRoot(zip), + new Error("Zip contains file 'bar.txt' in root dir") + ); + }); + + it("should throw if there are multiple root dirs", function() { + let zip = new JSZip(); + zip.file("root-1/foo.txt", ""); + zip.file("root-2/bar.txt", ""); + + assert.throws( + () => findRoot(zip), + new Error("Zip contains multiple root folders") + ); + }); + + it("should throw if given an empty zip file", function() { + let zip = new JSZip(); + + assert.throws( + () => findRoot(zip), + new Error("Empty zip file") + ); + }); + }); +}); diff --git a/test/slave.js b/test/slave.js index 36772faa..d8d34788 100644 --- a/test/slave.js +++ b/test/slave.js @@ -1,11 +1,8 @@ "use strict"; const assert = require("assert").strict; const path = require("path"); -const fs = require("fs-extra"); -const libLink = require("@clusterio/lib/link"); const libConfig = require("@clusterio/lib/config"); -const Instance = require("@clusterio/slave/src/Instance"); const Slave = require("@clusterio/slave/src/Slave"); describe("Slave testing", function() { @@ -13,166 +10,6 @@ describe("Slave testing", function() { libConfig.InstanceConfig.finalize(); }); - describe("checkFilename()", function() { - it("should allow a basic name", function() { - Slave._checkFilename("file"); - }); - - function check(item, msg) { - assert.throws(() => Slave._checkFilename(item), new Error(msg)); - } - - it("should throw on non-string", function() { - check(undefined, "must be a string"); - check(null, "must be a string"); - check({}, "must be a string"); - check([], "must be a string"); - check(0, "must be a string"); - check(false, "must be a string"); - }); - - it("should throw on empty name", function() { - check("", "cannot be empty"); - }); - - it("should throw on <>:\"\\/|?* \\x00\\r\\n\\t", function() { - for (let char of '<>:"\\/|?*\x00\r\n\t') { - check(char, 'cannot contain <>:"\\/|=* or control characters'); - } - }); - - it("should throw on CON, PRN, AUX, NUL, COM1, LPT1", function() { - for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { - check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); - } - }); - - it("should throw on . and ..", function() { - for (let bad of [".", ".."]) { - check(bad, "cannot be named any of . .. CON PRN AUX NUL COM1-9 and LPT1-9"); - } - }); - - it("should throw on names ending with . or space", function() { - check("a ", "cannot end with . or space"); - check("a.", "cannot end with . or space"); - }); - }); - - describe("cleanFilename()", function() { - function clean(item, expected) { - assert.equal(Slave._cleanFilename(item), expected); - } - - it("should allow a basic name", function() { - clean("file", "file"); - }); - - function check(item, msg) { - assert.throws(() => Slave._cleanFilename(item), new Error(msg)); - } - - it("should throw on non-string", function() { - check(undefined, "name must be a string"); - check(null, "name must be a string"); - check({}, "name must be a string"); - check([], "name must be a string"); - check(0, "name must be a string"); - check(false, "name must be a string"); - }); - - it("should clean empty name", function() { - clean("", "_"); - }); - - it("should clean <>:\"\\/|?* \\x00\\r\\n\\t", function() { - for (let char of '<>:"\\/|?*\x00\r\n\t') { - clean(char, "_"); - } - }); - - it("should clean CON, PRN, AUX, NUL, COM1, LPT1", function() { - for (let bad of ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1"]) { - clean(bad, `${bad}_`); - } - }); - - it("should clean . and ..", function() { - for (let bad of [".", ".."]) { - clean(bad, `${bad}_`); - } - }); - - it("should clean names ending with . or space", function() { - clean("a ", "a_"); - clean("a.", "a_"); - }); - }); - - describe("symlinkMods()", function() { - let testDir = path.join("temp", "test", "symlink"); - - let instance; - before(async function() { - // Remove previous test output - await fs.remove(testDir); - - // Add some test mods - await fs.outputFile(path.join(testDir, "shared", "mod_a.zip"), "a"); - await fs.outputFile(path.join(testDir, "shared", "mod_b.zip"), "b"); - await fs.outputFile(path.join(testDir, "shared", "mod.dat"), "c"); - await fs.outputFile(path.join(testDir, "shared", "mod-list.json"), "d"); - - let instanceConfig = new libConfig.InstanceConfig("slave"); - await instanceConfig.init(); - instanceConfig.set("instance.name", "test"); - instance = new Instance( - {}, new libLink.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig - ); - await fs.outputFile(instance.path("mods", "mod_i.zip"), "i"); - }); - - it("should link mods and data files", async function() { - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert.equal(await fs.readFile(instance.path("mods", "mod_a.zip"), "utf-8"), "a"); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - - it("should ignore directories", async function() { - await fs.ensureDir(path.join(testDir, "shared", "dir")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "dir"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should ignore files", async function() { - await fs.outputFile(path.join(testDir, "shared", "file"), "f"); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - assert(!await fs.exists(instance.path("mods", "file"), "utf-8"), "dir was unxpectedly linked"); - }); - - it("should unlink removed mods", async function() { - // This does not work on Windows - if (process.platform === "win32") { - this.skip(); - } - - await fs.unlink(path.join(testDir, "shared", "mod_a.zip")); - await Instance._symlinkMods(instance, path.join(testDir, "shared")); - - await assert.rejects(fs.lstat(instance.path("mods", "mod_a.zip")), { code: "ENOENT" }); - assert.equal(await fs.readFile(instance.path("mods", "mod_b.zip"), "utf-8"), "b"); - assert.equal(await fs.readFile(instance.path("mods", "mod.dat"), "utf-8"), "c"); - assert.equal(await fs.readFile(instance.path("mods", "mod-list.json"), "utf-8"), "d"); - assert.equal(await fs.readFile(instance.path("mods", "mod_i.zip"), "utf-8"), "i"); - }); - }); - describe("discoverInstances()", function() { it("should discover test instance", async function() { let instancePath = path.join("test", "file", "instances");