From 13e12f733592fdb5111001e481e0e6f60d61c6ea Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 31 Jan 2023 04:18:01 +0100 Subject: [PATCH] Tie data exports to mod packs Store the export manifest in ModPack to allow mulitple mod packs to have different data exported. This is necessary as the export contains the settings prototypes needed to create a proper UI for mod settings. For now the useLocale and useItemMetadata interfaces uses the manifest for a default mod pack set in the master config. --- packages/ctl/src/commands.js | 6 +++ packages/lib/config/definitions.js | 7 +++ packages/lib/data/ExportManifest.js | 46 +++++++++++++++++++ packages/lib/data/ModPack.js | 11 +++++ packages/lib/data/index.js | 2 + packages/lib/link/messages.js | 10 ++++ packages/master/src/BaseConnection.js | 12 +++++ packages/master/src/Master.js | 9 ---- packages/master/src/routes.js | 29 ++++++++---- packages/slave/src/Instance.js | 1 + packages/web_ui/src/model/export_manifest.jsx | 13 ++++-- packages/web_ui/src/model/item_metadata.jsx | 10 ++-- packages/web_ui/src/model/locale.jsx | 4 +- test/integration/clusterio.js | 13 +++--- test/integration/index.js | 3 +- test/lib/data/ModPack.js | 4 ++ 16 files changed, 146 insertions(+), 34 deletions(-) create mode 100644 packages/lib/data/ExportManifest.js diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index a99e9eb7..4f1c03ce 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -893,6 +893,12 @@ modPackCommands.add(new libCommand.Command({ print(` ${setting}: ${JSON.stringify(settingValue.value)}`); } } + } else if (field === "export_manifest") { + print(`${field}:`); + print(" assets:"); + for (let [name, fileName] of Object.entries(value.assets)) { + print(` ${name}: ${fileName}`); + } } else { print(`${field}: ${value}`); diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index b0926270..d6605c85 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -115,6 +115,13 @@ MasterGroup.define({ type: "number", initial_value: 15, }); +MasterGroup.define({ + name: "default_mod_pack_id", + title: "Default Mod Pack", + description: "Mod pack used by default for instances.", + type: "number", + optional: true, +}); MasterGroup.define({ name: "default_role_id", title: "Default role", diff --git a/packages/lib/data/ExportManifest.js b/packages/lib/data/ExportManifest.js new file mode 100644 index 00000000..a07d663e --- /dev/null +++ b/packages/lib/data/ExportManifest.js @@ -0,0 +1,46 @@ +"use strict"; + +/** + * Export manifest for a mod pack + * + * Tracks files stored on disk for data that extracted from Factorio using a + * given mod pack. + * @alias module:lib/data.ExportManifest + */ +class ExportManifest { + + /** + * Mapping between known assets and their file names. + * + * Currently exported assets are + * - settings: JSON of mod settings prototypes. + * - prototypes: JSON of all game prototypes except settings. + * - item-spritesheet: PNG containing icons from the game. + * - item-metadata: JSON of Map entries describing icons in item-spritesheet + * - locale: JSON of flattened Map entries of en locale strings from the game. + * @type {Object} + */ + assets = {}; + + static jsonSchema = { + type: "object", + additionalProperties: false, + required: ["assets"], + properties: { + "assets": { + type: "object", + additionalProperties: { type: "string" }, + }, + }, + }; + + constructor(json = {}) { + if (json.assets) { this.assets = json.assets; } + } + + toJSON() { + return { assets: this.assets }; + } +} + +module.exports = ExportManifest; diff --git a/packages/lib/data/ModPack.js b/packages/lib/data/ModPack.js index 0b58a0f2..286dcea7 100644 --- a/packages/lib/data/ModPack.js +++ b/packages/lib/data/ModPack.js @@ -3,6 +3,7 @@ const zlib = require("zlib"); const libSchema = require("../schema"); +const ExportManifest = require("./ExportManifest"); const { integerFactorioVersion } = require("./version"); @@ -99,6 +100,12 @@ class ModPack { "runtime-per-user": new Map(), }; + /** + * Mapping to files containing exported data for this modpack + * @type {module:lib/data.ExportManifest|undefined} + */ + exportManifest; + /** * True if this mod pack has been deleted from the list of mod packs. * @type {boolean} @@ -128,6 +135,7 @@ class ModPack { "runtime-per-user": ModSettingsJsonSchema, }, }, + "export_manifest": ExportManifest.jsonSchema, "is_deleted": { type: "boolean" }, }, }; @@ -151,6 +159,7 @@ class ModPack { "runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])), }; } + if (json.export_manifest) { this.exportManifest = new ExportManifest(json.export_manifest); } if (json.is_deleted) { this.isDeleted = json.is_deleted; } if (!this.mods.has("base")) { @@ -171,6 +180,7 @@ class ModPack { "runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]), }, }; + if (this.exportManifest) { json.export_manifest = this.exportManifest.toJSON(); } if (this.isDeleted) { json.is_deleted = this.isDeleted; } return json; } @@ -178,6 +188,7 @@ class ModPack { toModPackString() { const json = this.toJSON(); delete json.id; + delete json.export_manifest; // eslint-disable-next-line node/no-sync let buf = zlib.deflateSync(JSON.stringify(json)); diff --git a/packages/lib/data/index.js b/packages/lib/data/index.js index cec20aed..f3856763 100644 --- a/packages/lib/data/index.js +++ b/packages/lib/data/index.js @@ -4,11 +4,13 @@ * @author Hornwitser */ "use strict"; +const ExportManifest = require("./ExportManifest"); const ModInfo = require("./ModInfo"); const ModPack = require("./ModPack"); const version = require("./version"); module.exports = { + ExportManifest, ModInfo, ModPack, ...version, diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index 4404a25d..3da48aec 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -723,6 +723,16 @@ messages.getModPack = new Request({ }, }); +messages.getDefaultModPack = new Request({ + type: "get_default_mod_pack", + links: ["control-master", "instance-slave", "slave-master"], + permission: "core.mod_pack.get", + forwardTo: "master", + responseProperties: { + "mod_pack": libData.ModPack.jsonSchema, + }, +}); + messages.listModPacks = new Request({ type: "list_mod_packs", links: ["control-master"], diff --git a/packages/master/src/BaseConnection.js b/packages/master/src/BaseConnection.js index f22094cf..92e25c1d 100644 --- a/packages/master/src/BaseConnection.js +++ b/packages/master/src/BaseConnection.js @@ -93,6 +93,18 @@ class BaseConnection extends libLink.Link { } return { mod_pack: modPack.toJSON() }; } + + async getDefaultModPackRequestHandler(message) { + let id = this._master.config.get("master.default_mod_pack_id"); + if (id === null) { + throw new libErrors.RequestError("Default mod pack not set on master"); + } + let modPack = this._master.modPacks.get(id); + if (!modPack) { + throw new libErrors.RequestError(`Default mod pack configured (${id}) does not exist`); + } + return { mod_pack: modPack.toJSON() }; + } } module.exports = BaseConnection; diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index edb1288c..2bb3ccb7 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -177,7 +177,6 @@ class Master { this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json")); this.userManager = new UserManager(this.config); await this.userManager.load(path.join(databaseDirectory, "users.json")); - this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json")); let modsDirectory = this.config.get("master.mods_directory"); await fs.ensureDir(modsDirectory); @@ -445,14 +444,6 @@ class Master { return manifest; } - async updateExportManifest(newManifest) { - let databaseDirectory = this.config.get("master.database_directory"); - await libFileOps.safeOutputFile( - path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4) - ); - this.exportManifest = newManifest; - } - async deleteMod(name, version) { let filename = `${name}_${version}.zip`; let mod = this.mods.get(filename); diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 2ed9cd25..4f14fda3 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -187,6 +187,21 @@ async function uploadExport(req, res) { return; } + if (typeof req.query.mod_pack_id !== "string") { + res.sendStatus(400); + return; + } + let modPackId = Number.parseInt(req.query.mod_pack_id, 10); + if (!Number.isInteger(modPackId)) { + res.sendStatus(400); + return; + } + let modPack = res.app.locals.master.modPacks.get(modPackId); + if (!modPack) { + res.sendStatus(400); + return; + } + let data = []; for await (let chunk of req) { data.push(chunk); @@ -204,7 +219,7 @@ async function uploadExport(req, res) { "export/locale.json", ]; - let manifest = {}; + let assets = {}; for (let filePath of exportFiles) { let file = zip.file(filePath); if (!file) { @@ -213,19 +228,16 @@ async function uploadExport(req, res) { let { name, ext } = path.posix.parse(filePath); let hash = await libHash.hashStream(file.nodeStream()); - manifest[name] = `static/${name}.${hash}${ext}`; + assets[name] = `${name}.${hash}${ext}`; await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer")); } - await res.app.locals.master.updateExportManifest(manifest); + + modPack.exportManifest = new libData.ExportManifest({ assets }); + res.app.locals.master.modPackUpdated(modPack); res.sendStatus(200); } -async function getExportManifest(req, res) { - endpointHitCounter.labels(req.route.path).inc(); - res.json(res.app.locals.master.exportManifest); -} - async function createProxyStream(app) { let asyncRandomBytes = util.promisify(crypto.randomBytes); let id = (await asyncRandomBytes(8)).toString("hex"); @@ -601,7 +613,6 @@ function addRouteHandlers(app) { validateSlaveToken, (req, res, next) => uploadExport(req, res).catch(next) ); - app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next)); app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next)); app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next)); app.post("/api/upload-save", diff --git a/packages/slave/src/Instance.js b/packages/slave/src/Instance.js index 2148206e..a39287ef 100644 --- a/packages/slave/src/Instance.js +++ b/packages/slave/src/Instance.js @@ -1064,6 +1064,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " "); let content = await zip.generateAsync({ type: "nodebuffer" }); let url = new URL(this._slave.config.get("slave.master_url")); url.pathname += "api/upload-export"; + url.searchParams.set("mod_pack_id", this.config.get("factorio.mod_pack")); let response = await phin({ url, method: "PUT", data: content, diff --git a/packages/web_ui/src/model/export_manifest.jsx b/packages/web_ui/src/model/export_manifest.jsx index ebb436d9..50ef1467 100644 --- a/packages/web_ui/src/model/export_manifest.jsx +++ b/packages/web_ui/src/model/export_manifest.jsx @@ -1,15 +1,20 @@ -import { useEffect, useState } from "react"; +import { useEffect, useContext, useState } from "react"; + +import { libData, libLink } from "@clusterio/lib"; +import ControlContext from "../components/ControlContext"; let exportManifestCache = null; let emptyCache = {}; export function useExportManifest() { + const control = useContext(ControlContext); let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache); useEffect(() => { async function load() { - let response = await fetch(`${webRoot}api/export-manifest`); - if (response.ok) { - exportManifestCache = await response.json(); + let result = await libLink.messages.getDefaultModPack.send(control); + let modPack = new libData.ModPack(result.mod_pack); + if (modPack.exportManifest) { + exportManifestCache = modPack.exportManifest; setExportManifest(exportManifestCache); } } diff --git a/packages/web_ui/src/model/item_metadata.jsx b/packages/web_ui/src/model/item_metadata.jsx index 3858e296..8c1ae078 100644 --- a/packages/web_ui/src/model/item_metadata.jsx +++ b/packages/web_ui/src/model/item_metadata.jsx @@ -9,10 +9,14 @@ export function useItemMetadata() { let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map()); useEffect(() => { async function load() { - if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) { + if ( + !exportManifest + || !exportManifest.assets["item-metadata"] + || !exportManifest.assets["item-spritesheet"] + ) { return; } - let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`); + let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`); if (response.ok) { let data = await response.json(); itemMetadataCache = new Map(data); @@ -22,7 +26,7 @@ export function useItemMetadata() { for (let [name, meta] of itemMetadataCache) { style.sheet.insertRule( `.item-${CSS.escape(name)} { - background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}"); + background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}"); background-repeat: no-repeat; background-position: -${meta.x}px -${meta.y}px; height: ${meta.size}px; diff --git a/packages/web_ui/src/model/locale.jsx b/packages/web_ui/src/model/locale.jsx index 3a22e6d7..431833fc 100644 --- a/packages/web_ui/src/model/locale.jsx +++ b/packages/web_ui/src/model/locale.jsx @@ -9,10 +9,10 @@ export function useLocale() { let [locale, setLocale] = useState(localeCache || new Map()); useEffect(() => { async function load() { - if (!exportManifest["locale"]) { + if (!exportManifest || !exportManifest.assets["locale"]) { return; } - let response = await fetch(`${staticRoot}${exportManifest["locale"]}`); + let response = await fetch(`${staticRoot}static/${exportManifest.assets["locale"]}`); if (response.ok) { let data = await response.json(); localeCache = new Map(data); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 41bd159d..272d5064 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -307,17 +307,18 @@ describe("Integration of Clusterio", function() { let exportPath = path.join("temp", "test", "static"); await fs.remove(exportPath); await execCtl("instance export-data test"); - let response = await get("/api/export-manifest"); - let manifest = JSON.parse(response.body); - assert(Object.keys(manifest).length > 1, "Export manifest is empty"); + let result = await libLink.messages.getDefaultModPack.send(getControl()); + let modPack = new libData.ModPack(result.mod_pack); + let assets = modPack.exportManifest.assets; + assert(Object.keys(assets).length > 1, "Export assets is empty"); for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) { - assert(manifest[key], `Missing ${key} from manifest`); + assert(assets[key], `Missing ${key} from assets`); assert( - await fs.exists(path.join(exportPath, "..", manifest[key])), + await fs.exists(path.join(exportPath, assets[key])), `Manifest entry for ${key} was not written to filesystem` ); } - let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, "..", manifest["prototypes"]))); + let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, assets["prototypes"]))); assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types"); await checkInstanceStatus(44, "stopped"); }); diff --git a/test/integration/index.js b/test/integration/index.js index b604bb1f..54cf1c29 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -228,7 +228,8 @@ before(async function() { testPack.factorioVersion = "1.1.0"; testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); - libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); + await libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); + await libLink.messages.setMasterConfigField.send(control, { field: "master.default_mod_pack_id", value: "12" }); }); after(async function() { diff --git a/test/lib/data/ModPack.js b/test/lib/data/ModPack.js index 8a6fed3e..a5c105e5 100644 --- a/test/lib/data/ModPack.js +++ b/test/lib/data/ModPack.js @@ -3,6 +3,7 @@ const assert = require("assert").strict; const zlib = require("zlib"); const libSchema = require("@clusterio/lib/schema"); +const ExportManifest = require("@clusterio/lib/data/ExportManifest"); const ModPack = require("@clusterio/lib/data/ModPack"); @@ -18,6 +19,7 @@ describe("lib/data/ModPack", function() { assert.deepEqual(new ModPack(json), pack); const packStringed = ModPack.fromModPackString(pack.toModPackString()); packStringed.id = pack.id; + packStringed.exportManifest = pack.exportManifest; assert.deepEqual(packStringed, pack); } @@ -40,6 +42,7 @@ describe("lib/data/ModPack", function() { "string-setting": { "value": "a string" }, }, }})); + check(new ModPack({ export_manifest: { assets: { setting: "settings.json" }}})); check(new ModPack({ deleted: true })); check(new ModPack({ name: "Super pack", @@ -60,6 +63,7 @@ describe("lib/data/ModPack", function() { "string-setting": { "value": "a string" }, }, }, + export_manifest: { assets: { setting: "settings.json" }}, deleted: true, })); });