Tie data exports to mod packs

Store the export manifest in ModPack to allow mulitple mod packs to have
different data exported. This is necessary as the export contains the
settings prototypes needed to create a proper UI for mod settings.

For now the useLocale and useItemMetadata interfaces uses the manifest
for a default mod pack set in the master config.

packages/ctl/src/commands.js

@@ -893,6 +893,12 @@ modPackCommands.add(new libCommand.Command({
 						print(`    ${setting}: ${JSON.stringify(settingValue.value)}`);
 					}
 				}
+			} else if (field === "export_manifest") {
+				print(`${field}:`);
+				print("  assets:");
+				for (let [name, fileName] of Object.entries(value.assets)) {
+					print(`    ${name}: ${fileName}`);
+				}
 
 			} else {
 				print(`${field}: ${value}`);

packages/lib/config/definitions.js

@@ -115,6 +115,13 @@ MasterGroup.define({
 	type: "number",
 	initial_value: 15,
 });
+MasterGroup.define({
+	name: "default_mod_pack_id",
+	title: "Default Mod Pack",
+	description: "Mod pack used by default for instances.",
+	type: "number",
+	optional: true,
+});
 MasterGroup.define({
 	name: "default_role_id",
 	title: "Default role",

packages/lib/data/ExportManifest.js

@@ -0,0 +1,46 @@
+"use strict";
+
+/**
+ * Export manifest for a mod pack
+ *
+ * Tracks files stored on disk for data that extracted from Factorio using a
+ * given mod pack.
+ * @alias module:lib/data.ExportManifest
+ */
+class ExportManifest {
+
+	/**
+	 * Mapping between known assets and their file names.
+	 *
+	 * Currently exported assets are
+	 * - settings: JSON of mod settings prototypes.
+	 * - prototypes: JSON of all game prototypes except settings.
+	 * - item-spritesheet: PNG containing icons from the game.
+	 * - item-metadata: JSON of Map entries describing icons in item-spritesheet
+	 * - locale: JSON of flattened Map entries of en locale strings from the game.
+	 * @type {Object<string, string>}
+	 */
+	assets = {};
+
+	static jsonSchema = {
+		type: "object",
+		additionalProperties: false,
+		required: ["assets"],
+		properties: {
+			"assets": {
+				type: "object",
+				additionalProperties: { type: "string" },
+			},
+		},
+	};
+
+	constructor(json = {}) {
+		if (json.assets) { this.assets = json.assets; }
+	}
+
+	toJSON() {
+		return { assets: this.assets };
+	}
+}
+
+module.exports = ExportManifest;

packages/lib/data/ModPack.js

@@ -3,6 +3,7 @@ const zlib = require("zlib");
 
 const libSchema = require("../schema");
 
+const ExportManifest = require("./ExportManifest");
 const { integerFactorioVersion } = require("./version");
 
 
@@ -99,6 +100,12 @@ class ModPack {
 		"runtime-per-user": new Map(),
 	};
 
+	/**
+	 * Mapping to files containing exported data for this modpack
+	 * @type {module:lib/data.ExportManifest|undefined}
+	 */
+	exportManifest;
+
 	/**
 	 * True if this mod pack has been deleted from the list of mod packs.
 	 * @type {boolean}
@@ -128,6 +135,7 @@ class ModPack {
 					"runtime-per-user": ModSettingsJsonSchema,
 				},
 			},
+			"export_manifest": ExportManifest.jsonSchema,
 			"is_deleted": { type: "boolean" },
 		},
 	};
@@ -151,6 +159,7 @@ class ModPack {
 				"runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])),
 			};
 		}
+		if (json.export_manifest) { this.exportManifest = new ExportManifest(json.export_manifest); }
 		if (json.is_deleted) { this.isDeleted = json.is_deleted; }
 
 		if (!this.mods.has("base")) {
@@ -171,6 +180,7 @@ class ModPack {
 				"runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]),
 			},
 		};
+		if (this.exportManifest) { json.export_manifest = this.exportManifest.toJSON(); }
 		if (this.isDeleted) { json.is_deleted = this.isDeleted; }
 		return json;
 	}
@@ -178,6 +188,7 @@ class ModPack {
 	toModPackString() {
 		const json = this.toJSON();
 		delete json.id;
+		delete json.export_manifest;
 
 		// eslint-disable-next-line node/no-sync
 		let buf = zlib.deflateSync(JSON.stringify(json));

packages/lib/data/index.js

@@ -4,11 +4,13 @@
  * @author Hornwitser
  */
 "use strict";
+const ExportManifest = require("./ExportManifest");
 const ModInfo = require("./ModInfo");
 const ModPack = require("./ModPack");
 const version = require("./version");
 
 module.exports = {
+	ExportManifest,
 	ModInfo,
 	ModPack,
 	...version,

packages/lib/link/messages.js

@@ -723,6 +723,16 @@ messages.getModPack = new Request({
 	},
 });
 
+messages.getDefaultModPack = new Request({
+	type: "get_default_mod_pack",
+	links: ["control-master", "instance-slave", "slave-master"],
+	permission: "core.mod_pack.get",
+	forwardTo: "master",
+	responseProperties: {
+		"mod_pack": libData.ModPack.jsonSchema,
+	},
+});
+
 messages.listModPacks = new Request({
 	type: "list_mod_packs",
 	links: ["control-master"],

packages/master/src/BaseConnection.js

@@ -93,6 +93,18 @@ class BaseConnection extends libLink.Link {
 		}
 		return { mod_pack: modPack.toJSON() };
 	}
+
+	async getDefaultModPackRequestHandler(message) {
+		let id = this._master.config.get("master.default_mod_pack_id");
+		if (id === null) {
+			throw new libErrors.RequestError("Default mod pack not set on master");
+		}
+		let modPack = this._master.modPacks.get(id);
+		if (!modPack) {
+			throw new libErrors.RequestError(`Default mod pack configured (${id}) does not exist`);
+		}
+		return { mod_pack: modPack.toJSON() };
+	}
 }
 
 module.exports = BaseConnection;

packages/master/src/Master.js

@@ -177,7 +177,6 @@ class Master {
 		this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json"));
 		this.userManager = new UserManager(this.config);
 		await this.userManager.load(path.join(databaseDirectory, "users.json"));
-		this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json"));
 
 		let modsDirectory = this.config.get("master.mods_directory");
 		await fs.ensureDir(modsDirectory);
@@ -445,14 +444,6 @@ class Master {
 		return manifest;
 	}
 
-	async updateExportManifest(newManifest) {
-		let databaseDirectory = this.config.get("master.database_directory");
-		await libFileOps.safeOutputFile(
-			path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4)
-		);
-		this.exportManifest = newManifest;
-	}
-
 	async deleteMod(name, version) {
 		let filename = `${name}_${version}.zip`;
 		let mod = this.mods.get(filename);

packages/master/src/routes.js

@@ -187,6 +187,21 @@ async function uploadExport(req, res) {
 		return;
 	}
 
+	if (typeof req.query.mod_pack_id !== "string") {
+		res.sendStatus(400);
+		return;
+	}
+	let modPackId = Number.parseInt(req.query.mod_pack_id, 10);
+	if (!Number.isInteger(modPackId)) {
+		res.sendStatus(400);
+		return;
+	}
+	let modPack = res.app.locals.master.modPacks.get(modPackId);
+	if (!modPack) {
+		res.sendStatus(400);
+		return;
+	}
+
 	let data = [];
 	for await (let chunk of req) {
 		data.push(chunk);
@@ -204,7 +219,7 @@ async function uploadExport(req, res) {
 		"export/locale.json",
 	];
 
-	let manifest = {};
+	let assets = {};
 	for (let filePath of exportFiles) {
 		let file = zip.file(filePath);
 		if (!file) {
@@ -213,19 +228,16 @@ async function uploadExport(req, res) {
 
 		let { name, ext } = path.posix.parse(filePath);
 		let hash = await libHash.hashStream(file.nodeStream());
-		manifest[name] = `static/${name}.${hash}${ext}`;
+		assets[name] = `${name}.${hash}${ext}`;
 		await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
 	}
-	await res.app.locals.master.updateExportManifest(manifest);
+
+	modPack.exportManifest = new libData.ExportManifest({ assets });
+	res.app.locals.master.modPackUpdated(modPack);
 
 	res.sendStatus(200);
 }
 
-async function getExportManifest(req, res) {
-	endpointHitCounter.labels(req.route.path).inc();
-	res.json(res.app.locals.master.exportManifest);
-}
-
 async function createProxyStream(app) {
 	let asyncRandomBytes = util.promisify(crypto.randomBytes);
 	let id = (await asyncRandomBytes(8)).toString("hex");
@@ -601,7 +613,6 @@ function addRouteHandlers(app) {
 		validateSlaveToken,
 		(req, res, next) => uploadExport(req, res).catch(next)
 	);
-	app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next));
 	app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next));
 	app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next));
 	app.post("/api/upload-save",

packages/slave/src/Instance.js

@@ -1064,6 +1064,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
 			let content = await zip.generateAsync({ type: "nodebuffer" });
 			let url = new URL(this._slave.config.get("slave.master_url"));
 			url.pathname += "api/upload-export";
+			url.searchParams.set("mod_pack_id", this.config.get("factorio.mod_pack"));
 			let response = await phin({
 				url, method: "PUT",
 				data: content,

packages/web_ui/src/model/export_manifest.jsx

@@ -1,15 +1,20 @@
-import { useEffect, useState } from "react";
+import { useEffect, useContext, useState } from "react";
+
+import { libData, libLink } from "@clusterio/lib";
+import ControlContext from "../components/ControlContext";
 
 
 let exportManifestCache = null;
 let emptyCache = {};
 export function useExportManifest() {
+	const control = useContext(ControlContext);
 	let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache);
 	useEffect(() => {
 		async function load() {
-			let response = await fetch(`${webRoot}api/export-manifest`);
-			if (response.ok) {
-				exportManifestCache = await response.json();
+			let result = await libLink.messages.getDefaultModPack.send(control);
+			let modPack = new libData.ModPack(result.mod_pack);
+			if (modPack.exportManifest) {
+				exportManifestCache = modPack.exportManifest;
 				setExportManifest(exportManifestCache);
 			}
 		}

packages/web_ui/src/model/item_metadata.jsx

@@ -9,10 +9,14 @@ export function useItemMetadata() {
 	let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map());
 	useEffect(() => {
 		async function load() {
-			if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) {
+			if (
+				!exportManifest
+				|| !exportManifest.assets["item-metadata"]
+				|| !exportManifest.assets["item-spritesheet"]
+			) {
 				return;
 			}
-			let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`);
+			let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				itemMetadataCache = new Map(data);
@@ -22,7 +26,7 @@ export function useItemMetadata() {
 				for (let [name, meta] of itemMetadataCache) {
 					style.sheet.insertRule(
 						`.item-${CSS.escape(name)} {
-	background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}");
+	background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}");
 	background-repeat: no-repeat;
 	background-position: -${meta.x}px -${meta.y}px;
 	height: ${meta.size}px;

packages/web_ui/src/model/locale.jsx

@@ -9,10 +9,10 @@ export function useLocale() {
 	let [locale, setLocale] = useState(localeCache || new Map());
 	useEffect(() => {
 		async function load() {
-			if (!exportManifest["locale"]) {
+			if (!exportManifest || !exportManifest.assets["locale"]) {
 				return;
 			}
-			let response = await fetch(`${staticRoot}${exportManifest["locale"]}`);
+			let response = await fetch(`${staticRoot}static/${exportManifest.assets["locale"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				localeCache = new Map(data);

test/integration/clusterio.js

@@ -307,17 +307,18 @@ describe("Integration of Clusterio", function() {
 				let exportPath = path.join("temp", "test", "static");
 				await fs.remove(exportPath);
 				await execCtl("instance export-data test");
-				let response = await get("/api/export-manifest");
-				let manifest = JSON.parse(response.body);
-				assert(Object.keys(manifest).length > 1, "Export manifest is empty");
+				let result = await libLink.messages.getDefaultModPack.send(getControl());
+				let modPack = new libData.ModPack(result.mod_pack);
+				let assets = modPack.exportManifest.assets;
+				assert(Object.keys(assets).length > 1, "Export assets is empty");
 				for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) {
-					assert(manifest[key], `Missing ${key} from manifest`);
+					assert(assets[key], `Missing ${key} from assets`);
 					assert(
-						await fs.exists(path.join(exportPath, "..", manifest[key])),
+						await fs.exists(path.join(exportPath, assets[key])),
 						`Manifest entry for ${key} was not written to filesystem`
 					);
 				}
-				let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, "..", manifest["prototypes"])));
+				let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, assets["prototypes"])));
 				assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types");
 				await checkInstanceStatus(44, "stopped");
 			});

test/integration/index.js

@@ -228,7 +228,8 @@ before(async function() {
 	testPack.factorioVersion = "1.1.0";
 	testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" });
 	testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" });
-	libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() });
+	await libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() });
+	await libLink.messages.setMasterConfigField.send(control, { field: "master.default_mod_pack_id", value: "12" });
 });
 
 after(async function() {

test/lib/data/ModPack.js

@@ -3,6 +3,7 @@ const assert = require("assert").strict;
 const zlib = require("zlib");
 
 const libSchema = require("@clusterio/lib/schema");
+const ExportManifest = require("@clusterio/lib/data/ExportManifest");
 const ModPack = require("@clusterio/lib/data/ModPack");
 
 
@@ -18,6 +19,7 @@ describe("lib/data/ModPack", function() {
 				assert.deepEqual(new ModPack(json), pack);
 				const packStringed = ModPack.fromModPackString(pack.toModPackString());
 				packStringed.id = pack.id;
+				packStringed.exportManifest = pack.exportManifest;
 				assert.deepEqual(packStringed, pack);
 			}
 
@@ -40,6 +42,7 @@ describe("lib/data/ModPack", function() {
 					"string-setting": { "value": "a string" },
 				},
 			}}));
+			check(new ModPack({ export_manifest: { assets: { setting: "settings.json" }}}));
 			check(new ModPack({ deleted: true }));
 			check(new ModPack({
 				name: "Super pack",
@@ -60,6 +63,7 @@ describe("lib/data/ModPack", function() {
 						"string-setting": { "value": "a string" },
 					},
 				},
+				export_manifest: { assets: { setting: "settings.json" }},
 				deleted: true,
 			}));
 		});