Tie data exports to mod packs

Store the export manifest in ModPack to allow mulitple mod packs to have
different data exported. This is necessary as the export contains the
settings prototypes needed to create a proper UI for mod settings.

For now the useLocale and useItemMetadata interfaces uses the manifest
for a default mod pack set in the master config.
This commit is contained in:
Hornwitser committed 2023-02-02 00:28:58 +01:00
1 parent 7c6aaeb05d
commit 13e12f7335
16 files changed
+146 -34

No files matched your search

+6
View File
@@ -893,6 +893,12 @@ modPackCommands.add(new libCommand.Command({
print(` ${setting}: ${JSON.stringify(settingValue.value)}`); print(` ${setting}: ${JSON.stringify(settingValue.value)}`);
} }
} }
} else if (field === "export_manifest") {
print(`${field}:`);
print(" assets:");
for (let [name, fileName] of Object.entries(value.assets)) {
print(` ${name}: ${fileName}`);
}
} else { } else {
print(`${field}: ${value}`); print(`${field}: ${value}`);
+7
View File
@@ -115,6 +115,13 @@ MasterGroup.define({
type: "number", type: "number",
initial_value: 15, initial_value: 15,
}); });
MasterGroup.define({
name: "default_mod_pack_id",
title: "Default Mod Pack",
description: "Mod pack used by default for instances.",
type: "number",
optional: true,
});
MasterGroup.define({ MasterGroup.define({
name: "default_role_id", name: "default_role_id",
title: "Default role", title: "Default role",
+46
View File
@@ -0,0 +1,46 @@
"use strict";
/**
* Export manifest for a mod pack
*
* Tracks files stored on disk for data that extracted from Factorio using a
* given mod pack.
* @alias module:lib/data.ExportManifest
*/
class ExportManifest {
/**
* Mapping between known assets and their file names.
*
* Currently exported assets are
* - settings: JSON of mod settings prototypes.
* - prototypes: JSON of all game prototypes except settings.
* - item-spritesheet: PNG containing icons from the game.
* - item-metadata: JSON of Map entries describing icons in item-spritesheet
* - locale: JSON of flattened Map entries of en locale strings from the game.
* @type {Object<string, string>}
*/
assets = {};
static jsonSchema = {
type: "object",
additionalProperties: false,
required: ["assets"],
properties: {
"assets": {
type: "object",
additionalProperties: { type: "string" },
},
},
};
constructor(json = {}) {
if (json.assets) { this.assets = json.assets; }
}
toJSON() {
return { assets: this.assets };
}
}
module.exports = ExportManifest;
+11
View File
@@ -3,6 +3,7 @@ const zlib = require("zlib");
const libSchema = require("../schema"); const libSchema = require("../schema");
const ExportManifest = require("./ExportManifest");
const { integerFactorioVersion } = require("./version"); const { integerFactorioVersion } = require("./version");
@@ -99,6 +100,12 @@ class ModPack {
"runtime-per-user": new Map(), "runtime-per-user": new Map(),
}; };
/**
* Mapping to files containing exported data for this modpack
* @type {module:lib/data.ExportManifest|undefined}
*/
exportManifest;
/** /**
* True if this mod pack has been deleted from the list of mod packs. * True if this mod pack has been deleted from the list of mod packs.
* @type {boolean} * @type {boolean}
@@ -128,6 +135,7 @@ class ModPack {
"runtime-per-user": ModSettingsJsonSchema, "runtime-per-user": ModSettingsJsonSchema,
}, },
}, },
"export_manifest": ExportManifest.jsonSchema,
"is_deleted": { type: "boolean" }, "is_deleted": { type: "boolean" },
}, },
}; };
@@ -151,6 +159,7 @@ class ModPack {
"runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])), "runtime-per-user": new Map(Object.entries(json.settings["runtime-per-user"])),
}; };
} }
if (json.export_manifest) { this.exportManifest = new ExportManifest(json.export_manifest); }
if (json.is_deleted) { this.isDeleted = json.is_deleted; } if (json.is_deleted) { this.isDeleted = json.is_deleted; }
if (!this.mods.has("base")) { if (!this.mods.has("base")) {
@@ -171,6 +180,7 @@ class ModPack {
"runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]), "runtime-per-user": Object.fromEntries(this.settings["runtime-per-user"]),
}, },
}; };
if (this.exportManifest) { json.export_manifest = this.exportManifest.toJSON(); }
if (this.isDeleted) { json.is_deleted = this.isDeleted; } if (this.isDeleted) { json.is_deleted = this.isDeleted; }
return json; return json;
} }
@@ -178,6 +188,7 @@ class ModPack {
toModPackString() { toModPackString() {
const json = this.toJSON(); const json = this.toJSON();
delete json.id; delete json.id;
delete json.export_manifest;
// eslint-disable-next-line node/no-sync // eslint-disable-next-line node/no-sync
let buf = zlib.deflateSync(JSON.stringify(json)); let buf = zlib.deflateSync(JSON.stringify(json));
+2
View File
@@ -4,11 +4,13 @@
* @author Hornwitser * @author Hornwitser
*/ */
"use strict"; "use strict";
const ExportManifest = require("./ExportManifest");
const ModInfo = require("./ModInfo"); const ModInfo = require("./ModInfo");
const ModPack = require("./ModPack"); const ModPack = require("./ModPack");
const version = require("./version"); const version = require("./version");
module.exports = { module.exports = {
ExportManifest,
ModInfo, ModInfo,
ModPack, ModPack,
...version, ...version,
+10
View File
@@ -723,6 +723,16 @@ messages.getModPack = new Request({
}, },
}); });
messages.getDefaultModPack = new Request({
type: "get_default_mod_pack",
links: ["control-master", "instance-slave", "slave-master"],
permission: "core.mod_pack.get",
forwardTo: "master",
responseProperties: {
"mod_pack": libData.ModPack.jsonSchema,
},
});
messages.listModPacks = new Request({ messages.listModPacks = new Request({
type: "list_mod_packs", type: "list_mod_packs",
links: ["control-master"], links: ["control-master"],
+12
View File
@@ -93,6 +93,18 @@ class BaseConnection extends libLink.Link {
} }
return { mod_pack: modPack.toJSON() }; return { mod_pack: modPack.toJSON() };
} }
async getDefaultModPackRequestHandler(message) {
let id = this._master.config.get("master.default_mod_pack_id");
if (id === null) {
throw new libErrors.RequestError("Default mod pack not set on master");
}
let modPack = this._master.modPacks.get(id);
if (!modPack) {
throw new libErrors.RequestError(`Default mod pack configured (${id}) does not exist`);
}
return { mod_pack: modPack.toJSON() };
}
} }
module.exports = BaseConnection; module.exports = BaseConnection;
-9
View File
@@ -177,7 +177,6 @@ class Master {
this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json")); this.modPacks = await Master.loadModPacks(path.join(databaseDirectory, "mod-packs.json"));
this.userManager = new UserManager(this.config); this.userManager = new UserManager(this.config);
await this.userManager.load(path.join(databaseDirectory, "users.json")); await this.userManager.load(path.join(databaseDirectory, "users.json"));
this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json"));
let modsDirectory = this.config.get("master.mods_directory"); let modsDirectory = this.config.get("master.mods_directory");
await fs.ensureDir(modsDirectory); await fs.ensureDir(modsDirectory);
@@ -445,14 +444,6 @@ class Master {
return manifest; return manifest;
} }
async updateExportManifest(newManifest) {
let databaseDirectory = this.config.get("master.database_directory");
await libFileOps.safeOutputFile(
path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4)
);
this.exportManifest = newManifest;
}
async deleteMod(name, version) { async deleteMod(name, version) {
let filename = `${name}_${version}.zip`; let filename = `${name}_${version}.zip`;
let mod = this.mods.get(filename); let mod = this.mods.get(filename);
+20 -9
View File
@@ -187,6 +187,21 @@ async function uploadExport(req, res) {
return; return;
} }
if (typeof req.query.mod_pack_id !== "string") {
res.sendStatus(400);
return;
}
let modPackId = Number.parseInt(req.query.mod_pack_id, 10);
if (!Number.isInteger(modPackId)) {
res.sendStatus(400);
return;
}
let modPack = res.app.locals.master.modPacks.get(modPackId);
if (!modPack) {
res.sendStatus(400);
return;
}
let data = []; let data = [];
for await (let chunk of req) { for await (let chunk of req) {
data.push(chunk); data.push(chunk);
@@ -204,7 +219,7 @@ async function uploadExport(req, res) {
"export/locale.json", "export/locale.json",
]; ];
let manifest = {}; let assets = {};
for (let filePath of exportFiles) { for (let filePath of exportFiles) {
let file = zip.file(filePath); let file = zip.file(filePath);
if (!file) { if (!file) {
@@ -213,19 +228,16 @@ async function uploadExport(req, res) {
let { name, ext } = path.posix.parse(filePath); let { name, ext } = path.posix.parse(filePath);
let hash = await libHash.hashStream(file.nodeStream()); let hash = await libHash.hashStream(file.nodeStream());
manifest[name] = `static/${name}.${hash}${ext}`; assets[name] = `${name}.${hash}${ext}`;
await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer")); await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
} }
await res.app.locals.master.updateExportManifest(manifest);
modPack.exportManifest = new libData.ExportManifest({ assets });
res.app.locals.master.modPackUpdated(modPack);
res.sendStatus(200); res.sendStatus(200);
} }
async function getExportManifest(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.json(res.app.locals.master.exportManifest);
}
async function createProxyStream(app) { async function createProxyStream(app) {
let asyncRandomBytes = util.promisify(crypto.randomBytes); let asyncRandomBytes = util.promisify(crypto.randomBytes);
let id = (await asyncRandomBytes(8)).toString("hex"); let id = (await asyncRandomBytes(8)).toString("hex");
@@ -601,7 +613,6 @@ function addRouteHandlers(app) {
validateSlaveToken, validateSlaveToken,
(req, res, next) => uploadExport(req, res).catch(next) (req, res, next) => uploadExport(req, res).catch(next)
); );
app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next));
app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next)); app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next));
app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next)); app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next));
app.post("/api/upload-save", app.post("/api/upload-save",
+1
View File
@@ -1064,6 +1064,7 @@ rcon.print(game.table_to_json(players))`.replace(/\r?\n/g, " ");
let content = await zip.generateAsync({ type: "nodebuffer" }); let content = await zip.generateAsync({ type: "nodebuffer" });
let url = new URL(this._slave.config.get("slave.master_url")); let url = new URL(this._slave.config.get("slave.master_url"));
url.pathname += "api/upload-export"; url.pathname += "api/upload-export";
url.searchParams.set("mod_pack_id", this.config.get("factorio.mod_pack"));
let response = await phin({ let response = await phin({
url, method: "PUT", url, method: "PUT",
data: content, data: content,
@@ -1,15 +1,20 @@
import { useEffect, useState } from "react"; import { useEffect, useContext, useState } from "react";
import { libData, libLink } from "@clusterio/lib";
import ControlContext from "../components/ControlContext";
let exportManifestCache = null; let exportManifestCache = null;
let emptyCache = {}; let emptyCache = {};
export function useExportManifest() { export function useExportManifest() {
const control = useContext(ControlContext);
let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache); let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache);
useEffect(() => { useEffect(() => {
async function load() { async function load() {
let response = await fetch(`${webRoot}api/export-manifest`); let result = await libLink.messages.getDefaultModPack.send(control);
if (response.ok) { let modPack = new libData.ModPack(result.mod_pack);
exportManifestCache = await response.json(); if (modPack.exportManifest) {
exportManifestCache = modPack.exportManifest;
setExportManifest(exportManifestCache); setExportManifest(exportManifestCache);
} }
} }
+7 -3
View File
@@ -9,10 +9,14 @@ export function useItemMetadata() {
let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map()); let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map());
useEffect(() => { useEffect(() => {
async function load() { async function load() {
if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) { if (
!exportManifest
|| !exportManifest.assets["item-metadata"]
|| !exportManifest.assets["item-spritesheet"]
) {
return; return;
} }
let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`); let response = await fetch(`${staticRoot}static/${exportManifest.assets["item-metadata"]}`);
if (response.ok) { if (response.ok) {
let data = await response.json(); let data = await response.json();
itemMetadataCache = new Map(data); itemMetadataCache = new Map(data);
@@ -22,7 +26,7 @@ export function useItemMetadata() {
for (let [name, meta] of itemMetadataCache) { for (let [name, meta] of itemMetadataCache) {
style.sheet.insertRule( style.sheet.insertRule(
`.item-${CSS.escape(name)} { `.item-${CSS.escape(name)} {
background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}"); background-image: url("${staticRoot}static/${exportManifest.assets["item-spritesheet"]}");
background-repeat: no-repeat; background-repeat: no-repeat;
background-position: -${meta.x}px -${meta.y}px; background-position: -${meta.x}px -${meta.y}px;
height: ${meta.size}px; height: ${meta.size}px;
+2 -2
View File
@@ -9,10 +9,10 @@ export function useLocale() {
let [locale, setLocale] = useState(localeCache || new Map()); let [locale, setLocale] = useState(localeCache || new Map());
useEffect(() => { useEffect(() => {
async function load() { async function load() {
if (!exportManifest["locale"]) { if (!exportManifest || !exportManifest.assets["locale"]) {
return; return;
} }
let response = await fetch(`${staticRoot}${exportManifest["locale"]}`); let response = await fetch(`${staticRoot}static/${exportManifest.assets["locale"]}`);
if (response.ok) { if (response.ok) {
let data = await response.json(); let data = await response.json();
localeCache = new Map(data); localeCache = new Map(data);
+7 -6
View File
@@ -307,17 +307,18 @@ describe("Integration of Clusterio", function() {
let exportPath = path.join("temp", "test", "static"); let exportPath = path.join("temp", "test", "static");
await fs.remove(exportPath); await fs.remove(exportPath);
await execCtl("instance export-data test"); await execCtl("instance export-data test");
let response = await get("/api/export-manifest"); let result = await libLink.messages.getDefaultModPack.send(getControl());
let manifest = JSON.parse(response.body); let modPack = new libData.ModPack(result.mod_pack);
assert(Object.keys(manifest).length > 1, "Export manifest is empty"); let assets = modPack.exportManifest.assets;
assert(Object.keys(assets).length > 1, "Export assets is empty");
for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) { for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) {
assert(manifest[key], `Missing ${key} from manifest`); assert(assets[key], `Missing ${key} from assets`);
assert( assert(
await fs.exists(path.join(exportPath, "..", manifest[key])), await fs.exists(path.join(exportPath, assets[key])),
`Manifest entry for ${key} was not written to filesystem` `Manifest entry for ${key} was not written to filesystem`
); );
} }
let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, "..", manifest["prototypes"]))); let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, assets["prototypes"])));
assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types"); assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types");
await checkInstanceStatus(44, "stopped"); await checkInstanceStatus(44, "stopped");
}); });
+2 -1
View File
@@ -228,7 +228,8 @@ before(async function() {
testPack.factorioVersion = "1.1.0"; testPack.factorioVersion = "1.1.0";
testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" }); testPack.mods.set("subspace_storage", { name: "clusterio_lib", enabled: true, version: "0.1.2" });
testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" }); testPack.mods.set("subspace_storage", { name: "subspace_storage", enabled: true, version: "1.99.8" });
libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() }); await libLink.messages.createModPack.send(control, { mod_pack: testPack.toJSON() });
await libLink.messages.setMasterConfigField.send(control, { field: "master.default_mod_pack_id", value: "12" });
}); });
after(async function() { after(async function() {
+4
View File
@@ -3,6 +3,7 @@ const assert = require("assert").strict;
const zlib = require("zlib"); const zlib = require("zlib");
const libSchema = require("@clusterio/lib/schema"); const libSchema = require("@clusterio/lib/schema");
const ExportManifest = require("@clusterio/lib/data/ExportManifest");
const ModPack = require("@clusterio/lib/data/ModPack"); const ModPack = require("@clusterio/lib/data/ModPack");
@@ -18,6 +19,7 @@ describe("lib/data/ModPack", function() {
assert.deepEqual(new ModPack(json), pack); assert.deepEqual(new ModPack(json), pack);
const packStringed = ModPack.fromModPackString(pack.toModPackString()); const packStringed = ModPack.fromModPackString(pack.toModPackString());
packStringed.id = pack.id; packStringed.id = pack.id;
packStringed.exportManifest = pack.exportManifest;
assert.deepEqual(packStringed, pack); assert.deepEqual(packStringed, pack);
} }
@@ -40,6 +42,7 @@ describe("lib/data/ModPack", function() {
"string-setting": { "value": "a string" }, "string-setting": { "value": "a string" },
}, },
}})); }}));
check(new ModPack({ export_manifest: { assets: { setting: "settings.json" }}}));
check(new ModPack({ deleted: true })); check(new ModPack({ deleted: true }));
check(new ModPack({ check(new ModPack({
name: "Super pack", name: "Super pack",
@@ -60,6 +63,7 @@ describe("lib/data/ModPack", function() {
"string-setting": { "value": "a string" }, "string-setting": { "value": "a string" },
}, },
}, },
export_manifest: { assets: { setting: "settings.json" }},
deleted: true, deleted: true,
})); }));
}); });