Fix exported data loading in web interface

Store exported data files with a hash in the filename to allow the new
caching stategy for static files to work with it and provide a manifest
for the web interface to find the exported files with.

.gitignore

@@ -40,7 +40,7 @@ Factorio*
 /config-slave.json
 pnpm-lock.yaml
 mod_cache.json
-static/export
+/static/
 saves
 /temp/
 

packages/master/src/Master.js

@@ -163,6 +163,7 @@ class Master {
 		this.instances = await Master.loadInstances(path.join(databaseDirectory, "instances.json"));
 		this.userManager = new UserManager(this.config);
 		await this.userManager.load(path.join(databaseDirectory, "users.json"));
+		this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json"));
 
 		this.config.on("fieldChanged", (group, field, prev) => {
 			libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev);
@@ -371,6 +372,14 @@ class Master {
 		return manifest;
 	}
 
+	async updateExportManifest(newManifest) {
+		let databaseDirectory = this.config.get("master.database_directory");
+		await libFileOps.safeOutputFile(
+			path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4)
+		);
+		this.exportManifest = newManifest;
+	}
+
 	/**
 	 * Query master log
 	 *

packages/master/src/routes.js

@@ -9,6 +9,7 @@ const util = require("util");
 
 const libErrors = require("@clusterio/lib/errors");
 const libFileOps = require("@clusterio/lib/file_ops");
+const libHash = require("@clusterio/lib/hash");
 const libHelpers = require("@clusterio/lib/helpers");
 const libLink = require("@clusterio/lib/link");
 const libPlugin = require("@clusterio/lib/plugin");
@@ -196,19 +197,28 @@ async function uploadExport(req, res) {
 		"export/locale.json",
 	];
 
+	let manifest = {};
 	for (let filePath of exportFiles) {
 		let file = zip.file(filePath);
 		if (!file) {
 			continue;
 		}
 
-		let name = path.posix.basename(filePath);
-		await libFileOps.safeOutputFile(path.join("static", "export", name), await file.async("nodebuffer"));
+		let { name, ext } = path.posix.parse(filePath);
+		let hash = await libHash.hashStream(file.nodeStream());
+		manifest[name] = `static/${name}.${hash}${ext}`;
+		await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
 	}
+	await res.app.locals.master.updateExportManifest(manifest);
 
 	res.sendStatus(200);
 }
 
+async function getExportManifest(req, res) {
+	endpointHitCounter.labels(req.route.path).inc();
+	res.json(res.app.locals.master.exportManifest);
+}
+
 async function createProxyStream(app) {
 	let asyncRandomBytes = util.promisify(crypto.randomBytes);
 	let id = (await asyncRandomBytes(8)).toString("hex");
@@ -433,6 +443,7 @@ function addRouteHandlers(app) {
 		validateSlaveToken,
 		(req, res, next) => uploadExport(req, res).catch(next)
 	);
+	app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next));
 	app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next));
 	app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next));
 	app.post("/api/upload-save",

packages/web_ui/src/model/export_manifest.jsx

@@ -0,0 +1,23 @@
+import { useEffect, useState } from "react";
+
+
+let exportManifestCache = null;
+let emptyCache = {};
+export function useExportManifest() {
+	let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache);
+	useEffect(() => {
+		async function load() {
+			let response = await fetch(`${webRoot}api/export-manifest`);
+			if (response.ok) {
+				exportManifestCache = await response.json();
+				setExportManifest(exportManifestCache);
+			}
+		}
+
+		if (!exportManifestCache) {
+			load();
+		}
+	}, []);
+
+	return exportManifest;
+}

packages/web_ui/src/model/item_metadata.jsx

@@ -1,12 +1,18 @@
 import { useEffect, useState } from "react";
 
+import { useExportManifest } from "./export_manifest";
+
 
 let itemMetadataCache = null;
 export function useItemMetadata() {
+	let exportManifest = useExportManifest();
 	let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map());
 	useEffect(() => {
 		async function load() {
-			let response = await fetch(`${webRoot}export/item-metadata.json`);
+			if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) {
+				return;
+			}
+			let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				itemMetadataCache = new Map(data);
@@ -16,7 +22,7 @@ export function useItemMetadata() {
 				for (let [name, meta] of itemMetadataCache) {
 					style.sheet.insertRule(
 						`.item-${CSS.escape(name)} {
-	background-image: url("${webRoot}export/item-spritesheet.png");
+	background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}");
 	background-repeat: no-repeat;
 	background-position: -${meta.x}px -${meta.y}px;
 	height: ${meta.size}px;
@@ -31,7 +37,7 @@ export function useItemMetadata() {
 		if (!itemMetadataCache) {
 			load();
 		}
-	}, []);
+	}, [exportManifest]);
 
 	return itemMetadata;
 }

packages/web_ui/src/model/locale.jsx

@@ -1,12 +1,18 @@
 import React, { useEffect, useState } from "react";
 
+import { useExportManifest } from "./export_manifest";
+
 
 let localeCache = null;
 export function useLocale() {
+	let exportManifest = useExportManifest();
 	let [locale, setLocale] = useState(localeCache || new Map());
 	useEffect(() => {
 		async function load() {
-			let response = await fetch(`${webRoot}export/locale.json`);
+			if (!exportManifest["locale"]) {
+				return;
+			}
+			let response = await fetch(`${staticRoot}${exportManifest["locale"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				localeCache = new Map(data);
@@ -17,7 +23,7 @@ export function useLocale() {
 		if (!localeCache) {
 			load();
 		}
-	}, []);
+	}, [exportManifest]);
 
 	return locale;
 }

test/integration/clusterio.js

@@ -300,10 +300,16 @@ describe("Integration of Clusterio", function() {
 		describe("instance export-data", function() {
 			it("exports the data", async function() {
 				slowTest(this);
-				let exportPath = path.join("temp", "test", "static", "export", "locale.json");
+				let exportPath = path.join("temp", "test", "static");
 				await fs.remove(exportPath);
 				await execCtl("instance export-data test");
-				assert(await fs.exists(exportPath), "Export was not created");
+				let response = await get("/api/export-manifest");
+				let manifest = JSON.parse(response.body);
+				assert(Object.keys(manifest).length > 1, "Export manifest is empty");
+				assert(
+					await fs.exists(path.join(exportPath, "..", manifest["item-metadata"])),
+					"Export was not written to filesystem"
+				);
 				await checkInstanceStatus(44, "stopped");
 			});
 		});