From 2a221456d9d66bd2176592aab940962d1d9ff601 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 8 Jun 2022 15:36:47 +0200 Subject: [PATCH] Fix exported data loading in web interface Store exported data files with a hash in the filename to allow the new caching stategy for static files to work with it and provide a manifest for the web interface to find the exported files with. --- .gitignore | 2 +- packages/master/src/Master.js | 9 ++++++++ packages/master/src/routes.js | 15 ++++++++++-- packages/web_ui/src/model/export_manifest.jsx | 23 +++++++++++++++++++ packages/web_ui/src/model/item_metadata.jsx | 12 +++++++--- packages/web_ui/src/model/locale.jsx | 10 ++++++-- test/integration/clusterio.js | 10 ++++++-- 7 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 packages/web_ui/src/model/export_manifest.jsx diff --git a/.gitignore b/.gitignore index 72a782e6..84213197 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,7 @@ Factorio* /config-slave.json pnpm-lock.yaml mod_cache.json -static/export +/static/ saves /temp/ diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 65d0c9e8..eb292e62 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -163,6 +163,7 @@ class Master { this.instances = await Master.loadInstances(path.join(databaseDirectory, "instances.json")); this.userManager = new UserManager(this.config); await this.userManager.load(path.join(databaseDirectory, "users.json")); + this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json")); this.config.on("fieldChanged", (group, field, prev) => { libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev); @@ -371,6 +372,14 @@ class Master { return manifest; } + async updateExportManifest(newManifest) { + let databaseDirectory = this.config.get("master.database_directory"); + await libFileOps.safeOutputFile( + path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4) + ); + this.exportManifest = newManifest; + } + /** * Query master log * diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 36c39402..612aad16 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -9,6 +9,7 @@ const util = require("util"); const libErrors = require("@clusterio/lib/errors"); const libFileOps = require("@clusterio/lib/file_ops"); +const libHash = require("@clusterio/lib/hash"); const libHelpers = require("@clusterio/lib/helpers"); const libLink = require("@clusterio/lib/link"); const libPlugin = require("@clusterio/lib/plugin"); @@ -196,19 +197,28 @@ async function uploadExport(req, res) { "export/locale.json", ]; + let manifest = {}; for (let filePath of exportFiles) { let file = zip.file(filePath); if (!file) { continue; } - let name = path.posix.basename(filePath); - await libFileOps.safeOutputFile(path.join("static", "export", name), await file.async("nodebuffer")); + let { name, ext } = path.posix.parse(filePath); + let hash = await libHash.hashStream(file.nodeStream()); + manifest[name] = `static/${name}.${hash}${ext}`; + await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer")); } + await res.app.locals.master.updateExportManifest(manifest); res.sendStatus(200); } +async function getExportManifest(req, res) { + endpointHitCounter.labels(req.route.path).inc(); + res.json(res.app.locals.master.exportManifest); +} + async function createProxyStream(app) { let asyncRandomBytes = util.promisify(crypto.randomBytes); let id = (await asyncRandomBytes(8)).toString("hex"); @@ -433,6 +443,7 @@ function addRouteHandlers(app) { validateSlaveToken, (req, res, next) => uploadExport(req, res).catch(next) ); + app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next)); app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next)); app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next)); app.post("/api/upload-save", diff --git a/packages/web_ui/src/model/export_manifest.jsx b/packages/web_ui/src/model/export_manifest.jsx new file mode 100644 index 00000000..ebb436d9 --- /dev/null +++ b/packages/web_ui/src/model/export_manifest.jsx @@ -0,0 +1,23 @@ +import { useEffect, useState } from "react"; + + +let exportManifestCache = null; +let emptyCache = {}; +export function useExportManifest() { + let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache); + useEffect(() => { + async function load() { + let response = await fetch(`${webRoot}api/export-manifest`); + if (response.ok) { + exportManifestCache = await response.json(); + setExportManifest(exportManifestCache); + } + } + + if (!exportManifestCache) { + load(); + } + }, []); + + return exportManifest; +} diff --git a/packages/web_ui/src/model/item_metadata.jsx b/packages/web_ui/src/model/item_metadata.jsx index 95ed36ac..3858e296 100644 --- a/packages/web_ui/src/model/item_metadata.jsx +++ b/packages/web_ui/src/model/item_metadata.jsx @@ -1,12 +1,18 @@ import { useEffect, useState } from "react"; +import { useExportManifest } from "./export_manifest"; + let itemMetadataCache = null; export function useItemMetadata() { + let exportManifest = useExportManifest(); let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map()); useEffect(() => { async function load() { - let response = await fetch(`${webRoot}export/item-metadata.json`); + if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) { + return; + } + let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`); if (response.ok) { let data = await response.json(); itemMetadataCache = new Map(data); @@ -16,7 +22,7 @@ export function useItemMetadata() { for (let [name, meta] of itemMetadataCache) { style.sheet.insertRule( `.item-${CSS.escape(name)} { - background-image: url("${webRoot}export/item-spritesheet.png"); + background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}"); background-repeat: no-repeat; background-position: -${meta.x}px -${meta.y}px; height: ${meta.size}px; @@ -31,7 +37,7 @@ export function useItemMetadata() { if (!itemMetadataCache) { load(); } - }, []); + }, [exportManifest]); return itemMetadata; } diff --git a/packages/web_ui/src/model/locale.jsx b/packages/web_ui/src/model/locale.jsx index ad4b83d0..3a22e6d7 100644 --- a/packages/web_ui/src/model/locale.jsx +++ b/packages/web_ui/src/model/locale.jsx @@ -1,12 +1,18 @@ import React, { useEffect, useState } from "react"; +import { useExportManifest } from "./export_manifest"; + let localeCache = null; export function useLocale() { + let exportManifest = useExportManifest(); let [locale, setLocale] = useState(localeCache || new Map()); useEffect(() => { async function load() { - let response = await fetch(`${webRoot}export/locale.json`); + if (!exportManifest["locale"]) { + return; + } + let response = await fetch(`${staticRoot}${exportManifest["locale"]}`); if (response.ok) { let data = await response.json(); localeCache = new Map(data); @@ -17,7 +23,7 @@ export function useLocale() { if (!localeCache) { load(); } - }, []); + }, [exportManifest]); return locale; } diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index edeb4574..d4051043 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -300,10 +300,16 @@ describe("Integration of Clusterio", function() { describe("instance export-data", function() { it("exports the data", async function() { slowTest(this); - let exportPath = path.join("temp", "test", "static", "export", "locale.json"); + let exportPath = path.join("temp", "test", "static"); await fs.remove(exportPath); await execCtl("instance export-data test"); - assert(await fs.exists(exportPath), "Export was not created"); + let response = await get("/api/export-manifest"); + let manifest = JSON.parse(response.body); + assert(Object.keys(manifest).length > 1, "Export manifest is empty"); + assert( + await fs.exists(path.join(exportPath, "..", manifest["item-metadata"])), + "Export was not written to filesystem" + ); await checkInstanceStatus(44, "stopped"); }); });