Fix exported data loading in web interface

Store exported data files with a hash in the filename to allow the new
caching stategy for static files to work with it and provide a manifest
for the web interface to find the exported files with.
This commit is contained in:
Hornwitser committed 2022-06-08 21:01:36 +02:00
1 parent 48a57bf16c
commit 2a221456d9
7 files changed
+71 -10

No files matched your search

+9
View File
@@ -163,6 +163,7 @@ class Master {
this.instances = await Master.loadInstances(path.join(databaseDirectory, "instances.json"));
this.userManager = new UserManager(this.config);
await this.userManager.load(path.join(databaseDirectory, "users.json"));
this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json"));
this.config.on("fieldChanged", (group, field, prev) => {
libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev);
@@ -371,6 +372,14 @@ class Master {
return manifest;
}
async updateExportManifest(newManifest) {
let databaseDirectory = this.config.get("master.database_directory");
await libFileOps.safeOutputFile(
path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4)
);
this.exportManifest = newManifest;
}
/**
* Query master log
*
+13 -2
View File
@@ -9,6 +9,7 @@ const util = require("util");
const libErrors = require("@clusterio/lib/errors");
const libFileOps = require("@clusterio/lib/file_ops");
const libHash = require("@clusterio/lib/hash");
const libHelpers = require("@clusterio/lib/helpers");
const libLink = require("@clusterio/lib/link");
const libPlugin = require("@clusterio/lib/plugin");
@@ -196,19 +197,28 @@ async function uploadExport(req, res) {
"export/locale.json",
];
let manifest = {};
for (let filePath of exportFiles) {
let file = zip.file(filePath);
if (!file) {
continue;
}
let name = path.posix.basename(filePath);
await libFileOps.safeOutputFile(path.join("static", "export", name), await file.async("nodebuffer"));
let { name, ext } = path.posix.parse(filePath);
let hash = await libHash.hashStream(file.nodeStream());
manifest[name] = `static/${name}.${hash}${ext}`;
await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
}
await res.app.locals.master.updateExportManifest(manifest);
res.sendStatus(200);
}
async function getExportManifest(req, res) {
endpointHitCounter.labels(req.route.path).inc();
res.json(res.app.locals.master.exportManifest);
}
async function createProxyStream(app) {
let asyncRandomBytes = util.promisify(crypto.randomBytes);
let id = (await asyncRandomBytes(8)).toString("hex");
@@ -433,6 +443,7 @@ function addRouteHandlers(app) {
validateSlaveToken,
(req, res, next) => uploadExport(req, res).catch(next)
);
app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next));
app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next));
app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next));
app.post("/api/upload-save",