From 7625fd82e353e6cce420b625ebdbc0d67786ee43 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 6 Mar 2024 18:30:45 +0100 Subject: [PATCH] Cache ModInfo to reduce startup time (#606) Store and use a cache of the loaded ModInfo data to mod-info-cache.json in the mods directory during startup. For simplicity this does not cache mods added to the controller or host after startup, these get added to the cache on the next startup. --- packages/lib/src/ModStore.ts | 36 ++++++++++++++++++++++++++++++++ packages/lib/src/data/ModInfo.ts | 9 ++++++++ test/integration/clusterio.js | 1 + test/lib/data/ModInfo.js | 3 +++ 4 files changed, 49 insertions(+) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 1819e645..e53fdb39 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -1,8 +1,10 @@ import fs from "fs-extra"; import path from "path"; +import { Static } from "@sinclair/typebox"; import { logger } from "./logging"; import { ModInfo } from "./data"; import TypedEventEmitter from "./TypedEventEmitter"; +import { safeOutputFile } from "./file_ops"; export interface ModStoreEvents { /** A stored mod was created, updated or deleted */ @@ -74,6 +76,21 @@ export default class ModStore extends TypedEventEmitter[]; + const cacheFilename = path.join(modsDirectory, "mod-info-cache.json"); + let cache = new Map(); + try { + const content = await fs.readFile(cacheFilename, "utf8"); + cache = new Map((JSON.parse(content) as ModInfoCache).map(item => { + const mod = ModInfo.fromJSON(item); + return [mod.filename, mod]; + })); + } catch (err: any) { + if (err.code !== "ENOENT") { + logger.warn(`Error loading ${cacheFilename}:\n${err.stack}`); + } + } + const files = new Map(); for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) { if (entry.isDirectory()) { @@ -81,10 +98,23 @@ export default class ModStore extends TypedEventEmitter mod.toJSON()); + await safeOutputFile(cacheFilename, JSON.stringify(content)); + } + return new this(modsDirectory, files); } } diff --git a/packages/lib/src/data/ModInfo.ts b/packages/lib/src/data/ModInfo.ts index b555f0bb..e29db8d3 100644 --- a/packages/lib/src/data/ModInfo.ts +++ b/packages/lib/src/data/ModInfo.ts @@ -116,6 +116,11 @@ export default class ModInfo { */ size = 0; + /** + * Last modification time of the file. + */ + mtimeMs = 0; + /** * SHA1 hash of this mod */ @@ -147,6 +152,7 @@ export default class ModInfo { static jsonSchema = Type.Object({ ...this.infoJsonSchema.properties, "size": Type.Optional(Type.Integer()), + "mtime_ms": Type.Optional(Type.Number()), "sha1": Type.Optional(Type.String()), "updated_at_ms": Type.Optional(Type.Number()), "is_deleted": Type.Optional(Type.Boolean()), @@ -170,6 +176,7 @@ export default class ModInfo { // Additional data if (json.size) { modInfo.size = json.size; } + if (json.mtime_ms) { modInfo.mtimeMs = json.mtime_ms; } if (json.sha1) { modInfo.sha1 = json.sha1; } if (json.updated_at_ms) { modInfo.updatedAtMs = json.updated_at_ms; } if (json.is_deleted) { modInfo.isDeleted = json.is_deleted; } @@ -192,6 +199,7 @@ export default class ModInfo { json.dependencies = this.dependencies; } if (this.size) { json.size = this.size; } + if (this.mtimeMs) { json.mtime_ms = this.mtimeMs; } if (this.sha1) { json.sha1 = this.sha1; } if (this.updatedAtMs) { json.updated_at_ms = this.updatedAtMs; } if (this.isDeleted) { json.is_deleted = this.isDeleted; } @@ -229,6 +237,7 @@ export default class ModInfo { size: stat.size, sha1: await libHash.hashFile(modPath), + mtime_ms: stat.mtimeMs, updated_at_ms: stat.mtimeMs, is_deleted: false, }); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 4fe1f626..acf69b94 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -1054,6 +1054,7 @@ describe("Integration of Clusterio", function() { "factorioVersion: 1.1\n" + "dependencies:\n" + `size: ${stat.size}\n` + + `mtimeMs: ${stat.mtimeMs}\n` + `sha1: ${hash}\n` + `updatedAtMs: ${stat.mtimeMs}\n` + "isDeleted: false\n", diff --git a/test/lib/data/ModInfo.js b/test/lib/data/ModInfo.js index 50e104db..9e8d6c65 100644 --- a/test/lib/data/ModInfo.js +++ b/test/lib/data/ModInfo.js @@ -33,6 +33,7 @@ describe("lib/data/ModInfo", function() { check(ModInfo.fromJSON({ dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"] })); check(ModInfo.fromJSON({ filename: "MyMod_1.0.0.zip" })); check(ModInfo.fromJSON({ size: 1024 })); + check(ModInfo.fromJSON({ mtime_ms: new Date(2024, 0, 1).getTime() })); check(ModInfo.fromJSON({ sha1: "verified-as-MyMod" })); check(ModInfo.fromJSON({ is_deleted: true })); @@ -49,6 +50,7 @@ describe("lib/data/ModInfo", function() { dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"], filename: "MyMod_1.0.0.zip", size: 1024, + mtime_ms: new Date(2024, 0, 1).getTime(), sha1: "verified-as-MyMod", is_deleted: true, })); @@ -92,6 +94,7 @@ describe("lib/data/ModInfo", function() { dependencies: [], filename: "empty_mod_1.0.0.zip", size: stat.size, + mtime_ms: stat.mtimeMs, updated_at_ms: stat.mtimeMs, sha1: hash, })