diff --git a/docs/managing-a-cluster.md b/docs/managing-a-cluster.md index 476d1314..aa3e8587 100644 --- a/docs/managing-a-cluster.md +++ b/docs/managing-a-cluster.md @@ -247,6 +247,13 @@ This command lists all plugins currently known to the remote. Newly installed pl If remote plugin updates are enabled on the remote, this will execute `npm update --save `. Changes will not be applied until the remote is restarted. If the `--restart` flag is provided, the remote will automatically restart once the update is complete. +### Update All + + ctl> controller update --all [--restart] + ctl> host update --all [--restart] + +Updates the Clusterio package and all plugins installed from npm in a single `npm update --save` command. Updating everything at once avoids ending up with multiple versions of `@clusterio/lib` in the installation. Requires both remote updates and remote plugin updates to be enabled on the remote. + ### Install ctl> controller plugin install [--restart] diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 2551aee0..a61a0080 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -206,7 +206,7 @@ async function handleBootstrapCommand( console.log(content); } else { logger.info(`Writing ${args.output}`); - await lib.safeOutputFile(args.output, content); + await lib.safeOutputFile(args.output, content, { mode: 0o600 }); } } } diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 3dbe8934..b502429b 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -126,6 +126,7 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.PluginListRequest, this.handlePluginListRequest.bind(this)); this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this)); this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this)); + this.handle(lib.UpdateAllRequest, this.handleUpdateAllRequest.bind(this)); this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this)); } @@ -1305,6 +1306,16 @@ export default class ControlConnection extends BaseConnection { return await lib.handlePluginInstall(request.pluginPackage); } + async handleUpdateAllRequest(request: lib.UpdateAllRequest) { + if (!this._controller.config.get("controller.allow_remote_updates")) { + throw new lib.RequestError("Remote updates are disabled on this machine"); + } + if (!this._controller.config.get("controller.allow_plugin_updates")) { + throw new lib.RequestError("Plugin updates are disabled on this machine"); + } + return await lib.handleUpdateAll("@clusterio/controller", this._controller.pluginInfos); + } + async handlePluginListRequest(request: lib.PluginListRequest) { return this._controller.pluginInfos.map(pluginInfo => lib.PluginDetails.fromNodeEnvInfo( pluginInfo, diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 841304a1..d3b09b3a 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -1135,7 +1135,9 @@ export default class Controller { let manifestPath = path.posix.join(pluginInfo.requirePath, "dist", "web", "manifest.json"); pluginInfo.manifest = await Controller.loadJsonObject(import.meta.resolve(manifestPath), true); } catch (err) { - logger.warn(`Unable to load dist/web/manifest.json for plugin ${pluginInfo.name}`); + if (lib.pluginNeedsWebBuild(pluginInfo)) { + logger.warn(`Unable to load dist/web/manifest.json for plugin ${pluginInfo.name}`); + } } if (!this.config.get(`${pluginInfo.name}.load_plugin`)) { diff --git a/packages/controller/src/WsServerConnector.ts b/packages/controller/src/WsServerConnector.ts index 7277f8c3..439a2056 100644 --- a/packages/controller/src/WsServerConnector.ts +++ b/packages/controller/src/WsServerConnector.ts @@ -106,6 +106,9 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector { this._state = "connected"; this._attachSocketHandlers(); this._dropSendBufferSeq(lastSeq); + logger.verbose( + `Connector | resuming session with ${this.dst}, resending ${this._sendBuffer.length} buffered messages` + ); for (let message of this._sendBuffer) { this._socket.send(JSON.stringify(message)); } @@ -113,7 +116,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector { } _timedOut() { - logger.verbose("Connector | Connection timed out"); + logger.verbose(`Connector | Connection to ${this.dst} timed out`); this._close(); } diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index d54b785f..993f16b9 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -132,7 +132,7 @@ function getPlugins(req: Request, res: Response) { if (!web.main) { web.error = `Missing ${pluginInfo.name}.js entry in manifest.json`; } - } else { + } else if (lib.pluginNeedsWebBuild(pluginInfo)) { web.error = "Missing dist/web/manifest.json"; } if (web.main === "remoteEntry.js") { diff --git a/packages/create/create.js b/packages/create/create.js index 457dae2b..5980f9d2 100755 --- a/packages/create/create.js +++ b/packages/create/create.js @@ -234,7 +234,7 @@ async function migrateRename(args) { if (await pathExists(source) && !await pathExists(destination)) { await safeOutputFile(destination, JSON.stringify( renameConfig(JSON.parse(await fs.readFile(source))), null, "\t" - )); + ), { mode: 0o600 }); logger.info(`Migrated ${source} to ${destination}`); } } diff --git a/packages/ctl/src/commands_controller.ts b/packages/ctl/src/commands_controller.ts index 4bfbd41e..a6684abd 100644 --- a/packages/ctl/src/commands_controller.ts +++ b/packages/ctl/src/commands_controller.ts @@ -28,9 +28,14 @@ controllerCommands.add(new lib.Command({ controllerCommands.add(new lib.Command({ definition: ["update", "Update the controller", (yargs) => { yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" }); + yargs.option("all", { alias: "a", type: "boolean", description: "Also update all plugins" }); }], - handler: async function(args: { restart: boolean }, control: Control) { - await control.send(new lib.ControllerUpdateRequest()); + handler: async function(args: { restart: boolean, all: boolean }, control: Control) { + if (args.all) { + await control.send(new lib.UpdateAllRequest()); + } else { + await control.send(new lib.ControllerUpdateRequest()); + } if (args.restart) { await control.send(new lib.ControllerRestartRequest()); } else { diff --git a/packages/ctl/src/commands_host.ts b/packages/ctl/src/commands_host.ts index fc269e3b..fe408679 100644 --- a/packages/ctl/src/commands_host.ts +++ b/packages/ctl/src/commands_host.ts @@ -87,7 +87,7 @@ hostCommands.add(new lib.Command({ } else { logger.info(`Writing ${args.output}`); try { - await fs.writeFile(args.output, content, { flag: "wx" }); + await fs.writeFile(args.output, content, { flag: "wx", mode: 0o600 }); } catch (err: any) { if (err.code === "EEXIST") { throw new lib.CommandError(`File ${args.output} already exists`); @@ -101,10 +101,15 @@ hostCommands.add(new lib.Command({ definition: ["update ", "Update the host", (yargs) => { yargs.positional("host", { describe: "Host to update", type: "string" }); yargs.option("restart", { alias: "r", type: "boolean", description: "Restart after update" }); + yargs.option("all", { alias: "a", type: "boolean", description: "Also update all plugins" }); }], - handler: async function(args: { host: string, restart: boolean }, control: Control) { + handler: async function(args: { host: string, restart: boolean, all: boolean }, control: Control) { let hostId = await lib.resolveHost(control, args.host); - await control.sendTo({ hostId }, new lib.HostUpdateRequest()); + if (args.all) { + await control.sendTo({ hostId }, new lib.UpdateAllRequest()); + } else { + await control.sendTo({ hostId }, new lib.HostUpdateRequest()); + } if (args.restart) { await control.sendTo({ hostId }, new lib.HostRestartRequest()); } else { diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 747d3e0e..a586a982 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -401,6 +401,7 @@ export default class Host extends lib.Link { this.handle(lib.PluginListRequest, this.handlePluginListRequest.bind(this)); this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this)); this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this)); + this.handle(lib.UpdateAllRequest, this.handleUpdateAllRequest.bind(this)); this.snoopEvent(lib.InstanceAdminlistUpdateEvent, this.handleAdminlistUpdateEvent.bind(this)); this.snoopEvent(lib.InstanceBanlistUpdateEvent, this.handleBanlistUpdateEvent.bind(this)); @@ -1145,6 +1146,16 @@ export default class Host extends lib.Link { return await lib.handlePluginInstall(request.pluginPackage); } + async handleUpdateAllRequest(request: lib.UpdateAllRequest) { + if (!this.config.get("host.allow_remote_updates")) { + throw new lib.RequestError("Remote updates are disabled on this machine"); + } + if (!this.config.get("host.allow_plugin_updates")) { + throw new lib.RequestError("Plugin updates are disabled on this machine"); + } + return await lib.handleUpdateAll("@clusterio/host", this.pluginInfos); + } + async handlePluginListRequest(request: lib.PluginListRequest) { return this.pluginInfos.map(pluginInfo => lib.PluginDetails.fromNodeEnvInfo( pluginInfo, diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 4db05ea0..3f78c67b 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -791,6 +791,7 @@ end`.replace(/\r?\n/g, " "); } else { modPack = await this.sendTo("controller", new lib.ModPackGetRequest(modPackId)); } + this.checkModPackVersion(modPack); this.activeModPack = modPack; const mods = await this._host.fetchMods(modPack.mods.values(), modPack.factorioVersion); @@ -842,6 +843,25 @@ end`.replace(/\r?\n/g, " "); await fs.writeFile(this.path("mods", "mod-settings.dat"), this.activeModPack.toModSettingsDat()); } + /** + * Check that the mod pack does not target a newer Factorio than the server + * + * The mod pack's Factorio version is written into mod-settings.dat and + * Factorio refuses to read it if it is newer than the game. It then + * loads default settings instead, which can corrupt the save. + * + * @throws {lib.RequestError} if the mod pack version is higher than the server version. + */ + checkModPackVersion(modPack: lib.ModPack) { + const serverVersion = this.server.version; + if (serverVersion && modPack.integerFactorioVersion > lib.integerFullVersion(serverVersion)) { + throw new lib.RequestError( + `Mod pack ${modPack.name} is for Factorio ${modPack.factorioVersion} ` + + `which is newer than the Factorio ${serverVersion} this instance runs` + ); + } + } + /** * Prepare instance for starting * diff --git a/packages/lib/src/api.ts b/packages/lib/src/api.ts index 43289178..86a25393 100644 --- a/packages/lib/src/api.ts +++ b/packages/lib/src/api.ts @@ -4,7 +4,8 @@ export type PluginWebApi = { enabled: boolean; loaded: boolean; web: { - main: string; + /** Path to the web bundle, not present if the plugin has no web build. */ + main?: string; error?: string; }; /** diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index 6ca54746..769079a0 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -468,7 +468,7 @@ export class Config< if (!this.filepath) { throw new Error("Cannot save config which has no filepath"); } - await safeOutputFile(this.filepath, JSON.stringify(this, null, "\t")); + await safeOutputFile(this.filepath, JSON.stringify(this, null, "\t"), { mode: 0o600 }); this.dirty = false; } diff --git a/packages/lib/src/data/messages_plugin.ts b/packages/lib/src/data/messages_plugin.ts index 2bce3ab1..25eb0045 100644 --- a/packages/lib/src/data/messages_plugin.ts +++ b/packages/lib/src/data/messages_plugin.ts @@ -1,6 +1,8 @@ import { Type, Static } from "@sinclair/typebox"; import { jsonArray } from "./composites.js"; import type { PluginNodeEnvInfo } from "../plugin.js"; +import { Address, MessageRequest } from "./messages_core.js"; +import type { IUser } from "./UserDetails.js"; /* This is similar to other plugin definitions but with lots removed */ export class PluginDetails { @@ -91,3 +93,14 @@ export class PluginInstallRequest { return this.pluginPackage; } } + +export class UpdateAllRequest { + declare ["constructor"]: typeof UpdateAllRequest; + static type = "request" as const; + static src = "control" as const; + static dst = ["controller", "host"] as const; + static permission(user: IUser, message: MessageRequest) { + user.checkPermission(message.dst.type === Address.host ? "core.host.update" : "core.controller.update"); + user.checkPermission("core.plugin.update"); + } +} diff --git a/packages/lib/src/file_ops.ts b/packages/lib/src/file_ops.ts index b8899406..908f57a3 100644 --- a/packages/lib/src/file_ops.ts +++ b/packages/lib/src/file_ops.ts @@ -168,6 +168,10 @@ export async function safeOutputFile( throw err; } } + if (options?.mode !== undefined) { + // writeFile only applies mode when creating, a leftover temporary keeps its old mode. + await fs.chmod(temporary, options.mode); + } await fs.rename(temporary, file); } diff --git a/packages/lib/src/link/connectors.ts b/packages/lib/src/link/connectors.ts index 8ce84f8f..d14b89e3 100644 --- a/packages/lib/src/link/connectors.ts +++ b/packages/lib/src/link/connectors.ts @@ -713,12 +713,14 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector | EventClass) plugin.PluginListRequest, plugin.PluginUpdateRequest, plugin.PluginInstallRequest, + plugin.UpdateAllRequest, ]; diff --git a/packages/lib/src/plugin.ts b/packages/lib/src/plugin.ts index f406e12f..d1cfd359 100644 --- a/packages/lib/src/plugin.ts +++ b/packages/lib/src/plugin.ts @@ -40,6 +40,23 @@ export type PluginDeclaration = { routes?: string[]; } +/** + * Check if a plugin is expected to ship a web build. + * + * Mirrors the rule used by the create tool: a web build is generated when the + * plugin has a web or controller entrypoint or defines config fields. + */ +export function pluginNeedsWebBuild(info: PluginDeclaration) { + return Boolean( + info.webEntrypoint + || info.controllerEntrypoint + || info.controllerConfigFields + || info.hostConfigFields + || info.instanceConfigFields + || info.controlConfigFields + ); +} + export type PluginNodeEnvInfo = PluginDeclaration & { /** * Path to the folder with the static files that should be hosted on the web diff --git a/packages/lib/src/rce_ops.ts b/packages/lib/src/rce_ops.ts index d42ed3e7..1946a5eb 100644 --- a/packages/lib/src/rce_ops.ts +++ b/packages/lib/src/rce_ops.ts @@ -19,8 +19,8 @@ async function logExec(cmd: string) { } } -export async function updatePackage(name: string) { - return logExec(`npm update --save ${name}`); +export async function updatePackage(...names: string[]) { + return logExec(`npm update --save ${names.join(" ")}`); } export async function installPackage(name: string) { @@ -35,6 +35,13 @@ export async function handlePluginUpdate(pluginName: string, pluginInfos: Plugin return await updatePackage(pluginName); } +export async function handleUpdateAll(corePackage: string, pluginInfos: PluginNodeEnvInfo[]) { + const pluginPackages = pluginInfos + .map(plugin => plugin.npmPackage) + .filter((name): name is string => name !== undefined); + return await updatePackage(corePackage, ...pluginPackages); +} + export async function handlePluginInstall(pluginName: string) { if (pluginName.length > 214 || /[^a-zA-Z0-9\-_.+@\/]/.test(pluginName)) { // https://docs.npmjs.com/cli/v11/configuring-npm/package-json#name diff --git a/packages/web_ui/src/bootstrap.tsx b/packages/web_ui/src/bootstrap.tsx index 710dc76a..8c38c456 100644 --- a/packages/web_ui/src/bootstrap.tsx +++ b/packages/web_ui/src/bootstrap.tsx @@ -50,6 +50,9 @@ async function loadPluginInfos(): Promise<[lib.PluginWebpackEnvInfo[], string]> logger.error(`Failed to load plugin ${meta.name}: ${meta.web.error}`); continue; } + if (!meta.web.main) { + continue; + } try { await loadScript(`${staticRoot}${meta.web.main}`); let container: any = (window as { [key: string]: any })[`plugin_${meta.name}`]; diff --git a/packages/web_ui/src/components/NpmButton.tsx b/packages/web_ui/src/components/NpmButton.tsx index 1a5a83e2..e543cb43 100644 --- a/packages/web_ui/src/components/NpmButton.tsx +++ b/packages/web_ui/src/components/NpmButton.tsx @@ -27,6 +27,17 @@ function NpmRemoteUpdate({ setApplyAction, form, target }: NpmActionProps) { ; } +function NpmUpdateAll({ setApplyAction, form, target }: NpmActionProps) { + const control = useContext(ControlContext); + + setApplyAction(async () => { + await control.sendTo(target, new lib.UpdateAllRequest()); + }); + + return <> + ; +} + function NpmPluginUpdate({ setApplyAction, form, target }: NpmActionProps) { const control = useContext(ControlContext); const [plugins, setPlugins] = useState([]); @@ -128,6 +139,8 @@ export function NpmButton(props: { } }, [hostId]); + const coreUpdatePermission = `core.${props.target === "controller" ? "controller" : "host"}.update` as const; + function onValuesChange({ action } : { action?: string }) { if (action) { setFormAction(action); @@ -175,10 +188,17 @@ export function NpmButton(props: {
- {account.hasPermission(`core.${props.target === "controller" ? "controller" : "host"}.update`) + {account.hasPermission(coreUpdatePermission) ? Update Clusterio : undefined} + {account.hasPermission(coreUpdatePermission) && account.hasPermission("core.plugin.update") + ? + Update All + : undefined} {account.hasPermission("core.plugin.update") ? Update Plugin @@ -198,6 +218,8 @@ export function NpmButton(props: { } {formAction === "remote_update" ? : undefined} + {formAction === "update_all" + ? : undefined} {formAction === "plugin_update" ? : undefined} {formAction === "plugin_install" diff --git a/packages/web_ui/src/components/PluginViewPage.tsx b/packages/web_ui/src/components/PluginViewPage.tsx index d81220e1..233f549e 100644 --- a/packages/web_ui/src/components/PluginViewPage.tsx +++ b/packages/web_ui/src/components/PluginViewPage.tsx @@ -46,11 +46,14 @@ export default function PluginViewPage() { } const pluginError = pluginMeta.web.error ?? pluginInfo?.error; + const hasWebModule = Boolean(pluginMeta.web.main || pluginMeta.web.error); if (!loadedPlugin) { return {pluginMeta.version} - No + + {hasWebModule ? "No" : "No web module"} + {pluginMeta.loaded ? "Yes" : "No"} {pluginMeta.enabled ? "Yes" : "No"} diff --git a/packages/web_ui/src/components/PluginsPage.tsx b/packages/web_ui/src/components/PluginsPage.tsx index f10d0585..144b1155 100644 --- a/packages/web_ui/src/components/PluginsPage.tsx +++ b/packages/web_ui/src/components/PluginsPage.tsx @@ -86,6 +86,9 @@ export default function PluginsPage() { if (!plugin.meta.enabled) { return <> Disabled on controller; } + if (!plugin.meta.web.main && !plugin.meta.web.error) { + return plugin.meta.version; + } if (!plugin.meta.web.error && !control.pluginInfos.has(plugin.meta.name)) { return <> Reload page to load; } @@ -103,7 +106,15 @@ export default function PluginsPage() { { title: "Loaded", key: "loaded", - render: (_, plugin) => (plugin.package ? "Yes" : null), + render: (_, plugin) => { + if (plugin.package) { + return "Yes"; + } + if (!plugin.meta.web.main && !plugin.meta.web.error) { + return "No web module"; + } + return null; + }, sorter: (a, b) => Number(Boolean(a.package)) - Number(Boolean(b.package)), sortOrder: tableState.sortOrder("loaded"), responsive: ["sm"], diff --git a/plugins/inventory_sync/README.md b/plugins/inventory_sync/README.md index c39a8283..ecf5d615 100644 --- a/plugins/inventory_sync/README.md +++ b/plugins/inventory_sync/README.md @@ -32,8 +32,8 @@ The greatly simplified data flow is as follows: 3. If the player inventory was previously uploaded but is not the same then the player inventory is deleted and the player is turned into a spectator. 4. Otherwise the player inventory is kept. -5. Scenario script asks for the player inventory from the controller. -6. Controller sends it to the scenario in a stream of chunks. +5. Scenario script asks for the player inventory from the controller, including its current recipe notification state. +6. Controller sends it to the scenario in a stream of chunks, with only the recipe notifications that differ from the current state. 7. Scenario displays and updates a progress bar as each chunk is received. 8. Once all chunks have loaded the player's synced inventory is recreated from the data and the player can start playing. diff --git a/plugins/inventory_sync/instance.ts b/plugins/inventory_sync/instance.ts index bc05d896..831c7524 100644 --- a/plugins/inventory_sync/instance.ts +++ b/plugins/inventory_sync/instance.ts @@ -1,3 +1,5 @@ +import util from "util"; +import zlib from "zlib"; import * as lib from "@clusterio/lib"; import { BaseInstancePlugin } from "@clusterio/host"; import { @@ -8,6 +10,11 @@ type IpcPlayerName = { player_name: string } +type IpcDownloadRequest = { + player_name: string, + recipe_notifications?: string, +} + type IpcAcquireResponse = { player_name: string, status: string, @@ -26,6 +33,49 @@ function chunkify(chunkSize: number, string: string): string[] { return string.match(new RegExp(`.{1,${chunkSize}}`, "g")) || []; } +const inflate = util.promisify(zlib.inflate); +const deflate = util.promisify(zlib.deflate); + +/** + * Decode a string produced by helpers.encode_string in Factorio + */ +async function decodeLuaString(encoded: string): Promise { + return (await inflate(Buffer.from(encoded, "base64"))).toString("utf8"); +} + +/** + * Encode a string so that helpers.decode_string in Factorio can read it + */ +async function encodeLuaString(text: string): Promise { + return (await deflate(Buffer.from(text, "utf8"))).toString("base64"); +} + +export type RecipeNotificationDelta = { + add?: string[], + remove?: string[], +} + +/** + * Compute the changes needed to turn the current cleared recipe list into the stored one + * @param stored - Cleared recipe names stored on the controller + * @param current - Cleared recipe names the instance currently has + * @returns names to add to and remove from the current list + */ +export function recipeNotificationDelta(stored: string[], current: string[]): RecipeNotificationDelta { + const storedSet = new Set(stored); + const currentSet = new Set(current); + const delta: RecipeNotificationDelta = {}; + const add = stored.filter(name => !currentSet.has(name)); + const remove = current.filter(name => !storedSet.has(name)); + if (add.length) { + delta.add = add; + } + if (remove.length) { + delta.remove = remove; + } + return delta; +} + export class InstancePlugin extends BaseInstancePlugin { playersToRelease!: Set; disconnecting!: boolean; @@ -55,7 +105,7 @@ export class InstancePlugin extends BaseInstancePlugin { ); this.instance.server.on( "ipc-inventory_sync_download", - (request: IpcPlayerName) => this.handleDownload(request).catch( + (request: IpcDownloadRequest) => this.handleDownload(request).catch( err => this.logger.error(`Error handling ipc-inventory_sync_download:\n${err.stack}`) ), ); @@ -163,7 +213,27 @@ export class InstancePlugin extends BaseInstancePlugin { ); } - async handleDownload(request: IpcPlayerName) { + /** + * Replace the stored recipe notifications with the difference from the + * current state on the instance to reduce the amount of data sent to Lua. + */ + async applyRecipeNotificationDelta(playerData: IpcPlayerData, current?: string) { + if (!playerData.recipe_notifications) { + return; + } + try { + const delta = recipeNotificationDelta( + JSON.parse(await decodeLuaString(playerData.recipe_notifications)), + current ? JSON.parse(await decodeLuaString(current)) : [], + ); + playerData.recipe_notifications = await encodeLuaString(JSON.stringify(delta)); + } catch (err: any) { + this.logger.warn(`Dropping invalid recipe notifications for ${playerData.name}:\n${err.stack}`); + delete playerData.recipe_notifications; + } + } + + async handleDownload(request: IpcDownloadRequest) { const playerName = request.player_name; this.logger.verbose(`Downloading ${playerName}`); @@ -177,6 +247,8 @@ export class InstancePlugin extends BaseInstancePlugin { return; } + await this.applyRecipeNotificationDelta(response.playerData, request.recipe_notifications); + const chunkSize = this.instance.config.get("inventory_sync.rcon_chunk_size"); const chunks = chunkify(chunkSize, JSON.stringify(response.playerData)); this.logger.verbose(`Sending inventory for ${playerName} in ${chunks.length} chunks`); diff --git a/plugins/inventory_sync/module/inventory_sync.lua b/plugins/inventory_sync/module/inventory_sync.lua index bf56a57e..c6804251 100644 --- a/plugins/inventory_sync/module/inventory_sync.lua +++ b/plugins/inventory_sync/module/inventory_sync.lua @@ -17,6 +17,7 @@ local progress_dialog = require("modules/inventory_sync/gui/progress_dialog") local dialog_failed_download = require("modules/inventory_sync/gui/dialog_failed_download") local v2_remote_controller = compat.version_ge("2.0.0") +local recipe_notifications_api = compat.version_ge("2.0.67") -- Returns true if the player is currently in a cutscene local function is_in_cutscene(player) @@ -441,8 +442,15 @@ function inventory_sync.initiate_inventory_download(player, player_record, gener local script_data = get_script_data() script_data.active_downloads[player.name] = record + -- The plugin only sends back what differs from the current notification state + local recipe_notifications + if recipe_notifications_api then + recipe_notifications = serialize.serialize_crafting_notifications(player) + end + clusterio_api.send_json("inventory_sync_download", { - player_name = player.name + player_name = player.name, + recipe_notifications = recipe_notifications, }) -- If this is a synced player turn them into a spectator while the diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index 6d62a6aa..0ef6705e 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -533,62 +533,55 @@ function serialize.serialize_crafting_notifications(player, failed) recipes[recipe.name] = nil end - -- Convert to a list of names - local recipe_names = failed or {} -- Includes previous recipes if any - local index = #recipe_names + 1 + -- Convert to a list of names, including previously failed recipes if any + local recipe_names = {} + for _, name in ipairs(failed or {}) do + recipe_names[#recipe_names + 1] = name + end for name in pairs(recipes) do - recipe_names[index] = name - index = index + 1 + recipe_names[#recipe_names + 1] = name end return helpers.encode_string(helpers.table_to_json(recipe_names)) end +--- @class CraftingNotificationDelta +--- @field add string[]? Recipes cleared elsewhere that still have a notification here +--- @field remove string[]? Recipes with a notification elsewhere that are cleared here + --- @param player LuaPlayer ---- @param serialized string +--- @param serialized string Encoded delta against the state sent with the download request --- @return string[]? failed Recipe names that do not exist function serialize.deserialize_crafting_notifications(player, serialized) - -- Decode the recipe name strings - local recipe_names = helpers.json_to_table(assert(helpers.decode_string(serialized))) - assert(type(recipe_names) == "table", "wrong type decoded from json_to_table") - local clear_notification = player.clear_recipe_notification - local add_notification = player.add_recipe_notification - local notifications = {} - local failed = {} + --- @type CraftingNotificationDelta + local delta = helpers.json_to_table(assert(helpers.decode_string(serialized))) + assert(type(delta) == "table", "wrong type decoded from json_to_table") - -- Assume all recipes need a notification + -- Recipes which can have a notification on this server + local enabled = {} for _, recipe in pairs(player.force.recipes) do if recipe.enabled and not recipe.hidden then - notifications[recipe.name] = true + enabled[recipe.name] = true end end - -- Remove recipes the player has cleared - for _, recipe_name in pairs(recipe_names) do - if notifications[recipe_name] then - notifications[recipe_name] = false + -- Clear notifications the player has cleared elsewhere, this means "add seen notification" + local failed = {} + for _, recipe_name in pairs(delta.add or {}) do + if enabled[recipe_name] then + player.clear_recipe_notification(recipe_name) else - -- Recipe does not exist on this server - failed[#failed+1] = recipe_name + failed[#failed + 1] = recipe_name -- Recipe does not exist on this server end end - -- Clear ones which are ones which are not required - for _, recipe in pairs(player.get_recipe_notifications()) do - if notifications[recipe.name] == false then - clear_notification(recipe.name) - end - notifications[recipe.name] = nil -- Prevents double add below - end - - -- Add any which the player did not have - for recipe_name, should_add in pairs(notifications) do - if should_add then - add_notification(recipe_name) + -- Add notifications the player has elsewhere, this means "remove seen notification" + for _, recipe_name in pairs(delta.remove or {}) do + if enabled[recipe_name] then + player.add_recipe_notification(recipe_name) end end - -- Return failed recipes if any return next(failed) and failed or nil end diff --git a/plugins/inventory_sync/test/instance.js b/plugins/inventory_sync/test/instance.js new file mode 100644 index 00000000..020f5979 --- /dev/null +++ b/plugins/inventory_sync/test/instance.js @@ -0,0 +1,130 @@ +import assert from "node:assert/strict"; +import util from "node:util"; +import zlib from "node:zlib"; +import * as lib from "@clusterio/lib"; + +import * as mock from "../../../test/mock.js"; +import { recipeNotificationDelta, InstancePlugin } from "../dist/node/instance.js"; +import { DownloadResponse } from "../dist/node/messages.js"; +import { plugin as info } from "../dist/node/index.js"; + +const inflate = util.promisify(zlib.inflate); +const deflate = util.promisify(zlib.deflate); + +// Matches helpers.encode_string in Factorio +async function encode(value) { + return (await deflate(Buffer.from(JSON.stringify(value)))).toString("base64"); +} +async function decode(encoded) { + return JSON.parse((await inflate(Buffer.from(encoded, "base64"))).toString()); +} + +describe("inventory_sync", function() { + describe("recipeNotificationDelta()", function() { + it("should be empty when the lists match", function() { + assert.deepEqual(recipeNotificationDelta(["a", "b"], ["b", "a"]), {}); + }); + it("should add stored names missing from current", function() { + assert.deepEqual(recipeNotificationDelta(["a", "b", "c"], ["a"]), { add: ["b", "c"] }); + }); + it("should remove current names missing from stored", function() { + assert.deepEqual(recipeNotificationDelta(["a"], ["a", "b"]), { remove: ["b"] }); + }); + it("should handle both directions at once", function() { + assert.deepEqual(recipeNotificationDelta(["a", "b"], ["b", "c"]), { add: ["a"], remove: ["c"] }); + }); + it("should keep duplicates in stored out of the current list", function() { + assert.deepEqual(recipeNotificationDelta(["a", "a"], ["a"]), {}); + }); + }); + + describe("InstancePlugin.applyRecipeNotificationDelta()", function() { + const warnings = []; + const plugin = { logger: { warn: msg => warnings.push(msg) } }; + const apply = InstancePlugin.prototype.applyRecipeNotificationDelta.bind(plugin); + + it("should replace the stored list with an encoded delta", async function() { + const playerData = { name: "test", recipe_notifications: await encode(["a", "b"]) }; + await apply(playerData, await encode(["b", "c"])); + assert.deepEqual(await decode(playerData.recipe_notifications), { add: ["a"], remove: ["c"] }); + }); + it("should send an empty delta when nothing changed", async function() { + const playerData = { name: "test", recipe_notifications: await encode(["a"]) }; + await apply(playerData, await encode(["a"])); + assert.deepEqual(await decode(playerData.recipe_notifications), {}); + }); + it("should add everything without a current list", async function() { + const playerData = { name: "test", recipe_notifications: await encode(["a"]) }; + await apply(playerData, undefined); + assert.deepEqual(await decode(playerData.recipe_notifications), { add: ["a"] }); + }); + it("should do nothing without stored notifications", async function() { + const playerData = { name: "test" }; + await apply(playerData, await encode(["a"])); + assert.deepEqual(playerData, { name: "test" }); + }); + it("should drop the field on invalid input", async function() { + const playerData = { name: "test", recipe_notifications: "not base64 zlib" }; + await apply(playerData, await encode(["a"])); + assert.equal(playerData.recipe_notifications, undefined); + assert.equal(warnings.length, 1); + }); + }); + + describe("InstancePlugin.handleDownload()", function() { + let instancePlugin; + before(async function() { + try { + lib.registerPluginMessages([info]); + } catch (err) { + // Already registered by the full test suite + } + instancePlugin = await mock.createInstancePlugin(InstancePlugin, info); + instancePlugin.instance.mockConfigEntries.set("inventory_sync.rcon_chunk_size", 100000); + }); + beforeEach(function() { + instancePlugin.instance.server.reset(); + }); + + function respondWith(playerData) { + instancePlugin.instance.connector.once("send", message => { + instancePlugin.instance.connector.emit("message", new lib.MessageResponse( + 1, message.dst, message.src, new DownloadResponse(playerData) + )); + }); + } + function downloadCommand(playerData) { + const json = lib.escapeString(JSON.stringify(playerData)); + return `/sc inventory_sync.download_inventory('test','${json}',1,1)`; + } + + it("should send an empty download when the controller has no data", async function() { + respondWith(undefined); + await instancePlugin.handleDownload({ player_name: "test" }); + assert.deepEqual( + instancePlugin.instance.server.rconCommands, + ["/sc inventory_sync.download_inventory('test',nil,0,0)"] + ); + }); + it("should send everything as a delta when no snapshot was given", async function() { + respondWith({ generation: 1, name: "test", recipe_notifications: await encode(["a"]) }); + await instancePlugin.handleDownload({ player_name: "test" }); + const expected = { generation: 1, name: "test", recipe_notifications: await encode({ add: ["a"] }) }; + assert.deepEqual(instancePlugin.instance.server.rconCommands, [downloadCommand(expected)]); + }); + it("should send a delta when a snapshot was given", async function() { + respondWith({ generation: 1, name: "test", recipe_notifications: await encode(["a", "b"]) }); + await instancePlugin.handleDownload({ player_name: "test", recipe_notifications: await encode(["b"]) }); + const expected = { generation: 1, name: "test", recipe_notifications: await encode({ add: ["a"] }) }; + assert.deepEqual(instancePlugin.instance.server.rconCommands, [downloadCommand(expected)]); + }); + it("should be invoked by the download ipc", async function() { + respondWith(undefined); + instancePlugin.instance.server.emit("ipc-inventory_sync_download", { player_name: "test" }); + for (let i = 0; i < 100 && !instancePlugin.instance.server.rconCommands.length; i++) { + await new Promise(resolve => setImmediate(resolve)); + } + assert.equal(instancePlugin.instance.server.rconCommands.length, 1); + }); + }); +}); diff --git a/test/controller/routes.js b/test/controller/routes.js index 49c63783..7f122047 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -102,7 +102,7 @@ describe("controller/src/routes", function() { }); it("should report missing manifest and entrypoint", async function() { controller.pluginInfos = [ - { name: "foo", version: "1.0.0", npmPackage: "foo" }, + { name: "foo", version: "1.0.0", npmPackage: "foo", webEntrypoint: "dist/web/index.js" }, { name: "bar", version: "1.0.0", npmPackage: "bar", manifest: {} }, { name: "baz", version: "1.0.0", npmPackage: "baz", manifest: { "baz.js": "remoteEntry.js" } }, ]; @@ -116,6 +116,19 @@ describe("controller/src/routes", function() { assert.equal(data[1].loaded, false); assert.equal(data[1].enabled, false); }); + it("should not report a missing manifest for plugins without a web build", async function() { + controller.pluginInfos = [ + { name: "foo", version: "1.0.0", npmPackage: "foo", hostEntrypoint: "dist/node/host.js" }, + { name: "bar", version: "1.0.0", npmPackage: "bar", controllerEntrypoint: "dist/node/controller.js" }, + { name: "baz", version: "1.0.0", npmPackage: "baz", instanceConfigFields: {} }, + ]; + let response = await fetch(endpoint); + assert.equal(response.status, 200); + let data = await response.json(); + assert.deepEqual(data[0].web, {}); + assert.equal(data[1].web.error, "Missing dist/web/manifest.json"); + assert.equal(data[2].web.error, "Missing dist/web/manifest.json"); + }); }); describe("/api/cluster-name", function() { let endpoint; diff --git a/test/host/Instance.js b/test/host/Instance.js index 52623700..73e11c1a 100644 --- a/test/host/Instance.js +++ b/test/host/Instance.js @@ -38,6 +38,34 @@ describe("class Instance", function() { }); }); + describe(".checkModPackVersion()", function() { + function modPack(factorioVersion) { + return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion }); + } + + it("should accept a mod pack for an older or equal Factorio version", function() { + instance.server.version = "1.1.91"; + instance.checkModPackVersion(modPack("1.1")); + instance.checkModPackVersion(modPack("1.1.90")); + instance.checkModPackVersion(modPack("1.1.91")); + }); + + it("should reject a mod pack for a newer Factorio version", function() { + instance.server.version = "1.1.91"; + assert.throws( + () => instance.checkModPackVersion(modPack("1.1.110")), + new lib.RequestError( + "Mod pack pack is for Factorio 1.1.110 which is newer than the Factorio 1.1.91 this instance runs" + ) + ); + assert.throws(() => instance.checkModPackVersion(modPack("2.0")), lib.RequestError); + }); + + it("should skip the check when the server version is unknown", function() { + instance.checkModPackVersion(modPack("2.0")); + }); + }); + describe("._recordPlayerJoin()", function() { it("should add player to playersOnline", function() { instance._recordPlayerJoin("player"); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 7ef99463..b2388b5c 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -521,6 +521,9 @@ describe("Integration of Clusterio", function() { it("runs", async function() { await execCtl("controller update"); }); + it("accepts --all", async function() { + await execCtl("controller update --all"); + }); it("accepts --restart", async function() { // We cannot restart the controller, so we check for controller error instead await assert.rejects( @@ -603,6 +606,9 @@ describe("Integration of Clusterio", function() { it("runs", async function() { await execCtl("host update 4"); }); + it("accepts --all", async function() { + await execCtl("host update 4 --all"); + }); it("accepts --restart", async function() { // We cannot restart the host, so we check for host error instead await assert.rejects( diff --git a/test/lib/config/classes.js b/test/lib/config/classes.js index 17ed8100..c953ea13 100644 --- a/test/lib/config/classes.js +++ b/test/lib/config/classes.js @@ -395,6 +395,15 @@ describe("lib/config/classes", function() { const json = JSON.parse(await fs.readFile(filepath, "utf8")); assert.deepEqual(json, testInstance.toJSON()); }); + it("should not be readable by other users", async function() { + if (process.platform === "win32") { + this.skip(); + } + const testInstance = new TestConfig("local", { "alpha.foo": "a" }, filepath); + testInstance.set("alpha.foo", "b"); // Sets the dirty flag + await testInstance.save(); + assert.equal((await fs.stat(filepath)).mode & 0o777, 0o600); + }); it("should clear the dirty flag after saving", async function() { const testInstance = new TestConfig("local", { "alpha.foo": "a" }, filepath); testInstance.set("alpha.foo", "b"); // Sets the dirty flag diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index 036544a4..2f4993e7 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -100,6 +100,16 @@ describe("lib/file_ops", function() { await assert.rejects(fs.access(target.replace(".txt", ".tmp.txt")), "temporary was left behind"); assert.equal(await fs.readFile(target, "utf8"), "current"); }); + it("should apply mode to a leftover temporary file", async function() { + if (process.platform === "win32") { + this.skip(); + } + let target = path.join(baseDir, "safe", "mode.txt"); + await fs.writeFile(target.replace(".txt", ".tmp.txt"), "stale", { mode: 0o644 }); + await lib.safeOutputFile(target, "private", { mode: 0o600 }); + assert.equal((await fs.stat(target)).mode & 0o777, 0o600); + assert.equal(await fs.readFile(target, "utf8"), "private"); + }); it("should handle creating file in current working directory", async function() { let target = "temporary-file-made-to-test-cwd.txt"; try { diff --git a/test/lib/plugin.js b/test/lib/plugin.js new file mode 100644 index 00000000..176a3bf6 --- /dev/null +++ b/test/lib/plugin.js @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; + +import * as lib from "@clusterio/lib"; + + +describe("lib/plugin", function() { + describe("pluginNeedsWebBuild()", function() { + it("should be false for plugins with only host, instance or ctl code", function() { + assert.equal(lib.pluginNeedsWebBuild({ name: "foo", title: "Foo" }), false); + assert.equal(lib.pluginNeedsWebBuild({ + name: "foo", title: "Foo", hostEntrypoint: "host.js", instanceEntrypoint: "instance.js", + ctlEntrypoint: "ctl.js", messages: [], + }), false); + }); + it("should be true for plugins with web or controller code or config fields", function() { + assert.equal(lib.pluginNeedsWebBuild({ name: "foo", title: "Foo", webEntrypoint: "web.js" }), true); + assert.equal(lib.pluginNeedsWebBuild({ + name: "foo", title: "Foo", controllerEntrypoint: "controller.js", + }), true); + for (const field of [ + "controllerConfigFields", "hostConfigFields", "instanceConfigFields", "controlConfigFields", + ]) { + assert.equal(lib.pluginNeedsWebBuild({ name: "foo", title: "Foo", [field]: {} }), true, field); + } + }); + }); +}); diff --git a/test/lib/rce_ops.js b/test/lib/rce_ops.js index dc3c56d3..8ba32661 100644 --- a/test/lib/rce_ops.js +++ b/test/lib/rce_ops.js @@ -28,6 +28,20 @@ describe("rce_ops", function() { }); }); + describe("handleUpdateAll", function() { + it("updates the core package and all npm plugins in one command", async function() { + const _audit = lib.logger.audit; + const messages = []; + lib.logger.audit = (msg) => { messages.push(msg); }; + try { + await lib.handleUpdateAll("core", [{ npmPackage: "foo" }, { name: "local" }, { npmPackage: "bar" }]); + } finally { + lib.logger.audit = _audit; + } + assert.deepEqual(messages, ["RCE | npm update --save core foo bar"]); + }); + }); + describe("handlePluginInstall", function() { const _fetch = global.fetch; before(function() { diff --git a/test/messages/plugin.test.js b/test/messages/plugin.test.js index 621da7df..45e55048 100644 --- a/test/messages/plugin.test.js +++ b/test/messages/plugin.test.js @@ -133,6 +133,61 @@ describe("messages/plugin", function() { }); }); + describe("UpdateAllRequest", function() { + function checkedPermissions(dst) { + const permissions = []; + const user = { checkPermission(permission) { permissions.push(permission); } }; + lib.UpdateAllRequest.permission(user, { dst: lib.Address.fromShorthand(dst) }); + return permissions; + } + it("requires controller and plugin update permissions on the controller", function() { + assert.deepEqual(checkedPermissions("controller"), ["core.controller.update", "core.plugin.update"]); + }); + it("requires host and plugin update permissions on a host", function() { + assert.deepEqual(checkedPermissions({ hostId: 1 }), ["core.host.update", "core.plugin.update"]); + }); + it("runs on the controller", async function() { + controller.pluginInfos = [{ npmPackage: "foo" }]; + controller.config.set("controller.allow_remote_updates", true); + controller.config.set("controller.allow_plugin_updates", true); + await controlConnection.handleUpdateAllRequest(new lib.UpdateAllRequest()); + }); + it("rejects if remote updates are disabled on the controller", async function() { + controller.config.set("controller.allow_remote_updates", false); + await assert.rejects( + controlConnection.handleUpdateAllRequest(new lib.UpdateAllRequest()), + /Remote updates are disabled on this machine/ + ); + }); + it("rejects if plugin updates are disabled on the controller", async function() { + controller.config.set("controller.allow_plugin_updates", false); + await assert.rejects( + controlConnection.handleUpdateAllRequest(new lib.UpdateAllRequest()), + /Plugin updates are disabled on this machine/ + ); + }); + it("runs on a host", async function() { + host.pluginInfos = [{ npmPackage: "foo" }]; + host.config.set("host.allow_remote_updates", true); + host.config.set("host.allow_plugin_updates", true); + await host.handleUpdateAllRequest(new lib.UpdateAllRequest()); + }); + it("rejects if remote updates are disabled on the host", async function() { + host.config.set("host.allow_remote_updates", false); + await assert.rejects( + host.handleUpdateAllRequest(new lib.UpdateAllRequest()), + /Remote updates are disabled on this machine/ + ); + }); + it("rejects if plugin updates are disabled on the host", async function() { + host.config.set("host.allow_plugin_updates", false); + await assert.rejects( + host.handleUpdateAllRequest(new lib.UpdateAllRequest()), + /Plugin updates are disabled on this machine/ + ); + }); + }); + describe("PluginInstallRequest", function() { const _fetch = global.fetch; before(function() {