diff --git a/.gitignore b/.gitignore index d93623fb..56c8fbd4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,6 @@ dist/ node_modules Factorio* -factorio* /factorio /instances /sharedMods diff --git a/docs/configuration.md b/docs/configuration.md index 86d1a647..f78c609f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -109,6 +109,14 @@ This should be less than both the timeout and collection interval configured for Defaults to 8. +### master.stream_timeout + +Timeout in seconds for proxy streams through the master that has been created but is not yet sending data to be automatically closed. +Streams that have started sending data before this timeout is not affected. + +Defaults to 15. + + ### master.default_role_id Role to automatically grant to new users. diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 20243a75..f691fe08 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -25,6 +25,8 @@ const libSharedCommands = require("@clusterio/lib/shared_commands"); const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging"); const libLoggingUtils = require("@clusterio/lib/logging_utils"); const libHelpers = require("@clusterio/lib/helpers"); +const libFactorio = require("@clusterio/lib/factorio"); +const libFileOps = require("@clusterio/lib/file_ops"); function print(...content) { @@ -314,15 +316,175 @@ instanceCommands.add(new libCommand.Command({ }, })); +async function loadMapSettings(args) { + let seed = args.seed !== undefined ? args.seed : null; + let mapGenSettings = null; + let mapSettings = null; + if (args.mapExchangeString) { + let parsed = libFactorio.readMapExchangeString(args.mapExchangeString); + mapGenSettings = parsed.map_gen_settings; + mapSettings = parsed.map_settings; + } + if (args.mapGenSettings) { + mapGenSettings = JSON.parse(await fs.readFile(args.mapGenSettings)); + } + if (args.mapSettings) { + mapSettings = JSON.parse(await fs.readFile(args.mapSettings)); + } + + return { + seed, + mapGenSettings, + mapSettings, + }; +} + instanceCommands.add(new libCommand.Command({ - definition: ["create-save ", "Create a new save on an instance", (yargs) => { - yargs.positional("instance", { describe: "Instance to create on", type: "string" }); + definition: ["list-saves ", "list saves on an instance", (yargs) => { + yargs.positional("instance", { describe: "Instance to list saves on", type: "string" }); }], handler: async function(args, control) { let instanceId = await libCommand.resolveInstance(control, args.instance); + let response = await libLink.messages.listSaves.send(control, { instance_id: instanceId }); + for (let entry of response.list) { + entry.mtime = new Date(entry.mtime_ms).toLocaleString(); + delete entry.mtime_ms; + } + print(asTable(response.list)); + }, +})); + +instanceCommands.add(new libCommand.Command({ + definition: ["create-save [name]", "Create a new save on an instance", (yargs) => { + yargs.positional("instance", { describe: "Instance to create on", type: "string" }); + yargs.positional("name", { describe: "Name of save to create.", type: "string", default: "world.zip" }); + yargs.options({ + "seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" }, + "map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" }, + "map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" }, + "map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" }, + }); + }], + handler: async function(args, control) { + let instanceId = await libCommand.resolveInstance(control, args.instance); + let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args); await control.setLogSubscriptions({ instance_ids: [instanceId] }); let response = await libLink.messages.createSave.send(control, { instance_id: instanceId, + name: args.name, + seed, + map_gen_settings: mapGenSettings, + map_settings: mapSettings, + }); + }, +})); + +instanceCommands.add(new libCommand.Command({ + definition: ["upload-save ", "Upload a save to an instance", (yargs) => { + yargs.positional("instance", { describe: "Instance to upload to", type: "string" }); + yargs.positional("filepath", { describe: "Path to save to upload", type: "string" }); + yargs.options({ + "name": { describe: "Name to give save on server", nargs: 1, type: "string" }, + }); + }], + handler: async function(args, control) { + let filename = args.name || path.basename(args.filepath); + if (!filename.endsWith(".zip")) { + throw new libErrors.CommandError("Save name must end with .zip"); + } + // phin doesn't support streaming requests :( + let content = await fs.readFile(args.filepath); + + let instanceId = await libCommand.resolveInstance(control, args.instance); + let url = new URL(control.config.get("control.master_url")); + url.pathname += "api/upload-save"; + url.searchParams.append("instance_id", instanceId); + url.searchParams.append("filename", filename); + + let result = await phin({ + url, method: "POST", + headers: { + "X-Access-Token": control.config.get("control.master_token"), + "Content-Type": "application/zip", + }, + core: { ca: control.tlsCa }, + data: content, + parse: "json", + }); + + for (let error of result.body.errors || []) { + logger.error(error); + } + + for (let requestError of result.body.request_errors || []) { + logger.error(error); + } + + if (result.body.saves && result.body.saves.length) { + logger.info(`Successfully uploaded as ${result.body.saves[0]}`); + } + + if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + throw new libErrors.CommandError("Uploading save failed"); + } + }, +})); + +instanceCommands.add(new libCommand.Command({ + definition: ["download-save ", "Download a save from an instance", (yargs) => { + yargs.positional("instance", { describe: "Instance to download save from", type: "string" }); + yargs.positional("save", { describe: "Save to download", type: "string" }); + }], + handler: async function(args, control) { + let instanceId = await libCommand.resolveInstance(control, args.instance); + let result = await libLink.messages.downloadSave.send(control, { + instance_id: instanceId, + save: args.save, + }); + + let url = new URL(control.config.get("control.master_url")); + url.pathname += `api/stream/${result.stream_id}`; + let response = await phin({ + url, method: "GET", + core: { ca: control.tlsCa }, + stream: true, + }); + + let stream; + let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip"); + while (true) { + try { + stream = fs.createWriteStream(tempFilename, { flags: "wx" }); + await events.once(stream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(".", tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + response.pipe(stream); + await events.once(stream, "finish"); + + let filename = await libFileOps.findUnusedName(".", args.save, ".zip"); + await fs.rename(tempFilename, filename); + + logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`); + }, +})); + +instanceCommands.add(new libCommand.Command({ + definition: ["delete-save ", "Delete a save from an instance", (yargs) => { + yargs.positional("instance", { describe: "Instance to delete save from", type: "string" }); + yargs.positional("save", { describe: "Save to delete", type: "string" }); + }], + handler: async function(args, control) { + let instanceId = await libCommand.resolveInstance(control, args.instance); + await libLink.messages.deleteSave.send(control, { + instance_id: instanceId, + save: args.save, }); }, })); @@ -364,15 +526,23 @@ instanceCommands.add(new libCommand.Command({ yargs.positional("instance", { describe: "Instance to start", type: "string" }); yargs.positional("scenario", { describe: "Scenario to load", type: "string" }); yargs.options({ + "seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" }, + "map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" }, + "map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" }, + "map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" }, "keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false }, }); }], handler: async function(args, control) { let instanceId = await libCommand.resolveInstance(control, args.instance); await control.setLogSubscriptions({ instance_ids: [instanceId] }); + let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args); let response = await libLink.messages.loadScenario.send(control, { instance_id: instanceId, - scenario: args.scenario || null, + scenario: args.scenario, + seed, + map_gen_settings: mapGenSettings, + map_settings: mapSettings, }); control.keepOpen = args.keepOpen; }, @@ -748,6 +918,10 @@ class Control extends libLink.Link { this.keepOpen = false; } + async instanceUpdateEventHandler() { } + + async saveListUpdateEventHandler() { } + async setLogSubscriptions({ all = false, master = false, diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index 5b5aacc8..043611f9 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -101,6 +101,13 @@ MasterGroup.define({ type: "number", initial_value: 8, }); +MasterGroup.define({ + name: "proxy_stream_timeout", + title: "Proxy Stream Timeout", + description: "Timeout in seconds for proxy streams to start flowing", + type: "number", + initial_value: 15, +}); MasterGroup.define({ name: "default_role_id", title: "Default role", diff --git a/packages/lib/factorio/exchange_string.js b/packages/lib/factorio/exchange_string.js new file mode 100644 index 00000000..52698c0f --- /dev/null +++ b/packages/lib/factorio/exchange_string.js @@ -0,0 +1,393 @@ +"use strict"; +const zlib = require("zlib"); + + +class MapReaderState { + constructor(buf) { + this.pos = 0; + this.buf = buf; + this.last_position = { x: 0, y: 0 }; + } +} + +function readUInt8(state) { + let value = state.buf.readUInt8(state.pos); + state.pos += 1; + return value; +} + +function readBool(state) { + let value = readUInt8(state) !== 0; + return value; +} + +function readInt16(state) { + let value = state.buf.readInt16LE(state.pos); + state.pos += 2; + return value; +} + +function readUInt16(state) { + let value = state.buf.readUInt16LE(state.pos); + state.pos += 2; + return value; +} + +function readInt32(state) { + let value = state.buf.readInt32LE(state.pos); + state.pos += 4; + return value; +} + +function readUInt32(state) { + let value = state.buf.readUInt32LE(state.pos); + state.pos += 4; + return value; +} + +function readUInt32so(state) { + let value = readUInt8(state); + if (value === 0xff) { + return readUInt32(state); + } + + return value; +} + +function readFloat(state) { + let value = state.buf.readFloatLE(state.pos); + state.pos += 4; + return value; +} + +function readDouble(state) { + let value = state.buf.readDoubleLE(state.pos); + state.pos += 8; + return value; +} + +function readString(state) { + let size = readUInt32so(state); + let data = state.buf.slice(state.pos, state.pos + size).toString("utf-8"); + state.pos += size; + return data; +} + +function readOptional(state, readValue) { + let load = readUInt8(state) !== 0; + if (!load) { + return null; + } + return readValue(state); +} + +function readArray(state, readItem) { + let size = readUInt32so(state); + + let array = []; + for (let i = 0; i < size; i++) { + let item = readItem(state); + array.push(item); + } + + return array; +} + +function readDict(state, readKey, readValue) { + let size = readUInt32so(state); + + let mapping = new Map(); + for (let i = 0; i < size; i++) { + let key = readKey(state); + let value = readValue(state); + mapping.set(key, value); + } + + return mapping; +} + +function readVersion(state) { + let major = readUInt16(state); + let minor = readUInt16(state); + let patch = readUInt16(state); + let developer = readUInt16(state); + return [major, minor, patch, developer]; +} + +function readFrequencySizeRichness(state) { + return { + frequency: readFloat(state), + size: readFloat(state), + richness: readFloat(state), + }; +} + +function readAutoplaceSetting(state) { + return { + treat_missing_as_default: readBool(state), + settings: Object.fromEntries(readDict(state, readString, readFrequencySizeRichness)), + }; +} + +function readMapPosition(state) { + let x, y; + let x_diff = readInt16(state) / 256; + if (x_diff === 0x7fff / 256) { + x = readInt32(state) / 256; + y = readInt32(state) / 256; + } else { + let y_diff = readInt16(state) / 256; + x = state.last_position.x + x_diff; + y = state.last_position.y + y_diff; + } + state.last_position.x = x; + state.last_position.x = y; + return { x, y }; +} + +function readBoundingBox(state) { + return { + left_top: readMapPosition(state), + right_bottom: readMapPosition(state), + orientation: { + x: readInt16(state), + y: readInt16(state), + }, + }; +} + +function readCliffSettings(state) { + return { + name: readString(state), + elevation_0: readFloat(state), + elevation_interval: readFloat(state), + richness: readFloat(state), + }; +} + +function readMapGenSettings(state) { + return { + terrain_segmentation: readFloat(state), + water: readFloat(state), + autoplace_controls: Object.fromEntries(readDict(state, readString, readFrequencySizeRichness)), + autoplace_settings: Object.fromEntries(readDict(state, readString, readAutoplaceSetting)), + default_enable_all_autoplace_controls: readBool(state), + seed: readUInt32(state), + width: readUInt32(state), + height: readUInt32(state), + area_to_generate_at_start: readBoundingBox(state), + starting_area: readFloat(state), + peaceful_mode: readBool(state), + starting_points: readArray(state, readMapPosition), + property_expression_names: Object.fromEntries(readDict(state, readString, readString)), + cliff_settings: readCliffSettings(state), + }; +} + +function readPollution(state) { + return { + enabled: readOptional(state, readBool), + diffusion_ratio: readOptional(state, readDouble), + min_to_diffuse: readOptional(state, readDouble), + ageing: readOptional(state, readDouble), + expected_max_per_chunk: readOptional(state, readDouble), + min_to_show_per_chunk: readOptional(state, readDouble), + min_pollution_to_damage_trees: readOptional(state, readDouble), + pollution_with_max_forest_damage: readOptional(state, readDouble), + pollution_per_tree_damage: readOptional(state, readDouble), + pollution_restored_per_tree_damage: readOptional(state, readDouble), + max_pollution_to_restore_trees: readOptional(state, readDouble), + enemy_attack_pollution_consumption_modifier: readOptional(state, readDouble), + }; +} + +function readSteeringValue(state) { + return { + radius: readOptional(state, readDouble), + separation_factor: readOptional(state, readDouble), + separation_force: readOptional(state, readDouble), + force_unit_fuzzy_goto_behavior: readOptional(state, readBool), + }; + +} + +function readSteering(state) { + return { + default: readSteeringValue(state), + moving: readSteeringValue(state), + }; +} + +function readEnemyEvolution(state) { + return { + enabled: readOptional(state, readBool), + time_factor: readOptional(state, readDouble), + destroy_factor: readOptional(state, readDouble), + pollution_factor: readOptional(state, readDouble), + }; +} + +function readEnemyExpansion(state) { + return { + enabled: readOptional(state, readBool), + max_expansion_distance: readOptional(state, readUInt32), + friendly_base_influence_radius: readOptional(state, readUInt32), + enemy_building_influence_radius: readOptional(state, readUInt32), + building_coefficient: readOptional(state, readDouble), + other_base_coefficient: readOptional(state, readDouble), + neighbouring_chunk_coefficient: readOptional(state, readDouble), + neighbouring_base_chunk_coefficient: readOptional(state, readDouble), + max_colliding_tiles_coefficient: readOptional(state, readDouble), + settler_group_min_size: readOptional(state, readUInt32), + settler_group_max_size: readOptional(state, readUInt32), + min_expansion_cooldown: readOptional(state, readUInt32), + max_expansion_cooldown: readOptional(state, readUInt32), + }; +} + +function readUnitGroup(state) { + return { + min_group_gathering_time: readOptional(state, readUInt32), + max_group_gathering_time: readOptional(state, readUInt32), + max_wait_time_for_late_members: readOptional(state, readUInt32), + max_group_radius: readOptional(state, readDouble), + min_group_radius: readOptional(state, readDouble), + max_member_speedup_when_behind: readOptional(state, readDouble), + max_member_slowdown_when_ahead: readOptional(state, readDouble), + max_group_slowdown_factor: readOptional(state, readDouble), + max_group_member_fallback_factor: readOptional(state, readDouble), + member_disown_distance: readOptional(state, readDouble), + tick_tolerance_when_member_arrives: readOptional(state, readUInt32), + max_gathering_unit_groups: readOptional(state, readUInt32), + max_unit_group_size: readOptional(state, readUInt32), + }; +} + +function readPathFinder(state) { + return { + fwd2bwd_ratio: readOptional(state, readInt32), + goal_pressure_ratio: readOptional(state, readDouble), + use_path_cache: readOptional(state, readBool), + max_steps_worked_per_tick: readOptional(state, readDouble), + max_work_done_per_tick: readOptional(state, readUInt32), + short_cache_size: readOptional(state, readUInt32), + long_cache_size: readOptional(state, readUInt32), + short_cache_min_cacheable_distance: readOptional(state, readDouble), + short_cache_min_algo_steps_to_cache: readOptional(state, readUInt32), + long_cache_min_cacheable_distance: readOptional(state, readDouble), + cache_max_connect_to_cache_steps_multiplier: readOptional(state, readUInt32), + cache_accept_path_start_distance_ratio: readOptional(state, readDouble), + cache_accept_path_end_distance_ratio: readOptional(state, readDouble), + negative_cache_accept_path_start_distance_ratio: readOptional(state, readDouble), + negative_cache_accept_path_end_distance_ratio: readOptional(state, readDouble), + cache_path_start_distance_rating_multiplier: readOptional(state, readDouble), + cache_path_end_distance_rating_multiplier: readOptional(state, readDouble), + stale_enemy_with_same_destination_collision_penalty: readOptional(state, readDouble), + ignore_moving_enemy_collision_distance: readOptional(state, readDouble), + enemy_with_different_destination_collision_penalty: readOptional(state, readDouble), + general_entity_collision_penalty: readOptional(state, readDouble), + general_entity_subsequent_collision_penalty: readOptional(state, readDouble), + extended_collision_penalty: readOptional(state, readDouble), + max_clients_to_accept_any_new_request: readOptional(state, readUInt32), + max_clients_to_accept_short_new_request: readOptional(state, readUInt32), + direct_distance_to_consider_short_request: readOptional(state, readUInt32), + short_request_max_steps: readOptional(state, readUInt32), + short_request_ratio: readOptional(state, readDouble), + min_steps_to_check_path_find_termination: readOptional(state, readUInt32), + start_to_goal_cost_multiplier_to_terminate_path_find: readOptional(state, readDouble), + overload_levels: readOptional(state, (p) => readArray(p, readUInt32)), + overload_multipliers: readOptional(state, (p) => readArray(p, readDouble)), + negative_path_cache_delay_interval: readOptional(state, readUInt32), + }; +} + +function readDifficultySettings(state) { + return { + recipe_difficulty: readUInt8(state), + technology_difficulty: readUInt8(state), + technology_price_multiplier: readDouble(state), + research_queue_setting: ["always", "after-victory", "never"][readUInt8(state)], + }; +} + +function readMapSettings(state) { + return { + pollution: readPollution(state), + steering: readSteering(state), + enemy_evolution: readEnemyEvolution(state), + enemy_expansion: readEnemyExpansion(state), + unit_group: readUnitGroup(state), + path_finder: readPathFinder(state), + max_failed_behavior_count: readUInt32(state), + difficulty_settings: readDifficultySettings(state), + }; +} + +/** + * @typedef {object} module:lib/factorio.MapExchangeResult + * @property {array} version - + * Version of Factorio the string was created with. + * @property {object} map_gen_settings - + * Decoded map generator settings in the format the --map-gen-settings + * command line option to Factorio expect. + * @property {object} map_settings - + * Decoded map settings in the format the --map-settings command line option + * to Factorio expects. + * @property {number} checksum - checksum for the exchange string. + */ + +/** + * Parse a Map Exchange String + * + * Reads and decodes the data in the given map exchange string and returns + * data structures that can be fed into the Factorio server when creating a + * save to set the map gen settings and the map settings for the save. + * + * @param {string} exchangeString - Max Exchange String to parse. + * @returns {module:lib/factorio.MapExchangeResult} Parsed result. + * @memberof module:lib/factorio + */ +function readMapExchangeString(exchangeString) { + exchangeString = exchangeString.replace(/[ \t\n\r]+/g, ""); + if (!/>>>[0-9a-zA-Z\/+]+={0,3}<< true) { return newestFile; } +/** + * Modifies name in case it already exisist at the given directory + * + * Checks the directory passed if it already contains a file or folder with + * the given name, if it does not returns name unmodified, otherwise it + * returns a modified name that does not exist in the directory. + * + * Warning: this function should not be relied upon to be accurate for + * security sensitive applications. The selection process is inherently + * racy and a file or folder may have been created in the folder by the time + * this function returns. + * + * @param {string} directory - directory to check in. + * @param {string} name - file name to check for, may have extension. + * @param {string} extension - dot extension used for the file name. + * @returns {string} modified name with extension that likely does not exist + * in the folder + */ +async function findUnusedName(directory, name, extension = "") { + if (extension && name.endsWith(extension)) { + name = name.slice(0, -extension.length); + } + + while (true) { + if (!await fs.pathExists(path.join(directory, `${name}${extension}`))) { + return `${name}${extension}`; + } + + let match = /^(.*?)(-(\d+))?$/.exec(name); + if (!match[2]) { + name = `${match[1]}-2`; + } else { + name = `${match[1]}-${Number.parseInt(match[3], 10) + 1}`; + } + } +} + module.exports = { getNewestFile, + findUnusedName, }; diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index e4aedcb6..4e3c17e1 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -353,6 +353,26 @@ messages.listSlaves = new Request({ }, }); +let instanceProperties = { + "name": { type: "string" }, + "id": { type: "integer" }, + "assigned_slave": { type: ["null", "integer"] }, + "status": { enum: [ + "unknown", "unassigned", "stopped", "starting", "running", "stopping", + "creating_save", "exporting_data", + ]}, +}; + +messages.getInstance = new Request({ + type: "get_instance", + links: ["control-master"], + permission: "core.instance.get", + requestProperties: { + "id": { type: "integer" }, + }, + responseProperties: instanceProperties, +}); + messages.listInstances = new Request({ type: "list_instances", links: ["control-master"], @@ -363,20 +383,25 @@ messages.listInstances = new Request({ items: { additionalProperties: false, required: ["name", "id", "assigned_slave", "status"], - properties: { - "name": { type: "string" }, - "id": { type: "integer" }, - "assigned_slave": { type: ["null", "integer"] }, - "status": { enum: [ - "unknown", "unassigned", "stopped", "starting", "running", "stopping", - "creating_save", "exporting_data", - ]}, - }, + properties: instanceProperties, }, }, }, }); +messages.setInstanceSubscriptions = new Request({ + type: "set_instance_subscriptions", + links: ["control-master"], + permission: "core.instance.subscribe", + requestProperties: { + "all": { type: "boolean" }, + "instance_ids": { + type: "array", + items: { type: "integer" }, + }, + }, +}); + messages.generateSlaveToken = new Request({ type: "generate_slave_token", links: ["control-master"], @@ -468,11 +493,103 @@ messages.startInstance = new Request({ forwardTo: "instance", }); +let saveList = { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["type", "name", "size", "mtime_ms", "loaded"], + properties: { + "type": { enum: ["file", "directory", "special"] }, + "name": { type: "string" }, + "size": { type: "integer" }, + "mtime_ms": { type: "number" }, + "loaded": { type: "boolean" }, + "default": { type: "boolean" }, + }, + }, +}; + +messages.listSaves = new Request({ + type: "list_saves", + links: ["control-master", "master-slave", "slave-instance"], + permission: "core.instance.save.list", + forwardTo: "instance", + responseProperties: { + "list": saveList, + }, +}); + messages.createSave = new Request({ type: "create_save", links: ["control-master", "master-slave", "slave-instance"], - permission: "core.instance.create_save", + permission: "core.instance.save.create", forwardTo: "instance", + requestProperties: { + "name": { type: "string" }, + "seed": { type: ["integer", "null"] }, + "map_gen_settings": { type: ["object", "null"] }, + "map_settings": { type: ["object", "null"] }, + }, +}); + +messages.deleteSave = new Request({ + type: "delete_save", + links: ["control-master", "master-slave"], + permission: "core.instance.save.delete", + forwardTo: "instance", + requestProperties: { + "save": { type: "string" }, + }, +}); + +messages.downloadSave = new Request({ + type: "download_save", + links: ["control-master"], + permission: "core.instance.save.download", + requestProperties: { + "instance_id": { type: "integer" }, + "save": { type: "string" }, + }, + responseProperties: { + "stream_id": { type: "string" }, + }, +}); + +messages.pullSave = new Request({ + type: "pull_save", + links: ["master-slave"], + requestProperties: { + "instance_id": { type: "integer" }, + "stream_id": { type: "string" }, + "filename": { type: "string" }, + }, + responseProperties: { + "save": { type: "string" }, + }, +}); + +messages.pushSave = new Request({ + type: "push_save", + links: ["master-slave"], + requestProperties: { + "instance_id": { type: "integer" }, + "stream_id": { type: "string" }, + "save": { type: "string" }, + }, +}); + +messages.setSaveListSubscriptions = new Request({ + type: "set_save_list_subscriptions", + links: ["control-master"], + permission: "core.instance.save.list_subscribe", + requestProperties: { + "all": { type: "boolean" }, + "instance_ids": { + type: "array", + items: { type: "integer" }, + }, + }, }); messages.loadScenario = new Request({ @@ -482,6 +599,9 @@ messages.loadScenario = new Request({ forwardTo: "instance", requestProperties: { "scenario": { type: "string" }, + "seed": { type: ["integer", "null"] }, + "map_gen_settings": { type: ["object", "null"] }, + "map_settings": { type: ["object", "null"] }, }, }); @@ -985,6 +1105,22 @@ messages.instanceStatusChanged = new Event({ }, }); +messages.instanceUpdate = new Event({ + type: "instance_update", + links: ["master-control"], + eventProperties: instanceProperties, +}); + +messages.saveListUpdate = new Event({ + type: "save_list_update", + links: ["instance-slave", "slave-master", "master-control"], + forwardTo: "master", + eventProperties: { + "instance_id": { type: "integer" }, + "list": saveList, + }, +}); + messages.masterConnectionEvent = new Event({ type: "master_connection_event", links: ["slave-instance"], diff --git a/packages/lib/users.js b/packages/lib/users.js index dd9e0930..c6c0060a 100644 --- a/packages/lib/users.js +++ b/packages/lib/users.js @@ -333,12 +333,24 @@ definePermission({ description: "Create slave configs via the master server.", }); +definePermission({ + name: "core.instance.get", + title: "Get instance", + description: "Get the details of an instance in the cluster.", + grantByDefault: true, +}); definePermission({ name: "core.instance.list", title: "List instances", description: "Get the full list of instances in the cluster.", grantByDefault: true, }); +definePermission({ + name: "core.instance.subscribe", + title: "Subscribe to instance updates", + description: "Subscribe to be notified on updates on the details and status of instances.", + grantByDefault: true, +}); definePermission({ name: "core.instance.create", title: "Create instance", @@ -360,10 +372,35 @@ definePermission({ description: "Assign or reassign instances to a slave.", }); definePermission({ - name: "core.instance.create_save", + name: "core.instance.save.list", + title: "List saves on instance", + description: "List the saves currently on the instance.", +}); +definePermission({ + name: "core.instance.save.create", title: "Create new instance save", description: "Create new savegames on instances.", }); +definePermission({ + name: "core.instance.save.delete", + title: "Delete instance save", + description: "Delete savegames on instances.", +}); +definePermission({ + name: "core.instance.save.upload", + title: "Upload save to instance", + description: "Upload savegames to instances.", +}); +definePermission({ + name: "core.instance.save.download", + title: "Download save from instance", + description: "Download savegames from instances.", +}); +definePermission({ + name: "core.instance.save.list_subscribe", + title: "Subscribe to save list updatess", + description: "Subscribe to be notifed on updates to the list of saves of instances.", +}); definePermission({ name: "core.instance.export_data", title: "Export locale and icons", diff --git a/packages/master/package.json b/packages/master/package.json index 9e026116..b9116808 100644 --- a/packages/master/package.json +++ b/packages/master/package.json @@ -20,9 +20,9 @@ }, "dependencies": { "@clusterio/lib": "^2.0.0-alpha.6", + "busboy": "^0.3.1", "compression": "^1.7.2", "express": "^4.16.2", - "express-fileupload": "1.2.0", "fs-extra": "^8.1.0", "jsonwebtoken": "^8.2.1", "jszip": "^3.2.2", diff --git a/packages/master/src/BaseConnection.js b/packages/master/src/BaseConnection.js index 191f1bf7..232b1942 100644 --- a/packages/master/src/BaseConnection.js +++ b/packages/master/src/BaseConnection.js @@ -23,25 +23,7 @@ class BaseConnection extends libLink.Link { } async forwardRequestToInstance(message, request) { - let instance = this._master.instances.get(message.data.instance_id); - if (!instance) { - throw new libErrors.RequestError(`Instance with ID ${message.data.instance_id} does not exist`); - } - - let slaveId = instance.config.get("instance.assigned_slave"); - if (slaveId === null) { - throw new libErrors.RequestError("Instance is not assigned to a slave"); - } - - let connection = this._master.wsServer.slaveConnections.get(slaveId); - if (!connection) { - throw new libErrors.RequestError("Slave containing instance is not connected"); - } - if (request.plugin && !connection.plugins.has(request.plugin)) { - throw new libErrors.RequestError(`Slave containing instance does not have ${request.plugin} plugin`); - } - - return await request.send(connection, message.data); + return await this._master.forwardRequestToInstance(request, message.data); } async forwardEventToInstance(message, event) { diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index a592a6ce..4f1c4fbd 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -13,6 +13,7 @@ const libPrometheus = require("@clusterio/lib/prometheus"); const libUsers = require("@clusterio/lib/users"); const BaseConnection = require("./BaseConnection"); +const routes = require("./routes"); const lastQueryLogTime = new libPrometheus.Gauge( "clusterio_master_last_query_log_duration_seconds", @@ -38,6 +39,16 @@ class ControlConnection extends BaseConnection { */ this.user = user; + this.instanceSubscriptions = { + all: false, + instance_ids: [], + }; + + this.saveListSubscriptions = { + all: false, + instance_ids: [], + }; + this.logTransport = null; this.logSubscriptions = { all: false, @@ -122,6 +133,21 @@ class ControlConnection extends BaseConnection { return { serialized_config: slaveConfig.serialize() }; } + async getInstanceRequestHandler(message) { + let id = message.data.id; + let instance = this._master.instances.get(id); + if (!instance) { + throw new libErrors.RequestError(`Instance with ID ${id} does not exist`); + } + + return { + id, + name: instance.config.get("instance.name"), + assigned_slave: instance.config.get("instance.assigned_slave"), + status: instance.status, + }; + } + async listInstancesRequestHandler(message) { let list = []; for (let instance of this._master.instances.values()) { @@ -135,6 +161,24 @@ class ControlConnection extends BaseConnection { return { list }; } + async setInstanceSubscriptionsRequestHandler(message) { + this.instanceSubscriptions = message.data; + } + + instanceUpdated(instance) { + if ( + this.instanceSubscriptions.all + || this.instanceSubscriptions.instance_ids.includes(instance.config.get("instance.id")) + ) { + libLink.messages.instanceUpdate.send(this, { + id: instance.config.get("instance.id"), + name: instance.config.get("instance.name"), + assigned_slave: instance.config.get("instance.assigned_slave"), + status: instance.status, + }); + } + } + // XXX should probably add a hook for slave reuqests? async createInstanceRequestHandler(message) { let instanceConfig = new libConfig.InstanceConfig("master"); @@ -174,7 +218,7 @@ class ControlConnection extends BaseConnection { let instance = { config: instanceConfig, status: "unassigned" }; this._master.instances.set(instanceId, instance); await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", instance, null); - this._master.addInstancePluginHooks(instance); + this._master.addInstanceHooks(instance); } async deleteInstanceRequestHandler(message, request) { @@ -190,6 +234,7 @@ class ControlConnection extends BaseConnection { let prev = instance.status; instance.status = "deleted"; + this.instanceUpdated(instance); await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", instance, prev); } @@ -287,6 +332,42 @@ class ControlConnection extends BaseConnection { } } + async setSaveListSubscriptionsRequestHandler(message) { + this.saveListSubscriptions = message.data; + } + + saveListUpdate(data) { + if ( + this.saveListSubscriptions.all + || this.saveListSubscriptions.instance_ids.includes(data.instance_id) + ) { + libLink.messages.saveListUpdate.send(this, data); + } + } + + async downloadSaveRequestHandler(message) { + let { instance_id, save } = message.data; + let stream = await routes.createProxyStream(this._master.app); + stream.filename = save; + + let ready = new Promise((resolve, reject) => { + stream.events.on("source", resolve); + stream.events.on("timeout", () => reject( + new libErrors.RequestError("Timed out establishing stream from slave") + )); + }); + ready.catch(() => {}); + + let result = await this._master.forwardRequestToInstance(libLink.messages.pushSave, { + instance_id, + stream_id: stream.id, + save, + }); + + await ready; + return { stream_id: stream.id }; + } + async setLogSubscriptionsRequestHandler(message) { this.logSubscriptions = message.data; this.updateLogSubscriptions(); diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index ccadbf3a..8f97c975 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -2,7 +2,6 @@ const compression = require("compression"); const events = require("events"); const express = require("express"); -const expressFileupload = require("express-fileupload"); const fs = require("fs-extra"); const http = require("http"); const https = require("https"); @@ -49,6 +48,7 @@ class Master { this.app = express(); this.app.locals.master = this; + this.app.locals.streams = new Map(); /** * Mapping of slave id to slave info @@ -140,7 +140,7 @@ class Master { const compiler = webpack(webpackConfigs); this.devMiddleware = webpackDevMiddleware(compiler, {}); - app.use(this.devMiddleware); + this.app.use(this.devMiddleware); } let databaseDirectory = this.config.get("master.database_directory"); @@ -152,7 +152,7 @@ class Master { await this.userManager.load(path.join(databaseDirectory, "users.json")); for (let instance of this.instances.values()) { - this.addInstancePluginHooks(instance); + this.addInstanceHooks(instance); } // Make sure we're actually going to listen on a port @@ -243,7 +243,7 @@ class Master { await fs.outputFile(this.configPath, JSON.stringify(this.config.serialize(), null, 4)); if (this.devMiddleware) { - await new Promise((resolve, reject) => { devMiddleware.close(resolve); }); + await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); }); } let databaseDirectory = this.config.get("master.database_directory"); @@ -336,7 +336,6 @@ class Master { hitCounter.inc(); next(); }); - app.use(expressFileupload()); app.use(compression()); // Set folder to serve static content from (the website) @@ -361,10 +360,36 @@ class Master { } } - addInstancePluginHooks(instance) { + addInstanceHooks(instance) { instance.config.on("fieldChanged", (group, field, prev) => { + if (group.name === "instance" && field === "name") { + this.instanceUpdated(instance); + } + libPlugin.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, group, field, prev); }); + + this.instanceUpdated(instance); + } + + instanceUpdated(instance) { + for (let controlConnection of this.wsServer.controlConnections) { + if (controlConnection.connector.closing) { + continue; + } + + controlConnection.instanceUpdated(instance); + } + } + + saveListUpdate(data) { + for (let controlConnection of this.wsServer.controlConnections) { + if (controlConnection.connector.closing) { + continue; + } + + controlConnection.saveListUpdate(data); + } } async loadPlugins() { @@ -466,6 +491,41 @@ class Master { }); }; } + + /** + * Forward the given request to the slave contaning the instance + * + * Finds the slave which the instance with the given instance ID is + * currently assigned to and forwards the request to that instance. + * The request data must have an instance_id parameter. + * + * @param {module:lib/link.Request} request - The request to send. + * @param {object} data - Data to pass with the request. + * @param {number} data.instance_id - + * ID of instance to send this request towards + * @returns {object} data returned from the request. + */ + async forwardRequestToInstance(request, data) { + let instance = this.instances.get(data.instance_id); + if (!instance) { + throw new libErrors.RequestError(`Instance with ID ${data.instance_id} does not exist`); + } + + let slaveId = instance.config.get("instance.assigned_slave"); + if (slaveId === null) { + throw new libErrors.RequestError("Instance is not assigned to a slave"); + } + + let connection = this.wsServer.slaveConnections.get(slaveId); + if (!connection) { + throw new libErrors.RequestError("Slave containing instance is not connected"); + } + if (request.plugin && !connection.plugins.has(request.plugin)) { + throw new libErrors.RequestError(`Slave containing instance does not have ${request.plugin} plugin`); + } + + return await request.send(connection, data); + } } module.exports = Master; diff --git a/packages/master/src/SlaveConnection.js b/packages/master/src/SlaveConnection.js index 9bbdfd4e..f4c71d1b 100644 --- a/packages/master/src/SlaveConnection.js +++ b/packages/master/src/SlaveConnection.js @@ -62,9 +62,19 @@ class SlaveConnection extends BaseConnection { return; } + // We may receive status changed where the status hasn't changed + // from our perspective if the connection was down at the time it + // changed. Slaves also send status updates on assignInstance which + // for hacky reason is also used to push config changes and + // restablish status after a connection loss. + if (instance.status === message.data.status) { + return; + } + let prev = instance.status; instance.status = message.data.status; logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`); + this._master.instanceUpdated(instance); await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", instance, prev); } @@ -99,6 +109,7 @@ class SlaveConnection extends BaseConnection { let prev = masterInstance.status; masterInstance.status = instanceData.status; logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`); + this._master.instanceUpdated(instance); await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", masterInstance, prev); } continue; @@ -107,7 +118,7 @@ class SlaveConnection extends BaseConnection { instanceConfig.set("instance.assigned_slave", this._id); let newInstance = { config: instanceConfig, status: instanceData.status }; this._master.instances.set(instanceConfig.get("instance.id"), newInstance); - this._master.addInstancePluginHooks(newInstance); + this._master.addInstanceHooks(newInstance); await libLink.messages.assignInstance.send(this, { instance_id: instanceConfig.get("instance.id"), serialized_config: instanceConfig.serialize("slave"), @@ -135,6 +146,10 @@ class SlaveConnection extends BaseConnection { libLink.messages.syncUserLists.send(this, { adminlist, banlist, whitelist }); } + async saveListUpdateEventHandler(message) { + this._master.saveListUpdate(message.data); + } + async logMessageEventHandler(message) { this._master.clusterLogger.log({ ...message.data.info, diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 8ab2cba0..0843f79a 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -1,13 +1,18 @@ "use strict"; +const Busboy = require("busboy"); +const crypto = require("crypto"); +const events = require("events"); const fs = require("fs-extra"); const JSZip = require("jszip"); const jwt = require("jsonwebtoken"); const path = require("path"); +const util = require("util"); const libHelpers = require("@clusterio/lib/helpers"); const libLink = require("@clusterio/lib/link"); const libPlugin = require("@clusterio/lib/plugin"); const libPrometheus = require("@clusterio/lib/prometheus"); +const { logger } = require("@clusterio/lib/logging"); const { endpointHitCounter } = require("./metrics"); @@ -106,6 +111,36 @@ function validateSlaveToken(req, res, next) { next(); } +function validateUserToken(req, res, next) { + let token = req.header("x-access-token"); + if (!token) { + res.sendStatus(401); + return; + } + + try { + let tokenPayload = jwt.verify( + token, + req.app.locals.master.config.get("master.auth_secret"), + { audience: "user" } + ); + let user = req.app.locals.master.userManager.users.get(tokenPayload.user); + if (!user) { + throw new Error("invalid user"); + } + if (tokenPayload.iat < user.tokenValidAfter) { + throw new Error("invalid token"); + } + res.locals.user = user; + + } catch (err) { + res.sendStatus(401); + return; + } + + next(); +} + // Handle an uploaded export package. async function uploadExport(req, res) { endpointHitCounter.labels(req.route.path).inc(); @@ -142,6 +177,218 @@ async function uploadExport(req, res) { res.sendStatus(200); } +async function createProxyStream(app) { + let asyncRandomBytes = util.promisify(crypto.randomBytes); + let id = (await asyncRandomBytes(8)).toString("hex"); + let stream = { + id, + flowing: false, + size: null, + mime: null, + filename: null, + events: new events.EventEmitter(), + timeout: setTimeout(() => { + stream.events.emit("timeout"); + }, app.locals.master.config.get("master.proxy_stream_timeout") * 1000), + }; + stream.events.on("close", () => { + clearTimeout(stream.timeout); + app.locals.streams.delete(id); + }); + stream.events.on("timeout", () => { + stream.events.emit("close"); + }); + app.locals.streams.set(id, stream); + return stream; +} + +async function putStream(req, res) { + let stream = req.app.locals.streams.get(req.params.id); + if (!stream || stream.source) { + res.sendStatus(404); + return; + } + stream.source = req; + stream.mime = req.get("Content-Type"); + stream.size = req.get("Content-Length"); + + stream.events.emit("source"); + stream.events.on("close", () => { + if (!stream.flowing) { + req.resume(); + res.sendStatus(500); + } else { + res.sendStatus(200); + } + }); +} + +async function getStream(req, res) { + let stream = req.app.locals.streams.get(req.params.id); + if (!stream || stream.flowing) { + res.sendStatus(404); + return; + } + + function startStream() { + res.append("Content-Type", stream.mime); + if (stream.size) { + res.append("Content-Length", stream.size); + } + if (stream.filename) { + res.append("Content-Disposition", `attachment; filename="${stream.filename}"`); + } else { + res.append("Content-Disposition", "attachment"); + } + stream.source.pipe(res); + stream.flowing = true; + res.on("finish", () => { + stream.events.emit("close"); + }); + clearTimeout(stream.timeout); + } + + if (stream.source) { + startStream(); + } else { + stream.events.on("source", startStream); + stream.events.on("timeout", () => { + res.sendStatus(500); + }); + } +} + + +async function uploadSave(req, res) { + try { + res.locals.user.checkPermission("core.instance.save.upload"); + } catch (err) { + res.status(403).json({ request_errors: [err.message] }); + return; + } + + let contentType = req.get("Content-Type"); + let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType); + if (!match) { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + let mimeType = match[1].toLowerCase(); + + let tasks = []; + let errors = []; + let requestErrors = []; + let saves = []; + + async function handleFile(instanceId, stream, filename, streamMime) { + let proxyStream = await createProxyStream(req.app); + proxyStream.source = stream; + proxyStream.mime = streamMime; + let timeout = new Promise((_, reject) => { + proxyStream.events.on("timeout", () => { + stream.resume(); + reject(new Error("Timed out establishing stream to slave")); + }); + }); + + try { + let result = await Promise.race([ + req.app.locals.master.forwardRequestToInstance(libLink.messages.pullSave, { + instance_id: instanceId, + stream_id: proxyStream.id, + filename, + }), + timeout, + ]); + saves.push(result.save); + + } catch (err) { + proxyStream.events.emit("close"); + logger.error(`Error uploading save: ${err.message}`); + errors.push(err.message); + stream.resume(); + } + } + + if (mimeType === "multipart/form-data") { + await new Promise(resolve => { + let fields = {}; + let busboy = new Busboy({ headers: req.headers }); + busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => { + if (fields.instanceId === undefined) { + requestErrors.push("instance_id must come before files uploaded"); + } + + if (fileMime !== "application/zip") { + requestErrors.push("invalid file Content-Type"); + } + + if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + + if (errors.length || requestErrors.length) { + stream.resume(); + return; + } + + tasks.push(handleFile(fields.instanceId, stream, filename, fileMime)); + }); + busboy.on("field", (fieldname, value, fieldnameTruncated, valueTruncated, transferEncoding, valueMime) => { + if (fieldname === "instance_id") { + fields.instanceId = Number.parseInt(value, 10); + if (Number.isNaN(fields.instanceId)) { + requestErrors.push("invalid instance_id"); + } + } + }); + busboy.on("finish", resolve); + busboy.on("error", (err) => { + logger.error(`Error parsing multipart request in upload-save:\n${err.stack}`); + errors.push(err.message); + }); + req.pipe(busboy); + }); + + } else if (mimeType === "application/zip") { + let filename = req.query.filename; + if (typeof filename !== "string") { + requestErrors.push("Missing or invalid filename parameter"); + } else if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + let instanceId = Number.parseInt(req.query.instance_id, 10); + if (Number.isNaN(instanceId)) { + requestErrors.push("Missing or invalid instance_id parameter"); + } + + if (errors.length || requestErrors.length) { + req.resume(); + } else { + tasks.push(handleFile(instanceId, req, filename, mimeType)); + } + + } else { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + + await Promise.all(tasks); + + if (errors.length) { + res.status(500); + res.json({ errors, request_errors: requestErrors }); + return; + } + if (requestErrors.length) { + res.status(400); + res.json({ request_errors: requestErrors }); + return; + } + + res.json({ saves }); +} + function addRouteHandlers(app) { app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next)); app.get("/api/plugins", getPlugins); @@ -149,6 +396,12 @@ function addRouteHandlers(app) { validateSlaveToken, (req, res, next) => uploadExport(req, res).catch(next) ); + app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next)); + app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next)); + app.post("/api/upload-save", + validateUserToken, + (req, res, next) => uploadSave(req, res).catch(next) + ); } // Routes used in the web interface and served by the master server @@ -170,4 +423,5 @@ const webRoutes = [ module.exports = { addRouteHandlers, webRoutes, + createProxyStream, }; diff --git a/packages/slave/slave.js b/packages/slave/slave.js index 93f3eb79..11fe8b83 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -37,6 +37,7 @@ const libConfig = require("@clusterio/lib/config"); const libSharedCommands = require("@clusterio/lib/shared_commands"); const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging"); const libLoggingUtils = require("@clusterio/lib/logging_utils"); +const libHelpers = require("@clusterio/lib/helpers"); const instanceRconCommandDuration = new libPrometheus.Histogram( @@ -169,6 +170,7 @@ class Instance extends libLink.Link { this._status = "stopped"; this._running = false; + this._loadedSave = null; this.server = new libFactorio.FactorioServer( factorioDir, this._dir, serverOptions ); @@ -209,9 +211,50 @@ class Instance extends libLink.Link { } } + static async listSaves(savesDir, loadedSave) { + let defaultSave = null; + if (loadedSave === null) { + defaultSave = await libFileOps.getNewestFile( + savesDir, (name) => !name.endsWith(".tmp.zip") + ); + } + + let list = []; + for (let name of await fs.readdir(savesDir)) { + let type; + let stat = await fs.stat(path.join(savesDir, name)); + if (stat.isFile()) { + type = "file"; + } else if (stat.isDirectory()) { + type = "directory"; + } else { + type = "special"; + } + + list.push({ + name, + type, + size: stat.size, + mtime_ms: stat.mtimeMs, + loaded: name === loadedSave, + default: name === defaultSave, + }); + } + + return list; + } + + async sendSaveListUpdate() { + libLink.messages.saveListUpdate.send(this, { + instance_id: this.id, + list: await Instance.listSaves(this.server.writePath("saves"), this._loadedSave), + }); + } + async _autosave(name) { let stat = await fs.stat(this.path("saves", `${name}.zip`)); instanceFactorioAutosaveSize.labels(String(this.id)).set(stat.size); + await this.sendSaveListUpdate(); } notifyStatus(status) { @@ -234,6 +277,7 @@ class Instance extends libLink.Link { notifyExit() { this._running = false; + this._loadedSave = null; this.notifyStatus("stopped"); this.config.off("fieldChanged", this._configFieldChanged); @@ -252,6 +296,10 @@ class Instance extends libLink.Link { for (let pluginInstance of this.plugins.values()) { pluginInstance.onExit(); } + + this.sendSaveListUpdate().catch(err => { + this.logger.error(`Unexpected error updating save list:\n${err.stack}`); + }); } async _loadPlugin(pluginInfo, slave) { @@ -267,7 +315,12 @@ class Instance extends libLink.Link { async init(pluginInfos) { this.notifyStatus("starting"); - await this.server.init(); + try { + await this.server.init(); + } catch (err) { + this.notifyExit(); + throw err; + } // load plugins for (let pluginInfo of pluginInfos) { @@ -413,6 +466,23 @@ class Instance extends libLink.Link { saveName = "world.zip"; } + // Load a copy if it's autosave to prevent overwriting the autosave + if (saveName.startsWith("_autosave")) { + this.logger.info("Copying autosave"); + let now = new Date(); + let newName = util.format( + "%s-%s-%s %s%s %s", + now.getUTCFullYear(), + (now.getUTCMonth() + 1).toLocaleString("en", { minimumIntegerDigits: 2 }), + now.getUTCDate().toLocaleString("en", { minimumIntegerDigits: 2 }), + now.getUTCHours().toLocaleString("en", { minimumIntegerDigits: 2 }), + now.getUTCMinutes().toLocaleString("en", { minimumIntegerDigits: 2 }), + saveName, + ); + await fs.copy(this.path("saves", saveName), this.path("saves", newName)); + saveName = newName; + } + if (!this.config.get("factorio.enable_save_patching")) { return saveName; } @@ -496,6 +566,7 @@ class Instance extends libLink.Link { }); this.server.on("exit", () => this.notifyExit()); + this._loadedSave = saveName; await this.server.start(saveName); if (this.config.get("factorio.enable_save_patching")) { @@ -503,6 +574,7 @@ class Instance extends libLink.Link { await this.updateInstanceData(); } + await this.sendSaveListUpdate(); await libPlugin.invokeHook(this.plugins, "onStart"); this._running = true; @@ -516,14 +588,17 @@ class Instance extends libLink.Link { * scenario. * * @param {String} scenario - Name of scenario to load. + * @param {?number} seed - seed to use. + * @param {?object} mapGenSettings - MapGenSettings to use. + * @param {?object} mapSettings - MapSettings to use. */ - async startScenario(scenario) { + async startScenario(scenario, seed, mapGenSettings, mapSettings) { this.server.on("rcon-ready", () => { this.logger.verbose("RCON connection established"); }); this.server.on("exit", () => this.notifyExit()); - await this.server.startScenario(scenario); + await this.server.startScenario(scenario, seed, mapGenSettings, mapSettings); await libPlugin.invokeHook(this.plugins, "onStart"); @@ -678,15 +753,22 @@ class Instance extends libLink.Link { throw err; } + let { scenario, seed, map_gen_settings, map_settings } = message.data; try { - await this.startScenario(message.data.scenario); + await this.startScenario(scenario, seed, map_gen_settings, map_settings); } catch (err) { await this.stop(); throw err; } } - async createSaveRequestHandler() { + async listSavesRequestHandler(message) { + return { + list: await Instance.listSaves(this.server.writePath("saves"), this._loadedSave), + }; + } + + async createSaveRequestHandler(message) { this.notifyStatus("creating_save"); try { this.logger.verbose("Writing server-settings.json"); @@ -702,7 +784,8 @@ class Instance extends libLink.Link { } this.server.on("exit", () => this.notifyExit()); - await this.server.create("world"); + let { name, seed, map_gen_settings, map_settings } = message.data; + await this.server.create(name, seed, map_gen_settings, map_settings); this.logger.info("Successfully created save"); } @@ -1042,13 +1125,8 @@ class Slave extends libLink.Link { throw new Error(`Instance name ${err.message}`); } - // For now add dashes until an unused directory name is found - let dir = path.join(this.config.get("slave.instances_directory"), name); - while (await fs.pathExists(dir)) { - dir += "-"; - } - - return dir; + let instancesDir = this.config.get("slave.instances_directory"); + return path.join(instancesDir, await libFileOps.findUnusedName(instancesDir, name)); } async forwardRequestToInstance(message, request) { @@ -1296,12 +1374,147 @@ class Slave extends libLink.Link { return await request.send(instanceConnection, message.data); } + async listSavesRequestHandler(message, request) { + let instanceId = message.data.instance_id; + let instanceConnection = this.instanceConnections.get(instanceId); + if (instanceConnection) { + return await request.send(instanceConnection, message.data); + } + + let instanceInfo = this.instanceInfos.get(instanceId); + if (!instanceInfo) { + throw new libErrors.RequestError(`Instance with ID ${instanceId} does not exist`); + } + + return { + list: await Instance.listSaves(path.join(instanceInfo.path, "saves"), null), + }; + } + async createSaveRequestHandler(message, request) { let instanceId = message.data.instance_id; let instanceConnection = await this._connectInstance(instanceId); await request.send(instanceConnection, message.data); } + async sendSaveListUpdate(instance_id, savesDir) { + let instanceConnection = this.instanceConnections.get(instance_id); + let saveList; + if (instanceConnection) { + saveList = (await libLink.messages.listSaves.send(instanceConnection, { instance_id })).list; + } else { + saveList = await Instance.listSaves(savesDir, null); + } + + libLink.messages.saveListUpdate.send(this, { instance_id, list: saveList }); + } + + async deleteSaveRequestHandler(message) { + let { instance_id, save } = message.data; + try { + checkFilename(save); + } catch (err) { + throw new libErrors.RequestError(`Save name ${err.message}`); + } + let instanceInfo = this.instanceInfos.get(instance_id); + if (!instanceInfo) { + throw new libErrors.RequestError(`Instance with ID ${instance_id} does not exist`); + } + + try { + await fs.unlink(path.join(instanceInfo.path, "saves", save)); + } catch (err) { + if (err.code === "ENOENT") { + throw new libErrors.RequestError(`${save} does not exist`); + } + throw err; + } + await this.sendSaveListUpdate(instance_id, path.join(instanceInfo.path, "saves")); + } + + async pullSaveRequestHandler(message) { + let { instance_id, stream_id, filename } = message.data; + try { + checkFilename(filename); + } catch (err) { + throw new libErrors.RequestError(`Save name ${err.message}`); + } + let instanceInfo = this.instanceInfos.get(instance_id); + if (!instanceInfo) { + throw new libErrors.RequestError(`Instance with ID ${instance_id} does not exist`); + } + + let url = new URL(this.config.get("slave.master_url")); + url.pathname += `api/stream/${stream_id}`; + let response = await phin({ + url, method: "GET", + core: { ca: this.tlsCa }, + stream: true, + }); + + if (response.statusCode !== 200) { + let content = await libHelpers.readStream(response); + throw new libErrors.RequestError(`Stream returned ${response.statusCode}: ${content.toString()}`); + } + + let savesDir = path.join(instanceInfo.path, "saves"); + let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + let stream; + while (true) { + try { + stream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" }); + await events.once(stream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(savesDir, tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + response.pipe(stream); + await events.once(stream, "finish"); + + filename = await libFileOps.findUnusedName(savesDir, filename, ".zip"); + await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, filename)); + + await this.sendSaveListUpdate(instance_id, savesDir); + return { save: filename }; + } + + async pushSaveRequestHandler(message) { + let { instance_id, stream_id, save } = message.data; + try { + checkFilename(save); + } catch (err) { + throw new libErrors.RequestError(`Save name ${err.message}`); + } + let instanceInfo = this.instanceInfos.get(instance_id); + if (!instanceInfo) { + throw new libErrors.RequestError(`Instance with ID ${instance_id} does not exist`); + } + + let content; + try { + // phin doesn't support streaming requests :( + content = await fs.readFile(path.join(instanceInfo.path, "saves", save)); + } catch (err) { + if (err.code === "ENOENT") { + throw new libErrors.RequestError(`${save} does not exist`); + } + throw err; + } + + let url = new URL(this.config.get("slave.master_url")); + url.pathname += `api/stream/${stream_id}`; + phin({ + url, method: "PUT", + core: { ca: this.tlsCa }, + data: content, + }).catch(err => logger.error(`Error pushing save to master:\n${err.stack}`)); + } + async exportDataRequestHandler(message, request) { let instanceId = message.data.instance_id; let instanceConnection = await this._connectInstance(instanceId); diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 7a64620d..c594a56a 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -19,8 +19,10 @@ "@babel/core": "^7.13.8", "@babel/preset-react": "^7.12.13", "@clusterio/lib": "^2.0.0-alpha.6", - "antd": "^4.6.5", + "antd": "^4.16.0", + "assert": "^2.0.0", "babel-loader": "^8.2.2", + "browserify-zlib": "^0.2.0", "buffer": "^6.0.2", "clean-webpack-plugin": "^3.0.0", "css-loader": "^5.1.0", @@ -33,6 +35,7 @@ "react": "^16.13.1", "react-dom": "^16.13.1", "react-router-dom": "^5.2.0", + "stream-browserify": "^3.0.0", "style-loader": "^2.0.0", "util": "^0.12.3", "webpack": "^5.36.2" diff --git a/packages/web_ui/src/components/CreateSaveModal.jsx b/packages/web_ui/src/components/CreateSaveModal.jsx new file mode 100644 index 00000000..f067635b --- /dev/null +++ b/packages/web_ui/src/components/CreateSaveModal.jsx @@ -0,0 +1,140 @@ +import React, { useContext, useState } from "react"; +import { Button, Col, Form, Input, Modal, Row, Typography } from "antd"; + +import libLink from "@clusterio/lib/link"; +import { readMapExchangeString } from "@clusterio/lib/factorio/exchange_string"; + +import ControlContext from "./ControlContext"; +import { notifyErrorHandler } from "../util/notify"; + + +export default function CreateSaveModal(props) { + let [visible, setVisible] = useState(false); + let [creatingSave, setCreatingSave] = useState(false); + let [form] = Form.useForm(); + let control = useContext(ControlContext); + + function createSave() { + let values = form.getFieldsValue(); + let name = values.saveName || "world.zip"; + let seed = values.seed === undefined ? null : Number.parseInt(values.seed, 10); + let mapGenSettings = null; + let mapSettings = null; + if (values.exchangeString && values.exchangeString.trim()) { + let result; + try { + result = readMapExchangeString(values.exchangeString); + } catch (err) { + form.setFields([{ name: "exchangeString", errors: [err.message] }]); + return; + } + form.setFields([{ name: "exchangeString", errors: [] }]); + mapGenSettings = result.map_gen_settings; + mapSettings = result.map_settings; + } + if (values.mapGenSettings && values.mapGenSettings.trim()) { + try { + mapGenSettings = JSON.parse(values.mapGenSettings); + } catch (err) { + form.setFields([{ name: "mapGenSettings", errors: [err.message] }]); + return; + } + form.setFields([{ name: "mapGenSettings", errors: [] }]); + } + if (values.mapSettings && values.mapSettings.trim()) { + try { + mapSettings = JSON.parse(values.mapGenSettings); + } catch (err) { + form.setFields([{ name: "mapSettings", errors: [err.message] }]); + return; + } + form.setFields([{ name: "mapSettings", errors: [] }]); + } + + setCreatingSave(true); + libLink.messages.createSave.send(control, { + instance_id: props.instance.id, + name, + seed, + map_gen_settings: mapGenSettings, + map_settings: mapSettings, + }).then(() => { + setVisible(false); + }).catch( + notifyErrorHandler("Error creating save") + ).finally(() => { + setCreatingSave(false); + }); + } + + function convertExchangeString() { + let exchangeString = form.getFieldValue("exchangeString"); + let result; + try { + result = readMapExchangeString(exchangeString); + } catch (err) { + form.setFields([{ name: "exchangeString", errors: [err.message] }]); + return; + } + form.setFields([{ name: "exchangeString", errors: [], value: "" }]); + form.setFieldsValue({ + mapGenSettings: JSON.stringify(result.map_gen_settings, null, 4), + mapSettings: JSON.stringify(result.map_settings, null, 4), + }); + } + + function handleCancel() { + setVisible(false); + } + + return <> + + +
+ + + + + + + + + +
+ +
+ + + + + + +
+ + ; +} diff --git a/packages/web_ui/src/components/InstanceViewPage.jsx b/packages/web_ui/src/components/InstanceViewPage.jsx index f44b5c34..bd96d526 100644 --- a/packages/web_ui/src/components/InstanceViewPage.jsx +++ b/packages/web_ui/src/components/InstanceViewPage.jsx @@ -12,6 +12,8 @@ import InstanceConsole from "./InstanceConsole"; import InstanceRcon from "./InstanceRcon"; import AssignInstanceModal from "./AssignInstanceModal"; import StartStopInstanceButton from "./StartStopInstanceButton"; +import LoadScenarioModal from "./LoadScenarioModal"; +import SavesList from "./SavesList"; import { notifyErrorHandler } from "../util/notify"; import { useInstance } from "../model/instance"; import { useSlave } from "../model/slave"; @@ -26,10 +28,10 @@ export default function InstanceViewPage(props) { let history = useHistory(); let control = useContext(ControlContext); - let [instance, updateInstance] = useInstance(instanceId); + let [instance] = useInstance(instanceId); let [slave] = useSlave(Number(instance["assigned_slave"])); - let [creatingSave, setCreatingSave] = useState(false); + let [exportingData, setExportingData] = useState(false); let nav = [{ name: "Instances", path: "/instances" }, { name: instance.name || "Unknown" }]; if (instance.loading) { @@ -44,24 +46,24 @@ export default function InstanceViewPage(props) { } let instanceButtons = - - {instance.status === "stopped" && } + Export data + {instance["status"]} + Console - + ; } diff --git a/packages/web_ui/src/components/LoadScenarioModal.jsx b/packages/web_ui/src/components/LoadScenarioModal.jsx new file mode 100644 index 00000000..448e6d0a --- /dev/null +++ b/packages/web_ui/src/components/LoadScenarioModal.jsx @@ -0,0 +1,140 @@ +import React, { useContext, useState } from "react"; +import { Button, Col, Form, Input, Modal, Row, Typography } from "antd"; + +import libLink from "@clusterio/lib/link"; +import { readMapExchangeString } from "@clusterio/lib/factorio/exchange_string"; + +import ControlContext from "./ControlContext"; +import { notifyErrorHandler } from "../util/notify"; + + +export default function LoadScenarioModal(props) { + let [visible, setVisible] = useState(false); + let [loadingScenario, setLoadingScenario] = useState(false); + let [form] = Form.useForm(); + let control = useContext(ControlContext); + + function loadScenario() { + let values = form.getFieldsValue(); + let scenario = values.scenario || "base/freeplay"; + let seed = values.seed === undefined ? null : Number.parseInt(values.seed, 10); + let mapGenSettings = null; + let mapSettings = null; + if (values.exchangeString && values.exchangeString.trim()) { + let result; + try { + result = readMapExchangeString(values.exchangeString); + } catch (err) { + form.setFields([{ name: "exchangeString", errors: [err.message] }]); + return; + } + form.setFields([{ name: "exchangeString", errors: [] }]); + mapGenSettings = result.map_gen_settings; + mapSettings = result.map_settings; + } + if (values.mapGenSettings && values.mapGenSettings.trim()) { + try { + mapGenSettings = JSON.parse(values.mapGenSettings); + } catch (err) { + form.setFields([{ name: "mapGenSettings", errors: [err.message] }]); + return; + } + form.setFields([{ name: "mapGenSettings", errors: [] }]); + } + if (values.mapSettings && values.mapSettings.trim()) { + try { + mapSettings = JSON.parse(values.mapGenSettings); + } catch (err) { + form.setFields([{ name: "mapSettings", errors: [err.message] }]); + return; + } + form.setFields([{ name: "mapSettings", errors: [] }]); + } + + setLoadingScenario(true); + libLink.messages.loadScenario.send(control, { + instance_id: props.instance.id, + scenario, + seed, + map_gen_settings: mapGenSettings, + map_settings: mapSettings, + }).then(() => { + setVisible(false); + }).catch( + notifyErrorHandler("Error creating save") + ).finally(() => { + setLoadingScenario(false); + }); + } + + function convertExchangeString() { + let exchangeString = form.getFieldValue("exchangeString"); + let result; + try { + result = readMapExchangeString(exchangeString); + } catch (err) { + form.setFields([{ name: "exchangeString", errors: [err.message] }]); + return; + } + form.setFields([{ name: "exchangeString", errors: [], value: "" }]); + form.setFieldsValue({ + mapGenSettings: JSON.stringify(result.map_gen_settings, null, 4), + mapSettings: JSON.stringify(result.map_settings, null, 4), + }); + } + + function handleCancel() { + setVisible(false); + } + + return <> + + +
+ + + + + + + + + +
+ +
+ + + + + + +
+ + ; +} diff --git a/packages/web_ui/src/components/SavesList.jsx b/packages/web_ui/src/components/SavesList.jsx new file mode 100644 index 00000000..448577c1 --- /dev/null +++ b/packages/web_ui/src/components/SavesList.jsx @@ -0,0 +1,135 @@ +import React, { useContext, useState } from "react"; +import { Button, Popconfirm, Space, Table, Tooltip, Upload } from "antd"; +import CaretLeftOutlined from "@ant-design/icons/CaretLeftOutlined"; +import LeftOutlined from "@ant-design/icons/LeftOutlined"; + +import libLink from "@clusterio/lib/link"; + +import ControlContext from "./ControlContext"; +import CreateSaveModal from "./CreateSaveModal"; +import SectionHeader from "./SectionHeader"; +import { useSaves } from "../model/saves"; +import { notifyErrorHandler } from "../util/notify"; + + +function formatBytes(bytes) { + if (bytes === 0) { + return "0 Bytes"; + } + + let units = [" Bytes", " kB", " MB", " GB", " TB"]; + let factor = 1000; + let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length); + return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power]; +} + +export default function SavesList(props) { + let control = useContext(ControlContext); + let saves = useSaves(props.instance.id); + let [starting, setStarting] = useState(false); + + const saveTable = <> + {save.name} + {save.loaded && } + {save.default && } + , + sorter: (a, b) => a.name.localeCompare(b.name), + }, + { + title: "Size", + key: "size", + responsive: ["sm"], + render: save => formatBytes(save.size), + align: "right", + sorter: (a, b) => a.size - b.size, + }, + { + title: "Last Modified", + key: "mtime_ms", + render: save => new Date(save.mtime_ms).toLocaleString(), + sorter: (a, b) => a.mtime_ms - b.mtime_ms, + defaultSortOrder: "descend", + }, + ]} + dataSource={saves} + rowKey={save => save.name} + pagination={false} + expandable={{ + columnWidth: 33, + expandRowByClick: true, + expandedRowRender: save => + + + { + libLink.messages.deleteSave.send( + control, { instance_id: props.instance.id, save: save.name } + ).catch(notifyErrorHandler("Error deleting save")); + }} + > + + + , + }} + />; + + let uploadProps = { + name: "file", + accept: ".zip", + headers: { + "X-Access-Token": control.connector.token, + }, + data: { + instance_id: props.instance.id, + }, + showUploadList: false, + action: `${webRoot}api/upload-save`, + // TODO: show progress to user. + }; + + return
+ + + + + + /> + + {saveTable} + +
; +} diff --git a/packages/web_ui/src/components/SectionHeader.jsx b/packages/web_ui/src/components/SectionHeader.jsx new file mode 100644 index 00000000..13323316 --- /dev/null +++ b/packages/web_ui/src/components/SectionHeader.jsx @@ -0,0 +1,8 @@ +import React, { useContext, useState } from "react"; + +export default function SectionHeader(props) { + return
+
{props.title}
+ {props.extra &&
{props.extra}
} +
; +} diff --git a/packages/web_ui/src/index.css b/packages/web_ui/src/index.css index 4f8cf73f..97d8a4a6 100644 --- a/packages/web_ui/src/index.css +++ b/packages/web_ui/src/index.css @@ -67,6 +67,27 @@ body { padding-right: 64px; } +.section-header { + display: flex; + align-items: center; + margin-top: 16px; + margin-bottom: 8px; +} + +.section-header-title { + flex: auto; + overflow: hidden; + font-weight: bold; + font-size: 16px; + line-height: 1.5715; + white-space: nowrap; + text-overflow: ellipsis; +} + +.section-header-extra { + margin-left: auto; +} + .site-name { position: absolute; top: -10px; @@ -143,6 +164,10 @@ body { white-space: pre-wrap; } +.save-list-dragger .ant-upload.ant-upload-btn { + padding: 0; +} + .rcon-result code { display: block; background: #000; diff --git a/packages/web_ui/src/index.jsx b/packages/web_ui/src/index.jsx index 2f783d08..a692e723 100644 --- a/packages/web_ui/src/index.jsx +++ b/packages/web_ui/src/index.jsx @@ -18,3 +18,5 @@ export { default as InstanceConsole } from "./components/InstanceConsole"; export { default as InstanceRcon } from "./components/InstanceRcon"; export { default as PageLayout } from "./components/PageLayout"; +export { default as SavesList } from "./components/SavesList"; +export { default as SectionHeader } from "./components/SectionHeader"; diff --git a/packages/web_ui/src/model/instance.jsx b/packages/web_ui/src/model/instance.jsx index c30227ab..4048a135 100644 --- a/packages/web_ui/src/model/instance.jsx +++ b/packages/web_ui/src/model/instance.jsx @@ -2,6 +2,7 @@ import React, { useEffect, useContext, useState } from "react"; import ControlContext from "../components/ControlContext"; import libLink from "@clusterio/lib/link"; +import { logger } from "@clusterio/lib/logging"; export function useInstance(id) { @@ -14,19 +15,26 @@ export function useInstance(id) { return; } - // XXX optimize by requesting only the instance in question - libLink.messages.listInstances.send(control).then(result => { - let match = result.list.find(i => i.id === id); - if (!match) { - setInstance({ missing: true }); - } else { - setInstance({ ...match, present: true }); - } + libLink.messages.getInstance.send(control, { id }).then(result => { + setInstance({ ...result, present: true }); + }).catch(err => { + logger.log(`Failed to get instance: ${err}`); + setInstance({ missing: true }); }); + } useEffect(() => { updateInstance(); + + function updateHandler(newInstance) { + setInstance({ ...newInstance, present: true }); + } + + control.onInstanceUpdate(id, updateHandler); + return () => { + control.offInstanceUpdate(id, updateHandler); + }; }, [id]); return [instance, updateInstance]; diff --git a/packages/web_ui/src/model/saves.jsx b/packages/web_ui/src/model/saves.jsx new file mode 100644 index 00000000..7d9e1a5e --- /dev/null +++ b/packages/web_ui/src/model/saves.jsx @@ -0,0 +1,34 @@ +import React, { useEffect, useContext, useState } from "react"; +import ControlContext from "../components/ControlContext"; + +import libLink from "@clusterio/lib/link"; +import { logger } from "@clusterio/lib/logging"; + + +export function useSaves(instanceId) { + let control = useContext(ControlContext); + let [saves, setSaves] = useState([]); + + function updateSaves() { + libLink.messages.listSaves.send(control, { instance_id: instanceId }).then(result => { + setSaves(result.list); + }).catch(err => { + logger.error(`Failed to list instance saves: ${err}`); + }); + } + + useEffect(() => { + updateSaves(); + + function updateHandler(data) { + setSaves(data.list); + } + + control.onSaveListUpdate(instanceId, updateHandler); + return () => { + control.offSaveListUpdate(instanceId, updateHandler); + }; + }, [instanceId]); + + return saves; +} diff --git a/packages/web_ui/src/util/websocket.jsx b/packages/web_ui/src/util/websocket.jsx index f683fea0..63d7d689 100644 --- a/packages/web_ui/src/util/websocket.jsx +++ b/packages/web_ui/src/util/websocket.jsx @@ -62,10 +62,18 @@ export class Control extends libLink.Link { */ this.accountName = null; + this.instanceUpdateHandlers = new Map(); + this.saveListUpdateHandlers = new Map(); this.instanceLogHandlers = new Map(); this.connector.on("connect", data => { this.accountName = data.account.name; + this.updateInstanceSubscriptions().catch(err => logger.error( + `Unexpected error updating instance subscriptions:\n${err.stack}` + )); + this.updateSaveListSubscriptions().catch(err => logger.error( + `Unexpected error updating save list subscriptions:\n${err.stack}` + )); this.updateLogSubscriptions().catch(err => logger.error( `Unexpected error updating log subscriptions:\n${err.stack}` )); @@ -84,6 +92,114 @@ export class Control extends libLink.Link { } } + async instanceUpdateEventHandler(message) { + let handlers = this.instanceUpdateHandlers.get(message.data.id); + for (let handler of handlers || []) { + handler(message.data); + } + }; + + async onInstanceUpdate(id, handler) { + if (!Number.isInteger(id)) { + throw new Error("Invalid instance id"); + } + + let handlers = this.instanceUpdateHandlers.get(id); + if (!handlers) { + handlers = []; + this.instanceUpdateHandlers.set(id, handlers); + } + + handlers.push(handler); + + if (handlers.length === 1) { + await this.updateInstanceSubscriptions(); + } + } + + async offInstanceUpdate(id, handler) { + let handlers = this.instanceUpdateHandlers.get(id); + if (!handlers || !handlers.length) { + throw new Error(`No handlers for instance ${id} exist`); + } + + let index = handlers.lastIndexOf(handler); + if (index === -1) { + throw new Error(`Given handler is not registered for instance ${id}`); + } + + handlers.splice(index, 1); + if (!handlers.length) { + this.instanceUpdateHandlers.delete(id); + await this.updateInstanceSubscriptions(); + } + } + + async updateInstanceSubscriptions() { + if (!this.connector.connected) { + return; + } + + await libLink.messages.setInstanceSubscriptions.send(this, { + all: false, + instance_ids: [...this.instanceUpdateHandlers.keys()], + }); + } + + async saveListUpdateEventHandler(message) { + let handlers = this.saveListUpdateHandlers.get(message.data.instance_id); + for (let handler of handlers || []) { + handler(message.data); + } + }; + + async onSaveListUpdate(id, handler) { + if (!Number.isInteger(id)) { + throw new Error("Invalid instance id"); + } + + let handlers = this.saveListUpdateHandlers.get(id); + if (!handlers) { + handlers = []; + this.saveListUpdateHandlers.set(id, handlers); + } + + handlers.push(handler); + + if (handlers.length === 1) { + await this.updateSaveListSubscriptions(); + } + } + + async offSaveListUpdate(id, handler) { + let handlers = this.saveListUpdateHandlers.get(id); + if (!handlers) { + throw new Error(`No handlers for instance ${id} exists`); + } + + let index = handlers.lastIndexOf(handler); + if (index === -1) { + throw new Error(`Given handler is not registered for instance ${id}`); + + handlers.splice(index, 1); + if (!handlers.length) { + this.saveListUpdateHandlers.delete(id); + await this.updateSaveListSubscriptions(); + } + } + } + + async updateSaveListSubscriptions() { + if (!this.connector.connected) { + return; + } + + await libLink.messages.setSaveListSubscriptions.send(this, { + all: false, + instance_ids: [...this.saveListUpdateHandlers.keys()], + }); + } + async logMessageEventHandler(message) { let info = message.data.info; diff --git a/packages/web_ui/webpack.common.js b/packages/web_ui/webpack.common.js index df0fac39..6a850827 100644 --- a/packages/web_ui/webpack.common.js +++ b/packages/web_ui/webpack.common.js @@ -92,8 +92,13 @@ module.exports = (env = {}) => ({ "util": require.resolve("util/"), "os": require.resolve("os-browserify/browser"), + // Required for zlib + "assert": require.resolve("assert"), + "stream": require.resolve("stream-browserify"), + "events": require.resolve("events/"), "path": require.resolve("path-browserify"), + "zlib": require.resolve("browserify-zlib"), }, }, }); diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index ce545166..ed0e7058 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -26,7 +26,7 @@ "devDependencies": { "@clusterio/lib": "^2.0.0-alpha.6", "@clusterio/web_ui": "^2.0.0-alpha.6", - "antd": "^4.6.5", + "antd": "^4.16.0", "phin": "^3.5.1", "react": "^16.13.1", "react-dom": "^16.13.1", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index 84a62545..66e246c4 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -25,7 +25,7 @@ "devDependencies": { "@clusterio/lib": "^2.0.0-alpha.6", "@clusterio/web_ui": "^2.0.0-alpha.6", - "antd": "^4.6.5", + "antd": "^4.16.0", "react": "^16.13.1", "react-dom": "^16.13.1", "webpack": "^5.36.2", diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 389bd86c..c73c3d32 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -7,6 +7,7 @@ const events = require("events"); const libLink = require("@clusterio/lib/link"); const libUsers = require("@clusterio/lib/users"); +const testStrings = require("../lib/factorio/test_strings"); const { TestControl, TestControlConnector, url, controlToken, slowTest, get, exec, execCtl, sendRcon, getControl, instancesDir, @@ -145,6 +146,21 @@ describe("Integration of Clusterio", function() { }); }); + describe("saveListUpdateEventHandler()", function() { + it("should have triggered for the created save", function() { + slowTest(this); + assert.equal(getControl().saveListUpdates.slice(-1)[0].list[0].name, "world.zip"); + }); + }); + + describe("instance list-saves", function() { + it("lists the created save", async function() { + slowTest(this); + let result = await execCtl("instance list-saves test"); + assert(/world\.zip/.test(result.stdout), "world.zip not present in list save output"); + }); + }); + describe("instance export-data", function() { it("exports the data", async function() { slowTest(this); @@ -157,11 +173,31 @@ describe("Integration of Clusterio", function() { }); describe("instance start", function() { - it("starts the instance", async function() { + it("should not hang if factorio version does not exist", async function() { slowTest(this); - await execCtl("instance start test"); + try { + await execCtl("instance config set 44 factorio.version 0.1.2"); + await assert.rejects(execCtl("instance start test")); + + } finally { + await execCtl("instance config set 44 factorio.version latest"); + } + }); + it("starts the given save", async function() { + slowTest(this); + await execCtl("instance start test --save world.zip"); await checkInstanceStatus(44, "running"); }); + it("copies the save if an autosave is the target", async function() { + slowTest(this); + await execCtl("instance stop 44"); + let savesDir = path.join("temp", "test", "instances", "test", "saves"); + await fs.copy(path.join(savesDir, "world.zip"), path.join(savesDir, "_autosave1.zip")); + await execCtl("instance start test"); + let result = await libLink.messages.listSaves.send(getControl(), { instance_id: 44 }); + let running = result.list.find(s => s.loaded); + assert(running.name !== "_autosave1.zip"); + }); }); describe("instance send-rcon", function() { @@ -184,7 +220,7 @@ describe("Integration of Clusterio", function() { "afk-auto-kick", "Kick if AFK for more than 2 minutes.", ], [ - "allow_commands", true, + "allow_commands", "true", "allow-commands", "Allow Lua commands: Yes.", ], [ @@ -315,6 +351,83 @@ describe("Integration of Clusterio", function() { }); }); + describe("instance load-scenario", function() { + it("starts the instance with the given settings", async function() { + slowTest(this); + await execCtl("instance config set 44 factorio.enable_save_patching false"); + await execCtl("instance config set 44 player_auth.load_plugin false"); + await execCtl("instance config set 44 research_sync.load_plugin false"); + await execCtl("instance config set 44 statistics_exporter.load_plugin false"); + await execCtl("instance config set 44 subspace_storage.load_plugin false"); + try { + let exchangeString = testStrings.modified.replace(/[\n\r]+/g, ""); + let args = `base/freeplay --seed 1234 --map-exchange-string "${exchangeString}"`; + await execCtl(`instance load-scenario test ${args}`); + await checkInstanceStatus(44, "running"); + await sendRcon(44, '/c game.print("disable achievements")'); + await sendRcon(44, '/c game.print("disable achievements")'); + assert.equal(await sendRcon(44, "/c rcon.print(game.default_map_gen_settings.seed)"), "1234\n"); + assert.equal(await sendRcon(44, "/c rcon.print(game.map_settings.pollution.ageing)"), "1.5\n"); + assert.equal( + await sendRcon(44, "/c rcon.print(game.difficulty_settings.research_queue_setting)"), "never\n" + ); + + } finally { + await execCtl("instance stop test"); + } + }); + }); + + describe("instance upload-save", function() { + it("should upload a zip file", async function() { + await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test"); + await execCtl("instance upload-save 44 upload.zip"); + assert( + await fs.pathExists(path.join("temp", "test", "instances", "test", "saves", "upload.zip")), + "file not uploaded to saves directory" + ); + }); + it("should reject non-zip files", async function() { + await fs.outputFile(path.join("temp", "test", "invalid"), "a test"); + await assert.rejects(execCtl("instance upload-save 44 invalid")); + }); + it("should reject path traversal attacks", async function() { + await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test"); + await assert.rejects(execCtl("instance upload-save 44 upload.zip --name ../traversal.zip")); + }); + }); + + describe("instance download-save", function() { + it("should download a save", async function() { + await fs.remove(path.join("temp", "test", "upload.zip")); + await execCtl("instance download-save 44 upload.zip"); + assert(await fs.pathExists(path.join("temp", "test", "upload.zip"))); + }); + it("should error if save does not exist", async function() { + await assert.rejects(execCtl("instance download-save 44 invalid")); + }); + it("should error on path traversal attacks", async function() { + await assert.rejects(execCtl("instance download-save 44 ../factorio-current.log")); + }); + }); + + describe("instance delete-save", function() { + it("should delete a save", async function() { + await execCtl("instance delete-save 44 upload.zip"); + assert( + !await fs.pathExists(path.join("temp", "test", "instances", "test", "saves", "upload.zip")), + "file not deleted" + ); + }); + it("should error if save does not exist", async function() { + await assert.rejects(execCtl("instance delete-save 44 upload.zip")); + }); + it("should error on path traversal attacks", async function() { + await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test"); + await assert.rejects(execCtl("instance delete-save 44 ../../upload.zip")); + }); + }); + describe("instance delete", function() { it("deletes the instance", async function() { slowTest(this); @@ -325,6 +438,22 @@ describe("Integration of Clusterio", function() { }); }); + describe("instanceUpdateEventHandler()", function() { + it("should have triggered for the previous instance status updates", function() { + let statusesToCheck = new Set( + ["unassigned", "stopped", "creating_save", "exporting_data", "starting", "running", "stopping"] + ); + let statusesNotSeen = new Set(statusesToCheck); + + for (let update of getControl().instanceUpdates) { + assert(statusesToCheck.has(update.status), `Missing check for status ${update.status}`); + statusesNotSeen.delete(update.status); + } + + assert(statusesNotSeen.size === 0, `Did not see the statuses ${[...statusesNotSeen]}`); + }); + }); + describe("permission list", function() { it("runs", async function() { await execCtl("permission list"); diff --git a/test/integration/index.js b/test/integration/index.js index 3b4b6766..ad2fa030 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -25,6 +25,17 @@ class TestControl extends libLink.Link { constructor(connector) { super("control", "master", connector); libLink.attachAllMessages(this); + this.instanceUpdates = []; + this.saveListUpdates = []; + + this.connector.on("connect", () => { + libLink.messages.setInstanceSubscriptions.send( + this, { all: true, instance_ids: [] } + ).catch(err => logger.error(`Error setting instance subscriptions:\n${err.stack}`)); + libLink.messages.setSaveListSubscriptions.send( + this, { all: true, instance_ids: [] } + ).catch(err => logger.error(`Error setting save list subscriptions:\n${err.stack}`)); + }); } async prepareDisconnectRequestHandler(message, request) { @@ -34,6 +45,14 @@ class TestControl extends libLink.Link { async debugWsMessageEventHandler() { } + async instanceUpdateEventHandler(message) { + this.instanceUpdates.push(message.data); + } + + async saveListUpdateEventHandler(message) { + this.saveListUpdates.push(message.data); + } + async logMessageEventHandler() { } } @@ -183,7 +202,9 @@ after(async function() { masterProcess.kill("SIGINT"); await events.once(masterProcess, "exit"); } - await control.connector.close(); + if (control) { + await control.connector.close(); + } }); // Ensure the test processes are stopped. diff --git a/test/lib/factorio/exchange_string.js b/test/lib/factorio/exchange_string.js new file mode 100644 index 00000000..8164a241 --- /dev/null +++ b/test/lib/factorio/exchange_string.js @@ -0,0 +1,73 @@ +"use strict"; +const assert = require("assert").strict; +const zlib = require("zlib"); + +const libFactorio = require("@clusterio/lib/factorio"); + +const testStrings = require("./test_strings"); + + +// Simple linear congruential random number generator +class Random { + constructor(seed) { + this.prev = seed; + } + + next() { + let curr = (1103515245 * this.prev + 12345) & 0x7fffffff; + this.prev = curr; + return curr >> 16; + } +} + +describe("lib/factorio/exchange_string", function() { + describe("readMapExchangeString", function() { + it("should parse a valid string", function() { + let result = libFactorio.readMapExchangeString(testStrings.default); + assert.equal(result.map_gen_settings.seed, 1234567890); + + result = libFactorio.readMapExchangeString(testStrings.modified); + assert.equal(result.checksum, 4092204126); + }); + + it("should handle malformed strings", function() { + assert.throws( + () => libFactorio.readMapExchangeString("<>"), + new Error("Not a map exchange string") + ); + + assert.throws( + () => libFactorio.readMapExchangeString(testStrings.default.slice(0, 100)), + new Error("Not a map exchange string") + ); + + assert.throws( + () => libFactorio.readMapExchangeString(`>>>${Buffer.from("abk430ia404ah3b4").toString("base64")}<<<`), + new Error("Malformed map exchange string: zlib inflate failed") + ); + + for (let i = 0; i < 100; i++) { + let gen = new Random(i); + let size = gen.next() % 200 + 100; + let data = Buffer.alloc(size); + for (let j = 0; j < size; j++) { + data[j] = gen.next() % 256; + } + // eslint-disable-next-line node/no-sync + data = zlib.deflateSync(data); + assert.throws( + () => libFactorio.readMapExchangeString(`>>>${data.toString("base64")}<<<`), + new Error("Malformed map exchange string: reached end before finishing parsing") + ); + } + + let pastEnd = Buffer.from(testStrings.default.replace(/><\n/g, ""), "base64"); + // eslint-disable-next-line node/no-sync + pastEnd = zlib.deflateSync(Buffer.concat([zlib.inflateSync(pastEnd), Buffer.from("junk")])); + assert.throws( + () => libFactorio.readMapExchangeString(`>>>${pastEnd.toString("base64")}<<<`), + new Error("Malformed map exchange string: data after end") + ); + }); + }); +}); diff --git a/test/lib/factorio/test_strings.js b/test/lib/factorio/test_strings.js new file mode 100644 index 00000000..243d4a01 --- /dev/null +++ b/test/lib/factorio/test_strings.js @@ -0,0 +1,38 @@ +"use strict"; +// Exchange strings for testing with + +let default_ = ` +>>>eNpjZGBkUGQAgwZ7EOZgSc5PzIHxQJgrOb+gILVIN78oFVmYM +7moNCVVNz8TVXFqXmpupW5SYjGKYo7Movw8dBNYi0vy81BFSopSU +4uRRbhLixLzMktz0fUyMF5imubZ0CLHAML/6xkU/v8HYSDrAdAvI +MzA2ABRCRSDAdbknMy0NAYGBUcgdgJJMzIyVousc39YNcWeEaJGz +wHK+AAVOZAEE/GEMfwccEqpwBgmSOYYg8FnJAbE0hKgFVBVHA4IB +kSyBSTJyNj7duuC78cu2DH+Wfnxkm9Sgj2joavIuw9G6+yAkuwgf +zLBiVkzQWAnzCsMMDMf2EOlbtoznj0DAm/sGVlBOkRAhIMFkDjgz +czAKMAHZC3oARIKMgwwp9nBjBFxYEwDg28wnzyGMS7bo/sDGBA2I +MPlQMQJEAG2EO4yRgjTod+B0UEeJiuJUALUb8SA7IYUhA9Pwqw9j +GQ/mkMwIwLZH2giKg5YooELZGEKnHjBDHcNMDwvsMN4DvMdGJlBD +JCqL0AxCA8kAzMKQgs4gIObmQEBgGmjrO/lHgDxa6G4<<< +`; + +let modified = ` +>>>eNp1UzFs00AUvZ8mbRIEypCBSlCCyFAhOQqBKULxlQUxMCCxgnCc +S2rh2On5LAgMZOjAgISEurQLXSmCBTGwRWKhEkgIJraiMgBiaGmFOiC +FO5/PsUz6pfv+9/6/99+/kwEBOo2k6c83NhrZtOkaNkIDXa286fZ6hG +ouJaJIwTmT+i2iuZYdQ3GeOKTb15qGJ4oxQsMglbWo60iGYcSQ8Zjrh +GUhwighHpcRSBHIEZ8ajuV35dmBwIPK1DRxmMX6ADnqmre1Rb/D86UF +hNYX1lZncZpZNkGpmQ41PE87J6RFTe4YjND4hPDq943NwfIcEmv0AJV +GI7F4tMXViYVgECgCjoWWPm66DqOurXmEMcvp1A3/br1pGV5Oq1ZqVW +Hzk0ralCz5xDH79a5vM6tnW4Rma5XgQPVk8kTXtTzmU5Jg1g6tm0hfr +ZwPLGPaVrstL6p0SYwEAPeLLy9/vbeigxysgsPgIESGTYVcCYMnL/Bh +qbRKoQsq2NFBdt+LBbIp4y3CqiweBzK5LJIAu4vbD18f7Dfg77PdT1e +bt3S4/q2w5J39IbQfFW+TitzaqrA3ahSkOLf0MPVFhw/vhf3SYUqcOC +PczkoKQfXaNILCMb5df8Rd6QRS0hqKpoihHdgfNcm2Cj7ryTnKGC4K8 +jnh3gmXQRElVwYyxI8x4FMqOzsu4edrKK6hNZ5wU7V9G+ufEFL+7yHi +cySQMp7wDHnRsBW571ORGn6fH2fUDj/FwV0iUbXPMbmTP7akkt8ChiL +/BPceEs3j1M3iz71/ObT9tA==<<<`; + +module.exports = { + default: default_, + modified, +}; diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index da153c92..5b02656d 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -13,6 +13,13 @@ describe("lib/file_ops", function() { await fs.outputFile(path.join(baseDir, "test", "file.txt"), "contents"); await fs.outputFile(path.join(baseDir, "test", "another file.txt"), "more contents"); + + await fs.outputFile(path.join(baseDir, "find", "file"), "contents"); + await fs.outputFile(path.join(baseDir, "find", "file.txt"), "contents"); + await fs.outputFile(path.join(baseDir, "find", "foo-1"), "contents"); + await fs.outputFile(path.join(baseDir, "find", "foo-2"), "contents"); + await fs.outputFile(path.join(baseDir, "find", "bar-1.txt"), "contents"); + await fs.outputFile(path.join(baseDir, "find", "bar-2.txt"), "contents"); } before(setupTestingEnv); @@ -31,4 +38,32 @@ describe("lib/file_ops", function() { assert.equal(newest, null); }); }); + + describe("findUnusedName()", function() { + it("should return named unchanged if it does not exist", async function() { + let cases = [ + [["file"], "file"], + [["file", ".txt"], "file.txt"], + [["file.txt", ".txt"], "file.txt"], + ]; + for (let [args, expected] of cases) { + let actual = await libFileOps.findUnusedName(path.join(baseDir, "test", "folder"), ...args); + assert.equal(actual, expected); + } + }); + it("should return changed name if it does exist", async function() { + let cases = [ + [["file"], "file-2"], + [["file", ".txt"], "file-2.txt"], + [["file.txt", ".txt"], "file-2.txt"], + [["foo-1"], "foo-3"], + [["bar-1", ".txt"], "bar-3.txt"], + [["bar-1.txt", ".txt"], "bar-3.txt"], + ]; + for (let [args, expected] of cases) { + let actual = await libFileOps.findUnusedName(path.join(baseDir, "find"), ...args); + assert.equal(actual, expected); + } + }); + }); }); diff --git a/test/master.js b/test/master.js deleted file mode 100644 index 50510de8..00000000 --- a/test/master.js +++ /dev/null @@ -1,9 +0,0 @@ -"use strict"; -const assert = require("assert").strict; - -const mock = require("./mock"); -const master = require("@clusterio/master/master"); - - -describe("Master testing", function() { -}); diff --git a/test/master/routes.js b/test/master/routes.js new file mode 100644 index 00000000..7037be75 --- /dev/null +++ b/test/master/routes.js @@ -0,0 +1,329 @@ +"use strict"; +const assert = require("assert").strict; +const events = require("events"); +const FormData = require("form-data"); +const jwt = require("jsonwebtoken"); +const http = require("http"); +const phin = require("phin"); + +const { wait } = require("@clusterio/lib/helpers"); +const routes = require("@clusterio/master/src/routes"); +const mock = require("../mock"); + + +describe("master/src/routes", function() { + let master; + let server; + let port; + beforeEach(async function() { + master = new mock.MockMaster(); + routes.addRouteHandlers(master.app); + server = http.createServer(master.app); + server.listen(0, "localhost"); + await events.once(server, "listening"); + port = server.address().port; + }); + afterEach(function() { + server.close(); + }); + describe("/api/stream/:id", function() { + it("should respond with 404 if stream doesn't exist", async function() { + let response; + response = await phin(`http://localhost:${port}/api/stream/01020304`); + assert.equal(response.statusCode, 404); + response = await phin({ + url: `http://localhost:${port}/api/stream/01020304`, method: "PUT", + data: "test stream", + }); + assert.equal(response.statusCode, 404); + }); + it("should respond with 500 if stream times out", async function() { + let stream = await routes.createProxyStream(master.app); + let response = await phin(`http://localhost:${port}/api/stream/${stream.id}`); + assert.equal(response.statusCode, 500); + }); + it("should passthrough a stream", async function() { + let stream; + let responses; + stream = await routes.createProxyStream(master.app); + responses = await Promise.all([ + (async function() { + await wait(100); + return await phin({ + url: `http://localhost:${port}/api/stream/${stream.id}`, method: "PUT", + data: "test content", + }); + }()), + phin(`http://localhost:${port}/api/stream/${stream.id}`), + ]); + assert.equal(responses[0].statusCode, 200); + assert.equal(responses[1].statusCode, 200); + assert.equal(responses[1].body.toString(), "test content"); + stream = await routes.createProxyStream(master.app); + responses = await Promise.all([ + phin({ + url: `http://localhost:${port}/api/stream/${stream.id}`, method: "PUT", + data: "test content", + }), + (async function() { + await wait(100); + return await phin(`http://localhost:${port}/api/stream/${stream.id}`); + }()), + ]); + assert.equal(responses[0].statusCode, 200); + assert.equal(responses[1].statusCode, 200); + assert.equal(responses[1].body.toString(), "test content"); + }); + }); + describe("/api/upload-save", function() { + it("should respond with 401 with invalid token", async function() { + let response; + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { "Content-Type": "application/zip" }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 401); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { + "Content-Type": "application/zip", + "X-Access-Token": "invalid", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 401); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { + "Content-Type": "application/zip", + "X-Access-Token": jwt.sign({ aud: "user", user: "invalid" }, "TestSecretDoNotUse"), + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 401); + master.userManager.users.get("test").tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { + "Content-Type": "application/zip", + "X-Access-Token": jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse"), + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 401); + }); + + it("should respond with 403 when user has insufficient permission", async function() { + let response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { + "Content-Type": "application/zip", + "X-Access-Token": jwt.sign({ aud: "user", user: "player" }, "TestSecretDoNotUse"), + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 403); + }); + + let testToken = jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse"); + it("should respond with 415 if content type is missing or invalid", async function() { + let response; + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { + "X-Access-Token": testToken, + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 415); + assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "invalid", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 415); + assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "text/plain", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 415); + assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] }); + }); + + it("should respond with 400 if invalid multipart request", async function() { + let response; + let form; + form = new FormData(); + form.append("instance_id", "123"); + form.append("file", "totally a zip file", { filename: "test.zip", contentType: "text/plain" }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save`, method: "POST", + headers: { + "X-Access-Token": testToken, + ...form.getHeaders(), + }, + data: form.getBuffer(), + }); + assert.equal(response.statusCode, 400); + assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid file Content-Type"] }); + form = new FormData(); + form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" }); + form.append("instance_id", "123"); + response = await phin({ + url: `http://localhost:${port}/api/upload-save`, method: "POST", + headers: { + "X-Access-Token": testToken, + ...form.getHeaders(), + }, + data: form.getBuffer(), + }); + assert.equal(response.statusCode, 400); + assert.deepEqual( + JSON.parse(response.body), { request_errors: ["instance_id must come before files uploaded"] } + ); + form = new FormData(); + form.append("instance_id", "123"); + form.append("file", "totally a zip file", { filename: "test.txt", contentType: "application/zip" }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save`, method: "POST", + headers: { + "X-Access-Token": testToken, + ...form.getHeaders(), + }, + data: form.getBuffer(), + }); + assert.equal(response.statusCode, 400); + assert.deepEqual( + JSON.parse(response.body), { request_errors: ["filename must end with .zip"] } + ); + form = new FormData(); + form.append("instance_id", "invalid"); + form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save`, method: "POST", + headers: { + "X-Access-Token": testToken, + ...form.getHeaders(), + }, + data: form.getBuffer(), + }); + assert.equal(response.statusCode, 400); + assert.deepEqual( + JSON.parse(response.body), { request_errors: ["invalid instance_id"] } + ); + }); + + it("should respond with 400 if invalid file request", async function() { + let response; + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "application/zip", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 400); + assert.deepEqual(JSON.parse(response.body), { request_errors: ["Missing or invalid filename parameter"] }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.txt`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "application/zip", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 400); + assert.deepEqual(JSON.parse(response.body), { request_errors: ["filename must end with .zip"] }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?filename=test.zip`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "application/zip", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 400); + assert.deepEqual( + JSON.parse(response.body), { request_errors: ["Missing or invalid instance_id parameter"] } + ); + }); + + it("should respond with 500 if the transfer failed", async function() { + master.forwardRequestToInstance = async (data, request) => { + throw new Error("Something went wrong"); + }; + let response; + let form; + form = new FormData(); + form.append("instance_id", "123"); + form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save`, method: "POST", + headers: { + "X-Access-Token": testToken, + ...form.getHeaders(), + }, + data: form.getBuffer(), + }); + assert.equal(response.statusCode, 500); + assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] }); + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "application/zip", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 500); + assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] }); + master.forwardRequestToInstance = async (data, request) => { + await new Promise(() => {}); + }; + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "application/zip", + }, + data: "totally a zip file", + }); + assert.equal(response.statusCode, 500); + assert.deepEqual( + JSON.parse(response.body), { errors: ["Timed out establishing stream to slave"], request_errors: [] } + ); + }); + + it("should complete a valid transfer", async function() { + master.forwardRequestToInstance = async (request, data) => { + let stream = await phin({ + url: `http://localhost:${port}/api/stream/${data.stream_id}`, + }); + assert.equal(stream.body.toString(), "totally a zip file"); + return { save: data.filename }; + }; + let response; + response = await phin({ + url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST", + headers: { + "X-Access-Token": testToken, + "Content-Type": "application/zip", + }, + data: "totally a zip file", + }); + assert.deepEqual(JSON.parse(response.body), { saves: ["test.zip"] }); + assert.equal(response.statusCode, 200); + }); + }); +}); diff --git a/test/mock.js b/test/mock.js index 5d11a8e0..0787c403 100644 --- a/test/mock.js +++ b/test/mock.js @@ -150,9 +150,12 @@ class MockControl extends libLink.Link { class MockMaster { constructor() { this.app = express(); + this.app.locals.master = this; + this.app.locals.streams = new Map(); this.mockConfigEntries = new Map([ ["master.external_address", "test"], ["master.auth_secret", "TestSecretDoNotUse"], + ["master.proxy_stream_timeout", 1], ]); this.config = { get: (name) => { @@ -227,6 +230,7 @@ module.exports = { MockInstance, MockSlave, MockControl, + MockMaster, createMasterPlugin, createInstancePlugin, diff --git a/test/package.json b/test/package.json index 205930a2..539f5007 100644 --- a/test/package.json +++ b/test/package.json @@ -10,6 +10,7 @@ "@clusterio/master": "^2.0.0-alpha.6", "@clusterio/slave": "^2.0.0-alpha.6", "express": "^4.16.2", + "form-data": "^4.0.0", "fs-extra": "^8.1.0", "jsonwebtoken": "^8.2.1", "jszip": "^3.2.2",