From 1e6b807ae3166a0564cd9dced9c0eb2d26a2feb7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 12 Mar 2020 15:39:25 +0100 Subject: [PATCH] Add Clusterio Library mod Add a library mod for wrapping the Clusterio remote API and providing related utilities. This eases development of Clusterio related Lua code by providing the same interface for both mods and modules and also makes the serialize library available without having to duplicate it. It is not possible to implement this through save patching as mods cannot load Lua code from the level code like they can with code from other mods. --- build-mod.js | 92 +++++++++++++++++++++ docs/developing-for-clusterio.md | 137 ++++++++++++++++++++++++++++--- mod/api.lua | 67 +++++++++++++++ mod/changelog.txt | 5 ++ mod/info.json | 9 ++ modules/clusterio/api.lua | 36 ++------ modules/clusterio/impl.lua | 40 +++++---- package.json | 1 + 8 files changed, 328 insertions(+), 59 deletions(-) create mode 100644 build-mod.js create mode 100644 mod/api.lua create mode 100644 mod/changelog.txt create mode 100644 mod/info.json diff --git a/build-mod.js b/build-mod.js new file mode 100644 index 00000000..ef47f8a4 --- /dev/null +++ b/build-mod.js @@ -0,0 +1,92 @@ +"use strict"; +const events = require("events"); +const fs = require("fs-extra"); +const JSZip = require("jszip"); +const klaw = require("klaw"); +const path = require("path"); +const yargs = require("yargs"); + + +// Location of mod source files +const sourceDir = "mod"; + +// Paths to additional files to put into the mod's folder. +const additionalFiles = [ + path.join("modules", "clusterio", "serialize.lua"), +]; + +async function main() { + const args = yargs + .scriptName("build") + .options({ + 'clean': { describe: "Remove previous builds", type: 'boolean', default: false }, + 'build': { describe: "Build mod", type: 'boolean', default: true }, + 'pack': { describe: "Pack into zip file", type: 'boolean', default: true }, + 'output-dir': { describe: "Path to output built mod", nargs: 1, type: 'string', default: "dist" }, + }) + .argv + ; + + let info = JSON.parse(await fs.readFile(path.join(sourceDir, "info.json"))); + + if (args.clean) { + let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/ + for (let entry of await fs.readdir(args.outputDir)) { + let match = splitter.exec(entry); + if (match) { + let [, name, version] = match; + if (name === info.name) { + let modPath = path.join(args.outputDir, entry); + console.log(`Removing ${modPath}`); + await fs.remove(modPath); + } + } + } + } + + if (args.build) { + await fs.ensureDir(args.outputDir); + let modName = `${info.name}_${info.version}`; + + if (args.pack) { + let zip = new JSZip(); + let walker = klaw(sourceDir) + .on('data', item => { + if (item.stats.isFile()) { + // On Windows the path created uses backslashes as the directory sepparator + // but the zip file needs to use forward slashes. We can't use the posix + // version of relative here as it doesn't work with Windows style paths. + let basePath = path.relative(sourceDir, item.path).replace(/\\/g, "/"); + zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path)); + } + }) + ; + await events.once(walker, 'end'); + + for (let filePath of additionalFiles) { + zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath)); + } + + let modPath = path.join(args.outputDir, `${modName}.zip`); + console.log(`Writing ${modPath}`); + let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath)); + await events.once(writeStream, 'finish'); + + } else { + let modDir = path.join(args.outputDir, modName); + if (await fs.exists(modDir)) { + console.log(`Removing existing build ${modDir}`); + await fs.remove(modDir); + } + console.log(`Building ${modDir}`); + await fs.copy(sourceDir, modDir); + for (let filePath of additionalFiles) { + await fs.copy(filePath, path.join(modDir, path.basename(filePath))); + } + } + } +} + +if (module === require.main) { + main().catch(err => { console.log(err) }); +} diff --git a/docs/developing-for-clusterio.md b/docs/developing-for-clusterio.md index 4ada6fd8..5e2273fe 100644 --- a/docs/developing-for-clusterio.md +++ b/docs/developing-for-clusterio.md @@ -14,7 +14,7 @@ Contents - [event_handler interface](#event_handler-interface) - [Communicating with Clusterio](#communicating-with-clusterio) - [Clusterio Modules](#clusterio-modules) - - [Clusterio Module API](#clusterio-module-api) +- [Clusterio Lua API](#clusterio-lua-api) - [Clusterio Plugins](#clusterio-plugins) @@ -216,7 +216,8 @@ The following entries are supported in the module.json file: supported. The events for dependencies is invoked before the events of the dependent, and starting an instance will fail if the dependencies cannot be satisfied. If not specified it will default - to depending on `clusterio`. + to depending on `clusterio`. Make sure to add `clusterio` to your + dependencies if you add other dependencies and depend on the Lua API. - `require`: Lua files to require in `control.lua`. This should only really be needed if you want to make a global function available for use in @@ -250,21 +251,133 @@ module. Because modules can be patched into an existing game you cannot rely on the `on_init` callback to be called in Clusterio Modules. Nor can you rely on the `on_configuration_changed` callback, as this is not called -when level code changes. The `clusterio` module provides the custom +when level code changes. The Clusterio Lua API provides the custom `on_server_startup` event that can be used as a substitute, see the next section. -### Clusterio Module API +Clusterio Lua API +----------------- -There's a builtin clusterio module that provides some tools for getting -instance info and communicating to plugins, script events to listen for -and a Factorio remote call API. The available interfaces is documented -in [modules/clusterio/api.lua](../modules/clusterio/api.lua). To use -this API from another module, require it using something along the lines -of `local clusterio_api = require("modules/clusterio/api")`. Make sure -to add `clusterio` as a dependency in your module.json if you use the -dependency entry in it. +Clusterio provides a save patched module as well as a regular Factorio +mod for interfacing with Clusterio. This includes tools sending data to +plugins, script events to listen for, and an item serialization module. + + +### clusterio_api library + +**require path** +From a module +`local clusterio_api = require("modules/clusterio/api")` +From a mod +`local clusterio_api = require("__clusterio_lib__/api")` + +Provides the main interface to Clusterio from within the game. When +using the mod version you must call `init` before it'll function +properly. + +#### init + +Mod version only + +Initialize the API. This should be called from both the `on_init` and +`on_load` events in your mod, and before this is called certain features +will not be available, most notably the events table. + +#### clusterio_api.events + +Table of events raised by clusterio. If the game is started outside of +Clusterio then none of these events will be raised. + +In the mod version this table does not exist before the `init` function +is called. + +##### on_instance_updated + +Raised after the name and id of an instance has been updated. This may +occur even if the id and name didn't change. + +Event data: +- `instance_id`: The id of the instance. +- `instance_name`: the name of the instance. + +##### on_server_startup + +Raised when Clusterio is starting up the server. It can be used to +initialize clusterio related features in a mod, and as a stand-in for +the `on_init` and `on_configuration_changed` event for modules. It is +invoked on the first tick the server runs after the save has been +patched, before most other events. + +Use this event to initialize and and/or migrate the data structures +you need. Keep in mind that both mods and Clusterio modules can switch +from any version to any version so it should be able to handle both +forwards and backwards migrations gracefully. + +#### clusterio_api.send_json(channel, data) + +Send json data to Clusterio over the given channel. The `data` argument +should be table that can be serialized with `game.table_to_json`. +Clusterio plugins can listen to channels and will receive an event with +the data sent here. See the [Communicating with Factorio +section](writing-plugins.md#communicating-with-factorio) in the Writing +Plugins document for more information. + +If the game is started outside of Clusterio the data will be sent, but +since there's no code following stdout to pick it up it will be lost. + +**Note**: Payloads greater than 4 MB will cause stuttering in the game. + +**Note**: This is not a binary safe way of sending data. Strings +embedded into the tables sent must be valid UTF-8 text. + +Parameters: +- `channel`: string identifying which channel to send it on. +- `data`: table that can be converted to JSON with game.table_to_json + + +### serialize library + +**require path** +From a module +`local serialize = require("modules/clusterio/serialize")` +From a mod +`local serialize = require("__clusterio_lib__/serialize")` + +Serializes inventories and item stacks into a table that in turn can be +converted into JSON with `game.table_to_json`. Properly handles the +extra data for most items. + +#### serialize.serialize_equipment_grid(LuaEquipmentGrid) + +Returns a table containing a serialized representation of the equipment +grid passed. + +#### serialize.deserialize_equipment_grid(LuaEquipmentGrid, serialized_grid) + +Deserialize a previously serialized equipment grid into the target +LuaEquipmentGrid. Overwrites all content in the target. + +#### serialize.serialize_item_stack(LuaItemStack, destination_table) + +Serialize the LuaItemStack passed into the target destination_table. +The destination_table should be a new empty table. + +#### serialize.deserialize_item_stack(LuaItemStack, serialized_stack) + +Deserialize a previously serialized item stack into the target +LuaItemStack. Overwrites the content of the item stack. + +#### serialize.serialize_inventory(LuaInventory) + +Returns a table containing a serialized representation of the inventory +passed. + +#### serialize.deserialize_inventory(LuaInventory, serialized_inventory) + +Deserialize a previously serialized inventory into the target +LuaInventory. Overwrites slots that has content in the serialized +inventory. Clusterio Plugins diff --git a/mod/api.lua b/mod/api.lua new file mode 100644 index 00000000..c800b035 --- /dev/null +++ b/mod/api.lua @@ -0,0 +1,67 @@ +-- Important: Keep this API in sync with modules/clusterio/api.lua +local api = {} + + +local initialized = false +function api.init() + if initialized then + return + end + initialized = true + + if remote.interfaces.clusterio_api then + api.events = remote.call("clusterio_api", "get_events") + else + local null_event = script.generate_event_name() + api.events = { + on_instance_updated = null_event, + on_server_startup = null_event, + } + end +end + +local function call_api(fn, ...) + if remote.interfaces.clusterio_api and remote.interfaces.clusterio_api[fn] then + return remote.call("clusterio_api", fn, ...) + else + return nil + end +end + +function api.get_instance_name() + return call_api("get_instance_name") +end + +function api.get_instance_id() + return call_api("get_instance_id") +end + + +function api.send_json(channel, data) + if not remote.interfaces.clusterio_api then + return + end + + -- Escape bad characters. The question mark is used for separating the + -- channel name from the payload. + channel = channel:gsub("([\x00-\x1f?\\])", function(match) + return "\\x" .. string.format("%02x", match:byte()) + end) + + data = game.table_to_json(data) + + -- If there's more than about 4kB of data users running with the Windows + -- console open will start to experience stuttering, otherwise we could + -- output about 1 MB of data at a time through stdout. + if #data < 4000 then + print("\f$ipc:" .. channel .. "?j" .. data) + else + local file_no = remote.call("clusterio_api", "get_file_no") + local file_name = "clst_" .. file_no .. ".json" + game.write_file(file_name, data, false, 0) + print("\f$ipc:" .. channel .. "?f" .. file_name) + end +end + + +return api diff --git a/mod/changelog.txt b/mod/changelog.txt new file mode 100644 index 00000000..50e3310e --- /dev/null +++ b/mod/changelog.txt @@ -0,0 +1,5 @@ +--------------------------------------------------------------------------------------------------- +Version: 0.1.0 + Major Features: + - API for mods to interact with Clusterio + - Serialization library for converting items to JSON compatible tables. diff --git a/mod/info.json b/mod/info.json new file mode 100644 index 00000000..26a80f92 --- /dev/null +++ b/mod/info.json @@ -0,0 +1,9 @@ +{ + "name": "clusterio_lib", + "version": "0.1.0", + "title": "Clusterio Library", + "author": "Clusterio Team", + "homepage": "https://github.com/clusterio/factorioClusterio", + "description": "Library for interfacing with Clusterio.", + "factorio_version": "0.17" +} diff --git a/modules/clusterio/api.lua b/modules/clusterio/api.lua index 2132f4fa..f0e932dd 100644 --- a/modules/clusterio/api.lua +++ b/modules/clusterio/api.lua @@ -1,42 +1,20 @@ +-- Important: Keep this API in sync with mod/api.lua local api = {} --- Table of events raised by clusterio api.events = { - -- Raised after the name and id of an instance has been updated - -- This may occur even if the id and name didn't change. - - -- Event data: - -- instance_id: The id of the instance. - -- instance_name: the name of the instance. on_instance_updated = script.generate_event_name(), - - -- Raised when the server is starting up - -- This event serves as the stand-in for the on_init and - -- on_configuration_changed events for modules. It is invoked as early - -- as possible, before most but not all other events, every time the - -- server is started up. - - -- Use this event to initialize and and/or migrate the data structures - -- you need. Keep in mind that Clusterio can switch from any version of - -- your module to any version of your module so it should be able to - -- handle both forwards and backwards migrations. on_server_startup = script.generate_event_name(), } --- Send a table as json to Clusterio --- Send data to Clusterio over the given channel. Clusterio plugins can --- listen to channels and will receive an event with the data sent here. --- See docs/writing-plugins.md for more information. +function api.get_instance_name() + return global.clusterio.instance_name +end --- Note: Payloads greater than 4 MB will cause stuttering. +function api.get_instance_id() + return global.clusterio.instance_id +end --- Note: This is not a binary safe way of sending data. Strings embedded --- into the tables sent must be valid UTF-8 text. - --- Parameters: --- channel: string identifying which channel to send it on. --- data: table that can be converted to json with game.table_to_json function api.send_json(channel, data) -- Escape bad characters. The question mark is used for separating the diff --git a/modules/clusterio/impl.lua b/modules/clusterio/impl.lua index d4989221..3ae9593f 100644 --- a/modules/clusterio/impl.lua +++ b/modules/clusterio/impl.lua @@ -32,26 +32,30 @@ function clusterio_private.update_instance(new_id, new_name) end -function impl.add_remote_interface() - remote.add_interface('clusterio_api', { - -- Returns a table of events raised by clusterio - get_events = function() - return api.events - end, +-- This is not part of the add_remote_interface callback to ensure it is +-- available when the clusterio_lib mod is loaded. The reason this is +-- neccessary is that on_init for newly added mods happen before on_load +-- for existing mods, and the add_remote_interface callback is done in +-- on_load. See https://forums.factorio.com/viewtopic.php?f=25&t=81552 for +-- more details. +remote.add_interface('clusterio_api', { + get_events = function() + return api.events + end, - -- Returns the instance id the game is run under - -- This may change over time and/or be nil. - get_instance_id = function() - return global.clusterio.instance_id - end, + get_instance_id = function() + return global.clusterio.instance_id + end, - -- Returns the name of the instance the game runs under. - -- This may change over time and/or be nil. - get_instance_name = function() - return global.clusterio.instance_name - end, - }) -end + get_instance_name = function() + return global.clusterio.instance_name + end, + + get_file_no = function() + global.clusterio_file_no = (global.clusterio_file_no or 0) + 1 + return global.clusterio_file_no + end, +}) return impl diff --git a/package.json b/package.json index a4afb539..2b258eb0 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ }, "devDependencies": { "html5-validator": "^1.2.1", + "klaw": "^3.0.0", "mocha": "^6.2.0", "mocha-sinon": "^2.1.0", "mocha.parallel": "^0.15.5",