Add Clusterio Library mod

Add a library mod for wrapping the Clusterio remote API and providing
related utilities.  This eases development of Clusterio related Lua code
by providing the same interface for both mods and modules and also makes
the serialize library available without having to duplicate it.

It is not possible to implement this through save patching as mods
cannot load Lua code from the level code like they can with code from
other mods.

build-mod.js

@@ -0,0 +1,92 @@
+"use strict";
+const events = require("events");
+const fs = require("fs-extra");
+const JSZip = require("jszip");
+const klaw = require("klaw");
+const path = require("path");
+const yargs = require("yargs");
+
+
+// Location of mod source files
+const sourceDir = "mod";
+
+// Paths to additional files to put into the mod's folder.
+const additionalFiles = [
+	path.join("modules", "clusterio", "serialize.lua"),
+];
+
+async function main() {
+	const args = yargs
+		.scriptName("build")
+		.options({
+			'clean': { describe: "Remove previous builds", type: 'boolean', default: false },
+			'build': { describe: "Build mod", type: 'boolean', default: true },
+			'pack': { describe: "Pack into zip file", type: 'boolean', default: true },
+			'output-dir': { describe: "Path to output built mod", nargs: 1, type: 'string', default: "dist" },
+		})
+		.argv
+	;
+
+	let info = JSON.parse(await fs.readFile(path.join(sourceDir, "info.json")));
+
+	if (args.clean) {
+		let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/
+		for (let entry of await fs.readdir(args.outputDir)) {
+			let match = splitter.exec(entry);
+			if (match) {
+				let [, name, version] = match;
+				if (name === info.name) {
+					let modPath = path.join(args.outputDir, entry);
+					console.log(`Removing ${modPath}`);
+					await fs.remove(modPath);
+				}
+			}
+		}
+	}
+
+	if (args.build) {
+		await fs.ensureDir(args.outputDir);
+		let modName = `${info.name}_${info.version}`;
+
+		if (args.pack) {
+			let zip = new JSZip();
+			let walker = klaw(sourceDir)
+				.on('data', item => {
+					if (item.stats.isFile()) {
+						// On Windows the path created uses backslashes as the directory sepparator
+						// but the zip file needs to use forward slashes.  We can't use the posix
+						// version of relative here as it doesn't work with Windows style paths.
+						let basePath = path.relative(sourceDir, item.path).replace(/\\/g, "/");
+						zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
+					}
+				})
+			;
+			await events.once(walker, 'end');
+
+			for (let filePath of additionalFiles) {
+				zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath));
+			}
+
+			let modPath = path.join(args.outputDir, `${modName}.zip`);
+			console.log(`Writing ${modPath}`);
+			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
+			await events.once(writeStream, 'finish');
+
+		} else {
+			let modDir = path.join(args.outputDir, modName);
+			if (await fs.exists(modDir)) {
+				console.log(`Removing existing build ${modDir}`);
+				await fs.remove(modDir);
+			}
+			console.log(`Building ${modDir}`);
+			await fs.copy(sourceDir, modDir);
+			for (let filePath of additionalFiles) {
+				await fs.copy(filePath, path.join(modDir, path.basename(filePath)));
+			}
+		}
+	}
+}
+
+if (module === require.main) {
+	main().catch(err => { console.log(err) });
+}

docs/developing-for-clusterio.md

@@ -14,7 +14,7 @@ Contents
     - [event_handler interface](#event_handler-interface)
 - [Communicating with Clusterio](#communicating-with-clusterio)
 - [Clusterio Modules](#clusterio-modules)
-    - [Clusterio Module API](#clusterio-module-api)
+- [Clusterio Lua API](#clusterio-lua-api)
 - [Clusterio Plugins](#clusterio-plugins)
 
 
@@ -216,7 +216,8 @@ The following entries are supported in the module.json file:
     supported.  The events for dependencies is invoked before the events
     of the dependent, and starting an instance will fail if the
     dependencies cannot be satisfied.  If not specified it will default
-    to depending on `clusterio`.
+    to depending on `clusterio`.  Make sure to add `clusterio` to your
+    dependencies if you add other dependencies and depend on the Lua API.
 - `require`:
     Lua files to require in `control.lua`.  This should only really be
     needed if you want to make a global function available for use in
@@ -250,21 +251,133 @@ module.
 Because modules can be patched into an existing game you cannot rely on
 the `on_init` callback to be called in Clusterio Modules.  Nor can you
 rely on the `on_configuration_changed` callback, as this is not called
-when level code changes.  The `clusterio` module provides the custom
+when level code changes.  The Clusterio Lua API provides the custom
 `on_server_startup` event that can be used as a substitute, see the next
 section.
 
 
-### Clusterio Module API
+Clusterio Lua API
+-----------------
 
-There's a builtin clusterio module that provides some tools for getting
-instance info and communicating to plugins, script events to listen for
-and a Factorio remote call API.  The available interfaces is documented
-in [modules/clusterio/api.lua](../modules/clusterio/api.lua).  To use
-this API from another module, require it using something along the lines
-of `local clusterio_api = require("modules/clusterio/api")`.  Make sure
-to add `clusterio` as a dependency in your module.json if you use the
-dependency entry in it.
+Clusterio provides a save patched module as well as a regular Factorio
+mod for interfacing with Clusterio.  This includes tools sending data to
+plugins, script events to listen for, and an item serialization module.
+
+
+### clusterio_api library
+
+**require path**  
+From a module  
+`local clusterio_api = require("modules/clusterio/api")`  
+From a mod  
+`local clusterio_api = require("__clusterio_lib__/api")`
+
+Provides the main interface to Clusterio from within the game.  When
+using the mod version you must call `init` before it'll function
+properly.
+
+#### init
+
+<sub>Mod version only</sub>
+
+Initialize the API.  This should be called from both the `on_init` and
+`on_load` events in your mod, and before this is called certain features
+will not be available, most notably the events table.
+
+#### clusterio_api.events
+
+Table of events raised by clusterio.  If the game is started outside of
+Clusterio then none of these events will be raised.
+
+In the mod version this table does not exist before the `init` function
+is called.
+
+##### on_instance_updated
+
+Raised after the name and id of an instance has been updated.  This may
+occur even if the id and name didn't change.
+
+Event data:
+- `instance_id`: The id of the instance.
+- `instance_name`: the name of the instance.
+
+##### on_server_startup
+
+Raised when Clusterio is starting up the server.  It can be used to
+initialize clusterio related features in a mod, and as a stand-in for
+the `on_init` and `on_configuration_changed` event for modules.  It is
+invoked on the first tick the server runs after the save has been
+patched, before most other events.
+
+Use this event to initialize and and/or migrate the data structures
+you need.  Keep in mind that both mods and Clusterio modules can switch
+from any version to any version so it should be able to handle both
+forwards and backwards migrations gracefully.
+
+#### clusterio_api.send_json(channel, data)
+
+Send json data to Clusterio over the given channel.  The `data` argument
+should be table that can be serialized with `game.table_to_json`.
+Clusterio plugins can listen to channels and will receive an event with
+the data sent here.  See the [Communicating with Factorio
+section](writing-plugins.md#communicating-with-factorio) in the Writing
+Plugins document for more information.
+
+If the game is started outside of Clusterio the data will be sent, but
+since there's no code following stdout to pick it up it will be lost.
+
+**Note**: Payloads greater than 4 MB will cause stuttering in the game.
+
+**Note**: This is not a binary safe way of sending data.  Strings
+embedded into the tables sent must be valid UTF-8 text.
+
+Parameters:
+- `channel`: string identifying which channel to send it on.
+- `data`: table that can be converted to JSON with game.table_to_json
+
+
+### serialize library
+
+**require path**  
+From a module  
+`local serialize = require("modules/clusterio/serialize")`  
+From a mod  
+`local serialize = require("__clusterio_lib__/serialize")`
+
+Serializes inventories and item stacks into a table that in turn can be
+converted into JSON with `game.table_to_json`.  Properly handles the
+extra data for most items.
+
+#### serialize.serialize_equipment_grid(LuaEquipmentGrid)
+
+Returns a table containing a serialized representation of the equipment
+grid passed.
+
+#### serialize.deserialize_equipment_grid(LuaEquipmentGrid, serialized_grid)
+
+Deserialize a previously serialized equipment grid into the target
+LuaEquipmentGrid.  Overwrites all content in the target.
+
+#### serialize.serialize_item_stack(LuaItemStack, destination_table)
+
+Serialize the LuaItemStack passed into the target destination_table.
+The destination_table should be a new empty table.
+
+#### serialize.deserialize_item_stack(LuaItemStack, serialized_stack)
+
+Deserialize a previously serialized item stack into the target
+LuaItemStack.  Overwrites the content of the item stack.
+
+#### serialize.serialize_inventory(LuaInventory)
+
+Returns a table containing a serialized representation of the inventory
+passed.
+
+#### serialize.deserialize_inventory(LuaInventory, serialized_inventory)
+
+Deserialize a previously serialized inventory into the target
+LuaInventory.  Overwrites slots that has content in the serialized
+inventory.
 
 
 Clusterio Plugins

mod/api.lua

@@ -0,0 +1,67 @@
+-- Important: Keep this API in sync with modules/clusterio/api.lua
+local api = {}
+
+
+local initialized = false
+function api.init()
+    if initialized then
+        return
+    end
+    initialized = true
+
+    if remote.interfaces.clusterio_api then
+        api.events = remote.call("clusterio_api", "get_events")
+    else
+        local null_event = script.generate_event_name()
+        api.events = {
+            on_instance_updated = null_event,
+            on_server_startup = null_event,
+        }
+    end
+end
+
+local function call_api(fn, ...)
+    if remote.interfaces.clusterio_api and remote.interfaces.clusterio_api[fn] then
+        return remote.call("clusterio_api", fn, ...)
+    else
+        return nil
+    end
+end
+
+function api.get_instance_name()
+    return call_api("get_instance_name")
+end
+
+function api.get_instance_id()
+    return call_api("get_instance_id")
+end
+
+
+function api.send_json(channel, data)
+    if not remote.interfaces.clusterio_api then
+        return
+    end
+
+    -- Escape bad characters.  The question mark is used for separating the
+    -- channel name from the payload.
+    channel = channel:gsub("([\x00-\x1f?\\])", function(match)
+        return "\\x" .. string.format("%02x", match:byte())
+    end)
+
+    data = game.table_to_json(data)
+
+    -- If there's more than about 4kB of data users running with the Windows
+    -- console open will start to experience stuttering, otherwise we could
+    -- output about 1 MB of data at a time through stdout.
+    if #data < 4000 then
+        print("\f$ipc:" .. channel .. "?j" .. data)
+    else
+        local file_no = remote.call("clusterio_api", "get_file_no")
+        local file_name = "clst_" .. file_no .. ".json"
+        game.write_file(file_name, data, false, 0)
+        print("\f$ipc:" .. channel .. "?f" .. file_name)
+    end
+end
+
+
+return api

mod/changelog.txt

@@ -0,0 +1,5 @@
+---------------------------------------------------------------------------------------------------
+Version: 0.1.0
+  Major Features:
+    - API for mods to interact with Clusterio
+    - Serialization library for converting items to JSON compatible tables.

mod/info.json

@@ -0,0 +1,9 @@
+{
+    "name": "clusterio_lib",
+    "version": "0.1.0",
+    "title": "Clusterio Library",
+    "author": "Clusterio Team",
+    "homepage": "https://github.com/clusterio/factorioClusterio",
+    "description": "Library for interfacing with Clusterio.",
+    "factorio_version": "0.17"
+}

modules/clusterio/api.lua

@@ -1,42 +1,20 @@
+-- Important: Keep this API in sync with mod/api.lua
 local api = {}
 
 
--- Table of events raised by clusterio
 api.events = {
-    -- Raised after the name and id of an instance has been updated
-    -- This may occur even if the id and name didn't change.
-
-    -- Event data:
-    --   instance_id: The id of the instance.
-    --   instance_name: the name of the instance.
     on_instance_updated = script.generate_event_name(),
-
-    -- Raised when the server is starting up
-    -- This event serves as the stand-in for the on_init and
-    -- on_configuration_changed events for modules.  It is invoked as early
-    -- as possible, before most but not all other events, every time the
-    -- server is started up.
-
-    -- Use this event to initialize and and/or migrate the data structures
-    -- you need.  Keep in mind that Clusterio can switch from any version of
-    -- your module to any version of your module so it should be able to
-    -- handle both forwards and backwards migrations.
     on_server_startup = script.generate_event_name(),
 }
 
--- Send a table as json to Clusterio
--- Send data to Clusterio over the given channel.  Clusterio plugins can
--- listen to channels and will receive an event with the data sent here.
--- See docs/writing-plugins.md for more information.
+function api.get_instance_name()
+    return global.clusterio.instance_name
+end
 
--- Note: Payloads greater than 4 MB will cause stuttering.
+function api.get_instance_id()
+    return global.clusterio.instance_id
+end
 
--- Note: This is not a binary safe way of sending data.  Strings embedded
--- into the tables sent must be valid UTF-8 text.
-
--- Parameters:
---   channel: string identifying which channel to send it on.
---   data: table that can be converted to json with game.table_to_json
 function api.send_json(channel, data)
 
     -- Escape bad characters.  The question mark is used for separating the

modules/clusterio/impl.lua

@@ -32,26 +32,30 @@ function clusterio_private.update_instance(new_id, new_name)
 end
 
 
-function impl.add_remote_interface()
-    remote.add_interface('clusterio_api', {
-        -- Returns a table of events raised by clusterio
-        get_events = function()
-            return api.events
-        end,
+-- This is not part of the add_remote_interface callback to ensure it is
+-- available when the clusterio_lib mod is loaded.  The reason this is
+-- neccessary is that on_init for newly added mods happen before on_load
+-- for existing mods, and the add_remote_interface callback is done in
+-- on_load.  See https://forums.factorio.com/viewtopic.php?f=25&t=81552 for
+-- more details.
+remote.add_interface('clusterio_api', {
+    get_events = function()
+        return api.events
+    end,
 
-        -- Returns the instance id the game is run under
-        -- This may change over time and/or be nil.
-        get_instance_id = function()
-            return global.clusterio.instance_id
-        end,
+    get_instance_id = function()
+        return global.clusterio.instance_id
+    end,
 
-        -- Returns the name of the instance the game runs under.
-        -- This may change over time and/or be nil.
-        get_instance_name = function()
-            return global.clusterio.instance_name
-        end,
-    })
-end
+    get_instance_name = function()
+        return global.clusterio.instance_name
+    end,
+
+    get_file_no = function()
+        global.clusterio_file_no = (global.clusterio_file_no or 0) + 1
+        return global.clusterio_file_no
+    end,
+})
 
 
 return impl

package.json

@@ -69,6 +69,7 @@
   },
   "devDependencies": {
     "html5-validator": "^1.2.1",
+    "klaw": "^3.0.0",
     "mocha": "^6.2.0",
     "mocha-sinon": "^2.1.0",
     "mocha.parallel": "^0.15.5",