Export settings and all prototypes

Add settings and all prototypes to the export of data done.  Settings
are needed for a settings GUI to function, while prototype data would be
useful to display any sort of information about things from the game.

packages/lib/factorio/export.js

@@ -207,6 +207,42 @@ async function loadLayeredIcon(server, modVersions, item, size, iconCache) {
 	return icon;
 }
 
+const itemTypes = new Set([
+	"item",
+	"ammo",
+	"capsule",
+	"gun",
+	"item-with-entity-data",
+	"item-with-label",
+	"item-with-inventory",
+	"blueprint-book",
+	"item-with-tags",
+	"selection-tool",
+	"blueprint",
+	"copy-paste-tool",
+	"deconstruction-item",
+	"upgrade-item",
+	"module",
+	"rail-planner",
+	"spidertron-remote",
+	"tool",
+	"armor",
+	"mining-tool",
+	"repair-tool",
+
+	// XXX Bad hack to get icons for fluids. These should be treated as a
+	// separate namespace from items as they may be named the same as items.
+	"fluid",
+	// TODO consider adding virtual signals and recipes
+]);
+
+function filterItems(prototypes) {
+	return Object.entries(prototypes)
+		.filter(([type, _]) => itemTypes.has(type))
+		.flatMap(([_, typePrototypes]) => Object.values(typePrototypes))
+	;
+}
+
 /**
  * Export item icons and data
  *
@@ -348,11 +384,29 @@ async function exportLocale(server, modVersions, modOrder, languageCode) {
 async function exportData(server) {
 	await generateExportMod(server);
 
-	let items = [];
+	let settings = {};
+	let prototypes = {};
 	let modVersions = new Map();
 	let modOrder = [];
 
-	server.on("ipc-item_export", data => items.push(data));
+	function add(obj, prototype) {
+		if (!Object.prototype.hasOwnProperty.call(obj, prototype.type)) {
+			obj[prototype.type] = {};
+		}
+		obj[prototype.type][prototype.name] = prototype;
+	}
+
+	server.on("ipc-prototype_export", data => add(prototypes, data));
+	server.on("ipc-settings_export", data => add(settings, data));
+	server.on("ipc-mod_setting_mod", ({ name, mod }) => {
+		for (let type of Object.values(settings)) {
+			if (Object.prototype.hasOwnProperty.call(type, name)) {
+				type[name].mod = mod;
+				return;
+			}
+		}
+		server._logger.error(`Unable to find ${name} in settings prototypes`);
+	});
 	server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); });
 	server.on("output", parsed => {
 		if (parsed.format === "seconds" && parsed.type === "generic") {
@@ -370,8 +424,8 @@ async function exportData(server) {
 		await fs.unlink(server.writePath("mods", "export_0.0.0.zip"));
 	}
 
-	if (!items.length) {
-		throw new Error("No items got exported");
+	if (!Object.keys(prototypes).length) {
+		throw new Error("No prototypes got exported");
 	}
 
 	// Some mod authors put leading zeros into the versions of their zip files.
@@ -396,13 +450,15 @@ async function exportData(server) {
 		}
 	}
 
-	let { iconSheet, itemData } = await exportItems(server, modVersions, items);
+	let { iconSheet, itemData } = await exportItems(server, modVersions, filterItems(prototypes));
 	let locale = await exportLocale(server, modVersions, modOrder, "en");
 
 	// Free up the memory used by zip files loaded during the export.
 	zipCache.clear();
 
 	let zip = new JSZip();
+	zip.file("export/settings.json", JSON.stringify(settings));
+	zip.file("export/prototypes.json", JSON.stringify(prototypes));
 	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
 	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
 	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));

packages/master/src/routes.js

@@ -197,6 +197,8 @@ async function uploadExport(req, res) {
 
 	// This is hardcoded to prevent path expansion attacks
 	let exportFiles = [
+		"export/settings.json",
+		"export/prototypes.json",
 		"export/item-spritesheet.png",
 		"export/item-metadata.json",
 		"export/locale.json",

packages/slave/lua/export/clusterio_api.lua

@@ -0,0 +1,73 @@
+local function is_array(t)
+	local count = 0
+	for k in pairs(t) do
+		if type(k) ~= "number" then
+			return false
+		end
+		count = count + 1
+	end
+
+	for i = 1, count do
+		if not t[i] then
+			return false
+		end
+	end
+
+	return count ~= 0
+end
+
+-- no game.table_to_json here :(
+local function table_to_json(data)
+	if type(data) == "number" then
+		if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it.
+		if data == -math.huge then return "-1e500" end
+		if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead.
+		return string.format("%.17g", data)
+
+	elseif type(data) == "string" then
+		data = data:gsub('([\x00-\x1f\\"])', function(match)
+			return "\\u00" .. string.format("%02x", match:byte())
+		end)
+		return '"' .. data .. '"'
+
+	elseif data == nil then
+		return "null"
+
+	elseif data == true then
+		return "true"
+	elseif data == false then
+		return "false"
+
+	elseif type(data) == "table" then
+		local r
+		if is_array(data) then
+			r = {"["}
+			for i=1, #data do
+				r[#r+1] = table_to_json(data[i])
+				r[#r+1] = ","
+			end
+
+			-- overwrite last comma, empty table is an object
+			r[#r] = "]"
+
+		else
+			r = {"{"}
+			for k, v in pairs(data) do
+				r[#r+1] = table_to_json(tostring(k))
+				r[#r+1] = ":"
+				r[#r+1] = table_to_json(v)
+				r[#r+1] = ","
+			end
+			r[#r + (r[#r] == "," and 0 or 1)] = "}"
+		end
+
+		return table.concat(r)
+	end
+end
+
+local clusterio_api = {}
+function clusterio_api.send_json(channel, data)
+	data = table_to_json(data)
+	print("\f$ipc:" .. channel .. "?j" .. data)
+end
+return clusterio_api

packages/slave/lua/export/control.lua

@@ -5,4 +5,7 @@ end
 
 script.on_init(function()
 	send_json("mod_list", game.active_mods)
+	for name, mod_setting in pairs(game.mod_setting_prototypes) do
+		send_json("mod_setting_mod", { name = name, mod = mod_setting.mod })
+	end
 end)

packages/slave/lua/export/data-final-fixes.lua

@@ -1,104 +1,8 @@
-local item_types = {
-	-- Item categories
-	"item",
-	"ammo",
-	"capsule",
-	"gun",
-	"item-with-entity-data",
-	"item-with-label",
-	"item-with-inventory",
-	"blueprint-book",
-	"item-with-tags",
-	"selection-tool",
-	"blueprint",
-	"deconstruction-item",
-	"copy-paste-tool",
-	"upgrade-item",
-	"module",
-	"rail-planner",
-	"tool",
-	"armor",
-	"mining-tool",
-	"repair-tool",
+local clusterio_api = require("clusterio_api")
 
-	-- Fluids
-	"fluid",
-}
-
-local function is_array(t)
-	local count = 0
-	for k in pairs(t) do
-		if type(k) ~= "number" then
-			return false
-		end
-		count = count + 1
-	end
-
-	for i = 1, count do
-		if not t[i] then
-			return false
-		end
-	end
-
-	return count ~= 0
-end
-
--- no game.table_to_json here :(
-local function table_to_json(data)
-	if type(data) == "number" then
-		if data == math.huge then return "1e500" end -- Inf is not representable in JSON, but 1e500 will overflow to it.
-		if data == -math.huge then return "-1e500" end
-		if data ~= data then return "0" end -- NaN is not representable in JSON, treat as 0 instead.
-		return string.format("%.17g", data)
-
-	elseif type(data) == "string" then
-		data = data:gsub('([\x00-\x1f\\"])', function(match)
-			return "\\u00" .. string.format("%02x", match:byte())
-		end)
-		return '"' .. data .. '"'
-
-	elseif data == nil then
-		return "null"
-
-	elseif data == true then
-		return "true"
-	elseif data == false then
-		return "false"
-
-	elseif type(data) == "table" then
-		local r
-		if is_array(data) then
-			r = {"["}
-			for i=1, #data do
-				r[#r+1] = table_to_json(data[i])
-				r[#r+1] = ","
-			end
-
-			-- overwrite last comma, empty table is an object
-			r[#r] = "]"
-
-		else
-			r = {"{"}
-			for k, v in pairs(data) do
-				r[#r+1] = table_to_json(tostring(k))
-				r[#r+1] = ":"
-				r[#r+1] = table_to_json(v)
-				r[#r+1] = ","
-			end
-			r[#r + (r[#r] == "," and 0 or 1)] = "}"
-		end
-
-		return table.concat(r)
-	end
-end
-
-local function send_json(channel, data)
-	data = table_to_json(data)
-	print("\f$ipc:" .. channel .. "?j" .. data)
-end
-
-for _, group in ipairs(item_types) do
-	for _, item in pairs(data.raw[group]) do
-		send_json("item_export", item)
+log("Exporting prototypes")
+for group, prototypes in pairs(data.raw) do
+	for name, prototype in pairs(prototypes) do
+		clusterio_api.send_json("prototype_export", prototype)
 	end
 end

packages/slave/lua/export/settings-final-fixes.lua

@@ -0,0 +1,8 @@
+local clusterio_api = require("clusterio_api")
+
+log("Exporting settings")
+for group, prototypes in pairs(data.raw) do
+	for name, prototype in pairs(prototypes) do
+		clusterio_api.send_json("settings_export", prototype)
+	end
+end

test/integration/clusterio.js

@@ -310,10 +310,15 @@ describe("Integration of Clusterio", function() {
 				let response = await get("/api/export-manifest");
 				let manifest = JSON.parse(response.body);
 				assert(Object.keys(manifest).length > 1, "Export manifest is empty");
-				assert(
-					await fs.exists(path.join(exportPath, "..", manifest["item-metadata"])),
-					"Export was not written to filesystem"
-				);
+				for (let key of ["settings", "prototypes", "item-metadata", "item-spritesheet", "locale"]) {
+					assert(manifest[key], `Missing ${key} from manifest`);
+					assert(
+						await fs.exists(path.join(exportPath, "..", manifest[key])),
+						`Manifest entry for ${key} was not written to filesystem`
+					);
+				}
+				let prototypes = JSON.parse(await fs.readFile(path.join(exportPath, "..", manifest["prototypes"])));
+				assert(Object.keys(prototypes).length > 50, "Expected there to be more than 50 prototype types");
 				await checkInstanceStatus(44, "stopped");
 			});
 		});