Implement export of item icons and locale

Add mod and command for exporting the Factorio locale including mods,
and item icons assemled into a sprite sheet including mods.

.gitignore

@@ -41,8 +41,7 @@ Factorio*
 package-lock.json
 secret-api-token.txt
 mod_cache.json
-static/pictures/spritesheet.js
-static/pictures/spritesheet.png
+static/export
 static/external
 saves
 /temp/

CHANGELOG.md

@@ -39,6 +39,7 @@ Version 2.0.0
 - Added commands sent statistic.
 - Added Factorio server CPU and memory usage statistic.
 - Added Factorio autosave size statistic.
+- Added support for extracting locale and item icons from Factorio and mods.
 
 ### Changes
 

build-mod.js

@@ -40,6 +40,10 @@ async function build(args) {
 		info.factorio_version = args.factorioVersion;
 	}
 
+	if (args.dependencies) {
+		info.dependencies = args.dependencies;
+	}
+
 	if (args.clean) {
 		let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/
 		for (let entry of await fs.readdir(args.outputDir)) {

clusterctl.js

@@ -255,6 +255,21 @@ commands.push(new Command({
 	},
 }));
 
+commands.push(new Command({
+	definition: ['export-data', "Export item icons and locale from instance", (yargs) => {
+		yargs.options({
+			'instance': { describe: "Instance to export from", nargs: 1, type: 'string', demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await resolveInstance(control, args.instance);
+		await link.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		let response = await link.messages.exportData.send(control, {
+			instance_id: instanceId,
+		});
+	},
+}));
+
 commands.push(new Command({
 	definition: ['start-instance', "Start instance", (yargs) => {
 		yargs.options({

lib/factorio/export.js

@@ -0,0 +1,360 @@
+// Export of item icons and locale
+"use strict";
+const fs = require("fs-extra");
+const path = require("path");
+const ini = require("ini");
+const Jimp = require("jimp");
+const JSZip = require("jszip");
+
+const buildMod = require("../../build-mod");
+
+/**
+ * Generate the export mod needed for exportData
+ *
+ * Packs the lua export mod located in lua/export into the mods folder for
+ * the server given, with dependencies generated for all the mods present in
+ * the server's mods folder.
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to generate the export mod for.
+ * @memberof module:lib/factorio
+ */
+async function generateExportMod(server) {
+	let dependencies = [];
+	let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/
+	for (let entry of await fs.readdir(server.writePath("mods"))) {
+		let match = splitter.exec(entry);
+		if (match && match[1] !== "export") {
+			dependencies.push(`? ${match[1]}`);
+		}
+	}
+
+	await buildMod.build({
+		clean: false,
+		build: true,
+		pack: true,
+		sourceDir: path.join("lua", "export"),
+		outputDir: server.writePath("mods"),
+		bumpPatch: false,
+		factorioVersion: server.version.replace(/\.\d+$/, ""),
+		dependencies,
+	});
+}
+
+let zipCache = new Map();
+async function loadZip(server, modVersions, mod) {
+	let modVersion = modVersions.get(mod);
+	if (!modVersion) {
+		throw new Error(`Got path for unknown mod ${mod}`);
+	}
+
+	let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`);
+	let zip = zipCache.get(zipPath);
+	if (!zip) {
+		zip = await JSZip.loadAsync(await fs.readFile(zipPath));
+		zipCache.set(zipPath, zip);
+	}
+
+	return zip.folder(`${mod}_${modVersion}`);
+}
+
+/**
+ * Load the given Factorio file path into a Buffer
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to load the file from.
+ * @param {Map<string, string>} modVersions - Mapping of mod to version used.
+ * @param {string} modPath - Factorio style path to the file to load.
+ * @returns {?Buffer} The content of the file or null if not found.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
+ */
+async function loadFile(server, modVersions, modPath) {
+	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
+	if (!match) {
+		throw new Error(`Bad mod path ${match}`);
+	}
+
+	let [, mod, filePath] = match;
+
+	if (["core", "base"].includes(mod)) {
+		try {
+			return await fs.readFile(server.dataPath(mod, filePath));
+		} catch (err) {
+			if (err.code === "ENOENT") {
+				return null;
+			}
+			throw err;
+		}
+	}
+
+	let zip = await loadZip(server, modVersions, mod);
+	let file = zip.file(filePath);
+	if (!file) {
+		return null;
+	}
+
+	return await file.async("nodebuffer");
+}
+
+/**
+ * Export item icons and data
+ *
+ * Assembles and packs the icons for the item prototypes given into a single
+ * spritesheet and json file with meta data.
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to generate the export mod for.
+ * @param {Array<Object>} items - Array of item prototypes.
+ * @return {itemSheet: Jimp, itemData: Map<string, Object>} Item spritesheet and metadata.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
+ */
+async function exportItems(server, modVersions, items) {
+
+	// Size to render icons at
+	let size = 32;
+
+	// Width of spritesheet
+	let width = 1024;
+
+	let rows = Math.ceil(items.length / (width / size));
+	let iconSheet = await Jimp.create(width, rows * size);
+	let itemData = new Map();
+	let pos = 0;
+
+	let iconCache = new Map();
+	async function loadIcon(path, iconSize, iconMipmaps) {
+		let icon = iconCache.get(path);
+		if (icon === undefined) {
+			let fileContent = await loadFile(server, modVersions, path);
+			if (fileContent) {
+				icon = await Jimp.read(fileContent);
+				icon.crop(0, 0, iconSize, iconSize);
+				iconCache.set(path, icon);
+			} else {
+				icon = null;
+				console.log(`Warning: ${path} not found`);
+			}
+			iconCache.set(path, icon);
+		}
+		return icon;
+	}
+
+	let simpleIcons = new Map();
+	for (let item of items) {
+		let icon;
+		let iconPos;
+		if (item.icons) {
+			let baseLayerSize = item.icons[0].icon_size || item.icon_size;
+			icon = await Jimp.create(size, size);
+			iconPos = pos;
+
+			// The scaling factor of the base layer
+			let baseLayerScale = item.icons[0].scale || 32 / baseLayerSize;
+
+			// The size in pixels of one unit
+			let baseUnit = size / (baseLayerSize * baseLayerScale);
+
+			for (let layer of item.icons) {
+				let layerSize = layer.icon_size || item.icon_size;
+				let iconLayer = await loadIcon(layer.icon, layerSize, layer.icon_mipmaps || 0);
+
+				if (!iconLayer) {
+					continue;
+				}
+
+				iconLayer = iconLayer.clone();
+
+				let tint;
+				if (layer.tint) {
+					let divisor = (layer.tint.r > 1 || layer.tint.g > 1 || layer.tint.b > 1) ? 255 : 1;
+					tint = {
+						r: (layer.tint.r || 0) / divisor,
+						g: (layer.tint.g || 0) / divisor,
+						b: (layer.tint.b || 0) / divisor,
+						a: layer.tint.a || 1,
+					};
+				} else {
+					tint = { r: 1, b: 1, g: 1, a: 1 };
+				}
+
+				let layerScale = layer.scale || 32 / layerSize;
+				let [xs, ys] = layer.shift || [0, 0];
+				let realScale = layerScale * baseUnit;
+				let sizeShift = (size - layerSize * realScale) / 2;
+
+				xs = xs * baseUnit + sizeShift;
+				ys = ys * baseUnit + sizeShift;
+
+				if (realScale !== 1) {
+					iconLayer.scale(realScale);
+				}
+
+				iconLayer.scan(0, 0, iconLayer.bitmap.width, iconLayer.bitmap.height, function(x, y, sidx) {
+					x += xs;
+					y += ys;
+					if (x < 0 || x >= size || y < 0 || y >= size) {
+						return;
+					}
+					let sdata = this.bitmap.data;
+					let ddata = icon.bitmap.data;
+					let didx = icon.getPixelIndex(x, y);
+
+					let sa = sdata[sidx + 3] / 255;
+					let da = ddata[didx + 3] / 255;
+					let db = da * (1 - sa * tint.a);
+					let cb = sa + da * (1 - sa);
+					ddata[didx + 0] = Math.min(255, (sdata[sidx + 0] * sa * tint.r + ddata[didx + 0] * db) / cb);
+					ddata[didx + 1] = Math.min(255, (sdata[sidx + 1] * sa * tint.g + ddata[didx + 1] * db) / cb);
+					ddata[didx + 2] = Math.min(255, (sdata[sidx + 2] * sa * tint.b + ddata[didx + 2] * db) / cb);
+					ddata[didx + 3] = Math.min(255, sdata[sidx + 3] + ddata[didx + 3] * (1 - sa));
+				});
+			}
+
+		} else {
+			iconPos = simpleIcons.get(item.icon);
+			if (iconPos === undefined) {
+				icon = await loadIcon(item.icon, item.icon_size, 0);
+				if (icon) {
+					iconPos = pos;
+					simpleIcons.set(item.icon, pos);
+					let iconScale = size / item.icon_size;
+					if (iconScale !== 1) {
+						icon = icon.clone();
+						icon.scale(iconScale);
+					}
+				}
+			}
+		}
+
+		if (iconPos !== undefined) {
+			itemData.set(item.name, {
+				x: iconPos * size % width,
+				y: Math.floor(iconPos / (width / size)) * size,
+				size,
+				localised_name: item.localised_name,
+				localised_description: item.localised_description,
+			});
+		}
+
+		if (icon) {
+			iconSheet.composite(icon, pos * size % width, Math.floor(pos / (width / size)) * size);
+			pos += 1;
+		}
+	}
+
+	iconSheet.crop(0, 0, width, Math.ceil(pos / (width / size)) * size);
+
+	return {iconSheet, itemData};
+}
+
+/**
+ * Exports the locale files for the base game and the given mods
+ *
+ * Parses and merges all the locales for the all the mods given through
+ * `modVersions` and `modOrder`.
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to export the locale from.
+ * @param modVersions {Map<string, string>} modVersions -
+ *     Mapping of mod name to version to export locale from.
+ * @param modOrder {Array<string>} modOrder - Load order of the mods.
+ * @returns {Map<string, string>} merged locale information
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
+ */
+async function exportLocale(server, modVersions, modOrder, languageCode) {
+	let mergedLocales = new Map();
+
+	function mergeLocale(locale) {
+		for (let [category, entries] of Object.entries(locale)) {
+			if (typeof entries === "string") {
+				mergedLocales.set(category, entries);
+
+			} else {
+				for (let [key, value] of Object.entries(entries)) {
+					mergedLocales.set(`${category}.${key}`, value);
+				}
+			}
+		}
+	}
+
+	let baseLocaleFilePath = server.dataPath("base", "locale", languageCode, "base.cfg");
+	mergeLocale(ini.parse(await fs.readFile(baseLocaleFilePath, "utf8")));
+
+	for (let mod of modOrder) {
+		if (["core", "base", "export"].includes(mod)) {
+			continue;
+		}
+
+		let zip = await loadZip(server, modVersions, mod);
+		for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) {
+			let content = await file.async("nodebuffer");
+			mergeLocale(ini.parse(content.toString("utf8")));
+		}
+	}
+
+	return mergedLocales;
+}
+
+/**
+ * Export the locale and item icons for the given factorio server
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to export the data from.
+ * @memberof module:lib/factorio
+ */
+async function exportData(server) {
+	await generateExportMod(server);
+
+	let items = [];
+	let modVersions = new Map();
+	let modOrder = [];
+
+	server.on("ipc-item_export", data => items.push(data));
+	server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); });
+	server.on("output", data => {
+		if (data.format === "seconds" && data.type === "generic") {
+			let match = /^Checksum of (.*): \d+$/.exec(data.message);
+			if (match) {
+				modOrder.push(match[1]);
+			}
+		}
+	});
+
+	try {
+		await server.startScenario("base/freeplay");
+		await server.kill();
+	} finally {
+		await fs.unlink(server.writePath("mods", "export_0.0.0.zip"));
+	}
+
+	if (!items.length) {
+		throw new Error("No items got exported");
+	}
+
+	let {iconSheet, itemData} = await exportItems(server, modVersions, items);
+	let locale = await exportLocale(server, modVersions, modOrder, "en");
+
+	// Free up the memory used by zip files loaded during the export.
+	zipCache.clear();
+
+	let zip = new JSZip();
+	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
+	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
+	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
+	let content = await zip.generateAsync({ type: "nodebuffer" });
+	await fs.outputFile("temp/export.zip", content);
+}
+
+
+module.exports = {
+	exportData,
+
+	// For testing only
+	_exportLocale: exportLocale,
+};

lib/factorio/index.js

@@ -4,7 +4,7 @@
  * @author Hornwitser
  */
 module.exports = {
-	...require('lib/factorio/locale'),
+	...require("lib/factorio/export"),
 	...require('lib/factorio/patch'),
 	...require('lib/factorio/server'),
 };

lib/factorio/locale.js

@@ -1,26 +0,0 @@
-// Function for parsing factorio locale from a local factorio install
-"use strict";
-const fs = require("fs-extra");
-const path = require("path");
-const ini = require("ini");
-
-/**
- * Gets factorios locale as an object, does not respect mods.
- *
- * @param {string} factorioDirectory
- * @param {string} languageCode
- * @param {function} callback
- * @memberof module:lib/factorio
- */
-async function getLocale(dataDir, languageCode) {
-	if (typeof dataDir !== "string") throw new TypeError("dataDir must be a string");
-	if (typeof languageCode !== "string") throw new TypeError("languageCode must be a string");
-
-	let localeFilePath = path.join(dataDir, "base", "locale", languageCode, "base.cfg");
-	let content = await fs.readFile(localeFilePath, "utf8");
-	return ini.parse(content);
-}
-
-module.exports = {
-	getLocale,
-};

lib/factorio/server.js

@@ -955,6 +955,18 @@ class FactorioServer extends events.EventEmitter {
 		}
 	}
 
+	/**
+	 * Kill the server
+	 *
+	 * Terminates the server without any cleanup or saving.
+	 */
+	async kill() {
+		this._check(["running", "stopping"]);
+		this._state = "stopping";
+		this._server.kill("SIGKILL");
+		await events.once(this._server, "exit");
+	}
+
 	/**
 	 * Disable achievements on the running server
 	 *

lib/link/messages.js

@@ -345,6 +345,12 @@ messages.createSave = new Request({
 	forwardTo: 'instance',
 });
 
+messages.exportData = new Request({
+	type: "export_data",
+	links: ["control-master", "master-slave", "slave-instance"],
+	forwardTo: "instance",
+});
+
 messages.stopInstance = new Request({
 	type: 'stop_instance',
 	links: ['control-master', 'master-slave', 'slave-instance'],

lib/npmPostinstall.js

@@ -56,17 +56,6 @@ var directories = [
 	"instances",
 ];
 let downloadList = [
-	// {
-		// url: "http://fbpviewer.trakos.pl/images/spritesheet.json",
-		// file: "static/pictures/spritesheet.js",
-		// callback: function(file){
-			// fs.writeFileSync("static/pictures/spritesheet.js", "export default\n"+fs.readFileSync("static/pictures/spritesheet.js"));
-		// },
-	// },
-	// {
-		// url: "http://fbpviewer.trakos.pl/images/spritesheet.png",
-		// file: "static/pictures/spritesheet.png",
-	// },
 	{
 		url: "https://code.jquery.com/jquery.js",
 		file: "static/external/jquery.js",

lua/export/control.lua

@@ -0,0 +1,8 @@
+local function send_json(channel, data)
+    data = game.table_to_json(data)
+    print("\f$ipc:" .. channel .. "?j" .. data)
+end
+
+script.on_init(function()
+    send_json("mod_list", game.active_mods)
+end)

lua/export/data-final-fixes.lua

@@ -0,0 +1,101 @@
+local item_types = {
+    -- Item categories
+    "item",
+    "ammo",
+    "capsule",
+    "gun",
+    "item-with-entity-data",
+    "item-with-label",
+    "item-with-inventory",
+    "blueprint-book",
+    "item-with-tags",
+    "selection-tool",
+    "blueprint",
+    "deconstruction-item",
+    "copy-paste-tool",
+    "upgrade-item",
+    "module",
+    "rail-planner",
+    "tool",
+    "armor",
+    "mining-tool",
+    "repair-tool",
+
+    -- Fluids
+    "fluid",
+}
+
+local function is_array(t)
+    local count = 0
+    for k in pairs(t) do
+        if type(k) ~= "number" then
+            return false
+        end
+        count = count + 1
+    end
+
+    for i = 1, count do
+        if not t[i] then
+            return false
+        end
+    end
+
+    return count ~= 0
+end
+
+-- no game.table_to_json here :(
+local function table_to_json(data)
+    if type(data) == "number" then
+        return string.format("%.17g", data)
+
+    elseif type(data) == "string" then
+        data = data:gsub('([\x00-\x1f\\"])', function(match)
+            return "\\x" .. string.format("%02x", match:byte())
+        end)
+        return '"' .. data .. '"'
+
+    elseif data == nil then
+        return "null"
+
+    elseif data == true then
+        return "true"
+    elseif data == false then
+        return "false"
+
+    elseif type(data) == "table" then
+        local r
+        if is_array(data) then
+            r = {"["}
+            for i=1, #data do
+                r[#r+1] = table_to_json(data[i])
+                r[#r+1] = ","
+            end
+
+            -- overwrite last comma, empty table is an object
+            r[#r] = "]"
+
+        else
+            r = {"{"}
+            for k, v in pairs(data) do
+                r[#r+1] = table_to_json(tostring(k))
+                r[#r+1] = ":"
+                r[#r+1] = table_to_json(v)
+                r[#r+1] = ","
+            end
+            r[#r + (r[#r] == "," and 0 or 1)] = "}"
+        end
+
+        return table.concat(r)
+    end
+end
+
+local function send_json(channel, data)
+    data = table_to_json(data)
+    print("\f$ipc:" .. channel .. "?j" .. data)
+end
+
+for _, group in ipairs(item_types) do
+    for _, item in pairs(data.raw[group]) do
+        send_json("item_export", item)
+    end
+end

lua/export/info.json

@@ -0,0 +1,8 @@
+{
+    "name": "export",
+    "version": "0.0.0",
+    "title": "Icon exporter",
+    "author": "Clusterio Team",
+    "homepage": "https://github.com/clusterio/factorioClusterio",
+    "description": "Exports item prototypes on stdout during the data stage."
+}

master.js

@@ -39,7 +39,6 @@ let pluginList = {};
 // homebrew modules
 const generateSSLcert = require("lib/generateSSLcert");
 const database = require("lib/database");
-const factorio = require("lib/factorio");
 const schema = require("lib/schema");
 const link = require("lib/link");
 const errors = require("lib/errors");
@@ -88,7 +87,6 @@ app.use(function(req, res, next){
 });
 
 require("./routes")(app);
-require("./routes/api/getPictures")(app);
 // Set folder to serve static content from (the website)
 app.use(express.static('static'));
 
@@ -327,27 +325,6 @@ app.get("/api/modmeta", async function(req, res) {
 });
 
 
-var localeCache;
-/**
-GET endpoint. Returns factorio's base locale as a JSON object.
-
-@memberof clusterioMaster
-@instance
-@alias api/getFactorioLocale
-@returns {object<string, object>} 2 deep nested object with base game factorio locale as key:value pairs
-*/
-app.get("/api/getFactorioLocale", function(req,res){
-	endpointHitCounter.labels(req.route.path).inc();
-	if (!localeCache) {
-		factorio.getLocale(path.join(masterConfig.get("master.factorio_directory"), "data"), "en").then(locale => {
-			localeCache = locale;
-			res.send(localeCache);
-		});
-	} else {
-		res.send(localeCache);
-	}
-});
-
 /**
  * Base class for master server connections
  *

package.json

@@ -45,6 +45,7 @@
     "follow-redirects": "^1.5.0",
     "fs-extra": "^8.1.0",
     "ini": "^1.3.5",
+    "jimp": "^0.12.1",
     "jsonwebtoken": "^8.2.1",
     "jszip": "^3.2.2",
     "lib": "file:./lib",

plugins/subspace_storage/static/storage.js

@@ -26,14 +26,14 @@ function drawcontents(data) {
 		let row = rows[i];
 		if(!row) {
 			row = document.createElement('tr');
-			row.innerHTML = "<td><img width=32 height=32></td><td class=name></td><td class=count></td>";
+			row.innerHTML = "<td><div></div></td><td class=name></td><td class=count></td>";
 			table.appendChild(row);
 		}
 		
-		const img = row.querySelector('img');
-		const imgName = getImageFromName(item.name);
-		if(img.getAttribute('src') !== imgName) {
-			img.setAttribute('src',imgName);
+		const itemIcon = row.querySelector("div");
+		const iconClass = getItemIconClass(item.name);
+		if (itemIcon.getAttribute("class") !== iconClass) {
+			itemIcon.setAttribute("class", iconClass);
 		}
 		
 		const name = row.querySelector('.name');

plugins/subspace_storage/views/storage.ejs

@@ -23,7 +23,7 @@
                 <thead class="thead-dark">
                 <tr>
                     <th scope="col">Icon</th>
-                    <th scope="col">Item ID</th>
+                    <th scope="col">Resource</th>
                     <th scope="col">Quantity</th>
                 </tr>
                 </thead>

routes/api/getPictures.js

@@ -1,16 +0,0 @@
-const fs = require("fs-extra")
-const path = require("path")
-
-module.exports = function(app){
-	app.get("/api/getPictures", async (req, res) => {
-		let pictures = (await fs.readdir(path.join(__dirname, `../../static/pictures`)))
-		.filter(path => path.includes(".png"))
-		.map(name => ({
-			name: name.replace(".png",""),
-			path: `/pictures/${name}`
-		}))
-		
-		
-		res.send(pictures)
-	})
-}

slave.js

@@ -438,6 +438,20 @@ class Instance extends link.Link{
 		console.log("Clusterio | Successfully created save");
 	}
 
+	async exportDataRequestHandler() {
+		try {
+			console.log("Clusterio | Writing server-settings.json");
+			await this.writeServerSettings();
+
+			console.log("Exporting data .....");
+			await symlinkMods(this, "sharedMods", console);
+			await factorio.exportData(this.server);
+
+		} finally {
+			this.notifyExit();
+		}
+	}
+
 	async stopInstanceRequestHandler() {
 		await this.stop();
 	}
@@ -834,6 +848,12 @@ class Slave extends link.Link {
 		await request.send(instanceConnection, message.data);
 	}
 
+	async exportDataRequestHandler(message, request) {
+		let instanceId = message.data.instance_id;
+		let instanceConnection = await this._connectInstance(instanceId);
+		await request.send(instanceConnection, message.data);
+	}
+
 	async stopInstance(instanceId) {
 		let instanceConnection = this.instanceConnections.get(instanceId);
 		await link.messages.stopInstance.send(instanceConnection, { instance_id: instanceId });

static/main.js

@@ -185,73 +185,34 @@ function replaceAll(target, search, replacement) {
 		return target.split(search).join(replacement);
 };
 
-function getImageFromName(name){
-	if(imagelinks[name]){
-		return root + imagelinks[name].slice(1);
-	} else if(imagedata[name]) {
-		return "https://wiki.factorio.com/images/" + imagedata[name] + ".png";
-	} else if(factorioLocale["entity-name"] && factorioLocale["entity-name"][name]){
-		return "https://wiki.factorio.com/images/" + capitalizeFirstLetter(replaceAll(factorioLocale["entity-name"][name], ' ', '_').toLowerCase()) + ".png";
-	} else if(factorioLocale["item-name"] && factorioLocale["item-name"][name]){
-		return "https://wiki.factorio.com/images/" + capitalizeFirstLetter(replaceAll(factorioLocale["item-name"][name], ' ', '_').toLowerCase()) + ".png";
-	} else if(factorioLocale["fluid-name"] && factorioLocale["fluid-name"][name]){
-		return "https://wiki.factorio.com/images/" + capitalizeFirstLetter(replaceAll(factorioLocale["fluid-name"][name], ' ', '_').toLowerCase()) + ".png";
-	} else if(factorioLocale["equipment-name"] && factorioLocale["equipment-name"][name]){
-		return "https://wiki.factorio.com/images/" + capitalizeFirstLetter(replaceAll(factorioLocale["equipment-name"][name], ' ', '_').toLowerCase()) + ".png";
-	} else return `${root}pictures/unknown-item.png`;
+function getItemIconClass(name) {
+	if (factorioItemMetadata.has(name)) {
+		return `item-${name}`;
+	} else {
+		return "item-unknown-item";
+	}
 }
-window.factorioLocale = {};
-$.getJSON(`${root}api/getFactorioLocale`, (locale) => {
-	window.factorioLocale = locale;
+window.factorioLocale = new Map();
+$.getJSON(`${root}export/locale.json`, (locale) => {
+	window.factorioLocale = new Map(locale);
 });
-// image data
-// key is the name of the item in the database, value is the name of the image on wiki.factorio.com/images/*
-// this list has to include all entities that doesn't just follow the simple capitalize first letter convention
-/* Notice from bilka: All icon images hosted on this Wiki have been renamed to use their in-game names, with underscores. If your external tool references these icons, please correct your links. */
-var imagedata = {
-	"put-combinator": "Decider_combinator",
-	"smart-train-stop": "Train_stop",
-	"raw-fish": "Fish",
-	"solar-panel-equipment": "Portable_solar_panel",
-	"diesel-locomotive": "Locomotive", // pre 0.15 loco name
-	"small-lamp": "Lamp",
-	"tree-02": "Green_tree",
-	"tree-02-red": "Green_tree",
-	"tree-03": "Green_tree",
-	"tree-04": "Green_tree",
-	"tree-05": "Green_tree",
-	"tree-07": "Green_tree",
-	"tree-09": "Green_tree",
-	"tree-09-brown": "Green_tree",
-	"tree-09-red": "Green_tree",
-	"dead-tree-desert": "Dead_tree_desert",
-	"sand-rock-big": "Rock_big",
-	// "biter-spawner" missing, but can't find png for it on wiki
-	// "spitter-spawner"
-	
-}
+window.factorioItemMetadata = new Map();
+$.getJSON(`${root}export/item-metadata.json`, (itemMetadata) => {
+	window.factorioItemMetadata = new Map(itemMetadata);
+	console.log(factorioItemMetadata.get("electronic-circuit"));
+	let style = document.createElement("style");
+	style.type = "text/css";
+	document.head.appendChild(style);
 
-// value:link pairs where you can add modded item icons that are missing
-var imagelinks = {
-	"crude-oil-barrel": "/pictures/Crude_oil_barrel.png",
-	"heavy-oil-barrel": "/pictures/Heavy_oil_barrel.png",
-	"light-oil-barrel": "/pictures/Light_oil_barrel.png",
-	"petroleum-gas-barrel": "/pictures/Petroleum_gas_barrel.png",
-	"sulfuric-acid-barrel": "/pictures/Sulfuric_acid_barrel.png",
-	"water-barrel": "/pictures/Water_barrel.png",
-	"lubricant-barrel": "/pictures/Lubricant_barrel.png",
-	"power-armor-mk2": "/pictures/Power_armor_MK2.png",
-	"fictional-modded-item": "/pictures/fictional-modded-item.png",
-	"raw-fish": "/pictures/Fish.png",
-	"rail": "/pictures/rail.png",
-	"effectivity-module-1": "/pictures/green_module_1.png",
-	"effectivity-module-2": "/pictures/green_module_2.png",
-	"effectivity-module-3": "/pictures/green_module_3.png",
-	"piercing-shotgun-shell": "/pictures/piercing_shotgun_shells.png",
-	"energy-shield-mk2-equipment": "/pictures/Energy_shield_MK2.png",
-	"battery-mk2-equipment": "/pictures/Battery_MK2.png",
-	"personal-roboport-mk2-equipment": "/pictures/Personal_roboport_MK2.png",
-}
-var populateImageLinks = async () => (await getJSON(`${root}api/getPictures`)).forEach(img => imagelinks[img.name] = img.path)
-
-populateImageLinks()
+	for (let [name, meta] of itemMetadata) {
+		style.sheet.insertRule([
+			`.item-${name} {`,
+			` background-image: url("${root}export/item-spritesheet.png");`,
+			" background-repeat: no-repeat;",
+			` background-position: -${meta.x}px -${meta.y}px;`,
+			` height: ${meta.size}px;`,
+			` width: ${meta.size}px;`,
+			"}\n",
+		].join(""));
+	}
+});

static/style.css

@@ -79,4 +79,10 @@ input:checked + .slider:before {
 
 .slider.round:before {
     border-radius: 50%;
-}
+}
+
+.item-unknown-item {
+    background: url("pictures/unknown-item.png");
+    height: 32px;
+    width: 32px;
+}

test/file/factorio/data/base/locale/en/base.cfg

@@ -1,4 +1,4 @@
-; Test data for getLocale
+; Test data for exportLocale
 [test]
 key-a=1
 key-b=2

test/integration/clusterio.js

@@ -8,19 +8,6 @@ const { slowTest, get, exec, controlConfigPath, instancesDir } = require("./inde
 
 
 describe("Integration of Clusterio", function() {
-	describe("master http api", function() {
-		describe("GET /api/getFactorioLocale", function() {
-			it("should get the basegame factorio locale", async function() {
-				let res = await get("/api/getFactorioLocale");
-				let object = res.body;
-
-				// test that it is looks like a factorio locale
-				assert.equal(typeof object, "object");
-				assert.equal(object["entity-name"]["fish"], "Fish");
-				assert.equal(object["entity-name"]["small-lamp"], "Lamp");
-			});
-		});
-	});
 	parallel("master web interface", function() {
 		this.timeout(6000);
 
@@ -71,6 +58,14 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("export-data", function() {
+			it("exports the data", async function() {
+				slowTest(this);
+				await exec(`node clusterctl --config ${controlConfigPath} export-data --instance test`);
+				assert(await fs.exists(path.join("temp", "export.zip")), "Export was not created");
+			});
+		});
+
 		describe("start-instance", function() {
 			it("starts the instance", async function() {
 				slowTest(this);

test/integration/export.js

@@ -0,0 +1,33 @@
+const assert = require("assert").strict;
+const path = require("path");
+
+const factorio = require("lib/factorio");
+
+
+describe("Integration of lib/factorio/export", function() {
+	describe("exportLocale()", function() {
+		it("returns a nested Map with base game locale information", async function() {
+			let writePath = path.join("temp", "test", "server");
+			let testServer = new factorio.FactorioServer("factorio", writePath, {});
+			await testServer.init();
+
+			let locale = await factorio._exportLocale(testServer, new Map(), ["base"], "en");
+
+			assert(locale instanceof Map, "locale is not a map");
+			assert.equal(locale.get("entity-name.fish"), "Fish");
+			assert.equal(locale.get("entity-name.small-lamp"), "Lamp");
+			let empty_keys = 0;
+			for (let [key, value] of locale) {
+				// key is always a string
+				assert.equal(typeof key, "string");
+
+				// Value is always a string, nearly always truthy
+				if (!value) {
+					empty_keys++;
+				}
+				assert.equal(typeof value, "string");
+			}
+			assert(empty_keys < 10, `Got unusually high amount (${empty_keys}) of empty keys`);
+		});
+	});
+});

test/integration/locale.js

@@ -1,62 +0,0 @@
-const assert = require("assert").strict;
-const path = require("path");
-
-const factorio = require("lib/factorio");
-
-
-describe("Integration of lib/factorio/locale", function() {
-	describe("getLocale()", function() {
-		let locale;
-		before(async function() {
-			locale = await factorio.getLocale(path.join("factorio", "data"), "en");
-		});
-
-		it("returns a nested object with base game locale information", async function() {
-			assert.equal(typeof locale, "object");
-			assert.equal(locale["entity-name"]["fish"], "Fish");
-			assert.equal(locale["entity-name"]["small-lamp"], "Lamp");
-			let empty_keys = 0;
-			Object.keys(locale).forEach(key => {
-				// first level of the nested object is always an object
-				assert.equal(typeof locale[key], "object");
-
-				Object.keys(locale[key]).forEach(key2 => {
-					// second level of the nested object is always a string, nearly always truthy
-					if (!locale[key][key2]) {
-						empty_keys++;
-					}
-					assert.equal(typeof locale[key][key2], "string");
-				});
-
-			});
-			assert(empty_keys < 10, `Got unusually high amount (${empty_keys}) of empty keys`);
-		});
-
-		it("has some duplicate keys", async function() {
-			let arrayOfKeys = [];
-			// collect all keys in a single array
-			Object.keys(locale).forEach(key => {
-				arrayOfKeys.push(key);
-				Object.keys(locale[key]).forEach(key2 => {
-					arrayOfKeys.push(key2);
-				});
-			});
-
-			// check for duplicates
-			let duplicateKeys = [];
-			let i = 0;
-			arrayOfKeys.forEach(key => {
-				let o = 0;
-				arrayOfKeys.forEach(key2 => {
-					if(i != o && key == key2){
-						duplicateKeys.push(key);
-					}
-					o++
-				});
-				i++;
-			});
-			// we got 704 but this is a very unexact number
-			assert(duplicateKeys.length > 10)
-		});
-	});
-});

test/lib/factorio/export.js

@@ -0,0 +1,21 @@
+const assert = require("assert").strict;
+const path = require("path");
+
+const factorio = require("lib/factorio");
+
+
+describe("lib/factorio/export", function() {
+	describe("exportLocale()", function() {
+		let testServer;
+		before(async function() {
+			let writePath = path.join("temp", "test", "server");
+			testServer = new factorio.FactorioServer(path.join("test", "file", "factorio"), writePath, {});
+			await testServer.init();
+		});
+
+		it("returns a flattened mapping with locale definitions", async function() {
+			let locale = await factorio._exportLocale(testServer, new Map(), ["base"], "en");
+			assert.deepEqual(locale, new Map([["test.key-a", "1"], ["test.key-b", "2"]]));
+		});
+	});;
+});

test/lib/factorio/locale.js

@@ -1,18 +0,0 @@
-const assert = require("assert").strict;
-const path = require("path");
-
-const factorio = require("lib/factorio");
-
-
-describe("lib/factorio/locale", function() {
-	describe("getLocale()", function() {
-		it("verifies types of the arguments", async function() {
-			await assert.rejects(factorio.getLocale(1, "a"), new TypeError("dataDir must be a string"));
-			await assert.rejects(factorio.getLocale("a", 1), new TypeError("languageCode must be a string"));
-		});
-		it("returns a nested object with locale definitions", async function() {
-			let locale = await factorio.getLocale(path.join("test", "file", "factorio", "data"), "en");
-			assert.deepEqual(locale, { test: { "key-a": "1", "key-b": "2" } });
-		});
-	});;
-});