mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-23 00:13:13 +09:00
Merge pull request #297 from Hornwitser/automate-export
Automate export process
This commit is contained in:
7 files changed
+71
-17
No files matched your search
@@ -19,8 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
|
|
||||||
The icons for angels modded items located in /static/pictures/ are licensed
|
|
||||||
by https://mods.factorio.com/user/Arch666Angel under CC BY-NC-ND 4.0
|
|
||||||
https://creativecommons.org/licenses/by-nc-nd/4.0/
|
|
||||||
|
|
||||||
@@ -306,6 +306,7 @@ async function exportLocale(server, modVersions, modOrder, languageCode) {
|
|||||||
*
|
*
|
||||||
* @param {module:lib/factorio.FactorioServer} server -
|
* @param {module:lib/factorio.FactorioServer} server -
|
||||||
* The server to export the data from.
|
* The server to export the data from.
|
||||||
|
* @returns {JSZip} zip file with exported data.
|
||||||
* @memberof module:lib/factorio
|
* @memberof module:lib/factorio
|
||||||
*/
|
*/
|
||||||
async function exportData(server) {
|
async function exportData(server) {
|
||||||
@@ -347,8 +348,7 @@ async function exportData(server) {
|
|||||||
zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
|
zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
|
||||||
zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
|
zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
|
||||||
zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
|
zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
|
||||||
let content = await zip.generateAsync({ type: "nodebuffer" });
|
return zip;
|
||||||
await fs.outputFile("temp/export.zip", content);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ const setBlocking = require("set-blocking");
|
|||||||
const events = require("events");
|
const events = require("events");
|
||||||
const yargs = require("yargs");
|
const yargs = require("yargs");
|
||||||
const WebSocket = require("ws");
|
const WebSocket = require("ws");
|
||||||
|
const JSZip = require("jszip");
|
||||||
const version = require("./package").version;
|
const version = require("./package").version;
|
||||||
|
|
||||||
// ugly globals
|
// ugly globals
|
||||||
@@ -174,6 +175,45 @@ async function getMetrics(req, res, next) {
|
|||||||
}
|
}
|
||||||
app.get('/metrics', (req, res, next) => getMetrics(req, res, next).catch(next));
|
app.get('/metrics', (req, res, next) => getMetrics(req, res, next).catch(next));
|
||||||
|
|
||||||
|
// Handle an uploaded export package.
|
||||||
|
async function uploadExport(req, res, next) {
|
||||||
|
endpointHitCounter.labels(req.route.path).inc();
|
||||||
|
if (req.get("Content-Type") !== "application/zip") {
|
||||||
|
res.sendStatus(415);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = [];
|
||||||
|
for await (let chunk of req) {
|
||||||
|
data.push(chunk);
|
||||||
|
}
|
||||||
|
data = Buffer.concat(data);
|
||||||
|
let zip = await JSZip.loadAsync(data);
|
||||||
|
delete data;
|
||||||
|
|
||||||
|
// This is hardcoded to prevent path expansion attacks
|
||||||
|
let exportFiles = [
|
||||||
|
"export/item-spritesheet.png",
|
||||||
|
"export/item-metadata.json",
|
||||||
|
"export/locale.json",
|
||||||
|
];
|
||||||
|
|
||||||
|
for (let filePath of exportFiles) {
|
||||||
|
let file = zip.file(filePath);
|
||||||
|
if (!file) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let name = path.posix.basename(filePath);
|
||||||
|
await fs.outputFile(path.join("static", "export", name), await file.async("nodebuffer"));
|
||||||
|
}
|
||||||
|
|
||||||
|
res.sendStatus(200);
|
||||||
|
}
|
||||||
|
app.put("/api/upload-export",
|
||||||
|
authenticate.middleware,
|
||||||
|
(req, res, next) => uploadExport(req, res, next).catch(next)
|
||||||
|
);
|
||||||
|
|
||||||
const masterConnectedClientsCount = new prometheus.Gauge(
|
const masterConnectedClientsCount = new prometheus.Gauge(
|
||||||
'clusterio_master_connected_clients_count', "How many clients are currently connected to this master server",
|
'clusterio_master_connected_clients_count', "How many clients are currently connected to this master server",
|
||||||
|
|||||||
@@ -55,6 +55,7 @@
|
|||||||
"needle": "^2.4.0",
|
"needle": "^2.4.0",
|
||||||
"node-factorio-api": "^0.3.8",
|
"node-factorio-api": "^0.3.8",
|
||||||
"node-forge": "^0.7.1",
|
"node-forge": "^0.7.1",
|
||||||
|
"phin": "^3.4.1",
|
||||||
"pidusage": "^2.0.18",
|
"pidusage": "^2.0.18",
|
||||||
"rcon-client": "^4.2.0",
|
"rcon-client": "^4.2.0",
|
||||||
"request": "^2.83.0",
|
"request": "^2.83.0",
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ const yargs = require("yargs");
|
|||||||
const events = require("events");
|
const events = require("events");
|
||||||
const pidusage = require("pidusage");
|
const pidusage = require("pidusage");
|
||||||
const setBlocking = require("set-blocking");
|
const setBlocking = require("set-blocking");
|
||||||
|
const phin = require("phin");
|
||||||
const version = require("./package").version;
|
const version = require("./package").version;
|
||||||
|
|
||||||
// internal libraries
|
// internal libraries
|
||||||
@@ -58,9 +59,10 @@ const instanceFactorioAutosaveSize = new prometheus.Gauge(
|
|||||||
* Keeps track of the runtime parameters of an instance
|
* Keeps track of the runtime parameters of an instance
|
||||||
*/
|
*/
|
||||||
class Instance extends link.Link{
|
class Instance extends link.Link{
|
||||||
constructor(connector, dir, factorioDir, instanceConfig) {
|
constructor(slave, connector, dir, factorioDir, instanceConfig) {
|
||||||
super('instance', 'slave', connector);
|
super('instance', 'slave', connector);
|
||||||
link.attachAllMessages(this);
|
link.attachAllMessages(this);
|
||||||
|
this._slave = slave;
|
||||||
this._dir = dir;
|
this._dir = dir;
|
||||||
|
|
||||||
this.plugins = new Map();
|
this.plugins = new Map();
|
||||||
@@ -141,21 +143,21 @@ class Instance extends link.Link{
|
|||||||
console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`);
|
console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`);
|
||||||
}
|
}
|
||||||
|
|
||||||
async init(pluginInfos, slave) {
|
async init(pluginInfos) {
|
||||||
await this.server.init();
|
await this.server.init();
|
||||||
|
|
||||||
// load plugins
|
// load plugins
|
||||||
for (let pluginInfo of pluginInfos) {
|
for (let pluginInfo of pluginInfos) {
|
||||||
if (
|
if (
|
||||||
!pluginInfo.instanceEntrypoint
|
!pluginInfo.instanceEntrypoint
|
||||||
|| !slave.serverPlugins.has(pluginInfo.name)
|
|| !this._slave.serverPlugins.has(pluginInfo.name)
|
||||||
|| !this.config.group(pluginInfo.name).get("enabled")
|
|| !this.config.group(pluginInfo.name).get("enabled")
|
||||||
) {
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this._loadPlugin(pluginInfo, slave);
|
await this._loadPlugin(pluginInfo, this._slave);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.notifyExit();
|
this.notifyExit();
|
||||||
throw err;
|
throw err;
|
||||||
@@ -445,7 +447,23 @@ class Instance extends link.Link{
|
|||||||
|
|
||||||
console.log("Exporting data .....");
|
console.log("Exporting data .....");
|
||||||
await symlinkMods(this, "sharedMods", console);
|
await symlinkMods(this, "sharedMods", console);
|
||||||
await factorio.exportData(this.server);
|
let zip = await factorio.exportData(this.server);
|
||||||
|
|
||||||
|
let content = await zip.generateAsync({ type: "nodebuffer" });
|
||||||
|
let url = new URL(this._slave.config.get("slave.master_url"));
|
||||||
|
url.pathname += "api/upload-export";
|
||||||
|
let response = await phin({
|
||||||
|
url, method: "PUT",
|
||||||
|
data: content,
|
||||||
|
core: { rejectUnauthorized: false },
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/zip",
|
||||||
|
"x-access-token": this._slave.config.get("slave.master_token"),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (response.statusCode !== 200) {
|
||||||
|
throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`)
|
||||||
|
}
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
this.notifyExit();
|
this.notifyExit();
|
||||||
@@ -801,9 +819,9 @@ class Slave extends link.Link {
|
|||||||
let [connectionClient, connectionServer] = link.VirtualConnector.makePair();
|
let [connectionClient, connectionServer] = link.VirtualConnector.makePair();
|
||||||
let instanceConnection = new InstanceConnection(connectionServer, this, instanceId);
|
let instanceConnection = new InstanceConnection(connectionServer, this, instanceId);
|
||||||
let instance = new Instance(
|
let instance = new Instance(
|
||||||
connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config
|
this, connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config
|
||||||
);
|
);
|
||||||
await instance.init(this.pluginInfos, this);
|
await instance.init(this.pluginInfos);
|
||||||
|
|
||||||
// XXX: race condition on multiple simultanious calls
|
// XXX: race condition on multiple simultanious calls
|
||||||
this.instanceConnections.set(instanceId, instanceConnection);
|
this.instanceConnections.set(instanceId, instanceConnection);
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ describe("Integration of Clusterio", function() {
|
|||||||
it("exports the data", async function() {
|
it("exports the data", async function() {
|
||||||
slowTest(this);
|
slowTest(this);
|
||||||
await exec(`node clusterctl --config ${controlConfigPath} export-data --instance test`);
|
await exec(`node clusterctl --config ${controlConfigPath} export-data --instance test`);
|
||||||
assert(await fs.exists(path.join("temp", "export.zip")), "Export was not created");
|
assert(await fs.exists(path.join("static", "export", "locale.json")), "Export was not created");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -17,7 +17,7 @@ describe("Slave testing", function() {
|
|||||||
let instanceConfig = new config.InstanceConfig();
|
let instanceConfig = new config.InstanceConfig();
|
||||||
await instanceConfig.init();
|
await instanceConfig.init();
|
||||||
instanceConfig.set("instance.name", "foo");
|
instanceConfig.set("instance.name", "foo");
|
||||||
instance = new slave._Instance(new link.VirtualConnector(), "dir", "factorioDir", instanceConfig);
|
instance = new slave._Instance({}, new link.VirtualConnector(), "dir", "factorioDir", instanceConfig);
|
||||||
});
|
});
|
||||||
|
|
||||||
describe(".name", function() {
|
describe(".name", function() {
|
||||||
@@ -109,7 +109,7 @@ describe("Slave testing", function() {
|
|||||||
await instanceConfig.init();
|
await instanceConfig.init();
|
||||||
instanceConfig.set("instance.name", "test");
|
instanceConfig.set("instance.name", "test");
|
||||||
instance = new slave._Instance(
|
instance = new slave._Instance(
|
||||||
new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig
|
{}, new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig
|
||||||
);
|
);
|
||||||
await fs.outputFile(instance.path("mods", "mod_i.zip"), "i");
|
await fs.outputFile(instance.path("mods", "mod_i.zip"), "i");
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in new issue
Block a user