From 0cfa048d42842510eb67b51179bdf99d690a9def Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:55:18 +0000 Subject: [PATCH 1/2] Add NO_EXTERNAL_TEST flag to skip tests using external APIs Adds an externalTest(this) helper alongside slowTest that skips the test when NO_EXTERNAL_TEST is set and raises the timeout to 60s, plus a no-external-test script. Marks the tests hitting factorio.com, the mod portal and GitHub with it. Co-Authored-By: Claude Fable 5 --- docs/contributing.md | 5 +++++ package.json | 1 + test/host/server.js | 6 +++++- test/integration/index.js | 14 ++++++++++++++ test/lib/ModStore.test.js | 4 +++- test/lib/external/FactorioVersions.test.js | 3 ++- test/lib/external/LatestReleases.test.js | 3 ++- test/messages/mod.test.js | 4 ++-- 8 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index a35de4fc..1ae0d1d2 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -125,6 +125,11 @@ Please do add tests for any code you add and run both: To check that your changes pass the integration tests as well as the ESLint rules set up for the project. +The full test suite can take a while to run. +Use `pnpm run fast-test` to skip tests marked as slow, or `pnpm run no-external-test` to skip tests that depend on external APIs such as the Factorio mod portal or GitHub. +These correspond to setting the `FAST_TEST` and `NO_EXTERNAL_TEST` environment variables and can be combined. +When writing tests that take a lot of time call `slowTest(this)` at the start of the test, and for tests that call out to external APIs call `externalTest(this)`, both are exported from `test/integration/index.js`. + ### Submitting a Pull Request diff --git a/package.json b/package.json index 74368d2f..dc551a8c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"", "silent-test": "SILENT_TEST=y pnpm test", "fast-test": "FAST_TEST=y pnpm test", + "no-external-test": "NO_EXTERNAL_TEST=y pnpm test", "cover": "nyc pnpm test", "fast-cover": "FAST_TEST=y nyc pnpm test", "ci-cover": "nyc --reporter=lcovonly pnpm run-script test", diff --git a/test/host/server.js b/test/host/server.js index adbec5e1..f5858c85 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -8,7 +8,7 @@ const hostServer = require("@clusterio/host/dist/node/src/server"); const lib = require("@clusterio/lib"); const { wait } = lib; const { testLines } = require("../lib/factorio/lines"); -const { slowTest } = require("../integration"); +const { slowTest, externalTest } = require("../integration"); describe("host/server", function() { @@ -137,6 +137,7 @@ describe("host/server", function() { it("works", async function() { slowTest(this); + externalTest(this); const url = "https://github.com/clusterio/clusterio/archive/refs/tags/v2.0.0-alpha.22.zip"; const downloads = path.join("temp", "test", "downloads"); await fs.rm(downloads, { force: true, recursive: true, maxRetries: 10 }); @@ -160,6 +161,8 @@ describe("host/server", function() { }); it("works", async function() { + slowTest(this); + externalTest(this); const url = "https://github.com/clusterio/clusterio/archive/refs/tags/v2.0.0-alpha.22.tar.gz"; const downloads = path.join("temp", "test", "downloads"); await fs.rm(downloads, { force: true, recursive: true, maxRetries: 10 }); @@ -479,6 +482,7 @@ describe("host/server", function() { } it("should download a version correctly (live api)", async function() { slowTest(this); + externalTest(this); if (_platform !== "linux") { this.skip(); } diff --git a/test/integration/index.js b/test/integration/index.js index 1dbf56c3..2b81631a 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -128,6 +128,15 @@ function slowTest(test) { test.timeout(30000); } +// Mark that this test depends on an external API and may be slow or flaky. +function externalTest(test) { + if (process.env.NO_EXTERNAL_TEST) { + test.skip(); + } + + test.timeout(60000); +} + // Mark that this test or suite of tests requires a factorio install to run. function requiresFactorio(testOrSuite) { if (testOrSuite.skip) { @@ -320,6 +329,10 @@ before(async function() { console.log("FAST_TEST is present in env, slow tests will be skipped."); } + if (process.env.NO_EXTERNAL_TEST) { + console.log("NO_EXTERNAL_TEST is present in env, tests depending on external APIs will be skipped."); + } + await fs.rm(instancesDir, { force: true, recursive: true, maxRetries: 10 }); await fs.rm(modsDir, { force: true, recursive: true, maxRetries: 10 }); await fs.rm(databaseDir, { force: true, recursive: true, maxRetries: 10 }); @@ -433,6 +446,7 @@ module.exports = { execController, execCtlProcess, slowTest, + externalTest, requiresFactorio, get, exec, diff --git a/test/lib/ModStore.test.js b/test/lib/ModStore.test.js index 8f2037a3..b329d185 100644 --- a/test/lib/ModStore.test.js +++ b/test/lib/ModStore.test.js @@ -9,6 +9,7 @@ const JSZip = require("jszip"); // Added for creating mock zips const nativeFetch = global.fetch; const { ModStore, ModInfo, ModVersionEquality } = require("@clusterio/lib"); // Adjust path based on compiled output +const { externalTest } = require("../integration"); const MODS_DIR = path.join("temp", "test", "mod_store", "mods"); const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json"); @@ -582,7 +583,7 @@ describe("lib/ModStore", function () { // It requires network access and may take a while to run. // It might also break if the API changes or is unavailable. it("should fetch all mods from the LIVE portal API and check structure/count", async function () { - this.timeout(60000); // Increase timeout to 60 seconds for live API call + externalTest(this); // Ensure we are using the true native fetch we captured earlier global.fetch = nativeFetch; assert.strictEqual( @@ -700,6 +701,7 @@ describe("lib/ModStore", function () { await assert.rejects(ModStore.fetchModReleases("my-mod")); }); it("should fetch from the correct api url (live)", async function() { + externalTest(this); global.fetch = nativeFetch; const result = await ModStore.fetchModReleases("clusterio_lib"); assert(result.releases && result.releases.length > 1, "Failed to fetch releases"); diff --git a/test/lib/external/FactorioVersions.test.js b/test/lib/external/FactorioVersions.test.js index dee4b41c..b92d03a7 100644 --- a/test/lib/external/FactorioVersions.test.js +++ b/test/lib/external/FactorioVersions.test.js @@ -2,7 +2,7 @@ const assert = require("assert").strict; const { readFile } = require("node:fs/promises"); const { fetchFactorioVersions } = require("@clusterio/lib"); -const { slowTest } = require("../../integration"); +const { slowTest, externalTest } = require("../../integration"); describe("FactorioVersions", function() { describe("fetchFactorioVersions", function() { @@ -129,6 +129,7 @@ describe("FactorioVersions", function() { }); it("parses versions from live api", async function() { slowTest(this); + externalTest(this); global.fetch = originalFetch; const versions = await fetchFactorioVersions(); diff --git a/test/lib/external/LatestReleases.test.js b/test/lib/external/LatestReleases.test.js index 01d1d0df..4fb79eff 100644 --- a/test/lib/external/LatestReleases.test.js +++ b/test/lib/external/LatestReleases.test.js @@ -1,7 +1,7 @@ const assert = require("assert").strict; const { fetchLatestReleases, resolveReleaseChannel } = require("@clusterio/lib"); -const { slowTest } = require("../../integration"); +const { slowTest, externalTest } = require("../../integration"); describe("LatestReleases", function() { describe("fetchLatestReleases", function() { @@ -54,6 +54,7 @@ describe("LatestReleases", function() { }); it("fetches from the live api", async function() { slowTest(this); + externalTest(this); global.fetch = originalFetch; const releases = await fetchLatestReleases(); diff --git a/test/messages/mod.test.js b/test/messages/mod.test.js index f782d5fc..200b7417 100644 --- a/test/messages/mod.test.js +++ b/test/messages/mod.test.js @@ -7,7 +7,7 @@ const { ModDependencyResolveRequest, ModDependency, ModInfo } = lib; const { testMatrix, testRoundTripJsonSerialisable } = require("../common"); const { Controller, ControlConnection } = require("@clusterio/controller"); -const { slowTest } = require("../integration"); +const { externalTest } = require("../integration"); describe("messages/mod", function() { /** @type {Controller} */ @@ -467,7 +467,7 @@ describe("messages/mod", function() { } }); it("resolves dependencies (live)", async function() { - this.timeout(60000); // Increase timeout to 60 seconds for live API call + externalTest(this); lib.ModStore.fetchModReleases = _ModStore_fetchModReleases; const result = await controlConnection.handleModDependencyResolveRequest( From ffc79ea0e79648f4a6d71221833ffc379bfdeae1 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:26:30 +0000 Subject: [PATCH 2/2] Disable user verification in start-all and auto_start tests Factorio contacts auth.factorio.com on startup when require_user_verification is enabled, which made these tests fail offline. Match the other integration tests by turning it off. Co-Authored-By: Claude Fable 5 --- test/integration/clusterio.js | 3 +++ test/integration/instance.js | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 780a4dcc..b44a5e18 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -301,6 +301,9 @@ describe("Integration of Clusterio", function() { await execCtl("instance create alt-start --id 97"); await execCtl("instance assign alt-start 5"); await execCtl("instance config set alt-start instance.auto_start true"); + const visibility = jsonArg({ lan: false, public: false }); + await execCtlProcess(`instance config set-prop alt-start factorio.settings visibility ${visibility}`); + await execCtl("instance config set-prop alt-start factorio.settings require_user_verification false"); await stopAltHost(hostProcess); hostProcess = await spawnAltHost(); // Stop the host immediatly to test the handling of stopping diff --git a/test/integration/instance.js b/test/integration/instance.js index ccd94d30..56f150b7 100644 --- a/test/integration/instance.js +++ b/test/integration/instance.js @@ -241,6 +241,13 @@ describe("start-all and stop-all commands", function() { await execCtl(`instance assign ${testInstName1} 4`); await execCtl(`instance assign ${testInstName2} 4`); await execCtl(`instance assign ${testInstName3} 4`); + const visibility = JSON.stringify({ lan: false, public: false }).replace( + /"/g, process.platform === "win32" ? '""' : '\\"' + ); + for (const name of [testInstName1, testInstName2, testInstName3]) { + await execCtlProcess(`instance config set-prop ${name} factorio.settings visibility "${visibility}"`); + await execCtl(`instance config set-prop ${name} factorio.settings require_user_verification false`); + } // Exclude testInstName3 from start-all by default await execCtl(`instance config set ${testInstName3} instance.exclude_from_start_all true`);