Add NO_EXTERNAL_TEST flag to skip tests using external APIs (#976)

* Add NO_EXTERNAL_TEST flag to skip tests using external APIs

Adds an externalTest(this) helper alongside slowTest that skips the test
when NO_EXTERNAL_TEST is set and raises the timeout to 60s, plus a
no-external-test script. Marks the tests hitting factorio.com, the mod
portal and GitHub with it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Disable user verification in start-all and auto_start tests

Factorio contacts auth.factorio.com on startup when
require_user_verification is enabled, which made these tests fail
offline. Match the other integration tests by turning it off.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

docs/contributing.md

@@ -125,6 +125,11 @@ Please do add tests for any code you add and run both:
 
 To check that your changes pass the integration tests as well as the ESLint rules set up for the project.
 
+The full test suite can take a while to run.
+Use `pnpm run fast-test` to skip tests marked as slow, or `pnpm run no-external-test` to skip tests that depend on external APIs such as the Factorio mod portal or GitHub.
+These correspond to setting the `FAST_TEST` and `NO_EXTERNAL_TEST` environment variables and can be combined.
+When writing tests that take a lot of time call `slowTest(this)` at the start of the test, and for tests that call out to external APIs call `externalTest(this)`, both are exported from `test/integration/index.js`.
+
 
 ### Submitting a Pull Request
 

package.json

@@ -10,6 +10,7 @@
 		"test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"",
 		"silent-test": "SILENT_TEST=y pnpm test",
 		"fast-test": "FAST_TEST=y pnpm test",
+		"no-external-test": "NO_EXTERNAL_TEST=y pnpm test",
 		"cover": "nyc pnpm test",
 		"fast-cover": "FAST_TEST=y nyc pnpm test",
 		"ci-cover": "nyc --reporter=lcovonly pnpm run-script test",

test/host/server.js

@@ -8,7 +8,7 @@ const hostServer = require("@clusterio/host/dist/node/src/server");
 const lib = require("@clusterio/lib");
 const { wait } = lib;
 const { testLines } = require("../lib/factorio/lines");
-const { slowTest } = require("../integration");
+const { slowTest, externalTest } = require("../integration");
 
 
 describe("host/server", function() {
@@ -137,6 +137,7 @@ describe("host/server", function() {
 
 		it("works", async function() {
 			slowTest(this);
+			externalTest(this);
 			const url = "https://github.com/clusterio/clusterio/archive/refs/tags/v2.0.0-alpha.22.zip";
 			const downloads = path.join("temp", "test", "downloads");
 			await fs.rm(downloads, { force: true, recursive: true, maxRetries: 10 });
@@ -160,6 +161,8 @@ describe("host/server", function() {
 		});
 
 		it("works", async function() {
+			slowTest(this);
+			externalTest(this);
 			const url = "https://github.com/clusterio/clusterio/archive/refs/tags/v2.0.0-alpha.22.tar.gz";
 			const downloads = path.join("temp", "test", "downloads");
 			await fs.rm(downloads, { force: true, recursive: true, maxRetries: 10 });
@@ -479,6 +482,7 @@ describe("host/server", function() {
 			}
 			it("should download a version correctly (live api)", async function() {
 				slowTest(this);
+				externalTest(this);
 				if (_platform !== "linux") {
 					this.skip();
 				}

test/integration/clusterio.js

@@ -301,6 +301,9 @@ describe("Integration of Clusterio", function() {
 				await execCtl("instance create alt-start --id 97");
 				await execCtl("instance assign alt-start 5");
 				await execCtl("instance config set alt-start instance.auto_start true");
+				const visibility = jsonArg({ lan: false, public: false });
+				await execCtlProcess(`instance config set-prop alt-start factorio.settings visibility ${visibility}`);
+				await execCtl("instance config set-prop alt-start factorio.settings require_user_verification false");
 				await stopAltHost(hostProcess);
 				hostProcess = await spawnAltHost();
 				// Stop the host immediatly to test the handling of stopping

test/integration/index.js

@@ -128,6 +128,15 @@ function slowTest(test) {
 	test.timeout(30000);
 }
 
+// Mark that this test depends on an external API and may be slow or flaky.
+function externalTest(test) {
+	if (process.env.NO_EXTERNAL_TEST) {
+		test.skip();
+	}
+
+	test.timeout(60000);
+}
+
 // Mark that this test or suite of tests requires a factorio install to run.
 function requiresFactorio(testOrSuite) {
 	if (testOrSuite.skip) {
@@ -320,6 +329,10 @@ before(async function() {
 		console.log("FAST_TEST is present in env, slow tests will be skipped.");
 	}
 
+	if (process.env.NO_EXTERNAL_TEST) {
+		console.log("NO_EXTERNAL_TEST is present in env, tests depending on external APIs will be skipped.");
+	}
+
 	await fs.rm(instancesDir, { force: true, recursive: true, maxRetries: 10 });
 	await fs.rm(modsDir, { force: true, recursive: true, maxRetries: 10 });
 	await fs.rm(databaseDir, { force: true, recursive: true, maxRetries: 10 });
@@ -433,6 +446,7 @@ module.exports = {
 	execController,
 	execCtlProcess,
 	slowTest,
+	externalTest,
 	requiresFactorio,
 	get,
 	exec,

test/integration/instance.js

@@ -241,6 +241,13 @@ describe("start-all and stop-all commands", function() {
 		await execCtl(`instance assign ${testInstName1} 4`);
 		await execCtl(`instance assign ${testInstName2} 4`);
 		await execCtl(`instance assign ${testInstName3} 4`);
+		const visibility = JSON.stringify({ lan: false, public: false }).replace(
+			/"/g, process.platform === "win32" ? '""' : '\\"'
+		);
+		for (const name of [testInstName1, testInstName2, testInstName3]) {
+			await execCtlProcess(`instance config set-prop ${name} factorio.settings visibility "${visibility}"`);
+			await execCtl(`instance config set-prop ${name} factorio.settings require_user_verification false`);
+		}
 
 		// Exclude testInstName3 from start-all by default
 		await execCtl(`instance config set ${testInstName3} instance.exclude_from_start_all true`);

test/lib/ModStore.test.js

@@ -9,6 +9,7 @@ const JSZip = require("jszip"); // Added for creating mock zips
 const nativeFetch = global.fetch;
 
 const { ModStore, ModInfo, ModVersionEquality } = require("@clusterio/lib"); // Adjust path based on compiled output
+const { externalTest } = require("../integration");
 
 const MODS_DIR = path.join("temp", "test", "mod_store", "mods");
 const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json");
@@ -582,7 +583,7 @@ describe("lib/ModStore", function () {
 		// It requires network access and may take a while to run.
 		// It might also break if the API changes or is unavailable.
 		it("should fetch all mods from the LIVE portal API and check structure/count", async function () {
-			this.timeout(60000); // Increase timeout to 60 seconds for live API call
+			externalTest(this);
 			// Ensure we are using the true native fetch we captured earlier
 			global.fetch = nativeFetch;
 			assert.strictEqual(
@@ -700,6 +701,7 @@ describe("lib/ModStore", function () {
 			await assert.rejects(ModStore.fetchModReleases("my-mod"));
 		});
 		it("should fetch from the correct api url (live)", async function() {
+			externalTest(this);
 			global.fetch = nativeFetch;
 			const result = await ModStore.fetchModReleases("clusterio_lib");
 			assert(result.releases && result.releases.length > 1, "Failed to fetch releases");

test/lib/external/FactorioVersions.test.js

@@ -2,7 +2,7 @@ const assert = require("assert").strict;
 const { readFile } = require("node:fs/promises");
 
 const { fetchFactorioVersions } = require("@clusterio/lib");
-const { slowTest } = require("../../integration");
+const { slowTest, externalTest } = require("../../integration");
 
 describe("FactorioVersions", function() {
 	describe("fetchFactorioVersions", function() {
@@ -129,6 +129,7 @@ describe("FactorioVersions", function() {
 		});
 		it("parses versions from live api", async function() {
 			slowTest(this);
+			externalTest(this);
 			global.fetch = originalFetch;
 
 			const versions = await fetchFactorioVersions();

test/lib/external/LatestReleases.test.js

@@ -1,7 +1,7 @@
 const assert = require("assert").strict;
 
 const { fetchLatestReleases, resolveReleaseChannel } = require("@clusterio/lib");
-const { slowTest } = require("../../integration");
+const { slowTest, externalTest } = require("../../integration");
 
 describe("LatestReleases", function() {
 	describe("fetchLatestReleases", function() {
@@ -54,6 +54,7 @@ describe("LatestReleases", function() {
 		});
 		it("fetches from the live api", async function() {
 			slowTest(this);
+			externalTest(this);
 			global.fetch = originalFetch;
 
 			const releases = await fetchLatestReleases();

test/messages/mod.test.js

@@ -7,7 +7,7 @@ const { ModDependencyResolveRequest, ModDependency, ModInfo } = lib;
 const { testMatrix, testRoundTripJsonSerialisable } = require("../common");
 
 const { Controller, ControlConnection } = require("@clusterio/controller");
-const { slowTest } = require("../integration");
+const { externalTest } = require("../integration");
 
 describe("messages/mod", function() {
 	/** @type {Controller} */
@@ -467,7 +467,7 @@ describe("messages/mod", function() {
 				}
 			});
 			it("resolves dependencies (live)", async function() {
-				this.timeout(60000); // Increase timeout to 60 seconds for live API call
+				externalTest(this);
 				lib.ModStore.fetchModReleases = _ModStore_fetchModReleases;
 
 				const result = await controlConnection.handleModDependencyResolveRequest(