diff --git a/docs/contributing.md b/docs/contributing.md index a35de4fc..1ae0d1d2 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -125,6 +125,11 @@ Please do add tests for any code you add and run both: To check that your changes pass the integration tests as well as the ESLint rules set up for the project. +The full test suite can take a while to run. +Use `pnpm run fast-test` to skip tests marked as slow, or `pnpm run no-external-test` to skip tests that depend on external APIs such as the Factorio mod portal or GitHub. +These correspond to setting the `FAST_TEST` and `NO_EXTERNAL_TEST` environment variables and can be combined. +When writing tests that take a lot of time call `slowTest(this)` at the start of the test, and for tests that call out to external APIs call `externalTest(this)`, both are exported from `test/integration/index.js`. + ### Submitting a Pull Request diff --git a/package.json b/package.json index 74368d2f..dc551a8c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"", "silent-test": "SILENT_TEST=y pnpm test", "fast-test": "FAST_TEST=y pnpm test", + "no-external-test": "NO_EXTERNAL_TEST=y pnpm test", "cover": "nyc pnpm test", "fast-cover": "FAST_TEST=y nyc pnpm test", "ci-cover": "nyc --reporter=lcovonly pnpm run-script test", diff --git a/test/host/server.js b/test/host/server.js index adbec5e1..f5858c85 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -8,7 +8,7 @@ const hostServer = require("@clusterio/host/dist/node/src/server"); const lib = require("@clusterio/lib"); const { wait } = lib; const { testLines } = require("../lib/factorio/lines"); -const { slowTest } = require("../integration"); +const { slowTest, externalTest } = require("../integration"); describe("host/server", function() { @@ -137,6 +137,7 @@ describe("host/server", function() { it("works", async function() { slowTest(this); + externalTest(this); const url = "https://github.com/clusterio/clusterio/archive/refs/tags/v2.0.0-alpha.22.zip"; const downloads = path.join("temp", "test", "downloads"); await fs.rm(downloads, { force: true, recursive: true, maxRetries: 10 }); @@ -160,6 +161,8 @@ describe("host/server", function() { }); it("works", async function() { + slowTest(this); + externalTest(this); const url = "https://github.com/clusterio/clusterio/archive/refs/tags/v2.0.0-alpha.22.tar.gz"; const downloads = path.join("temp", "test", "downloads"); await fs.rm(downloads, { force: true, recursive: true, maxRetries: 10 }); @@ -479,6 +482,7 @@ describe("host/server", function() { } it("should download a version correctly (live api)", async function() { slowTest(this); + externalTest(this); if (_platform !== "linux") { this.skip(); } diff --git a/test/integration/index.js b/test/integration/index.js index 1dbf56c3..2b81631a 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -128,6 +128,15 @@ function slowTest(test) { test.timeout(30000); } +// Mark that this test depends on an external API and may be slow or flaky. +function externalTest(test) { + if (process.env.NO_EXTERNAL_TEST) { + test.skip(); + } + + test.timeout(60000); +} + // Mark that this test or suite of tests requires a factorio install to run. function requiresFactorio(testOrSuite) { if (testOrSuite.skip) { @@ -320,6 +329,10 @@ before(async function() { console.log("FAST_TEST is present in env, slow tests will be skipped."); } + if (process.env.NO_EXTERNAL_TEST) { + console.log("NO_EXTERNAL_TEST is present in env, tests depending on external APIs will be skipped."); + } + await fs.rm(instancesDir, { force: true, recursive: true, maxRetries: 10 }); await fs.rm(modsDir, { force: true, recursive: true, maxRetries: 10 }); await fs.rm(databaseDir, { force: true, recursive: true, maxRetries: 10 }); @@ -433,6 +446,7 @@ module.exports = { execController, execCtlProcess, slowTest, + externalTest, requiresFactorio, get, exec, diff --git a/test/lib/ModStore.test.js b/test/lib/ModStore.test.js index 8f2037a3..b329d185 100644 --- a/test/lib/ModStore.test.js +++ b/test/lib/ModStore.test.js @@ -9,6 +9,7 @@ const JSZip = require("jszip"); // Added for creating mock zips const nativeFetch = global.fetch; const { ModStore, ModInfo, ModVersionEquality } = require("@clusterio/lib"); // Adjust path based on compiled output +const { externalTest } = require("../integration"); const MODS_DIR = path.join("temp", "test", "mod_store", "mods"); const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json"); @@ -582,7 +583,7 @@ describe("lib/ModStore", function () { // It requires network access and may take a while to run. // It might also break if the API changes or is unavailable. it("should fetch all mods from the LIVE portal API and check structure/count", async function () { - this.timeout(60000); // Increase timeout to 60 seconds for live API call + externalTest(this); // Ensure we are using the true native fetch we captured earlier global.fetch = nativeFetch; assert.strictEqual( @@ -700,6 +701,7 @@ describe("lib/ModStore", function () { await assert.rejects(ModStore.fetchModReleases("my-mod")); }); it("should fetch from the correct api url (live)", async function() { + externalTest(this); global.fetch = nativeFetch; const result = await ModStore.fetchModReleases("clusterio_lib"); assert(result.releases && result.releases.length > 1, "Failed to fetch releases"); diff --git a/test/lib/external/FactorioVersions.test.js b/test/lib/external/FactorioVersions.test.js index dee4b41c..b92d03a7 100644 --- a/test/lib/external/FactorioVersions.test.js +++ b/test/lib/external/FactorioVersions.test.js @@ -2,7 +2,7 @@ const assert = require("assert").strict; const { readFile } = require("node:fs/promises"); const { fetchFactorioVersions } = require("@clusterio/lib"); -const { slowTest } = require("../../integration"); +const { slowTest, externalTest } = require("../../integration"); describe("FactorioVersions", function() { describe("fetchFactorioVersions", function() { @@ -129,6 +129,7 @@ describe("FactorioVersions", function() { }); it("parses versions from live api", async function() { slowTest(this); + externalTest(this); global.fetch = originalFetch; const versions = await fetchFactorioVersions(); diff --git a/test/lib/external/LatestReleases.test.js b/test/lib/external/LatestReleases.test.js index 01d1d0df..4fb79eff 100644 --- a/test/lib/external/LatestReleases.test.js +++ b/test/lib/external/LatestReleases.test.js @@ -1,7 +1,7 @@ const assert = require("assert").strict; const { fetchLatestReleases, resolveReleaseChannel } = require("@clusterio/lib"); -const { slowTest } = require("../../integration"); +const { slowTest, externalTest } = require("../../integration"); describe("LatestReleases", function() { describe("fetchLatestReleases", function() { @@ -54,6 +54,7 @@ describe("LatestReleases", function() { }); it("fetches from the live api", async function() { slowTest(this); + externalTest(this); global.fetch = originalFetch; const releases = await fetchLatestReleases(); diff --git a/test/messages/mod.test.js b/test/messages/mod.test.js index f782d5fc..200b7417 100644 --- a/test/messages/mod.test.js +++ b/test/messages/mod.test.js @@ -7,7 +7,7 @@ const { ModDependencyResolveRequest, ModDependency, ModInfo } = lib; const { testMatrix, testRoundTripJsonSerialisable } = require("../common"); const { Controller, ControlConnection } = require("@clusterio/controller"); -const { slowTest } = require("../integration"); +const { externalTest } = require("../integration"); describe("messages/mod", function() { /** @type {Controller} */ @@ -467,7 +467,7 @@ describe("messages/mod", function() { } }); it("resolves dependencies (live)", async function() { - this.timeout(60000); // Increase timeout to 60 seconds for live API call + externalTest(this); lib.ModStore.fetchModReleases = _ModStore_fetchModReleases; const result = await controlConnection.handleModDependencyResolveRequest(