From 838a43428b86a1c22ec9ff8abdcfd033c8d78af7 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:13:10 +0000 Subject: [PATCH 1/2] Only print test output for failing tests A spec reporter subclass buffers console output, including winston's ConsoleTransport and the integration child process lines, and prints it above the failure line when a test or hook fails. VERBOSE_TEST streams everything as before, and CI sets it on debug re-runs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/node.js.yml | 6 ++++++ docs/contributing.md | 2 ++ package.json | 2 +- test/reporter.cjs | 38 +++++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 test/reporter.cjs diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index cb033ec7..4fb432e2 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -67,9 +67,15 @@ jobs: run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/${{ matrix.factorio-version }}/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz - name: Run tests if: ${{ !matrix.coverage }} + env: + # Re-run with debug logging to see output from passing tests + VERBOSE_TEST: ${{ runner.debug }} run: pnpm test --grep "^(Integration of Clusterio (clusterioctl|plugin system)|Clusterio Instance) " ${{ matrix.tests == 'other' && '--invert' || '' }} - name: Run coverage if: ${{ matrix.coverage }} + env: + # Re-run with debug logging to see output from passing tests + VERBOSE_TEST: ${{ runner.debug }} run: pnpm run ci-cover --grep "^(Integration of Clusterio (clusterioctl|plugin system)|Clusterio Instance) " ${{ matrix.tests == 'other' && '--invert' || '' }} - name: Upload coverage to Codecov if: ${{ matrix.coverage }} diff --git a/docs/contributing.md b/docs/contributing.md index 0692fd8b..6e5b4925 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -197,6 +197,8 @@ Use `pnpm run fast-test` to skip tests marked as slow, or `pnpm run no-external- These correspond to setting the `FAST_TEST` and `NO_EXTERNAL_TEST` environment variables and can be combined. For example, `FAST_TEST=y NO_EXTERNAL_TEST=y pnpm test` combines both exclusions. These flags still run the remaining integration setup and tests; neither produces an isolated unit-test run. +Console and log output is only printed for tests and hooks that fail, just above the failure. +Set `VERBOSE_TEST=y` to print all of it as it happens, or re-run a CI job with debug logging enabled. When writing slow tests call `slowTest(this)`, and for external API tests call `externalTest(this)`; both are exported from `test/common.js`. For tests that need a Factorio installation, call `requiresFactorio(this)` in the test or pass the suite to `requiresFactorio`, imported from `test/integration/index.js`. `test/common.js` also provides JSON round-trip/schema helpers, and `test/mock.js` provides shared mocks. diff --git a/package.json b/package.json index 7c8c4b7d..66909833 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "build": "tsc --build", "watch": "tsc --build --watch", "build-mod": "pnpm run -r build-mod --output-dir ../../dist", - "test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"", + "test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R ./test/reporter.cjs --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"", "silent-test": "SILENT_TEST=y pnpm test", "fast-test": "FAST_TEST=y pnpm test", "no-external-test": "NO_EXTERNAL_TEST=y pnpm test", diff --git a/test/reporter.cjs b/test/reporter.cjs new file mode 100644 index 00000000..561cc823 --- /dev/null +++ b/test/reporter.cjs @@ -0,0 +1,38 @@ +// Spec reporter that holds back console output and prints it only for +// failing tests and hooks. Set VERBOSE_TEST to print everything as it happens. +"use strict"; +const util = require("node:util"); +const Mocha = require("mocha"); + +const { Spec } = Mocha.reporters; +const { EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants; +const methods = ["log", "info", "debug", "warn", "error"]; + +class CaptureReporter extends Spec { + constructor(runner, options) { + super(runner, options); + if (process.env.VERBOSE_TEST) { + return; + } + + const original = {}; + const captured = []; + for (const method of methods) { + original[method] = console[method]; + console[method] = (...args) => { captured.push([method, util.format(...args)]); }; + } + + const discard = () => { captured.length = 0; }; + runner.on(EVENT_TEST_PASS, discard); + runner.on(EVENT_TEST_PENDING, discard); + // Prepend so the output lands above the failure line. + runner.prependListener(EVENT_TEST_FAIL, () => { + for (const [method, text] of captured) { + original[method](text); + } + discard(); + }); + } +} + +module.exports = CaptureReporter; From 6363f15cb963c27614bbca567a1ea031fe246832 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:31:45 +0000 Subject: [PATCH 2/2] Replace SILENT_TEST and VERBOSE_TEST with TEST_OUTPUT The reporter now handles all three modes: verbose prints everything, normal prints output from failing tests and hooks, silent prints only the results. The integration spawn helper and setup_logging no longer filter output themselves. npm install output in plugin install now goes through the logger instead of inherited stdio, so it follows the log level and is captured in tests like the rest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/node.js.yml | 4 ++-- docs/contributing.md | 3 ++- package.json | 2 +- packages/lib/src/shared_commands.ts | 15 +++++++++++++-- test/integration/index.js | 22 ++-------------------- test/reporter.cjs | 16 ++++++++++++---- test/setup_logging.js | 3 --- 7 files changed, 32 insertions(+), 33 deletions(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 4fb432e2..7389d896 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -69,13 +69,13 @@ jobs: if: ${{ !matrix.coverage }} env: # Re-run with debug logging to see output from passing tests - VERBOSE_TEST: ${{ runner.debug }} + TEST_OUTPUT: ${{ runner.debug == '1' && 'verbose' || 'normal' }} run: pnpm test --grep "^(Integration of Clusterio (clusterioctl|plugin system)|Clusterio Instance) " ${{ matrix.tests == 'other' && '--invert' || '' }} - name: Run coverage if: ${{ matrix.coverage }} env: # Re-run with debug logging to see output from passing tests - VERBOSE_TEST: ${{ runner.debug }} + TEST_OUTPUT: ${{ runner.debug == '1' && 'verbose' || 'normal' }} run: pnpm run ci-cover --grep "^(Integration of Clusterio (clusterioctl|plugin system)|Clusterio Instance) " ${{ matrix.tests == 'other' && '--invert' || '' }} - name: Upload coverage to Codecov if: ${{ matrix.coverage }} diff --git a/docs/contributing.md b/docs/contributing.md index 6e5b4925..684b2e57 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -198,7 +198,8 @@ These correspond to setting the `FAST_TEST` and `NO_EXTERNAL_TEST` environment v For example, `FAST_TEST=y NO_EXTERNAL_TEST=y pnpm test` combines both exclusions. These flags still run the remaining integration setup and tests; neither produces an isolated unit-test run. Console and log output is only printed for tests and hooks that fail, just above the failure. -Set `VERBOSE_TEST=y` to print all of it as it happens, or re-run a CI job with debug logging enabled. +Set `TEST_OUTPUT=verbose` to print all of it as it happens, or `TEST_OUTPUT=silent` (`pnpm run silent-test`) to print only the results. +In CI, re-running a job with debug logging enabled gives verbose output. When writing slow tests call `slowTest(this)`, and for external API tests call `externalTest(this)`; both are exported from `test/common.js`. For tests that need a Factorio installation, call `requiresFactorio(this)` in the test or pass the suite to `requiresFactorio`, imported from `test/integration/index.js`. `test/common.js` also provides JSON round-trip/schema helpers, and `test/mock.js` provides shared mocks. diff --git a/package.json b/package.json index 66909833..b8fdd877 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "watch": "tsc --build --watch", "build-mod": "pnpm run -r build-mod --output-dir ../../dist", "test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R ./test/reporter.cjs --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"", - "silent-test": "SILENT_TEST=y pnpm test", + "silent-test": "TEST_OUTPUT=silent pnpm test", "fast-test": "FAST_TEST=y pnpm test", "no-external-test": "NO_EXTERNAL_TEST=y pnpm test", "cover": "c8 pnpm test", diff --git a/packages/lib/src/shared_commands.ts b/packages/lib/src/shared_commands.ts index 155be8f3..cf0db1d6 100644 --- a/packages/lib/src/shared_commands.ts +++ b/packages/lib/src/shared_commands.ts @@ -13,6 +13,7 @@ import { InvalidFieldError, InvalidValueError } from "./errors.js"; import { logger } from "./logging.js"; import * as libHelpers from "./helpers.js"; import { LockFile } from "./LockFile.js"; +import { LineSplitter } from "./stream.js"; import { pathToFileURL } from "url"; @@ -88,11 +89,21 @@ function npmInstall(packageSpec: string) { const npm = process.platform === "win32" ? "npm.cmd" : "npm"; return new Promise((resolve, reject) => { const child = spawn(npm, ["install", "--save", "--no-audit", "--no-fund", packageSpec], { - stdio: "inherit", + stdio: ["ignore", "pipe", "pipe"], shell: process.platform === "win32", }); + for (const [output, level] of [[child.stdout, "info"], [child.stderr, "warn"]] as const) { + const lines = new LineSplitter({ readableObjectMode: true }); + lines.on("data", (line: Buffer) => { + const text = line.toString("utf8").trimEnd(); + if (text) { + logger.log(level, text); + } + }); + output.pipe(lines); + } child.on("error", reject); - child.on("exit", code => { + child.on("close", code => { if (code === 0) { resolve(); } else { diff --git a/test/integration/index.js b/test/integration/index.js index 781f5cf1..09407e23 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -297,16 +297,8 @@ export async function writeHostConfig(configPath, pairing, fields = {}) { /** @returns {Promise} */ export function spawn(name, cmd, waitFor, options = {}) { - const silent = process.env.SILENT_TEST; - const bootstrap = !controllerProcess || !hostProcess; - function log(...args) { - if (!silent || bootstrap) { - console.log(...args); - } - } - return new Promise((resolve, reject) => { - log(cmd); + console.log(cmd); let parts = cmd.split(" "); let process = child_process.spawn(parts[0], parts.slice(1), childOptions(options)); let stdout = new LineSplitter({ readableObjectMode: true }); @@ -314,13 +306,9 @@ export function spawn(name, cmd, waitFor, options = {}) { let onData = line => { line = line.toString("utf8"); if (waitFor.test(line)) { - if (silent) { - stdout.off("data", onData); - stderr.off("data", onData); - } resolve(process); } - log(name, line); + console.log(name, line); }; stdout.on("data", onData); stderr.on("data", onData); @@ -336,12 +324,6 @@ export function spawnNode(name, cmd, waitFor, options) { before(async function() { this.timeout(40000); - - const silent = process.env.SILENT_TEST; - if (silent) { - console.log("SILENT_TEST is present in env, loggers after bootstrap will be muted."); - } - // If fast test is enabled then output that it is if (process.env.FAST_TEST) { diff --git a/test/reporter.cjs b/test/reporter.cjs index 561cc823..1f558bda 100644 --- a/test/reporter.cjs +++ b/test/reporter.cjs @@ -1,5 +1,6 @@ -// Spec reporter that holds back console output and prints it only for -// failing tests and hooks. Set VERBOSE_TEST to print everything as it happens. +// Spec reporter controlling console output with TEST_OUTPUT: +// verbose prints everything, normal (default) prints only output from failing +// tests and hooks, silent prints only the results. "use strict"; const util = require("node:util"); const Mocha = require("mocha"); @@ -7,11 +8,16 @@ const Mocha = require("mocha"); const { Spec } = Mocha.reporters; const { EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants; const methods = ["log", "info", "debug", "warn", "error"]; +const modes = ["verbose", "normal", "silent"]; class CaptureReporter extends Spec { constructor(runner, options) { super(runner, options); - if (process.env.VERBOSE_TEST) { + const mode = process.env.TEST_OUTPUT || "normal"; + if (!modes.includes(mode)) { + throw new Error(`Invalid TEST_OUTPUT "${mode}", expected one of ${modes.join(", ")}`); + } + if (mode === "verbose") { return; } @@ -19,7 +25,9 @@ class CaptureReporter extends Spec { const captured = []; for (const method of methods) { original[method] = console[method]; - console[method] = (...args) => { captured.push([method, util.format(...args)]); }; + console[method] = mode === "silent" + ? () => {} + : (...args) => { captured.push([method, util.format(...args)]); }; } const discard = () => { captured.length = 0; }; diff --git a/test/setup_logging.js b/test/setup_logging.js index f5d8cec4..5cd4fa04 100644 --- a/test/setup_logging.js +++ b/test/setup_logging.js @@ -5,11 +5,8 @@ import { ConsoleTransport, TerminalFormat, logger } from "@clusterio/lib"; // running tests individually. before(function() { - const silent = process.env.SILENT_TEST; - logger.add(new ConsoleTransport({ level: "info", format: new TerminalFormat(), - filter: () => !silent, })); });