Allow known breaks in the version compatibility test

The version-compat job fails every pull request once master breaks
compatibility with the latest release on purpose. List those breaks in
scripts/version-compat-breaks.json keyed by the published version they
break against, each with a pr number and reason. Listed messages warn
instead of failing, connect skips the cluster test, entries for other
versions are ignored, and a listed message that did not change fails.

Add the two src changes from #1129 against 2.0.3 so master passes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/contributing.md

@@ -18,6 +18,7 @@ Read our [past decisions](/docs/decisions.md) for compatibility policies and des
     - [Checking your code for errors](#checking-your-code-for-errors)
     - [Cleaning generated and runtime files](#cleaning-generated-and-runtime-files)
     - [Submitting a Pull Request](#submitting-a-pull-request)
+    - [Breaking Compatibility with the Last Release](#breaking-compatibility-with-the-last-release)
     - [Editing your Feature Branch](#editing-your-feature-branch)
     - [Resolving Merge Conflicts](#resolving-merge-conflicts)
     - [Updating Your GitHub Fork](#updating-your-github-fork)
@@ -250,6 +251,30 @@ If there are more changes you want to have merged or mistakes you need to fix yo
 Pull request descriptions should contain a `### Changelog` section with additions for the changelog contained inside a code block, see the [CHANGELOG.md](/CHANGELOG.md) file for the formatting of changelog entries and possible sections to use.
 If there are no changelog entries to add, remove the code block and start the changelog section with the text `None.`, followed by an optional reason for why it is not is needed.
 
+### Breaking Compatibility with the Last Release
+
+The version-compat job in CI runs the local controller with the latest published host, and the published controller with the local host.
+It fails if a link message changed its src, dst, data or response schema since that release.
+If the break is intended, add an entry for the published version to [scripts/version-compat-breaks.json](../scripts/version-compat-breaks.json) in the same pull request:
+
+```json
+{
+	"2.0.3": {
+		"messages": {
+			"request FactorioVersionsRequest": { "pr": 1129, "reason": "Why it breaks and what stops working." }
+		},
+		"connect": { "pr": 1234, "reason": "Why a host cannot connect at all." }
+	}
+}
+```
+
+Listed messages are reported as warnings with the reason instead of failing the job.
+Only add `connect` when the published and local versions cannot connect at all, it skips starting the cluster.
+Listing a message that did not change fails the job.
+Entries only apply while their version is the latest release, after the next release they are ignored and can be removed.
+Describe the break in the Breaking Changes section of the changelog.
+
+
 ### Editing your Feature Branch
 
 Until your feature branch is merged you can edit it as you see fit by using the rebase tool.

scripts/version-compat-breaks.js

@@ -0,0 +1,98 @@
+/**
+ * Known compatibility breaks for the version compatibility test
+ *
+ * scripts/version-compat-breaks.json lists the intended breaks against a
+ * published version, keyed by that version:
+ *
+ *   {
+ *     "2.0.3": {
+ *       "messages": {
+ *         "request FactorioVersionsRequest": { "pr": 1129, "reason": "..." }
+ *       },
+ *       "connect": { "pr": 1234, "reason": "..." }
+ *     }
+ *   }
+ *
+ * Listed messages are warned about instead of failing the test, and an entry
+ * with connect skips starting the cluster.  Entries for other versions are
+ * ignored.  See docs/contributing.md for when to add an entry.
+ */
+"use strict";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export const breaksFile = path.join(import.meta.dirname, "version-compat-breaks.json");
+
+function validateBreak(value, where) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) {
+		throw new Error(`${where} must be an object`);
+	}
+	if (!Number.isInteger(value.pr) || value.pr <= 0) {
+		throw new Error(`${where} must have a pr number`);
+	}
+	if (typeof value.reason !== "string" || !value.reason.trim()) {
+		throw new Error(`${where} must have a reason`);
+	}
+	const unknown = Object.keys(value).filter(key => !["pr", "reason"].includes(key));
+	if (unknown.length) {
+		throw new Error(`${where} has unknown properties ${unknown.join(", ")}`);
+	}
+}
+
+export function validateBreaks(breaks) {
+	if (!breaks || typeof breaks !== "object" || Array.isArray(breaks)) {
+		throw new Error("Known breaks must be an object keyed by version");
+	}
+	for (const [version, entry] of Object.entries(breaks)) {
+		if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+			throw new Error(`Known breaks for ${version} must be an object`);
+		}
+		const unknown = Object.keys(entry).filter(key => !["messages", "connect"].includes(key));
+		if (unknown.length) {
+			throw new Error(`Known breaks for ${version} has unknown properties ${unknown.join(", ")}`);
+		}
+		if (entry.messages !== undefined) {
+			if (!entry.messages || typeof entry.messages !== "object" || Array.isArray(entry.messages)) {
+				throw new Error(`Known breaks for ${version} messages must be an object`);
+			}
+			for (const [name, value] of Object.entries(entry.messages)) {
+				if (!/^(request|event) \S+$/.test(name)) {
+					throw new Error(`Known break ${version} ${name} must be named "request <Name>" or "event <Name>"`);
+				}
+				validateBreak(value, `Known break ${version} ${name}`);
+			}
+		}
+		if (entry.connect !== undefined) {
+			validateBreak(entry.connect, `Known break ${version} connect`);
+		}
+	}
+	return breaks;
+}
+
+export async function loadBreaks(file = breaksFile) {
+	return validateBreaks(JSON.parse(await fs.readFile(file, "utf8")));
+}
+
+/**
+ * Pick the entry for the published version and list the versions ignored.
+ */
+export function selectBreaks(breaks, version) {
+	return {
+		messages: new Map(Object.entries(breaks[version]?.messages ?? {})),
+		connect: breaks[version]?.connect,
+		ignored: Object.keys(breaks).filter(key => key !== version),
+	};
+}
+
+/**
+ * Split changed message names into known and unknown breaks, and list the
+ * known breaks that did not change.
+ */
+export function matchBreaks(messages, changedNames) {
+	const changed = new Set(changedNames);
+	return {
+		known: changedNames.filter(name => messages.has(name)),
+		unknown: changedNames.filter(name => !messages.has(name)),
+		stale: [...messages.keys()].filter(name => !changed.has(name)),
+	};
+}

scripts/version-compat-breaks.json

@@ -0,0 +1,14 @@
+{
+	"2.0.3": {
+		"messages": {
+			"request FactorioVersionsRequest": {
+				"pr": 1129,
+				"reason": "Hosts send it while starting an instance, so a newer host cannot start instances on a 2.0.3 controller."
+			},
+			"request LatestReleasesRequest": {
+				"pr": 1129,
+				"reason": "Hosts send it while starting an instance that targets a release channel such as stable, which a 2.0.3 controller rejects."
+			}
+		}
+	}
+}

scripts/version-compat-test.js

@@ -14,6 +14,9 @@
  * version has are left out of the comparison, as the other version ignores
  * them when received and does not need them when sent.
  *
+ * Intended breaks are listed in scripts/version-compat-breaks.json, see
+ * scripts/version-compat-breaks.js for the format.
+ *
  * Usage:
  *   node scripts/version-compat-test.js --controller local --host published
  *   node scripts/version-compat-test.js --controller published --host local --published-version 2.0.0-alpha.27
@@ -28,6 +31,8 @@ import path from "node:path";
 import readline from "node:readline";
 import util from "node:util";
 
+import { loadBreaks, matchBreaks, selectBreaks } from "./version-compat-breaks.js";
+
 const execFile = util.promisify(child_process.execFile);
 
 const repoRoot = path.join(import.meta.dirname, "..");
@@ -196,7 +201,15 @@ async function messageSpecs(libPath) {
 	return new Map(Object.entries(JSON.parse(result.stdout)));
 }
 
-async function checkMessageCompat(publishedVersion) {
+function printChange(print, name, fields, localSpec, publishedSpec, publishedVersion) {
+	print(`Link ${name} changed ${fields.join(", ")} since ${publishedVersion}:`);
+	for (const field of fields) {
+		print(`  ${field} in ${publishedVersion}: ${stableStringify(publishedSpec[field])}`);
+		print(`  ${field} in local: ${stableStringify(localSpec[field])}`);
+	}
+}
+
+async function checkMessageCompat(publishedVersion, knownBreaks) {
 	const localSpecs = await messageSpecs(path.join(repoRoot, "packages", "lib"));
 	const publishedSpecs = await messageSpecs(path.join(publishedDir, "node_modules", "@clusterio", "lib"));
 	console.log(`| comparing ${localSpecs.size} local with ${publishedSpecs.size} published link messages`);
@@ -231,15 +244,24 @@ async function checkMessageCompat(publishedVersion) {
 		}
 	}
 
+	const { known, unknown, stale } = matchBreaks(knownBreaks, changed.map(({ name }) => name));
 	for (const { name, fields, localSpec, publishedSpec } of changed) {
-		console.error(`Link ${name} changed ${fields.join(", ")} since ${publishedVersion}:`);
-		for (const field of fields) {
-			console.error(`  ${field} in ${publishedVersion}: ${stableStringify(publishedSpec[field])}`);
-			console.error(`  ${field} in local: ${stableStringify(localSpec[field])}`);
+		if (known.includes(name)) {
+			const { pr, reason } = knownBreaks.get(name);
+			annotate(`Link ${name} is a known break since ${publishedVersion} from #${pr}: ${reason}`);
+			printChange(console.log, name, fields, localSpec, publishedSpec, publishedVersion);
+		} else {
+			printChange(console.error, name, fields, localSpec, publishedSpec, publishedVersion);
 		}
 	}
-	if (changed.length) {
-		throw new Error(`${changed.length} link message(s) changed serialization since ${publishedVersion}`);
+	for (const name of stale) {
+		console.error(`Link ${name} is listed as a known break but did not change since ${publishedVersion}`);
+	}
+	const problems = [];
+	if (unknown.length) { problems.push(`${unknown.length} link message(s) changed serialization`); }
+	if (stale.length) { problems.push(`${stale.length} known break(s) did not change`); }
+	if (problems.length) {
+		throw new Error(`${problems.join(" and ")} since ${publishedVersion}`);
 	}
 }
 
@@ -259,6 +281,7 @@ async function pollCtl(ctl, cliArgs, expectRegex, description, timeoutMs = 30e3)
 
 async function main() {
 	const options = parseArgs();
+	const breaks = await loadBreaks();
 
 	await fs.rm(workDir, { force: true, recursive: true, maxRetries: 10 });
 	await fs.mkdir(controllerDir, { recursive: true });
@@ -279,7 +302,17 @@ async function main() {
 	const publishedLib = JSON.parse(await fs.readFile(
 		path.join(publishedDir, "node_modules", "@clusterio", "lib", "package.json"), "utf8"
 	));
-	await checkMessageCompat(publishedLib.version);
+	const knownBreaks = selectBreaks(breaks, publishedLib.version);
+	for (const version of knownBreaks.ignored) {
+		annotate(`Known breaks for ${version} do not apply to ${publishedLib.version} and can be removed`);
+	}
+	await checkMessageCompat(publishedLib.version, knownBreaks.messages);
+	if (knownBreaks.connect) {
+		const { pr, reason } = knownBreaks.connect;
+		annotate(`Skipping cluster test, known break since ${publishedLib.version} from #${pr}: ${reason}`);
+		console.log(`PASS: link messages match ${publishedLib.version} apart from known breaks`);
+		return;
+	}
 
 	const controller = packageDir("controller", options.controller);
 	const host = packageDir("host", options.host);

test/version_compat_breaks.js

@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+
+import {
+	loadBreaks, matchBreaks, selectBreaks, validateBreaks,
+} from "../scripts/version-compat-breaks.js";
+
+
+describe("version compatibility known breaks", function() {
+	const breaks = {
+		"2.0.3": {
+			messages: {
+				"request ListedRequest": { pr: 1, reason: "Listed." },
+				"event UnchangedEvent": { pr: 2, reason: "Unchanged." },
+			},
+			connect: { pr: 3, reason: "Cannot connect." },
+		},
+		"2.0.2": {
+			messages: { "request OldRequest": { pr: 4, reason: "Old." } },
+		},
+	};
+
+	describe("validateBreaks()", function() {
+		it("should accept the committed file", async function() {
+			await loadBreaks();
+		});
+		it("should accept a valid file", function() {
+			validateBreaks(breaks);
+			validateBreaks({});
+		});
+		it("should require a pr number and reason", function() {
+			assert.throws(
+				() => validateBreaks({ "1.0.0": { messages: { "request A": { reason: "A." } } } }),
+				/must have a pr number/
+			);
+			assert.throws(
+				() => validateBreaks({ "1.0.0": { messages: { "request A": { pr: 1, reason: " " } } } }),
+				/must have a reason/
+			);
+			assert.throws(
+				() => validateBreaks({ "1.0.0": { connect: { pr: 1 } } }),
+				/connect must have a reason/
+			);
+		});
+		it("should reject unknown properties and bad message names", function() {
+			assert.throws(() => validateBreaks({ "1.0.0": { connected: false } }), /unknown properties connected/);
+			assert.throws(
+				() => validateBreaks({ "1.0.0": { messages: { "request A": { pr: 1, reason: "A.", x: 1 } } } }),
+				/unknown properties x/
+			);
+			assert.throws(
+				() => validateBreaks({ "1.0.0": { messages: { "A": { pr: 1, reason: "A." } } } }),
+				/must be named/
+			);
+		});
+	});
+
+	describe("selectBreaks()", function() {
+		it("should pick the entry for the published version", function() {
+			const selected = selectBreaks(breaks, "2.0.3");
+			assert.deepEqual([...selected.messages.keys()], ["request ListedRequest", "event UnchangedEvent"]);
+			assert.deepEqual(selected.connect, { pr: 3, reason: "Cannot connect." });
+			assert.deepEqual(selected.ignored, ["2.0.2"]);
+		});
+		it("should ignore every entry when the version is not listed", function() {
+			const selected = selectBreaks(breaks, "2.0.4");
+			assert.equal(selected.messages.size, 0);
+			assert.equal(selected.connect, undefined);
+			assert.deepEqual(selected.ignored, ["2.0.3", "2.0.2"]);
+		});
+	});
+
+	describe("matchBreaks()", function() {
+		const { messages } = selectBreaks(breaks, "2.0.3");
+		it("should split changed messages into known and unknown", function() {
+			const result = matchBreaks(messages, ["request ListedRequest", "event UnchangedEvent", "event OtherEvent"]);
+			assert.deepEqual(result, {
+				known: ["request ListedRequest", "event UnchangedEvent"],
+				unknown: ["event OtherEvent"],
+				stale: [],
+			});
+		});
+		it("should report listed messages that did not change as stale", function() {
+			const result = matchBreaks(messages, ["request ListedRequest"]);
+			assert.deepEqual(result.stale, ["event UnchangedEvent"]);
+		});
+		it("should not match messages listed for another version", function() {
+			const result = matchBreaks(selectBreaks(breaks, "2.0.3").messages, ["request OldRequest"]);
+			assert.deepEqual(result.unknown, ["request OldRequest"]);
+		});
+	});
+});