Allow known breaks in the version compatibility test

The version-compat job fails every pull request once master breaks
compatibility with the latest release on purpose. List those breaks in
scripts/version-compat-breaks.json keyed by the published version they
break against, each with a pr number and reason. Listed messages warn
instead of failing, connect skips the cluster test, entries for other
versions are ignored, and a listed message that did not change fails.

Add the two src changes from #1129 against 2.0.3 so master passes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/version-compat-breaks.js

@@ -0,0 +1,98 @@
+/**
+ * Known compatibility breaks for the version compatibility test
+ *
+ * scripts/version-compat-breaks.json lists the intended breaks against a
+ * published version, keyed by that version:
+ *
+ *   {
+ *     "2.0.3": {
+ *       "messages": {
+ *         "request FactorioVersionsRequest": { "pr": 1129, "reason": "..." }
+ *       },
+ *       "connect": { "pr": 1234, "reason": "..." }
+ *     }
+ *   }
+ *
+ * Listed messages are warned about instead of failing the test, and an entry
+ * with connect skips starting the cluster.  Entries for other versions are
+ * ignored.  See docs/contributing.md for when to add an entry.
+ */
+"use strict";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export const breaksFile = path.join(import.meta.dirname, "version-compat-breaks.json");
+
+function validateBreak(value, where) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) {
+		throw new Error(`${where} must be an object`);
+	}
+	if (!Number.isInteger(value.pr) || value.pr <= 0) {
+		throw new Error(`${where} must have a pr number`);
+	}
+	if (typeof value.reason !== "string" || !value.reason.trim()) {
+		throw new Error(`${where} must have a reason`);
+	}
+	const unknown = Object.keys(value).filter(key => !["pr", "reason"].includes(key));
+	if (unknown.length) {
+		throw new Error(`${where} has unknown properties ${unknown.join(", ")}`);
+	}
+}
+
+export function validateBreaks(breaks) {
+	if (!breaks || typeof breaks !== "object" || Array.isArray(breaks)) {
+		throw new Error("Known breaks must be an object keyed by version");
+	}
+	for (const [version, entry] of Object.entries(breaks)) {
+		if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+			throw new Error(`Known breaks for ${version} must be an object`);
+		}
+		const unknown = Object.keys(entry).filter(key => !["messages", "connect"].includes(key));
+		if (unknown.length) {
+			throw new Error(`Known breaks for ${version} has unknown properties ${unknown.join(", ")}`);
+		}
+		if (entry.messages !== undefined) {
+			if (!entry.messages || typeof entry.messages !== "object" || Array.isArray(entry.messages)) {
+				throw new Error(`Known breaks for ${version} messages must be an object`);
+			}
+			for (const [name, value] of Object.entries(entry.messages)) {
+				if (!/^(request|event) \S+$/.test(name)) {
+					throw new Error(`Known break ${version} ${name} must be named "request <Name>" or "event <Name>"`);
+				}
+				validateBreak(value, `Known break ${version} ${name}`);
+			}
+		}
+		if (entry.connect !== undefined) {
+			validateBreak(entry.connect, `Known break ${version} connect`);
+		}
+	}
+	return breaks;
+}
+
+export async function loadBreaks(file = breaksFile) {
+	return validateBreaks(JSON.parse(await fs.readFile(file, "utf8")));
+}
+
+/**
+ * Pick the entry for the published version and list the versions ignored.
+ */
+export function selectBreaks(breaks, version) {
+	return {
+		messages: new Map(Object.entries(breaks[version]?.messages ?? {})),
+		connect: breaks[version]?.connect,
+		ignored: Object.keys(breaks).filter(key => key !== version),
+	};
+}
+
+/**
+ * Split changed message names into known and unknown breaks, and list the
+ * known breaks that did not change.
+ */
+export function matchBreaks(messages, changedNames) {
+	const changed = new Set(changedNames);
+	return {
+		known: changedNames.filter(name => messages.has(name)),
+		unknown: changedNames.filter(name => !messages.has(name)),
+		stale: [...messages.keys()].filter(name => !changed.has(name)),
+	};
+}

scripts/version-compat-breaks.json

@@ -0,0 +1,14 @@
+{
+	"2.0.3": {
+		"messages": {
+			"request FactorioVersionsRequest": {
+				"pr": 1129,
+				"reason": "Hosts send it while starting an instance, so a newer host cannot start instances on a 2.0.3 controller."
+			},
+			"request LatestReleasesRequest": {
+				"pr": 1129,
+				"reason": "Hosts send it while starting an instance that targets a release channel such as stable, which a 2.0.3 controller rejects."
+			}
+		}
+	}
+}

scripts/version-compat-test.js

@@ -14,6 +14,9 @@
  * version has are left out of the comparison, as the other version ignores
  * them when received and does not need them when sent.
  *
+ * Intended breaks are listed in scripts/version-compat-breaks.json, see
+ * scripts/version-compat-breaks.js for the format.
+ *
  * Usage:
  *   node scripts/version-compat-test.js --controller local --host published
  *   node scripts/version-compat-test.js --controller published --host local --published-version 2.0.0-alpha.27
@@ -28,6 +31,8 @@ import path from "node:path";
 import readline from "node:readline";
 import util from "node:util";
 
+import { loadBreaks, matchBreaks, selectBreaks } from "./version-compat-breaks.js";
+
 const execFile = util.promisify(child_process.execFile);
 
 const repoRoot = path.join(import.meta.dirname, "..");
@@ -196,7 +201,15 @@ async function messageSpecs(libPath) {
 	return new Map(Object.entries(JSON.parse(result.stdout)));
 }
 
-async function checkMessageCompat(publishedVersion) {
+function printChange(print, name, fields, localSpec, publishedSpec, publishedVersion) {
+	print(`Link ${name} changed ${fields.join(", ")} since ${publishedVersion}:`);
+	for (const field of fields) {
+		print(`  ${field} in ${publishedVersion}: ${stableStringify(publishedSpec[field])}`);
+		print(`  ${field} in local: ${stableStringify(localSpec[field])}`);
+	}
+}
+
+async function checkMessageCompat(publishedVersion, knownBreaks) {
 	const localSpecs = await messageSpecs(path.join(repoRoot, "packages", "lib"));
 	const publishedSpecs = await messageSpecs(path.join(publishedDir, "node_modules", "@clusterio", "lib"));
 	console.log(`| comparing ${localSpecs.size} local with ${publishedSpecs.size} published link messages`);
@@ -231,15 +244,24 @@ async function checkMessageCompat(publishedVersion) {
 		}
 	}
 
+	const { known, unknown, stale } = matchBreaks(knownBreaks, changed.map(({ name }) => name));
 	for (const { name, fields, localSpec, publishedSpec } of changed) {
-		console.error(`Link ${name} changed ${fields.join(", ")} since ${publishedVersion}:`);
-		for (const field of fields) {
-			console.error(`  ${field} in ${publishedVersion}: ${stableStringify(publishedSpec[field])}`);
-			console.error(`  ${field} in local: ${stableStringify(localSpec[field])}`);
+		if (known.includes(name)) {
+			const { pr, reason } = knownBreaks.get(name);
+			annotate(`Link ${name} is a known break since ${publishedVersion} from #${pr}: ${reason}`);
+			printChange(console.log, name, fields, localSpec, publishedSpec, publishedVersion);
+		} else {
+			printChange(console.error, name, fields, localSpec, publishedSpec, publishedVersion);
 		}
 	}
-	if (changed.length) {
-		throw new Error(`${changed.length} link message(s) changed serialization since ${publishedVersion}`);
+	for (const name of stale) {
+		console.error(`Link ${name} is listed as a known break but did not change since ${publishedVersion}`);
+	}
+	const problems = [];
+	if (unknown.length) { problems.push(`${unknown.length} link message(s) changed serialization`); }
+	if (stale.length) { problems.push(`${stale.length} known break(s) did not change`); }
+	if (problems.length) {
+		throw new Error(`${problems.join(" and ")} since ${publishedVersion}`);
 	}
 }
 
@@ -259,6 +281,7 @@ async function pollCtl(ctl, cliArgs, expectRegex, description, timeoutMs = 30e3)
 
 async function main() {
 	const options = parseArgs();
+	const breaks = await loadBreaks();
 
 	await fs.rm(workDir, { force: true, recursive: true, maxRetries: 10 });
 	await fs.mkdir(controllerDir, { recursive: true });
@@ -279,7 +302,17 @@ async function main() {
 	const publishedLib = JSON.parse(await fs.readFile(
 		path.join(publishedDir, "node_modules", "@clusterio", "lib", "package.json"), "utf8"
 	));
-	await checkMessageCompat(publishedLib.version);
+	const knownBreaks = selectBreaks(breaks, publishedLib.version);
+	for (const version of knownBreaks.ignored) {
+		annotate(`Known breaks for ${version} do not apply to ${publishedLib.version} and can be removed`);
+	}
+	await checkMessageCompat(publishedLib.version, knownBreaks.messages);
+	if (knownBreaks.connect) {
+		const { pr, reason } = knownBreaks.connect;
+		annotate(`Skipping cluster test, known break since ${publishedLib.version} from #${pr}: ${reason}`);
+		console.log(`PASS: link messages match ${publishedLib.version} apart from known breaks`);
+		return;
+	}
 
 	const controller = packageDir("controller", options.controller);
 	const host = packageDir("host", options.host);