Add changelog script parsing entries from PRs (#871)

Add script which uses GitHub's REST API to fetch issues and pull
requests that have changed since the last release, parse out the
changelog entries from merged pull requests, and print a formatted
version of the changelog suitable for copying into the release
description, the CHANGELOG.md file and the Discord announcement.

The script expects Pull Requests to have a `### Changelog` section in
them with a code block containing changelog entries.  As an added bonus
the script can parse out `#<number>` references so that you do not need to
write it out in the ugly `[#id](long-url-with-id-repeated)` format.

Updates documentation and pull request template to describe the
changelog format and usage of the formatting script.

Includes improvements from review and a second look.

.github/pull_request_template.md

@@ -1,10 +1,15 @@
-Describe why and what has been changed by this pullrequest, feel free to also ask for input on decisions you are unsure about etc.
-The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferrentially link to the associated issues. If there are no relevant issues, link to the pull request itself.
+<!--
+Describe why and what has been changed by this pull request, feel free to also ask for input on decisions you are unsure about etc.
 
-## Changelog
+The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferentially link to the associated issues. If there are no relevant issues, link to the pull request itself.
+
+If there are no relevant changelog entries replace the code block below with "None." followed by an optional reason for why it is not needed.
+-->
+
+### Changelog
 ```
 ### Features
-- Added doodad to web interface. [#999](https://github.com/clusterio/clusterio/issues/999)
+- Added doodad to web interface. #999
 
 ### Changes
 - ...
@@ -12,6 +17,6 @@ The changelog lines below will be copied directly from merged pull requests to t
 ### Fixes
 - ...
 
-### Breaking changes
+### Breaking Changes
 - ...
 ```

docs/contributing.md

@@ -137,6 +137,8 @@ You can also pass the `--set-upstream` (or `-u`) flag to make `git push` do the
 After your feature branch have been merged you should consider it final.
 If there are more changes you want to have merged or mistakes you need to fix you must start a new feature branch and submit a new pull request.
 
+Pull request descriptions should contain a `### Changelog` section with additions for the changelog contained inside a code block, see the [CHANGELOG.md](/CHANGELOG.md) file for the formatting of changelog entries and possible sections to use.
+If there are no changelog entries to add, remove the code block and start the changelog section with the text `None.`, followed by an optional reason for why it is not is needed.
 
 ### Editing your Feature Branch
 

docs/release-workflow.md

@@ -5,7 +5,9 @@
 - CHANGELOG.md in the repository root
 - The changelog.txt file in mod directories
 
-Pull requests will usually have suggestions for changelog entries.
+Pull requests will usually have suggestions for changelog entries. These can be automatically parsed and formatted by running `node scripts/pr_to_changelog.mts`, though you will have to manually sort through which should go in the changelog in the repository root and which should go in the changelogs in mod directories.
+
+If the script fails due to GitHub's 60/hour public access rate limit, you can set the `GH_TOKEN` environment variable to a fine-grained personal access token which has a much higher rate limits.
 
 ## Packages
 

package.json

@@ -44,6 +44,7 @@
 		"@eslint/js": "^9.32.0",
 		"@stylistic/eslint-plugin": "^5.2.2",
 		"@swc/core": "^1.13.3",
+		"@types/node": "catalog:",
 		"@typescript-eslint/eslint-plugin": "^8.38.0",
 		"@typescript-eslint/parser": "^8.38.0",
 		"eslint": "^9.32.0",

pnpm-workspace.yaml

@@ -6,33 +6,33 @@ packages:
   - "test/"
 
 catalog:
-  "@types/node": ^20.17.19
-  "@types/fs-extra": ^11.0.1
-  "@types/express": ^4.17.17
-  "@types/finalhandler": ^1.2.0
-  "@types/jsonwebtoken": ^9.0.9
-  "@types/set-blocking": ^2.0.0
-  "@types/react": ^18.3.18
-  "@types/ws": ^8.5.14
   "@sinclair/typebox": ^0.30.4
   "@swc/core": ^1.13.3
+  "@types/express": ^4.17.17
+  "@types/finalhandler": ^1.2.0
+  "@types/fs-extra": ^11.0.1
+  "@types/jsonwebtoken": ^9.0.9
+  "@types/node": ^20.17.19
+  "@types/react": ^18.3.18
+  "@types/set-blocking": ^2.0.0
+  "@types/ws": ^8.5.14
   "antd": ^5.24.2
+  "chalk": ^4.1.2
+  "express": ^5.1.0
+  "fs-extra": ^8.1.0
+  "jsonwebtoken": ^9.0.2
+  "jszip": ^3.10.1
   "react": ^18.2.0
   "react-dom": ^18.2.0
   "react-router-dom": ^6.29.0
+  "set-blocking": ^2.0.0
   "typescript": ^5.7.3
   "webpack": ^5.98.0
   "webpack-cli": ^5.1.4
   "webpack-merge": ^5.9.0
-  "fs-extra": ^8.1.0
-  "express": ^5.1.0
   "winston": ^3.17.0
-  "winston-transport": ^4.9.0
   "winston-daily-rotate-file": ^4.7.1
-  "jsonwebtoken": ^9.0.2
-  "set-blocking": ^2.0.0
-  "jszip": ^3.10.1
-  "chalk": ^4.1.2
+  "winston-transport": ^4.9.0
   "ws": ^8.18.1
 
 catalogMode: strict

scripts/github_api.mts

@@ -0,0 +1,136 @@
+export async function githubFetch(
+	path: string,
+	query: Record<string, string>= {},
+	init: RequestInit & { headers?: Record<string, string> } = {},
+) {
+	const url = new URL("https://api.github.com/");
+	url.pathname = path;
+	for (const [name, value] of Object.entries(query)) {
+		url.searchParams.set(name, value);
+	}
+	init = {
+		...init,
+		headers: {
+			...init.headers as Record<string, string> ?? {},
+			"Accept": "application/vnd.github+json",
+		}
+	};
+	if (process.env.GH_TOKEN) {
+		init.headers!["Authorization"] = `Bearer ${process.env.GH_TOKEN}`;
+	}
+	console.log("Fetching", url.href);
+	const response = await fetch(url, init);
+	if (!response.ok) {
+		console.log(response.headers);
+		throw new Error(`GitHub replied with ${response.status} ${response.statusText}: ${await response.text()}`);
+	}
+	if (!response.headers.get("Content-Type")?.startsWith("application/json")) {
+		throw new Error(
+			`GitHub replied with ${response.status} ${response.statusText}: ${response.headers.get("Content-Type")}`
+		);
+	}
+	return response;
+}
+
+export async function githubFetchJson<T>(
+	path: string,
+	query: Record<string, string> = {},
+	init: RequestInit & { headers?: Record<string, string> } = {},
+): Promise<T> {
+	const response = await githubFetch(path, query, init);
+	return await response.json();
+}
+
+export async function githubFetchJsonPaginated<T>(
+	path: string,
+	query: Record<string, string> = {},
+	init: RequestInit & { headers?: Record<string, string> } = {},
+	shouldContinue: (result: T[]) => boolean = () => true,
+): Promise<T[]> {
+	const data: T[] = [];
+	let pagesRemaining = true;
+	query = {
+		...query,
+		per_page: "100",
+	};
+	while (pagesRemaining) {
+		const response = await githubFetch(path, query, init);
+		const responseItems = await response.json();
+		data.push(...responseItems);
+		const linkHeader = response.headers.get("Link");
+		pagesRemaining = linkHeader?.includes(`rel="next"`) ?? false;
+		if (pagesRemaining) {
+			pagesRemaining = shouldContinue(responseItems);
+		}
+		if (pagesRemaining) {
+			query = {
+				...query,
+				page: String((query.page ? (Number(query.page)) : 1) + 1),
+			}
+		}
+	}
+	return data;
+}
+
+// Minimal types that include only what is used and useful for our scripts.  See
+// https://docs.github.com/en/rest for complete reference.
+export interface PullRequest {
+	number: number,
+	url: string,
+	html_url: string,
+	state: string,
+	title: string,
+	body: string | null,
+	created_at: string,
+	updated_at: string,
+	closed_at: string | null,
+	merged_at: string | null,
+	merge_commit_sha: string | null,
+}
+
+export interface Issue {
+	number: number,
+	title: string,
+	url: string,
+	html_url: string,
+	state: string,
+	body?: string | null,
+	pull_request?: {
+		url: string,
+		html_url: string,
+		merged_at?: string | null,
+	},
+}
+
+export interface Release {
+	tag_name: string,
+	target_commitish: string,
+	name: string | null,
+	body: string | null,
+	created_at: string,
+	published_at: string | null,
+	updated_at?: string | null,
+}
+
+export interface Commit {
+	sha: string,
+	commit: {
+		message: string,
+	},
+	author: null | {} | {
+		name?: null | string,
+		login: string,
+		type: string,
+	},
+	parents: {
+		sha: string,
+	}[],
+}
+
+export interface Reference {
+	ref: string,
+	object: {
+		type: string,
+		sha: string,
+	},
+}

scripts/pr_to_changelog.mts

@@ -0,0 +1,233 @@
+import {
+	type Release,
+	type Issue,
+	githubFetchJson,
+	githubFetchJsonPaginated,
+} from "./github_api.mts";
+
+type Changelog = Record<string, string[]>;
+
+const changelogSections = [
+	"Major Features",
+	"Features",
+	"Fixes",
+	"Changes",
+	"Breaking Changes",
+	"Meta",
+];
+
+// Generic result codes
+const success = 0;
+const failure = 1;
+
+type ParserState = {
+	pos: number,
+	lines: string[],
+};
+
+function createParser(lines: string[]): ParserState {
+	return { pos: 0, lines };
+}
+
+function atEnd(parser: ParserState) {
+	return parser.pos >= parser.lines.length;
+}
+
+function currentLine(parser: ParserState) {
+	if (atEnd(parser)) {
+		throw new Error("Attempt to access line past end");
+	}
+	return parser.lines[parser.pos];
+}
+
+function skipToLine(
+	parser: ParserState,
+	pattern: RegExp
+) {
+	for (; parser.pos < parser.lines.length; parser.pos++) {
+		if (pattern.test(currentLine(parser))) {
+			return success;
+		}
+	}
+	return failure;
+}
+
+function skipEmptyLines(parser: ParserState) {
+	while (parser.pos < parser.lines.length && /^ *$/.test(currentLine(parser))) {
+		parser.pos += 1;
+	}
+}
+
+function extractPullRequestChangelog(
+	parser: ParserState,
+):
+	| [typeof success, string[]]
+	| [typeof failure, undefined, string]
+{
+	const headerFail = skipToLine(parser, /^###? Changelog *$/);
+	if (headerFail) {
+		return [failure, , "Header not found"];
+	}
+	parser.pos += 1;
+	skipEmptyLines(parser);
+	if (!atEnd(parser) && /^None/i.test(currentLine(parser))) {
+		parser.pos += 1;
+		return [success, []];
+	}
+	const blockStartFail = skipToLine(parser, /```/);
+	if (blockStartFail) {
+		return [failure, , "Code block not found after header"];
+	}
+	parser.pos += 1
+	const blockStartPos = parser.pos;
+	const blockEndFail = skipToLine(parser, /```/);
+	if (blockEndFail) {
+		return [failure, , "End of code block not found after header"];
+	}
+	return [success, parser.lines.slice(blockStartPos, parser.pos)];
+}
+
+function parseChangelog(parser: ParserState, pr: Issue, issues: Issue[]): [Changelog, string[]] {
+	let section: string | undefined;
+	let changelog: Changelog = Object.create(null);
+	let warnings: string[] = [];
+	while (!atEnd(parser)) {
+		const line = currentLine(parser);
+		parser.pos += 1;
+		if (line.trim() === "") {
+			continue;
+		}
+		if (line.startsWith("###")) {
+			section = line.slice(4).trim();
+			if (!changelogSections.includes(section)) {
+				warnings.push(
+					`Changelog section ${section} on line ${parser.pos} is not one of ${changelogSections.join(", ")}`
+				);
+			}
+			changelog[section] = changelog[section] ?? [];
+			continue;
+		}
+		if (!section) {
+			warnings.push(`Unexpected content "${line}" outside of a changelog section on line ${parser.pos}`);
+			continue;
+		}
+		if (line.startsWith("- ")) {
+			const entry = line
+				.slice(2)
+				.trim()
+				.replace(/\[#(\d+)\]\(https:\/\/github.com\/clusterio\/clusterio\/(issues|pull)\/\d+\)/g, "#$1");
+			let hasRef = false
+			for (const ref of entry.matchAll(/#(\d+)/g)) {
+				hasRef = true;
+				const id = Number.parseInt(ref[1], 10);
+				const refIssue = issues.find(issue => issue.number === id);
+				if (!refIssue) {
+					warnings.push(
+						`Issue reference ${ref[0]} on line ${parser.pos} does not exist ` +
+						"or has not been updated since the last release"
+					);
+				} else if (refIssue.pull_request && refIssue.number !== pr.number) {
+					warnings.push(
+						`Pull request reference ${ref[0]} on line ${parser.pos} is to a ` +
+						"different Pull request than this one."
+					);
+				}
+			}
+			if (!hasRef) {
+				warnings.push(`Changelog entry on line ${parser.pos} does not reference an issue`);
+			}
+			changelog[section].push(entry);
+			continue;
+		}
+		warnings.push(`Unexpected content "${line}" on line ${parser.pos}`);
+	}
+	return [changelog, warnings];
+}
+
+function changelogFromPullRequests(pullRequests: Issue[], issues: Issue[]) {
+	const mergedChangelog: Changelog = Object.create(null);
+	for (const pr of pullRequests) {
+		const [
+			changelogFail,
+			changelogLines,
+			changelogFailReason
+		] = extractPullRequestChangelog(createParser(pr.body!.split(/\r?\n/)));
+		if (changelogFail) {
+			console.error(`Failed to extract changelog in ${pr.html_url}: ${changelogFailReason}`)
+			continue;
+		}
+		const [changelog, warnings] = parseChangelog(createParser(changelogLines), pr, issues);
+		for (const warning of warnings) {
+			console.warn(`Warning in changelog for ${pr.html_url}: ${warning}`)
+		}
+		for (const [section, items] of Object.entries(changelog)) {
+			mergedChangelog[section] = [...mergedChangelog[section] ?? [], ...items];
+		}
+	}
+	return mergedChangelog;
+}
+
+async function fetchLastRelease() {
+	// Assumes releases are ordered by their creation or tag time in descending order.
+	const releases = await githubFetchJson<Release[]>("/repos/clusterio/clusterio/releases");
+	const latest = releases.find(r => r.target_commitish === "master");
+	if (!latest) {
+		throw new Error("Unable to find latest release on master");
+	}
+	return latest;
+}
+
+async function fetchIssuesUpdatedSince(since: string) {
+	return await githubFetchJsonPaginated<Issue>(
+		"/repos/clusterio/clusterio/issues",
+		{ state: "all", since, sort: "updated", direction: "asc" },
+	);
+}
+
+function printMarkdown(changelog: Changelog, issues: Issue[], refText: (issue: Issue) => string) {
+	const sections = new Set([...changelogSections, ...Object.keys(changelog)]);
+	for (const section of sections) {
+		if (section in changelog) {
+			console.log(`### ${section}`);
+			const entries = changelog[section];
+			console.log(entries.map(text => (
+				`- ${text.replaceAll(/#(\d+)/g, (ref, idAsText) => {
+					const id = Number.parseInt(idAsText, 10);
+					if (!Number.isFinite(id)) {
+						return ref;
+					}
+					const issue = issues.find(issue => issue.number === id);
+					if (!issue) {
+						return ref;
+					}
+					return refText(issue);
+				})}`
+			)).join("\n"));
+			console.log();
+		}
+	}
+}
+
+async function main() {
+	const lastRelease = await fetchLastRelease();
+	const issues = await fetchIssuesUpdatedSince(lastRelease.created_at);
+
+	const pullRequests = issues.filter(issue => (
+		issue.pull_request
+		&& issue.pull_request.merged_at
+		&& issue.pull_request.merged_at > lastRelease.created_at
+	));
+
+	const changelog = changelogFromPullRequests(pullRequests, issues);
+
+	console.log();
+	console.log("=== Github markdown ===");
+	printMarkdown(changelog, issues, issue => `clusterio/clusterio#${issue.number}`);
+	console.log();
+	console.log("=== Discord markdown ===");
+	printMarkdown(changelog, issues, issue => `[#${issue.number}](<${issue.html_url}>`);
+}
+
+if (import.meta.main) {
+	main().catch(console.error);
+}