diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c374ea3e..c91ff2ee 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,10 +1,15 @@ -Describe why and what has been changed by this pullrequest, feel free to also ask for input on decisions you are unsure about etc. -The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferrentially link to the associated issues. If there are no relevant issues, link to the pull request itself. + + +### Changelog ``` ### Features -- Added doodad to web interface. [#999](https://github.com/clusterio/clusterio/issues/999) +- Added doodad to web interface. #999 ### Changes - ... @@ -12,6 +17,6 @@ The changelog lines below will be copied directly from merged pull requests to t ### Fixes - ... -### Breaking changes +### Breaking Changes - ... ``` diff --git a/docs/contributing.md b/docs/contributing.md index 6a5fe1da..a35de4fc 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -137,6 +137,8 @@ You can also pass the `--set-upstream` (or `-u`) flag to make `git push` do the After your feature branch have been merged you should consider it final. If there are more changes you want to have merged or mistakes you need to fix you must start a new feature branch and submit a new pull request. +Pull request descriptions should contain a `### Changelog` section with additions for the changelog contained inside a code block, see the [CHANGELOG.md](/CHANGELOG.md) file for the formatting of changelog entries and possible sections to use. +If there are no changelog entries to add, remove the code block and start the changelog section with the text `None.`, followed by an optional reason for why it is not is needed. ### Editing your Feature Branch diff --git a/docs/release-workflow.md b/docs/release-workflow.md index 8c0adf44..691afec2 100644 --- a/docs/release-workflow.md +++ b/docs/release-workflow.md @@ -5,7 +5,9 @@ - CHANGELOG.md in the repository root - The changelog.txt file in mod directories -Pull requests will usually have suggestions for changelog entries. +Pull requests will usually have suggestions for changelog entries. These can be automatically parsed and formatted by running `node scripts/pr_to_changelog.mts`, though you will have to manually sort through which should go in the changelog in the repository root and which should go in the changelogs in mod directories. + +If the script fails due to GitHub's 60/hour public access rate limit, you can set the `GH_TOKEN` environment variable to a fine-grained personal access token which has a much higher rate limits. ## Packages diff --git a/package.json b/package.json index 6b2eb80e..023ee107 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@eslint/js": "^9.32.0", "@stylistic/eslint-plugin": "^5.2.2", "@swc/core": "^1.13.3", + "@types/node": "catalog:", "@typescript-eslint/eslint-plugin": "^8.38.0", "@typescript-eslint/parser": "^8.38.0", "eslint": "^9.32.0", diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 11bbe69b..fc8d08c6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,33 +6,33 @@ packages: - "test/" catalog: - "@types/node": ^20.17.19 - "@types/fs-extra": ^11.0.1 - "@types/express": ^4.17.17 - "@types/finalhandler": ^1.2.0 - "@types/jsonwebtoken": ^9.0.9 - "@types/set-blocking": ^2.0.0 - "@types/react": ^18.3.18 - "@types/ws": ^8.5.14 "@sinclair/typebox": ^0.30.4 "@swc/core": ^1.13.3 + "@types/express": ^4.17.17 + "@types/finalhandler": ^1.2.0 + "@types/fs-extra": ^11.0.1 + "@types/jsonwebtoken": ^9.0.9 + "@types/node": ^20.17.19 + "@types/react": ^18.3.18 + "@types/set-blocking": ^2.0.0 + "@types/ws": ^8.5.14 "antd": ^5.24.2 + "chalk": ^4.1.2 + "express": ^5.1.0 + "fs-extra": ^8.1.0 + "jsonwebtoken": ^9.0.2 + "jszip": ^3.10.1 "react": ^18.2.0 "react-dom": ^18.2.0 "react-router-dom": ^6.29.0 + "set-blocking": ^2.0.0 "typescript": ^5.7.3 "webpack": ^5.98.0 "webpack-cli": ^5.1.4 "webpack-merge": ^5.9.0 - "fs-extra": ^8.1.0 - "express": ^5.1.0 "winston": ^3.17.0 - "winston-transport": ^4.9.0 "winston-daily-rotate-file": ^4.7.1 - "jsonwebtoken": ^9.0.2 - "set-blocking": ^2.0.0 - "jszip": ^3.10.1 - "chalk": ^4.1.2 + "winston-transport": ^4.9.0 "ws": ^8.18.1 catalogMode: strict diff --git a/scripts/github_api.mts b/scripts/github_api.mts new file mode 100644 index 00000000..3b4f16ea --- /dev/null +++ b/scripts/github_api.mts @@ -0,0 +1,136 @@ +export async function githubFetch( + path: string, + query: Record= {}, + init: RequestInit & { headers?: Record } = {}, +) { + const url = new URL("https://api.github.com/"); + url.pathname = path; + for (const [name, value] of Object.entries(query)) { + url.searchParams.set(name, value); + } + init = { + ...init, + headers: { + ...init.headers as Record ?? {}, + "Accept": "application/vnd.github+json", + } + }; + if (process.env.GH_TOKEN) { + init.headers!["Authorization"] = `Bearer ${process.env.GH_TOKEN}`; + } + console.log("Fetching", url.href); + const response = await fetch(url, init); + if (!response.ok) { + console.log(response.headers); + throw new Error(`GitHub replied with ${response.status} ${response.statusText}: ${await response.text()}`); + } + if (!response.headers.get("Content-Type")?.startsWith("application/json")) { + throw new Error( + `GitHub replied with ${response.status} ${response.statusText}: ${response.headers.get("Content-Type")}` + ); + } + return response; +} + +export async function githubFetchJson( + path: string, + query: Record = {}, + init: RequestInit & { headers?: Record } = {}, +): Promise { + const response = await githubFetch(path, query, init); + return await response.json(); +} + +export async function githubFetchJsonPaginated( + path: string, + query: Record = {}, + init: RequestInit & { headers?: Record } = {}, + shouldContinue: (result: T[]) => boolean = () => true, +): Promise { + const data: T[] = []; + let pagesRemaining = true; + query = { + ...query, + per_page: "100", + }; + while (pagesRemaining) { + const response = await githubFetch(path, query, init); + const responseItems = await response.json(); + data.push(...responseItems); + const linkHeader = response.headers.get("Link"); + pagesRemaining = linkHeader?.includes(`rel="next"`) ?? false; + if (pagesRemaining) { + pagesRemaining = shouldContinue(responseItems); + } + if (pagesRemaining) { + query = { + ...query, + page: String((query.page ? (Number(query.page)) : 1) + 1), + } + } + } + return data; +} + +// Minimal types that include only what is used and useful for our scripts. See +// https://docs.github.com/en/rest for complete reference. +export interface PullRequest { + number: number, + url: string, + html_url: string, + state: string, + title: string, + body: string | null, + created_at: string, + updated_at: string, + closed_at: string | null, + merged_at: string | null, + merge_commit_sha: string | null, +} + +export interface Issue { + number: number, + title: string, + url: string, + html_url: string, + state: string, + body?: string | null, + pull_request?: { + url: string, + html_url: string, + merged_at?: string | null, + }, +} + +export interface Release { + tag_name: string, + target_commitish: string, + name: string | null, + body: string | null, + created_at: string, + published_at: string | null, + updated_at?: string | null, +} + +export interface Commit { + sha: string, + commit: { + message: string, + }, + author: null | {} | { + name?: null | string, + login: string, + type: string, + }, + parents: { + sha: string, + }[], +} + +export interface Reference { + ref: string, + object: { + type: string, + sha: string, + }, +} diff --git a/scripts/pr_to_changelog.mts b/scripts/pr_to_changelog.mts new file mode 100644 index 00000000..d974937f --- /dev/null +++ b/scripts/pr_to_changelog.mts @@ -0,0 +1,233 @@ +import { + type Release, + type Issue, + githubFetchJson, + githubFetchJsonPaginated, +} from "./github_api.mts"; + +type Changelog = Record; + +const changelogSections = [ + "Major Features", + "Features", + "Fixes", + "Changes", + "Breaking Changes", + "Meta", +]; + +// Generic result codes +const success = 0; +const failure = 1; + +type ParserState = { + pos: number, + lines: string[], +}; + +function createParser(lines: string[]): ParserState { + return { pos: 0, lines }; +} + +function atEnd(parser: ParserState) { + return parser.pos >= parser.lines.length; +} + +function currentLine(parser: ParserState) { + if (atEnd(parser)) { + throw new Error("Attempt to access line past end"); + } + return parser.lines[parser.pos]; +} + +function skipToLine( + parser: ParserState, + pattern: RegExp +) { + for (; parser.pos < parser.lines.length; parser.pos++) { + if (pattern.test(currentLine(parser))) { + return success; + } + } + return failure; +} + +function skipEmptyLines(parser: ParserState) { + while (parser.pos < parser.lines.length && /^ *$/.test(currentLine(parser))) { + parser.pos += 1; + } +} + +function extractPullRequestChangelog( + parser: ParserState, +): + | [typeof success, string[]] + | [typeof failure, undefined, string] +{ + const headerFail = skipToLine(parser, /^###? Changelog *$/); + if (headerFail) { + return [failure, , "Header not found"]; + } + parser.pos += 1; + skipEmptyLines(parser); + if (!atEnd(parser) && /^None/i.test(currentLine(parser))) { + parser.pos += 1; + return [success, []]; + } + const blockStartFail = skipToLine(parser, /```/); + if (blockStartFail) { + return [failure, , "Code block not found after header"]; + } + parser.pos += 1 + const blockStartPos = parser.pos; + const blockEndFail = skipToLine(parser, /```/); + if (blockEndFail) { + return [failure, , "End of code block not found after header"]; + } + return [success, parser.lines.slice(blockStartPos, parser.pos)]; +} + +function parseChangelog(parser: ParserState, pr: Issue, issues: Issue[]): [Changelog, string[]] { + let section: string | undefined; + let changelog: Changelog = Object.create(null); + let warnings: string[] = []; + while (!atEnd(parser)) { + const line = currentLine(parser); + parser.pos += 1; + if (line.trim() === "") { + continue; + } + if (line.startsWith("###")) { + section = line.slice(4).trim(); + if (!changelogSections.includes(section)) { + warnings.push( + `Changelog section ${section} on line ${parser.pos} is not one of ${changelogSections.join(", ")}` + ); + } + changelog[section] = changelog[section] ?? []; + continue; + } + if (!section) { + warnings.push(`Unexpected content "${line}" outside of a changelog section on line ${parser.pos}`); + continue; + } + if (line.startsWith("- ")) { + const entry = line + .slice(2) + .trim() + .replace(/\[#(\d+)\]\(https:\/\/github.com\/clusterio\/clusterio\/(issues|pull)\/\d+\)/g, "#$1"); + let hasRef = false + for (const ref of entry.matchAll(/#(\d+)/g)) { + hasRef = true; + const id = Number.parseInt(ref[1], 10); + const refIssue = issues.find(issue => issue.number === id); + if (!refIssue) { + warnings.push( + `Issue reference ${ref[0]} on line ${parser.pos} does not exist ` + + "or has not been updated since the last release" + ); + } else if (refIssue.pull_request && refIssue.number !== pr.number) { + warnings.push( + `Pull request reference ${ref[0]} on line ${parser.pos} is to a ` + + "different Pull request than this one." + ); + } + } + if (!hasRef) { + warnings.push(`Changelog entry on line ${parser.pos} does not reference an issue`); + } + changelog[section].push(entry); + continue; + } + warnings.push(`Unexpected content "${line}" on line ${parser.pos}`); + } + return [changelog, warnings]; +} + +function changelogFromPullRequests(pullRequests: Issue[], issues: Issue[]) { + const mergedChangelog: Changelog = Object.create(null); + for (const pr of pullRequests) { + const [ + changelogFail, + changelogLines, + changelogFailReason + ] = extractPullRequestChangelog(createParser(pr.body!.split(/\r?\n/))); + if (changelogFail) { + console.error(`Failed to extract changelog in ${pr.html_url}: ${changelogFailReason}`) + continue; + } + const [changelog, warnings] = parseChangelog(createParser(changelogLines), pr, issues); + for (const warning of warnings) { + console.warn(`Warning in changelog for ${pr.html_url}: ${warning}`) + } + for (const [section, items] of Object.entries(changelog)) { + mergedChangelog[section] = [...mergedChangelog[section] ?? [], ...items]; + } + } + return mergedChangelog; +} + +async function fetchLastRelease() { + // Assumes releases are ordered by their creation or tag time in descending order. + const releases = await githubFetchJson("/repos/clusterio/clusterio/releases"); + const latest = releases.find(r => r.target_commitish === "master"); + if (!latest) { + throw new Error("Unable to find latest release on master"); + } + return latest; +} + +async function fetchIssuesUpdatedSince(since: string) { + return await githubFetchJsonPaginated( + "/repos/clusterio/clusterio/issues", + { state: "all", since, sort: "updated", direction: "asc" }, + ); +} + +function printMarkdown(changelog: Changelog, issues: Issue[], refText: (issue: Issue) => string) { + const sections = new Set([...changelogSections, ...Object.keys(changelog)]); + for (const section of sections) { + if (section in changelog) { + console.log(`### ${section}`); + const entries = changelog[section]; + console.log(entries.map(text => ( + `- ${text.replaceAll(/#(\d+)/g, (ref, idAsText) => { + const id = Number.parseInt(idAsText, 10); + if (!Number.isFinite(id)) { + return ref; + } + const issue = issues.find(issue => issue.number === id); + if (!issue) { + return ref; + } + return refText(issue); + })}` + )).join("\n")); + console.log(); + } + } +} + +async function main() { + const lastRelease = await fetchLastRelease(); + const issues = await fetchIssuesUpdatedSince(lastRelease.created_at); + + const pullRequests = issues.filter(issue => ( + issue.pull_request + && issue.pull_request.merged_at + && issue.pull_request.merged_at > lastRelease.created_at + )); + + const changelog = changelogFromPullRequests(pullRequests, issues); + + console.log(); + console.log("=== Github markdown ==="); + printMarkdown(changelog, issues, issue => `clusterio/clusterio#${issue.number}`); + console.log(); + console.log("=== Discord markdown ==="); + printMarkdown(changelog, issues, issue => `[#${issue.number}](<${issue.html_url}>`); +} + +if (import.meta.main) { + main().catch(console.error); +}