Use binary storage for recipe overlay for 8x space reduction

plugins/minimap/controller.ts

@@ -7,9 +7,9 @@ import {
 	ChartData,
 	ChartTagDataEvent,
 	ChartTagData,
-    RecipeDataEvent,
-    RecipeData,
-    GetRawRecipeTileRequest,
+	RecipeDataEvent,
+	RecipeData,
+	GetRawRecipeTileRequest,
 } from "./messages";
 import * as fs from "fs-extra";
 import * as path from "path";
@@ -17,10 +17,12 @@ import * as zlib from "zlib";
 import { promisify } from "util";
 import sharp from "sharp";
 import {
-	renderChunkToPixels, 
-	extractChunkFromTile, 
+	renderChunkToPixels,
+	extractChunkFromTile,
 	renderTileToPixels,
-	pixelsToRGBA
+	pixelsToRGBA,
+	parseRecipeTileBinary,
+	ParsedRecipeTile
 } from "./tile-utils";
 
 const CHUNK_SIZE = 32;
@@ -36,10 +38,13 @@ interface PixelChange {
 export class ControllerPlugin extends BaseControllerPlugin {
 	private tilesPath: string = "";
 	private chartTagsPath: string = "";
-    private recipeTilesPath: string = "";
+	private recipeTilesPath: string = "";
 	private chunkSavingQueue = new Map<string, Map<string, { data: ChartData, tick: number }>>();
 	private chartTagQueues = new Map<string, ChartTagData[]>();
-    private recipeSavingQueue = new Map<string, RecipeData[]>();
+	private recipeSavingQueue = new Map<string, Buffer[]>();
+	// Keep per-tile dictionary of recipe -> uint16 index as well as next free index
+	private recipeDictionaries = new Map<string, Map<string, number>>();
+	private nextRecipeIndex = new Map<string, number>();
 	private lastSeenTagContent = new Map<string, Map<number, {
 		position: [number, number];
 		text: string;
@@ -73,8 +78,8 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.controller.handle(ChartTagDataEvent, this.handleChartTagDataEvent.bind(this));
 		this.controller.handle(GetRawTileRequest, this.handleGetRawTileRequest.bind(this));
 		this.controller.handle(GetChartTagsRequest, this.handleGetChartTagsRequest.bind(this));
-        this.controller.handle(RecipeDataEvent, this.handleRecipeDataEvent.bind(this));
-        this.controller.handle(GetRawRecipeTileRequest, this.handleGetRawRecipeTileRequest.bind(this));
+		this.controller.handle(RecipeDataEvent, this.handleRecipeDataEvent.bind(this));
+		this.controller.handle(GetRawRecipeTileRequest, this.handleGetRawRecipeTileRequest.bind(this));
 
 		// Set up HTTP routes for serving tiles
 		this.setupTileRoutes();
@@ -87,7 +92,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.saveInterval = setInterval(() => {
 			this.saveTiles().catch(err => this.logger.error(`Error saving tiles: ${err}`));
 			this.saveChartTags().catch(err => this.logger.error(`Error saving chart tags: ${err}`));
-            this.saveRecipeTiles().catch(err => this.logger.error(`Error saving recipe tiles: ${err}`));
+			this.saveRecipeTiles().catch(err => this.logger.error(`Error saving recipe tiles: ${err}`));
 		}, 5000);
 	}
 
@@ -96,7 +101,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			clearInterval(this.saveInterval);
 			this.saveInterval = null;
 		}
-		
+
 		// Perform final save before shutdown
 		await this.saveTiles();
 		await this.saveChartTags();
@@ -106,18 +111,18 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	// Compare two pixel arrays and return changes
 	private comparePixels(oldPixels: Uint16Array, newPixels: Uint16Array, chunkX: number, chunkY: number): PixelChange[] {
 		const changes: PixelChange[] = [];
-		
+
 		for (let y = 0; y < 32; y++) {
 			for (let x = 0; x < 32; x++) {
 				const index = y * 32 + x;
 				const oldColor = oldPixels[index];
 				const newColor = newPixels[index];
-				
+
 				if (oldColor !== newColor) {
 					// Calculate absolute position within the 256x256 tile
 					const absoluteX = chunkX * 32 + x;
 					const absoluteY = chunkY * 32 + y;
-					
+
 					changes.push({
 						x: absoluteX,
 						y: absoluteY,
@@ -127,7 +132,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 				}
 			}
 		}
-		
+
 		return changes;
 	}
 
@@ -136,27 +141,27 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		if (changes.length === 0) {
 			return Buffer.alloc(0);
 		}
-		
+
 		// Type (1) + Tick (4) + Length (2) + Pixel data (6 bytes per pixel)
 		const headerSize = 7;
 		const pixelDataSize = changes.length * 6; // 6 bytes per pixel change
 		const totalSize = headerSize + pixelDataSize;
 		const buffer = Buffer.alloc(totalSize);
-		
+
 		let offset = 0;
-		
+
 		// Type 2 for pixels
 		buffer.writeUInt8(2, offset);
 		offset += 1;
-		
+
 		// Tick (converted to seconds)
 		buffer.writeUInt32BE(Math.floor(tick / 60), offset);
 		offset += 4;
-		
+
 		// Length (number of pixel changes, not bytes)
 		buffer.writeUInt16BE(changes.length, offset);
 		offset += 2;
-		
+
 		// Write pixel changes
 		for (const change of changes) {
 			buffer.writeUInt8(change.x, offset);
@@ -168,7 +173,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			buffer.writeUInt16BE(change.oldColor, offset);
 			offset += 2;
 		}
-		
+
 		return buffer;
 	}
 
@@ -230,14 +235,14 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	async renderTile(tileData: Buffer): Promise<Buffer> {
 		// Use renderTileToPixels to get the current state, then convert to image
 		const currentPixels = await renderTileToPixels(tileData);
-		
+
 		// Convert RGB565 pixel data to RGBA buffer for Sharp  
 		const rgbaData = pixelsToRGBA(currentPixels);
 		const raw = Buffer.from(rgbaData);
 
 		// Create and return the image directly
-		return sharp(raw, { 
-			raw: { width: 256, height: 256, channels: 4 } 
+		return sharp(raw, {
+			raw: { width: 256, height: 256, channels: 4 }
 		}).png().toBuffer();
 	}
 
@@ -281,7 +286,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	async updateTile(existingTile: Buffer, newChunks: Map<string, { data: ChartData, tick: number }>): Promise<Buffer | undefined> {
 		const existingChunks = new Map<string, { tick: number, data: Buffer }>();
-		
+
 		// Parse existing tile data to find existing chunks
 		let offset = 0;
 		while (offset < existingTile.length) {
@@ -289,28 +294,28 @@ export class ControllerPlugin extends BaseControllerPlugin {
 				this.logger.error(`Invalid tile data: cannot read type at offset ${offset}`);
 				break;
 			}
-			
+
 			const type = existingTile.readUInt8(offset);
 			offset += 1;
-			
+
 			if (type === 1) { // Chunk
 				if (offset + 7 > existingTile.length) {
 					this.logger.error(`Invalid tile data: insufficient chunk header data at offset ${offset}`);
 					break;
 				}
-				
+
 				const tick = existingTile.readUInt32BE(offset);
 				offset += 4;
 				const chunkCoordsByte = existingTile.readUInt8(offset);
 				offset += 1;
 				const length = existingTile.readUInt16BE(offset);
 				offset += 2;
-				
+
 				if (offset + length > existingTile.length) {
 					this.logger.error(`Invalid tile data: insufficient chunk data at offset ${offset}`);
 					break;
 				}
-				
+
 				const chunkX = chunkCoordsByte >> 4;
 				const chunkY = chunkCoordsByte & 0x0F;
 				const chunkName = `${chunkX}_${chunkY}`;
@@ -323,18 +328,18 @@ export class ControllerPlugin extends BaseControllerPlugin {
 					this.logger.error(`Invalid tile data: insufficient pixel header data at offset ${offset}`);
 					break;
 				}
-				
+
 				const tick = existingTile.readUInt32BE(offset);
 				offset += 4;
 				const pixelCount = existingTile.readUInt16BE(offset);
 				offset += 2;
 				const pixelDataLength = pixelCount * 6;
-				
+
 				if (offset + pixelDataLength > existingTile.length) {
 					this.logger.error(`Invalid tile data: insufficient pixel data at offset ${offset}`);
 					break;
 				}
-				
+
 				// Skip pixel data since we're not rebuilding the file
 				offset += pixelDataLength;
 			} else {
@@ -352,11 +357,11 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		// Process new chunks and determine what to append
 		const appendBuffers: Buffer[] = [];
 		let hasChanges = false;
-		
+
 		for (const [chunkName, newChunk] of newChunks) {
 			const newChunkData = Buffer.from(newChunk.data.chart_data, "base64");
 			const [chunkX, chunkY] = chunkName.split("_").map(Number);
-			
+
 			if (!existingChunks.has(chunkName)) {
 				// New chunk - append as chunk data (type 1)
 				this.logger.info(`Adding new chunk ${chunkName} to tile`);
@@ -370,7 +375,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			} else {
 				// Existing chunk - compare current tile state with new chunk data
 				const newPixels = await renderChunkToPixels(newChunkData);
-				
+
 				let oldPixels: Uint16Array;
 				if (currentTilePixels) {
 					// Extract current chunk area from the rendered tile
@@ -379,7 +384,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 					// Fallback to black pixels if tile couldn't be rendered
 					oldPixels = new Uint16Array(32 * 32);
 				}
-				
+
 				const changes = this.comparePixels(oldPixels, newPixels, chunkX, chunkY);
 				if (changes.length > 0) {
 					this.logger.info(`Adding ${changes.length} pixel changes for chunk ${chunkName} to tile`);
@@ -412,7 +417,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			for (const file of tagFiles) {
 				const fileKey = file.replace('_chart_tags.json', '');
 				const filePath = path.join(this.chartTagsPath, file);
-				
+
 				if (!this.lastSeenTagContent.has(fileKey)) {
 					this.lastSeenTagContent.set(fileKey, new Map());
 				}
@@ -421,7 +426,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 				try {
 					const content = await fs.readFile(filePath, 'utf-8');
 					const lines = content.trim().split('\n').filter(line => line.trim());
-					
+
 					for (const line of lines) {
 						try {
 							const tagData = JSON.parse(line);
@@ -438,7 +443,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 							this.logger.warn(`Failed to parse chart tag line in ${file}: ${parseErr}`);
 						}
 					}
-					
+
 					this.logger.info(`Loaded ${seenTags.size} existing tag contents from ${file}`);
 				} catch (readErr) {
 					this.logger.warn(`Failed to read chart tag file ${file}: ${readErr}`);
@@ -462,38 +467,21 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			}
 
 			const files = await fs.readdir(this.recipeTilesPath);
-			const recipeFiles = files.filter(file => file.endsWith("_recipes.jsonl"));
+			const recipeFiles = files.filter(file => file.endsWith(".recipes"));
 
 			for (const file of recipeFiles) {
-				const tileKey = file.replace("_recipes.jsonl", "");
+				const tileKey = file.replace(".recipes", "");
 				const filePath = path.join(this.recipeTilesPath, file);
 
-				if (!this.lastSeenRecipeContent.has(tileKey)) {
-					this.lastSeenRecipeContent.set(tileKey, new Map());
-				}
-				const seenRecipes = this.lastSeenRecipeContent.get(tileKey)!;
+				// Initialise structures
+				this.lastSeenRecipeContent.set(tileKey, new Map());
+				this.recipeDictionaries.set(tileKey, new Map());
+				this.nextRecipeIndex.set(tileKey, 0);
 
 				try {
-					const content = await fs.readFile(filePath, "utf-8");
-					const lines = content.trim().split("\n").filter(line => line.trim());
-
-					for (const line of lines) {
-						try {
-							const recipeData = JSON.parse(line) as RecipeData;
-							const posKey = `${recipeData.position[0]}_${recipeData.position[1]}`;
-
-							// Update map: if the recipe has an end_tick it's no longer active, otherwise store/update it.
-							if (recipeData.end_tick !== undefined) {
-								seenRecipes.delete(posKey);
-							} else {
-								seenRecipes.set(posKey, recipeData.recipe);
-							}
-						} catch (parseErr) {
-							this.logger.warn(`Failed to parse recipe line in ${file}: ${parseErr}`);
-						}
-					}
-
-					this.logger.info(`Loaded ${seenRecipes.size} existing recipe entries from ${file}`);
+					const data = await fs.readFile(filePath);
+					await this.parseRecipeTileForState(tileKey, data);
+					this.logger.info(`Loaded recipe state for ${tileKey}`);
 				} catch (readErr) {
 					this.logger.warn(`Failed to read recipe file ${file}: ${readErr}`);
 				}
@@ -503,6 +491,73 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		}
 	}
 
+	// Parse binary recipe tile and populate lastSeenRecipeContent & recipeDictionaries
+	private async parseRecipeTileForState(tileKey: string, buffer: Buffer) {
+		// Derive tile coordinates from key (last two underscore-separated parts)
+		const parts = tileKey.split("_");
+		const tileY = parseInt(parts.pop()!); // last
+		const tileX = parseInt(parts.pop()!); // second last
+		const parsed: ParsedRecipeTile = parseRecipeTileBinary(tileX, tileY, buffer, null);
+
+		// Populate dictionary mapping (name -> index)
+		const dict = this.recipeDictionaries.get(tileKey)!;
+		for (const [id, name] of parsed.dictionary) {
+			dict.set(name, id);
+			const currentNext = this.nextRecipeIndex.get(tileKey) ?? 0;
+			if (id + 1 > currentNext) this.nextRecipeIndex.set(tileKey, id + 1);
+		}
+
+		// Populate active recipe cache
+		const seen = this.lastSeenRecipeContent.get(tileKey)!;
+		for (const [posKey, recipe] of parsed.activeRecipes) {
+			seen.set(posKey, recipe);
+		}
+	}
+
+	// Encode helpers
+	private getOrCreateRecipeIndex(tileKey: string, recipe: string): { index: number, dictEntry?: Buffer } {
+		if (!this.recipeDictionaries.has(tileKey)) {
+			this.recipeDictionaries.set(tileKey, new Map());
+			this.nextRecipeIndex.set(tileKey, 0);
+		}
+		const dict = this.recipeDictionaries.get(tileKey)!;
+		if (dict.has(recipe)) {
+			return { index: dict.get(recipe)! };
+		}
+		const nextIdx = this.nextRecipeIndex.get(tileKey)!;
+		dict.set(recipe, nextIdx);
+		this.nextRecipeIndex.set(tileKey, nextIdx + 1);
+		const nameBuf = Buffer.from(recipe, "utf-8");
+		const dictEntry = Buffer.alloc(1 + 2 + 1 + nameBuf.length);
+		dictEntry.writeUInt8(0, 0);
+		dictEntry.writeUInt16BE(nextIdx, 1);
+		dictEntry.writeUInt8(nameBuf.length, 3);
+		nameBuf.copy(dictEntry, 4);
+		return { index: nextIdx, dictEntry };
+	}
+
+	private encodeSet(tileKey: string, tick: number, px: number, py: number, recipe: string): Buffer[] {
+		const { index, dictEntry } = this.getOrCreateRecipeIndex(tileKey, recipe);
+		const buf = Buffer.alloc(1 + 4 + 1 + 1 + 2);
+		let o = 0;
+		buf.writeUInt8(1, o); o += 1;
+		buf.writeUInt32BE(Math.floor(tick / 60), o); o += 4;
+		buf.writeUInt8(px, o); o += 1;
+		buf.writeUInt8(py, o); o += 1;
+		buf.writeUInt16BE(index, o);
+		return dictEntry ? [dictEntry, buf] : [buf];
+	}
+
+	private encodeClear(tick: number, px: number, py: number): Buffer {
+		const buf = Buffer.alloc(1 + 4 + 1 + 1);
+		let o = 0;
+		buf.writeUInt8(2, o); o += 1;
+		buf.writeUInt32BE(Math.floor(tick / 60), o); o += 4;
+		buf.writeUInt8(px, o); o += 1;
+		buf.writeUInt8(py, o);
+		return buf;
+	}
+
 	async saveChartTags() {
 		if (this.chartTagQueues.size === 0) {
 			return;
@@ -525,12 +580,12 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 				try {
 					const filePath = path.join(this.chartTagsPath, `${fileKey}_chart_tags.json`);
-					
+
 					// Convert chart tags to newline-delimited JSON format
 					const jsonLines = tagsToSave.map(tag => JSON.stringify(tag)).join('\n');
-					
+
 					await fs.appendFile(filePath, jsonLines + '\n');
-					
+
 					// Mark tags as successfully saved
 					if (!this.lastSeenTagContent.has(fileKey)) {
 						this.lastSeenTagContent.set(fileKey, new Map());
@@ -544,7 +599,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 							last_user: tag.last_user
 						});
 					}
-					
+
 					this.logger.info(`Saved ${tagsToSave.length} chart tag entries to ${fileKey}`);
 				} catch (err) {
 					this.logger.error(`Error saving chart tags for ${fileKey}: ${err}`);
@@ -575,19 +630,19 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			// Check for duplicate content, but allow deletion events (end_tick set) even if we've seen the tag before
 			const seenTags = this.lastSeenTagContent.get(fileKey)!;
 			const lastSeenContent = seenTags.get(tag_data.tag_number);
-			
+
 			// For deletion events (end_tick set), always allow through
 			if (tag_data.end_tick !== undefined) {
 				// Don't skip deletion events
 			} else if (lastSeenContent) {
 				// For creation/modification events, check if content has actually changed
-				const contentUnchanged = 
+				const contentUnchanged =
 					lastSeenContent.position[0] === tag_data.position[0] &&
 					lastSeenContent.position[1] === tag_data.position[1] &&
 					lastSeenContent.text === tag_data.text &&
 					JSON.stringify(lastSeenContent.icon) === JSON.stringify(tag_data.icon) &&
 					lastSeenContent.last_user === tag_data.last_user;
-				
+
 				if (contentUnchanged) {
 					return; // Skip if content hasn't changed
 				}
@@ -621,7 +676,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 			// Send live update to connected web clients
 			const updateEvent = new ChartTagDataEvent(instance_id, tag_data);
-			
+
 			// Broadcast to web clients (overriding the formal src/dst routing)
 			this.controller.sendTo(new lib.Address(lib.Address.broadcast, lib.Address.control), updateEvent);
 
@@ -659,7 +714,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 				tick,
 				chunk
 			);
-			
+
 			// Broadcast to web clients (overriding the formal src/dst routing)
 			this.controller.sendTo(new lib.Address(lib.Address.broadcast, lib.Address.control), updateEvent);
 
@@ -670,7 +725,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	async handleGetRawTileRequest(request: GetRawTileRequest) {
 		const { instance_id, surface, force, tile_x, tile_y, tick } = request;
-		
+
 		const tileName = `${instance_id}_${surface}_${force}_${tile_x}_${tile_y}.bin`;
 		const tilePath = path.join(this.tilesPath, tileName);
 
@@ -680,10 +735,10 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 		try {
 			const tileData = await fs.readFile(tilePath);
-			
+
 			const base64Data = tileData.toString('base64');
-			
-			return { 
+
+			return {
 				tile_data: base64Data
 			};
 		} catch (err) {
@@ -694,7 +749,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	async handleGetChartTagsRequest(request: GetChartTagsRequest) {
 		const { instance_id, surface, force } = request;
-		
+
 		const fileKey = `${instance_id}_${surface}_${force}`;
 		const filePath = path.join(this.chartTagsPath, `${fileKey}_chart_tags.json`);
 
@@ -705,7 +760,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		try {
 			const content = await fs.readFile(filePath, 'utf-8');
 			const lines = content.trim().split('\n').filter(line => line.trim());
-			
+
 			const chartTags: ChartTagData[] = [];
 			for (const line of lines) {
 				try {
@@ -715,7 +770,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 					this.logger.warn(`Failed to parse chart tag line: ${parseErr}`);
 				}
 			}
-			
+
 			return { chart_tags: chartTags };
 		} catch (err) {
 			this.logger.error(`Error reading chart tag file: ${err}`);
@@ -737,12 +792,12 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.recipeSavingQueue = new Map();
 
 		try {
-			const promises = [] as Promise<void>[];
-			for (const [tileName, list] of queue) {
+			const promises: Promise<void>[] = [];
+			for (const [tileName, buffers] of queue) {
 				const promise = (async () => {
-					const tilePath = path.join(this.recipeTilesPath, `${tileName}_recipes.jsonl`);
-					const lines = list.map(rec => JSON.stringify(rec)).join("\n") + "\n";
-					await fs.appendFile(tilePath, lines, "utf-8");
+					const tilePath = path.join(this.recipeTilesPath, `${tileName}.recipes`);
+					const concat = Buffer.concat(buffers);
+					await fs.appendFile(tilePath, concat);
 				})();
 				promises.push(promise);
 			}
@@ -764,36 +819,44 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			const tileKey = `${instance_id}_${recipe_data.surface}_${recipe_data.force}_${tileX}_${tileY}`;
 			const posKey = `${x}_${y}`;
 
-			// Initialise the deduplication map for this tile if needed
+			// Initialise dedup map
 			if (!this.lastSeenRecipeContent.has(tileKey)) {
 				this.lastSeenRecipeContent.set(tileKey, new Map());
 			}
 			const seenRecipes = this.lastSeenRecipeContent.get(tileKey)!;
 
-			let shouldEnqueue = true;
+			let buffersToWrite: Buffer[] = [];
+
+			// Translate world pos to local px/py
+			const px = Math.round(x - tileX * 256);
+			const py = Math.round(y - tileY * 256);
 
 			if (recipe_data.end_tick !== undefined) {
-				// A recipe is being removed/ended – always enqueue the event but update our cache
-				seenRecipes.delete(posKey);
-			} else {
-				const existingRecipe = seenRecipes.get(posKey);
-				if (existingRecipe === recipe_data.recipe) {
-					// Duplicate event – skip persisting
-					shouldEnqueue = false;
+				// Clear recipe
+				if (seenRecipes.has(posKey)) {
+					seenRecipes.delete(posKey);
+					buffersToWrite.push(this.encodeClear(recipe_data.end_tick, px, py));
 				} else {
-					// New or changed recipe – remember it
-					seenRecipes.set(posKey, recipe_data.recipe);
+					return; // nothing changed
 				}
+			} else {
+				if (!recipe_data.recipe) {
+					return; // nothing to set
+				}
+				const existing = seenRecipes.get(posKey);
+				if (existing === recipe_data.recipe) {
+					return; // duplicate
+				}
+				seenRecipes.set(posKey, recipe_data.recipe);
+				buffersToWrite = this.encodeSet(tileKey, recipe_data.start_tick ?? 0, px, py, recipe_data.recipe);
 			}
 
-			if (!shouldEnqueue) {
-				return;
-			}
+			if (buffersToWrite.length === 0) return;
 
 			if (!this.recipeSavingQueue.has(tileKey)) {
 				this.recipeSavingQueue.set(tileKey, []);
 			}
-			this.recipeSavingQueue.get(tileKey)!.push(recipe_data);
+			this.recipeSavingQueue.get(tileKey)!.push(...buffersToWrite);
 
 			// broadcast live update
 			this.controller.sendTo(new lib.Address(lib.Address.broadcast, lib.Address.control), event);
@@ -806,15 +869,15 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	async handleGetRawRecipeTileRequest(request: GetRawRecipeTileRequest) {
 		const { instance_id, surface, force, tile_x, tile_y } = request;
 		const tileKey = `${instance_id}_${surface}_${force}_${tile_x}_${tile_y}`;
-		const filePath = path.join(this.recipeTilesPath, `${tileKey}_recipes.jsonl`);
+		const filePath = path.join(this.recipeTilesPath, `${tileKey}.recipes`);
 
 		if (!await fs.pathExists(filePath)) {
 			return { recipe_tile: null };
 		}
 
 		try {
-			const data = await fs.readFile(filePath, "utf-8");
-			const base64 = Buffer.from(data, "utf-8").toString("base64");
+			const data = await fs.readFile(filePath);
+			const base64 = data.toString("base64");
 			return { recipe_tile: base64 };
 		} catch (err) {
 			this.logger.error(`Error reading recipe tile: ${err}`);

plugins/minimap/docs/minimap_recipe_overlay.md

@@ -0,0 +1,88 @@
+# Crafting-Recipe Overlay for Minimap & Timelapse
+
+## Overview
+Records which recipe every crafting machine is running, streams live changes to web clients, and replays them frame-perfectly in timelapse mode. The solution:
+
+* Works for all forces & surfaces
+* Scales to millions of events using compact binary storage
+* Allows cheap frustum-culling using 256×256 map tiles
+* Uses append-only storage so historical data is never rewritten
+
+## Architecture
+
+### Event Flow
+```mermaid
+sequenceDiagram
+    participant Lua as Factorio Lua (minimap.lua)
+    participant Host as Host/Instance plugin (instance.ts)
+    participant Ctl as Controller plugin (controller.ts)
+    participant UI as Web clients
+
+    Lua->>Host: ipc-minimap:recipe_data (JSON)
+    Host->>Ctl: RecipeDataEvent
+    Ctl-->>UI: RecipeDataEvent (broadcast)
+    UI->>Ctl: GetRawRecipeTileRequest (HTTP / cluster bus)
+    Ctl-->>UI: Raw recipe tile (.recipes)
+```
+
+### Messages
+| Message                            | Direction                   | Purpose                     |
+| ---------------------------------- | --------------------------- | --------------------------- |
+| `RecipeDataEvent`                  | instance → controller → web | Single recipe change (live) |
+| `GetRawRecipeTileRequest/Response` | web → controller            | Fetches binary recipe tile  |
+
+## Storage Format
+
+### Tile Organization
+Uses same 256×256 world-unit tiles as terrain data. 
+Filename: `<instance>_<surface>_<force>_<tileX>_<tileY>.recipes`
+Stored in: `database_directory/minimap_recipe_tiles/`
+
+### Binary Record Format (append-only)
+All numbers **big-endian**.
+
+| Bytes               | Field            | Description                                   |
+| ------------------- | ---------------- | --------------------------------------------- |
+| 1                   | `type`           | `0=DictEntry`, `1=SetRecipe`, `2=ClearRecipe` |
+| **DictEntry (0)**   |                  |
+| 2                   | `index` (uint16) | Dictionary ID (0-65535)                       |
+| 1                   | `len` (uint8)    | UTF-8 byte length                             |
+| n                   | name bytes       | Recipe name string                            |
+| **SetRecipe (1)**   |                  |
+| 4                   | `sec` (uint32)   | ⌊tick / 60⌋ timestamp                         |
+| 1                   | `px` (uint8)     | X position in tile (0-255)                    |
+| 1                   | `py` (uint8)     | Y position in tile (0-255)                    |
+| 2                   | `index` (uint16) | Dictionary ID                                 |
+| **ClearRecipe (2)** |                  |
+| 4                   | `sec` (uint32)   | ⌊tick / 60⌋ timestamp                         |
+| 1                   | `px` (uint8)     | X position in tile (0-255)                    |
+| 1                   | `py` (uint8)     | Y position in tile (0-255)                    |
+
+Dictionary entries are written only when a previously unseen recipe name is required for that tile.
+
+### Space Efficiency
+- ~8x more compact than JSON format
+- Dictionary compression eliminates recipe name repetition
+- Per-tile dictionaries keep indices small (uint16 sufficient)
+
+## Playback
+
+### Live Mode
+- `RecipeDataEvent` broadcasts like `TileDataEvent`
+- Web UI maintains `Map<posKey, recipeName>` for instant icon updates
+- Icons render using Factorio sprite metadata
+
+### Timelapse Mode
+- Downloads `.recipes` files via `GetRawRecipeTileRequest`
+- Parses binary format using shared `parseRecipeTileBinary()` utility
+- Applies Set/Clear records up to selected timeline tick
+- Integrates with existing terrain timelapse system
+
+## Scalability
+| Metric                         | Estimate                      |
+| ------------------------------ | ----------------------------- |
+| Assemblers per large factory   | ~50,000                       |
+| Avg recipe changes per machine | ~5 lifetime                   |
+| Bytes per Set/Clear record     | 8 bytes                       |
+| Total storage per instance     | ~2 MB                         |
+| Max recipes per tile           | <1,000 (well under 65K limit) |

plugins/minimap/tile-utils.ts

@@ -74,19 +74,19 @@ export async function renderChunkToPixels(chunkData: BinaryData): Promise<Uint16
 	try {
 		const decompressed = await inflateData(chunkData);
 		const pixels = new Uint16Array(32 * 32);
-		
+
 		// Ensure we don't read beyond available data
 		const maxPixels = Math.min(decompressed.length / 2, 32 * 32);
-		
+
 		for (let i = 0; i < maxPixels; i++) {
 			pixels[i] = readUInt16LE(decompressed, i * 2);
 		}
-		
+
 		// Fill remaining pixels with black (0) if we have less data than expected
 		for (let i = maxPixels; i < 32 * 32; i++) {
 			pixels[i] = 0;
 		}
-		
+
 		return pixels;
 	} catch (decompressError) {
 		console.error(`Failed to decompress chunk data: ${decompressError}`);
@@ -100,7 +100,7 @@ export async function renderChunkToPixels(chunkData: BinaryData): Promise<Uint16
  */
 export function extractChunkFromTile(tilePixels: Uint16Array, chunkX: number, chunkY: number): Uint16Array {
 	const chunkPixels = new Uint16Array(32 * 32);
-	
+
 	for (let y = 0; y < 32; y++) {
 		for (let x = 0; x < 32; x++) {
 			const tileX = chunkX * 32 + x;
@@ -110,7 +110,7 @@ export function extractChunkFromTile(tilePixels: Uint16Array, chunkX: number, ch
 			chunkPixels[chunkIndex] = tilePixels[tileIndex];
 		}
 	}
-	
+
 	return chunkPixels;
 }
 
@@ -120,13 +120,13 @@ export function extractChunkFromTile(tilePixels: Uint16Array, chunkX: number, ch
  */
 export async function renderTileToPixels(tileData: BinaryData, maxTick?: number): Promise<Uint16Array> {
 	const currentPixels = new Uint16Array(256 * 256); // 256x256 tile
-	
+
 	let offset = 0;
 	while (offset < tileData.length) {
 		if (offset + 1 > tileData.length) {
 			break;
 		}
-		
+
 		const type = readUInt8(tileData, offset);
 		offset += 1;
 
@@ -134,10 +134,10 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
 			if (offset + 7 > tileData.length) {
 				break;
 			}
-			
+
 			const tick = readUInt32BE(tileData, offset) * 60; // Convert back to actual tick
 			offset += 4;
-			
+
 			// Skip this chunk if it's beyond our target tick
 			if (maxTick !== undefined && tick > maxTick) {
 				const chunkCoordsByte = readUInt8(tileData, offset);
@@ -147,7 +147,7 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
 				offset += length; // Skip chunk data
 				continue;
 			}
-			
+
 			const chunkCoordsByte = readUInt8(tileData, offset);
 			offset += 1;
 			const length = readUInt16BE(tileData, offset);
@@ -163,7 +163,7 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
 
 			try {
 				const chunkPixels = await renderChunkToPixels(chunkData);
-				
+
 				// Update current pixel state
 				for (let y = 0; y < 32; y++) {
 					for (let x = 0; x < 32; x++) {
@@ -177,29 +177,29 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
 			} catch (decompressError) {
 				console.error(`Failed to decompress chunk data: ${decompressError}`);
 			}
-			
+
 			offset += length;
 		} else if (type === 2) { // Pixels
 			if (offset + 6 > tileData.length) {
 				break;
 			}
-			
+
 			const tick = readUInt32BE(tileData, offset) * 60; // Convert back to actual tick
 			offset += 4;
 			const pixelCount = readUInt16BE(tileData, offset);
 			offset += 2;
-			
+
 			const expectedDataLength = pixelCount * 6;
 			if (offset + expectedDataLength > tileData.length) {
 				break;
 			}
-			
+
 			// Skip this pixel changeset if it's beyond our target tick
 			if (maxTick !== undefined && tick > maxTick) {
 				offset += expectedDataLength;
 				continue;
 			}
-			
+
 			// Apply pixel changes to current state
 			for (let i = 0; i < pixelCount; i++) {
 				const pixelOffset = offset + i * 6;
@@ -207,14 +207,14 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
 				const y = readUInt8(tileData, pixelOffset + 1);
 				const newColor = readUInt16BE(tileData, pixelOffset + 2);
 				// oldColor at pixelOffset + 4 is not needed for rendering
-				
+
 				// Update current pixel state
 				if (x < 256 && y < 256) {
 					const tileIndex = y * 256 + x;
 					currentPixels[tileIndex] = newColor;
 				}
 			}
-			
+
 			offset += expectedDataLength;
 		} else {
 			break;
@@ -230,18 +230,18 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
  */
 export function pixelsToRGBA(pixels: Uint16Array): Uint8ClampedArray {
 	const imageData = new Uint8ClampedArray(256 * 256 * 4);
-	
+
 	for (let i = 0; i < 256 * 256; i++) {
 		const rgb565Value = pixels[i];
 		const [r, g, b] = rgb565ToRgb888(rgb565Value);
 		const bufferIndex = i * 4;
-		
+
 		imageData[bufferIndex] = r;
 		imageData[bufferIndex + 1] = g;
 		imageData[bufferIndex + 2] = b;
 		imageData[bufferIndex + 3] = 255; // Alpha
 	}
-	
+
 	return imageData;
 }
 
@@ -261,13 +261,13 @@ export function pixelsToImageData(pixels: Uint16Array): any {
  */
 export function extractAvailableTicks(tileData: BinaryData): number[] {
 	const ticks = new Set<number>();
-	
+
 	let offset = 0;
 	while (offset < tileData.length) {
 		if (offset + 1 > tileData.length) {
 			break;
 		}
-		
+
 		const type = readUInt8(tileData, offset);
 		offset += 1;
 
@@ -275,11 +275,11 @@ export function extractAvailableTicks(tileData: BinaryData): number[] {
 			if (offset + 7 > tileData.length) {
 				break;
 			}
-			
+
 			const tick = readUInt32BE(tileData, offset) * 60; // Convert back to actual tick
 			ticks.add(tick);
 			offset += 4;
-			
+
 			const chunkCoordsByte = readUInt8(tileData, offset);
 			offset += 1;
 			const length = readUInt16BE(tileData, offset);
@@ -289,19 +289,19 @@ export function extractAvailableTicks(tileData: BinaryData): number[] {
 			if (offset + 6 > tileData.length) {
 				break;
 			}
-			
+
 			const tick = readUInt32BE(tileData, offset) * 60; // Convert back to actual tick
 			ticks.add(tick);
 			offset += 4;
-			
+
 			const pixelCount = readUInt16BE(tileData, offset);
 			offset += 2;
-			
+
 			const expectedDataLength = pixelCount * 6;
 			if (offset + expectedDataLength > tileData.length) {
 				break;
 			}
-			
+
 			offset += expectedDataLength;
 		} else {
 			break;
@@ -355,13 +355,13 @@ export function parseTileData(tileData: BinaryData): ParsedTileData {
 	const chunks: ChunkRecord[] = [];
 	const pixelChanges: PixelChangeRecord[] = [];
 	const ticks = new Set<number>();
-	
+
 	let offset = 0;
 	while (offset < tileData.length) {
 		if (offset + 1 > tileData.length) {
 			break;
 		}
-		
+
 		const type = readUInt8(tileData, offset);
 		offset += 1;
 
@@ -369,11 +369,11 @@ export function parseTileData(tileData: BinaryData): ParsedTileData {
 			if (offset + 7 > tileData.length) {
 				break;
 			}
-			
+
 			const tick = readUInt32BE(tileData, offset) * 60; // Convert back to actual tick
 			ticks.add(tick);
 			offset += 4;
-			
+
 			const chunkCoordsByte = readUInt8(tileData, offset);
 			offset += 1;
 			const length = readUInt16BE(tileData, offset);
@@ -393,24 +393,24 @@ export function parseTileData(tileData: BinaryData): ParsedTileData {
 				chunkY,
 				data
 			});
-			
+
 			offset += length;
 		} else if (type === 2) { // Pixels
 			if (offset + 6 > tileData.length) {
 				break;
 			}
-			
+
 			const tick = readUInt32BE(tileData, offset) * 60; // Convert back to actual tick
 			ticks.add(tick);
 			offset += 4;
 			const pixelCount = readUInt16BE(tileData, offset);
 			offset += 2;
-			
+
 			const expectedDataLength = pixelCount * 6;
 			if (offset + expectedDataLength > tileData.length) {
 				break;
 			}
-			
+
 			const changes: PixelChange[] = [];
 			for (let i = 0; i < pixelCount; i++) {
 				const pixelOffset = offset + i * 6;
@@ -418,19 +418,19 @@ export function parseTileData(tileData: BinaryData): ParsedTileData {
 				const y = readUInt8(tileData, pixelOffset + 1);
 				const newColor = readUInt16BE(tileData, pixelOffset + 2);
 				const oldColor = readUInt16BE(tileData, pixelOffset + 4);
-				
+
 				if (x < 256 && y < 256) {
 					changes.push({ x, y, newColor, oldColor });
 				}
 			}
-			
+
 			if (changes.length > 0) {
 				pixelChanges.push({
 					tick,
 					changes
 				});
 			}
-			
+
 			offset += expectedDataLength;
 		} else {
 			break;
@@ -450,7 +450,7 @@ export function parseTileData(tileData: BinaryData): ParsedTileData {
 export async function applyChunkToPixels(pixels: Uint16Array, chunkRecord: ChunkRecord): Promise<void> {
 	try {
 		const chunkPixels = await renderChunkToPixels(chunkRecord.data);
-		
+
 		// Update tile pixels with chunk data
 		for (let y = 0; y < 32; y++) {
 			for (let x = 0; x < 32; x++) {
@@ -505,29 +505,29 @@ export function revertPixelChanges(pixels: Uint16Array, changes: PixelChange[]):
  * Much more efficient than re-rendering from scratch
  */
 export async function renderTileIncremental(
-	parsedData: ParsedTileData, 
-	currentPixels: Uint16Array, 
-	fromTick: number, 
+	parsedData: ParsedTileData,
+	currentPixels: Uint16Array,
+	fromTick: number,
 	toTick: number
 ): Promise<void> {
 	const { chunks, pixelChanges } = parsedData;
-	
+
 	if (fromTick === toTick) {
 		return; // No changes needed
 	}
-	
+
 	const forward = toTick > fromTick;
-	
+
 	if (forward) {
 		// Apply changes forward from fromTick to toTick
-		
+
 		// Apply chunks that occur in this range
 		for (const chunk of chunks) {
 			if (chunk.tick > fromTick && chunk.tick <= toTick) {
 				await applyChunkToPixels(currentPixels, chunk);
 			}
 		}
-		
+
 		// Apply pixel changes that occur in this range
 		for (const record of pixelChanges) {
 			if (record.tick > fromTick && record.tick <= toTick) {
@@ -536,7 +536,7 @@ export async function renderTileIncremental(
 		}
 	} else {
 		// Revert changes backward from fromTick to toTick
-		
+
 		// Revert pixel changes in reverse order
 		for (let i = pixelChanges.length - 1; i >= 0; i--) {
 			const record = pixelChanges[i];
@@ -544,7 +544,7 @@ export async function renderTileIncremental(
 				revertPixelChanges(currentPixels, record.changes);
 			}
 		}
-		
+
 		// Revert chunks (clear to black if we're going before their creation time)
 		for (let i = chunks.length - 1; i >= 0; i--) {
 			const chunk = chunks[i];
@@ -553,4 +553,91 @@ export async function renderTileIncremental(
 			}
 		}
 	}
+}
+
+/**
+ * Interface for parsed recipe tile data
+ */
+export interface ParsedRecipeTile {
+	ticks: number[]; // ticks present in this tile
+	activeRecipes: Map<string, string>; // posKey -> recipeName
+	dictionary: Map<number, string>; // id -> recipeName
+}
+
+/**
+ * Parse binary recipe tile data and return timeline ticks and active recipes at target tick
+ * @param tileX Tile X coordinate for world position calculation
+ * @param tileY Tile Y coordinate for world position calculation  
+ * @param buf Binary recipe tile data
+ * @param targetTick Target tick for filtering (null = latest state)
+ * @returns Parsed recipe data with ticks and active recipes
+ */
+export function parseRecipeTileBinary(tileX: number, tileY: number, buf: BinaryData, targetTick: number | null): ParsedRecipeTile {
+	const ticks: number[] = [];
+	const dict = new Map<number, string>();
+	const active = new Map<string, string>();
+	const targetSec = targetTick !== null ? Math.floor(targetTick / 60) : Infinity;
+
+	let offset = 0;
+	while (offset < buf.length) {
+		const type = readUInt8(buf, offset);
+		offset += 1;
+
+		if (type === 0) { // Dictionary entry
+			const idx = readUInt16BE(buf, offset);
+			offset += 2;
+			const len = readUInt8(buf, offset);
+			offset += 1;
+			const nameBytes = sliceData(buf, offset, offset + len);
+			offset += len;
+
+			// Convert to string using cross-platform method
+			let recipeName: string;
+			if (isNode && nameBytes instanceof Buffer) {
+				recipeName = nameBytes.toString('utf-8');
+			} else {
+				recipeName = new TextDecoder().decode(nameBytes as Uint8Array);
+			}
+
+			dict.set(idx, recipeName);
+		} else if (type === 1 || type === 2) { // Set recipe (1) or Clear recipe (2)
+			const sec = readUInt32BE(buf, offset);
+			offset += 4;
+			const px = readUInt8(buf, offset);
+			offset += 1;
+			const py = readUInt8(buf, offset);
+			offset += 1;
+
+			let idx: number | undefined;
+			if (type === 1) {
+				idx = readUInt16BE(buf, offset);
+				offset += 2;
+			}
+
+			const tickVal = sec * 60;
+			ticks.push(tickVal);
+
+			if (sec > targetSec) {
+				// Future to target time – we can skip rest if file is chronological
+				// but to be safe just continue without applying
+				continue;
+			}
+
+			const worldX = tileX * 256 + px;
+			const worldY = tileY * 256 + py;
+			const posKey = `${worldX},${worldY}`;
+
+			if (type === 1 && idx !== undefined) {
+				const name = dict.get(idx);
+				if (name) active.set(posKey, name);
+			} else if (type === 2) {
+				active.delete(posKey);
+			}
+		} else {
+			console.warn("Unknown recipe record type", type);
+			break;
+		}
+	}
+
+	return { ticks, activeRecipes: active, dictionary: dict };
 } 

plugins/minimap/web/CanvasMinimapPage.tsx

@@ -10,7 +10,8 @@ import {
 	extractAvailableTicks,
 	parseTileData,
 	renderTileIncremental,
-	ParsedTileData
+	ParsedTileData,
+	parseRecipeTileBinary
 } from "../tile-utils";
 
 const { Text } = Typography;
@@ -527,15 +528,11 @@ export default function CanvasMinimapPage() {
 							tileY
 						));
 						if (recipeResp.recipe_tile) {
-							const text = atob(recipeResp.recipe_tile);
-							const lines = text.trim().split('\n').filter(l => l.trim());
-							for (const line of lines) {
-								try {
-									const rec = JSON.parse(line);
-									if (rec.start_tick !== undefined) { allTicks.add(rec.start_tick); recipeTicksAdded++; }
-									if (rec.end_tick !== undefined) { allTicks.add(rec.end_tick); recipeTicksAdded++; }
-								} catch { /* ignore parse errors */ }
-							}
+							const binStr = atob(recipeResp.recipe_tile);
+							const buf = new Uint8Array(binStr.length);
+							for (let i = 0; i < binStr.length; i++) buf[i] = binStr.charCodeAt(i);
+							const parsed = parseRecipeTileBinary(tileX, tileY, buf, null);
+							for (const t of parsed.ticks) { allTicks.add(t); recipeTicksAdded++; }
 						}
 					} catch (e) {
 						console.warn(`Failed to load recipe tile ${tileX},${tileY}:`, e);
@@ -1690,22 +1687,15 @@ export default function CanvasMinimapPage() {
 				cs.selectedInstance, cs.selectedSurface, cs.selectedForce, tileX, tileY
 			));
 			if (!resp.recipe_tile) return;
-			const text = atob(resp.recipe_tile);
-			const lines = text.trim().split('\n').filter(l => l.trim());
-			for (const line of lines) {
-				try {
-					const rec = JSON.parse(line);
-					const key = `${rec.position[0]},${rec.position[1]}`;
-					// Timelapse filter
-					if (cs.isTimelapseMode && cs.currentTick !== null) {
-						const t = cs.currentTick;
-						if (rec.start_tick !== undefined && rec.start_tick > t) continue;
-						if (rec.end_tick !== undefined && rec.end_tick <= t) { recipeCache.current.delete(key); continue; }
-					} else {
-						if (rec.end_tick !== undefined) { recipeCache.current.delete(key); continue; }
-					}
-					if (rec.recipe) { recipeCache.current.set(key, rec.recipe); }
-				} catch { }
+			const binStr = atob(resp.recipe_tile);
+			const buf = new Uint8Array(binStr.length);
+			for (let i = 0; i < binStr.length; i++) buf[i] = binStr.charCodeAt(i);
+
+			const parsed = parseRecipeTileBinary(tileX, tileY, buf, cs.isTimelapseMode ? cs.currentTick : null);
+
+			// Apply parsed active recipes to cache
+			for (const [coord, recName] of parsed.activeRecipes) {
+				recipeCache.current.set(coord, recName);
 			}
 		} catch (e) { console.warn('loadRecipeTile', e); }
 	};