From ae7a1744adf1a9395e0a24832f00489eff8c4525 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Mon, 21 Jul 2025 18:11:08 +0200 Subject: [PATCH] Further compress tile pixel delta data 7 bytes header + 6 bytes per pixel -> 11 byte header + 2 bytes per pixel --- plugins/minimap/controller.ts | 63 ++++++++++++----------- plugins/minimap/docs/tile_data_storage.md | 36 +++++++------ plugins/minimap/tile-utils.ts | 46 +++++++---------- 3 files changed, 70 insertions(+), 75 deletions(-) diff --git a/plugins/minimap/controller.ts b/plugins/minimap/controller.ts index 661b5420..5c4c4757 100644 --- a/plugins/minimap/controller.ts +++ b/plugins/minimap/controller.ts @@ -142,39 +142,40 @@ export class ControllerPlugin extends BaseControllerPlugin { return Buffer.alloc(0); } - // Type (1) + Tick (4) + Length (2) + Pixel data (6 bytes per pixel) - const headerSize = 7; - const pixelDataSize = changes.length * 6; // 6 bytes per pixel change - const totalSize = headerSize + pixelDataSize; - const buffer = Buffer.alloc(totalSize); - - let offset = 0; - - // Type 2 for pixels - buffer.writeUInt8(2, offset); - offset += 1; - - // Tick (converted to seconds) - buffer.writeUInt32BE(Math.floor(tick / 60), offset); - offset += 4; - - // Length (number of pixel changes, not bytes) - buffer.writeUInt16BE(changes.length, offset); - offset += 2; - - // Write pixel changes - for (const change of changes) { - buffer.writeUInt8(change.x, offset); - offset += 1; - buffer.writeUInt8(change.y, offset); - offset += 1; - buffer.writeUInt16BE(change.newColor, offset); - offset += 2; - buffer.writeUInt16BE(change.oldColor, offset); - offset += 2; + // Group changes by identical new/old color pair for better compression + const groups = new Map(); + for (const ch of changes) { + const key = `${ch.newColor}_${ch.oldColor}`; + if (!groups.has(key)) groups.set(key, []); + groups.get(key)!.push(ch); } - return buffer; + // Build buffers for each group and concatenate + const groupBuffers: Buffer[] = []; + for (const [key, group] of groups) { + const sample = group[0]; + const count = group.length; + + // New format: Type(1) + Tick(4) + Count(2) + NewColor(2) + OldColor(2) + (x,y)*count + const headerSize = 1 + 4 + 2 + 2 + 2; + const pixelDataSize = count * 2; // 1 byte x + 1 byte y per pixel + const buf = Buffer.alloc(headerSize + pixelDataSize); + let offset = 0; + buf.writeUInt8(2, offset); offset += 1; + buf.writeUInt32BE(Math.floor(tick / 60), offset); offset += 4; + buf.writeUInt16BE(count, offset); offset += 2; + buf.writeUInt16BE(sample.newColor, offset); offset += 2; + buf.writeUInt16BE(sample.oldColor, offset); offset += 2; + + for (const c of group) { + buf.writeUInt8(c.x, offset); offset += 1; + buf.writeUInt8(c.y, offset); offset += 1; + } + + groupBuffers.push(buf); + } + + return Buffer.concat(groupBuffers); } private setupTileRoutes() { diff --git a/plugins/minimap/docs/tile_data_storage.md b/plugins/minimap/docs/tile_data_storage.md index b279b62d..2bb8b3cb 100644 --- a/plugins/minimap/docs/tile_data_storage.md +++ b/plugins/minimap/docs/tile_data_storage.md @@ -55,32 +55,35 @@ Initial exploration data for a 32×32 area. - **Compression**: Deflate (zlib) #### Pixel Changes Record (Type 2) -Incremental updates for individual pixels. +Incremental updates for pixels sharing the same color transition. -| Bytes | Field | Description | -| ----- | ---------------- | ----------------------- | -| 4 | `tick` (uint32) | ⌊tick / 60⌋ timestamp | -| 2 | `count` (uint16) | Number of pixel changes | +| Bytes | Field | Description | +| ----- | ------------------- | --------------------------- | +| 4 | `tick` (uint32) | ⌊tick / 60⌋ timestamp | +| 2 | `count` (uint16) | Number of pixels changed | +| 2 | `new_color` (uint16)| New RGB565 value | +| 2 | `old_color` (uint16)| Previous RGB565 value | -**Per pixel change (6 bytes each):** -| Bytes | Field | Description | -| ----- | -------------------- | ---------------------------- | -| 1 | `x` (uint8) | X coordinate in tile (0-255) | -| 1 | `y` (uint8) | Y coordinate in tile (0-255) | -| 2 | `new_color` (uint16) | New RGB565 value | -| 2 | `old_color` (uint16) | Previous RGB565 value | +**Per pixel coordinate (2 bytes each):** +| Bytes | Field | Description | +| ----- | ----------- | ---------------------------- | +| 1 | `x` (uint8) | X coordinate in tile (0-255) | +| 1 | `y` (uint8) | Y coordinate in tile (0-255) | + +**Note**: Multiple Type 2 records may be generated for a single tick if pixels have different color transitions. Each record groups pixels that share the same `old_color` -> `new_color` transition. ## Processing Strategy ### Live Updates 1. **Exploration**: New chunks sent as Type 1 records -2. **Changes**: Individual pixel updates as Type 2 records -3. **Optimization**: Multiple changes at same tick can be merged +2. **Changes**: Individual pixel updates grouped by color transition as Type 2 records +3. **Optimization**: Pixels with identical color changes are batched together ### Storage Efficiency - **Chunks**: ~100-500 bytes after compression (vs 2048 uncompressed) -- **Pixels**: 6 bytes per change (compact for sparse updates) +- **Pixels**: 2 bytes per coordinate + 11-byte header per color group (vs 6 bytes per pixel in old format) - **Timeline**: Old color values enable fast rewind without recalculation +- **Compression**: Grouping by color transitions provides significant space savings when many pixels change to the same colors ### Timelapse Playback 1. **Forward**: Apply chunks and pixel changes up to target tick @@ -95,7 +98,8 @@ Incremental updates for individual pixels. 2. Every 5 seconds, process queued chunks: - Load existing tile (if exists) - Compare new chunk pixels with current tile state - - Generate pixel change records for differences + - Group pixel differences by color transition + - Generate compressed pixel change records - Append new data to tile file (never modify existing) ``` diff --git a/plugins/minimap/tile-utils.ts b/plugins/minimap/tile-utils.ts index e84f4c0b..1438bc22 100644 --- a/plugins/minimap/tile-utils.ts +++ b/plugins/minimap/tile-utils.ts @@ -188,30 +188,28 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number) offset += 4; const pixelCount = readUInt16BE(tileData, offset); offset += 2; + const newColor = readUInt16BE(tileData, offset); + offset += 2; + const oldColor = readUInt16BE(tileData, offset); + offset += 2; - const expectedDataLength = pixelCount * 6; + const expectedDataLength = pixelCount * 2; // x,y per pixel if (offset + expectedDataLength > tileData.length) { break; } - // Skip this pixel changeset if it's beyond our target tick + // Skip if beyond target tick if (maxTick !== undefined && tick > maxTick) { offset += expectedDataLength; continue; } - // Apply pixel changes to current state for (let i = 0; i < pixelCount; i++) { - const pixelOffset = offset + i * 6; + const pixelOffset = offset + i * 2; const x = readUInt8(tileData, pixelOffset); const y = readUInt8(tileData, pixelOffset + 1); - const newColor = readUInt16BE(tileData, pixelOffset + 2); - // oldColor at pixelOffset + 4 is not needed for rendering - - // Update current pixel state if (x < 256 && y < 256) { - const tileIndex = y * 256 + x; - currentPixels[tileIndex] = newColor; + currentPixels[y * 256 + x] = newColor; } } @@ -295,9 +293,8 @@ export function extractAvailableTicks(tileData: BinaryData): number[] { offset += 4; const pixelCount = readUInt16BE(tileData, offset); - offset += 2; - - const expectedDataLength = pixelCount * 6; + offset += 2 + 2 + 2; // skip newColor oldColor too + const expectedDataLength = pixelCount * 2; if (offset + expectedDataLength > tileData.length) { break; } @@ -405,32 +402,25 @@ export function parseTileData(tileData: BinaryData): ParsedTileData { offset += 4; const pixelCount = readUInt16BE(tileData, offset); offset += 2; + const newColor = readUInt16BE(tileData, offset); + offset += 2; + const oldColor = readUInt16BE(tileData, offset); + offset += 2; - const expectedDataLength = pixelCount * 6; + const expectedDataLength = pixelCount * 2; if (offset + expectedDataLength > tileData.length) { break; } const changes: PixelChange[] = []; for (let i = 0; i < pixelCount; i++) { - const pixelOffset = offset + i * 6; + const pixelOffset = offset + i * 2; const x = readUInt8(tileData, pixelOffset); const y = readUInt8(tileData, pixelOffset + 1); - const newColor = readUInt16BE(tileData, pixelOffset + 2); - const oldColor = readUInt16BE(tileData, pixelOffset + 4); - - if (x < 256 && y < 256) { - changes.push({ x, y, newColor, oldColor }); - } - } - - if (changes.length > 0) { - pixelChanges.push({ - tick, - changes - }); + changes.push({ x, y, newColor, oldColor }); } + pixelChanges.push({ tick, changes }); offset += expectedDataLength; } else { break;