Further compress tile pixel delta data

7 bytes header + 6 bytes per pixel -> 11 byte header + 2 bytes per pixel
This commit is contained in:
Danielv123 committed 2025-08-10 00:28:48 +02:00
1 parent ab8321ff4b
commit ae7a1744ad
3 files changed
+70 -75

No files matched your search

+32 -31
View File
@@ -142,39 +142,40 @@ export class ControllerPlugin extends BaseControllerPlugin {
return Buffer.alloc(0);
}
// Type (1) + Tick (4) + Length (2) + Pixel data (6 bytes per pixel)
const headerSize = 7;
const pixelDataSize = changes.length * 6; // 6 bytes per pixel change
const totalSize = headerSize + pixelDataSize;
const buffer = Buffer.alloc(totalSize);
let offset = 0;
// Type 2 for pixels
buffer.writeUInt8(2, offset);
offset += 1;
// Tick (converted to seconds)
buffer.writeUInt32BE(Math.floor(tick / 60), offset);
offset += 4;
// Length (number of pixel changes, not bytes)
buffer.writeUInt16BE(changes.length, offset);
offset += 2;
// Write pixel changes
for (const change of changes) {
buffer.writeUInt8(change.x, offset);
offset += 1;
buffer.writeUInt8(change.y, offset);
offset += 1;
buffer.writeUInt16BE(change.newColor, offset);
offset += 2;
buffer.writeUInt16BE(change.oldColor, offset);
offset += 2;
// Group changes by identical new/old color pair for better compression
const groups = new Map<string, PixelChange[]>();
for (const ch of changes) {
const key = `${ch.newColor}_${ch.oldColor}`;
if (!groups.has(key)) groups.set(key, []);
groups.get(key)!.push(ch);
}
return buffer;
// Build buffers for each group and concatenate
const groupBuffers: Buffer[] = [];
for (const [key, group] of groups) {
const sample = group[0];
const count = group.length;
// New format: Type(1) + Tick(4) + Count(2) + NewColor(2) + OldColor(2) + (x,y)*count
const headerSize = 1 + 4 + 2 + 2 + 2;
const pixelDataSize = count * 2; // 1 byte x + 1 byte y per pixel
const buf = Buffer.alloc(headerSize + pixelDataSize);
let offset = 0;
buf.writeUInt8(2, offset); offset += 1;
buf.writeUInt32BE(Math.floor(tick / 60), offset); offset += 4;
buf.writeUInt16BE(count, offset); offset += 2;
buf.writeUInt16BE(sample.newColor, offset); offset += 2;
buf.writeUInt16BE(sample.oldColor, offset); offset += 2;
for (const c of group) {
buf.writeUInt8(c.x, offset); offset += 1;
buf.writeUInt8(c.y, offset); offset += 1;
}
groupBuffers.push(buf);
}
return Buffer.concat(groupBuffers);
}
private setupTileRoutes() {
+20 -16
View File
@@ -55,32 +55,35 @@ Initial exploration data for a 32×32 area.
- **Compression**: Deflate (zlib)
#### Pixel Changes Record (Type 2)
Incremental updates for individual pixels.
Incremental updates for pixels sharing the same color transition.
| Bytes | Field | Description |
| ----- | ---------------- | ----------------------- |
| 4 | `tick` (uint32) | ⌊tick / 60⌋ timestamp |
| 2 | `count` (uint16) | Number of pixel changes |
| Bytes | Field | Description |
| ----- | ------------------- | --------------------------- |
| 4 | `tick` (uint32) | ⌊tick / 60⌋ timestamp |
| 2 | `count` (uint16) | Number of pixels changed |
| 2 | `new_color` (uint16)| New RGB565 value |
| 2 | `old_color` (uint16)| Previous RGB565 value |
**Per pixel change (6 bytes each):**
| Bytes | Field | Description |
| ----- | -------------------- | ---------------------------- |
| 1 | `x` (uint8) | X coordinate in tile (0-255) |
| 1 | `y` (uint8) | Y coordinate in tile (0-255) |
| 2 | `new_color` (uint16) | New RGB565 value |
| 2 | `old_color` (uint16) | Previous RGB565 value |
**Per pixel coordinate (2 bytes each):**
| Bytes | Field | Description |
| ----- | ----------- | ---------------------------- |
| 1 | `x` (uint8) | X coordinate in tile (0-255) |
| 1 | `y` (uint8) | Y coordinate in tile (0-255) |
**Note**: Multiple Type 2 records may be generated for a single tick if pixels have different color transitions. Each record groups pixels that share the same `old_color` -> `new_color` transition.
## Processing Strategy
### Live Updates
1. **Exploration**: New chunks sent as Type 1 records
2. **Changes**: Individual pixel updates as Type 2 records
3. **Optimization**: Multiple changes at same tick can be merged
2. **Changes**: Individual pixel updates grouped by color transition as Type 2 records
3. **Optimization**: Pixels with identical color changes are batched together
### Storage Efficiency
- **Chunks**: ~100-500 bytes after compression (vs 2048 uncompressed)
- **Pixels**: 6 bytes per change (compact for sparse updates)
- **Pixels**: 2 bytes per coordinate + 11-byte header per color group (vs 6 bytes per pixel in old format)
- **Timeline**: Old color values enable fast rewind without recalculation
- **Compression**: Grouping by color transitions provides significant space savings when many pixels change to the same colors
### Timelapse Playback
1. **Forward**: Apply chunks and pixel changes up to target tick
@@ -95,7 +98,8 @@ Incremental updates for individual pixels.
2. Every 5 seconds, process queued chunks:
- Load existing tile (if exists)
- Compare new chunk pixels with current tile state
- Generate pixel change records for differences
- Group pixel differences by color transition
- Generate compressed pixel change records
- Append new data to tile file (never modify existing)
```
+18 -28
View File
@@ -188,30 +188,28 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
offset += 4;
const pixelCount = readUInt16BE(tileData, offset);
offset += 2;
const newColor = readUInt16BE(tileData, offset);
offset += 2;
const oldColor = readUInt16BE(tileData, offset);
offset += 2;
const expectedDataLength = pixelCount * 6;
const expectedDataLength = pixelCount * 2; // x,y per pixel
if (offset + expectedDataLength > tileData.length) {
break;
}
// Skip this pixel changeset if it's beyond our target tick
// Skip if beyond target tick
if (maxTick !== undefined && tick > maxTick) {
offset += expectedDataLength;
continue;
}
// Apply pixel changes to current state
for (let i = 0; i < pixelCount; i++) {
const pixelOffset = offset + i * 6;
const pixelOffset = offset + i * 2;
const x = readUInt8(tileData, pixelOffset);
const y = readUInt8(tileData, pixelOffset + 1);
const newColor = readUInt16BE(tileData, pixelOffset + 2);
// oldColor at pixelOffset + 4 is not needed for rendering
// Update current pixel state
if (x < 256 && y < 256) {
const tileIndex = y * 256 + x;
currentPixels[tileIndex] = newColor;
currentPixels[y * 256 + x] = newColor;
}
}
@@ -295,9 +293,8 @@ export function extractAvailableTicks(tileData: BinaryData): number[] {
offset += 4;
const pixelCount = readUInt16BE(tileData, offset);
offset += 2;
const expectedDataLength = pixelCount * 6;
offset += 2 + 2 + 2; // skip newColor oldColor too
const expectedDataLength = pixelCount * 2;
if (offset + expectedDataLength > tileData.length) {
break;
}
@@ -405,32 +402,25 @@ export function parseTileData(tileData: BinaryData): ParsedTileData {
offset += 4;
const pixelCount = readUInt16BE(tileData, offset);
offset += 2;
const newColor = readUInt16BE(tileData, offset);
offset += 2;
const oldColor = readUInt16BE(tileData, offset);
offset += 2;
const expectedDataLength = pixelCount * 6;
const expectedDataLength = pixelCount * 2;
if (offset + expectedDataLength > tileData.length) {
break;
}
const changes: PixelChange[] = [];
for (let i = 0; i < pixelCount; i++) {
const pixelOffset = offset + i * 6;
const pixelOffset = offset + i * 2;
const x = readUInt8(tileData, pixelOffset);
const y = readUInt8(tileData, pixelOffset + 1);
const newColor = readUInt16BE(tileData, pixelOffset + 2);
const oldColor = readUInt16BE(tileData, pixelOffset + 4);
if (x < 256 && y < 256) {
changes.push({ x, y, newColor, oldColor });
}
}
if (changes.length > 0) {
pixelChanges.push({
tick,
changes
});
changes.push({ x, y, newColor, oldColor });
}
pixelChanges.push({ tick, changes });
offset += expectedDataLength;
} else {
break;