mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-17 15:33:12 +09:00
Further compress tile pixel delta data
7 bytes header + 6 bytes per pixel -> 11 byte header + 2 bytes per pixel
This commit is contained in:
1 parent
ab8321ff4b
commit
ae7a1744ad
3 files changed
+70
-75
No files matched your search
@@ -142,39 +142,40 @@ export class ControllerPlugin extends BaseControllerPlugin {
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
|
||||
// Type (1) + Tick (4) + Length (2) + Pixel data (6 bytes per pixel)
|
||||
const headerSize = 7;
|
||||
const pixelDataSize = changes.length * 6; // 6 bytes per pixel change
|
||||
const totalSize = headerSize + pixelDataSize;
|
||||
const buffer = Buffer.alloc(totalSize);
|
||||
|
||||
let offset = 0;
|
||||
|
||||
// Type 2 for pixels
|
||||
buffer.writeUInt8(2, offset);
|
||||
offset += 1;
|
||||
|
||||
// Tick (converted to seconds)
|
||||
buffer.writeUInt32BE(Math.floor(tick / 60), offset);
|
||||
offset += 4;
|
||||
|
||||
// Length (number of pixel changes, not bytes)
|
||||
buffer.writeUInt16BE(changes.length, offset);
|
||||
offset += 2;
|
||||
|
||||
// Write pixel changes
|
||||
for (const change of changes) {
|
||||
buffer.writeUInt8(change.x, offset);
|
||||
offset += 1;
|
||||
buffer.writeUInt8(change.y, offset);
|
||||
offset += 1;
|
||||
buffer.writeUInt16BE(change.newColor, offset);
|
||||
offset += 2;
|
||||
buffer.writeUInt16BE(change.oldColor, offset);
|
||||
offset += 2;
|
||||
// Group changes by identical new/old color pair for better compression
|
||||
const groups = new Map<string, PixelChange[]>();
|
||||
for (const ch of changes) {
|
||||
const key = `${ch.newColor}_${ch.oldColor}`;
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key)!.push(ch);
|
||||
}
|
||||
|
||||
return buffer;
|
||||
// Build buffers for each group and concatenate
|
||||
const groupBuffers: Buffer[] = [];
|
||||
for (const [key, group] of groups) {
|
||||
const sample = group[0];
|
||||
const count = group.length;
|
||||
|
||||
// New format: Type(1) + Tick(4) + Count(2) + NewColor(2) + OldColor(2) + (x,y)*count
|
||||
const headerSize = 1 + 4 + 2 + 2 + 2;
|
||||
const pixelDataSize = count * 2; // 1 byte x + 1 byte y per pixel
|
||||
const buf = Buffer.alloc(headerSize + pixelDataSize);
|
||||
let offset = 0;
|
||||
buf.writeUInt8(2, offset); offset += 1;
|
||||
buf.writeUInt32BE(Math.floor(tick / 60), offset); offset += 4;
|
||||
buf.writeUInt16BE(count, offset); offset += 2;
|
||||
buf.writeUInt16BE(sample.newColor, offset); offset += 2;
|
||||
buf.writeUInt16BE(sample.oldColor, offset); offset += 2;
|
||||
|
||||
for (const c of group) {
|
||||
buf.writeUInt8(c.x, offset); offset += 1;
|
||||
buf.writeUInt8(c.y, offset); offset += 1;
|
||||
}
|
||||
|
||||
groupBuffers.push(buf);
|
||||
}
|
||||
|
||||
return Buffer.concat(groupBuffers);
|
||||
}
|
||||
|
||||
private setupTileRoutes() {
|
||||
|
||||
@@ -55,32 +55,35 @@ Initial exploration data for a 32×32 area.
|
||||
- **Compression**: Deflate (zlib)
|
||||
|
||||
#### Pixel Changes Record (Type 2)
|
||||
Incremental updates for individual pixels.
|
||||
Incremental updates for pixels sharing the same color transition.
|
||||
|
||||
| Bytes | Field | Description |
|
||||
| ----- | ---------------- | ----------------------- |
|
||||
| 4 | `tick` (uint32) | ⌊tick / 60⌋ timestamp |
|
||||
| 2 | `count` (uint16) | Number of pixel changes |
|
||||
| Bytes | Field | Description |
|
||||
| ----- | ------------------- | --------------------------- |
|
||||
| 4 | `tick` (uint32) | ⌊tick / 60⌋ timestamp |
|
||||
| 2 | `count` (uint16) | Number of pixels changed |
|
||||
| 2 | `new_color` (uint16)| New RGB565 value |
|
||||
| 2 | `old_color` (uint16)| Previous RGB565 value |
|
||||
|
||||
**Per pixel change (6 bytes each):**
|
||||
| Bytes | Field | Description |
|
||||
| ----- | -------------------- | ---------------------------- |
|
||||
| 1 | `x` (uint8) | X coordinate in tile (0-255) |
|
||||
| 1 | `y` (uint8) | Y coordinate in tile (0-255) |
|
||||
| 2 | `new_color` (uint16) | New RGB565 value |
|
||||
| 2 | `old_color` (uint16) | Previous RGB565 value |
|
||||
**Per pixel coordinate (2 bytes each):**
|
||||
| Bytes | Field | Description |
|
||||
| ----- | ----------- | ---------------------------- |
|
||||
| 1 | `x` (uint8) | X coordinate in tile (0-255) |
|
||||
| 1 | `y` (uint8) | Y coordinate in tile (0-255) |
|
||||
|
||||
**Note**: Multiple Type 2 records may be generated for a single tick if pixels have different color transitions. Each record groups pixels that share the same `old_color` -> `new_color` transition.
|
||||
|
||||
## Processing Strategy
|
||||
|
||||
### Live Updates
|
||||
1. **Exploration**: New chunks sent as Type 1 records
|
||||
2. **Changes**: Individual pixel updates as Type 2 records
|
||||
3. **Optimization**: Multiple changes at same tick can be merged
|
||||
2. **Changes**: Individual pixel updates grouped by color transition as Type 2 records
|
||||
3. **Optimization**: Pixels with identical color changes are batched together
|
||||
|
||||
### Storage Efficiency
|
||||
- **Chunks**: ~100-500 bytes after compression (vs 2048 uncompressed)
|
||||
- **Pixels**: 6 bytes per change (compact for sparse updates)
|
||||
- **Pixels**: 2 bytes per coordinate + 11-byte header per color group (vs 6 bytes per pixel in old format)
|
||||
- **Timeline**: Old color values enable fast rewind without recalculation
|
||||
- **Compression**: Grouping by color transitions provides significant space savings when many pixels change to the same colors
|
||||
|
||||
### Timelapse Playback
|
||||
1. **Forward**: Apply chunks and pixel changes up to target tick
|
||||
@@ -95,7 +98,8 @@ Incremental updates for individual pixels.
|
||||
2. Every 5 seconds, process queued chunks:
|
||||
- Load existing tile (if exists)
|
||||
- Compare new chunk pixels with current tile state
|
||||
- Generate pixel change records for differences
|
||||
- Group pixel differences by color transition
|
||||
- Generate compressed pixel change records
|
||||
- Append new data to tile file (never modify existing)
|
||||
```
|
||||
|
||||
|
||||
@@ -188,30 +188,28 @@ export async function renderTileToPixels(tileData: BinaryData, maxTick?: number)
|
||||
offset += 4;
|
||||
const pixelCount = readUInt16BE(tileData, offset);
|
||||
offset += 2;
|
||||
const newColor = readUInt16BE(tileData, offset);
|
||||
offset += 2;
|
||||
const oldColor = readUInt16BE(tileData, offset);
|
||||
offset += 2;
|
||||
|
||||
const expectedDataLength = pixelCount * 6;
|
||||
const expectedDataLength = pixelCount * 2; // x,y per pixel
|
||||
if (offset + expectedDataLength > tileData.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip this pixel changeset if it's beyond our target tick
|
||||
// Skip if beyond target tick
|
||||
if (maxTick !== undefined && tick > maxTick) {
|
||||
offset += expectedDataLength;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Apply pixel changes to current state
|
||||
for (let i = 0; i < pixelCount; i++) {
|
||||
const pixelOffset = offset + i * 6;
|
||||
const pixelOffset = offset + i * 2;
|
||||
const x = readUInt8(tileData, pixelOffset);
|
||||
const y = readUInt8(tileData, pixelOffset + 1);
|
||||
const newColor = readUInt16BE(tileData, pixelOffset + 2);
|
||||
// oldColor at pixelOffset + 4 is not needed for rendering
|
||||
|
||||
// Update current pixel state
|
||||
if (x < 256 && y < 256) {
|
||||
const tileIndex = y * 256 + x;
|
||||
currentPixels[tileIndex] = newColor;
|
||||
currentPixels[y * 256 + x] = newColor;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,9 +293,8 @@ export function extractAvailableTicks(tileData: BinaryData): number[] {
|
||||
offset += 4;
|
||||
|
||||
const pixelCount = readUInt16BE(tileData, offset);
|
||||
offset += 2;
|
||||
|
||||
const expectedDataLength = pixelCount * 6;
|
||||
offset += 2 + 2 + 2; // skip newColor oldColor too
|
||||
const expectedDataLength = pixelCount * 2;
|
||||
if (offset + expectedDataLength > tileData.length) {
|
||||
break;
|
||||
}
|
||||
@@ -405,32 +402,25 @@ export function parseTileData(tileData: BinaryData): ParsedTileData {
|
||||
offset += 4;
|
||||
const pixelCount = readUInt16BE(tileData, offset);
|
||||
offset += 2;
|
||||
const newColor = readUInt16BE(tileData, offset);
|
||||
offset += 2;
|
||||
const oldColor = readUInt16BE(tileData, offset);
|
||||
offset += 2;
|
||||
|
||||
const expectedDataLength = pixelCount * 6;
|
||||
const expectedDataLength = pixelCount * 2;
|
||||
if (offset + expectedDataLength > tileData.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
const changes: PixelChange[] = [];
|
||||
for (let i = 0; i < pixelCount; i++) {
|
||||
const pixelOffset = offset + i * 6;
|
||||
const pixelOffset = offset + i * 2;
|
||||
const x = readUInt8(tileData, pixelOffset);
|
||||
const y = readUInt8(tileData, pixelOffset + 1);
|
||||
const newColor = readUInt16BE(tileData, pixelOffset + 2);
|
||||
const oldColor = readUInt16BE(tileData, pixelOffset + 4);
|
||||
|
||||
if (x < 256 && y < 256) {
|
||||
changes.push({ x, y, newColor, oldColor });
|
||||
}
|
||||
}
|
||||
|
||||
if (changes.length > 0) {
|
||||
pixelChanges.push({
|
||||
tick,
|
||||
changes
|
||||
});
|
||||
changes.push({ x, y, newColor, oldColor });
|
||||
}
|
||||
|
||||
pixelChanges.push({ tick, changes });
|
||||
offset += expectedDataLength;
|
||||
} else {
|
||||
break;
|
||||
|
||||
Reference in new issue
Block a user