mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-21 08:03:13 +09:00
Implement new tile storage format with tentative timelapse support
This commit is contained in:
1 parent
2dcc92c96b
commit
a516b6b129
7 files changed
+348
-297
No files matched your search
@@ -39,17 +39,17 @@ The tile format is as follows:
|
|||||||
- 2 = pixels
|
- 2 = pixels
|
||||||
|
|
||||||
For chunk:
|
For chunk:
|
||||||
- 2 byte: length, number of data bytes
|
|
||||||
- 4 byte: math.floor(tick / 60)
|
- 4 byte: math.floor(tick / 60)
|
||||||
- 4 bits: chunk relative X coordinate
|
- 4 bits: chunk relative X coordinate
|
||||||
- 4 bits: chunk relative Y coordinate
|
- 4 bits: chunk relative Y coordinate
|
||||||
|
- 2 byte: length, number of data bytes
|
||||||
- n bytes of 32x32 rgb565, with deflate compression
|
- n bytes of 32x32 rgb565, with deflate compression
|
||||||
|
|
||||||
Type 1 chunks are assumed to only occur upon first exploration - further changes usually happen as individual pixels. For that reason, there is no rewind information tied to chunks, and they revert to black when rewinded.
|
Type 1 chunks are assumed to only occur upon first exploration - further changes usually happen as individual pixels. For that reason, there is no rewind information tied to chunks, and they revert to black when rewinded.
|
||||||
|
|
||||||
For pixels:
|
For pixels:
|
||||||
- 2 byte: length, number of pixels
|
|
||||||
- 4 byte: math.floor(tick / 60)
|
- 4 byte: math.floor(tick / 60)
|
||||||
|
- 2 byte: length, number of pixels
|
||||||
- Each pixel:
|
- Each pixel:
|
||||||
- 1 byte X coordinate
|
- 1 byte X coordinate
|
||||||
- 1 byte Y coordinate
|
- 1 byte Y coordinate
|
||||||
|
|||||||
+233
-210
@@ -4,7 +4,6 @@ import {
|
|||||||
TileDataEvent,
|
TileDataEvent,
|
||||||
GetInstanceBoundsRequest,
|
GetInstanceBoundsRequest,
|
||||||
ChartData,
|
ChartData,
|
||||||
ChunkUpdateEvent,
|
|
||||||
} from "./messages";
|
} from "./messages";
|
||||||
import * as fs from "fs-extra";
|
import * as fs from "fs-extra";
|
||||||
import * as path from "path";
|
import * as path from "path";
|
||||||
@@ -18,22 +17,27 @@ const inflateAsync = promisify(zlib.inflate);
|
|||||||
|
|
||||||
export class ControllerPlugin extends BaseControllerPlugin {
|
export class ControllerPlugin extends BaseControllerPlugin {
|
||||||
private tilesPath: string = "";
|
private tilesPath: string = "";
|
||||||
|
private chunkSavingQueue = new Map<string, Map<string, { data: ChartData, tick: number }>>();
|
||||||
|
private savingTiles: boolean = false;
|
||||||
// Set to true if colors appear inverted - this will try big-endian byte order
|
// Set to true if colors appear inverted - this will try big-endian byte order
|
||||||
private useBigEndian: boolean = false;
|
private useBigEndian: boolean = false;
|
||||||
|
|
||||||
async init() {
|
async init() {
|
||||||
this.tilesPath = path.resolve(
|
this.tilesPath = path.resolve(
|
||||||
this.controller.config.get("controller.database_directory"),
|
this.controller.config.get("controller.database_directory"),
|
||||||
"minimap_chunks"
|
"minimap_tiles"
|
||||||
);
|
);
|
||||||
await fs.ensureDir(this.tilesPath);
|
await fs.ensureDir(this.tilesPath);
|
||||||
|
|
||||||
this.controller.handle(TileDataEvent, this.handleTileDataEvent.bind(this));
|
this.controller.handle(TileDataEvent, this.handleTileDataEvent.bind(this));
|
||||||
|
|
||||||
this.controller.handle(GetInstanceBoundsRequest, this.handleGetInstanceBoundsRequest.bind(this));
|
this.controller.handle(GetInstanceBoundsRequest, this.handleGetInstanceBoundsRequest.bind(this));
|
||||||
|
|
||||||
// Set up HTTP routes for serving tiles
|
// Set up HTTP routes for serving tiles
|
||||||
this.setupTileRoutes();
|
this.setupTileRoutes();
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
this.saveTiles().catch(err => this.logger.error(`Error saving tiles: ${err}`));
|
||||||
|
}, 5000);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert RGB565 to RGB888 values using Factorio's exact method
|
// Convert RGB565 to RGB888 values using Factorio's exact method
|
||||||
@@ -48,268 +52,287 @@ export class ControllerPlugin extends BaseControllerPlugin {
|
|||||||
private setupTileRoutes() {
|
private setupTileRoutes() {
|
||||||
const app = this.controller.app;
|
const app = this.controller.app;
|
||||||
|
|
||||||
// Create black placeholder image for 512x512 tiles
|
app.get("/api/minimap/tile/:instanceId/:surface/:force/:z/:x/:y.png", async (req, res) => {
|
||||||
const createBlackTile = async () => {
|
|
||||||
return await sharp({
|
|
||||||
create: {
|
|
||||||
width: 512,
|
|
||||||
height: 512,
|
|
||||||
channels: 4,
|
|
||||||
background: { r: 0, g: 0, b: 0, alpha: 0 },
|
|
||||||
},
|
|
||||||
}).png().toBuffer();
|
|
||||||
};
|
|
||||||
|
|
||||||
// Serve chart tiles with surface and force support - now serving 512x512 tiles
|
|
||||||
app.get("/api/minimap/chart/:instanceId/:surface/:force/:z/:x/:y.png", async (req, res) => {
|
|
||||||
try {
|
try {
|
||||||
const { instanceId, surface, force, z, x, y } = req.params;
|
const { instanceId, surface, force, z, x, y } = req.params;
|
||||||
const tileX = parseInt(x);
|
const tileX = parseInt(x);
|
||||||
const tileY = parseInt(y);
|
const tileY = parseInt(y);
|
||||||
|
|
||||||
// Each 512x512 tile contains 16x16 chunks of 32x32 pixels
|
const tileName = `${instanceId}_${surface}_${force}_${tileX}_${tileY}.bin`;
|
||||||
const chunksPerTile = 16;
|
const tilePath = path.join(this.tilesPath, tileName);
|
||||||
const tileSize = 512;
|
|
||||||
|
|
||||||
// Calculate the range of chunk coordinates that make up this 512x512 tile
|
if (!await fs.pathExists(tilePath)) {
|
||||||
const startChunkX = tileX * chunksPerTile;
|
res.status(404).send("Tile not found");
|
||||||
const startChunkY = tileY * chunksPerTile;
|
return;
|
||||||
const endChunkX = startChunkX + chunksPerTile;
|
|
||||||
const endChunkY = startChunkY + chunksPerTile;
|
|
||||||
|
|
||||||
// Create a 512x512 canvas
|
|
||||||
const canvas = sharp({
|
|
||||||
create: {
|
|
||||||
width: tileSize,
|
|
||||||
height: tileSize,
|
|
||||||
channels: 4,
|
|
||||||
background: { r: 0, g: 0, b: 0, alpha: 0 },
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
// Load all the 32x32 chunks and composite them
|
|
||||||
const compositeImages = [];
|
|
||||||
let chunksFound = 0;
|
|
||||||
|
|
||||||
for (let chunkY = startChunkY; chunkY < endChunkY; chunkY++) {
|
|
||||||
for (let chunkX = startChunkX; chunkX < endChunkX; chunkX++) {
|
|
||||||
const filename = `${instanceId}/${surface}/${force}/${chunkX}_${chunkY}.png`;
|
|
||||||
const filePath = path.resolve(this.tilesPath, filename);
|
|
||||||
|
|
||||||
if (await fs.pathExists(filePath)) {
|
|
||||||
const relativeX = (chunkX - startChunkX) * CHUNK_SIZE;
|
|
||||||
const relativeY = (chunkY - startChunkY) * CHUNK_SIZE;
|
|
||||||
|
|
||||||
compositeImages.push({
|
|
||||||
input: filePath,
|
|
||||||
left: relativeX,
|
|
||||||
top: relativeY,
|
|
||||||
});
|
|
||||||
chunksFound++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if(chunksFound > 0) {
|
const tileData = await fs.readFile(tilePath);
|
||||||
this.logger.info(`Tile ${tileX},${tileY}: Found ${chunksFound} chunks, chunk range: ${startChunkX}-${endChunkX-1}, ${startChunkY}-${endChunkY-1}`);
|
const image = await this.renderTile(tileData);
|
||||||
}
|
|
||||||
|
|
||||||
// If we have any chunks to composite, create the merged image
|
|
||||||
if (compositeImages.length > 0) {
|
|
||||||
const mergedImage = await canvas.composite(compositeImages).png().toBuffer();
|
|
||||||
res.setHeader("Content-Type", "image/png");
|
|
||||||
res.send(mergedImage);
|
|
||||||
} else {
|
|
||||||
// No chunks found, return black tile
|
|
||||||
res.setHeader("Content-Type", "image/png");
|
|
||||||
res.send(await createBlackTile());
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
this.logger.error(`Error serving chart tile: ${err}`);
|
|
||||||
|
|
||||||
// Return black tile on error
|
|
||||||
res.setHeader("Content-Type", "image/png");
|
res.setHeader("Content-Type", "image/png");
|
||||||
res.send(await createBlackTile());
|
res.send(image);
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.error(`Error serving tile: ${err}`);
|
||||||
|
res.status(500).send("Error serving tile");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// List available surfaces and forces
|
|
||||||
app.get("/api/minimap/surfaces", async (req, res) => {
|
app.get("/api/minimap/surfaces", async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const instances = new Set<string>();
|
const files = await fs.readdir(this.tilesPath);
|
||||||
const surfaces = new Set<string>();
|
const surfaces = new Set<string>();
|
||||||
const forces = new Set<string>();
|
const forces = new Set<string>();
|
||||||
|
|
||||||
// Traverse the hierarchical directory structure: instanceId/surface/force/
|
for (const file of files) {
|
||||||
const instanceDirs = await fs.readdir(this.tilesPath, { withFileTypes: true });
|
if (file.endsWith(".bin")) {
|
||||||
|
const parts = file.split("_");
|
||||||
for (const instanceDir of instanceDirs) {
|
if (parts.length === 5) {
|
||||||
if (instanceDir.isDirectory()) {
|
surfaces.add(parts[1]);
|
||||||
instances.add(instanceDir.name);
|
forces.add(parts[2]);
|
||||||
|
|
||||||
const instancePath = path.join(this.tilesPath, instanceDir.name);
|
|
||||||
const surfaceDirs = await fs.readdir(instancePath, { withFileTypes: true });
|
|
||||||
|
|
||||||
for (const surfaceDir of surfaceDirs) {
|
|
||||||
if (surfaceDir.isDirectory()) {
|
|
||||||
surfaces.add(surfaceDir.name);
|
|
||||||
|
|
||||||
const surfacePath = path.join(instancePath, surfaceDir.name);
|
|
||||||
const forceDirs = await fs.readdir(surfacePath, { withFileTypes: true });
|
|
||||||
|
|
||||||
for (const forceDir of forceDirs) {
|
|
||||||
if (forceDir.isDirectory()) {
|
|
||||||
forces.add(forceDir.name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res.json({
|
res.json({
|
||||||
instances: Array.from(instances),
|
|
||||||
surfaces: Array.from(surfaces),
|
surfaces: Array.from(surfaces),
|
||||||
forces: Array.from(forces),
|
forces: Array.from(forces),
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.logger.error(`Error listing surfaces: ${err}`);
|
this.logger.error(`Error listing surfaces and forces: ${err}`);
|
||||||
res.status(500).json({ error: "Failed to list surfaces" });
|
res.status(500).json({ error: "Failed to list surfaces and forces" });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async handleTileDataEvent(event: TileDataEvent) {
|
async renderTile(tileData: Buffer): Promise<Buffer> {
|
||||||
try {
|
const tileSize = 256;
|
||||||
const { type, data, position, instanceId } = event;
|
const canvas = sharp({
|
||||||
|
create: {
|
||||||
|
width: tileSize,
|
||||||
|
height: tileSize,
|
||||||
|
channels: 4,
|
||||||
|
background: { r: 0, g: 0, b: 0, alpha: 0 },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
if (type === "chart") {
|
const compositeImages: any[] = [];
|
||||||
await this.processChartData(data as ChartData[], position, instanceId);
|
let offset = 0;
|
||||||
|
while (offset < tileData.length) {
|
||||||
|
if (offset + 1 > tileData.length) {
|
||||||
|
this.logger.error(`Invalid tile data: cannot read type at offset ${offset}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const type = tileData.readUInt8(offset);
|
||||||
|
offset += 1; // After type
|
||||||
|
|
||||||
|
if (type === 1) { // Chunk
|
||||||
|
// Check if we have enough bytes for the header
|
||||||
|
if (offset + 7 > tileData.length) {
|
||||||
|
this.logger.error(`Invalid tile data: insufficient header data at offset ${offset}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const tick = tileData.readUInt32BE(offset); // Read tick
|
||||||
|
offset += 4; // After tick
|
||||||
|
const chunkCoordsByte = tileData.readUInt8(offset); // Read coords
|
||||||
|
offset += 1; // After coords
|
||||||
|
const length = tileData.readUInt16BE(offset); // Read length
|
||||||
|
offset += 2; // After length
|
||||||
|
|
||||||
|
// Check if we have enough bytes for the data
|
||||||
|
if (offset + length > tileData.length) {
|
||||||
|
this.logger.error(`Invalid tile data: insufficient chunk data at offset ${offset}, need ${length} bytes but only ${tileData.length - offset} available`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const chunkX = chunkCoordsByte >> 4;
|
||||||
|
const chunkY = chunkCoordsByte & 0x0F;
|
||||||
|
const chunkData = tileData.slice(offset, offset + length);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const decompressed = await inflateAsync(chunkData);
|
||||||
|
|
||||||
|
const raw = Buffer.alloc(32 * 32 * 4);
|
||||||
|
// Initialize buffer to black (all zeros with alpha=255)
|
||||||
|
for (let i = 0; i < 32 * 32; i++) {
|
||||||
|
raw[i * 4 + 3] = 255; // Set alpha to 255 for all pixels
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure we don't read beyond available data
|
||||||
|
const maxPixels = Math.min(decompressed.length / 2, 32 * 32);
|
||||||
|
|
||||||
|
for (let i = 0; i < maxPixels * 2; i += 2) {
|
||||||
|
const rgb565Value = decompressed.readUInt16LE(i);
|
||||||
|
const [r, g, b] = this.rgb565ToRgb888(rgb565Value);
|
||||||
|
const pixelIndex = i / 2;
|
||||||
|
const bufferIndex = pixelIndex * 4;
|
||||||
|
raw[bufferIndex] = r;
|
||||||
|
raw[bufferIndex + 1] = g;
|
||||||
|
raw[bufferIndex + 2] = b;
|
||||||
|
raw[bufferIndex + 3] = 255;
|
||||||
|
}
|
||||||
|
|
||||||
|
compositeImages.push({
|
||||||
|
input: await sharp(raw, { raw: { width: 32, height: 32, channels: 4 } }).png().toBuffer(),
|
||||||
|
left: chunkX * 32,
|
||||||
|
top: chunkY * 32,
|
||||||
|
});
|
||||||
|
} catch (decompressError) {
|
||||||
|
this.logger.error(`Failed to decompress chunk data: ${decompressError}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += length; // Advance offset by the data block length
|
||||||
|
} else if (type === 2) { // Pixels
|
||||||
|
// Check if we have enough bytes for the header
|
||||||
|
if (offset + 6 > tileData.length) {
|
||||||
|
this.logger.error(`Invalid tile data: insufficient pixel header data at offset ${offset}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read tick (4 bytes) and length (2 bytes) first
|
||||||
|
offset += 4; // Skip tick
|
||||||
|
const length = tileData.readUInt16BE(offset); // Read length
|
||||||
|
offset += 2; // After length
|
||||||
|
|
||||||
|
// Check if we have enough bytes for the data
|
||||||
|
if (offset + length > tileData.length) {
|
||||||
|
this.logger.error(`Invalid tile data: insufficient pixel data at offset ${offset}, need ${length} bytes but only ${tileData.length - offset} available`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += length; // Advance offset by the data block length
|
||||||
|
} else {
|
||||||
|
this.logger.error(`Unknown tile data type: ${type} at offset ${offset}`);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
} catch (err) {
|
|
||||||
this.logger.error(`Error processing tile data: ${err}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return canvas.composite(compositeImages).png().toBuffer();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processChartData(chartData: ChartData[], position: [number, number], instanceId: number) {
|
async saveTiles() {
|
||||||
if (!position || !instanceId) {
|
if (this.savingTiles) {
|
||||||
this.logger.error("Chart data requires position and instanceId");
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const [worldX, worldY] = position;
|
const queue = this.chunkSavingQueue;
|
||||||
|
if (queue.size === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.logger.info(`Starting save for ${queue.size} tiles.`);
|
||||||
|
|
||||||
// Convert world coordinates to chunk coordinates
|
this.savingTiles = true;
|
||||||
const chunkX = Math.floor(worldX / 32);
|
this.chunkSavingQueue = new Map();
|
||||||
const chunkY = Math.floor(worldY / 32);
|
|
||||||
|
|
||||||
// Process each surface/force combination
|
try {
|
||||||
for (const chart of chartData) {
|
const savePromises = [];
|
||||||
const { surface, force, chart_data } = chart;
|
for (const [tileName, chunks] of queue) {
|
||||||
|
const promise = (async () => {
|
||||||
// Create filename using chunk coordinates
|
const tilePath = path.join(this.tilesPath, tileName);
|
||||||
const filename = `${instanceId}/${surface}/${force}/${chunkX}_${chunkY}.png`;
|
let existingTile = Buffer.alloc(0);
|
||||||
|
if (await fs.pathExists(tilePath)) {
|
||||||
// Process the 32x32 chunk directly and get the processed image data
|
existingTile = await fs.readFile(tilePath);
|
||||||
const imageDataBase64 = await this.saveChunkImage(filename, chart_data);
|
}
|
||||||
|
const newTile = await this.updateTile(existingTile, chunks);
|
||||||
// Send live update to connected web clients
|
await fs.writeFile(tilePath, newTile);
|
||||||
if (imageDataBase64) {
|
})();
|
||||||
const updateEvent = new ChunkUpdateEvent(
|
savePromises.push(promise);
|
||||||
instanceId,
|
|
||||||
surface,
|
|
||||||
force,
|
|
||||||
chunkX,
|
|
||||||
chunkY,
|
|
||||||
imageDataBase64
|
|
||||||
);
|
|
||||||
|
|
||||||
// Broadcast to all connected control clients
|
|
||||||
this.controller.sendEvent(updateEvent, new lib.Address(lib.Address.broadcast, lib.Address.control));
|
|
||||||
}
|
}
|
||||||
|
await Promise.all(savePromises);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.error(`Error saving tiles: ${err}`);
|
||||||
|
} finally {
|
||||||
|
this.savingTiles = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async saveChunkImage(filename: string, chartData: string): Promise<string | null> {
|
async updateTile(existingTile: Buffer, newChunks: Map<string, { data: ChartData, tick: number }>): Promise<Buffer> {
|
||||||
const imagePath = path.resolve(this.tilesPath, filename);
|
const existingChunks = new Map<string, { tick: number, data: Buffer }>();
|
||||||
|
let offset = 0;
|
||||||
|
while (offset < existingTile.length) {
|
||||||
|
const type = existingTile.readUInt8(offset);
|
||||||
|
offset += 1;
|
||||||
|
|
||||||
// Ensure the directory structure exists
|
if (type === 1) { // Chunk
|
||||||
await fs.ensureDir(path.dirname(imagePath));
|
const tick = existingTile.readUInt32BE(offset);
|
||||||
|
offset += 4;
|
||||||
|
const chunkCoordsByte = existingTile.readUInt8(offset);
|
||||||
|
offset += 1;
|
||||||
|
const length = existingTile.readUInt16BE(offset);
|
||||||
|
offset += 2;
|
||||||
|
|
||||||
// Create raw image buffer for 32x32 RGBA
|
const chunkX = chunkCoordsByte >> 4;
|
||||||
const raw = Buffer.alloc(CHUNK_SIZE * CHUNK_SIZE * 4);
|
const chunkY = chunkCoordsByte & 0x0F;
|
||||||
|
const chunkName = `${chunkX}_${chunkY}`;
|
||||||
|
const data = existingTile.slice(offset, offset + length);
|
||||||
|
offset += length;
|
||||||
|
existingChunks.set(chunkName, { tick, data });
|
||||||
|
|
||||||
try {
|
} else if (type === 2) { // Pixels
|
||||||
// Decode base64 to get the deflate-compressed data
|
const tick = existingTile.readUInt32BE(offset);
|
||||||
const compressedBuffer = Buffer.from(chartData, 'base64');
|
offset += 4;
|
||||||
|
const length = existingTile.readUInt16BE(offset);
|
||||||
// Decompress the deflate data to get the original binary chart data
|
offset += 2;
|
||||||
const binaryBuffer = await inflateAsync(compressedBuffer);
|
offset += length; // Skip pixel data
|
||||||
const dataLength = Math.min(binaryBuffer.length, CHUNK_SIZE * CHUNK_SIZE * 2);
|
|
||||||
|
|
||||||
for (let i = 0; i < dataLength; i += 2) {
|
|
||||||
// Read bytes properly from binary buffer
|
|
||||||
const byte1 = binaryBuffer[i];
|
|
||||||
const byte2 = binaryBuffer[i + 1];
|
|
||||||
|
|
||||||
if (byte1 === undefined || byte2 === undefined) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Choose endianness based on flag
|
|
||||||
const rgb565Value = this.useBigEndian ?
|
|
||||||
(byte1 << 8) + byte2 : // Big-endian: byte1 is high byte
|
|
||||||
byte1 + (byte2 << 8); // Little-endian: byte1 is low byte
|
|
||||||
|
|
||||||
const [r, g, b] = this.rgb565ToRgb888(rgb565Value);
|
|
||||||
|
|
||||||
// Calculate pixel position within the chunk
|
|
||||||
const pixelIndex = i / 2;
|
|
||||||
const x = pixelIndex % CHUNK_SIZE;
|
|
||||||
const y = Math.floor(pixelIndex / CHUNK_SIZE);
|
|
||||||
|
|
||||||
// Set pixel in raw buffer (RGBA order - Sharp expects this)
|
|
||||||
const bufferIndex = pixelIndex * 4;
|
|
||||||
if (bufferIndex >= 0 && bufferIndex < raw.length - 3) {
|
|
||||||
raw[bufferIndex] = r; // R
|
|
||||||
raw[bufferIndex + 1] = g; // G
|
|
||||||
raw[bufferIndex + 2] = b; // B
|
|
||||||
raw[bufferIndex + 3] = 255; // A (full alpha)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Create the PNG buffer for both saving and live updates
|
for (const [chunkName, chunk] of newChunks) {
|
||||||
const pngBuffer = await sharp(raw, {
|
// chart_data is already compressed by Factorio, just decode base64
|
||||||
raw: {
|
const chunkData = Buffer.from(chunk.data.chart_data, "base64");
|
||||||
width: CHUNK_SIZE,
|
existingChunks.set(chunkName, { tick: chunk.tick, data: chunkData });
|
||||||
height: CHUNK_SIZE,
|
}
|
||||||
channels: 4,
|
|
||||||
},
|
|
||||||
}).png().toBuffer();
|
|
||||||
|
|
||||||
// Save the chunk image to disk
|
const newTileParts: Buffer[] = [];
|
||||||
await fs.writeFile(imagePath, pngBuffer);
|
for (const [chunkName, chunk] of existingChunks) {
|
||||||
|
const [chunkX, chunkY] = chunkName.split("_").map(Number);
|
||||||
|
const header = Buffer.alloc(8); // Type (1) + Tick (4) + Coords (1) + Length (2)
|
||||||
|
header.writeUInt8(1, 0); // Type
|
||||||
|
header.writeUInt32BE(Math.floor(chunk.tick / 60), 1); // Tick
|
||||||
|
header.writeUInt8((chunkX << 4) | chunkY, 5); // Coords
|
||||||
|
header.writeUInt16BE(chunk.data.length, 6); // Length of data block only
|
||||||
|
newTileParts.push(header, chunk.data);
|
||||||
|
}
|
||||||
|
|
||||||
// Convert ImageData to base64 for live updates
|
return Buffer.concat(newTileParts);
|
||||||
// Create ImageData-like structure that can be sent over WebSocket
|
}
|
||||||
const imageDataArray = new Uint8ClampedArray(raw);
|
|
||||||
const imageDataJson = {
|
|
||||||
data: Array.from(imageDataArray),
|
|
||||||
width: CHUNK_SIZE,
|
|
||||||
height: CHUNK_SIZE,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Get the final PNG file size for comparison
|
async handleTileDataEvent(event: TileDataEvent) {
|
||||||
const stats = await fs.stat(imagePath);
|
try {
|
||||||
const pngSize = stats.size;
|
const { instance_id, surface, force, x, y, tick, chunk } = event;
|
||||||
this.logger.info(`Chunk ${filename}: raw ${raw.length}, decompressed ${binaryBuffer.length}, compressed ${compressedBuffer.length}, PNG ${pngSize} bytes`);
|
|
||||||
|
|
||||||
// Return base64 encoded ImageData for live updates
|
// Calculate tile coordinates for 8x8 chunks per tile (256x256 pixels)
|
||||||
return Buffer.from(JSON.stringify(imageDataJson)).toString('base64');
|
const tileX = Math.floor(x / 32 / 8);
|
||||||
|
const tileY = Math.floor(y / 32 / 8);
|
||||||
|
const tileName = `${instance_id}_${surface}_${force}_${tileX}_${tileY}.bin`;
|
||||||
|
|
||||||
|
// Fix negative modulo results by using proper modulo operation (0-7 range for 8x8)
|
||||||
|
const chunkX = ((Math.floor(x / 32) % 8) + 8) % 8;
|
||||||
|
const chunkY = ((Math.floor(y / 32) % 8) + 8) % 8;
|
||||||
|
const chunkName = `${chunkX}_${chunkY}`;
|
||||||
|
|
||||||
|
if (!this.chunkSavingQueue.has(tileName)) {
|
||||||
|
this.chunkSavingQueue.set(tileName, new Map());
|
||||||
|
}
|
||||||
|
this.chunkSavingQueue.get(tileName)!.set(chunkName, { data: chunk, tick });
|
||||||
|
|
||||||
|
// Send live update to connected web clients
|
||||||
|
const updateEvent = new TileDataEvent(
|
||||||
|
instance_id,
|
||||||
|
surface,
|
||||||
|
force,
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
tick,
|
||||||
|
chunk
|
||||||
|
);
|
||||||
|
|
||||||
|
// Broadcast to web clients (overriding the formal src/dst routing)
|
||||||
|
this.controller.sendTo(new lib.Address(lib.Address.broadcast, lib.Address.control), updateEvent);
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.logger.error(`Error processing chunk image ${filename}: ${err}`);
|
this.logger.error(`Error processing tile data: ${err}`);
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ import * as lib from "@clusterio/lib";
|
|||||||
import {
|
import {
|
||||||
TileDataEvent,
|
TileDataEvent,
|
||||||
GetInstanceBoundsRequest,
|
GetInstanceBoundsRequest,
|
||||||
ChunkUpdateEvent,
|
|
||||||
} from "./messages";
|
} from "./messages";
|
||||||
|
|
||||||
// Define permissions
|
// Define permissions
|
||||||
@@ -25,7 +24,6 @@ export const plugin: lib.PluginDeclaration = {
|
|||||||
messages: [
|
messages: [
|
||||||
TileDataEvent,
|
TileDataEvent,
|
||||||
GetInstanceBoundsRequest,
|
GetInstanceBoundsRequest,
|
||||||
ChunkUpdateEvent,
|
|
||||||
],
|
],
|
||||||
routes: ["/minimap"],
|
routes: ["/minimap"],
|
||||||
};
|
};
|
||||||
@@ -39,10 +39,13 @@ export class InstancePlugin extends BaseInstancePlugin {
|
|||||||
|
|
||||||
// Send chart data to controller
|
// Send chart data to controller
|
||||||
const event = new TileDataEvent(
|
const event = new TileDataEvent(
|
||||||
"chart",
|
this.instance.config.get("instance.id"),
|
||||||
chartData,
|
chartData[0].surface,
|
||||||
position!,
|
chartData[0].force,
|
||||||
this.instance.config.get("instance.id")
|
position[0],
|
||||||
|
position[1],
|
||||||
|
0, // TODO: Get tick from lua
|
||||||
|
chartData[0]
|
||||||
);
|
);
|
||||||
|
|
||||||
return this.instance.sendTo("controller", event);
|
return this.instance.sendTo("controller", event);
|
||||||
|
|||||||
+31
-51
@@ -10,6 +10,17 @@ export const ChartDataSchema = Type.Object({
|
|||||||
|
|
||||||
export type ChartData = Static<typeof ChartDataSchema>;
|
export type ChartData = Static<typeof ChartDataSchema>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TileDataEvent: Used for both Factorio Instance -> Controller and Controller -> Web Clients
|
||||||
|
*
|
||||||
|
* Contains compressed RGB565 chart data from Factorio's internal mapping system.
|
||||||
|
* When sent from instance to controller: triggers storage in persistent tile files.
|
||||||
|
* When sent from controller to web clients: enables real-time minimap updates.
|
||||||
|
*
|
||||||
|
* Data flow:
|
||||||
|
* - Lua module -> Instance plugin -> Controller plugin (for persistence)
|
||||||
|
* - Controller plugin -> Web UI plugin -> Canvas renderer (for live updates)
|
||||||
|
*/
|
||||||
export class TileDataEvent {
|
export class TileDataEvent {
|
||||||
declare ["constructor"]: typeof TileDataEvent;
|
declare ["constructor"]: typeof TileDataEvent;
|
||||||
static type = "event" as const;
|
static type = "event" as const;
|
||||||
@@ -19,65 +30,34 @@ export class TileDataEvent {
|
|||||||
static permission = null;
|
static permission = null;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
public type: "chart",
|
public instance_id: number,
|
||||||
public data: ChartData[],
|
public surface: string,
|
||||||
public position: [number, number],
|
public force: string,
|
||||||
public instanceId: number
|
public x: number,
|
||||||
) {
|
public y: number,
|
||||||
}
|
public tick: number,
|
||||||
|
public chunk: ChartData,
|
||||||
|
) { }
|
||||||
|
|
||||||
static jsonSchema = Type.Object({
|
static jsonSchema = Type.Object({
|
||||||
"type": Type.Union([Type.Literal("chart")]),
|
"instance_id": Type.Number(),
|
||||||
"data": Type.Array(ChartDataSchema),
|
"surface": Type.String(),
|
||||||
"position": Type.Array(Type.Number(), { minItems: 2, maxItems: 2 }),
|
"force": Type.String(),
|
||||||
"instanceId": Type.Number(),
|
"x": Type.Number(),
|
||||||
|
"y": Type.Number(),
|
||||||
|
"tick": Type.Number(),
|
||||||
|
"chunk": ChartDataSchema,
|
||||||
});
|
});
|
||||||
|
|
||||||
static fromJSON(json: Static<typeof TileDataEvent.jsonSchema>) {
|
static fromJSON(json: Static<typeof TileDataEvent.jsonSchema>) {
|
||||||
return new this(
|
return new this(
|
||||||
json.type,
|
json.instance_id,
|
||||||
json.data,
|
|
||||||
json.position as [number, number],
|
|
||||||
json.instanceId,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ChunkUpdateEvent {
|
|
||||||
declare ["constructor"]: typeof ChunkUpdateEvent;
|
|
||||||
static type = "event" as const;
|
|
||||||
static src = "controller" as const;
|
|
||||||
static dst = "control" as const;
|
|
||||||
static plugin = "minimap" as const;
|
|
||||||
static permission = "minimap.view";
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
public instanceId: number,
|
|
||||||
public surface: string,
|
|
||||||
public force: string,
|
|
||||||
public chunkX: number,
|
|
||||||
public chunkY: number,
|
|
||||||
public imageData: string, // base64 encoded ImageData
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
static jsonSchema = Type.Object({
|
|
||||||
"instanceId": Type.Number(),
|
|
||||||
"surface": Type.String(),
|
|
||||||
"force": Type.String(),
|
|
||||||
"chunkX": Type.Number(),
|
|
||||||
"chunkY": Type.Number(),
|
|
||||||
"imageData": Type.String(),
|
|
||||||
});
|
|
||||||
|
|
||||||
static fromJSON(json: Static<typeof ChunkUpdateEvent.jsonSchema>) {
|
|
||||||
return new this(
|
|
||||||
json.instanceId,
|
|
||||||
json.surface,
|
json.surface,
|
||||||
json.force,
|
json.force,
|
||||||
json.chunkX,
|
json.x,
|
||||||
json.chunkY,
|
json.y,
|
||||||
json.imageData,
|
json.tick,
|
||||||
|
json.chunk
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import React, { useState, useEffect, useContext, useRef, useCallback } from "react";
|
import React, { useState, useEffect, useContext, useRef, useCallback } from "react";
|
||||||
import { Row, Col, Button, Card, Space, Select } from "antd";
|
import { Row, Col, Button, Card, Space, Select } from "antd";
|
||||||
import { ControlContext } from "@clusterio/web_ui";
|
import { ControlContext } from "@clusterio/web_ui";
|
||||||
import { GetInstanceBoundsRequest, ChunkUpdateEvent } from "../messages";
|
import { GetInstanceBoundsRequest, TileDataEvent } from "../messages";
|
||||||
|
import * as zlib from "zlib";
|
||||||
|
|
||||||
interface Instance {
|
interface Instance {
|
||||||
instanceId: number;
|
instanceId: number;
|
||||||
@@ -26,8 +27,53 @@ interface ViewState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const CHUNK_SIZE = 32; // 32x32 pixels per chunk
|
const CHUNK_SIZE = 32; // 32x32 pixels per chunk
|
||||||
const TILE_SIZE = 512; // 512x512 pixels per tile (16x16 chunks)
|
const TILE_SIZE = 256; // 256x256 pixels per tile (8x8 chunks)
|
||||||
const CHUNKS_PER_TILE = 16;
|
const CHUNKS_PER_TILE = 8;
|
||||||
|
|
||||||
|
// Convert RGB565 to RGB888 values using Factorio's exact method
|
||||||
|
function rgb565ToRgb888(rgb565Value: number): [number, number, number] {
|
||||||
|
// This matches ByteColor(uint16_t rgb5) constructor in Factorio source
|
||||||
|
const r = ((rgb565Value >> 11) & 0x1F) << 3; // 5 bits -> 8 bits
|
||||||
|
const g = ((rgb565Value >> 5) & 0x3F) << 2; // 6 bits -> 8 bits
|
||||||
|
const b = (rgb565Value & 0x1F) << 3; // 5 bits -> 8 bits
|
||||||
|
return [r, g, b];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert raw chart data to ImageData
|
||||||
|
async function chartDataToImageData(chartData: string): Promise<ImageData> {
|
||||||
|
// Decode base64 and decompress
|
||||||
|
const compressedData = Buffer.from(chartData, "base64");
|
||||||
|
const decompressed = zlib.inflateSync(compressedData);
|
||||||
|
|
||||||
|
// Create RGBA image data (32x32 pixels)
|
||||||
|
const imageData = new Uint8ClampedArray(32 * 32 * 4);
|
||||||
|
|
||||||
|
// Ensure we don't read beyond available data
|
||||||
|
const maxPixels = Math.min(decompressed.length / 2, 32 * 32);
|
||||||
|
|
||||||
|
for (let i = 0; i < maxPixels * 2; i += 2) {
|
||||||
|
const rgb565Value = decompressed.readUInt16LE(i);
|
||||||
|
const [r, g, b] = rgb565ToRgb888(rgb565Value);
|
||||||
|
const pixelIndex = i / 2;
|
||||||
|
const bufferIndex = pixelIndex * 4;
|
||||||
|
|
||||||
|
imageData[bufferIndex] = r;
|
||||||
|
imageData[bufferIndex + 1] = g;
|
||||||
|
imageData[bufferIndex + 2] = b;
|
||||||
|
imageData[bufferIndex + 3] = 255; // Alpha
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fill remaining pixels with black if we have less data than expected
|
||||||
|
for (let pixelIndex = maxPixels; pixelIndex < 32 * 32; pixelIndex++) {
|
||||||
|
const bufferIndex = pixelIndex * 4;
|
||||||
|
imageData[bufferIndex] = 0; // R
|
||||||
|
imageData[bufferIndex + 1] = 0; // G
|
||||||
|
imageData[bufferIndex + 2] = 0; // B
|
||||||
|
imageData[bufferIndex + 3] = 255; // A
|
||||||
|
}
|
||||||
|
|
||||||
|
return new ImageData(imageData, 32, 32);
|
||||||
|
}
|
||||||
|
|
||||||
export default function CanvasMinimapPage() {
|
export default function CanvasMinimapPage() {
|
||||||
const [instances, setInstances] = useState<Instance[]>([]);
|
const [instances, setInstances] = useState<Instance[]>([]);
|
||||||
@@ -310,7 +356,7 @@ export default function CanvasMinimapPage() {
|
|||||||
try {
|
try {
|
||||||
// Load the entire tile from server
|
// Load the entire tile from server
|
||||||
const response = await fetch(
|
const response = await fetch(
|
||||||
`${window.location.origin}/api/minimap/chart/${selectedInstance}/${selectedSurface}/${selectedForce}/10/${tileX}/${tileY}.png`
|
`${window.location.origin}/api/minimap/tile/${selectedInstance}/${selectedSurface}/${selectedForce}/10/${tileX}/${tileY}.png`
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
@@ -703,23 +749,24 @@ export default function CanvasMinimapPage() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const handleChunkUpdate = (event: ChunkUpdateEvent) => {
|
const handleChunkUpdate = (event: TileDataEvent) => {
|
||||||
// Only process updates for the currently selected instance/surface/force
|
// Only process updates for the currently selected instance/surface/force
|
||||||
if (event.instanceId === selectedInstance &&
|
if (event.instance_id === selectedInstance &&
|
||||||
event.surface === selectedSurface &&
|
event.surface === selectedSurface &&
|
||||||
event.force === selectedForce) {
|
event.force === selectedForce) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Decode the base64 ImageData
|
// Calculate chunk coordinates from world coordinates
|
||||||
const imageDataJson = JSON.parse(atob(event.imageData));
|
const chunkX = Math.floor(event.x / 32);
|
||||||
const imageData = new ImageData(
|
const chunkY = Math.floor(event.y / 32);
|
||||||
new Uint8ClampedArray(imageDataJson.data),
|
|
||||||
imageDataJson.width,
|
|
||||||
imageDataJson.height
|
|
||||||
);
|
|
||||||
|
|
||||||
// Update the chunk in our cache
|
// Convert raw chart data to ImageData
|
||||||
updateChunk(event.chunkX, event.chunkY, imageData);
|
chartDataToImageData(event.chunk.chart_data).then(imageData => {
|
||||||
|
// Update the chunk in our cache
|
||||||
|
updateChunk(chunkX, chunkY, imageData);
|
||||||
|
}).catch(err => {
|
||||||
|
console.error("Failed to convert chart data to ImageData:", err);
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error("Failed to process live chunk update:", err);
|
console.error("Failed to process live chunk update:", err);
|
||||||
}
|
}
|
||||||
@@ -727,10 +774,10 @@ export default function CanvasMinimapPage() {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Subscribe to chunk update events through the plugin
|
// Subscribe to chunk update events through the plugin
|
||||||
plugin.onChunkUpdate(handleChunkUpdate);
|
plugin.onTileUpdate(handleChunkUpdate);
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
plugin.offChunkUpdate(handleChunkUpdate);
|
plugin.offTileUpdate(handleChunkUpdate);
|
||||||
};
|
};
|
||||||
}, [selectedInstance, selectedSurface, selectedForce, updateChunk, control]);
|
}, [selectedInstance, selectedSurface, selectedForce, updateChunk, control]);
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
import React from "react";
|
import React from "react";
|
||||||
import { BaseWebPlugin } from "@clusterio/web_ui";
|
import { BaseWebPlugin } from "@clusterio/web_ui";
|
||||||
import CanvasMinimapPage from "./CanvasMinimapPage";
|
import CanvasMinimapPage from "./CanvasMinimapPage";
|
||||||
import { ChunkUpdateEvent } from "../messages";
|
import { TileDataEvent } from "../messages";
|
||||||
|
|
||||||
export class WebPlugin extends BaseWebPlugin {
|
export class WebPlugin extends BaseWebPlugin {
|
||||||
chunkUpdateCallbacks: Array<(event: ChunkUpdateEvent) => void> = [];
|
tileUpdateCallbacks: Array<(event: TileDataEvent) => void> = [];
|
||||||
|
|
||||||
async init() {
|
async init() {
|
||||||
this.pages = [
|
this.pages = [
|
||||||
@@ -16,26 +16,26 @@ export class WebPlugin extends BaseWebPlugin {
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
// Handle chunk update events from the controller
|
// Handle tile update events from the controller
|
||||||
this.control.handle(ChunkUpdateEvent, this.handleChunkUpdateEvent.bind(this));
|
this.control.handle(TileDataEvent, this.handleTileDataEvent.bind(this));
|
||||||
}
|
}
|
||||||
|
|
||||||
async handleChunkUpdateEvent(event: ChunkUpdateEvent) {
|
async handleTileDataEvent(event: TileDataEvent) {
|
||||||
// Notify all registered callbacks
|
// Notify all registered callbacks
|
||||||
for (let callback of this.chunkUpdateCallbacks) {
|
for (let callback of this.tileUpdateCallbacks) {
|
||||||
callback(event);
|
callback(event);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onChunkUpdate(callback: (event: ChunkUpdateEvent) => void) {
|
onTileUpdate(callback: (event: TileDataEvent) => void) {
|
||||||
this.chunkUpdateCallbacks.push(callback);
|
this.tileUpdateCallbacks.push(callback);
|
||||||
}
|
}
|
||||||
|
|
||||||
offChunkUpdate(callback: (event: ChunkUpdateEvent) => void) {
|
offTileUpdate(callback: (event: TileDataEvent) => void) {
|
||||||
let index = this.chunkUpdateCallbacks.lastIndexOf(callback);
|
let index = this.tileUpdateCallbacks.lastIndexOf(callback);
|
||||||
if (index === -1) {
|
if (index === -1) {
|
||||||
throw new Error("callback is not registered");
|
throw new Error("callback is not registered");
|
||||||
}
|
}
|
||||||
this.chunkUpdateCallbacks.splice(index, 1);
|
this.tileUpdateCallbacks.splice(index, 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user