Implement new tile storage format with tentative timelapse support

This commit is contained in:
Danielv123 committed 2025-08-10 00:28:48 +02:00
1 parent 2dcc92c96b
commit a516b6b129
7 files changed
+358 -307

No files matched your search

+242 -219
View File
@@ -4,7 +4,6 @@ import {
TileDataEvent,
GetInstanceBoundsRequest,
ChartData,
ChunkUpdateEvent,
} from "./messages";
import * as fs from "fs-extra";
import * as path from "path";
@@ -18,22 +17,27 @@ const inflateAsync = promisify(zlib.inflate);
export class ControllerPlugin extends BaseControllerPlugin {
private tilesPath: string = "";
private chunkSavingQueue = new Map<string, Map<string, { data: ChartData, tick: number }>>();
private savingTiles: boolean = false;
// Set to true if colors appear inverted - this will try big-endian byte order
private useBigEndian: boolean = false;
async init() {
this.tilesPath = path.resolve(
this.controller.config.get("controller.database_directory"),
"minimap_chunks"
"minimap_tiles"
);
await fs.ensureDir(this.tilesPath);
this.controller.handle(TileDataEvent, this.handleTileDataEvent.bind(this));
this.controller.handle(GetInstanceBoundsRequest, this.handleGetInstanceBoundsRequest.bind(this));
// Set up HTTP routes for serving tiles
this.setupTileRoutes();
setInterval(() => {
this.saveTiles().catch(err => this.logger.error(`Error saving tiles: ${err}`));
}, 5000);
}
// Convert RGB565 to RGB888 values using Factorio's exact method
@@ -48,268 +52,287 @@ export class ControllerPlugin extends BaseControllerPlugin {
private setupTileRoutes() {
const app = this.controller.app;
// Create black placeholder image for 512x512 tiles
const createBlackTile = async () => {
return await sharp({
create: {
width: 512,
height: 512,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 },
},
}).png().toBuffer();
};
// Serve chart tiles with surface and force support - now serving 512x512 tiles
app.get("/api/minimap/chart/:instanceId/:surface/:force/:z/:x/:y.png", async (req, res) => {
app.get("/api/minimap/tile/:instanceId/:surface/:force/:z/:x/:y.png", async (req, res) => {
try {
const { instanceId, surface, force, z, x, y } = req.params;
const tileX = parseInt(x);
const tileY = parseInt(y);
// Each 512x512 tile contains 16x16 chunks of 32x32 pixels
const chunksPerTile = 16;
const tileSize = 512;
// Calculate the range of chunk coordinates that make up this 512x512 tile
const startChunkX = tileX * chunksPerTile;
const startChunkY = tileY * chunksPerTile;
const endChunkX = startChunkX + chunksPerTile;
const endChunkY = startChunkY + chunksPerTile;
// Create a 512x512 canvas
const canvas = sharp({
create: {
width: tileSize,
height: tileSize,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 },
},
});
// Load all the 32x32 chunks and composite them
const compositeImages = [];
let chunksFound = 0;
for (let chunkY = startChunkY; chunkY < endChunkY; chunkY++) {
for (let chunkX = startChunkX; chunkX < endChunkX; chunkX++) {
const filename = `${instanceId}/${surface}/${force}/${chunkX}_${chunkY}.png`;
const filePath = path.resolve(this.tilesPath, filename);
if (await fs.pathExists(filePath)) {
const relativeX = (chunkX - startChunkX) * CHUNK_SIZE;
const relativeY = (chunkY - startChunkY) * CHUNK_SIZE;
compositeImages.push({
input: filePath,
left: relativeX,
top: relativeY,
});
chunksFound++;
}
}
const tileName = `${instanceId}_${surface}_${force}_${tileX}_${tileY}.bin`;
const tilePath = path.join(this.tilesPath, tileName);
if (!await fs.pathExists(tilePath)) {
res.status(404).send("Tile not found");
return;
}
if(chunksFound > 0) {
this.logger.info(`Tile ${tileX},${tileY}: Found ${chunksFound} chunks, chunk range: ${startChunkX}-${endChunkX-1}, ${startChunkY}-${endChunkY-1}`);
}
// If we have any chunks to composite, create the merged image
if (compositeImages.length > 0) {
const mergedImage = await canvas.composite(compositeImages).png().toBuffer();
res.setHeader("Content-Type", "image/png");
res.send(mergedImage);
} else {
// No chunks found, return black tile
res.setHeader("Content-Type", "image/png");
res.send(await createBlackTile());
}
} catch (err) {
this.logger.error(`Error serving chart tile: ${err}`);
// Return black tile on error
const tileData = await fs.readFile(tilePath);
const image = await this.renderTile(tileData);
res.setHeader("Content-Type", "image/png");
res.send(await createBlackTile());
res.send(image);
} catch (err) {
this.logger.error(`Error serving tile: ${err}`);
res.status(500).send("Error serving tile");
}
});
// List available surfaces and forces
app.get("/api/minimap/surfaces", async (req, res) => {
try {
const instances = new Set<string>();
const files = await fs.readdir(this.tilesPath);
const surfaces = new Set<string>();
const forces = new Set<string>();
// Traverse the hierarchical directory structure: instanceId/surface/force/
const instanceDirs = await fs.readdir(this.tilesPath, { withFileTypes: true });
for (const instanceDir of instanceDirs) {
if (instanceDir.isDirectory()) {
instances.add(instanceDir.name);
const instancePath = path.join(this.tilesPath, instanceDir.name);
const surfaceDirs = await fs.readdir(instancePath, { withFileTypes: true });
for (const surfaceDir of surfaceDirs) {
if (surfaceDir.isDirectory()) {
surfaces.add(surfaceDir.name);
const surfacePath = path.join(instancePath, surfaceDir.name);
const forceDirs = await fs.readdir(surfacePath, { withFileTypes: true });
for (const forceDir of forceDirs) {
if (forceDir.isDirectory()) {
forces.add(forceDir.name);
}
}
}
for (const file of files) {
if (file.endsWith(".bin")) {
const parts = file.split("_");
if (parts.length === 5) {
surfaces.add(parts[1]);
forces.add(parts[2]);
}
}
}
res.json({
instances: Array.from(instances),
surfaces: Array.from(surfaces),
forces: Array.from(forces),
});
} catch (err) {
this.logger.error(`Error listing surfaces: ${err}`);
res.status(500).json({ error: "Failed to list surfaces" });
this.logger.error(`Error listing surfaces and forces: ${err}`);
res.status(500).json({ error: "Failed to list surfaces and forces" });
}
});
}
async handleTileDataEvent(event: TileDataEvent) {
try {
const { type, data, position, instanceId } = event;
async renderTile(tileData: Buffer): Promise<Buffer> {
const tileSize = 256;
const canvas = sharp({
create: {
width: tileSize,
height: tileSize,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 },
},
});
if (type === "chart") {
await this.processChartData(data as ChartData[], position, instanceId);
const compositeImages: any[] = [];
let offset = 0;
while (offset < tileData.length) {
if (offset + 1 > tileData.length) {
this.logger.error(`Invalid tile data: cannot read type at offset ${offset}`);
break;
}
const type = tileData.readUInt8(offset);
offset += 1; // After type
if (type === 1) { // Chunk
// Check if we have enough bytes for the header
if (offset + 7 > tileData.length) {
this.logger.error(`Invalid tile data: insufficient header data at offset ${offset}`);
break;
}
const tick = tileData.readUInt32BE(offset); // Read tick
offset += 4; // After tick
const chunkCoordsByte = tileData.readUInt8(offset); // Read coords
offset += 1; // After coords
const length = tileData.readUInt16BE(offset); // Read length
offset += 2; // After length
// Check if we have enough bytes for the data
if (offset + length > tileData.length) {
this.logger.error(`Invalid tile data: insufficient chunk data at offset ${offset}, need ${length} bytes but only ${tileData.length - offset} available`);
break;
}
const chunkX = chunkCoordsByte >> 4;
const chunkY = chunkCoordsByte & 0x0F;
const chunkData = tileData.slice(offset, offset + length);
try {
const decompressed = await inflateAsync(chunkData);
const raw = Buffer.alloc(32 * 32 * 4);
// Initialize buffer to black (all zeros with alpha=255)
for (let i = 0; i < 32 * 32; i++) {
raw[i * 4 + 3] = 255; // Set alpha to 255 for all pixels
}
// Ensure we don't read beyond available data
const maxPixels = Math.min(decompressed.length / 2, 32 * 32);
for (let i = 0; i < maxPixels * 2; i += 2) {
const rgb565Value = decompressed.readUInt16LE(i);
const [r, g, b] = this.rgb565ToRgb888(rgb565Value);
const pixelIndex = i / 2;
const bufferIndex = pixelIndex * 4;
raw[bufferIndex] = r;
raw[bufferIndex + 1] = g;
raw[bufferIndex + 2] = b;
raw[bufferIndex + 3] = 255;
}
compositeImages.push({
input: await sharp(raw, { raw: { width: 32, height: 32, channels: 4 } }).png().toBuffer(),
left: chunkX * 32,
top: chunkY * 32,
});
} catch (decompressError) {
this.logger.error(`Failed to decompress chunk data: ${decompressError}`);
}
offset += length; // Advance offset by the data block length
} else if (type === 2) { // Pixels
// Check if we have enough bytes for the header
if (offset + 6 > tileData.length) {
this.logger.error(`Invalid tile data: insufficient pixel header data at offset ${offset}`);
break;
}
// Read tick (4 bytes) and length (2 bytes) first
offset += 4; // Skip tick
const length = tileData.readUInt16BE(offset); // Read length
offset += 2; // After length
// Check if we have enough bytes for the data
if (offset + length > tileData.length) {
this.logger.error(`Invalid tile data: insufficient pixel data at offset ${offset}, need ${length} bytes but only ${tileData.length - offset} available`);
break;
}
offset += length; // Advance offset by the data block length
} else {
this.logger.error(`Unknown tile data type: ${type} at offset ${offset}`);
break;
}
} catch (err) {
this.logger.error(`Error processing tile data: ${err}`);
}
return canvas.composite(compositeImages).png().toBuffer();
}
private async processChartData(chartData: ChartData[], position: [number, number], instanceId: number) {
if (!position || !instanceId) {
this.logger.error("Chart data requires position and instanceId");
async saveTiles() {
if (this.savingTiles) {
return;
}
const [worldX, worldY] = position;
// Convert world coordinates to chunk coordinates
const chunkX = Math.floor(worldX / 32);
const chunkY = Math.floor(worldY / 32);
const queue = this.chunkSavingQueue;
if (queue.size === 0) {
return;
}
this.logger.info(`Starting save for ${queue.size} tiles.`);
// Process each surface/force combination
for (const chart of chartData) {
const { surface, force, chart_data } = chart;
// Create filename using chunk coordinates
const filename = `${instanceId}/${surface}/${force}/${chunkX}_${chunkY}.png`;
// Process the 32x32 chunk directly and get the processed image data
const imageDataBase64 = await this.saveChunkImage(filename, chart_data);
// Send live update to connected web clients
if (imageDataBase64) {
const updateEvent = new ChunkUpdateEvent(
instanceId,
surface,
force,
chunkX,
chunkY,
imageDataBase64
);
// Broadcast to all connected control clients
this.controller.sendEvent(updateEvent, new lib.Address(lib.Address.broadcast, lib.Address.control));
this.savingTiles = true;
this.chunkSavingQueue = new Map();
try {
const savePromises = [];
for (const [tileName, chunks] of queue) {
const promise = (async () => {
const tilePath = path.join(this.tilesPath, tileName);
let existingTile = Buffer.alloc(0);
if (await fs.pathExists(tilePath)) {
existingTile = await fs.readFile(tilePath);
}
const newTile = await this.updateTile(existingTile, chunks);
await fs.writeFile(tilePath, newTile);
})();
savePromises.push(promise);
}
await Promise.all(savePromises);
} catch (err) {
this.logger.error(`Error saving tiles: ${err}`);
} finally {
this.savingTiles = false;
}
}
private async saveChunkImage(filename: string, chartData: string): Promise<string | null> {
const imagePath = path.resolve(this.tilesPath, filename);
// Ensure the directory structure exists
await fs.ensureDir(path.dirname(imagePath));
// Create raw image buffer for 32x32 RGBA
const raw = Buffer.alloc(CHUNK_SIZE * CHUNK_SIZE * 4);
try {
// Decode base64 to get the deflate-compressed data
const compressedBuffer = Buffer.from(chartData, 'base64');
async updateTile(existingTile: Buffer, newChunks: Map<string, { data: ChartData, tick: number }>): Promise<Buffer> {
const existingChunks = new Map<string, { tick: number, data: Buffer }>();
let offset = 0;
while (offset < existingTile.length) {
const type = existingTile.readUInt8(offset);
offset += 1;
// Decompress the deflate data to get the original binary chart data
const binaryBuffer = await inflateAsync(compressedBuffer);
const dataLength = Math.min(binaryBuffer.length, CHUNK_SIZE * CHUNK_SIZE * 2);
for (let i = 0; i < dataLength; i += 2) {
// Read bytes properly from binary buffer
const byte1 = binaryBuffer[i];
const byte2 = binaryBuffer[i + 1];
if (type === 1) { // Chunk
const tick = existingTile.readUInt32BE(offset);
offset += 4;
const chunkCoordsByte = existingTile.readUInt8(offset);
offset += 1;
const length = existingTile.readUInt16BE(offset);
offset += 2;
if (byte1 === undefined || byte2 === undefined) {
continue;
}
const chunkX = chunkCoordsByte >> 4;
const chunkY = chunkCoordsByte & 0x0F;
const chunkName = `${chunkX}_${chunkY}`;
const data = existingTile.slice(offset, offset + length);
offset += length;
existingChunks.set(chunkName, { tick, data });
// Choose endianness based on flag
const rgb565Value = this.useBigEndian ?
(byte1 << 8) + byte2 : // Big-endian: byte1 is high byte
byte1 + (byte2 << 8); // Little-endian: byte1 is low byte
const [r, g, b] = this.rgb565ToRgb888(rgb565Value);
// Calculate pixel position within the chunk
const pixelIndex = i / 2;
const x = pixelIndex % CHUNK_SIZE;
const y = Math.floor(pixelIndex / CHUNK_SIZE);
// Set pixel in raw buffer (RGBA order - Sharp expects this)
const bufferIndex = pixelIndex * 4;
if (bufferIndex >= 0 && bufferIndex < raw.length - 3) {
raw[bufferIndex] = r; // R
raw[bufferIndex + 1] = g; // G
raw[bufferIndex + 2] = b; // B
raw[bufferIndex + 3] = 255; // A (full alpha)
}
} else if (type === 2) { // Pixels
const tick = existingTile.readUInt32BE(offset);
offset += 4;
const length = existingTile.readUInt16BE(offset);
offset += 2;
offset += length; // Skip pixel data
}
}
// Create the PNG buffer for both saving and live updates
const pngBuffer = await sharp(raw, {
raw: {
width: CHUNK_SIZE,
height: CHUNK_SIZE,
channels: 4,
},
}).png().toBuffer();
// Save the chunk image to disk
await fs.writeFile(imagePath, pngBuffer);
// Convert ImageData to base64 for live updates
// Create ImageData-like structure that can be sent over WebSocket
const imageDataArray = new Uint8ClampedArray(raw);
const imageDataJson = {
data: Array.from(imageDataArray),
width: CHUNK_SIZE,
height: CHUNK_SIZE,
};
// Get the final PNG file size for comparison
const stats = await fs.stat(imagePath);
const pngSize = stats.size;
this.logger.info(`Chunk ${filename}: raw ${raw.length}, decompressed ${binaryBuffer.length}, compressed ${compressedBuffer.length}, PNG ${pngSize} bytes`);
for (const [chunkName, chunk] of newChunks) {
// chart_data is already compressed by Factorio, just decode base64
const chunkData = Buffer.from(chunk.data.chart_data, "base64");
existingChunks.set(chunkName, { tick: chunk.tick, data: chunkData });
}
// Return base64 encoded ImageData for live updates
return Buffer.from(JSON.stringify(imageDataJson)).toString('base64');
const newTileParts: Buffer[] = [];
for (const [chunkName, chunk] of existingChunks) {
const [chunkX, chunkY] = chunkName.split("_").map(Number);
const header = Buffer.alloc(8); // Type (1) + Tick (4) + Coords (1) + Length (2)
header.writeUInt8(1, 0); // Type
header.writeUInt32BE(Math.floor(chunk.tick / 60), 1); // Tick
header.writeUInt8((chunkX << 4) | chunkY, 5); // Coords
header.writeUInt16BE(chunk.data.length, 6); // Length of data block only
newTileParts.push(header, chunk.data);
}
return Buffer.concat(newTileParts);
}
async handleTileDataEvent(event: TileDataEvent) {
try {
const { instance_id, surface, force, x, y, tick, chunk } = event;
// Calculate tile coordinates for 8x8 chunks per tile (256x256 pixels)
const tileX = Math.floor(x / 32 / 8);
const tileY = Math.floor(y / 32 / 8);
const tileName = `${instance_id}_${surface}_${force}_${tileX}_${tileY}.bin`;
// Fix negative modulo results by using proper modulo operation (0-7 range for 8x8)
const chunkX = ((Math.floor(x / 32) % 8) + 8) % 8;
const chunkY = ((Math.floor(y / 32) % 8) + 8) % 8;
const chunkName = `${chunkX}_${chunkY}`;
if (!this.chunkSavingQueue.has(tileName)) {
this.chunkSavingQueue.set(tileName, new Map());
}
this.chunkSavingQueue.get(tileName)!.set(chunkName, { data: chunk, tick });
// Send live update to connected web clients
const updateEvent = new TileDataEvent(
instance_id,
surface,
force,
x,
y,
tick,
chunk
);
// Broadcast to web clients (overriding the formal src/dst routing)
this.controller.sendTo(new lib.Address(lib.Address.broadcast, lib.Address.control), updateEvent);
} catch (err) {
this.logger.error(`Error processing chunk image ${filename}: ${err}`);
return null;
this.logger.error(`Error processing tile data: ${err}`);
}
}