mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-21 16:13:12 +09:00
Add datastore to lib
This commit is contained in:
2 files changed
+216
No files matched your search
@@ -28,6 +28,7 @@ export * from "./src/stream";
|
|||||||
export * from "./src/system_collectors";
|
export * from "./src/system_collectors";
|
||||||
export * from "./src/zip_ops";
|
export * from "./src/zip_ops";
|
||||||
export * from "./src/subscriptions";
|
export * from "./src/subscriptions";
|
||||||
|
export * from "./src/datastore";
|
||||||
|
|
||||||
export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
|
export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
|
||||||
export { default as ModStore } from "./src/ModStore";
|
export { default as ModStore } from "./src/ModStore";
|
||||||
|
|||||||
@@ -0,0 +1,215 @@
|
|||||||
|
import fs from "fs-extra";
|
||||||
|
|
||||||
|
import { safeOutputFile } from "./file_ops";
|
||||||
|
import { SubscribableValue } from "./subscriptions";
|
||||||
|
import { JSONDeserialisable } from "./data/composites";
|
||||||
|
|
||||||
|
type DatastoreKey = string | number;
|
||||||
|
type DatastoreValue = string | number | boolean | object;
|
||||||
|
|
||||||
|
// Deep copy required to support the read only property of datastore values
|
||||||
|
function deepCopy<V extends DatastoreValue>(value: V): V {
|
||||||
|
if (value instanceof Object) {
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value).map(([k, v]) => [k, deepCopy(v)])
|
||||||
|
) as V;
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Abstract class which can provide saving and loading capabilities to a datastore
|
||||||
|
export abstract class DatastoreProvider<
|
||||||
|
K extends DatastoreKey,
|
||||||
|
V extends DatastoreValue,
|
||||||
|
> {
|
||||||
|
abstract save(data: Map<K, V>): Promise<void>
|
||||||
|
abstract load(): Promise<Map<K, V>>
|
||||||
|
|
||||||
|
// Load data from the source and return it, this should be passed to the constructor of a datastore
|
||||||
|
async bootstrap() {
|
||||||
|
return [this, await this.load()] as const;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store all data in memory, used as a default and should be avoided
|
||||||
|
export class MemoryDatastoreProvider<
|
||||||
|
K extends DatastoreKey,
|
||||||
|
V extends DatastoreValue,
|
||||||
|
> extends DatastoreProvider<K, V> {
|
||||||
|
async save(data: Map<K, V>) {
|
||||||
|
}
|
||||||
|
|
||||||
|
async load() {
|
||||||
|
return new Map();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store all data within a json object file and validate the result against a schema
|
||||||
|
export class JsonDatastoreProvider<
|
||||||
|
K extends DatastoreKey,
|
||||||
|
V extends DatastoreValue,
|
||||||
|
> extends DatastoreProvider<K, V> {
|
||||||
|
constructor(
|
||||||
|
private filePath: string,
|
||||||
|
private dataClass: JSONDeserialisable<V>,
|
||||||
|
private migrations: (rawJson: unknown) => Array<[K, unknown]> = v => v as any,
|
||||||
|
) {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the data to the json file
|
||||||
|
async save(data: Map<K, V>) {
|
||||||
|
await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the data from the json file
|
||||||
|
async load() {
|
||||||
|
// Read the raw json from the file
|
||||||
|
let rawJson;
|
||||||
|
try {
|
||||||
|
rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" }));
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.code !== "ENOENT") {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
return new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply migrations
|
||||||
|
const serialized = this.migrations(rawJson);
|
||||||
|
|
||||||
|
// Convert to data class objects
|
||||||
|
return new Map(serialized.map((k, v) => [k, this.dataClass.fromJSON(v)]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store all data within a json array file where each element has a unique id
|
||||||
|
export class JsonIdDatastoreProvider<
|
||||||
|
K extends DatastoreKey,
|
||||||
|
V extends DatastoreValue & { id: K },
|
||||||
|
> extends DatastoreProvider<K, V> {
|
||||||
|
constructor(
|
||||||
|
private filePath: string,
|
||||||
|
private dataClass: JSONDeserialisable<V>,
|
||||||
|
private migrations: (rawJson: unknown) => Array<{ id: K }> = v => v as any,
|
||||||
|
) {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the data to the json file
|
||||||
|
async save(data: Map<K, V>) {
|
||||||
|
await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the data from the json file
|
||||||
|
async load() {
|
||||||
|
// Read the raw json from the file
|
||||||
|
let rawJson;
|
||||||
|
try {
|
||||||
|
rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" }));
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err.code !== "ENOENT") {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
return new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply migrations
|
||||||
|
const serialized = this.migrations(rawJson);
|
||||||
|
|
||||||
|
// Convert to data class objects
|
||||||
|
return new Map(serialized.map((e) => [e.id, this.dataClass.fromJSON(e)]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Implements a similar interface to a map
|
||||||
|
export abstract class BaseDatastore<
|
||||||
|
K extends DatastoreKey,
|
||||||
|
V extends DatastoreValue,
|
||||||
|
> {
|
||||||
|
protected dirty = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
protected provider: DatastoreProvider<K, V> = new MemoryDatastoreProvider(),
|
||||||
|
protected data = new Map<K, V>(),
|
||||||
|
) {}
|
||||||
|
|
||||||
|
// Save the datastore to the provider
|
||||||
|
async save() {
|
||||||
|
if (this.dirty) {
|
||||||
|
await this.provider.save(this.data);
|
||||||
|
this.dirty = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the datastore from the provider
|
||||||
|
async load() {
|
||||||
|
this.data = await this.provider.load();
|
||||||
|
this.dirty = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns true if the datastore has the value, false otherwise
|
||||||
|
has(key: K) {
|
||||||
|
return this.data.has(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get a value from from the datastore
|
||||||
|
get(key: K) {
|
||||||
|
return this.data.get(key) as Readonly<V> ?? undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get a copy of a value from the datastore
|
||||||
|
getCopy(key: K) {
|
||||||
|
const v = this.data.get(key);
|
||||||
|
return v === undefined ? v : deepCopy(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns all values in the datastore
|
||||||
|
values() {
|
||||||
|
return this.data.values() as IterableIterator<Readonly<V>>;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// General key value mapping which can be saved and loaded with the file system
|
||||||
|
export class Datastore<
|
||||||
|
K extends DatastoreKey,
|
||||||
|
V extends DatastoreValue,
|
||||||
|
> extends BaseDatastore<K, V> {
|
||||||
|
// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
|
||||||
|
set(key: K, value: V) {
|
||||||
|
this.data.set(key, value);
|
||||||
|
this.dirty = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set many values at once from an array of key value pairs
|
||||||
|
setMany(pairs: [[K, V]]) {
|
||||||
|
this.dirty ||= pairs.length > 0;
|
||||||
|
for (const [k, v] of pairs) {
|
||||||
|
this.data.set(k, v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A special implementation of the datastore to work directly with subscribable values
|
||||||
|
// TODO broadcast value changes to subscribers
|
||||||
|
export class SubscribableDatastore<
|
||||||
|
V extends SubscribableValue,
|
||||||
|
> extends BaseDatastore<V["id"], V> {
|
||||||
|
// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
|
||||||
|
set(value: V) {
|
||||||
|
value.updatedAtMs = Date.now();
|
||||||
|
this.data.set(value.id, value);
|
||||||
|
this.dirty = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set many values at once from an array of key value pairs
|
||||||
|
setMany(values: V[]) {
|
||||||
|
const nowMs = Date.now();
|
||||||
|
this.dirty ||= values.length > 0;
|
||||||
|
for (const value of values) {
|
||||||
|
value.updatedAtMs = nowMs;
|
||||||
|
this.data.set(value.id, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user