mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-21 08:03:13 +09:00
Add recovery startup mode (#757)
* Add recovery startup mode * Change error code to RecoveryMode
This commit is contained in:
1 parent
7cf13a8b3b
commit
be071be8f2
7 files changed
+58
No files matched your search
@@ -270,6 +270,10 @@ async function initialize(): Promise<InitializeParameters> {
|
|||||||
type: "boolean", nargs: 0, default: false,
|
type: "boolean", nargs: 0, default: false,
|
||||||
describe: "Indicate that a process monitor will restart the controller on failure",
|
describe: "Indicate that a process monitor will restart the controller on failure",
|
||||||
});
|
});
|
||||||
|
yargs.option("recovery", {
|
||||||
|
type: "boolean", nargs: 0, default: false,
|
||||||
|
describe: "Start the controller in recovery mode with all plugins disabled and hosts disconnected",
|
||||||
|
});
|
||||||
yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 });
|
yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 });
|
||||||
yargs.option("dev-plugin", { hidden: true, type: "array" });
|
yargs.option("dev-plugin", { hidden: true, type: "array" });
|
||||||
})
|
})
|
||||||
@@ -313,6 +317,9 @@ async function initialize(): Promise<InitializeParameters> {
|
|||||||
let shouldRun = false;
|
let shouldRun = false;
|
||||||
if (command === "run") {
|
if (command === "run") {
|
||||||
logger.info(`Starting Clusterio controller ${version}`);
|
logger.info(`Starting Clusterio controller ${version}`);
|
||||||
|
if (args.recovery) {
|
||||||
|
logger.warn("Controller recovery mode enabled. Some features will be disabled.");
|
||||||
|
}
|
||||||
shouldRun = true;
|
shouldRun = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -406,6 +413,7 @@ async function startup() {
|
|||||||
controllerConfigPath,
|
controllerConfigPath,
|
||||||
controllerConfig,
|
controllerConfig,
|
||||||
Boolean(args.canRestart),
|
Boolean(args.canRestart),
|
||||||
|
Boolean(args.recovery),
|
||||||
...await Controller.bootstrap(controllerConfig)
|
...await Controller.bootstrap(controllerConfig)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -163,6 +163,11 @@ export default class Controller {
|
|||||||
* will restart the controller on non-zero exit codes.
|
* will restart the controller on non-zero exit codes.
|
||||||
*/
|
*/
|
||||||
public canRestart: boolean = false,
|
public canRestart: boolean = false,
|
||||||
|
/**
|
||||||
|
* If true indicates the controller is in recovery mode and should
|
||||||
|
* disable certain actions such as loading plugins or connecting to hosts
|
||||||
|
*/
|
||||||
|
public recoveryMode: boolean = false,
|
||||||
public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
|
public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
|
||||||
/** Mapping of host id to host info */
|
/** Mapping of host id to host info */
|
||||||
public hosts = new lib.SubscribableDatastore<HostInfo>(),
|
public hosts = new lib.SubscribableDatastore<HostInfo>(),
|
||||||
@@ -538,6 +543,10 @@ export default class Controller {
|
|||||||
clearInterval(this.autosaveInterval);
|
clearInterval(this.autosaveInterval);
|
||||||
this.autosaveInterval = undefined;
|
this.autosaveInterval = undefined;
|
||||||
}
|
}
|
||||||
|
if (this.recoveryMode) {
|
||||||
|
logger.warn("Recovery | autosaving disabled");
|
||||||
|
return;
|
||||||
|
}
|
||||||
const autosaveIntervalSeconds = this.config.get("controller.autosave_interval");
|
const autosaveIntervalSeconds = this.config.get("controller.autosave_interval");
|
||||||
if (autosaveIntervalSeconds > 0) {
|
if (autosaveIntervalSeconds > 0) {
|
||||||
this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000);
|
this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000);
|
||||||
@@ -1058,6 +1067,11 @@ export default class Controller {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this.recoveryMode) {
|
||||||
|
logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let ControllerPluginClass = BaseControllerPlugin;
|
let ControllerPluginClass = BaseControllerPlugin;
|
||||||
try {
|
try {
|
||||||
if (pluginInfo.controllerEntrypoint) {
|
if (pluginInfo.controllerEntrypoint) {
|
||||||
|
|||||||
@@ -239,6 +239,13 @@ ${err.stack}`
|
|||||||
}
|
}
|
||||||
|
|
||||||
async registerHost(data: lib.RegisterHostData, socket: WebSocket, req: IncomingMessage) {
|
async registerHost(data: lib.RegisterHostData, socket: WebSocket, req: IncomingMessage) {
|
||||||
|
if (this.controller.recoveryMode) {
|
||||||
|
logger.warn(`Recovery | rejected host connection attempt for ${this.remoteAddr(req)}`);
|
||||||
|
wsRejectedConnectionsCounter.inc();
|
||||||
|
socket.close(lib.ConnectionClosed.RecoveryMode, "Connection Refused: Controller in recovery mode");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let tokenPayload = jwt.verify(
|
let tokenPayload = jwt.verify(
|
||||||
data.token,
|
data.token,
|
||||||
|
|||||||
@@ -120,6 +120,10 @@ async function startHost() {
|
|||||||
type: "boolean", nargs: 0, default: false,
|
type: "boolean", nargs: 0, default: false,
|
||||||
describe: "Indicate that a process monitor will restart this host on failure",
|
describe: "Indicate that a process monitor will restart this host on failure",
|
||||||
});
|
});
|
||||||
|
yargs.option("recovery", {
|
||||||
|
type: "boolean", nargs: 0, default: false,
|
||||||
|
describe: "Start the host in recovery mode with all plugins disabled and controller disconnected",
|
||||||
|
});
|
||||||
})
|
})
|
||||||
.demandCommand(1, "You need to specify a command to run")
|
.demandCommand(1, "You need to specify a command to run")
|
||||||
.strict()
|
.strict()
|
||||||
@@ -144,6 +148,9 @@ async function startHost() {
|
|||||||
let command = args._[0];
|
let command = args._[0];
|
||||||
if (command === "run") {
|
if (command === "run") {
|
||||||
logger.info(`Starting Clusterio host ${version}`);
|
logger.info(`Starting Clusterio host ${version}`);
|
||||||
|
if (args.recovery) {
|
||||||
|
logger.warn("Host recovery mode enabled. Some features will be disabled.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`Loading available plugins from ${args.pluginList}`);
|
logger.info(`Loading available plugins from ${args.pluginList}`);
|
||||||
@@ -226,6 +233,7 @@ async function startHost() {
|
|||||||
tlsCa,
|
tlsCa,
|
||||||
pluginInfos,
|
pluginInfos,
|
||||||
Boolean(args.canRestart),
|
Boolean(args.canRestart),
|
||||||
|
Boolean(args.recovery),
|
||||||
...await Host.bootstrap(hostConfig)
|
...await Host.bootstrap(hostConfig)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -298,6 +298,11 @@ export default class Host extends lib.Link {
|
|||||||
* will restart this host on a non-zero exit codes.
|
* will restart this host on a non-zero exit codes.
|
||||||
*/
|
*/
|
||||||
public canRestart = false,
|
public canRestart = false,
|
||||||
|
/**
|
||||||
|
* If true indicates the host is in recovery mode and should
|
||||||
|
* disable certain actions such as loading plugins or instance autostart
|
||||||
|
*/
|
||||||
|
public recoveryMode = false,
|
||||||
public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
|
public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()),
|
||||||
) {
|
) {
|
||||||
super(connector);
|
super(connector);
|
||||||
@@ -403,6 +408,11 @@ export default class Host extends lib.Link {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this.recoveryMode) {
|
||||||
|
logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let HostPluginClass = BaseHostPlugin;
|
let HostPluginClass = BaseHostPlugin;
|
||||||
try {
|
try {
|
||||||
if (pluginInfo.hostEntrypoint) {
|
if (pluginInfo.hostEntrypoint) {
|
||||||
@@ -1080,6 +1090,10 @@ export default class Host extends lib.Link {
|
|||||||
|
|
||||||
for (let [instanceId, instanceInfo] of this.instanceInfos) {
|
for (let [instanceId, instanceInfo] of this.instanceInfos) {
|
||||||
if (instanceInfo.config.get("instance.auto_start")) {
|
if (instanceInfo.config.get("instance.auto_start")) {
|
||||||
|
if (this.recoveryMode) {
|
||||||
|
logger.warn(`Recovery | skipping auto startup for ${instanceInfo.config.get("instance.name")}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
let instanceConnection = await this._connectInstance(instanceId);
|
let instanceConnection = await this._connectInstance(instanceId);
|
||||||
await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
|
await instanceConnection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
|
||||||
|
|||||||
@@ -646,6 +646,11 @@ end`.replace(/\r?\n/g, " ");
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this._host.recoveryMode) {
|
||||||
|
this.logger.warn(`Recovery | force disabled plugin ${pluginInfo.name}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this._loadPlugin(pluginInfo, this._host);
|
await this._loadPlugin(pluginInfo, this._host);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ export enum ConnectionClosed {
|
|||||||
ProtocolError = 1002, // Protocol not followed
|
ProtocolError = 1002, // Protocol not followed
|
||||||
PolicyViolation = 1008, // Generic code for any endpoint policy
|
PolicyViolation = 1008, // Generic code for any endpoint policy
|
||||||
InternalError = 1011, // Endpoint failed to fulfil request
|
InternalError = 1011, // Endpoint failed to fulfil request
|
||||||
|
TryAgainLater = 1013, // Temporary server condition blocking requests
|
||||||
|
|
||||||
// Codes after 3000 are available for frameworks
|
// Codes after 3000 are available for frameworks
|
||||||
// However they should be registered to IANA
|
// However they should be registered to IANA
|
||||||
@@ -32,6 +33,7 @@ export enum ConnectionClosed {
|
|||||||
|
|
||||||
// Codes after 4000 are available for applications
|
// Codes after 4000 are available for applications
|
||||||
MalformedMessage = 4000,
|
MalformedMessage = 4000,
|
||||||
|
RecoveryMode = 4001,
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in new issue
Block a user