Support running PS in a single process again (#11331)

Author: Slayer95

config/config-example.js

@@ -15,23 +15,6 @@ exports.port = 8000;
  */
 exports.bindaddress = '0.0.0.0';
 
-/**
- * workers - the number of networking child processes to spawn
- *   This should be no greater than the number of threads available on your
- *   server's CPU. If you're not sure how many you have, you can check from a
- *   terminal by running:
- *
- *   $ node -e "console.log(require('os').cpus().length)"
- *
- *   Using more workers than there are available threads will cause performance
- *   issues. Keeping a couple threads available for use for OS-related work and
- *   other PS processes will likely give you the best performance, if your
- *   server's CPU is capable of multithreading. If you don't know what any of
- *   this means or you are unfamiliar with PS' networking code, leave this set
- *   to 1.
- */
-exports.workers = 1;
-
 /**
  * wsdeflate - compresses WebSocket messages
  *  Toggles use of the Sec-WebSocket-Extension permessage-deflate extension.
@@ -92,6 +75,50 @@ Main's SSL deploy script from Let's Encrypt looks like:
  */
 exports.proxyip = false;
 
+// subprocesses - the number of child processes to use for various tasks.
+//   Can be set to `0` instead of `{...}` to stop using subprocesses, if you're running out of RAM.
+exports.subprocesses = {
+	/**
+	 * network - the number of networking child processes to spawn
+	 *   This should be no greater than the number of threads available on your
+	 *   server's CPU. If you're not sure how many you have, you can check from a
+	 *   terminal by running:
+	 *
+	 *   $ node -e "console.log(require('os').cpus().length)"
+	 *
+	 *   Using more workers than there are available threads will cause performance
+	 *   issues. Keeping a couple threads available for use for OS-related work and
+	 *   other PS processes will likely give you the best performance, if your
+	 *   server's CPU is capable of multithreading. If you don't know what any of
+	 *   this means or you are unfamiliar with PS' networking code, leave this set
+	 *   to 1.
+	 */
+	network: 1,
+	/**
+	 * for simulating battles
+	 *   You should leave this at 1 unless your server has a very large
+	 *   amount of traffic (i.e. hundreds of concurrent battles).
+	 */
+	simulator: 1,
+
+	// beyond this point, it'd be very weird if you needed more than one of each of these
+
+	/** for validating teams */
+	validator: 1,
+	/** for user authentication */
+	verifier: 1,
+	localartemis: 1,
+	remoteartemis: 1,
+	friends: 1,
+	chatdb: 1,
+	modlog: 1,
+	pm: 1,
+	/** for the battlesearch chat plugin */
+	battlesearch: 1,
+	/** datasearch - for the datasearch chat plugin */
+	datasearch: 1,
+};
+
 /**
  * Various debug options
  *
@@ -401,15 +428,6 @@ exports.logchallenges = false;
  */
 exports.loguserstats = 1000 * 60 * 10; // 10 minutes
 
-/**
- * validatorprocesses - the number of processes to use for validating teams
- * simulatorprocesses - the number of processes to use for handling battles
- * You should leave both of these at 1 unless your server has a very large
- * amount of traffic (i.e. hundreds of concurrent battles).
- */
-exports.validatorprocesses = 1;
-exports.simulatorprocesses = 1;
-
 /**
  * inactiveuserthreshold - how long a user must be inactive before being pruned
  * from the `users` array. The default is 1 hour.

server/artemis/local.ts

@@ -173,5 +173,5 @@ if (require.main === module) {
 	// eslint-disable-next-line no-eval
 	Repl.start(`abusemonitor-local-${process.pid}`, cmd => eval(cmd));
 } else if (!process.send) {
-	PM.spawn(Config.localartemisprocesses || 1);
+	PM.spawn(global.Config?.subprocessescache?.localartemis ?? 1);
 }

server/artemis/remote.ts

@@ -123,7 +123,7 @@ if (require.main === module) {
 	// eslint-disable-next-line no-eval
 	Repl.start(`abusemonitor-remote-${process.pid}`, cmd => eval(cmd));
 } else if (!process.send) {
-	PM.spawn(Config.remoteartemisprocesses || 1);
+	PM.spawn(global.Config?.subprocessescache?.remoteartemis ?? 1);
 }
 
 export class RemoteClassifier {

server/chat-plugins/battlesearch.ts

@@ -26,7 +26,6 @@ interface BattleSearchResults {
 	timesBattled: { [k: string]: number };
 }
 
-const MAX_BATTLESEARCH_PROCESSES = 1;
 export async function runBattleSearch(userids: ID[], month: string, tierid: ID, turnLimit?: number) {
 	const useRipgrep = await checkRipgrepAvailability();
 	const pathString = `${month}/${tierid}/`;
@@ -499,5 +498,5 @@ if (!PM.isParentProcess) {
 	// eslint-disable-next-line no-eval
 	Repl.start('battlesearch', cmd => eval(cmd));
 } else {
-	PM.spawn(MAX_BATTLESEARCH_PROCESSES);
+	PM.spawn(global.Config?.subprocessescache?.battlesearch ?? 1);
 }

server/chat-plugins/datasearch.ts

@@ -50,7 +50,6 @@ interface MoveOrGroup {
 
 type Direction = 'less' | 'greater' | 'equal';
 
-const MAX_PROCESSES = 1;
 const RESULTS_MAX_LENGTH = 10;
 const MAX_RANDOM_RESULTS = 30;
 const dexesHelpMods = Object.keys((global.Dex?.dexes || {})).filter(x => x !== 'sourceMaps').join('</code>, <code>');
@@ -3119,7 +3118,7 @@ if (!PM.isParentProcess) {
 	// eslint-disable-next-line no-eval
 	require('../../lib/repl').Repl.start('dexsearch', (cmd: string) => eval(cmd));
 } else {
-	PM.spawn(MAX_PROCESSES);
+	PM.spawn(global.Config?.subprocessescache?.datasearch ?? 1);
 }
 
 export const testables = {

server/chat.ts

@@ -1821,7 +1821,7 @@ export const Chat = new class {
 	 */
 	database = SQL(module, {
 		file: global.Config?.nofswriting ? ':memory:' : PLUGIN_DATABASE_PATH,
-		processes: global.Config?.chatdbprocesses,
+		processes: global.Config?.subprocessescache?.chatdb ?? 1,
 	});
 	databaseReadyPromise: Promise<void> | null = null;
 
@@ -2708,7 +2708,7 @@ export interface Monitor {
 
 // explicitly check this so it doesn't happen in other child processes
 if (!process.send) {
-	Chat.database.spawn(Config.chatdbprocesses || 1);
+	Chat.database.spawn(global.Config?.subprocessescache?.chatdb ?? 1);
 	Chat.databaseReadyPromise = Chat.prepareDatabase();
 	// we need to make sure it is explicitly JUST the child of the original parent db process
 	// no other child processes

server/config-loader.ts

@@ -9,17 +9,36 @@ import * as defaults from '../config/config-example';
 import type { GroupInfo, EffectiveGroupSymbol } from './user-groups';
 import { ProcessManager, FS } from '../lib';
 
-export type ConfigType = typeof defaults & {
+type DefaultConfig = typeof defaults;
+
+type InputConfig = Omit<DefaultConfig, 'subprocesses'> & {
+	subprocesses: null | 0 | 1 | SubProcessesConfig,
+};
+
+type ProcessType = (
+	'localartemis' | 'remoteartemis' | 'battlesearch' | 'datasearch' | 'friends' |
+	'chatdb' | 'pm' | 'modlog' | 'network' | 'simulator' | 'validator' | 'verifier'
+);
+
+type SubProcessesConfig = Partial<Record<ProcessType, number>>;
+
+export type ConfigType = InputConfig & {
 	groups: { [symbol: string]: GroupInfo },
 	groupsranking: EffectiveGroupSymbol[],
 	greatergroupscache: { [combo: string]: GroupSymbol },
+	subprocessescache: SubProcessesConfig,
 	[k: string]: any,
 };
 /** Map<process flag, config settings for it to turn on> */
 const FLAG_PRESETS = new Map([
 	['--no-security', ['nothrottle', 'noguestsecurity', 'noipchecks']],
 ]);
 
+const processTypes: ProcessType[] = [
+	'localartemis', 'remoteartemis', 'battlesearch', 'datasearch', 'friends',
+	'chatdb', 'pm', 'modlog', 'network', 'simulator', 'validator', 'verifier',
+];
+
 const CONFIG_PATH = FS('./config/config.js').path;
 
 export function load(invalidate = false) {
@@ -28,12 +47,25 @@ export function load(invalidate = false) {
 	// config.routes is nested - we need to ensure values are set for its keys as well.
 	config.routes = { ...defaults.routes, ...config.routes };
 
-	// Automatically stop startup if better-sqlite3 isn't installed and SQLite is enabled
-	if (config.usesqlite) {
-		try {
-			require('better-sqlite3');
-		} catch {
-			throw new Error(`better-sqlite3 is not installed or could not be loaded, but Config.usesqlite is enabled.`);
+	if (!process.send) {
+		// Automatically stop startup if optional dependencies are enabled yet missing
+		if (config.usesqlite) {
+			try {
+				require.resolve('better-sqlite3');
+			} catch {
+				throw new Error(`better-sqlite3 is not installed or could not be loaded, but Config.usesqlite is enabled.`);
+			}
+		}
+
+		if (config.ofemain) {
+			try {
+				require.resolve('node-oom-heapdump');
+			} catch {
+				throw new Error(
+					`node-oom-heapdump is not installed, but it is a required dependency if Config.ofemain is set to true! ` +
+					`Run npm install node-oom-heapdump and restart the server.`
+				);
+			}
 		}
 	}
 
@@ -43,18 +75,57 @@ export function load(invalidate = false) {
 		}
 	}
 
+	cacheSubProcesses(config);
 	cacheGroupData(config);
 	return config;
 }
 
+function cacheSubProcesses(config: ConfigType) {
+	if (config.subprocesses !== undefined) {
+		// Leniently accept all other falsy values, including `null`.
+		const value = config.subprocesses || 0;
+		if (value === 0 || value === 1) {
+			// https://github.com/microsoft/TypeScript/issues/35745
+			config.subprocessescache = (Object.fromEntries(
+				processTypes.map(k => [k, value])
+			) as Record<ProcessType, number>);
+		} else if (typeof value === 'object') {
+			config.subprocessescache = value;
+		} else {
+			reportError(`Invalid \`subprocesses\` specification. Use any of 0, 1, or a plain old object.`);
+		}
+	}
+	config.subprocessescache ??= {};
+	const deprecatedKeys = [];
+	if ('workers' in config) {
+		deprecatedKeys.push('workers');
+		config.subprocessescache.network = config.workers;
+	}
+	for (const processType of processTypes) {
+		if (processType === 'network') continue;
+		const compatKey = `${processType}processes`;
+		if (compatKey in config) {
+			deprecatedKeys.push(compatKey);
+			config.subprocessescache[processType] = config[compatKey];
+		}
+	}
+	for (const compatKey of deprecatedKeys) {
+		reportError(
+			`You are using \`${compatKey}\`, which is deprecated\n` +
+			`Support for this may be removed.\n` +
+			`Please ensure that you update your config.js to use \`subprocesses\` (see config-example.js, line 80).\n`
+		);
+	}
+}
+
 export function cacheGroupData(config: ConfigType) {
 	if (config.groups) {
 		// Support for old config groups format.
 		// Should be removed soon.
 		reportError(
 			`You are using a deprecated version of user group specification in config.\n` +
-			`Support for this will be removed soon.\n` +
-			`Please ensure that you update your config.js to the new format (see config-example.js, line 457).\n`
+			`Support for this may be removed.\n` +
+			`Please ensure that you update your config.js to the new format (see config-example.js, line 521).\n`
 		);
 	} else {
 		config.punishgroups = Object.create(null);
@@ -162,6 +233,6 @@ function reportError(msg: string) {
 	// This module generally loads before Monitor, so we put this in a setImmediate to wait for it to load.
 	// Most child processes don't have Monitor.error, but the main process should always have them, and Config
 	// errors should always be the same across processes, so this is a neat way to avoid unnecessary logging.
-	setImmediate(() => global.Monitor?.error?.(msg));
+	setImmediate(() => global.Monitor?.error?.(`[CONFIG] ${msg}`));
 }
 export const Config = load();

server/friends.ts

@@ -454,5 +454,5 @@ if (require.main === module) {
 		Repl.start(`friends-${process.pid}`, cmd => eval(cmd));
 	}
 } else if (!process.send) {
-	PM.spawn(Config.friendsprocesses || 1);
+	PM.spawn(global.Config?.subprocessescache?.friends ?? 1);
 }

server/index.ts

@@ -131,7 +131,6 @@ function setupGlobals() {
 
 	const Verifier = require('./verifier');
 	global.Verifier = Verifier;
-	Verifier.PM.spawn();
 
 	const { Tournaments } = require('./tournaments');
 	global.Tournaments = Tournaments;
@@ -157,7 +156,7 @@ if (Config.crashguard) {
  * Start networking processes to be connected to
  *********************************************************/
 
-import { Sockets } from './sockets';
+const { Sockets } = require('./sockets');
 global.Sockets = Sockets;
 
 export function listen(port: number, bindAddress: string, workerCount: number) {
@@ -182,9 +181,8 @@ if (require.main === module) {
  * Set up our last global
  *********************************************************/
 
-import * as TeamValidatorAsync from './team-validator-async';
+const { TeamValidatorAsync } = require('./team-validator-async');
 global.TeamValidatorAsync = TeamValidatorAsync;
-TeamValidatorAsync.PM.spawn();
 
 /*********************************************************
  * Start up the REPL server
@@ -202,16 +200,6 @@ if (Config.startuphook) {
 }
 
 if (Config.ofemain) {
-	try {
-		require.resolve('node-oom-heapdump');
-	} catch (e: any) {
-		if (e.code !== 'MODULE_NOT_FOUND') throw e; // should never happen
-		throw new Error(
-			'node-oom-heapdump is not installed, but it is a required dependency if Config.ofe is set to true! ' +
-			'Run npm install node-oom-heapdump and restart the server.'
-		);
-	}
-
 	// Create a heapdump if the process runs out of memory.
 	global.nodeOomHeapdump = (require as any)('node-oom-heapdump')({
 		addTimestamp: true,

server/modlog/index.ts

@@ -113,7 +113,7 @@ export class Modlog {
 
 		if (Config.usesqlite) {
 			if (this.database.isParentProcess) {
-				this.database.spawn(Config.modlogprocesses || 1);
+				this.database.spawn(global.Config?.subprocessescache?.modlog ?? 1);
 			} else {
 				global.Monitor = {
 					crashlog(error: Error, source = 'A modlog child process', details: AnyObject | null = null) {