Skip to content

Commit 8dbe19b

Browse files
committed
Do not create new logs folder each time worker is spawned
1 parent 9ecf164 commit 8dbe19b

9 files changed

Lines changed: 179 additions & 6 deletions

File tree

docs/architecture.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ Console output includes all levels with colorization (`colorize: true`, `coloriz
9494
2. In test environment (`NODE_ENV === 'test'`), transport is disabled — plain pino logger without file output.
9595
3. Log files are pre-created with `fs.ensureFileSync` to ensure target paths exist before async transport starts.
9696
4. Process-level handlers for `uncaughtException`, `unhandledRejection`, `SIGINT`, and `SIGTERM` write fatal logs before termination.
97+
5. Worker threads receive the log folder path from the main thread via `workerData.logsFolderPath`, so all workers write to the same log folder that was created at application startup instead of generating a new timestamped folder. The `workerData` shape is defined by a shared zod schema in `src/1 - replays/workers/workerData.ts`, which is used both for validation in the logger and for typing in `WorkerPool`.
9798

9899
## 6. Scheduler (Production Flow)
99100

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import path from 'path';
2+
3+
const logsPath = path.join(require('os').homedir(), 'sg_stats', 'logs');
4+
5+
const importLogsFolderPath = async (
6+
workerData: unknown,
7+
): Promise<string> => {
8+
let logsFolderPathValue: string = '';
9+
10+
jest.isolateModules(() => {
11+
jest.doMock('worker_threads', () => ({ workerData }));
12+
13+
// eslint-disable-next-line global-require
14+
const { logsFolderPath } = require('../../../0 - utils/logger') as { logsFolderPath: string };
15+
16+
logsFolderPathValue = logsFolderPath;
17+
});
18+
19+
return logsFolderPathValue;
20+
};
21+
22+
describe('logger logsFolderPath in worker context', () => {
23+
test('should use logsFolderPath from workerData when valid', async () => {
24+
const expectedPath = '/home/user/sg_stats/logs/17.02.2026 13:00';
25+
26+
const result = await importLogsFolderPath({ logsFolderPath: expectedPath });
27+
28+
expect(result).toBe(expectedPath);
29+
});
30+
31+
test('should not contain current timestamp when workerData has logsFolderPath', async () => {
32+
const mainThreadPath = '/home/user/sg_stats/logs/01.01.2020 00:00';
33+
34+
const result = await importLogsFolderPath({ logsFolderPath: mainThreadPath });
35+
36+
expect(result).toBe(mainThreadPath);
37+
expect(result).not.toContain(new Date().getFullYear().toString());
38+
});
39+
40+
test('should generate new path when workerData is null', async () => {
41+
const result = await importLogsFolderPath(null);
42+
43+
expect(result).not.toBe('');
44+
expect(result.startsWith(logsPath)).toBe(true);
45+
});
46+
47+
test('should generate new path when workerData is undefined', async () => {
48+
const result = await importLogsFolderPath(undefined);
49+
50+
expect(result).not.toBe('');
51+
expect(result.startsWith(logsPath)).toBe(true);
52+
});
53+
54+
test('should generate new path when workerData has invalid shape', async () => {
55+
const result = await importLogsFolderPath({ wrongField: 123 });
56+
57+
expect(result).not.toBe('');
58+
expect(result.startsWith(logsPath)).toBe(true);
59+
});
60+
61+
test('should generate new path when workerData has non-string logsFolderPath', async () => {
62+
const result = await importLogsFolderPath({ logsFolderPath: 42 });
63+
64+
expect(result).not.toBe('');
65+
expect(result.startsWith(logsPath)).toBe(true);
66+
});
67+
68+
test('should use exact workerData path without appending timestamp', async () => {
69+
const exactPath = '/custom/logs/fixed-folder';
70+
71+
const result = await importLogsFolderPath({ logsFolderPath: exactPath });
72+
73+
expect(result).toBe(exactPath);
74+
});
75+
});
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import { workerDataSchema, WorkerData } from '../../../../1 - replays/workers/workerData';
2+
3+
const satisfies = <T>(value: T): T => value;
4+
5+
describe('workerDataSchema', () => {
6+
test('should accept valid workerData', () => {
7+
const input = { logsFolderPath: '/home/user/sg_stats/logs/17.02.2026 13:00' };
8+
9+
const result = workerDataSchema.safeParse(input);
10+
11+
expect(result.success).toBe(true);
12+
13+
if (result.success) {
14+
expect(result.data.logsFolderPath).toBe(input.logsFolderPath);
15+
}
16+
});
17+
18+
test('should reject when logsFolderPath is missing', () => {
19+
const result = workerDataSchema.safeParse({});
20+
21+
expect(result.success).toBe(false);
22+
});
23+
24+
test('should reject when logsFolderPath is not a string', () => {
25+
const result = workerDataSchema.safeParse({ logsFolderPath: 123 });
26+
27+
expect(result.success).toBe(false);
28+
});
29+
30+
test('should reject null', () => {
31+
const result = workerDataSchema.safeParse(null);
32+
33+
expect(result.success).toBe(false);
34+
});
35+
36+
test('should reject undefined', () => {
37+
const result = workerDataSchema.safeParse(undefined);
38+
39+
expect(result.success).toBe(false);
40+
});
41+
42+
test('should strip extra fields', () => {
43+
const input = { logsFolderPath: '/logs/path', extra: 'value' };
44+
45+
const result = workerDataSchema.safeParse(input);
46+
47+
expect(result.success).toBe(true);
48+
});
49+
});
50+
51+
describe('WorkerData type', () => {
52+
test('should be assignable from valid object', () => {
53+
const data = satisfies<WorkerData>({
54+
logsFolderPath: '/home/user/sg_stats/logs/17.02.2026 13:00',
55+
});
56+
57+
expect(data.logsFolderPath).toBe('/home/user/sg_stats/logs/17.02.2026 13:00');
58+
});
59+
});

src/!tests/unit-tests/1 - replays, 2 - parseReplayInfo/workers/workerPool.test.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ jest.mock('worker_threads', () => ({
2323

2424
public terminate = jest.fn(async () => undefined);
2525

26-
constructor(scriptPath: string) {
26+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
27+
constructor(scriptPath: string, _options?: Record<string, unknown>) {
2728
super();
2829

2930
this.scriptPath = scriptPath;

src/!yearStatistics/index.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import fs from 'fs-extra';
55
import { dayjsUTC } from '../0 - utils/dayjs';
66
import generateBasicFolders from '../0 - utils/generateBasicFolders';
77
import { isInInterval } from '../0 - utils/isInInterval';
8-
import logger from '../0 - utils/logger';
8+
import logger, { logsFolderPath } from '../0 - utils/logger';
99
import { prepareNamesList } from '../0 - utils/namesHelper/prepareNamesList';
1010
import { yearResultsPath } from '../0 - utils/paths';
1111
import pipe from '../0 - utils/pipe';
@@ -42,6 +42,7 @@ import { printFinish } from './utils/printText';
4242
const workerPool = new WorkerPool({
4343
workerCount: 1,
4444
workerScriptPath: path.join(__dirname, '../1 - replays/workers/parseReplayWorker.js'),
45+
workerData: { logsFolderPath },
4546
});
4647

4748
logger.info(`Replays count: ${replays.length}`);

src/0 - utils/logger.ts

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,40 @@
11
import path from 'path';
2+
import { workerData as rawWorkerData } from 'worker_threads';
23

34
import fs from 'fs-extra';
45
import pino from 'pino';
56

7+
import { workerDataSchema } from '../1 - replays/workers/workerData';
68
import { dayjsUTC } from './dayjs';
79
import { dateFormat } from './namesHelper/utils/consts';
810
import { logsPath } from './paths';
911

12+
const parseWorkerData = (): string | null => {
13+
if (rawWorkerData === null || rawWorkerData === undefined) return null;
14+
15+
const result = workerDataSchema.safeParse(rawWorkerData);
16+
17+
return result.success ? result.data.logsFolderPath : null;
18+
};
19+
1020
const pinoPrettyDefaultOptions = { sync: true, colorize: true, colorizeObjects: true };
1121
const loggerLevel = process.env.LOG_LEVEL || 'debug';
1222

23+
const resolveLogsFolderPath = (): string => {
24+
const workerLogsFolderPath = parseWorkerData();
25+
26+
if (workerLogsFolderPath) {
27+
return workerLogsFolderPath;
28+
}
29+
30+
return path.join(logsPath, dayjsUTC().tz('Europe/Moscow').format(dateFormat));
31+
};
32+
33+
export const logsFolderPath = resolveLogsFolderPath();
34+
1335
const getTransport = () => {
1436
fs.ensureDirSync(logsPath);
1537

16-
const logsFolderPath = path.join(logsPath, dayjsUTC().tz('Europe/Moscow').format(dateFormat));
17-
1838
const debugFilePath = path.join(logsFolderPath, 'debug.log');
1939
const infoFilePath = path.join(logsFolderPath, 'info.log');
2040
const warningFilePath = path.join(logsFolderPath, 'warning.log');
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import z from 'zod';
2+
3+
export const workerDataSchema = z.object({
4+
logsFolderPath: z.string(),
5+
});
6+
7+
export type WorkerData = z.infer<typeof workerDataSchema>;

src/1 - replays/workers/workerPool.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import {
44
ParseReplayTaskMessage,
55
ParseReplayTaskResponseMessage,
66
} from './types';
7+
import { WorkerData } from './workerData';
78

89
type WorkerPoolConfig = {
910
workerCount: number;
1011
workerScriptPath: string;
12+
workerData?: WorkerData;
1113
};
1214

1315
type TaskWithoutId = Omit<ParseReplayTaskMessage, 'taskId'>;
@@ -30,6 +32,8 @@ const toError = (error: unknown): Error => {
3032
export class WorkerPool {
3133
private readonly workerScriptPath: string;
3234

35+
private readonly workerData: WorkerData | undefined;
36+
3337
private readonly workers: Worker[] = [];
3438

3539
private readonly queuedTasks: QueuedTask[] = [];
@@ -48,6 +52,7 @@ export class WorkerPool {
4852
}
4953

5054
this.workerScriptPath = config.workerScriptPath;
55+
this.workerData = config.workerData;
5156

5257
for (let index = 0; index < config.workerCount; index += 1) {
5358
this.spawnWorker();
@@ -93,7 +98,10 @@ export class WorkerPool {
9398
}
9499

95100
private spawnWorker(): void {
96-
const worker = new Worker(this.workerScriptPath);
101+
const worker = new Worker(
102+
this.workerScriptPath,
103+
{ workerData: this.workerData },
104+
);
97105

98106
this.workers.push(worker);
99107
this.workerActiveTaskId.set(worker, null);

src/index.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { dayjsUTC } from './0 - utils/dayjs';
77
import filterPlayersByTotalPlayedGames from './0 - utils/filterPlayersByTotalPlayedGames';
88
import formatGameType from './0 - utils/formatGameType';
99
import generateBasicFolders from './0 - utils/generateBasicFolders';
10-
import logger from './0 - utils/logger';
10+
import logger, { logsFolderPath } from './0 - utils/logger';
1111
import { prepareNamesList } from './0 - utils/namesHelper/prepareNamesList';
1212
import {
1313
commitParsingStatus,
@@ -64,6 +64,7 @@ const startParsingReplays = async () => {
6464
const workerPool = new WorkerPool({
6565
workerCount: getRuntimeConfig().workerCount,
6666
workerScriptPath: path.join(__dirname, '1 - replays/workers/parseReplayWorker.js'),
67+
workerData: { logsFolderPath },
6768
});
6869

6970
logger.info('Started parsing replays.');

0 commit comments

Comments
 (0)