diff --git a/README.md b/README.md index 43e66e9..dc0916d 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,9 @@ rpClient.checkConnect().then(() => { | launchUuidPrint | Optional | false | Whether to print the current launch UUID. | | launchUuidPrintOutput | Optional | 'STDOUT' | Launch UUID printing output. Possible values: 'STDOUT', 'STDERR', 'FILE', 'ENVIRONMENT'. Works only if `launchUuidPrint` set to `true`. File format: `rp-launch-uuid-${launch_uuid}.tmp`. Env variable: `RP_LAUNCH_UUID`. | | skippedIsNotIssue | Optional | False | ReportPortal provides feature to mark skipped tests as not 'To Investigate'. Option could be equal boolean values: `true` - skipped tests will not be marked as 'To Investigate' on application. `false` - skipped tests considered as issues and will be marked as 'To Investigate' on application. | +| batchLogs | Optional | false | Enable batch logging to optimize network requests by grouping multiple logs into a single request. When enabled, logs are accumulated and sent in batches instead of individually. | +| batchLogsSize | Optional | 20 | Maximum number of logs in a single batch. Only applicable when `batchLogs` is `true`. Logs are sent when this limit is reached. | +| batchPayloadLimit | Optional | 67108864 | Maximum batch payload size in bytes (default: 64MB). Only applicable when `batchLogs` is `true`. Logs are sent when adding a new log would exceed this limit. | ### HTTP client options @@ -592,6 +595,45 @@ The method takes three arguments: | type | Required | | The file mimeType, example 'image/png' (support types: 'image/*', application/['xml', 'javascript', 'json', 'css', 'php'], other formats will be opened in reportportal in a new browser tab only). | | content | Required | | base64 encoded file content. | +### flushLogs + +`flushLogs` - manually flushes all pending logs from the batch queue. This method is only relevant when batch logging is enabled (`batchLogs: true`). + +```javascript +// Enable batch logging in client configuration +const rpClient = new RPClient({ + apiKey: 'reportportalApiKey', + endpoint: 'http://your-instance.com:8080/api/v1', + launch: 'LAUNCH_NAME', + project: 'PROJECT_NAME', + batchLogs: true, + batchLogsSize: 20 +}); + +// Send multiple logs (they will be batched) +rpClient.sendLog(stepObj.tempId, { + level: 'INFO', + message: 'First log', + time: rpClient.helpers.now() +}); + +rpClient.sendLog(stepObj.tempId, { + level: 'INFO', + message: 'Second log', + time: rpClient.helpers.now() +}); + +// Manually flush remaining logs before finishing +await rpClient.flushLogs(); +``` + +**Note:** When batch logging is enabled, logs are automatically sent when: +- The batch reaches `batchLogsSize` (default: 20 logs) +- The batch payload exceeds `batchPayloadLimit` (default: 64MB) +- `finishLaunch()` is called (automatic flush) + +You typically don't need to call `flushLogs()` manually unless you want to ensure logs are sent at a specific point in your test execution. + ### mergeLaunches `mergeLaunches` - merges already completed runs into one (useful when running tests in multiple threads on the same machine). diff --git a/__tests__/config.spec.js b/__tests__/config.spec.js index d566182..42acc51 100644 --- a/__tests__/config.spec.js +++ b/__tests__/config.spec.js @@ -109,5 +109,33 @@ describe('Config commons test suite', () => { expect(console.dir).toHaveBeenCalledWith(new ReportPortalRequiredOptionError('apiKey')); }); + + it('should fall back to defaults for invalid batch options', () => { + const config = getClientConfig({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + batchLogsSize: 0, + batchPayloadLimit: -1, + }); + + expect(config.batchLogsSize).toBe(20); + expect(config.batchPayloadLimit).toBe(64 * 1024 * 1024); + }); + + it('should fall back to defaults for non-number batch options', () => { + const config = getClientConfig({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + batchLogsSize: 'abc', + batchPayloadLimit: 'def', + }); + + expect(config.batchLogsSize).toBe(20); + expect(config.batchPayloadLimit).toBe(64 * 1024 * 1024); + }); }); }); diff --git a/__tests__/log-batcher.spec.js b/__tests__/log-batcher.spec.js new file mode 100644 index 0000000..02075cf --- /dev/null +++ b/__tests__/log-batcher.spec.js @@ -0,0 +1,350 @@ +const LogBatcher = require('../lib/logs/batcher'); +const { MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE } = require('../lib/logs/constants'); + +describe('LogBatcher', () => { + describe('constructor', () => { + it('should create instance with default values', () => { + const batcher = new LogBatcher(); + + expect(batcher.entryNum).toBe(MAX_LOG_BATCH_SIZE); + expect(batcher.payloadLimit).toBe(MAX_LOG_BATCH_PAYLOAD_SIZE); + expect(batcher.batchSize).toBe(0); + expect(batcher.currentPayloadSize).toBe(0); + }); + + it('should create instance with custom values', () => { + const batcher = new LogBatcher(5, 1024); + + expect(batcher.entryNum).toBe(5); + expect(batcher.payloadLimit).toBe(1024); + }); + }); + + describe('append', () => { + it('should add log to batch and return null when not full', () => { + const batcher = new LogBatcher(3, 10000); + + const logReq = { + payload: { message: 'Test log', level: 'info' }, + file: null, + }; + + const result = batcher.append(logReq); + + expect(result).toBeNull(); + expect(batcher.batchSize).toBe(1); + }); + + it('should return batch when entry limit is reached', () => { + const batcher = new LogBatcher(2, 10000); + + const logReq1 = { + payload: { message: 'Test log 1', level: 'info' }, + file: null, + }; + const logReq2 = { + payload: { message: 'Test log 2', level: 'info' }, + file: null, + }; + + const result1 = batcher.append(logReq1); + expect(result1).toBeNull(); + + const result2 = batcher.append(logReq2); + expect(result2).toBeInstanceOf(Array); + expect(result2).toHaveLength(2); + expect(result2[0]).toBe(logReq1); + expect(result2[1]).toBe(logReq2); + expect(batcher.batchSize).toBe(0); + }); + + it('should return batch when payload limit is exceeded', () => { + const batcher = new LogBatcher(10, 300); // Small payload limit + + const logReq1 = { + payload: { message: 'Small log', level: 'info' }, + file: null, + }; + const logReq2 = { + payload: { + message: 'Large log with lots of content to exceed the payload limit', + level: 'info', + }, + file: null, + }; + + const result1 = batcher.append(logReq1); + expect(result1).toBeNull(); + + const result2 = batcher.append(logReq2); + expect(result2).toBeInstanceOf(Array); + expect(result2).toHaveLength(1); + expect(result2[0]).toBe(logReq1); + expect(batcher.batchSize).toBe(1); + }); + + it('should handle logs with file attachments', () => { + const batcher = new LogBatcher(3, 10000); + + const logReq = { + payload: { message: 'Test log with file', level: 'info' }, + file: { + name: 'test.png', + type: 'image/png', + content: Buffer.from('fake image content').toString('base64'), + }, + }; + + const result = batcher.append(logReq); + + expect(result).toBeNull(); + expect(batcher.batchSize).toBe(1); + }); + }); + + describe('oversized log handling', () => { + it('should return existing batch when adding an oversized log', () => { + jest.spyOn(console, 'warn').mockImplementation(); + const batcher = new LogBatcher(10, 300); + + const normalLog = { + payload: { message: 'Small log', level: 'info' }, + file: null, + }; + + const oversizedLog = { + payload: { + message: 'x'.repeat(1000), // This will exceed the 300 byte limit + level: 'info', + }, + file: null, + }; + + // Add a normal log first + const result1 = batcher.append(normalLog); + expect(result1).toBeNull(); + expect(batcher.batchSize).toBe(1); + + // Add an oversized log - should return the existing batch and queue the oversized log + const result2 = batcher.append(oversizedLog); + expect(console.warn).toHaveBeenCalledWith( + expect.stringContaining('exceeds payload limit'), + ); + expect(result2).toBeInstanceOf(Array); + expect(result2).toHaveLength(1); + expect(result2[0]).toBe(normalLog); + expect(batcher.batchSize).toBe(1); // oversized log is queued for next flush + }); + + it('should return oversized log as single-item batch when batch is empty', () => { + const batcher = new LogBatcher(10, 300); + + const oversizedLog = { + payload: { + message: 'x'.repeat(1000), // This will exceed the 300 byte limit + level: 'info', + }, + file: null, + }; + + // Add an oversized log to empty batch + const result = batcher.append(oversizedLog); + expect(result).toBeInstanceOf(Array); + expect(result).toHaveLength(1); + expect(result[0]).toBe(oversizedLog); + expect(batcher.batchSize).toBe(0); + }); + + it('should handle consecutive oversized logs correctly', () => { + const batcher = new LogBatcher(10, 300); + + const oversizedLog1 = { + payload: { message: 'x'.repeat(1000), level: 'info' }, + file: null, + }; + + const oversizedLog2 = { + payload: { message: 'y'.repeat(1000), level: 'info' }, + file: null, + }; + + // First oversized log should be returned immediately + const result1 = batcher.append(oversizedLog1); + expect(result1).toBeInstanceOf(Array); + expect(result1).toHaveLength(1); + expect(result1[0]).toBe(oversizedLog1); + + // Second oversized log should also be returned immediately + const result2 = batcher.append(oversizedLog2); + expect(result2).toBeInstanceOf(Array); + expect(result2).toHaveLength(1); + expect(result2[0]).toBe(oversizedLog2); + + expect(batcher.batchSize).toBe(0); + }); + }); + + describe('flush', () => { + it('should return null when batch is empty', () => { + const batcher = new LogBatcher(); + + const result = batcher.flush(); + + expect(result).toBeNull(); + }); + + it('should return current batch and clear it', () => { + const batcher = new LogBatcher(10, 10000); + + const logReq1 = { + payload: { message: 'Test log 1', level: 'info' }, + file: null, + }; + const logReq2 = { + payload: { message: 'Test log 2', level: 'info' }, + file: null, + }; + + batcher.append(logReq1); + batcher.append(logReq2); + + const result = batcher.flush(); + + expect(result).toBeInstanceOf(Array); + expect(result).toHaveLength(2); + expect(result[0]).toBe(logReq1); + expect(result[1]).toBe(logReq2); + expect(batcher.batchSize).toBe(0); + expect(batcher.currentPayloadSize).toBe(0); + }); + + it('should return null on second flush call', () => { + const batcher = new LogBatcher(10, 10000); + + const logReq = { + payload: { message: 'Test log', level: 'info' }, + file: null, + }; + + batcher.append(logReq); + batcher.flush(); + + const result = batcher.flush(); + + expect(result).toBeNull(); + }); + }); + + describe('batch accumulation', () => { + it('should accumulate multiple batches correctly', () => { + const batcher = new LogBatcher(2, 10000); + + const logReqs = [ + { payload: { message: 'Log 1' }, file: null }, + { payload: { message: 'Log 2' }, file: null }, + { payload: { message: 'Log 3' }, file: null }, + { payload: { message: 'Log 4' }, file: null }, + ]; + + const batch1 = batcher.append(logReqs[0]); + expect(batch1).toBeNull(); + + const batch2 = batcher.append(logReqs[1]); + expect(batch2).toHaveLength(2); + + const batch3 = batcher.append(logReqs[2]); + expect(batch3).toBeNull(); + + const batch4 = batcher.append(logReqs[3]); + expect(batch4).toHaveLength(2); + }); + }); + + describe('edge cases', () => { + it('should handle empty payload', () => { + const batcher = new LogBatcher(); + + const logReq = { + payload: {}, + file: null, + }; + + const result = batcher.append(logReq); + + expect(result).toBeNull(); + expect(batcher.batchSize).toBe(1); + }); + + it('should handle very large single log', () => { + const batcher = new LogBatcher(10, 100); + + const largeLogReq = { + payload: { + message: 'x'.repeat(1000), + }, + file: null, + }; + + const result = batcher.append(largeLogReq); + + // Should return immediately as single-item batch + expect(result).toBeInstanceOf(Array); + expect(result).toHaveLength(1); + expect(batcher.batchSize).toBe(0); + }); + + it('should handle file content as Buffer', () => { + const batcher = new LogBatcher(3, 10000); + + const logReq = { + payload: { message: 'Test log with Buffer', level: 'info' }, + file: { + name: 'test.txt', + type: 'text/plain', + content: Buffer.from('test content'), + }, + }; + + const result = batcher.append(logReq); + + expect(result).toBeNull(); + expect(batcher.batchSize).toBe(1); + }); + + it('should handle file content as string', () => { + const batcher = new LogBatcher(3, 10000); + + const logReq = { + payload: { message: 'Test log with string', level: 'info' }, + file: { + name: 'test.txt', + type: 'text/plain', + content: 'test content as string', + }, + }; + + const result = batcher.append(logReq); + + expect(result).toBeNull(); + expect(batcher.batchSize).toBe(1); + }); + + it('should handle file content as array-like object', () => { + const batcher = new LogBatcher(3, 10000); + + const logReq = { + payload: { message: 'Test log with array-like', level: 'info' }, + file: { + name: 'test.txt', + type: 'text/plain', + content: [1, 2, 3, 4, 5], + }, + }; + + const result = batcher.append(logReq); + + expect(result).toBeNull(); + expect(batcher.batchSize).toBe(1); + }); + }); +}); diff --git a/__tests__/report-portal-client.spec.js b/__tests__/report-portal-client.spec.js index e0ced6e..946d6a8 100644 --- a/__tests__/report-portal-client.spec.js +++ b/__tests__/report-portal-client.spec.js @@ -1120,4 +1120,207 @@ describe('ReportPortal javascript client', () => { expect(result.catch).toBeDefined(); }); }); + + describe('batch logging', () => { + it('should send batch when logBatcher returns a full batch', async () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + }); + client.launchUuid = 'launchUuid'; + const itemObj = { + promiseStart: Promise.resolve({ id: 'itemId' }), + children: [], + }; + // When logBatcher.append returns a non-null batch, it means the batch is full + const batch = [{ payload: { message: 'test' } }]; + jest.spyOn(client.logBatcher, 'append').mockReturnValue(batch); + jest.spyOn(client, 'sendLogBatch').mockResolvedValue({ success: true }); + jest.spyOn(client, 'getUniqId').mockReturnValue('logTempId'); + + const result = client.sendLogViaBatcher(itemObj, { message: 'test' }, null); + await result.promise; + + expect(client.logBatcher.append).toHaveBeenCalledWith({ + payload: expect.objectContaining({ + message: 'test', + launchUuid: 'launchUuid', + itemUuid: 'itemId', + }), + file: null, + }); + expect(client.sendLogBatch).toHaveBeenCalledWith(batch); + expect(result.tempId).toEqual('logTempId'); + }); + + it('should queue log when batch is not full', async () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + }); + client.launchUuid = 'launchUuid'; + const itemObj = { + promiseStart: Promise.resolve({ id: 'itemId' }), + children: [], + }; + // When logBatcher.append returns null, it means the batch is not full yet + jest.spyOn(client.logBatcher, 'append').mockReturnValue(null); + jest.spyOn(client, 'sendLogBatch').mockResolvedValue({ success: true }); + jest.spyOn(client, 'getUniqId').mockReturnValue('logTempId'); + + const result = client.sendLogViaBatcher(itemObj, { message: 'test' }, null); + await result.promise; + + expect(client.sendLogBatch).not.toHaveBeenCalled(); + expect(result.tempId).toEqual('logTempId'); + }); + + it('should send batch to server with correct parameters', async () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + }); + const batch = [{ payload: { message: 'test1' } }, { payload: { message: 'test2' } }]; + jest.spyOn(client, 'buildBatchMultiPartStream').mockReturnValue(Buffer.from('data')); + jest.spyOn(client.restClient, 'create').mockResolvedValue({ success: true }); + + await client.sendLogBatch(batch); + + expect(client.buildBatchMultiPartStream).toHaveBeenCalledWith(batch, expect.any(String)); + expect(client.restClient.create).toHaveBeenCalledWith( + 'log', + expect.any(Buffer), + expect.objectContaining({ + headers: expect.objectContaining({ + 'Content-Type': expect.stringContaining('multipart/form-data; boundary='), + }), + }), + ); + }); + + it('should return early when batch is empty', async () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + }); + jest.spyOn(client.restClient, 'create'); + + await client.sendLogBatch([]); + + expect(client.restClient.create).not.toHaveBeenCalled(); + }); + + it('should flush pending logs', async () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + }); + const batch = [{ payload: { message: 'test' } }]; + jest.spyOn(client.logBatcher, 'flush').mockReturnValue(batch); + jest.spyOn(client, 'sendLogBatch').mockResolvedValue({ success: true }); + + await client.flushLogs(); + + expect(client.logBatcher.flush).toHaveBeenCalled(); + expect(client.sendLogBatch).toHaveBeenCalledWith(batch); + }); + + it('should resolve immediately when flushing without batcher', async () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: false, + }); + jest.spyOn(client, 'sendLogBatch'); + + const result = await client.flushLogs(); + + expect(result).toBeUndefined(); + expect(client.sendLogBatch).not.toHaveBeenCalled(); + }); + + it('should use batcher when enabled', () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + batchLogs: true, + }); + client.map = { + itemTempId: { + promiseStart: Promise.resolve({ id: 'itemId' }), + children: [], + }, + }; + jest.spyOn(client, 'sendLogViaBatcher').mockReturnValue({ tempId: 'logId', promise: Promise.resolve() }); + + const result = client.sendLogWithoutFile('itemTempId', { message: 'test' }); + + expect(client.sendLogViaBatcher).toHaveBeenCalledWith( + client.map.itemTempId, + { message: 'test' }, + null, + ); + expect(result.tempId).toEqual('logId'); + expect(result.promise).toBeDefined(); + }); + + it('should build multipart stream with file attachments', () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + }); + const logRequests = [ + { + payload: { message: 'test1', launchUuid: 'uuid', file: { name: 'test.png' } }, + file: { name: 'test.png', content: 'base64content', type: 'image/png' }, + }, + ]; + + const result = client.buildBatchMultiPartStream(logRequests, 'boundary123'); + + expect(result).toBeInstanceOf(Buffer); + expect(result.toString()).toContain('test.png'); + expect(result.toString()).toContain('boundary123'); + expect(result.toString()).toContain('Content-Type: image/png'); + }); + + it('should build multipart stream with multiple logs', () => { + const client = new RPClient({ + apiKey: 'test', + project: 'test', + endpoint: 'https://abc.com', + }); + const logRequests = [ + { + payload: { message: 'test1', launchUuid: 'uuid' }, + file: null, + }, + { + payload: { message: 'test2', launchUuid: 'uuid', file: { name: 'screenshot.png' } }, + file: { name: 'screenshot.png', content: 'base64data', type: 'image/png' }, + }, + ]; + + const result = client.buildBatchMultiPartStream(logRequests, 'boundary456'); + + expect(result).toBeInstanceOf(Buffer); + const resultStr = result.toString(); + expect(resultStr).toContain('test1'); + expect(resultStr).toContain('test2'); + expect(resultStr).toContain('screenshot.png'); + }); + }); }); diff --git a/index.d.ts b/index.d.ts index 8c7e993..0933f80 100644 --- a/index.d.ts +++ b/index.d.ts @@ -160,6 +160,21 @@ declare module '@reportportal/client-javascript' { * OAuth 2.0 configuration object. When provided, OAuth authentication will be used instead of API key. */ oauth?: OAuthConfig; + /** + * Enable batch logging to optimize network requests by grouping multiple logs into a single request. + * @default false + */ + batchLogs?: boolean; + /** + * Maximum number of logs in a single batch. + * @default 20 + */ + batchLogsSize?: number; + /** + * Maximum batch payload size in bytes. + * @default 67108864 (64MB) + */ + batchPayloadLimit?: number; } /** @@ -389,6 +404,43 @@ declare module '@reportportal/client-javascript' { */ sendLog(itemId: string, options: LogOptions): Promise; + /** + * Flushes all pending logs from the batch queue. + * This method is only relevant when batch logging is enabled (batchLogs: true). + * It sends any accumulated logs immediately, without waiting for the batch to fill. + * + * @returns Promise that resolves when all pending logs are sent + * + * @example + * ```typescript + * // Enable batch logging in config + * const rpClient = new ReportPortalClient({ + * endpoint: 'https://your.reportportal.server/api/v1', + * project: 'your_project_name', + * apiKey: 'your_api_key', + * batchLogs: true, + * batchLogsSize: 20 + * }); + * + * // Send multiple logs (they will be batched) + * await rpClient.sendLog(testItem.tempId, { + * level: 'INFO', + * message: 'First log', + * time: rpClient.helpers.now() + * }); + * + * await rpClient.sendLog(testItem.tempId, { + * level: 'INFO', + * message: 'Second log', + * time: rpClient.helpers.now() + * }); + * + * // Manually flush remaining logs before finishing + * await rpClient.flushLogs(); + * ``` + */ + flushLogs(): Promise; + /** * Waits for all test items to be finished. * @example diff --git a/lib/commons/config.js b/lib/commons/config.js index 0700b92..0c6a6a7 100644 --- a/lib/commons/config.js +++ b/lib/commons/config.js @@ -1,5 +1,6 @@ const { ReportPortalRequiredOptionError, ReportPortalValidationError } = require('./errors'); const { OUTPUT_TYPES } = require('../constants/outputs'); +const { MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE } = require('../logs/constants'); const getOption = (options, optionName, defaultValue) => { if (!Object.prototype.hasOwnProperty.call(options, optionName) || !options[optionName]) { @@ -63,6 +64,15 @@ const getOAuthConfig = (options) => { }; }; +const getPositiveNumberOption = (options, optionName, defaultValue) => { + if (!Object.prototype.hasOwnProperty.call(options, optionName)) { + return defaultValue; + } + + const value = options[optionName]; + return Number.isFinite(value) && value > 0 ? value : defaultValue; +}; + const getClientConfig = (options) => { let calculatedOptions = options; try { @@ -116,6 +126,13 @@ const getClientConfig = (options) => { launchUuidPrint: options.launchUuidPrint, launchUuidPrintOutput, skippedIsNotIssue: !!options.skippedIsNotIssue, + batchLogs: options.batchLogs ?? false, + batchLogsSize: getPositiveNumberOption(options, 'batchLogsSize', MAX_LOG_BATCH_SIZE), + batchPayloadLimit: getPositiveNumberOption( + options, + 'batchPayloadLimit', + MAX_LOG_BATCH_PAYLOAD_SIZE, + ), }; } catch (error) { // don't throw the error up to not break the entire process diff --git a/lib/logs/batcher.js b/lib/logs/batcher.js new file mode 100644 index 0000000..bc7ac88 --- /dev/null +++ b/lib/logs/batcher.js @@ -0,0 +1,106 @@ +const { calculateMultipartSize } = require('./helpers'); +const { MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE } = require('./constants'); + +/** + * Batches log entries to optimize network requests. + * Accumulates logs until reaching size or count limits, then returns complete batches. + */ +class LogBatcher { + /** + * Creates a new LogBatcher instance. + * @param {number} entryNum - maximum number of log entries per batch + * @param {number} payloadLimit - maximum payload size in bytes per batch + */ + constructor(entryNum = MAX_LOG_BATCH_SIZE, payloadLimit = MAX_LOG_BATCH_PAYLOAD_SIZE) { + this.entryNum = entryNum; + this.payloadLimit = payloadLimit; + this.batch = []; + this.payloadSize = 0; + } + + /** + * Internal method to append a log request with known size. + * @private + * @param {number} size - calculated size of the log request in bytes + * @param {Object} logReq - log request object with payload and optional file + * @returns {Array|null} batch array if ready to send, null otherwise + */ + appendInternal(size, logReq) { + if (size >= this.payloadLimit) { + console.warn( + `Single log entry size (${size} bytes) exceeds payload limit (${this.payloadLimit} bytes).`, + ); + if (this.batch.length > 0) { + const { batch } = this; + this.batch = [logReq]; + this.payloadSize = size; + return batch; + } + return [logReq]; + } + + if (this.payloadSize + size >= this.payloadLimit) { + if (this.batch.length > 0) { + const { batch } = this; + this.batch = [logReq]; + this.payloadSize = size; + return batch; + } + } + + this.batch.push(logReq); + this.payloadSize += size; + + if (this.batch.length < this.entryNum) { + return null; + } + + const { batch } = this; + this.batch = []; + this.payloadSize = 0; + return batch; + } + + /** + * Appends a log request to the batch. + * @param {Object} logReq - log request with payload and optional file + * @returns {Array|null} batch if ready, null otherwise + */ + append(logReq) { + const size = calculateMultipartSize(logReq.payload, logReq.file); + return this.appendInternal(size, logReq); + } + + /** + * Flushes all pending log requests in the current batch. + * @returns {Array|null} batch array if there are pending logs, null if empty + */ + flush() { + if (this.batch.length <= 0) { + return null; + } + + const { batch } = this; + this.batch = []; + this.payloadSize = 0; + return batch; + } + + /** + * Gets the current number of log entries in the batch. + * @returns {number} number of pending log entries + */ + get batchSize() { + return this.batch.length; + } + + /** + * Gets the current payload size of the batch in bytes. + * @returns {number} current payload size in bytes + */ + get currentPayloadSize() { + return this.payloadSize; + } +} + +module.exports = LogBatcher; diff --git a/lib/logs/constants.js b/lib/logs/constants.js new file mode 100644 index 0000000..ca8a01f --- /dev/null +++ b/lib/logs/constants.js @@ -0,0 +1,7 @@ +const MAX_LOG_BATCH_SIZE = 20; +const MAX_LOG_BATCH_PAYLOAD_SIZE = 64 * 1024 * 1024; + +module.exports = { + MAX_LOG_BATCH_SIZE, + MAX_LOG_BATCH_PAYLOAD_SIZE, +}; diff --git a/lib/logs/helpers.js b/lib/logs/helpers.js new file mode 100644 index 0000000..f390fe8 --- /dev/null +++ b/lib/logs/helpers.js @@ -0,0 +1,64 @@ +/** + * Calculates multipart request size including boundaries and headers. + * @param {Object} payload - JSON payload object + * @returns {number} estimated size in bytes + */ +function calculateJsonPartSize(payload) { + if (!payload) { + return 0; + } + + const jsonString = JSON.stringify(payload); + const jsonSize = Buffer.byteLength(jsonString, 'utf8'); + // Overhead for multipart boundaries and headers + const boundaryOverhead = 100; + return jsonSize + boundaryOverhead; +} + +/** + * Calculates the size of a file part in a multipart request. + * @param {Object} file - file object with content and metadata + * @param {Buffer|string|ArrayBuffer} file.content - file content + * @param {string} file.name - file name + * @returns {number} estimated size in bytes including multipart overhead + */ +function calculateFilePartSize(file) { + if (!file || !file.content) { + return 0; + } + + let fileSize = 0; + + if (Buffer.isBuffer(file.content)) { + fileSize = file.content.length; + } else if (typeof file.content === 'string') { + fileSize = Buffer.byteLength(file.content, 'utf8'); + } else if (file.content.length !== undefined) { + fileSize = file.content.length; + } + + // Overhead for multipart boundaries, headers, and filename + const boundaryOverhead = 150; + return fileSize + boundaryOverhead; +} + +/** + * Calculates the total size of a multipart request for a log entry. + * @param {Object} payload - JSON payload object for the log + * @param {Object} [file] - optional file attachment object + * @returns {number} total estimated size in bytes + */ +function calculateMultipartSize(payload, file) { + const jsonSize = calculateJsonPartSize(payload); + const fileSize = calculateFilePartSize(file); + // Final boundary overhead (--boundary--\r\n) + const finalBoundaryOverhead = 50; + + return jsonSize + fileSize + finalBoundaryOverhead; +} + +module.exports = { + calculateJsonPartSize, + calculateFilePartSize, + calculateMultipartSize, +}; diff --git a/lib/logs/index.js b/lib/logs/index.js new file mode 100644 index 0000000..3f2a95b --- /dev/null +++ b/lib/logs/index.js @@ -0,0 +1,16 @@ +const LogBatcher = require('./batcher'); +const { + calculateJsonPartSize, + calculateFilePartSize, + calculateMultipartSize, +} = require('./helpers'); +const { MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE } = require('./constants'); + +module.exports = { + LogBatcher, + calculateJsonPartSize, + calculateFilePartSize, + calculateMultipartSize, + MAX_LOG_BATCH_SIZE, + MAX_LOG_BATCH_PAYLOAD_SIZE, +}; diff --git a/lib/report-portal-client.js b/lib/report-portal-client.js index 289aa76..c7baaf2 100644 --- a/lib/report-portal-client.js +++ b/lib/report-portal-client.js @@ -7,6 +7,7 @@ const { getClientConfig } = require('./commons/config'); const Statistics = require('../statistics/statistics'); const { EVENT_NAME } = require('../statistics/constants'); const { RP_STATUSES } = require('./constants/statuses'); +const { LogBatcher } = require('./logs'); const MULTIPART_BOUNDARY = Math.floor(Math.random() * 10000000000).toString(); @@ -65,6 +66,10 @@ class RPClient { this.launchUuid = ''; this.itemRetriesChainMap = new Map(); this.itemRetriesChainKeyMapByTempId = new Map(); + + if (this.config.batchLogs) { + this.logBatcher = new LogBatcher(this.config.batchLogsSize, this.config.batchPayloadLimit); + } } // eslint-disable-next-line valid-jsdoc @@ -276,28 +281,41 @@ class RPClient { const finishExecutionData = { endTime: this.helpers.now(), ...finishExecutionRQ }; launchObj.finishSend = true; + + const doFinishLaunch = () => { + launchObj.promiseStart.then( + () => { + this.logDebug(`Finish launch with tempId ${launchTempId}`, finishExecutionData); + const url = ['launch', launchObj.realId, 'finish'].join('/'); + this.restClient.update(url, finishExecutionData).then( + (response) => { + this.logDebug(`Success finish launch with tempId ${launchTempId}`, response); + console.log(`\nReportPortal Launch Link: ${response.link}`); + launchObj.resolveFinish(response); + }, + (error) => { + this.logDebug(`Error finish launch with tempId ${launchTempId}`, error); + console.dir(error); + launchObj.rejectFinish(error); + }, + ); + }, + (error) => { + console.dir(error); + launchObj.rejectFinish(error); + }, + ); + }; + Promise.all(launchObj.children.map((itemId) => this.map[itemId].promiseFinish)).then( () => { - launchObj.promiseStart.then( + this.flushLogs().then( () => { - this.logDebug(`Finish launch with tempId ${launchTempId}`, finishExecutionData); - const url = ['launch', launchObj.realId, 'finish'].join('/'); - this.restClient.update(url, finishExecutionData).then( - (response) => { - this.logDebug(`Success finish launch with tempId ${launchTempId}`, response); - console.log(`\nReportPortal Launch Link: ${response.link}`); - launchObj.resolveFinish(response); - }, - (error) => { - this.logDebug(`Error finish launch with tempId ${launchTempId}`, error); - console.dir(error); - launchObj.rejectFinish(error); - }, - ); + doFinishLaunch(); }, - (error) => { - console.dir(error); - launchObj.rejectFinish(error); + (flushError) => { + console.dir(flushError); + doFinishLaunch(); }, ); }, @@ -741,6 +759,10 @@ class RPClient { ); } + if (this.logBatcher) { + return this.sendLogViaBatcher(itemObj, saveLogRQ, null); + } + const requestPromise = (itemUuid, launchUuid) => { const url = 'log'; const isItemUuid = itemUuid !== launchUuid; @@ -782,6 +804,10 @@ class RPClient { ); } + if (this.logBatcher) { + return this.sendLogViaBatcher(itemObj, saveLogRQ, fileObj); + } + const requestPromise = (itemUuid, launchUuid) => { const isItemUuid = itemUuid !== launchUuid; @@ -794,6 +820,62 @@ class RPClient { return this.saveLog(itemObj, requestPromise); } + /** + * Send log via batcher (batch logging mode). + * @private + * @param {Object} itemObj - item object from the map + * @param {Object} saveLogRQ - log request data + * @param {Object} fileObj - file object (optional) + * @returns {Object} - an object which contains a tempId and a promise + */ + sendLogViaBatcher(itemObj, saveLogRQ, fileObj) { + const tempId = this.getUniqId(); + + this.map[tempId] = this.getNewItemObj((resolve, reject) => { + itemObj.promiseStart.then((itemResponse) => { + const itemUuid = itemResponse.id; + const isItemUuid = itemUuid !== this.launchUuid; + + const payload = { + ...saveLogRQ, + launchUuid: this.launchUuid, + ...(isItemUuid && { itemUuid }), + }; + + if (fileObj) { + payload.file = { name: fileObj.name }; + } + + const logRequest = { + payload, + file: fileObj, + }; + + const batch = this.logBatcher.append(logRequest); + + if (batch) { + this.sendLogBatch(batch).then(resolve, reject); + } else { + resolve({ queued: true }); + } + }, reject); + }); + + itemObj.children.push(tempId); + + const logObj = this.map[tempId]; + logObj.finishSend = true; + logObj.promiseStart.then( + (response) => logObj.resolveFinish(response), + (error) => logObj.rejectFinish(error), + ); + + return { + tempId, + promise: this.map[tempId].promiseFinish, + }; + } + getRequestLogWithFile(saveLogRQ, fileObj) { const url = 'log'; // eslint-disable-next-line no-param-reassign @@ -858,6 +940,99 @@ class RPClient { return Buffer.concat(buffers); } + /** + * Build multipart stream for batch logs. + * @private + * @param {Array} logRequests - array of log request objects + * @param {string} boundary - multipart boundary + * @returns {Buffer} - multipart stream buffer + */ + buildBatchMultiPartStream(logRequests, boundary) { + const eol = '\r\n'; + const bx = `--${boundary}`; + const buffers = []; + + // Collect all payloads into a single JSON array + const payloads = logRequests.map((logReq) => logReq.payload); + + // Add single JSON part containing array of all log payloads + buffers.push( + Buffer.from( + `${bx}${eol}Content-Disposition: form-data; name="json_request_part"${eol}Content-Type: application/json${eol}${eol}${eol}${JSON.stringify( + payloads, + )}${eol}`, + ), + ); + + // Add file parts for logs with attachments + logRequests.forEach((logReq) => { + const { file } = logReq; + + if (file && file.content) { + buffers.push( + Buffer.from( + `${bx}${eol}Content-Disposition: form-data; name="file"; filename="${file.name}"${eol}Content-Type: ${file.type}${eol}${eol}`, + ), + ); + buffers.push(Buffer.from(file.content, 'base64')); + buffers.push(Buffer.from(eol)); + } + }); + + // Add final boundary + buffers.push(Buffer.from(`${bx}--${eol}`)); + + return Buffer.concat(buffers); + } + + /** + * Send a batch of logs to ReportPortal. + * @private + * @param {Array} logBatch - array of log request objects + * @returns {Promise} - promise that resolves when batch is sent + */ + sendLogBatch(logBatch) { + if (!logBatch || logBatch.length === 0) { + return Promise.resolve(); + } + + const url = 'log'; + const boundary = Math.floor(Math.random() * 10000000000).toString(); + + this.logDebug(`Sending batch of ${logBatch.length} logs`); + + return this.restClient + .create(url, this.buildBatchMultiPartStream(logBatch, boundary), { + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + }, + }) + .then((response) => { + this.logDebug(`Successfully sent batch of ${logBatch.length} logs`, response); + return response; + }) + .catch((error) => { + this.logDebug(`Error sending log batch`, error); + console.dir(error); + throw error; + }); + } + + /** + * Flush all pending logs from the batcher. + * This is a public method that can be called to manually flush logs. + * + * @returns {Promise} - promise that resolves when all pending logs are sent + */ + flushLogs() { + if (!this.logBatcher) { + return Promise.resolve(); + } + + const batch = this.logBatcher.flush(); + return this.sendLogBatch(batch); + } + finishTestItemPromiseStart(itemObj, itemTempId, finishTestItemData) { itemObj.promiseStart.then( () => {