Limited requests count to 10 in 1 minute

Afgan0r · Afgan0r · commit 093acc1cfa8c · 2026-02-12T19:14:32.000+07:00
diff --git a/src/!tests/unit-tests/0 - utils/request.test.ts b/src/!tests/unit-tests/0 - utils/request.test.ts
@@ -0,0 +1,86 @@
+import fetch from 'node-fetch';
+
+import getProxiedRequest from '../../../0 - utils/getProxiedRequest';
+import request, {
+  getSgZoneRequestQueueState,
+  waitForSgZoneRequestQueueToDrain,
+} from '../../../0 - utils/request';
+
+jest.mock('node-fetch', () => jest.fn());
+jest.mock('../../../0 - utils/getProxiedRequest', () => jest.fn());
+
+const mockedFetch = fetch as jest.MockedFunction<typeof fetch>;
+const mockedGetProxiedRequest = getProxiedRequest as jest.MockedFunction<typeof getProxiedRequest>;
+const minuteInMs = 60 * 1000;
+let fakeNow = new Date('2026-01-01T00:00:00.000Z').getTime();
+
+beforeEach(() => {
+  jest.useFakeTimers({ doNotFake: ['performance'] });
+  jest.setSystemTime(new Date(fakeNow));
+  fakeNow += minuteInMs * 2;
+  mockedFetch.mockReset();
+  mockedGetProxiedRequest.mockReset();
+  mockedGetProxiedRequest.mockResolvedValue(null);
+});
+
+afterEach(() => {
+  jest.useRealTimers();
+});
+
+test('should limit sg.zone requests to 10 per minute', async () => {
+  mockedFetch.mockResolvedValue({ status: 200 } as Response);
+
+  const pendingRequests = Array.from({ length: 11 }, (_item, index) => (
+    request(`https://sg.zone/replays?p=${index + 1}`)
+  ));
+  const firstTenRequests = pendingRequests.slice(0, 10);
+  const eleventhRequest = pendingRequests[10];
+  let isEleventhRequestResolved = false;
+
+  eleventhRequest.then(() => {
+    isEleventhRequestResolved = true;
+  });
+
+  await expect(Promise.all(firstTenRequests)).resolves.toHaveLength(10);
+  expect(mockedFetch).toHaveBeenCalledTimes(10);
+  expect(isEleventhRequestResolved).toBe(false);
+
+  jest.advanceTimersByTime(59_999);
+  await Promise.resolve();
+
+  expect(mockedFetch).toHaveBeenCalledTimes(10);
+  expect(isEleventhRequestResolved).toBe(false);
+
+  jest.advanceTimersByTime(1);
+  await expect(eleventhRequest).resolves.toEqual({ status: 200 });
+  expect(mockedFetch).toHaveBeenCalledTimes(11);
+});
+
+test('should wait until sg.zone request queue is drained', async () => {
+  jest.useRealTimers();
+  mockedFetch.mockResolvedValue({ status: 200 } as Response);
+  const pendingRequest = request('https://sg.zone/replays?p=1');
+
+  const waitForDrainPromise = waitForSgZoneRequestQueueToDrain();
+  let isQueueDrained = false;
+
+  waitForDrainPromise.then(() => {
+    isQueueDrained = true;
+  });
+
+  await Promise.resolve();
+  await Promise.resolve();
+
+  expect(isQueueDrained).toBe(false);
+
+  await expect(pendingRequest).resolves.toEqual({ status: 200 });
+  await expect(waitForDrainPromise).resolves.toBeUndefined();
+});
+
+test('should expose sg.zone queue state for diagnostics', () => {
+  const queueState = getSgZoneRequestQueueState();
+
+  expect(queueState.pending).toBeGreaterThanOrEqual(0);
+  expect(queueState.active).toBeGreaterThanOrEqual(0);
+  expect(queueState.total).toBe(queueState.pending + queueState.active);
+});
diff --git a/src/0 - utils/request.ts b/src/0 - utils/request.ts
@@ -4,12 +4,101 @@ import getProxiedRequest from './getProxiedRequest';
 import logger from './logger';
 
 const defaultRetryCount = 3;
+const requestsPerMinuteLimit = 10;
+const minuteInMs = 60 * 1000;
+const sgZoneHost = 'sg.zone';
+const requestTimestamps: number[] = [];
+let slotReservationQueue: Promise<void> = Promise.resolve();
+let pendingSgZoneRequests = 0;
+let activeSgZoneRequests = 0;
+
+const getSgZoneRequestQueueState = () => ({
+  pending: pendingSgZoneRequests,
+  active: activeSgZoneRequests,
+  total: pendingSgZoneRequests + activeSgZoneRequests,
+});
+
+const sleep = async (ms: number): Promise<void> => (
+  new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  })
+);
+
+const isSgZoneRequest = (url: string): boolean => {
+  try {
+    const parsedUrl = new URL(url);
+
+    return parsedUrl.protocol === 'https:' && parsedUrl.hostname === sgZoneHost;
+  } catch (err) {
+    return false;
+  }
+};
+
+const clearExpiredTimestamps = (now: number): void => {
+  while (
+    requestTimestamps.length > 0
+    && (now - requestTimestamps[0]) >= minuteInMs
+  ) {
+    requestTimestamps.shift();
+  }
+};
+
+const waitForSgZoneRequestSlot = async (): Promise<void> => {
+  const now = Date.now();
+
+  clearExpiredTimestamps(now);
+
+  if (requestTimestamps.length < requestsPerMinuteLimit) {
+    requestTimestamps.push(now);
+
+    return;
+  }
+
+  const oldestRequestTimestamp = requestTimestamps[0];
+  const waitTime = minuteInMs - (now - oldestRequestTimestamp);
+
+  await sleep(waitTime);
+
+  await waitForSgZoneRequestSlot();
+};
+
+const reserveSgZoneRequestSlot = async (): Promise<void> => {
+  const slotReservation = slotReservationQueue.then(waitForSgZoneRequestSlot);
+
+  slotReservationQueue = slotReservation.catch(() => undefined);
+
+  await slotReservation;
+};
+
+const waitForSgZoneRequestQueueToDrain = async (): Promise<void> => {
+  if (getSgZoneRequestQueueState().total === 0) return;
+
+  await sleep(20);
+
+  await waitForSgZoneRequestQueueToDrain();
+};
 
 const request = async (
   url: string,
   retryCount: number = defaultRetryCount,
 ): Promise<Response | null> => {
+  const isSgZoneUrl = isSgZoneRequest(url);
+  let isSgZoneRequestActive = false;
+
   try {
+    if (isSgZoneUrl) {
+      pendingSgZoneRequests += 1;
+
+      try {
+        await reserveSgZoneRequestSlot();
+      } finally {
+        pendingSgZoneRequests -= 1;
+      }
+
+      activeSgZoneRequests += 1;
+      isSgZoneRequestActive = true;
+    }
+
     const proxiedResponse = await getProxiedRequest(url);
 
     if (proxiedResponse) return proxiedResponse;
@@ -26,8 +115,19 @@ const request = async (
       throw err;
     }
 
+    if (isSgZoneRequestActive) {
+      activeSgZoneRequests -= 1;
+      isSgZoneRequestActive = false;
+    }
+
     return await request(url, retryCount - 1);
+  } finally {
+    if (isSgZoneRequestActive) {
+      activeSgZoneRequests -= 1;
+    }
   }
 };
 
+export { getSgZoneRequestQueueState, waitForSgZoneRequestQueueToDrain };
+
 export default request;
diff --git a/src/schedule.ts b/src/schedule.ts
@@ -6,21 +6,61 @@ import startParsingReplays from '.';
 import generateBasicFolders from './0 - utils/generateBasicFolders';
 import logger from './0 - utils/logger';
 import { tempResultsPath } from './0 - utils/paths';
+import {
+  getSgZoneRequestQueueState,
+  waitForSgZoneRequestQueueToDrain,
+} from './0 - utils/request';
 import generateMaceList from './jobs/generateMaceListHTML';
 import generateMissionMakersList from './jobs/generateMissionMakersList';
 import startFetchingReplays from './jobs/prepareReplaysList';
 
 generateBasicFolders();
 
+let siteRequestJobsQueue: Promise<void> = Promise.resolve();
+
+const formatQueueStateForLog = () => {
+  const queueState = getSgZoneRequestQueueState();
+
+  return `pending=${queueState.pending}, active=${queueState.active}, total=${queueState.total}`;
+};
+
+const runSiteRequestJob = async (
+  jobName: string,
+  jobCallback: () => Promise<void>,
+): Promise<void> => {
+  const queuedJob = siteRequestJobsQueue.then(
+    async () => {
+      const waitStartTimestamp = Date.now();
+
+      logger.info(`[schedule][${jobName}] Waiting for sg.zone queue to drain. ${formatQueueStateForLog()}`);
+      await waitForSgZoneRequestQueueToDrain();
+      logger.info(
+        `[schedule][${jobName}] Queue drained after ${Date.now() - waitStartTimestamp}ms. ${formatQueueStateForLog()}. Starting job.`,
+      );
+      await jobCallback();
+      logger.info(`[schedule][${jobName}] Job finished. ${formatQueueStateForLog()}`);
+    },
+  );
+
+  siteRequestJobsQueue = queuedJob.catch(() => undefined);
+
+  await queuedJob;
+};
+
 Cron(
-  '*/5 * * * *',
+  '5 */2 * * *',
   { protect: true },
   async () => {
-    try {
-      await generateMissionMakersList();
-    } catch (err) {
-      logger.error(`Error during fetching mission makers list. Trace: ${err.stack}`);
-    }
+    await runSiteRequestJob(
+      'generateMissionMakersList',
+      async () => {
+        try {
+          await generateMissionMakersList();
+        } catch (err) {
+          logger.error(`Error during fetching mission makers list. Trace: ${err.stack}`);
+        }
+      },
+    );
   },
 );
 
@@ -33,16 +73,21 @@ const generateMaceListJob = async () => {
 };
 
 const replaysFetcherJob = Cron(
-  '*/5 * * * *',
+  '5 */2 * * *',
   { protect: true },
   async () => {
-    try {
-      await startFetchingReplays();
-    } catch (err) {
-      logger.error(`Error during fetching replays list. Trace: ${err.stack}`);
-    }
+    await runSiteRequestJob(
+      'startFetchingReplays',
+      async () => {
+        try {
+          await startFetchingReplays();
+        } catch (err) {
+          logger.error(`Error during fetching replays list. Trace: ${err.stack}`);
+        }
 
-    generateMaceListJob();
+        generateMaceListJob();
+      },
+    );
   },
 );
 
@@ -58,7 +103,7 @@ const waitReplaysFetchingToFinish = async (): Promise<void> => (
 );
 
 Cron(
-  '4 */1 * * *',
+  '0 1-23/2 * * *',
   { protect: true },
   async () => {
     if (replaysFetcherJob.isBusy()) {
