diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..627c962 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# If you're using Redis, enter the URL below (e.g. redis://127.0.0.1:6379) +REDIS_URL= \ No newline at end of file diff --git a/.gitignore b/.gitignore index 12ac647..4d9d37a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ node_modules/ -.DS_Store \ No newline at end of file +.DS_Store +dist/ +.env \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1a4ec01..40c2a4e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,9 @@ "license": "ISC", "dependencies": { "async-mutex": "^0.5.0", + "dotenv": "^17.2.3", "express": "^4.21.2", + "ioredis": "^5.8.2", "jsdom": "^26.0.0", "puppeteer": "^24.2.1" }, @@ -163,6 +165,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -185,10 +188,17 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } }, + "node_modules/@ioredis/commands": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz", + "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==", + "license": "MIT" + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -374,6 +384,7 @@ "integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.20.0" } @@ -699,9 +710,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -830,6 +841,15 @@ "node": ">=12" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1009,6 +1029,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1032,7 +1061,8 @@ "version": "0.0.1402036", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1402036.tgz", "integrity": "sha512-JwAYQgEvm3yD45CHB+RmF5kMbWtXBaOGwuxa87sZogHcLCv8c/IqnThaoQ1y60d7pXWjSKWQphPEc+1rAScVdg==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/diff": { "version": "4.0.2", @@ -1044,6 +1074,18 @@ "node": ">=0.3.1" } }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1148,6 +1190,21 @@ "node": ">= 0.4" } }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -1360,13 +1417,15 @@ } }, "node_modules/form-data": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", - "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", "mime-types": "^2.1.12" }, "engines": { @@ -1560,6 +1619,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -1713,6 +1787,53 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ioredis": { + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz", + "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.4.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/ip-address": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", @@ -1809,9 +1930,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -1878,6 +1999,18 @@ "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lru-cache": { "version": "7.18.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", @@ -2455,6 +2588,27 @@ "node": ">=8.10.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -2751,6 +2905,12 @@ "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", "license": "BSD-3-Clause" }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -2819,9 +2979,9 @@ "license": "MIT" }, "node_modules/tar-fs": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", - "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", + "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", "license": "MIT", "dependencies": { "pump": "^3.0.0", @@ -3001,6 +3161,7 @@ "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index 628d6ef..820ffc1 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,9 @@ "description": "", "dependencies": { "async-mutex": "^0.5.0", + "dotenv": "^17.2.3", "express": "^4.21.2", + "ioredis": "^5.8.2", "jsdom": "^26.0.0", "puppeteer": "^24.2.1" }, diff --git a/src/aceChecker.ts b/src/aceChecker.ts index aefde41..d4a8d6a 100644 --- a/src/aceChecker.ts +++ b/src/aceChecker.ts @@ -27,10 +27,13 @@ class PagePool { if (this.pages.length < this.maxSize) { const page = await this.browser.newPage(); + page.setDefaultNavigationTimeout(30000); + await page.setOfflineMode(true); await page.setRequestInterception(true); + page.on('request', (request) => { - if (['image', 'stylesheet', 'font', 'media'].includes(request.resourceType())) { + if (['image', 'stylesheet', 'font', 'media', 'xhr', 'fetch', 'websocket'].includes(request.resourceType())) { request.abort(); } else { request.continue(); @@ -102,13 +105,24 @@ export async function aceCheck(html: string, browser: puppeteer.Browser, guideli } const { page, hasScript } = await pagePool.getPage(); - let scriptAdded = false; try { - await page.setContent(html, { waitUntil: 'domcontentloaded' }); + try { + await page.setContent(html, { waitUntil: 'domcontentloaded', timeout: 5000 }); + } + catch (err: any) { + // sometimes a page takes too long to load, so retry it before we give up + if (err.name === 'TimeoutError') { + console.warn('Page failed to load in 5 seconds, trying with 60 seconds...'); + await page.setContent(html, { waitUntil: 'domcontentloaded', timeout: 60000 }); + } + else { + throw err; + } + } - let scriptAdded = hasScript; + scriptAdded = hasScript; if (!hasScript) { await page.addScriptTag({ path: require.resolve(acePath) @@ -160,6 +174,17 @@ export async function aceCheck(html: string, browser: puppeteer.Browser, guideli return combos.has(key); }); + // remove a few unneeded properties from the final JSON + delete report['nls']; + delete report['numExecuted']; + delete report['ruleTime']; + delete report['totalTime']; + + // remove ruleTime from each result + for (const result of report.results) { + delete result['ruleTime']; + } + return report; } finally { pagePool.releasePage(page, scriptAdded); diff --git a/src/server.ts b/src/server.ts index 662c5ca..d0a95f2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,10 @@ import { Request, Response, NextFunction } from 'express'; import { aceCheck, initializePagePool, closePagePool } from './aceChecker'; import bodyParser from 'body-parser'; import * as puppeteer from 'puppeteer'; +import Redis from 'ioredis'; +import dotenv from 'dotenv'; + +dotenv.config(); const app = express(); app.use(express.json({limit: '50mb'})); @@ -12,19 +16,16 @@ app.use(express.urlencoded({limit: '50mb', extended: true})); const PORT = process.env.PORT || 3000; const DEFAULT_ID = 'WCAG_2_1'; const DEFAULT_REPORT_LEVELS = ['violation', 'potentialviolation', 'manual']; - app.use(bodyParser.json()); - let browser: puppeteer.Browser; +const redis = new Redis(process.env.REDIS_URL || 'redis://localhost:6379'); +const SCAN_QUEUE = 'scan_queue'; + const asyncHandler = (fn: (req: Request, res: Response, next: NextFunction) => Promise) => (req: Request, res: Response, next: NextFunction) => Promise.resolve(fn(req, res, next)).catch(next); -app.get('/', (_req, res) => { - res.send('Hello, World!'); -}); - /** * The main scan endpoint that takes in the HTML content and the guideline IDs to scan against. * @param {string} html - The HTML content to scan. @@ -53,14 +54,71 @@ app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => { res.status(500).json({ error: err.message }); }); +async function pollRedisQueue() { + while (true) { + try { + // pop message from scan queue + const result = await redis.blpop(SCAN_QUEUE, 0); + const message = result ? result[1] : null; + + if (message) { + const { scanId, uuid, html, guidelineIds, reportLevels } = JSON.parse(message); + try { + const report: Report = await aceCheck(html, browser, guidelineIds || DEFAULT_ID, reportLevels || DEFAULT_REPORT_LEVELS); + const resultKey = `result:${scanId}:${uuid}`; + // set result in the Redis queue with an expiration time of 10 minutes + await redis.set( + resultKey, + JSON.stringify(report), + 'EX', 600 + ); + + console.log(`Stored result: ${resultKey}`); + } + catch (err: any) { + console.error("Error scanning page:", err); + if (err instanceof puppeteer.TimeoutError) { + console.log("Attempting to requeue..."); + await redis.rpush(SCAN_QUEUE, message); + } + await new Promise(resolve => setTimeout(resolve, 1000)); + } + } + } + catch (err) { + console.error("Error processing message from Redis queue:", err); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + } +} + (async () => { try { browser = await puppeteer.launch({ - args: ['--no-sandbox', '--disable-setuid-sandbox'] + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--no-first-run', + '--no-zygote', + '--single-process', + '--disable-gpu' + ], + headless: true, }); await initializePagePool(browser, 5); - app.listen(PORT, () => { - }); + + // if REDIS_URL is set, then only start redis queue polling + if (process.env.REDIS_URL) { + console.log("REDIS_URL set, starting Redis queue polling..."); + pollRedisQueue(); + } + else { + console.log(`Starting server on port ${PORT}...`); + app.listen(PORT, () => {}); + } + } catch (err) { console.error("Error launching Puppeteer:", err); process.exit(1);