diff --git a/handleRequest.test.ts b/handleRequest.test.ts index c26373e..5db8fcb 100644 --- a/handleRequest.test.ts +++ b/handleRequest.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { createRequestHandler, resolvePluginOrSchemaUrl } from "./handleRequest.js"; +import { createRequestHandler, resolvePluginOrSchemaUrl, rewriteGithubUrls } from "./handleRequest.js"; it("should get info.json", { timeout: 10_000 }, async () => { const { handleRequest } = createRequestHandler(); @@ -345,6 +345,17 @@ it("should not redirect wasm plugins", async () => { expect(response.status).not.toEqual(302); }); +it("should redirect non-allowed org asset to GitHub", async () => { + const { handleRequest } = createRequestHandler(); + const response = await handleRequest( + new Request("https://plugins.dprint.dev/someone/some-repo/0.1.0/asset/file.zip"), + ); + expect(response.status).toEqual(302); + expect(response.headers.get("location")).toEqual( + "https://github.com/someone/some-repo/releases/download/0.1.0/file.zip", + ); +}); + it("should return 404 for asset not found", async () => { const { handleRequest } = createRequestHandler(); const response = await handleRequest( @@ -353,6 +364,50 @@ it("should return 404 for asset not found", async () => { expect(response.status).toEqual(404); }); +it("rewriteGithubUrls", () => { + const githubUrl = "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/plugin.json"; + const origin = "https://plugins.dprint.dev"; + const json = JSON.stringify({ + "schemaVersion": 2, + "kind": "process", + "darwin-aarch64": { + "reference": + "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/dprint-plugin-exec-aarch64-apple-darwin.zip", + "checksum": "abc123", + }, + "linux-x86_64": { + "reference": + "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/dprint-plugin-exec-x86_64-unknown-linux-gnu.zip", + "checksum": "def456", + }, + }); + const body = new TextEncoder().encode(json).buffer as ArrayBuffer; + const result = rewriteGithubUrls(githubUrl, body, origin); + expect(typeof result).toEqual("string"); + const parsed = JSON.parse(result as string); + expect(parsed["darwin-aarch64"].reference).toEqual( + "https://plugins.dprint.dev/dprint/dprint-plugin-exec/0.6.0/asset/dprint-plugin-exec-aarch64-apple-darwin.zip", + ); + expect(parsed["linux-x86_64"].reference).toEqual( + "https://plugins.dprint.dev/dprint/dprint-plugin-exec/0.6.0/asset/dprint-plugin-exec-x86_64-unknown-linux-gnu.zip", + ); +}); + +it("rewriteGithubUrls should not rewrite non-plugin.json", () => { + const githubUrl = "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/schema.json"; + const body = new TextEncoder().encode("{}").buffer as ArrayBuffer; + const result = rewriteGithubUrls(githubUrl, body, "https://plugins.dprint.dev"); + expect(result).toBe(body); +}); + +it("rewriteGithubUrls should return original buffer when no URLs match", () => { + const githubUrl = "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/plugin.json"; + const json = JSON.stringify({ "key": "value" }); + const body = new TextEncoder().encode(json).buffer as ArrayBuffer; + const result = rewriteGithubUrls(githubUrl, body, "https://plugins.dprint.dev"); + expect(result).toBe(body); +}); + // todo: mock github api for these tests it("tryResolveSchemaUrl", async () => { diff --git a/handleRequest.ts b/handleRequest.ts index 2b878fb..8910f9f 100644 --- a/handleRequest.ts +++ b/handleRequest.ts @@ -1,6 +1,12 @@ import { renderHome } from "./home.jsx"; import oldMappings from "./old_redirects.json" with { type: "json" }; -import { tryResolveAssetUrl, tryResolveLatestJson, tryResolvePluginUrl, tryResolveSchemaUrl } from "./plugins.js"; +import { + isAssetAllowedRepo, + tryResolveAssetUrl, + tryResolveLatestJson, + tryResolvePluginUrl, + tryResolveSchemaUrl, +} from "./plugins.js"; import { readInfoFile } from "./readInfoFile.js"; import robotsTxt from "./robots.txt"; import styleCSS from "./style.css"; @@ -20,13 +26,16 @@ const contentTypes = { }; export function createRequestHandler() { - const memoryCache = new LruCache({ size: 50 }); + const memoryCache = new LruCache({ size: 50 }); return { async handleRequest(request: Request, ctx?: ExecutionContext) { const url = new URL(request.url); - const assetUrl = tryResolveAssetUrl(url); - if (assetUrl != null) { - return servePlugin(request, assetUrl, ctx); + const assetResult = tryResolveAssetUrl(url); + if (assetResult != null) { + if (!assetResult.shouldCache) { + return Response.redirect(assetResult.githubUrl, 302); + } + return servePlugin(request, url, assetResult.githubUrl, ctx); } const githubUrl = await resolvePluginOrSchemaUrl(url); @@ -36,10 +45,10 @@ export function createRequestHandler() { // plugin.json files resolve correctly const assetPath = githubUrlToAssetPath(githubUrl); if (assetPath != null) { - return Response.redirect(new URL(assetPath, url.origin).href, 302); + return Response.redirect(`${url.origin}${assetPath}`, 302); } } - return servePlugin(request, githubUrl, ctx); + return servePlugin(request, url, githubUrl, ctx); } const userLatestInfo = await tryResolveLatestJson(url); @@ -57,7 +66,7 @@ export function createRequestHandler() { } if (url.pathname.startsWith("/info.json")) { - const infoFileData = await readInfoFile(); + const infoFileData = await readInfoFile(url.origin); return createJsonResponse( JSON.stringify(infoFileData, null, 2), request, @@ -107,8 +116,8 @@ export function createRequestHandler() { }, }; - async function servePlugin(request: Request, githubUrl: string, ctx?: ExecutionContext) { - const result = await resolveBody(githubUrl, ctx); + async function servePlugin(request: Request, requestUrl: URL, githubUrl: string, ctx?: ExecutionContext) { + const result = await resolveBodyWithMemoryCache(githubUrl, requestUrl, ctx); return new Response(result.body, { headers: { "content-type": result.contentType, @@ -118,28 +127,46 @@ export function createRequestHandler() { }); } - async function resolveBody( + async function resolveBodyWithMemoryCache( githubUrl: string, + requestUrl: URL, ctx?: ExecutionContext, - ): Promise<{ body: ArrayBuffer | ReadableStream | null; status: number; contentType: string }> { - // L1: in-memory cache + ): Promise<{ body: ArrayBuffer | ReadableStream | string | null; status: number; contentType: string }> { + // L1: in-memory cache (already rewritten) const cached = memoryCache.get(githubUrl); if (cached != null) { return { body: cached.body, status: 200, contentType: cached.contentType }; } - // L2: R2 + const result = await fetchBody(githubUrl, requestUrl, ctx); + if (result.status !== 200 || !(result.body instanceof ArrayBuffer)) { + return result; + } + + // rewrite GitHub URLs in JSON files to use the local asset path + const body = rewriteGithubUrls(githubUrl, result.body, requestUrl.origin); + const size = typeof body === "string" ? body.length : body.byteLength; + if (size <= MAX_MEM_CACHE_BODY_SIZE) { + memoryCache.set(githubUrl, { body, contentType: result.contentType }); + } + + return { body, status: 200, contentType: result.contentType }; + } + + async function fetchBody( + githubUrl: string, + requestUrl: URL, + ctx?: ExecutionContext, + ): Promise<{ body: ArrayBuffer | ReadableStream | string | null; status: number; contentType: string }> { + // L2: R2 (stores original content) const r2Object = await r2Get(githubUrl); if (r2Object != null) { - const r2ContentType = r2Object.httpMetadata?.contentType ?? contentTypeForUrl(githubUrl); - // small enough for L1 — buffer and cache + const contentType = r2Object.httpMetadata?.contentType ?? contentTypeForUrl(githubUrl); if (r2Object.size <= MAX_MEM_CACHE_BODY_SIZE) { - const buffer = await r2Object.arrayBuffer(); - memoryCache.set(githubUrl, { body: buffer, contentType: r2ContentType }); - return { body: buffer, status: 200, contentType: r2ContentType }; + return { body: await r2Object.arrayBuffer(), status: 200, contentType }; } // large — stream directly without buffering - return { body: r2Object.body, status: 200, contentType: r2ContentType }; + return { body: r2Object.body, status: 200, contentType }; } // L3: fetch from GitHub @@ -155,21 +182,18 @@ export function createRequestHandler() { }; } - const responseContentType = response.headers.get("content-type") ?? contentTypeForUrl(githubUrl); + const contentType = response.headers.get("content-type") ?? contentTypeForUrl(githubUrl); const body = await response.arrayBuffer(); - // populate caches - const r2Promise = r2Put(githubUrl, body, responseContentType); + // store original in R2 + const r2Promise = r2Put(githubUrl, body, contentType); if (ctx != null) { ctx.waitUntil(r2Promise); } else { await r2Promise; } - if (body.byteLength <= MAX_MEM_CACHE_BODY_SIZE) { - memoryCache.set(githubUrl, { body, contentType: responseContentType }); - } - return { body, status: 200, contentType: responseContentType }; + return { body, status: 200, contentType }; } } @@ -203,6 +227,7 @@ function createJsonResponse(text: string, request: Request) { // converts a GitHub release URL to a local asset path // e.g. https://github.com/dprint/dprint-plugin-prettier/releases/download/0.7.0/plugin.json // -> /dprint/dprint-plugin-prettier/0.7.0/asset/plugin.json +const githubReleasePatternGlobal = /https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/download\/([^/]+)\/([^\s"]+)/g; const githubReleasePattern = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/download\/([^/]+)\/(.+)$/; function githubUrlToAssetPath(githubUrl: string) { const match = githubReleasePattern.exec(githubUrl); @@ -213,6 +238,34 @@ function githubUrlToAssetPath(githubUrl: string) { return `/${username}/${repo}/${tag}/asset/${fileName}`; } +const MAX_JSON_REWRITE_SIZE = 1024 * 1024; // 1MB + +export function rewriteGithubUrls(githubUrl: string, body: ArrayBuffer, origin: string): ArrayBuffer | string { + const match = githubReleasePattern.exec(githubUrl); + if ( + !githubUrl.endsWith("/plugin.json") + || body.byteLength > MAX_JSON_REWRITE_SIZE + || match == null + || !isAssetAllowedRepo(match[1], match[2]) + ) { + return body; + } + const text = new TextDecoder().decode(body); + const rewritten = text.replaceAll( + githubReleasePatternGlobal, + (match, username, repo, tag, fileName) => { + if (!isAssetAllowedRepo(username, repo)) { + return match; + } + return `${origin}/${username}/${repo}/${tag}/asset/${fileName}`; + }, + ); + if (rewritten === text) { + return body; + } + return rewritten; +} + function contentTypeForUrl(url: string) { if (url.endsWith(".wasm")) return contentTypes.wasm; if (url.endsWith(".json") || url.endsWith(".exe-plugin")) return contentTypes.json; diff --git a/plugins.test.ts b/plugins.test.ts index 5c4cc01..b2aedb0 100644 --- a/plugins.test.ts +++ b/plugins.test.ts @@ -7,33 +7,43 @@ function resolveAsset(url: string) { } it("tryResolveAssetUrl", () => { - // allowed repo + // allowed repo — should cache expect( resolveAsset( "https://plugins.dprint.dev/dprint/dprint-plugin-prettier/0.67.0/asset/dprint-plugin-prettier-x86_64-apple-darwin.zip", ), - ).toEqual( - "https://github.com/dprint/dprint-plugin-prettier/releases/download/0.67.0/dprint-plugin-prettier-x86_64-apple-darwin.zip", - ); + ).toEqual({ + githubUrl: + "https://github.com/dprint/dprint-plugin-prettier/releases/download/0.67.0/dprint-plugin-prettier-x86_64-apple-darwin.zip", + shouldCache: true, + }); - // latest tag is not allowed + // latest tag — still resolves but not cached expect( resolveAsset( "https://plugins.dprint.dev/dprint/dprint-plugin-prettier/latest/asset/dprint-plugin-prettier-x86_64-apple-darwin.zip", ), - ).toEqual(undefined); + ).toEqual({ + githubUrl: + "https://github.com/dprint/dprint-plugin-prettier/releases/download/latest/dprint-plugin-prettier-x86_64-apple-darwin.zip", + shouldCache: true, + }); - // different repo in dprint org also works + // different repo in dprint org — should cache expect( resolveAsset("https://plugins.dprint.dev/dprint/dprint-plugin-exec/0.5.0/asset/some-binary.zip"), - ).toEqual( - "https://github.com/dprint/dprint-plugin-exec/releases/download/0.5.0/some-binary.zip", - ); + ).toEqual({ + githubUrl: "https://github.com/dprint/dprint-plugin-exec/releases/download/0.5.0/some-binary.zip", + shouldCache: true, + }); - // org not on allow list + // org not on allow list — resolves but should not cache (redirect) expect( resolveAsset("https://plugins.dprint.dev/someone/some-repo/0.1.0/asset/file.zip"), - ).toEqual(undefined); + ).toEqual({ + githubUrl: "https://github.com/someone/some-repo/releases/download/0.1.0/file.zip", + shouldCache: false, + }); // non-matching URL expect( diff --git a/plugins.ts b/plugins.ts index ce9f239..0a9009e 100644 --- a/plugins.ts +++ b/plugins.ts @@ -56,32 +56,32 @@ const KNOWN_NON_PREFIXED_REPOS = new Set([ "lucacasonato/mf2-tools", ]); -// orgs/users allowed to serve release assets directly -const ASSET_ALLOWED_ORGS = new Set([ - "dprint", -]); +export function isAssetAllowedRepo(username: string, _repo: string) { + switch (username) { + case "dprint": + return true; + default: + return false; + } +} const assetNamePattern = "([A-Za-z0-9\\-\\._]+)"; const assetPattern = new URLPattern({ pathname: `/${userRepoPattern}/${tagPattern}/asset/${assetNamePattern}`, }); -export function tryResolveAssetUrl(url: URL) { +export function tryResolveAssetUrl(url: URL): { githubUrl: string; shouldCache: boolean } | undefined { const result = assetPattern.exec(url); if (!result) { return undefined; } const username = result.pathname.groups[0]!; const repo = result.pathname.groups[1]!; - if (!ASSET_ALLOWED_ORGS.has(username)) { - return undefined; - } const tag = result.pathname.groups[2]!; - if (tag === "latest") { - return undefined; - } const assetName = result.pathname.groups[3]!; - return `https://github.com/${username}/${repo}/releases/download/${tag}/${assetName}`; + const githubUrl = `https://github.com/${username}/${repo}/releases/download/${tag}/${assetName}`; + const shouldCache = isAssetAllowedRepo(username, repo); + return { githubUrl, shouldCache }; } export async function tryResolvePluginUrl(url: URL) { @@ -105,7 +105,7 @@ export async function tryResolveLatestJson(url: URL) { } const username = result.pathname.groups[0]!; const shortRepoName = result.pathname.groups[1]!; - const latestInfo = await getLatestInfo(username, shortRepoName); + const latestInfo = await getLatestInfo(username, shortRepoName, url.origin); if (latestInfo == null) { return 404; } @@ -120,7 +120,7 @@ export async function tryResolveLatestJson(url: URL) { }; } -export async function getLatestInfo(username: string, repoName: string) { +export async function getLatestInfo(username: string, repoName: string, origin: string) { repoName = await getFullRepoName(username, repoName); const releaseInfo = await getLatestReleaseInfo(username, repoName); if (releaseInfo == null) { @@ -134,8 +134,8 @@ export async function getLatestInfo(username: string, repoName: string) { return { schemaVersion: 1, url: username === "dprint" - ? `https://plugins.dprint.dev/${displayRepoName}-${releaseInfo.tagName}.${extension}` - : `https://plugins.dprint.dev/${username}/${displayRepoName}-${releaseInfo.tagName}.${extension}`, + ? `${origin}/${displayRepoName}-${releaseInfo.tagName}.${extension}` + : `${origin}/${username}/${displayRepoName}-${releaseInfo.tagName}.${extension}`, version: releaseInfo.tagName.replace(/^v/, ""), checksum: releaseInfo.checksum, downloadCount: releaseInfo.downloadCount, diff --git a/readInfoFile.ts b/readInfoFile.ts index 41a1164..2c07349 100644 --- a/readInfoFile.ts +++ b/readInfoFile.ts @@ -17,7 +17,7 @@ export interface PluginData { }; } -export async function readInfoFile(): Promise> { +export async function readInfoFile(origin: string): Promise> { return { ...infoJson, latest: await getLatest(infoJson.latest), @@ -28,8 +28,8 @@ export async function readInfoFile(): Promise> { for (const plugin of latest) { const [username, pluginName] = plugin.name.split("/"); const info = pluginName - ? await getLatestInfo(username, pluginName) - : await getLatestInfo("dprint", plugin.name); + ? await getLatestInfo(username, pluginName, origin) + : await getLatestInfo("dprint", plugin.name, origin); if (info != null) { results.push({ ...plugin,