Skip to content

Commit e3fe0d9

Browse files
dsherretdsherret
committed
Rewrite github asset urls to use plugin service
1 parent c76933f commit e3fe0d9

4 files changed

Lines changed: 130 additions & 38 deletions

File tree

‎handleRequest.test.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { expect, it } from "vitest";
2-
import { createRequestHandler, resolvePluginOrSchemaUrl } from "./handleRequest.js";
2+
import { createRequestHandler, resolvePluginOrSchemaUrl, rewriteGithubUrls } from "./handleRequest.js";
33

44
it("should get info.json", { timeout: 10_000 }, async () => {
55
const { handleRequest } = createRequestHandler();
@@ -353,6 +353,50 @@ it("should return 404 for asset not found", async () => {
353353
expect(response.status).toEqual(404);
354354
});
355355

356+
it("rewriteGithubUrls", () => {
357+
const githubUrl = "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/plugin.json";
358+
const origin = "https://plugins.dprint.dev";
359+
const json = JSON.stringify({
360+
"schemaVersion": 2,
361+
"kind": "process",
362+
"darwin-aarch64": {
363+
"reference":
364+
"https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/dprint-plugin-exec-aarch64-apple-darwin.zip",
365+
"checksum": "abc123",
366+
},
367+
"linux-x86_64": {
368+
"reference":
369+
"https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/dprint-plugin-exec-x86_64-unknown-linux-gnu.zip",
370+
"checksum": "def456",
371+
},
372+
});
373+
const body = new TextEncoder().encode(json).buffer as ArrayBuffer;
374+
const result = rewriteGithubUrls(githubUrl, body, origin);
375+
expect(typeof result).toEqual("string");
376+
const parsed = JSON.parse(result as string);
377+
expect(parsed["darwin-aarch64"].reference).toEqual(
378+
"https://plugins.dprint.dev/dprint/dprint-plugin-exec/0.6.0/asset/dprint-plugin-exec-aarch64-apple-darwin.zip",
379+
);
380+
expect(parsed["linux-x86_64"].reference).toEqual(
381+
"https://plugins.dprint.dev/dprint/dprint-plugin-exec/0.6.0/asset/dprint-plugin-exec-x86_64-unknown-linux-gnu.zip",
382+
);
383+
});
384+
385+
it("rewriteGithubUrls should not rewrite non-plugin.json", () => {
386+
const githubUrl = "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/schema.json";
387+
const body = new TextEncoder().encode("{}").buffer as ArrayBuffer;
388+
const result = rewriteGithubUrls(githubUrl, body, "https://plugins.dprint.dev");
389+
expect(result).toBe(body);
390+
});
391+
392+
it("rewriteGithubUrls should return original buffer when no URLs match", () => {
393+
const githubUrl = "https://github.com/dprint/dprint-plugin-exec/releases/download/0.6.0/plugin.json";
394+
const json = JSON.stringify({ "key": "value" });
395+
const body = new TextEncoder().encode(json).buffer as ArrayBuffer;
396+
const result = rewriteGithubUrls(githubUrl, body, "https://plugins.dprint.dev");
397+
expect(result).toBe(body);
398+
});
399+
356400
// todo: mock github api for these tests
357401

358402
it("tryResolveSchemaUrl", async () => {

‎handleRequest.ts‎

Lines changed: 69 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
import { renderHome } from "./home.jsx";
22
import oldMappings from "./old_redirects.json" with { type: "json" };
3-
import { tryResolveAssetUrl, tryResolveLatestJson, tryResolvePluginUrl, tryResolveSchemaUrl } from "./plugins.js";
3+
import {
4+
isAssetAllowedRepo,
5+
tryResolveAssetUrl,
6+
tryResolveLatestJson,
7+
tryResolvePluginUrl,
8+
tryResolveSchemaUrl,
9+
} from "./plugins.js";
410
import { readInfoFile } from "./readInfoFile.js";
511
import robotsTxt from "./robots.txt";
612
import styleCSS from "./style.css";
@@ -20,13 +26,13 @@ const contentTypes = {
2026
};
2127

2228
export function createRequestHandler() {
23-
const memoryCache = new LruCache<string, { body: ArrayBuffer; contentType: string }>({ size: 50 });
29+
const memoryCache = new LruCache<string, { body: ArrayBuffer | string; contentType: string }>({ size: 50 });
2430
return {
2531
async handleRequest(request: Request, ctx?: ExecutionContext) {
2632
const url = new URL(request.url);
2733
const assetUrl = tryResolveAssetUrl(url);
2834
if (assetUrl != null) {
29-
return servePlugin(request, assetUrl, ctx);
35+
return servePlugin(request, url, assetUrl, ctx);
3036
}
3137

3238
const githubUrl = await resolvePluginOrSchemaUrl(url);
@@ -36,10 +42,10 @@ export function createRequestHandler() {
3642
// plugin.json files resolve correctly
3743
const assetPath = githubUrlToAssetPath(githubUrl);
3844
if (assetPath != null) {
39-
return Response.redirect(new URL(assetPath, url.origin).href, 302);
45+
return Response.redirect(`${url.origin}${assetPath}`, 302);
4046
}
4147
}
42-
return servePlugin(request, githubUrl, ctx);
48+
return servePlugin(request, url, githubUrl, ctx);
4349
}
4450

4551
const userLatestInfo = await tryResolveLatestJson(url);
@@ -57,7 +63,7 @@ export function createRequestHandler() {
5763
}
5864

5965
if (url.pathname.startsWith("/info.json")) {
60-
const infoFileData = await readInfoFile();
66+
const infoFileData = await readInfoFile(url.origin);
6167
return createJsonResponse(
6268
JSON.stringify(infoFileData, null, 2),
6369
request,
@@ -107,8 +113,8 @@ export function createRequestHandler() {
107113
},
108114
};
109115

110-
async function servePlugin(request: Request, githubUrl: string, ctx?: ExecutionContext) {
111-
const result = await resolveBody(githubUrl, ctx);
116+
async function servePlugin(request: Request, requestUrl: URL, githubUrl: string, ctx?: ExecutionContext) {
117+
const result = await resolveBodyWithMemoryCache(githubUrl, requestUrl, ctx);
112118
return new Response(result.body, {
113119
headers: {
114120
"content-type": result.contentType,
@@ -118,28 +124,46 @@ export function createRequestHandler() {
118124
});
119125
}
120126

121-
async function resolveBody(
127+
async function resolveBodyWithMemoryCache(
122128
githubUrl: string,
129+
requestUrl: URL,
123130
ctx?: ExecutionContext,
124-
): Promise<{ body: ArrayBuffer | ReadableStream | null; status: number; contentType: string }> {
125-
// L1: in-memory cache
131+
): Promise<{ body: ArrayBuffer | ReadableStream | string | null; status: number; contentType: string }> {
132+
// L1: in-memory cache (already rewritten)
126133
const cached = memoryCache.get(githubUrl);
127134
if (cached != null) {
128135
return { body: cached.body, status: 200, contentType: cached.contentType };
129136
}
130137

131-
// L2: R2
138+
const result = await fetchBody(githubUrl, requestUrl, ctx);
139+
if (result.status !== 200 || !(result.body instanceof ArrayBuffer)) {
140+
return result;
141+
}
142+
143+
// rewrite GitHub URLs in JSON files to use the local asset path
144+
const body = rewriteGithubUrls(githubUrl, result.body, requestUrl.origin);
145+
const size = typeof body === "string" ? body.length : body.byteLength;
146+
if (size <= MAX_MEM_CACHE_BODY_SIZE) {
147+
memoryCache.set(githubUrl, { body, contentType: result.contentType });
148+
}
149+
150+
return { body, status: 200, contentType: result.contentType };
151+
}
152+
153+
async function fetchBody(
154+
githubUrl: string,
155+
requestUrl: URL,
156+
ctx?: ExecutionContext,
157+
): Promise<{ body: ArrayBuffer | ReadableStream | string | null; status: number; contentType: string }> {
158+
// L2: R2 (stores original content)
132159
const r2Object = await r2Get(githubUrl);
133160
if (r2Object != null) {
134-
const r2ContentType = r2Object.httpMetadata?.contentType ?? contentTypeForUrl(githubUrl);
135-
// small enough for L1 — buffer and cache
161+
const contentType = r2Object.httpMetadata?.contentType ?? contentTypeForUrl(githubUrl);
136162
if (r2Object.size <= MAX_MEM_CACHE_BODY_SIZE) {
137-
const buffer = await r2Object.arrayBuffer();
138-
memoryCache.set(githubUrl, { body: buffer, contentType: r2ContentType });
139-
return { body: buffer, status: 200, contentType: r2ContentType };
163+
return { body: await r2Object.arrayBuffer(), status: 200, contentType };
140164
}
141165
// large — stream directly without buffering
142-
return { body: r2Object.body, status: 200, contentType: r2ContentType };
166+
return { body: r2Object.body, status: 200, contentType };
143167
}
144168

145169
// L3: fetch from GitHub
@@ -155,21 +179,18 @@ export function createRequestHandler() {
155179
};
156180
}
157181

158-
const responseContentType = response.headers.get("content-type") ?? contentTypeForUrl(githubUrl);
182+
const contentType = response.headers.get("content-type") ?? contentTypeForUrl(githubUrl);
159183
const body = await response.arrayBuffer();
160184

161-
// populate caches
162-
const r2Promise = r2Put(githubUrl, body, responseContentType);
185+
// store original in R2
186+
const r2Promise = r2Put(githubUrl, body, contentType);
163187
if (ctx != null) {
164188
ctx.waitUntil(r2Promise);
165189
} else {
166190
await r2Promise;
167191
}
168-
if (body.byteLength <= MAX_MEM_CACHE_BODY_SIZE) {
169-
memoryCache.set(githubUrl, { body, contentType: responseContentType });
170-
}
171192

172-
return { body, status: 200, contentType: responseContentType };
193+
return { body, status: 200, contentType };
173194
}
174195
}
175196

@@ -203,6 +224,7 @@ function createJsonResponse(text: string, request: Request) {
203224
// converts a GitHub release URL to a local asset path
204225
// e.g. https://github.com/dprint/dprint-plugin-prettier/releases/download/0.7.0/plugin.json
205226
// -> /dprint/dprint-plugin-prettier/0.7.0/asset/plugin.json
227+
const githubReleasePatternGlobal = /https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/download\/([^/]+)\/([^\s"]+)/g;
206228
const githubReleasePattern = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/download\/([^/]+)\/(.+)$/;
207229
function githubUrlToAssetPath(githubUrl: string) {
208230
const match = githubReleasePattern.exec(githubUrl);
@@ -213,6 +235,28 @@ function githubUrlToAssetPath(githubUrl: string) {
213235
return `/${username}/${repo}/${tag}/asset/${fileName}`;
214236
}
215237

238+
const MAX_JSON_REWRITE_SIZE = 1024 * 1024; // 1MB
239+
240+
export function rewriteGithubUrls(githubUrl: string, body: ArrayBuffer, origin: string): ArrayBuffer | string {
241+
if (!githubUrl.endsWith("/plugin.json") || body.byteLength > MAX_JSON_REWRITE_SIZE) {
242+
return body;
243+
}
244+
const text = new TextDecoder().decode(body);
245+
const rewritten = text.replaceAll(
246+
githubReleasePatternGlobal,
247+
(match, username, repo, tag, fileName) => {
248+
if (!isAssetAllowedRepo(username, repo)) {
249+
return match;
250+
}
251+
return `${origin}/${username}/${repo}/${tag}/asset/${fileName}`;
252+
},
253+
);
254+
if (rewritten === text) {
255+
return body;
256+
}
257+
return rewritten;
258+
}
259+
216260
function contentTypeForUrl(url: string) {
217261
if (url.endsWith(".wasm")) return contentTypes.wasm;
218262
if (url.endsWith(".json") || url.endsWith(".exe-plugin")) return contentTypes.json;

‎plugins.ts‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,14 @@ const KNOWN_NON_PREFIXED_REPOS = new Set([
5656
"lucacasonato/mf2-tools",
5757
]);
5858

59-
// orgs/users allowed to serve release assets directly
60-
const ASSET_ALLOWED_ORGS = new Set([
61-
"dprint",
62-
]);
59+
export function isAssetAllowedRepo(username: string, _repo: string) {
60+
switch (username) {
61+
case "dprint":
62+
return true;
63+
default:
64+
return false;
65+
}
66+
}
6367

6468
const assetNamePattern = "([A-Za-z0-9\\-\\._]+)";
6569
const assetPattern = new URLPattern({
@@ -73,7 +77,7 @@ export function tryResolveAssetUrl(url: URL) {
7377
}
7478
const username = result.pathname.groups[0]!;
7579
const repo = result.pathname.groups[1]!;
76-
if (!ASSET_ALLOWED_ORGS.has(username)) {
80+
if (!isAssetAllowedRepo(username, repo)) {
7781
return undefined;
7882
}
7983
const tag = result.pathname.groups[2]!;
@@ -105,7 +109,7 @@ export async function tryResolveLatestJson(url: URL) {
105109
}
106110
const username = result.pathname.groups[0]!;
107111
const shortRepoName = result.pathname.groups[1]!;
108-
const latestInfo = await getLatestInfo(username, shortRepoName);
112+
const latestInfo = await getLatestInfo(username, shortRepoName, url.origin);
109113
if (latestInfo == null) {
110114
return 404;
111115
}
@@ -120,7 +124,7 @@ export async function tryResolveLatestJson(url: URL) {
120124
};
121125
}
122126

123-
export async function getLatestInfo(username: string, repoName: string) {
127+
export async function getLatestInfo(username: string, repoName: string, origin: string) {
124128
repoName = await getFullRepoName(username, repoName);
125129
const releaseInfo = await getLatestReleaseInfo(username, repoName);
126130
if (releaseInfo == null) {
@@ -134,8 +138,8 @@ export async function getLatestInfo(username: string, repoName: string) {
134138
return {
135139
schemaVersion: 1,
136140
url: username === "dprint"
137-
? `https://plugins.dprint.dev/${displayRepoName}-${releaseInfo.tagName}.${extension}`
138-
: `https://plugins.dprint.dev/${username}/${displayRepoName}-${releaseInfo.tagName}.${extension}`,
141+
? `${origin}/${displayRepoName}-${releaseInfo.tagName}.${extension}`
142+
: `${origin}/${username}/${displayRepoName}-${releaseInfo.tagName}.${extension}`,
139143
version: releaseInfo.tagName.replace(/^v/, ""),
140144
checksum: releaseInfo.checksum,
141145
downloadCount: releaseInfo.downloadCount,

‎readInfoFile.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ export interface PluginData {
1717
};
1818
}
1919

20-
export async function readInfoFile(): Promise<Readonly<PluginsData>> {
20+
export async function readInfoFile(origin: string): Promise<Readonly<PluginsData>> {
2121
return {
2222
...infoJson,
2323
latest: await getLatest(infoJson.latest),
@@ -28,8 +28,8 @@ export async function readInfoFile(): Promise<Readonly<PluginsData>> {
2828
for (const plugin of latest) {
2929
const [username, pluginName] = plugin.name.split("/");
3030
const info = pluginName
31-
? await getLatestInfo(username, pluginName)
32-
: await getLatestInfo("dprint", plugin.name);
31+
? await getLatestInfo(username, pluginName, origin)
32+
: await getLatestInfo("dprint", plugin.name, origin);
3333
if (info != null) {
3434
results.push({
3535
...plugin,

0 commit comments

Comments
 (0)