sailfork / t3code-github / packages/ source-control-github/ src/ server/ gitHubMediaFetch.ts
import type { SourceControlMediaRequest } from "@t3tools/source-control-core/server/driver";
import * as Clock from "effect/Clock";
import * as Effect from "effect/Effect";
import * as Redacted from "effect/Redacted";
import {
FetchHttpClient,
HttpClient,
HttpClientRequest,
type HttpClientResponse,
} from "effect/http";
import { GITHUB_CREDENTIALED_MEDIA_HOSTS } from "../shared/gitHubMedia.ts";
import * as GitHubCredentials from "./GitHubCredentials.ts";
import * as GitHubRest from "./GitHubRest.ts";
/**
* Everything a redirect leads to — the presigned object stores GitHub hands assets off to —
* authorizes with its own signature, and some of them reject a request that also carries a
* bearer token.
*/
const isCredentialedHost = (url: string) => {
const target = URL.parse(url);
return (
target !== null &&
target.protocol === "https:" &&
target.port === "" &&
GITHUB_CREDENTIALED_MEDIA_HOSTS.has(target.hostname.toLowerCase())
);
};
const MAX_REDIRECTS = 3;
const MANUAL_REDIRECT: RequestInit = { redirect: "manual" };
const MEDIA_CONTENT_TYPE_PATTERN = /^(?:image|video|audio)\/[\w!#Loading Sailfork
amp;^.+-]+$/i;
const ATTACHMENT_PATH_PATTERN =
/^\/(?:user-attachments\/assets\/[\w-]+|[^/]+\/[^/]+\/assets\/\d+\/[\w-]+)$/u;
const LEGACY_ATTACHMENT_REPOSITORY_PATTERN = /^\/([^/]+\/[^/]+)\/assets\//u;
const REPOSITORY_PATTERN = /^[\w.-]+\/[\w.-]+$/u;
// GitHub's signed upload links last about five minutes.
const SIGNED_URL_CACHE_TTL_MS = 4 * 60_000;
const SIGNED_URL_CACHE_MAX_ENTRIES = 64;
const contentTypeOf = (response: HttpClientResponse.HttpClientResponse) =>
response.headers["content-type"]?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
/** The repository an upload has to be rendered in, or `undefined` when it is not an upload. */
const uploadRepository = (request: SourceControlMediaRequest) => {
const target = URL.parse(request.url);
if (
target === null ||
target.protocol !== "https:" ||
target.hostname !== "github.com" ||
target.port !== "" ||
target.username !== "" ||
target.password !== "" ||
!ATTACHMENT_PATH_PATTERN.test(target.pathname)
) {
return undefined;
}
// A legacy upload names its repository; one the caller supplies cannot override that.
const repository =
LEGACY_ATTACHMENT_REPOSITORY_PATTERN.exec(target.pathname)?.[1] ?? request.repository;
// An unusable explicit context must not fall back to an unrelated checkout.
return repository !== undefined && REPOSITORY_PATTERN.test(repository) ? repository : undefined;
};
export const make = Effect.gen(function* () {
const credentials = yield* GitHubCredentials.GitHubCredentials;
const rest = yield* GitHubRest.GitHubRest;
const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);
const signedUrls = new Map<string, { readonly at: number; readonly url: string }>();
// Both media hosts are served by github.com's account, which is the host `gh` stores it under.
// Read on every fetch, so media loads as soon as the user signs in.
const credential = Effect.suspend(() => credentials.get("github.com")).pipe(
Effect.orElseSucceed(() => null),
);
const fetchFollowingRedirects = Effect.fnUntraced(function* (
url: string,
headers: Readonly<Record<string, string>>,
token: Redacted.Redacted<string> | null,
) {
let target = url;
for (let hop = 0; ; hop += 1) {
const authorization =
token !== null && isCredentialedHost(target) ? `Bearer ${Redacted.value(token)}` : null;
const response: HttpClientResponse.HttpClientResponse = yield* httpClient
.execute(
HttpClientRequest.get(target).pipe(
HttpClientRequest.setHeaders({
...headers,
// The bytes are streamed straight through, so never let an encoding layer in.
"accept-encoding": "identity",
...(authorization === null ? {} : { authorization }),
}),
),
)
.pipe(
Effect.provideService(FetchHttpClient.RequestInit, MANUAL_REDIRECT),
// A signed URL's query string is a bearer capability; keep it out of tracing spans.
Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
);
const location = response.headers.location;
if (response.status < 300 || response.status >= 400) return response;
if (!location || hop >= MAX_REDIRECTS) return null;
const next = URL.parse(location, target);
if (next === null || next.protocol !== "https:" || next.username || next.password) {
return null;
}
target = next.toString();
}
});
// An organization with SAML SSO answers a token-authenticated upload request with its sign-in
// page. Rendering the link as markdown in the repository's context hands back a signed URL.
const renderSignedUrl = Effect.fnUntraced(function* (url: string, repository: string) {
const rendered = yield* rest
.rest({
host: "github.com",
operation: "renderMarkdown",
method: "POST",
path: "markdown",
accept: "text/html",
body: { text: ``, mode: "gfm", context: repository },
})
.pipe(Effect.orElseSucceed(() => null));
if (rendered === null || rendered.status !== 200 || rendered.truncated) return null;
for (const match of rendered.body.matchAll(/<img\b[^>]*\ssrc\s*=\s*(["'])(.*?)\1/giu)) {
const signed = URL.parse(match[2]?.replaceAll("&", "&") ?? "");
if (
signed?.protocol === "https:" &&
signed.hostname === "private-user-images.githubusercontent.com" &&
signed.port === "" &&
signed.username === "" &&
signed.password === "" &&
signed.searchParams.get("jwt")
) {
return signed.toString();
}
}
return null;
});
return Effect.fn("GitHubMediaFetch.fetchMedia")(function* (request: SourceControlMediaRequest) {
const signedIn = yield* credential;
const repository = signedIn === null ? undefined : uploadRepository(request);
const cacheKey =
signedIn === null || repository === undefined
? null
: [signedIn.fingerprint, repository, request.url].map(encodeURIComponent).join("|");
const now = yield* Clock.currentTimeMillis;
const cached = cacheKey === null ? undefined : signedUrls.get(cacheKey);
let response: HttpClientResponse.HttpClientResponse | null;
if (cacheKey !== null && cached !== undefined && now - cached.at < SIGNED_URL_CACHE_TTL_MS) {
response = yield* fetchFollowingRedirects(cached.url, request.headers, null);
if (response === null || response.status >= 400) signedUrls.delete(cacheKey);
// A signed URL can expire before its TTL; only then is a fresh one worth rendering.
if (response?.status !== 403) return response;
} else {
if (cacheKey !== null) signedUrls.delete(cacheKey);
response = yield* fetchFollowingRedirects(
request.url,
request.headers,
signedIn?.token ?? null,
);
if (response === null || contentTypeOf(response) !== "text/html") return response;
}
if (cacheKey === null || repository === undefined || signedIn === null) return response;
const signed = yield* renderSignedUrl(request.url, repository);
if (signed === null) return response;
const media = yield* fetchFollowingRedirects(signed, request.headers, null);
if (
(media?.status === 200 || media?.status === 206) &&
MEDIA_CONTENT_TYPE_PATTERN.test(contentTypeOf(media))
) {
// The render can run under another account if the user switched mid-request.
const current = yield* credential;
if (current?.fingerprint === signedIn.fingerprint) {
if (signedUrls.size >= SIGNED_URL_CACHE_MAX_ENTRIES) {
const oldest = signedUrls.keys().next().value;
if (oldest !== undefined) signedUrls.delete(oldest);
}
signedUrls.set(cacheKey, { at: now, url: signed });
}
}
return media;
});
});