/** * Pull every externally-hosted image into our own bucket. * * pnpm assets:migrate * * The seed used to point at pravatar, picsum and Unsplash. That is fine for a * scratch database and wrong for anything anybody is shown: those services rate * limit, change what a URL returns, and go down — and when they do, a demo is a * grid of broken images with no way to fix it in the moment. * * This fetches each one once, stores it under a DETERMINISTIC key, and rewrites * the row to our own origin. Idempotent: a key that already exists is left * alone, so re-running after a reseed costs one HEAD per object rather than * re-downloading the internet. */ import { readFileSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { sql } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/postgres-js'; import postgres from 'postgres'; import { HeadObjectCommand, PutObjectCommand, S3Client, } from '@aws-sdk/client-s3'; import * as schema from './schema/index'; import { readSsl } from './client'; for (const line of readFileSync(new URL('../../../.env', import.meta.url), 'utf8').split('\n')) { const m = /^([A-Z_]+)=(.*)$/.exec(line.trim()); if (m?.[1]) process.env[m[1]] ??= m[2]; } const url = process.env.DATABASE_URL; if (!url) throw new Error('DATABASE_URL is not set'); const region = process.env.SPACES_REGION; const bucket = process.env.SPACES_BUCKET; if (!region || !bucket || !process.env.SPACES_KEY || !process.env.SPACES_SECRET) { throw new Error('Spaces is not configured. Set SPACES_REGION / BUCKET / KEY / SECRET.'); } const origin = process.env.SPACES_CDN_URL || `https://${bucket}.${region}.digitaloceanspaces.com`; const client = new S3Client({ region, endpoint: `https://${region}.digitaloceanspaces.com`, credentials: { accessKeyId: process.env.SPACES_KEY, secretAccessKey: process.env.SPACES_SECRET, }, }); const ssl = readSsl(); const pg = postgres(url, { max: 1, ...(ssl ? { ssl } : {}) }); const db = drizzle(pg, { schema }); const EXT: Record = { 'image/jpeg': 'jpg', 'image/png': 'png', 'image/webp': 'webp', 'image/avif': 'avif', }; /** Already ours? Then there is nothing to fetch. */ function isLocal(u: string): boolean { return u.startsWith(origin); } async function exists(key: string): Promise { try { await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); return true; } catch { return false; } } /** * Fetch one image and store it, returning the URL it now lives at. * * The key is a hash of the SOURCE url, so the same source always lands on the * same object: reseeding gives every pro a new uuid, and keying on that would * fill the bucket with a fresh copy of every photo on every run. */ async function adopt(source: string, prefix: string): Promise { const hash = createHash('sha1').update(source).digest('hex').slice(0, 16); const response = await fetch(source, { redirect: 'follow' }); if (!response.ok) { console.warn(` ! ${response.status} ${source.slice(0, 60)}`); return null; } const type = (response.headers.get('content-type') ?? 'image/jpeg').split(';')[0]!.trim(); const key = `${prefix}/${hash}.${EXT[type] ?? 'jpg'}`; if (await exists(key)) return `${origin}/${key}`; const body = Buffer.from(await response.arrayBuffer()); await client.send( new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, ContentType: type, // Public: these are the photos on the deck card. Credential documents are // a different prefix and are never given an ACL. ACL: 'public-read', // A year. The key is content-addressed by source, so a changed image is a // changed key rather than a stale cache. CacheControl: 'public, max-age=31536000, immutable', }), ); console.log(` + ${(body.length / 1024).toFixed(0)}kb ${key}`); return `${origin}/${key}`; } async function main() { console.log(`Migrating assets into ${origin}\n`); const media = await db .select({ id: schema.proMedia.id, url: schema.proMedia.url, kind: schema.proMedia.kind }) .from(schema.proMedia); const external = media.filter((m) => !isLocal(m.url)); console.log(`pro_media: ${media.length} rows, ${external.length} still external`); // Sequential on purpose. Twenty parallel fetches against a free image host is // how you get rate limited half way through and end up with a bucket that is // partly migrated and rows that disagree with it. let moved = 0; for (const row of external) { const hosted = await adopt(row.url, row.kind === 'photo' ? 'pro-media' : 'work-samples'); if (!hosted) continue; await db .update(schema.proMedia) .set({ url: hosted }) .where(sql`${schema.proMedia.id} = ${row.id}`); moved += 1; } // Job photos are uploaded by customers and already ours, but a seeded one // could point outward too. const jobs = await db .select({ id: schema.jobs.id, photos: schema.jobs.photos }) .from(schema.jobs); let jobPhotos = 0; for (const job of jobs) { const outward = job.photos.filter((p) => !isLocal(p)); if (outward.length === 0) continue; const rewritten: string[] = []; for (const photo of job.photos) { rewritten.push(isLocal(photo) ? photo : ((await adopt(photo, 'job-photos')) ?? photo)); } await db .update(schema.jobs) .set({ photos: rewritten }) .where(sql`${schema.jobs.id} = ${job.id}`); jobPhotos += outward.length; } console.log(`\n${moved} pro images and ${jobPhotos} job photos now served from ${origin}`); const left = ( await db.select({ url: schema.proMedia.url }).from(schema.proMedia) ).filter((m) => !isLocal(m.url)); if (left.length) console.warn(`${left.length} still external — rerun to retry.`); } await main(); await pg.end();