linkder/packages/db/src/migrate-assets.ts

/**
 * Pull every externally-hosted image into our own bucket.
 *
 *   pnpm assets:migrate
 *
 * The seed used to point at pravatar, picsum and Unsplash. That is fine for a
 * scratch database and wrong for anything anybody is shown: those services rate
 * limit, change what a URL returns, and go down — and when they do, a demo is a
 * grid of broken images with no way to fix it in the moment.
 *
 * This fetches each one once, stores it under a DETERMINISTIC key, and rewrites
 * the row to our own origin. Idempotent: a key that already exists is left
 * alone, so re-running after a reseed costs one HEAD per object rather than
 * re-downloading the internet.
 */
import { readFileSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { sql } from 'drizzle-orm';
import { drizzle } from 'drizzle-orm/postgres-js';
import postgres from 'postgres';
import {
  HeadObjectCommand,
  PutObjectCommand,
  S3Client,
} from '@aws-sdk/client-s3';
import * as schema from './schema/index';
import { readSsl } from './client';

for (const line of readFileSync(new URL('../../../.env', import.meta.url), 'utf8').split('\n')) {
  const m = /^([A-Z_]+)=(.*)$/.exec(line.trim());
  if (m?.[1]) process.env[m[1]] ??= m[2];
}

const url = process.env.DATABASE_URL;
if (!url) throw new Error('DATABASE_URL is not set');

const region = process.env.SPACES_REGION;
const bucket = process.env.SPACES_BUCKET;
if (!region || !bucket || !process.env.SPACES_KEY || !process.env.SPACES_SECRET) {
  throw new Error('Spaces is not configured. Set SPACES_REGION / BUCKET / KEY / SECRET.');
}

const origin = process.env.SPACES_CDN_URL || `https://${bucket}.${region}.digitaloceanspaces.com`;

const client = new S3Client({
  region,
  endpoint: `https://${region}.digitaloceanspaces.com`,
  credentials: {
    accessKeyId: process.env.SPACES_KEY,
    secretAccessKey: process.env.SPACES_SECRET,
  },
});

const ssl = readSsl();
const pg = postgres(url, { max: 1, ...(ssl ? { ssl } : {}) });
const db = drizzle(pg, { schema });

const EXT: Record<string, string> = {
  'image/jpeg': 'jpg',
  'image/png': 'png',
  'image/webp': 'webp',
  'image/avif': 'avif',
};

/** Already ours? Then there is nothing to fetch. */
function isLocal(u: string): boolean {
  return u.startsWith(origin);
}

async function exists(key: string): Promise<boolean> {
  try {
    await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
    return true;
  } catch {
    return false;
  }
}

/**
 * Fetch one image and store it, returning the URL it now lives at.
 *
 * The key is a hash of the SOURCE url, so the same source always lands on the
 * same object: reseeding gives every pro a new uuid, and keying on that would
 * fill the bucket with a fresh copy of every photo on every run.
 */
async function adopt(source: string, prefix: string): Promise<string | null> {
  const hash = createHash('sha1').update(source).digest('hex').slice(0, 16);

  const response = await fetch(source, { redirect: 'follow' });
  if (!response.ok) {
    console.warn(`  ! ${response.status} ${source.slice(0, 60)}`);
    return null;
  }

  const type = (response.headers.get('content-type') ?? 'image/jpeg').split(';')[0]!.trim();
  const key = `${prefix}/${hash}.${EXT[type] ?? 'jpg'}`;

  if (await exists(key)) return `${origin}/${key}`;

  const body = Buffer.from(await response.arrayBuffer());
  await client.send(
    new PutObjectCommand({
      Bucket: bucket,
      Key: key,
      Body: body,
      ContentType: type,
      // Public: these are the photos on the deck card. Credential documents are
      // a different prefix and are never given an ACL.
      ACL: 'public-read',
      // A year. The key is content-addressed by source, so a changed image is a
      // changed key rather than a stale cache.
      CacheControl: 'public, max-age=31536000, immutable',
    }),
  );

  console.log(`  + ${(body.length / 1024).toFixed(0)}kb  ${key}`);
  return `${origin}/${key}`;
}

async function main() {
  console.log(`Migrating assets into ${origin}\n`);

  const media = await db
    .select({ id: schema.proMedia.id, url: schema.proMedia.url, kind: schema.proMedia.kind })
    .from(schema.proMedia);

  const external = media.filter((m) => !isLocal(m.url));
  console.log(`pro_media: ${media.length} rows, ${external.length} still external`);

  // Sequential on purpose. Twenty parallel fetches against a free image host is
  // how you get rate limited half way through and end up with a bucket that is
  // partly migrated and rows that disagree with it.
  let moved = 0;
  for (const row of external) {
    const hosted = await adopt(row.url, row.kind === 'photo' ? 'pro-media' : 'work-samples');
    if (!hosted) continue;
    await db
      .update(schema.proMedia)
      .set({ url: hosted })
      .where(sql`${schema.proMedia.id} = ${row.id}`);
    moved += 1;
  }

  // Job photos are uploaded by customers and already ours, but a seeded one
  // could point outward too.
  const jobs = await db
    .select({ id: schema.jobs.id, photos: schema.jobs.photos })
    .from(schema.jobs);

  let jobPhotos = 0;
  for (const job of jobs) {
    const outward = job.photos.filter((p) => !isLocal(p));
    if (outward.length === 0) continue;

    const rewritten: string[] = [];
    for (const photo of job.photos) {
      rewritten.push(isLocal(photo) ? photo : ((await adopt(photo, 'job-photos')) ?? photo));
    }
    await db
      .update(schema.jobs)
      .set({ photos: rewritten })
      .where(sql`${schema.jobs.id} = ${job.id}`);
    jobPhotos += outward.length;
  }

  console.log(`\n${moved} pro images and ${jobPhotos} job photos now served from ${origin}`);

  const left = (
    await db.select({ url: schema.proMedia.url }).from(schema.proMedia)
  ).filter((m) => !isLocal(m.url));
  if (left.length) console.warn(`${left.length} still external — rerun to retry.`);
}

await main();
await pg.end();