scripts/prune-documents.ts

Linked repository file from this build.

#!/usr/bin/env bun
// Lists, and with --delete removes, stored PDFs that no review snapshot references.
// Every snapshot version counts, so files from earlier versions are kept.
import { config } from "dotenv";
import { sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg";
import {
  documentStorage,
  listLocalDocuments,
  orphanedObjects,
  removeLocalDocument,
} from "@/lib/documents/server-storage";
import { assertProductionConfirmed, errorDetails, resolveDatabaseTarget } from "./db/target";

config({ quiet: true });

async function referencedKeys(url: string) {
  const pool = new Pool({ connectionString: url });
  try {
    const result = await drizzle(pool).execute<{ key: string }>(sql`
      select distinct document->>'storageKey' as key
      from snap_review_snapshots, jsonb_array_elements(review->'documents') as document
      where document->>'storageKey' is not null
    `);
    return new Set(result.rows.map((row) => row.key));
  } finally {
    await pool.end();
  }
}

async function main() {
  const args = process.argv.slice(2);
  const remove = args.includes("--delete");
  const target = resolveDatabaseTarget(process.env.QC_DB_TARGET);
  if (remove)
    assertProductionConfirmed(
      target.target,
      args.includes("--confirm-production"),
      "Deleting PDFs",
    );
  const storage = documentStorage();
  // A local database does not know which bucket files production uses.
  if (remove && storage.mode === "neon" && target.target !== "production")
    throw new Error("Deleting from Neon storage needs QC_DB_TARGET=production.");
  console.log(`Database: ${target.target}, ${target.label}. Storage: ${storage.mode}.`);
  const objects = storage.mode === "neon" ? await storage.list() : await listLocalDocuments();
  const orphans = orphanedObjects(objects, await referencedKeys(target.url), new Date());
  console.log(`${objects.length} stored object(s), ${orphans.length} older than a day and unused.`);
  for (const { key } of orphans) console.log(`  ${key}`);
  if (!remove) {
    if (orphans.length) console.log("Dry run. Run again with --delete to remove them.");
    return;
  }
  for (const { key } of orphans)
    await (storage.mode === "neon" ? storage.remove(key) : removeLocalDocument(key));
  console.log(`Removed ${orphans.length} object(s).`);
}

if (import.meta.main)
  main().catch((error) => {
    console.error(
      JSON.stringify(
        errorDetails(error, [process.env.DATABASE_URL ?? "", process.env.PROD_DATABASE_URL ?? ""]),
        null,
        2,
      ),
    );
    process.exit(1);
  });