/**
 * Upload changed markdown docs to Contentstack Digital Concierge as
 * knowledge Sources.
 *
 * Flow per changed file (two-request handshake, see user's original spec):
 *   1. POST /digital-concierge/sources — multipart/form-data — returns { uid, ... }
 *   2. PUT  /digital-concierge/sources/{uid} — JSON body { metadata: {...} }
 *
 * Change detection: sha256 of the file's markdown content vs. the previously
 * uploaded sha stored in `.dc-upload-state.json`. First run uploads everything;
 * subsequent runs only re-upload files that changed.
 *
 * Metadata: `source_url` (and the mirroring `url` + `retrieved_at`) points at
 * the **published page URL** for that markdown file — not the raw .md path.
 * A doc page's URL comes from the docs tree (scripts/lib/docs-tree.ts, the
 * same rule the site uses); a Skills page publishes at /prompts/<name>. Both
 * are prefixed with `DC_SITE_BASE_URL`
 * (defaults to https://studio-documentation.contentstackapps.com).
 *
 * Concurrency: at most 2 files uploaded in parallel (POST + PUT for one file
 * = 1 "slot"). Prevents rate-limiting on the DC API and keeps logs readable.
 *
 * Opt-in via env — if `DC_AUTHTOKEN` is missing, this script exits 0 with an
 * informational message. Safe to chain into build:rte; won't fail the build.
 *
 * ────────────────────────────────────────────────────────────────────────────
 * REQUIRED ENV VARS
 * ────────────────────────────────────────────────────────────────────────────
 *   DC_AUTHTOKEN       Contentstack authtoken (SSO token, not a Bearer JWT).
 *   DC_ORG_UID         Organization uid.
 *   DC_API_KEY         Stack API key.
 *   DC_PROJECT_ID      Digital Concierge project id.
 *   DC_AGENT_ID        Agent id (the widget's apiKey).
 *   DC_USER_UID        User uid making the request.
 *
 * OPTIONAL ENV VARS
 *   DC_SITE_BASE_URL   Public URL prefix for the built docs. Default:
 *                      https://studio-documentation.contentstackapps.com
 *   DC_API_BASE_URL    DC API base. Default:
 *                      https://digital-concierge-api.contentstack.com/digital-concierge
 *
 * ────────────────────────────────────────────────────────────────────────────
 * CLI FLAGS
 * ────────────────────────────────────────────────────────────────────────────
 *   --dry-run          Log what would be sent; don't actually send. Prints the
 *                      exact URL, headers (redacted), multipart shape, and
 *                      metadata body for each file so you can verify against
 *                      DevTools captures.
 *   --all              Upload every markdown file (ignore the state file's
 *                      sha comparison). Use for the first-run backfill or
 *                      when you want to force a full re-index.
 *   --file <path>      Upload one specific file (path relative to docs/).
 *                      Useful for verifying the payload shape end-to-end.
 *
 * ────────────────────────────────────────────────────────────────────────────
 * UNCONFIRMED PAYLOAD SHAPES — the user flagged these in the original spec.
 * If DevTools "Copy as cURL" of the real POST / PUT differs, adjust here:
 *   - MULTIPART_FILE_FIELD_NAME: assumed "file". Some APIs use "source" or
 *     "upload"; DevTools will tell you.
 *   - metadata body: assumed a flat object `{ url, source_url, retrieved_at }`.
 *     Some APIs want an array of `[{ key, value }]` instead. Verify against
 *     the real PUT capture; adjust `buildMetadataBody()` if the shape differs.
 * ────────────────────────────────────────────────────────────────────────────
 */

import * as fs from "fs";
import * as path from "path";
import * as crypto from "crypto";
import { readDocsTree } from "./lib/docs-tree";

// ────────────────────────────────────────────────────────────────────────────
// Config
// ────────────────────────────────────────────────────────────────────────────

const ROOT = path.resolve(__dirname, "..");
const DOCS_DIR = path.join(ROOT, "docs");
const STATE_FILE = path.join(ROOT, ".dc-upload-state.json");

const DC_API_BASE = process.env.DC_API_BASE_URL
  || "https://digital-concierge-api.contentstack.com/digital-concierge";
const DC_SITE_BASE = (process.env.DC_SITE_BASE_URL
  || "https://studio-documentation.contentstackapps.com").replace(/\/+$/, "");

const CONCURRENCY = 2;

// Rate-limit + transient-failure retry policy. Kicks in on HTTP 429
// (rate-limited), 502/503/504 (transient upstream errors), or fetch-level
// network exceptions. Reads Retry-After header when present (seconds OR
// HTTP-date form). Backs off exponentially with jitter otherwise. Env-tunable.
const MAX_RETRIES     = Number(process.env.DC_MAX_RETRIES     ?? 5);
const INITIAL_BACKOFF = Number(process.env.DC_INITIAL_BACKOFF ?? 1000);   // ms
const MAX_BACKOFF     = Number(process.env.DC_MAX_BACKOFF     ?? 30000);  // ms

// How to handle a markdown file whose sha changed since the previous upload:
//   recreate    (default) DELETE old source uid → POST new. No orphan.
//   create-only            POST new only, leave old source in place (orphan).
//                          Use when DELETE isn't supported / permitted.
//   patch                  PATCH /sources/{uid} with new content (unconfirmed
//                          endpoint — may 404/405 depending on the DC API).
const UPDATE_MODE = process.env.DC_UPDATE_MODE || "recreate";

// Multipart POST /sources form shape — confirmed from a real DevTools capture:
//
//   Content-Disposition: form-data; name="content"; filename="<name>.md"
//   Content-Type: text/markdown
//   <file bytes>
//   ---
//   Content-Disposition: form-data; name="chunking_strategy"
//   {"type":"fixed","chunkSize":1000,"overlapSize":100}
//   ---
//   Content-Disposition: form-data; name="name"
//   <name>.md
//
// All three field names + the chunking JSON + Content-Type are overridable
// via env if DC ever changes the wire shape.
const MULTIPART_FILE_FIELD_NAME = process.env.DC_MULTIPART_FIELD_NAME || "content";
const MULTIPART_NAME_FIELD      = process.env.DC_NAME_FIELD           || "name";
const MULTIPART_CHUNK_FIELD     = process.env.DC_CHUNKING_FIELD       || "chunking_strategy";
const CHUNKING_STRATEGY = {
  type:        process.env.DC_CHUNK_TYPE    || "fixed",
  chunkSize:   Number(process.env.DC_CHUNK_SIZE    ?? 1000),
  overlapSize: Number(process.env.DC_CHUNK_OVERLAP ?? 100),
};

// Origin + referer confirmed from the DevTools OPTIONS preflight — DC only
// accepts requests originated from app.contentstack.com. Overridable via env
// if that ever changes.
const DC_ORIGIN  = process.env.DC_ORIGIN  || "https://app.contentstack.com";
const DC_REFERER = process.env.DC_REFERER || "https://app.contentstack.com/";

// The 6 credentials are prompted for at runtime, NOT read from env / persisted
// to disk (per user directive: "dont store and ask for each each time"). The
// only exception is in-memory reuse within a single script invocation.
type Creds = {
  authtoken:        string;
  organization_uid: string;
  api_key:          string;
  project_id:       string;
  agent_id:         string;
  user_uid:         string;
};
let CREDENTIALS: Creds | null = null;

// ────────────────────────────────────────────────────────────────────────────
// Types
// ────────────────────────────────────────────────────────────────────────────

// State tracked per markdown file. `uid` + `createdAt` are set after the POST
// succeeds; `metadataAt` is set after the PUT succeeds. If a run POSTs then
// crashes / errors on the PUT, `metadataAt` stays undefined — the next run
// treats that entry as "orphaned source, retry PUT only, do not re-POST"
// (avoids duplicate sources in DC when the metadata shape or the network
// blips).
type UploadState = Record<string, {
  uid:         string;
  sha:         string;
  createdAt:   string;
  metadataAt?: string;
}>;
type Args = { dryRun: boolean; all: boolean; file?: string; purge: boolean; purgeAll: boolean; yes: boolean; cleanupOrphans: boolean };

// ────────────────────────────────────────────────────────────────────────────
// Helpers
// ────────────────────────────────────────────────────────────────────────────

function stripNumPrefix(p: string): string {
  return p.split(/[\\/]/).map(seg => seg.replace(/^[0-9]+-/, "")).join("/");
}

// The docs tree decides where a doc page publishes, the same reader the build
// uses: 10-set-up/05-install-studio/05-prerequisites.md publishes at
// /install-studio/prerequisites, with the part folder dropped. A broken
// tree stops the upload, because the build would refuse to publish it too.
const DOCS_TREE = readDocsTree(DOCS_DIR);
if (DOCS_TREE.problems.length) {
  console.error(`✗ The docs tree has ${DOCS_TREE.problems.length} problem(s); run npm run lint:nav. Nothing uploaded.`);
  process.exit(1);
}
const TREE_PAGES = new Map(DOCS_TREE.pages.map(p => [p.srcRel, p]));

// A doc page lives in a numbered part folder; anything else in docs/ (the
// home page, the prompts mirror) publishes at its own path.
function isTreePath(mdRelPath: string): boolean {
  return /^\d+-/.test(mdRelPath.split("/")[0]);
}

function htmlUrlFor(mdRelPath: string): string {
  // docs/10-set-up/05-install-studio/05-prerequisites.md → /install-studio/prerequisites
  // docs/prompts/install-studio.md → /prompts/install-studio
  // docs/prompts/index.md → /prompts, docs/index.md → /
  const url = TREE_PAGES.get(mdRelPath)?.url
    ?? "/" + stripNumPrefix(mdRelPath).replace(/\.md$/, "").replace(/(^|\/)index$/, "");
  return `${DC_SITE_BASE}${url === "/" ? "/" : url.replace(/\/$/, "")}`;
}

// Rewrite relative `.md` links inside a source file to their absolute published
// HTML URLs, so DC's chat agent cites the correct public URL instead of the
// raw markdown path. Only touches inline `](…)` link targets — leaves external
// URLs, images, and anchors-only refs alone.
//
// `currentMdRelPath` is repo-relative (e.g. "docs/34-smart-containers/section-slots.md").
function rewriteMarkdownLinks(content: string, currentMdRelPath: string): string {
  const currentDir = path.posix.dirname(currentMdRelPath.replace(/\\/g, "/"));
  return content.replace(
    /\]\(([^)\s]+?\.md)(#[^)\s]*)?\)/g,
    (match, target: string, anchor: string | undefined) => {
      // Skip absolute URLs (http:, https:, mailto:, //cdn, etc.)
      if (/^([a-z][a-z0-9+.-]*:|\/\/)/i.test(target)) return match;
      // Resolve against the current file's dir, staying within the repo.
      const resolved = path.posix.normalize(path.posix.join(currentDir, target));
      // If normalization escapes above the repo root, leave it alone.
      if (resolved.startsWith("..")) return match;
      return `](${htmlUrlFor(resolved)}${anchor ?? ""})`;
    }
  );
}

function sha256(text: string): string {
  return crypto.createHash("sha256").update(text).digest("hex");
}

function loadState(): UploadState {
  if (!fs.existsSync(STATE_FILE)) return {};
  try {
    return JSON.parse(fs.readFileSync(STATE_FILE, "utf-8"));
  } catch {
    return {};
  }
}

function saveState(state: UploadState): void {
  fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2) + "\n");
}

// Match the set of markdown files the site publishes as pages:
// walk docs/ and docs/prompts/ recursively for .md files; skip build outputs,
// contributor-only docs, and hidden / underscore-prefixed dirs.
//
// Excluded on purpose (the site doesn't publish these as pages, so
// their derived source_url would 404):
//   - .*                  hidden files/dirs (.git, .obsidian, etc.)
//   - _<name>             research / working dirs (docs/_research)
//   - dist/               build output, not a source
//   - adr/                architecture decision records — contributor-only,
//                         not emitted as HTML
//   - assets/             SVG + screenshot assets, plus assets/svg/README.md
//                         which is an internal convention note, not user-facing
//   - AGENTS.md           contributor guidance, never emitted as public HTML
//   - draft pages         `draft: true` in the docs tree. Digital Concierge
//                         answers production visitors, so an unwritten page
//                         never reaches it, whatever the build includes.
const EXCLUDED_DIRS = new Set(["dist", "adr", "assets"]);
function walkMdFiles(dir: string, out: string[] = []): string[] {
  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
    if (
      entry.name.startsWith(".") ||
      entry.name.startsWith("_") ||
      EXCLUDED_DIRS.has(entry.name)
    ) continue;
    const p = path.join(dir, entry.name);
    if (entry.isDirectory()) {
      walkMdFiles(p, out);
    } else if (
      entry.name.endsWith(".md") &&
      entry.name !== "AGENTS.md"
    ) {
      const rel = path.relative(DOCS_DIR, p).split(path.sep).join("/");
      if (isTreePath(rel) && TREE_PAGES.get(rel)?.draft !== false) continue;
      out.push(p);
    }
  }
  return out;
}

function parseArgs(argv: string[]): Args {
  const args: Args = { dryRun: false, all: false, purge: false, purgeAll: false, yes: false, cleanupOrphans: false };
  for (let i = 0; i < argv.length; i++) {
    if      (argv[i] === "--dry-run")          args.dryRun = true;
    else if (argv[i] === "--all")              args.all = true;
    else if (argv[i] === "--file")             args.file = argv[++i];
    else if (argv[i] === "--purge")            args.purge = true;
    else if (argv[i] === "--purge-all")        args.purgeAll = true;      // LIST every source, DELETE all
    else if (argv[i] === "--yes")              args.yes = true;           // skip confirm-before-delete
    else if (argv[i] === "--cleanup-orphans")  args.cleanupOrphans = true; // opt-in orphan cleanup for renamed/deleted files
  }
  return args;
}

function authHeaders(): Record<string, string> {
  if (!CREDENTIALS) throw new Error("Credentials not collected — call promptCredentials() first.");
  return {
    "authtoken":        CREDENTIALS.authtoken,
    "organization_uid": CREDENTIALS.organization_uid,
    "x-api-key":        CREDENTIALS.api_key,
    "x-project-id":     CREDENTIALS.project_id,
    "x-agent-id":       CREDENTIALS.agent_id,
    "x-user-uid":       CREDENTIALS.user_uid,
    "origin":           DC_ORIGIN,
    "referer":          DC_REFERER,
  };
}

// ────────────────────────────────────────────────────────────────────────────
// Interactive credential prompts (no persistence)
// ────────────────────────────────────────────────────────────────────────────

function readLineVisible(prompt: string): Promise<string> {
  return new Promise(resolve => {
    process.stdout.write(prompt);
    let buf = "";
    const onData = (chunk: Buffer) => {
      const s = chunk.toString();
      for (const c of s) {
        if (c === "\n" || c === "\r") {
          process.stdin.off("data", onData);
          process.stdin.pause();
          process.stdout.write("\n");
          resolve(buf.trim());
          return;
        }
        buf += c;
      }
    };
    process.stdin.resume();
    process.stdin.on("data", onData);
  });
}

function readLineHidden(prompt: string): Promise<string> {
  return new Promise(resolve => {
    process.stdout.write(prompt);
    const stdin = process.stdin;
    const wasRaw = stdin.isRaw;
    stdin.setRawMode?.(true);
    stdin.resume();
    let buf = "";
    const onData = (chunk: Buffer) => {
      const s = chunk.toString();
      for (const c of s) {
        if (c === "\n" || c === "\r" || c === "") {   // enter / EOT
          stdin.off("data", onData);
          stdin.setRawMode?.(wasRaw ?? false);
          stdin.pause();
          process.stdout.write("\n");
          resolve(buf.trim());
          return;
        }
        if (c === "") process.exit(130);              // Ctrl-C
        if (c === "" || c === "\b") { buf = buf.slice(0, -1); continue; }   // backspace
        buf += c;
      }
    };
    stdin.on("data", onData);
  });
}

async function promptCredentials(): Promise<Creds | null> {
  // Non-TTY (piped stdin, CI, agent-run) — cannot prompt. Fall back to env vars
  // if they're set; otherwise skip. This is an ESCAPE HATCH for headless
  // scenarios only — the primary path is interactive prompting.
  if (!process.stdin.isTTY) {
    const envMap: Array<[keyof Creds, string]> = [
      ["authtoken",        "DC_AUTHTOKEN"],
      ["organization_uid", "DC_ORG_UID"],
      ["api_key",          "DC_API_KEY"],
      ["project_id",       "DC_PROJECT_ID"],
      ["agent_id",         "DC_AGENT_ID"],
      ["user_uid",         "DC_USER_UID"],
    ];
    const envCreds: Partial<Creds> = {};
    const missing: string[] = [];
    for (const [k, envName] of envMap) {
      const v = process.env[envName];
      if (v) envCreds[k] = v;
      else missing.push(envName);
    }
    if (missing.length === 0) {
      console.log("ℹ  Non-TTY detected — using DC_* env vars (headless fallback).");
      return envCreds as Creds;
    }
    console.log("ℹ  Non-TTY detected — cannot prompt for credentials interactively.");
    console.log(`   Missing env vars: ${missing.join(", ")}`);
    console.log("   Digital Concierge upload skipped.");
    console.log("   To run interactively: `npm run dc:upload` from a real terminal.");
    return null;
  }

  console.log("");
  console.log("Digital Concierge — credentials required");
  console.log("─────────────────────────────────────────");
  console.log("These are prompted for each run and NEVER persisted to disk.");
  console.log("Press Ctrl-C or answer 'n' below to skip the upload.");
  console.log("");

  const consent = (await readLineVisible("Provide credentials and upload changed docs? [y/N]: ")).toLowerCase();
  if (consent !== "y" && consent !== "yes") {
    console.log("");
    console.log("↩  Skipping Digital Concierge upload.");
    return null;
  }

  console.log("");
  console.log("Paste each value from your browser DevTools (Network → any DC request → Request Headers):");
  console.log("");

  const authtoken        = await readLineHidden ("  authtoken        (hidden): ");
  if (!authtoken) { console.log("  (empty authtoken — aborting)"); return null; }
  const organization_uid = await readLineVisible("  organization_uid        : ");
  const api_key          = await readLineVisible("  x-api-key               : ");
  const project_id       = await readLineVisible("  x-project-id            : ");
  const agent_id         = await readLineVisible("  x-agent-id              : ");
  const user_uid         = await readLineVisible("  x-user-uid              : ");

  const missing: string[] = [];
  if (!organization_uid) missing.push("organization_uid");
  if (!api_key)          missing.push("x-api-key");
  if (!project_id)       missing.push("x-project-id");
  if (!agent_id)         missing.push("x-agent-id");
  if (!user_uid)         missing.push("x-user-uid");
  if (missing.length) {
    console.log(`  (missing: ${missing.join(", ")} — aborting)`);
    return null;
  }

  console.log("");
  console.log("✓ Credentials collected (in-memory only, cleared on exit).");
  console.log("");
  return { authtoken, organization_uid, api_key, project_id, agent_id, user_uid };
}

function redactAuth(h: Record<string, string>): Record<string, string> {
  const out: Record<string, string> = {};
  for (const [k, v] of Object.entries(h)) {
    if (k === "authtoken") out[k] = v ? `${v.slice(0, 6)}…${v.slice(-4)} (${v.length} chars)` : "(missing)";
    else out[k] = v;
  }
  return out;
}

function buildMetadataBody(publicUrl: string) {
  // Real PUT payload shape confirmed from DevTools capture of the DC UI:
  //
  //   {
  //     "metadata": {
  //       "source_url":   { "value": "https://…/quickstart-setup.html",
  //                         "is_source": true },
  //       "retrieved_at": { "value": "2026-07-09T…",
  //                         "is_source": false }
  //     }
  //   }
  //
  // Key observations:
  //   - Wrapped in { metadata: {...} } (matches earlier 400 error that said
  //     "metadata must be an object with string keys and object values").
  //   - Only TWO metadata keys: source_url + retrieved_at (no `url` field).
  //   - Each value is an object with `value` + `is_source` flag. `is_source:
  //     true` marks the value that the DC RAG engine uses as the citation
  //     link when it surfaces this source in an answer.
  return {
    metadata: {
      source_url: {
        value:     publicUrl,
        is_source: true,
      },
      retrieved_at: {
        value:     new Date().toISOString(),
        is_source: false,
      },
    },
  };
}

// ────────────────────────────────────────────────────────────────────────────
// HTTP — retryable fetch with Retry-After + exponential backoff
// ────────────────────────────────────────────────────────────────────────────

function sleep(ms: number): Promise<void> {
  return new Promise(r => setTimeout(r, ms));
}

/** Parse a `Retry-After` header (RFC 7231): either integer seconds or an HTTP-date. */
function parseRetryAfterMs(header: string | null): number | null {
  if (!header) return null;
  const secs = Number(header);
  if (!Number.isNaN(secs)) return secs * 1000;
  const asDate = Date.parse(header);
  if (!Number.isNaN(asDate)) return Math.max(0, asDate - Date.now());
  return null;
}

/** Fetch that retries on 429/502/503/504 + network errors, with Retry-After + jittered exponential backoff. */
async function retryableFetch(url: string, init: RequestInit, opLabel: string): Promise<Response> {
  let lastErr: any = null;
  let backoff = INITIAL_BACKOFF;

  for (let attempt = 1; attempt <= MAX_RETRIES + 1; attempt++) {
    try {
      const res = await fetch(url, init);
      // Retryable statuses.
      if ([429, 502, 503, 504].includes(res.status)) {
        const retryAfter = parseRetryAfterMs(res.headers.get("retry-after"));
        const waitMs = retryAfter ?? backoff;
        if (attempt > MAX_RETRIES) {
          const body = await res.text().catch(() => "");
          throw new Error(`${opLabel} exhausted retries: HTTP ${res.status} — ${body.slice(0, 200)}`);
        }
        console.log(`  ⏸  ${opLabel} → HTTP ${res.status}${retryAfter ? " (Retry-After honored)" : ""}; retrying in ${Math.round(waitMs / 1000)}s (attempt ${attempt}/${MAX_RETRIES})`);
        await sleep(waitMs);
        backoff = Math.min(MAX_BACKOFF, backoff * 2 + Math.floor(Math.random() * 500));
        continue;
      }
      return res;   // success OR non-retryable error — caller decides
    } catch (e: any) {
      lastErr = e;
      if (attempt > MAX_RETRIES) throw new Error(`${opLabel} network error (${MAX_RETRIES} retries exhausted): ${e.message || e}`);
      console.log(`  ⏸  ${opLabel} → network error (${e.message || e}); retrying in ${Math.round(backoff / 1000)}s (attempt ${attempt}/${MAX_RETRIES})`);
      await sleep(backoff);
      backoff = Math.min(MAX_BACKOFF, backoff * 2 + Math.floor(Math.random() * 500));
    }
  }
  throw lastErr ?? new Error(`${opLabel} failed`);
}

// ────────────────────────────────────────────────────────────────────────────
// HTTP — endpoints
// ────────────────────────────────────────────────────────────────────────────

/**
 * List every source in the DC project via paginated GET /sources.
 * Endpoint shape confirmed from DevTools capture:
 *   GET /sources?limit=30&skip=<N>&include_count=true&include_users=true&sort=created_at&order=desc
 * Response uses `include_count=true` to signal a total, letting us paginate
 * until we've collected everything. Response body shape is unconfirmed —
 * parses defensively across common keys.
 */
type ListedSource = { uid: string; name?: string; source_url?: string };
async function listAllSources(dryRun: boolean): Promise<ListedSource[]> {
  if (dryRun) {
    console.log(`  [dry-run] GET ${DC_API_BASE}/sources?limit=30&skip=0&include_count=true&sort=created_at&order=desc`);
    console.log(`  [dry-run] (returning 0 items in dry-run — LIST-and-DELETE flow can only be previewed live)`);
    return [];
  }

  const pageSize = 100;   // larger than DC UI's 30 to reduce request count on big projects
  const all: ListedSource[] = [];
  let skip = 0;
  let total: number | null = null;

  while (true) {
    const params = new URLSearchParams({
      limit:          String(pageSize),
      skip:           String(skip),
      include_count:  "true",
      include_users:  "true",
      sort:           "created_at",
      order:          "desc",
    });
    const url = `${DC_API_BASE}/sources?${params.toString()}`;
    const res = await retryableFetch(url, {
      method: "GET",
      headers: authHeaders(),
    }, `GET /sources?skip=${skip}`);
    const text = await res.text();
    if (!res.ok) throw new Error(`GET /sources failed at skip=${skip}: HTTP ${res.status} — ${text.slice(0, 300)}`);
    let parsed: any;
    try { parsed = JSON.parse(text); }
    catch { throw new Error(`GET /sources returned non-JSON at skip=${skip}: ${text.slice(0, 300)}`); }

    // Defensively find the array of items across common wrapper shapes.
    const items: any[] =
      parsed?.sources ??
      parsed?.data    ??
      parsed?.items   ??
      parsed?.results ??
      (Array.isArray(parsed) ? parsed : []);
    const countThisPage = items.length;

    // Track the total from include_count response.
    if (total === null) {
      total = parsed?.count ?? parsed?.total ?? parsed?.total_count ?? parsed?.meta?.total ?? null;
      if (total !== null) console.log(`  ℹ  Total sources reported by API: ${total}`);
    }

    for (const it of items) {
      const uid = it?.uid || it?._id || it?.id;
      if (!uid) continue;
      all.push({
        uid,
        name:       it?.name || it?.filename || it?.title,
        source_url: it?.metadata?.source_url?.value || it?.metadata?.source_url,
      });
    }

    console.log(`  · fetched page: skip=${skip} count=${countThisPage} (total collected so far: ${all.length})`);
    if (countThisPage < pageSize) break;                              // last page
    if (total !== null && all.length >= total) break;                 // hit the count
    skip += pageSize;
    if (skip > 100_000) throw new Error("GET /sources pagination safety cap hit at skip=100000 — probable API loop, aborting");
  }

  return all;
}

/**
 * DELETE an existing source. Used by the `recreate` update flow to prevent
 * orphan accumulation when a markdown file changes. Non-fatal — if DC's DELETE
 * endpoint isn't available (404/405) we log a warning and let the caller
 * proceed with just POSTing the new source (orphan accepted).
 */
async function deleteSource(uid: string, dryRun: boolean): Promise<boolean> {
  const url = `${DC_API_BASE}/sources/${uid}`;
  if (dryRun) {
    console.log(`  [dry-run] DELETE ${url}`);
    return true;
  }
  const res = await retryableFetch(url, {
    method: "DELETE",
    headers: authHeaders(),
  }, `DELETE /sources/${uid}`);
  if (res.ok || res.status === 204) return true;
  const body = await res.text().catch(() => "");
  console.log(`  ⚠  DELETE /sources/${uid} returned HTTP ${res.status} — ${body.slice(0, 200)}`);
  return false;
}

async function createSource(mdRelPath: string, absFile: string, dryRun: boolean): Promise<string> {
  const url = `${DC_API_BASE}/sources`;
  const filename = path.basename(mdRelPath);
  const chunkingJson = JSON.stringify(CHUNKING_STRATEGY);

  if (dryRun) {
    console.log(`  [dry-run] POST ${url}`);
    console.log(`  [dry-run] Headers: ${JSON.stringify(redactAuth(authHeaders()))}`);
    console.log(`  [dry-run] multipart '${MULTIPART_FILE_FIELD_NAME}' → ${filename} (${fs.statSync(absFile).size} bytes, text/markdown)`);
    console.log(`  [dry-run] multipart '${MULTIPART_CHUNK_FIELD}'   → ${chunkingJson}`);
    console.log(`  [dry-run] multipart '${MULTIPART_NAME_FIELD}'    → ${filename}`);
    return "dry-run-uid-" + sha256(mdRelPath).slice(0, 24);
  }

  const raw = fs.readFileSync(absFile, "utf-8");
  const rewritten = rewriteMarkdownLinks(raw, mdRelPath);
  const bytes = Buffer.from(rewritten, "utf-8");
  const form = new FormData();
  // Order matches the DevTools capture: content (file) → chunking_strategy → name
  form.append(MULTIPART_FILE_FIELD_NAME, new Blob([bytes], { type: "text/markdown" }), filename);
  form.append(MULTIPART_CHUNK_FIELD,     chunkingJson);
  form.append(MULTIPART_NAME_FIELD,      filename);

  const res = await retryableFetch(url, {
    method: "POST",
    headers: authHeaders(),   // fetch sets Content-Type incl. boundary from FormData
    body: form as any,
  }, "POST /sources");
  const bodyText = await res.text();
  if (!res.ok) {
    throw new Error(`POST /sources failed: HTTP ${res.status} — ${bodyText.slice(0, 400)}`);
  }
  let parsed: any;
  try { parsed = JSON.parse(bodyText); }
  catch { throw new Error(`POST /sources returned non-JSON: ${bodyText.slice(0, 400)}`); }
  const uid = parsed?.uid || parsed?.data?.uid || parsed?.source?.uid;
  if (!uid) throw new Error(`POST /sources succeeded but no uid in response: ${bodyText.slice(0, 400)}`);
  return uid;
}

async function updateMetadata(uid: string, publicUrl: string, dryRun: boolean): Promise<void> {
  const url = `${DC_API_BASE}/sources/${uid}`;
  const body = buildMetadataBody(publicUrl);

  if (dryRun) {
    console.log(`  [dry-run] PUT ${url}`);
    console.log(`  [dry-run] Body: ${JSON.stringify(body)}`);
    return;
  }

  const res = await retryableFetch(url, {
    method: "PUT",
    headers: { ...authHeaders(), "Content-Type": "application/json" },
    body: JSON.stringify(body),
  }, `PUT /sources/${uid}`);
  const bodyText = await res.text();
  if (!res.ok) {
    throw new Error(`PUT /sources/${uid} failed: HTTP ${res.status} — ${bodyText.slice(0, 400)}`);
  }
}

// ────────────────────────────────────────────────────────────────────────────
// Concurrency
// ────────────────────────────────────────────────────────────────────────────

async function pool<T>(items: T[], size: number, worker: (item: T, index: number) => Promise<void>): Promise<void> {
  let cursor = 0;
  const runners = Array.from({ length: Math.min(size, items.length) }, async () => {
    while (true) {
      const i = cursor++;
      if (i >= items.length) return;
      await worker(items[i], i);
    }
  });
  await Promise.all(runners);
}

// ────────────────────────────────────────────────────────────────────────────
// Main
// ────────────────────────────────────────────────────────────────────────────

// ────────────────────────────────────────────────────────────────────────────
// Purge — DELETE every source uid tracked in the state file
// ────────────────────────────────────────────────────────────────────────────

// ────────────────────────────────────────────────────────────────────────────
// Orphan cleanup — for files that have been renamed / deleted locally
// ────────────────────────────────────────────────────────────────────────────
//
// The state file (`.dc-upload-state.json`) is keyed by the file's docs-relative
// path. When a file is renamed or deleted locally, `dc:upload` uploads the
// new path but does NOT touch the old state entry — leaving a stale DC source
// that keeps serving the old content indefinitely.
//
// `cleanupOrphans` closes that gap. Given the current state file + current
// contents of `docs/`, it finds every state entry whose local file no longer
// exists, DELETEs the DC source, and removes the state entry. Opt-in via
// `--cleanup-orphans` (or the `dc:upload:cleanup` npm script). Safe to run
// alongside a regular upload — the upload phase runs first, so a rename in
// the same run does both: upload new + delete old.
//
// Distinct from `--purge` (which deletes EVERY tracked source — nuclear).
// Distinct from `--purge-all` (which LISTs and deletes every DC source, tracked
// or not — most nuclear). `--cleanup-orphans` is the precise, per-file version.
async function cleanupOrphans(args: Args): Promise<{ deleted: number; failed: number; skipped: number }> {
  const state = loadState();
  const entries = Object.entries(state);

  const allFiles = walkMdFiles(DOCS_DIR);
  const presentRelPaths = new Set(
    allFiles.map(abs => path.relative(DOCS_DIR, abs).split(path.sep).join("/"))
  );

  const orphans = entries.filter(([rel]) => !presentRelPaths.has(rel));

  console.log("╭────────────────────────────────────────────────────────────╮");
  console.log("│ Digital Concierge — orphan cleanup                        │");
  console.log("╰────────────────────────────────────────────────────────────╯");
  console.log(`  State file:    ${path.relative(process.cwd(), STATE_FILE)}`);
  console.log(`  Tracked:       ${entries.length} uid${entries.length === 1 ? "" : "s"}`);
  console.log(`  Local files:   ${presentRelPaths.size}`);
  console.log(`  Orphans:       ${orphans.length}  (in state, no local file)`);
  console.log(`  Mode:          ${args.dryRun ? "DRY-RUN (no DELETEs sent)" : "LIVE"}`);
  console.log(``);

  if (orphans.length === 0) {
    console.log(`✓ No orphans — every state entry has a matching local file.`);
    console.log(``);
    return { deleted: 0, failed: 0, skipped: 0 };
  }

  // Preview.
  console.log(`  Orphans to DELETE from DC + remove from state:`);
  const previewLimit = 30;
  for (const [rel, entry] of orphans.slice(0, previewLimit)) {
    console.log(`    ${entry.uid}   ← ${rel}`);
  }
  if (orphans.length > previewLimit) console.log(`    …and ${orphans.length - previewLimit} more`);
  console.log(``);

  let deleted = 0;
  let failed = 0;
  let skipped = 0;

  await pool(orphans, CONCURRENCY, async ([rel, entry], index) => {
    const num = `[${String(index + 1).padStart(String(orphans.length).length, " ")}/${orphans.length}]`;
    try {
      console.log(`${num} DELETE ${entry.uid}  ← ${rel}`);
      const ok = await deleteSource(entry.uid, args.dryRun);
      if (ok) {
        deleted++;
        if (!args.dryRun) {
          const fresh = loadState();
          delete fresh[rel];
          saveState(fresh);
        }
        console.log(`${num}   ✓ deleted + state entry removed`);
      } else {
        failed++;
        console.log(`${num}   ✗ DELETE returned non-success — leaving state entry in place`);
      }
    } catch (e: any) {
      failed++;
      console.error(`${num}   ✗ FAILED for ${rel}: ${e.message}`);
    }
  });

  console.log(``);
  console.log(`  ────────────────────────────────────────`);
  console.log(`  Orphan cleanup done.`);
  console.log(`    ✓ ${deleted} deleted`);
  console.log(`    ✗ ${failed} failed`);
  if (skipped > 0) console.log(`    · ${skipped} skipped`);
  console.log(``);

  return { deleted, failed, skipped };
}

async function purge(args: Args): Promise<void> {
  const state = loadState();
  const entries = Object.entries(state);

  console.log("╭────────────────────────────────────────────────────────────╮");
  console.log("│ Digital Concierge — PURGE (delete tracked sources)        │");
  console.log("│ ⚠  NUCLEAR: deletes every source tracked in the state    │");
  console.log("│ ⚠  file. NOT for rename orphan cleanup — for that,       │");
  console.log("│ ⚠  use --cleanup-orphans (npm run dc:upload:cleanup).    │");
  console.log("╰────────────────────────────────────────────────────────────╯");
  console.log(`  State file:  ${path.relative(process.cwd(), STATE_FILE)}`);
  console.log(`  Tracked:     ${entries.length} uid${entries.length === 1 ? "" : "s"}`);
  console.log(`  Concurrency: ${CONCURRENCY}`);
  console.log(`  Mode:        ${args.dryRun ? "DRY-RUN (no DELETEs sent)" : "LIVE"}`);
  console.log(``);

  if (entries.length === 0) {
    console.log(`✓ State file is empty — nothing to purge.`);
    console.log(``);
    console.log(`⚠  If the DC project has sources that were POSTed but never had metadata`);
    console.log(`   attached (from earlier failed runs before orphan-recovery landed), they`);
    console.log(`   are NOT in the state file. Delete those from the DC UI manually, or`);
    console.log(`   send a GET /sources capture so we can wire a list-and-delete flow.`);
    return;
  }

  // Preview.
  console.log(`  About to delete:`);
  for (const [rel, entry] of entries.slice(0, 20)) {
    console.log(`    ${entry.uid}   ← ${rel}`);
  }
  if (entries.length > 20) console.log(`    …and ${entries.length - 20} more`);
  console.log(``);

  // Warn about untracked orphans.
  console.log(`⚠  This only deletes sources tracked in the local state file.`);
  console.log(`   Sources created by earlier failed runs (before orphan-recovery landed)`);
  console.log(`   are not in state → will NOT be deleted here. Check the DC UI after`);
  console.log(`   this completes and delete any leftovers manually.`);
  console.log(``);

  // Prompt for credentials (same flow as upload).
  if (args.dryRun) {
    CREDENTIALS = {
      authtoken: "DRY_RUN_PLACEHOLDER", organization_uid: "org_dry_run",
      api_key: "blt_dry_run", project_id: "proj_dry_run",
      agent_id: "agent_dry_run", user_uid: "usr_dry_run",
    };
    console.log("ℹ  Dry-run mode — using placeholder credentials, no requests will be sent.");
  } else {
    CREDENTIALS = await promptCredentials();
    if (!CREDENTIALS) { console.log("↩  Purge skipped."); return; }

    // Extra confirm before destructive action, unless --yes was passed.
    if (!args.yes) {
      if (!process.stdin.isTTY) {
        console.log("✗ Confirmation required for destructive purge (non-TTY session).");
        console.log("  Re-run with --yes to skip the confirm, or run in a real terminal.");
        return;
      }
      const confirm = (await readLineVisible(`\nType 'DELETE' (uppercase) to confirm purge of ${entries.length} source(s): `)).trim();
      if (confirm !== "DELETE") {
        console.log("↩  Confirmation not received — purge aborted. Nothing was deleted.");
        return;
      }
      console.log("");
    }
  }

  // DELETE with concurrency + rate-limit retry.
  console.log(`  Deleting…`);
  console.log(``);
  let deleted = 0, failed = 0;
  const startedAt = Date.now();
  const items = entries.map(([rel, entry]) => ({ rel, uid: entry.uid }));

  await pool(items, CONCURRENCY, async (item, index) => {
    const num = `[${String(index + 1).padStart(String(items.length).length, " ")}/${items.length}]`;
    try {
      console.log(`${num} DELETE ${item.uid}  ← ${item.rel}`);
      const ok = await deleteSource(item.uid, args.dryRun);
      if (ok) {
        console.log(`${num}   ✓ deleted`);
        if (!args.dryRun) {
          delete state[item.rel];
          saveState(state);
        }
        deleted++;
      } else {
        console.log(`${num}   ⚠ DELETE returned non-2xx — leaving in state`);
        failed++;
      }
    } catch (e: any) {
      failed++;
      console.error(`${num}   ✗ FAILED: ${e.message}`);
    }
  });

  const dt = ((Date.now() - startedAt) / 1000).toFixed(1);
  console.log(``);
  console.log(`  ────────────────────────────────────────`);
  console.log(`  Done in ${dt}s.`);
  console.log(`    ✓ ${deleted} deleted`);
  console.log(`    ✗ ${failed} failed`);
  console.log(``);
  if (deleted > 0 && !args.dryRun) {
    console.log(`✓ State entries for deleted sources removed. Next \`dc:upload:all\` will re-create fresh.`);
  }
  if (failed > 0) process.exit(1);
}

/**
 * LIST + DELETE every source in the DC project (not just what's in the local
 * state file). Use this when the project has orphans from failed runs — the
 * local `.dc-upload-state.json` only tracks sources whose POST *succeeded*, so
 * `--purge` misses anything created before orphan-recovery landed.
 */
async function purgeAll(args: Args): Promise<void> {
  console.log("╭────────────────────────────────────────────────────────────╮");
  console.log("│ Digital Concierge — PURGE ALL (LIST + DELETE every source) │");
  console.log("│ ⚠  MOST NUCLEAR: LISTs every source on the DC project    │");
  console.log("│ ⚠  and DELETEs all — even sources not tracked locally.   │");
  console.log("│ ⚠  For rename orphan cleanup, use --cleanup-orphans      │");
  console.log("│ ⚠  (npm run dc:upload:cleanup) instead.                  │");
  console.log("╰────────────────────────────────────────────────────────────╯");
  console.log(`  Concurrency: ${CONCURRENCY}`);
  console.log(`  Mode:        ${args.dryRun ? "DRY-RUN (no DELETEs sent, no LIST either)" : "LIVE"}`);
  console.log(``);
  console.log(`⚠  This deletes EVERY source in the DC project, whether tracked locally or not.`);
  console.log(``);

  // Credentials first — we need them for the LIST call.
  if (args.dryRun) {
    CREDENTIALS = {
      authtoken: "DRY_RUN_PLACEHOLDER", organization_uid: "org_dry_run",
      api_key: "blt_dry_run", project_id: "proj_dry_run",
      agent_id: "agent_dry_run", user_uid: "usr_dry_run",
    };
    console.log("ℹ  Dry-run mode — using placeholder credentials, no requests will be sent.");
  } else {
    CREDENTIALS = await promptCredentials();
    if (!CREDENTIALS) { console.log("↩  Purge-all skipped."); return; }
  }

  // LIST.
  console.log(``);
  console.log(`  Listing all sources…`);
  const sources = await listAllSources(args.dryRun);
  console.log(``);
  console.log(`  Found ${sources.length} source(s) in the DC project.`);

  if (sources.length === 0) {
    console.log(`✓ Nothing to delete.`);
    return;
  }

  // Preview.
  console.log(``);
  console.log(`  Sample of what will be deleted:`);
  for (const s of sources.slice(0, 20)) {
    console.log(`    ${s.uid}   ← ${s.name ?? "(no name)"}`);
  }
  if (sources.length > 20) console.log(`    …and ${sources.length - 20} more`);
  console.log(``);

  // Confirm.
  if (!args.dryRun && !args.yes) {
    if (!process.stdin.isTTY) {
      console.log("✗ Confirmation required for destructive purge-all (non-TTY session).");
      console.log("  Re-run with --yes to skip the confirm, or run in a real terminal.");
      return;
    }
    const confirm = (await readLineVisible(`\nType 'DELETE' (uppercase) to confirm purge of ${sources.length} source(s): `)).trim();
    if (confirm !== "DELETE") {
      console.log("↩  Confirmation not received — purge-all aborted. Nothing was deleted.");
      return;
    }
    console.log(``);
  }

  // DELETE with concurrency + rate-limit retry.
  console.log(`  Deleting…`);
  console.log(``);
  const state = loadState();
  const stateByUid = new Map<string, string>();
  for (const [rel, entry] of Object.entries(state)) stateByUid.set(entry.uid, rel);

  let deleted = 0, failed = 0;
  const startedAt = Date.now();

  await pool(sources, CONCURRENCY, async (item, index) => {
    const num = `[${String(index + 1).padStart(String(sources.length).length, " ")}/${sources.length}]`;
    try {
      console.log(`${num} DELETE ${item.uid}  ${item.name ? `← ${item.name}` : ""}`);
      const ok = await deleteSource(item.uid, args.dryRun);
      if (ok) {
        console.log(`${num}   ✓ deleted`);
        if (!args.dryRun) {
          const rel = stateByUid.get(item.uid);
          if (rel) { delete state[rel]; saveState(state); }
        }
        deleted++;
      } else {
        console.log(`${num}   ⚠ DELETE returned non-2xx`);
        failed++;
      }
    } catch (e: any) {
      failed++;
      console.error(`${num}   ✗ FAILED: ${e.message}`);
    }
  });

  const dt = ((Date.now() - startedAt) / 1000).toFixed(1);
  console.log(``);
  console.log(`  ────────────────────────────────────────`);
  console.log(`  Done in ${dt}s.`);
  console.log(`    ✓ ${deleted} deleted`);
  console.log(`    ✗ ${failed} failed`);
  console.log(``);
  if (failed > 0) process.exit(1);
}

async function main() {
  const args = parseArgs(process.argv.slice(2));

  // Purge modes short-circuit the upload flow.
  if (args.purgeAll) {
    await purgeAll(args);
    return;
  }
  if (args.purge) {
    await purge(args);
    return;
  }

  // `--cleanup-orphans` runs as a post-upload phase at the end of main().
  // If no files need uploading AND the flag is on, we still want to enter the
  // credential-prompt path (see the "nothing to upload" short-circuit below).

  // Collect files BEFORE prompting — if nothing changed there's no point asking.
  const stateForCheck = loadState();
  const allFilesForCheck = walkMdFiles(DOCS_DIR);
  const singleFileForCheck = args.file ? path.resolve(args.file) : null;
  const targetForCheck = singleFileForCheck
    ? allFilesForCheck.filter(f => path.resolve(f) === singleFileForCheck)
    : allFilesForCheck;

  let willUpload = 0;
  for (const abs of targetForCheck) {
    const rel = path.relative(DOCS_DIR, abs).split(path.sep).join("/");
    const hash = sha256(rewriteMarkdownLinks(fs.readFileSync(abs, "utf-8"), rel));
    const prev = stateForCheck[rel];
    if (args.all || !prev || prev.sha !== hash) willUpload++;
  }

  if (willUpload === 0 && !args.dryRun && !args.cleanupOrphans) {
    console.log("✓ Digital Concierge: nothing to upload — every doc's sha matches the state.");
    return;
  }
  if (willUpload === 0 && args.cleanupOrphans) {
    console.log("ℹ  Digital Concierge: nothing to upload, but --cleanup-orphans requested. Running orphan cleanup only.");
  }

  // Prompt for credentials each run (per user directive — no persistence).
  // Dry-run mode uses placeholder values so no real creds are exchanged.
  if (args.dryRun) {
    CREDENTIALS = {
      authtoken:        "DRY_RUN_PLACEHOLDER",
      organization_uid: "org_dry_run",
      api_key:          "blt_dry_run",
      project_id:       "proj_dry_run",
      agent_id:         "agent_dry_run",
      user_uid:         "usr_dry_run",
    };
    console.log("ℹ  Dry-run mode — using placeholder credentials, no requests will be sent.");
  } else {
    console.log(`ℹ  ${willUpload} doc${willUpload === 1 ? "" : "s"} changed since last upload.`);
    CREDENTIALS = await promptCredentials();
    if (!CREDENTIALS) process.exit(0);
  }

  // Collect files.
  const state = loadState();
  const allFiles = walkMdFiles(DOCS_DIR);
  const singleFileAbs = args.file ? path.resolve(args.file) : null;
  const targetFiles = singleFileAbs
    ? allFiles.filter(f => path.resolve(f) === singleFileAbs)
    : allFiles;

  if (singleFileAbs && targetFiles.length === 0) {
    console.error(`✗ --file ${args.file} not found under docs/`);
    process.exit(1);
  }

  // Categorize.
  const toUpload: Array<{ absFile: string; rel: string; hash: string; isUpdate: boolean }> = [];
  let unchanged = 0;
  for (const abs of targetFiles) {
    const rel = path.relative(DOCS_DIR, abs).split(path.sep).join("/");
    const text = rewriteMarkdownLinks(fs.readFileSync(abs, "utf-8"), rel);
    const hash = sha256(text);
    const prev = state[rel];
    // Skip only if sha matches AND we have a successful metadata timestamp.
    // A prior POST-succeeded-but-PUT-failed run leaves prev.uid set but
    // prev.metadataAt undefined — that entry MUST be reprocessed to attach
    // metadata to the existing DC source (won't duplicate — see orphan-
    // recovery branch in the pool worker).
    if (!args.all && prev && prev.sha === hash && prev.metadataAt) {
      unchanged++;
      continue;
    }
    toUpload.push({ absFile: abs, rel, hash, isUpdate: !!prev });
  }

  // Summary before work.
  console.log("╭────────────────────────────────────────────────────────────╮");
  console.log("│ Digital Concierge — Sources upload                        │");
  console.log("╰────────────────────────────────────────────────────────────╯");
  console.log(`  Docs root:        ${path.relative(process.cwd(), DOCS_DIR)}`);
  console.log(`  API base:         ${DC_API_BASE}`);
  console.log(`  Public site URL:  ${DC_SITE_BASE}`);
  console.log(`  Concurrency:      ${CONCURRENCY}`);
  console.log(`  State file:       ${path.relative(process.cwd(), STATE_FILE)}`);
  console.log(`  Mode:             ${args.dryRun ? "DRY-RUN (no requests sent)" : args.all ? "FULL RE-INDEX" : "CHANGED FILES ONLY"}`);
  console.log(``);
  console.log(`  Files scanned:    ${targetFiles.length}`);
  console.log(`  Unchanged:        ${unchanged} (skipping)`);
  console.log(`  To upload:        ${toUpload.length}`);
  console.log(``);

  if (toUpload.length === 0 && !args.cleanupOrphans) {
    console.log(`✓ Nothing to upload — every file's sha matches the state.`);
    return;
  }
  if (toUpload.length === 0 && args.cleanupOrphans) {
    console.log(`ℹ  Nothing new to upload. Running orphan cleanup only.`);
    console.log(``);
    await cleanupOrphans(args);
    return;
  }

  // List each file with the URL it'll get as metadata.
  console.log(`  Files to upload:`);
  for (const item of toUpload) {
    const action = item.isUpdate ? "UPDATE" : "CREATE";
    console.log(`    [${action}] ${item.rel}`);
    console.log(`        →  ${htmlUrlFor(item.rel)}`);
  }
  console.log(``);
  console.log(`  Starting upload (max ${CONCURRENCY} concurrent)…`);
  console.log(``);

  // Upload with a concurrency cap.
  let ok = 0, failed = 0;
  const startedAt = Date.now();

  await pool(toUpload, CONCURRENCY, async (item, index) => {
    const num = `[${String(index + 1).padStart(String(toUpload.length).length, " ")}/${toUpload.length}]`;
    const publicUrl = htmlUrlFor(item.rel);
    const prev = state[item.rel];
    let uid: string | undefined = prev?.uid;

    try {
      console.log(`${num} → ${item.rel}`);

      // ── Three branches based on prior state ──
      //
      // A) prev.uid + prev.metadataAt + sha changed
      //    → content update. Depending on DC_UPDATE_MODE:
      //       recreate    (default) DELETE old uid → POST new. No orphan.
      //       create-only            POST new only (old uid orphaned in DC).
      //       patch                  Reserved — not yet implemented; fall back to recreate.
      //
      // B) prev.uid + !prev.metadataAt (any sha)
      //    → orphan recovery. Skip POST; retry PUT metadata on the existing uid.
      //
      // C) no prev.uid
      //    → first-time upload. POST + PUT.
      //
      const shaChanged = prev?.sha && prev.sha !== item.hash;

      if (uid && prev?.metadataAt && shaChanged) {
        // Branch A — content update.
        if (UPDATE_MODE === "create-only") {
          console.log(`${num}   ⚠  content changed; UPDATE_MODE=create-only → old uid=${uid} will be orphaned`);
          uid = undefined;   // fall through to POST
        } else {
          // Default: DELETE old → POST new.
          console.log(`${num}   ↻ content changed; DELETE old uid=${uid} then POST new`);
          const deleted = await deleteSource(uid, args.dryRun);
          if (!deleted) {
            console.log(`${num}   ⚠  DELETE didn't succeed — new POST will leave the old uid orphaned in DC`);
          }
          uid = undefined;   // force POST below
        }
      }

      if (uid && !prev?.metadataAt) {
        // Branch B — orphan recovery.
        console.log(`${num}   ↻ recovering orphan (uid=${uid} exists, metadata never set)`);
      } else if (!uid) {
        // Branch C — first-time upload (or content-update after DELETE above).
        console.log(`${num}   POST /sources …`);
        uid = await createSource(item.rel, item.absFile, args.dryRun);
        console.log(`${num}   ✓ created uid=${uid}`);
        if (!args.dryRun) {
          // Persist uid IMMEDIATELY after POST — before attempting PUT. Ensures
          // that a PUT failure on a fresh POST leaves us in the same recovery
          // shape as an orphan from a prior run.
          state[item.rel] = { uid, sha: item.hash, createdAt: new Date().toISOString() };
          saveState(state);
        }
      } else {
        // uid + metadataAt + sha unchanged. Only reachable via --all (the
        // sha-unchanged filter above lets these through in that mode).
        // Interpretation: user asked for a re-index → re-PUT metadata to
        // refresh retrieved_at, keep the existing source.
        console.log(`${num}   ↻ metadata refresh (--all): uid=${uid}, sha unchanged`);
      }

      console.log(`${num}   PUT /sources/${uid} { metadata } …`);
      await updateMetadata(uid!, publicUrl, args.dryRun);
      console.log(`${num}   ✓ metadata set (source_url=${publicUrl})`);
      if (!args.dryRun) {
        state[item.rel] = {
          ...(state[item.rel] || { uid: uid!, sha: item.hash, createdAt: new Date().toISOString() }),
          metadataAt: new Date().toISOString(),
        };
        saveState(state);
      }
      ok++;
    } catch (e: any) {
      failed++;
      console.error(`${num}   ✗ FAILED ${item.rel}: ${e.message}`);
    }
  });

  const dt = ((Date.now() - startedAt) / 1000).toFixed(1);
  console.log(``);
  console.log(`  ────────────────────────────────────────`);
  console.log(`  Upload done in ${dt}s.`);
  console.log(`    ✓ ${ok} uploaded`);
  console.log(`    ✗ ${failed} failed`);
  console.log(`    · ${unchanged} unchanged (skipped)`);
  console.log(``);

  // Post-upload: orphan cleanup, if requested. Runs after the upload phase so
  // a rename in the same run gets the new page uploaded first, then the old
  // DC entry deleted.
  if (args.cleanupOrphans) {
    const result = await cleanupOrphans(args);
    if (result.failed > 0 && failed === 0) process.exit(1);
  }

  if (failed > 0) process.exit(1);
}

main().catch(e => {
  console.error(`✗ Unexpected error: ${e.stack || e.message}`);
  process.exit(1);
});