Quickstart

import { resolve } from "node:path";
import { pathToFileURL } from "node:url";

export const API_ORIGIN = "https://www.veridionmarkets.com";

export function recentFilters(now = new Date()) {
  const from = new Date(now);
  from.setUTCDate(from.getUTCDate() - 30);
  return { transaction_date_from: from.toISOString().slice(0, 10), limit: "25" };
}

async function request(url, fetcher, headers = {}) {
  const response = await fetcher(url, {
    headers: { Accept: "application/json", ...headers },
    redirect: "error",
    signal: AbortSignal.timeout(30_000),
  });
  if (response.status === 304) {
    if (!headers["If-None-Match"]) throw new Error("Unexpected 304; no state committed");
    return { response, body: null };
  }
  const contentType = response.headers.get("content-type") || "";
  if (!/\bapplication\/(?:problem\+)?json\b/i.test(contentType)) {
    throw new Error(`HTTP ${response.status}: expected JSON; no state committed`);
  }
  const body = await response.json();
  if (!response.ok) {
    const error = new Error(`HTTP ${response.status}: ${body.code || "request_failed"}`);
    Object.assign(error, {
      code: body.code, requestId: response.headers.get("x-request-id"),
      retryAfter: response.headers.get("retry-after"),
    });
    throw error;
  }
  return { response, body };
}

// Three requests: a recent page, its conditional read, then coverage.
export async function preview({ fetcher = fetch, now = new Date() } = {}) {
  const url = new URL("/api/v1/disclosures", API_ORIGIN);
  url.search = new URLSearchParams(recentFilters(now)).toString();
  const first = await request(url, fetcher);
  if (!Array.isArray(first.body?.data)) throw new Error("Disclosure page unavailable");
  const etag = first.response.headers.get("etag");
  if (!etag) throw new Error("ETag unavailable; conditional request not attempted");
  const conditional = await request(url, fetcher, { "If-None-Match": etag });
  const coverage = await request(new URL("/api/v1/coverage", API_ORIGIN), fetcher);
  return {
    firstPage: first.body,
    conditional: { status: conditional.response.status, body: conditional.body },
    coverage: coverage.body,
  };
}

function requiredString(value) {
  return typeof value === "string" && value.length > 0;
}

function timestamp(value) {
  return typeof value === "string"
    && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$/.test(value)
    && Number.isFinite(Date.parse(value));
}

// State is replaced only after every page succeeds. Keep one writer per replica.
/**
 * @typedef {{ disclosure_id: string, version_id: string, change: { type: string }, [field: string]: unknown }} Row
 * @typedef {{ transaction_date_from: string, limit: string }} Filters
 * @typedef {{ rows: Row[], filters: Filters, watermark: string }} State
 * @param {{ state?: State | null, filters?: Filters, fetcher?: typeof fetch, maxPages?: number }} options
 * @returns {Promise<State>}
 */
export async function walk({ state = null, filters = recentFilters(), fetcher = fetch, maxPages = 50 } = {}) {
  if (!Number.isSafeInteger(maxPages) || maxPages < 1) throw new Error("Invalid page budget");
  if (state && (!timestamp(state.watermark) || !Array.isArray(state.rows) || !state.filters)) {
    throw new Error("Invalid saved state; do not invent a watermark");
  }
  const fixedFilters = { ...(state ? state.filters : filters) };
  if (Object.keys(fixedFilters).some((key) => !["transaction_date_from", "limit"].includes(key))
    || !/^\d{4}-\d{2}-\d{2}$/.test(fixedFilters.transaction_date_from)
    || !/^([1-9]|1[0-9]|2[0-5])$/.test(String(fixedFilters.limit))) {
    throw new Error("Use the fixed recent-window filters and a free-tier limit from 1 to 25");
  }
  const rows = new Map();
  for (const row of state?.rows ?? []) {
    if (!requiredString(row?.disclosure_id) || !requiredString(row?.version_id) || rows.has(row.disclosure_id)) {
      throw new Error("Invalid saved rows; no state committed");
    }
    rows.set(row.disclosure_id, row);
  }
  let cursor = null;
  let pin = null;
  const seen = new Set();
  for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
    const url = new URL("/api/v1/disclosures", API_ORIGIN);
    url.search = new URLSearchParams(fixedFilters).toString();
    if (state) url.searchParams.set("updated_since", state.watermark);
    if (cursor) url.searchParams.set("cursor", cursor);
    const { body } = await request(url, fetcher);
    const page = body?.page;
    if (!Array.isArray(body?.data) || !page
      || body.coverage?.rows_dropped_failed_validation !== 0
      || body.coverage?.rows_returned !== body.data.length
      || page.mode !== (state ? "changes" : "snapshot")
      || !Number.isSafeInteger(page.snapshot_generation) || page.snapshot_generation < 1
      || !timestamp(page.snapshot_as_of)
      || page.requested_updated_since !== (state?.watermark ?? null)
      || page.next_updated_since !== page.snapshot_as_of
      || typeof page.has_more !== "boolean") {
      throw new Error("Incomplete or invalid page; no state committed");
    }
    const pagePin = JSON.stringify([page.snapshot_generation, page.snapshot_as_of, page.next_updated_since]);
    if (pin !== null && pin !== pagePin) throw new Error("Snapshot changed; no state committed");
    pin = pagePin;
    for (const row of body.data) {
      if (!requiredString(row?.disclosure_id) || !requiredString(row?.version_id)
        || !["upsert", "supersede"].includes(row.change?.type)
        || (!state && row.change.type !== "upsert")) {
        throw new Error("Invalid change; no state committed");
      }
      if (row.change.type === "upsert") rows.set(row.disclosure_id, row);
      else if (rows.get(row.disclosure_id)?.version_id === row.version_id) rows.delete(row.disclosure_id);
    }
    if (!page.has_more) {
      if (page.next_cursor !== null) throw new Error("Unexpected cursor; no state committed");
      return { filters: fixedFilters, rows: [...rows.values()], watermark: page.next_updated_since };
    }
    if (!requiredString(page.next_cursor) || seen.has(page.next_cursor)) {
      throw new Error("Missing or repeated cursor; no state committed");
    }
    cursor = page.next_cursor;
    seen.add(cursor);
  }
  throw new Error("Page budget reached; no state committed. Narrow the initial window or use a licensed integration.");
}

if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
  try {
    console.log(JSON.stringify(await preview(), null, 2));
  } catch (error) {
    console.error(JSON.stringify({ error: error.message, code: error.code, request_id: error.requestId, retry_after: error.retryAfter }));
    process.exitCode = 1;
  }
}