import { extractMetadata, extractSelector, extractTables } from "../extraction/extract.js";
import {
  extractWithSchema,
  extractWithSchemaDetailedAsync
} from "../extraction/schema.js";
import { extractJsonLd } from "../extraction/jsonld.js";
import { extractStructured } from "../extraction/structured.js";
import { getCachedContent, isExpired } from "../cache/store.js";
import { fetchWithPlaywright } from "../fetch/playwright-tier.js";
import { countTokens, truncateByTokens } from "../search/tokens.js";
import { createLogger } from "../logger.js";
import {
  isNamedSchemaType,
  extractNamedSchema,
  NAMED_SCHEMAS
} from "../extraction/v1/schemas/index.js";
import { isLocalLlmEnabled, extractWithLocalLlm } from "../extraction/v1/local-llm.js";
import { extractBrandAsync } from "../extraction/brand.js";
import { applyEvidenceFilter, getSourceText } from "../extraction/schema-truth.js";
const log = createLogger("extract");
function clampExtractData(data, maxTokens) {
  if (data === null || data === void 0) return data;
  const total = countTokens(JSON.stringify(data));
  if (total <= maxTokens) return data;
  if (Array.isArray(data)) {
    const out = [];
    let used = 2;
    for (const item of data) {
      const t = countTokens(JSON.stringify(item)) + 1;
      if (used + t > maxTokens) break;
      out.push(item);
      used += t;
    }
    return out;
  }
  if (typeof data === "object") {
    const obj = data;
    const out = {};
    const arrayKeys = [];
    const oversizeStringKeys = [];
    let used = 2;
    for (const [k, v] of Object.entries(obj)) {
      if (Array.isArray(v)) {
        arrayKeys.push(k);
        out[k] = [];
        continue;
      }
      const t = countTokens(JSON.stringify({ [k]: v })) - 1;
      if (used + t <= maxTokens) {
        out[k] = v;
        used += t;
        continue;
      }
      if (typeof v === "string" && v.length > 0) {
        oversizeStringKeys.push(k);
        out[k] = "";
        used += countTokens(JSON.stringify({ [k]: "" })) - 1;
      }
    }
    if (oversizeStringKeys.length > 0) {
      const perField = Math.max(64, Math.floor((maxTokens - used) / oversizeStringKeys.length));
      for (const k of oversizeStringKeys) {
        const original = obj[k];
        const truncated = truncateByTokens(original, perField);
        out[k] = truncated.length < original.length ? `${truncated.trimEnd()}\u2026` : truncated;
        used += countTokens(JSON.stringify({ [k]: out[k] })) - 1;
        used -= countTokens(JSON.stringify({ [k]: "" })) - 1;
      }
    }
    arrayKeys.sort(
      (a, b) => countTokens(JSON.stringify(obj[b])) - countTokens(JSON.stringify(obj[a]))
    );
    for (const k of arrayKeys) {
      const arr = obj[k];
      const kept = [];
      for (const item of arr) {
        const t = countTokens(JSON.stringify(item)) + 1;
        if (used + t > maxTokens) break;
        kept.push(item);
        used += t;
      }
      out[k] = kept;
    }
    return out;
  }
  return data;
}
const TABLES_DEFAULT_MAX_CHARS = 3e4;
function clampTablesToChars(data, maxChars) {
  if (!Array.isArray(data)) return { data, truncated: false };
  const serialized = JSON.stringify(data);
  if (serialized.length <= maxChars) return { data, truncated: false };
  const tables = data.map((t) => ({
    caption: t.caption,
    headers: [...t.headers],
    rows: [...t.rows]
  }));
  let currentSize = JSON.stringify(tables).length;
  while (currentSize > maxChars && tables.length > 0) {
    const last = tables[tables.length - 1];
    if (last.rows.length > 0) {
      const popped = last.rows.pop();
      currentSize -= JSON.stringify(popped).length + 1;
    } else {
      const poppedSize = JSON.stringify(tables[tables.length - 1]).length + 1;
      tables.pop();
      currentSize -= poppedSize;
    }
  }
  return { data: tables, truncated: true };
}
function buildSuccessOutput(data, sourceUrl, mode, maxTokens, warnings, startMs) {
  let finalData = data;
  let truncated = false;
  if (maxTokens !== void 0) {
    finalData = clampExtractData(data, maxTokens);
  } else if (mode === "tables") {
    const clamped = clampTablesToChars(data, TABLES_DEFAULT_MAX_CHARS);
    finalData = clamped.data;
    truncated = clamped.truncated;
  }
  const out = { data: finalData, source_url: sourceUrl, mode };
  if (warnings && warnings.length > 0) out.warnings = warnings;
  if (typeof startMs === "number") out.response_time_ms = Date.now() - startMs;
  if (truncated) out.truncated = true;
  return { ok: true, data: out };
}
async function resolveHtml(input, router) {
  if (input.execution_mode === "stealth" && input.url) {
    const pw = await fetchWithPlaywright(input.url);
    return { html: pw.html, sourceUrl: input.url };
  }
  if (input.url) {
    const cached = getCachedContent(input.url);
    if (cached && !isExpired(cached)) {
      log.info("Using cached HTML", { url: input.url });
      return { html: cached.rawHtml, sourceUrl: cached.url };
    }
    const raw = await router.fetch(input.url, {
      renderJs: "auto",
      useAuth: false
    });
    return { html: raw.html, sourceUrl: raw.finalUrl };
  }
  return { html: input.html };
}
async function handleExtract(input, router) {
  const mode = input.mode ?? "metadata";
  const _start = Date.now();
  if (!input.url && !input.html) {
    return {
      ok: false,
      error: "invalid_input",
      error_reason: "Either url or html must be provided",
      stage: "extract"
    };
  }
  if (input.named_schema && input.schema) {
    return {
      ok: false,
      error: "invalid_input",
      error_reason: "schema and named_schema are mutually exclusive",
      stage: "extract"
    };
  }
  if (input.named_schema && !isNamedSchemaType(input.named_schema)) {
    return {
      ok: false,
      error: "invalid_input",
      error_reason: `Unknown named_schema. Valid: ${NAMED_SCHEMAS.join(", ")}`,
      stage: "extract"
    };
  }
  if (mode === "selector" && !input.css_selector) {
    return {
      ok: false,
      error: "invalid_input",
      error_reason: 'css_selector is required when mode is "selector"',
      stage: "extract"
    };
  }
  if (mode === "schema" && !input.named_schema && (!input.schema || !input.schema.properties)) {
    return {
      ok: false,
      error: "invalid_input",
      error_reason: 'schema is required when mode is "schema" and must have properties',
      stage: "extract"
    };
  }
  try {
    const { html, sourceUrl } = await resolveHtml(input, router);
    if (input.named_schema) {
      const namedData = await extractNamedSchema(input.named_schema, html, sourceUrl ?? input.url ?? "");
      if (namedData === null) {
        return {
          ok: true,
          data: {
            data: {},
            source_url: sourceUrl,
            mode: "schema",
            error: `No ${input.named_schema} data found on page`
          }
        };
      }
      return buildSuccessOutput(
        namedData,
        sourceUrl,
        "schema",
        input.max_tokens_out,
        void 0,
        _start
      );
    }
    if (mode === "schema" && input.schema && isLocalLlmEnabled()) {
      const llmData = await extractWithLocalLlm({
        schema: input.schema,
        html,
        url: sourceUrl ?? input.url ?? ""
      });
      const raw = llmData ?? {};
      const allFields = Object.keys(raw);
      let finalData = raw;
      let warnings;
      if (allFields.length > 0) {
        const sourceText = getSourceText(html);
        const filtered = applyEvidenceFilter({
          values: raw,
          provenance: {},
          // local-llm path has no separate provenance map
          sourceText,
          fields: allFields
        });
        finalData = filtered.values;
        if (filtered.rejectedFields.length > 0) {
          warnings = [
            `evidence-only filter: nulled ${filtered.rejectedFields.length} field(s) the LLM proposed but were not present in source text (${filtered.rejectedFields.join(", ")}).`
          ];
        }
      }
      return buildSuccessOutput(
        finalData,
        sourceUrl,
        "schema",
        input.max_tokens_out,
        warnings,
        _start
      );
    }
    let data;
    switch (mode) {
      case "selector":
        data = extractSelector(html, input.css_selector, input.multiple ?? false);
        break;
      case "tables":
        data = extractTables(html);
        break;
      case "structured":
        data = extractStructured(html);
        break;
      case "brand": {
        data = await extractBrandAsync(html, {
          baseUrl: sourceUrl ?? input.url
        });
        break;
      }
      case "schema": {
        const schema = input.schema;
        if (Array.isArray(schema.required) && schema.required.length > 0) {
          const detailed = await extractWithSchemaDetailedAsync(html, schema);
          data = detailed.values;
          if (detailed.warnings.length > 0) {
            return buildSuccessOutput(data, sourceUrl, mode, input.max_tokens_out, detailed.warnings, _start);
          }
        } else {
          data = extractWithSchema(html, schema);
        }
        break;
      }
      case "metadata":
      default: {
        const meta = extractMetadata(html);
        const jsonld = extractJsonLd(html);
        if (jsonld.length > 0) {
          meta.jsonld = jsonld;
        }
        data = meta;
        break;
      }
    }
    if (mode === "tables" && Array.isArray(data) && data.length === 0) {
      const hint = input.execution_mode === "stealth" ? "no_tables_detected \u2014 page genuinely contains no tables" : 'no_tables_detected \u2014 page may require JavaScript; retry with execution_mode: "stealth"';
      return {
        ok: false,
        error: "no_tables_detected",
        error_reason: "No tables found on page",
        stage: "extract",
        hint
      };
    }
    return buildSuccessOutput(data, sourceUrl, mode, input.max_tokens_out, void 0, _start);
  } catch (err) {
    log.error("Extract failed", { url: input.url, error: String(err) });
    return {
      ok: false,
      error: "extract_failed",
      error_reason: err instanceof Error ? err.message : String(err),
      stage: "extract"
    };
  }
}
export {
  clampTablesToChars,
  handleExtract
};
//# sourceMappingURL=extract.js.map