import { readFile } from "node:fs/promises";
import { stat } from "node:fs/promises";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { chunkText } from "../lib/semantic-chunker.js";
import { embedBatch } from "../lib/embeddings.js";
const execFileAsync = promisify(execFile);
// ---------------------------------------------------------------------------
// Text extraction + server-side chunking for file attachments
//
// FILE ON DISK --> MIME CHECK --> EXTRACT TEXT --> CHUNK --> CACHE --> RETURN
//       │               │              │              │         │
//       v               v              v              v         v
//   [missing?]     [CSV? image?]   [pdftotext     semantic   Map keyed
//   [perm denied?]  → reject       not found?]    chunking   by attachmentId
//   [empty?]        with guidance   [corrupt PDF?] via Ollama  (consumed by
//                                                             memory-ingest)
//
// Chunking uses embedding-based topic boundary detection via Ollama
// (semantic-chunker). The model never has to echo raw content back in a tool
// call — only chunk previews. This keeps model output under token limits
// regardless of document size or model output capacity.
// ---------------------------------------------------------------------------
const REJECTED_MIME_TYPES = {
    "text/csv": "CSV files contain structured data. Use memory-write to create graph nodes from each row instead of ingesting as a knowledge document.",
};
const IMAGE_PREFIXES = ["image/"];
async function extractPdfText(storagePath) {
    try {
        const { stdout } = await execFileAsync("pdftotext", [storagePath, "-"], {
            maxBuffer: 10 * 1024 * 1024,
        });
        return stdout.trim();
    }
    catch (err) {
        const msg = err instanceof Error ? err.message : String(err);
        if (msg.includes("ENOENT") || msg.includes("not found")) {
            throw new Error("pdftotext not found. Install poppler-utils: sudo apt-get install -y poppler-utils");
        }
        throw new Error(`PDF extraction failed: ${msg}`);
    }
}
/** In-process cache consumed by memory-ingest. Keyed by attachmentId. */
export const extractCache = new Map();
/**
 * Extract text content from a file attachment and chunk it server-side.
 *
 * Supports: application/pdf, text/plain, text/markdown.
 * Rejects: text/csv (structured data → memory-write), images (metadata-only).
 *
 * The full chunk content is cached in-process (keyed by attachmentId) for
 * retrieval by memory-ingest. The response includes only chunk previews
 * so the model can generate summaries without echoing full content.
 */
export async function memoryIngestExtract(opts) {
    const { storagePath, filename, mimeType, attachmentId } = opts;
    if (!attachmentId) {
        throw new Error("memory-ingest-extract requires attachmentId (the UUID of the file attachment). " +
            "Without it, cached content cannot be retrieved by memory-ingest.");
    }
    const t0 = Date.now();
    const log = (stage, detail) => console.error(`[memory-ingest-extract] [${filename}] ${stage}${detail ? ` — ${detail}` : ""} (${Date.now() - t0}ms)`);
    log("start", `mime=${mimeType}`);
    // Check for rejected MIME types with guidance
    const rejection = REJECTED_MIME_TYPES[mimeType];
    if (rejection) {
        throw new Error(rejection);
    }
    // Images have no text to extract
    if (IMAGE_PREFIXES.some((p) => mimeType.startsWith(p))) {
        throw new Error(`Images are stored as metadata only (DigitalDocument). No knowledge hierarchy created for "${filename}".`);
    }
    let text;
    if (mimeType === "application/pdf") {
        log("extracting", "pdftotext");
        text = await extractPdfText(storagePath);
        log("extracted", `${text.length} chars`);
    }
    else if (mimeType === "text/plain" ||
        mimeType === "text/markdown") {
        text = await readFile(storagePath, "utf-8");
        log("read", `${text.length} chars`);
    }
    else {
        throw new Error(`Unsupported MIME type "${mimeType}" for knowledge document ingestion. Supported: application/pdf, text/plain, text/markdown.`);
    }
    if (!text || text.trim().length === 0) {
        throw new Error(`File "${filename}" contains no extractable text.`);
    }
    const trimmedText = text.trim();
    const fileStat = await stat(storagePath);
    // Chunk the text server-side using embedding-based topic boundary detection
    log("chunking", `${trimmedText.length} chars via semantic embeddings`);
    const sections = await chunkText(trimmedText, embedBatch);
    const totalChunks = sections.reduce((n, s) => n + s.chunks.length, 0);
    log("chunked", `${sections.length} sections, ${totalChunks} chunks`);
    // Cache the full content for memory-ingest to retrieve
    extractCache.set(attachmentId, { filename, mimeType, sections });
    log("cached", `attachmentId=${attachmentId}`);
    // Return only previews — the model generates summaries from these
    return {
        filename,
        mimeType,
        sizeBytes: fileStat.size,
        textLength: trimmedText.length,
        sections: sections.map((s) => ({
            title: s.title,
            chunkCount: s.chunks.length,
            chunks: s.chunks.map((c) => ({
                id: c.id,
                preview: c.preview,
            })),
        })),
    };
}
//# sourceMappingURL=memory-ingest-extract.js.map