// ---------------------------------------------------------------------------
// Semantic document chunking — embedding-based topic boundary detection
//
// TEXT --> PARAGRAPHS --> BLOCKS --> EMBED --> SIMILARITY --> VALLEYS --> SECTIONS
//
// Replaces heading-regex chunking. Works identically on plain text, PDF
// output, and markdown. Headings are tiebreakers, not drivers.
// ---------------------------------------------------------------------------
const MIN_BLOCK_SIZE = 300;
const PREVIEW_LENGTH = 150;
const MIN_CHUNK_SIZE = 200;
const MAX_CHUNK_SIZE = 2000;
const MAX_SECTIONS = 25;
const BYTES_PER_SECTION = 4000;
const HEADING_SNAP_WINDOW = 2;
const HEADING_RE = /^(#{1,6})\s+(.+)$/m;
const SENTENCE_RE = /(?<=[.!?])\s+(?=[A-Z])/;
export function splitIntoParagraphs(text) {
    const paragraphs = text.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean);
    if (paragraphs.length > 1)
        return paragraphs;
    const sentences = text.split(SENTENCE_RE).map((s) => s.trim()).filter(Boolean);
    if (sentences.length > 1)
        return sentences;
    return paragraphs.length > 0 ? paragraphs : [text.trim()];
}
export function aggregateBlocks(paragraphs, minBytes = MIN_BLOCK_SIZE) {
    const blocks = [];
    let currentParagraphs = [];
    let currentText = "";
    function flush() {
        if (!currentText)
            return;
        const headingMatch = currentText.match(HEADING_RE);
        blocks.push({
            text: currentText,
            paragraphs: [...currentParagraphs],
            headingLine: headingMatch ? headingMatch[0] : null,
        });
        currentParagraphs = [];
        currentText = "";
    }
    for (const para of paragraphs) {
        if (currentText && currentText.length >= minBytes) {
            flush();
        }
        currentParagraphs.push(para);
        currentText = currentText ? currentText + "\n\n" + para : para;
    }
    flush();
    return blocks;
}
export function cosine(a, b) {
    if (a.length !== b.length) {
        throw new Error(`Vector length mismatch: ${a.length} vs ${b.length}`);
    }
    let dot = 0;
    let magA = 0;
    let magB = 0;
    for (let i = 0; i < a.length; i++) {
        dot += a[i] * b[i];
        magA += a[i] * a[i];
        magB += b[i] * b[i];
    }
    const denom = Math.sqrt(magA) * Math.sqrt(magB);
    if (denom === 0)
        return 0;
    return dot / denom;
}
/**
 * Compute cosine similarity between each consecutive pair of embeddings.
 * Returns array of length (embeddings.length - 1).
 */
export function similarityCurve(embeddings) {
    const curve = [];
    for (let i = 0; i < embeddings.length - 1; i++) {
        curve.push(cosine(embeddings[i], embeddings[i + 1]));
    }
    return curve;
}
/**
 * Compute valley depth at each position in the similarity curve.
 *
 * depth(i) = (left_peak - curve[i]) + (right_peak - curve[i])
 * where left_peak and right_peak are the nearest local maxima on each side.
 *
 * Deep valleys indicate strong topic transitions.
 */
export function valleyDepths(curve) {
    const depths = new Array(curve.length).fill(0);
    for (let i = 0; i < curve.length; i++) {
        let leftPeak = curve[i];
        for (let j = i - 1; j >= 0; j--) {
            if (curve[j] > leftPeak)
                leftPeak = curve[j];
            else
                break;
        }
        let rightPeak = curve[i];
        for (let j = i + 1; j < curve.length; j++) {
            if (curve[j] > rightPeak)
                rightPeak = curve[j];
            else
                break;
        }
        depths[i] = (leftPeak - curve[i]) + (rightPeak - curve[i]);
    }
    return depths;
}
/**
 * Select the top N deepest valleys, sorted by position.
 * Skips positions with zero depth (no valley).
 */
export function selectBoundaries(depths, n) {
    const indexed = depths
        .map((d, i) => ({ depth: d, index: i }))
        .filter((v) => v.depth > 0);
    indexed.sort((a, b) => b.depth - a.depth);
    const selected = indexed.slice(0, n).map((v) => v.index);
    selected.sort((a, b) => a - b);
    return selected;
}
/**
 * Snap valley-based boundaries to nearby markdown headings.
 * Boundary values represent the first block index of the new section.
 */
export function snapToHeadings(boundaries, blocks, window = HEADING_SNAP_WINDOW) {
    return boundaries.map((pos) => {
        const sectionStart = pos + 1;
        let bestDist = Infinity;
        let bestIdx = sectionStart;
        for (let offset = -window; offset <= window; offset++) {
            const idx = sectionStart + offset;
            if (idx < 1 || idx >= blocks.length)
                continue;
            if (blocks[idx].headingLine && Math.abs(offset) < bestDist) {
                bestDist = Math.abs(offset);
                bestIdx = idx;
            }
        }
        return bestIdx;
    });
}
/**
 * Semantic document chunking using embedding-based topic boundary detection.
 *
 * Replaces heading-regex chunking. Works on plain text, PDF output, and
 * markdown. Headings are used as tiebreakers for boundary placement, not
 * as the primary segmentation signal.
 *
 * @param text - Document text to chunk
 * @param embedFn - Batch embedding function (injected for testability)
 * @returns ChunkedSection[] — same shape as the previous heading-regex chunker
 */
export async function chunkText(text, embedFn) {
    const trimmed = text.trim();
    const t0 = Date.now();
    const log = (stage, detail) => console.error(`[semantic-chunker] ${stage}${detail ? ` — ${detail}` : ""} (${Date.now() - t0}ms)`);
    log("start", `${trimmed.length} chars`);
    // Short-circuit: very short documents don't need embedding
    if (trimmed.length < MIN_BLOCK_SIZE) {
        log("short-circuit", "below MIN_BLOCK_SIZE");
        return [{
                title: deriveTitle(trimmed),
                chunks: [{
                        id: 0,
                        preview: trimmed.slice(0, PREVIEW_LENGTH),
                        content: trimmed,
                    }],
            }];
    }
    // 1. Split into paragraphs and aggregate into blocks
    const paragraphs = splitIntoParagraphs(trimmed);
    const blocks = aggregateBlocks(paragraphs);
    log("blocks", `${paragraphs.length} paragraphs → ${blocks.length} blocks`);
    // Short-circuit: too few blocks for boundary detection
    if (blocks.length <= 2) {
        log("short-circuit", "≤2 blocks, skipping embedding");
        return buildSections(blocks, []);
    }
    // 2. Batch embed all blocks
    log("embedding", `${blocks.length} blocks via Ollama`);
    const embeddings = await embedFn(blocks.map((b) => b.text));
    log("embedded", `${embeddings.length} vectors (dim=${embeddings[0]?.length ?? 0})`);
    // 3. Compute similarity curve and valley depths
    const curve = similarityCurve(embeddings);
    const depths = valleyDepths(curve);
    // 4. Select top N boundaries (adaptive with ceiling)
    const targetN = Math.min(Math.ceil(trimmed.length / BYTES_PER_SECTION), MAX_SECTIONS, curve.length);
    const rawBoundaries = selectBoundaries(depths, targetN);
    // 5. Snap to nearby headings
    const snappedBoundaries = snapToHeadings(rawBoundaries, blocks);
    // Deduplicate (snapping can cause two boundaries to land on the same block)
    const uniqueBoundaries = [...new Set(snappedBoundaries)].sort((a, b) => a - b);
    log("boundaries", `${uniqueBoundaries.length} section breaks from ${rawBoundaries.length} candidates`);
    // 6. Build sections with chunks
    const sections = buildSections(blocks, uniqueBoundaries);
    const totalChunks = sections.reduce((n, s) => n + s.chunks.length, 0);
    log("done", `${sections.length} sections, ${totalChunks} chunks`);
    return sections;
}
function deriveTitle(text) {
    const headingMatch = text.match(HEADING_RE);
    if (headingMatch)
        return headingMatch[2].trim().slice(0, 80);
    const sentenceEnd = text.search(/[.!?]\s/);
    return sentenceEnd > 0 && sentenceEnd < 80
        ? text.slice(0, sentenceEnd + 1)
        : text.slice(0, 80);
}
/**
 * Build ChunkedSection[] from blocks and section boundary positions.
 * Each boundary is a block index where a new section starts.
 * Within each section, paragraphs are grouped into chunks respecting
 * MIN_CHUNK_SIZE (200) and MAX_CHUNK_SIZE (2000) byte constraints.
 */
export function buildSections(blocks, boundaries) {
    const sections = [];
    let chunkId = 0;
    const starts = [0, ...boundaries];
    const ends = [...boundaries, blocks.length];
    for (let s = 0; s < starts.length; s++) {
        const sectionBlocks = blocks.slice(starts[s], ends[s]);
        if (sectionBlocks.length === 0)
            continue;
        const headingBlock = sectionBlocks.find((b) => b.headingLine);
        const title = headingBlock
            ? headingBlock.headingLine.replace(/^#+\s+/, "")
            : deriveTitle(sectionBlocks[0].paragraphs[0] || sectionBlocks[0].text);
        const allParagraphs = sectionBlocks.flatMap((b) => b.paragraphs);
        const chunks = [];
        let buffer = "";
        for (const para of allParagraphs) {
            if (buffer && buffer.length + para.length + 2 > MAX_CHUNK_SIZE) {
                chunks.push({
                    id: chunkId++,
                    preview: buffer.slice(0, PREVIEW_LENGTH),
                    content: buffer,
                });
                buffer = para;
            }
            else {
                buffer = buffer ? buffer + "\n\n" + para : para;
            }
        }
        if (buffer) {
            if (buffer.length < MIN_CHUNK_SIZE && chunks.length > 0) {
                const prev = chunks[chunks.length - 1];
                prev.content = prev.content + "\n\n" + buffer;
                prev.preview = prev.content.slice(0, PREVIEW_LENGTH);
            }
            else {
                chunks.push({
                    id: chunkId++,
                    preview: buffer.slice(0, PREVIEW_LENGTH),
                    content: buffer,
                });
            }
        }
        if (chunks.length > 0) {
            sections.push({ title, chunks });
        }
    }
    return sections;
}
//# sourceMappingURL=semantic-chunker.js.map