"use strict";
// SPDX-License-Identifier: AGPL-3.0-or-later
// Copyright (C) 2024-2026 Gracker (Chris)
// This file is part of SmartPerfetto. See LICENSE for details.
Object.defineProperty(exports, "__esModule", { value: true });
exports.__TEST_ONLY__ = exports.BlogKnowledgeIngester = void 0;
/**
 * BlogKnowledgeIngester — pulls posts from androidperformance.com (or any
 * source matching the `BlogFetcher` shape), splits each post into
 * paragraph-bounded chunks, and writes them to a `RagStore` under the
 * `androidperformance.com` source kind.
 *
 * M0 scope:
 * - Pluggable fetcher so tests stay offline. Production wiring (RSS +
 *   incremental refresh + cron) lands in M2 alongside the admin route.
 * - Paragraph-based chunking up to `maxChunkChars` characters. No
 *   overlap — chunks split on hard paragraph boundaries; if context
 *   bleed becomes a problem we add overlap in M1, not before.
 * - HTML stripped via regex (script/style blocks removed first, then
 *   tags, then whitespace collapsed). Full DOM parsing is overkill for
 *   well-formed blog HTML; a swap to happy-dom stays local to this file.
 * - Stable, content-addressable chunk ids: `sha256(url + offset)` first
 *   16 hex chars. Re-ingesting the same post replaces the same chunks.
 *
 * @module blogKnowledgeIngester
 */
const crypto_1 = require("crypto");
const DEFAULT_MAX_CHUNK_CHARS = 1500;
/** Decode the small set of HTML entities we expect from blog markup.
 * Numeric entities (`&#39;`, `&#x27;`) plus the named ones below cover
 * 99% of authored content; anything richer would warrant a real DOM
 * parser, which M0 does not include. */
function decodeHtmlEntities(text) {
    return text
        .replace(/&amp;/g, '&')
        .replace(/&lt;/g, '<')
        .replace(/&gt;/g, '>')
        .replace(/&quot;/g, '"')
        .replace(/&#39;/g, "'")
        .replace(/&#x27;/gi, "'")
        .replace(/&nbsp;/g, ' ')
        .replace(/&#(\d+);/g, (_m, dec) => String.fromCharCode(Number(dec)));
}
/** Strip HTML to plain text. Removes script/style first so their
 * contents don't leak into the snippet, then drops every remaining tag
 * and collapses whitespace. */
function stripHtml(html) {
    return decodeHtmlEntities(html
        .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
        .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ')
        .replace(/<[^>]+>/g, ' ')
        .replace(/\s+/g, ' ')
        .trim());
}
/** Normalize markdown to plain text. Keeps the prose intact; removes
 * fenced code block delimiters but leaves the code so token-overlap
 * search can still find symbol names. */
function normalizeMarkdown(markdown) {
    return markdown
        .replace(/```[a-zA-Z0-9_+-]*\n?/g, '')
        .replace(/```/g, '')
        .replace(/\r\n?/g, '\n')
        .trim();
}
/** Split text into paragraphs (double-newline separated). Paragraph
 * boundaries are the chunk boundary candidates. */
function splitParagraphs(text) {
    return text
        .split(/\n\s*\n+/)
        .map(p => p.trim())
        .filter(p => p.length > 0);
}
function packParagraphs(paragraphs, maxChars) {
    const out = [];
    let cursor = 0;
    let buf = '';
    let bufStart = 0;
    for (const p of paragraphs) {
        if (buf.length === 0) {
            buf = p;
            bufStart = cursor;
        }
        else if (buf.length + 2 + p.length <= maxChars) {
            buf += '\n\n' + p;
        }
        else {
            out.push({ text: buf, offset: bufStart });
            buf = p;
            bufStart = cursor;
        }
        // Advance cursor by paragraph length + the joining "\n\n" we
        // assumed in the source. This lets the next chunk's offset point
        // at the original character index even when paragraphs are packed.
        cursor += p.length + 2;
    }
    if (buf.length > 0) {
        out.push({ text: buf, offset: bufStart });
    }
    return out;
}
/** sha256(url + offset).slice(0, 16). Stable and short. */
function makeChunkId(url, offset) {
    return (0, crypto_1.createHash)('sha256')
        .update(`${url}|${offset}`)
        .digest('hex')
        .slice(0, 16);
}
class BlogKnowledgeIngester {
    constructor(store, fetcher) {
        this.store = store;
        this.fetcher = fetcher;
    }
    async ingest(opts = {}) {
        const maxChars = opts.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS;
        const result = {
            postsProcessed: 0,
            chunksAdded: 0,
            chunksSkipped: 0,
            errors: [],
        };
        let posts;
        try {
            posts = await this.fetcher.fetchRecent({ since: opts.since });
        }
        catch (err) {
            result.errors.push({
                url: '<fetcher>',
                reason: err instanceof Error ? err.message : String(err),
            });
            return result;
        }
        for (const post of posts) {
            result.postsProcessed++;
            try {
                const chunks = this.chunkPost(post, maxChars);
                for (const chunk of chunks) {
                    this.store.addChunk(chunk);
                    result.chunksAdded++;
                }
            }
            catch (err) {
                result.errors.push({
                    url: post.url,
                    reason: err instanceof Error ? err.message : String(err),
                });
                result.chunksSkipped++;
            }
        }
        return result;
    }
    /** Convert one post into chunks. Public for tests so we can poke the
     * chunking decision in isolation. */
    chunkPost(post, maxChars) {
        const text = post.contentType === 'html'
            ? stripHtml(post.content)
            : normalizeMarkdown(post.content);
        if (text.length === 0)
            return [];
        const paragraphs = splitParagraphs(text);
        if (paragraphs.length === 0)
            return [];
        const packed = packParagraphs(paragraphs, maxChars);
        const indexedAt = post.fetchedAt;
        return packed.map(p => ({
            chunkId: makeChunkId(post.url, p.offset),
            kind: 'androidperformance.com',
            uri: post.url,
            title: post.title,
            snippet: p.text,
            tokenCount: estimateTokenCount(p.text),
            indexedAt,
            author: post.author,
            verifiedAt: post.publishedAt,
        }));
    }
}
exports.BlogKnowledgeIngester = BlogKnowledgeIngester;
/** Rough token count: ~4 chars per token for English/Chinese mix.
 * Used only for downstream context budgeting; off by 30% is fine. */
function estimateTokenCount(text) {
    return Math.max(1, Math.round(text.length / 4));
}
/** Re-export internals for tests. Not part of the runtime API. */
exports.__TEST_ONLY__ = {
    decodeHtmlEntities,
    stripHtml,
    normalizeMarkdown,
    splitParagraphs,
    packParagraphs,
    makeChunkId,
};
//# sourceMappingURL=blogKnowledgeIngester.js.map