/**
 * Clustering Service - Orchestrates document clustering pipeline
 *
 * Pipeline:
 *   1. Fetch document-level embeddings (average-pool chunk embeddings, L2-normalize)
 *   2. Call Python clustering worker (HDBSCAN / agglomerative / kmeans)
 *   3. Store cluster + document_cluster records with provenance
 *
 * CRITICAL: NEVER use console.log() - stdout is JSON-RPC protocol.
 * Use console.error() for all logging.
 *
 * @module services/clustering/clustering-service
 */
import { v4 as uuidv4 } from 'uuid';
import { PythonShell } from 'python-shell';
import path from 'path';
import { fileURLToPath } from 'url';
import { getProvenanceTracker } from '../provenance/index.js';
import { ProvenanceType } from '../../models/provenance.js';
import { insertCluster, insertDocumentCluster } from '../storage/database/cluster-operations.js';
import { computeHash } from '../../utils/hash.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
class ClusteringError extends Error {
    code;
    details;
    constructor(message, code, details) {
        super(message);
        this.code = code;
        this.details = details;
        this.name = 'ClusteringError';
    }
}
// ═══════════════════════════════════════════════════════════════════════════════
// SERVICE
// ═══════════════════════════════════════════════════════════════════════════════
/** Worker timeout: 5 minutes */
const WORKER_TIMEOUT_MS = 300_000;
/** Max stderr accumulation: 10KB */
const MAX_STDERR_LENGTH = 10_240;
/**
 * Compute document-level embeddings by average-pooling chunk embeddings.
 *
 * For each document that has embeddings, fetches all chunk-based vectors
 * from vec_embeddings, computes the element-wise mean, and L2-normalizes.
 *
 * sqlite-vec has NO native vector averaging -- we extract to TypeScript.
 *
 * @param conn - Raw better-sqlite3 connection (for direct vec_embeddings queries)
 * @param documentIds - Optional filter; if empty, includes all documents with embeddings
 * @returns Array of DocumentEmbedding with 768-dim Float32Array per document
 */
export function computeDocumentEmbeddings(conn, documentIds) {
    // chunk_id IS NOT NULL ensures we only get chunk embeddings (not VLM or extraction)
    const hasFilter = documentIds && documentIds.length > 0;
    const filterClause = hasFilter
        ? ` AND e.document_id IN (${documentIds.map(() => '?').join(', ')})`
        : '';
    const rows = conn
        .prepare(`
    SELECT e.document_id, v.vector
    FROM vec_embeddings v
    JOIN embeddings e ON e.id = v.embedding_id
    WHERE e.chunk_id IS NOT NULL${filterClause}
    ORDER BY e.document_id, e.chunk_index
  `)
        .all(...(hasFilter ? documentIds : []));
    // Group by document_id
    const docVectors = new Map();
    for (const row of rows) {
        const existing = docVectors.get(row.document_id);
        if (existing) {
            existing.push(row.vector);
        }
        else {
            docVectors.set(row.document_id, [row.vector]);
        }
    }
    // Average-pool + L2-normalize per document
    const results = [];
    for (const [docId, vectors] of docVectors) {
        const averaged = averageVectors(vectors);
        results.push({
            document_id: docId,
            embedding: averaged,
            chunk_count: vectors.length,
        });
    }
    return results;
}
/**
 * Average-pool vectors and L2-normalize the result.
 *
 * @param vectors - Array of 768-dim vectors as Buffers (from sqlite-vec)
 * @returns L2-normalized 768-dim Float32Array
 */
function averageVectors(vectors) {
    if (vectors.length === 0) {
        return new Float32Array(768);
    }
    const dim = 768;
    const sum = new Float64Array(dim); // Use float64 for accumulation precision
    for (const buf of vectors) {
        const f32 = new Float32Array(buf.buffer, buf.byteOffset, dim);
        for (let i = 0; i < dim; i++) {
            sum[i] += f32[i];
        }
    }
    // Mean
    const n = vectors.length;
    const result = new Float32Array(dim);
    for (let i = 0; i < dim; i++) {
        result[i] = sum[i] / n;
    }
    // L2 normalize
    let norm = 0;
    for (let i = 0; i < dim; i++) {
        norm += result[i] * result[i];
    }
    norm = Math.sqrt(norm);
    if (norm > 0) {
        for (let i = 0; i < dim; i++) {
            result[i] /= norm;
        }
    }
    return result;
}
/**
 * Run the Python clustering worker via python-shell.
 *
 * Sends JSON to stdin, parses JSON from stdout.
 * Uses the same PythonShell pattern as embedding_worker.py.
 *
 * @param embeddings - 2D array of embeddings [n_docs][768]
 * @param documentIds - Document IDs matching embedding order
 * @param config - Clustering algorithm configuration
 * @param distanceMatrix - Optional precomputed distance matrix [n_docs][n_docs]
 * @returns WorkerResult from Python
 */
async function runClusteringWorker(embeddings, documentIds, config, distanceMatrix) {
    const workerPath = path.resolve(__dirname, '../../../python/clustering_worker.py');
    const workerInput = {
        embeddings,
        document_ids: documentIds,
        algorithm: config.algorithm,
        n_clusters: config.n_clusters,
        min_cluster_size: config.min_cluster_size,
        distance_threshold: config.distance_threshold,
        linkage: config.linkage,
    };
    if (distanceMatrix) {
        workerInput.distance_matrix = distanceMatrix;
    }
    const input = JSON.stringify(workerInput);
    return new Promise((resolve, reject) => {
        let settled = false;
        const options = {
            mode: 'text',
            pythonOptions: ['-u'],
            args: [],
        };
        const shell = new PythonShell(workerPath, options);
        let stderr = '';
        let sigkillTimer = null;
        const cleanup = () => {
            if (sigkillTimer) {
                clearTimeout(sigkillTimer);
                sigkillTimer = null;
            }
        };
        const timer = setTimeout(() => {
            if (settled)
                return;
            try {
                shell.kill();
            }
            catch (error) {
                console.error('[ClusteringService] Failed to kill shell on timeout:', error instanceof Error ? error.message : String(error));
                /* ignore */
            }
            // M-6: SIGKILL escalation if SIGTERM doesn't exit within 5s
            sigkillTimer = setTimeout(() => {
                if (!settled) {
                    console.error(`[ClusteringService] Process did not exit after SIGTERM, sending SIGKILL (pid: ${shell.childProcess?.pid})`);
                    try {
                        shell.childProcess?.kill('SIGKILL');
                    }
                    catch (error) {
                        console.error('[ClusteringService] Failed to SIGKILL process (may already be gone):', error instanceof Error ? error.message : String(error));
                    }
                }
                if (!settled) {
                    settled = true;
                    reject(new ClusteringError(`Clustering worker timeout after ${WORKER_TIMEOUT_MS}ms (SIGKILL after 5s grace)`, 'WORKER_TIMEOUT', { stderr: stderr.substring(0, 1000) }));
                }
            }, 5000);
        }, WORKER_TIMEOUT_MS);
        const outputChunks = [];
        shell.on('message', (msg) => {
            outputChunks.push(msg);
        });
        shell.on('stderr', (err) => {
            if (stderr.length < MAX_STDERR_LENGTH) {
                stderr += err + '\n';
            }
        });
        const handleEnd = (err) => {
            clearTimeout(timer);
            cleanup();
            if (settled)
                return;
            settled = true;
            if (err) {
                console.error('[ClusterWorker] Error:', err.message);
                if (stderr)
                    console.error('[ClusterWorker] Stderr:', stderr.substring(0, 1000));
            }
            const output = outputChunks.join('\n');
            if (!output.trim()) {
                if (err) {
                    reject(new ClusteringError(`Clustering worker failed: ${err.message}`, 'WORKER_FAILED', {
                        stderr: stderr.substring(0, 1000),
                    }));
                }
                else {
                    reject(new ClusteringError('Clustering worker produced no output', 'WORKER_FAILED', {
                        stderr: stderr.substring(0, 1000),
                    }));
                }
                return;
            }
            // Parse the last JSON line
            const lines = output.trim().split('\n');
            let parsed;
            for (let i = lines.length - 1; i >= 0; i--) {
                try {
                    parsed = JSON.parse(lines[i].trim());
                    break;
                }
                catch (error) {
                    console.error('[ClusteringService] JSON parse failed for output line, trying previous:', error instanceof Error ? error.message : String(error));
                    /* not JSON, try previous line */
                }
            }
            if (parsed !== undefined) {
                // Validate required fields exist on parsed worker result
                if (typeof parsed !== 'object' || parsed === null) {
                    reject(new ClusteringError('Clustering worker returned non-object result', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
                    return;
                }
                if (typeof parsed.success !== 'boolean') {
                    reject(new ClusteringError('Clustering worker result missing required "success" field', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
                    return;
                }
                if (parsed.success &&
                    parsed.silhouette_score === undefined &&
                    parsed.labels === undefined) {
                    reject(new ClusteringError('Clustering worker success=true but missing "labels" and "silhouette_score" fields', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
                    return;
                }
                resolve(parsed);
            }
            else {
                reject(new ClusteringError('Failed to parse clustering worker output as JSON', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
            }
        };
        shell.send(input);
        shell.end(handleEnd);
    });
}
/**
 * Compute cosine similarity between a document embedding and a centroid.
 * Both vectors are assumed to be L2-normalized, so similarity = dot product.
 */
export function cosineSimilarity(a, b) {
    let dot = 0;
    for (let i = 0; i < a.length; i++) {
        dot += (a[i] ?? 0) * (b[i] ?? 0);
    }
    // Clamp to [-1, 1] to handle floating point drift
    return Math.max(-1, Math.min(1, dot));
}
// ═══════════════════════════════════════════════════════════════════════════════
// MAIN CLUSTERING PIPELINE
// ═══════════════════════════════════════════════════════════════════════════════
/**
 * Run the full clustering pipeline:
 *   1. Compute document-level embeddings
 *   2. Validate minimum document count
 *   3. Call Python clustering worker
 *   4. Create provenance records
 *   5. Store clusters + document_cluster assignments
 *
 * @param db - DatabaseService instance
 * @param vector - VectorService instance (unused directly but validates vec loaded)
 * @param config - Clustering configuration
 * @param documentIds - Optional filter (empty = all documents with embeddings)
 * @returns ClusterRunResult with full results
 */
export async function runClustering(db, _vector, config, documentIds) {
    const startTime = performance.now();
    const runId = uuidv4();
    const conn = db.getConnection();
    // Step 1: Compute document-level embeddings
    console.error(`[CLUSTER] Computing document embeddings...`);
    const docEmbeddings = computeDocumentEmbeddings(conn, documentIds);
    if (docEmbeddings.length < 2) {
        throw new ClusteringError(`At least 2 documents with embeddings required for clustering, got ${docEmbeddings.length}`, 'INSUFFICIENT_DOCUMENTS', { found: docEmbeddings.length, requested: documentIds?.length ?? 'all' });
    }
    console.error(`[CLUSTER] ${docEmbeddings.length} documents with embeddings`);
    // Step 2: Prepare data for Python worker
    const orderedDocIds = docEmbeddings.map((d) => d.document_id);
    const embeddingMatrix = docEmbeddings.map((d) => Array.from(d.embedding));
    // Step 3: Call Python clustering worker
    console.error(`[CLUSTER] Running ${config.algorithm} clustering...`);
    const workerResult = await runClusteringWorker(embeddingMatrix, orderedDocIds, config);
    if (!workerResult.success) {
        throw new ClusteringError(`Clustering worker failed: ${workerResult.error}`, 'WORKER_FAILED', {
            error_type: workerResult.error_type,
            error: workerResult.error,
        });
    }
    // Step 4: Store results in database
    console.error(`[CLUSTER] Found ${workerResult.n_clusters} clusters, storing results...`);
    const tracker = getProvenanceTracker(db);
    const now = new Date().toISOString();
    const processingDurationMs = Math.round(performance.now() - startTime);
    const algorithmParamsJson = JSON.stringify({
        algorithm: config.algorithm,
        n_clusters: config.n_clusters,
        min_cluster_size: config.min_cluster_size,
        distance_threshold: config.distance_threshold,
        linkage: config.linkage,
    });
    // Build cluster result items and store cluster records
    const clusterItems = [];
    // Validate required fields before destructuring
    if (!workerResult.labels || !workerResult.probabilities || !workerResult.centroids || !workerResult.coherence_scores) {
        throw new ClusteringError('Clustering worker returned success but missing required fields (labels, probabilities, centroids, coherence_scores)', 'WORKER_PARSE_ERROR', {
            hasLabels: !!workerResult.labels,
            hasProbabilities: !!workerResult.probabilities,
            hasCentroids: !!workerResult.centroids,
            hasCoherenceScores: !!workerResult.coherence_scores,
        });
    }
    const labels = workerResult.labels;
    const probabilities = workerResult.probabilities;
    const centroids = workerResult.centroids;
    const coherenceScores = workerResult.coherence_scores;
    // Group documents by cluster label
    const clusterGroups = new Map(); // label -> doc indices
    for (let i = 0; i < labels.length; i++) {
        const label = labels[i];
        if (label === -1)
            continue; // Skip noise
        const existing = clusterGroups.get(label);
        if (existing) {
            existing.push(i);
        }
        else {
            clusterGroups.set(label, [i]);
        }
    }
    // Sort cluster labels
    const sortedLabels = Array.from(clusterGroups.keys()).sort((a, b) => a - b);
    const clusterIdMap = new Map(); // label -> cluster UUID
    // Use a transaction to store everything atomically
    const storeTransaction = conn.transaction(() => {
        for (let ci = 0; ci < sortedLabels.length; ci++) {
            const label = sortedLabels[ci];
            const docIndices = clusterGroups.get(label);
            const centroid = centroids[ci];
            const coherence = coherenceScores[ci];
            const clusterId = uuidv4();
            clusterIdMap.set(label, clusterId);
            // Content hash from centroid + run_id
            const contentHash = computeHash(JSON.stringify(centroid) + ':' + runId);
            // Find any document's provenance_id as the source_id for the cluster provenance
            // Use the first document in this cluster
            const firstDocId = orderedDocIds[docIndices[0]];
            const firstDoc = db.getDocument(firstDocId);
            const sourceProvId = firstDoc?.provenance_id ?? null;
            // Create CLUSTERING provenance record
            const provId = tracker.createProvenance({
                type: ProvenanceType.CLUSTERING,
                source_type: 'CLUSTERING',
                source_id: sourceProvId,
                root_document_id: firstDoc?.provenance_id ?? runId,
                content_hash: contentHash,
                input_hash: computeHash(algorithmParamsJson + ':' + docIndices.length),
                processor: 'clustering-service',
                processor_version: '1.0.0',
                processing_params: {
                    algorithm: config.algorithm,
                    run_id: runId,
                    cluster_index: ci,
                    document_count: docIndices.length,
                },
                processing_duration_ms: processingDurationMs,
                processing_quality_score: coherence,
            });
            // Insert cluster record
            const cluster = {
                id: clusterId,
                run_id: runId,
                cluster_index: ci,
                label: null,
                description: null,
                classification_tag: null,
                document_count: docIndices.length,
                centroid_json: JSON.stringify(centroid),
                top_terms_json: null,
                coherence_score: coherence,
                algorithm: config.algorithm,
                algorithm_params_json: algorithmParamsJson,
                silhouette_score: workerResult.silhouette_score ?? null,
                content_hash: contentHash,
                provenance_id: provId,
                created_at: now,
                processing_duration_ms: processingDurationMs,
            };
            insertCluster(conn, cluster);
            // Build result item
            const itemDocIds = [];
            const itemSimilarities = [];
            const itemProbabilities = [];
            for (const idx of docIndices) {
                itemDocIds.push(orderedDocIds[idx]);
                itemSimilarities.push(cosineSimilarity(docEmbeddings[idx].embedding, centroid));
                itemProbabilities.push(probabilities[idx]);
            }
            clusterItems.push({
                cluster_index: ci,
                document_count: docIndices.length,
                coherence_score: coherence,
                centroid,
                document_ids: itemDocIds,
                similarities: itemSimilarities,
                probabilities: itemProbabilities,
            });
        }
        // Store document-cluster assignments (both clustered and noise documents)
        for (let i = 0; i < labels.length; i++) {
            const label = labels[i];
            const isNoise = label === -1;
            const clusterId = isNoise ? null : (clusterIdMap.get(label) ?? null);
            const centroid = isNoise ? null : centroids[sortedLabels.indexOf(label)];
            const similarity = centroid ? cosineSimilarity(docEmbeddings[i].embedding, centroid) : 0;
            const dc = {
                id: uuidv4(),
                document_id: orderedDocIds[i],
                cluster_id: clusterId,
                run_id: runId,
                similarity_to_centroid: Math.round(similarity * 1000000) / 1000000,
                membership_probability: probabilities[i],
                is_noise: isNoise,
                assigned_at: now,
            };
            insertDocumentCluster(conn, dc);
        }
    });
    storeTransaction();
    const noiseDocIds = orderedDocIds.filter((_, i) => labels[i] === -1);
    const totalDurationMs = Math.round(performance.now() - startTime);
    console.error(`[CLUSTER] Done: ${workerResult.n_clusters} clusters, ${noiseDocIds.length} noise docs, ${totalDurationMs}ms`);
    return {
        run_id: runId,
        algorithm: config.algorithm,
        n_clusters: workerResult.n_clusters,
        total_documents: docEmbeddings.length,
        noise_document_ids: noiseDocIds,
        silhouette_score: workerResult.silhouette_score ?? 0,
        clusters: clusterItems,
        processing_duration_ms: totalDurationMs,
    };
}
//# sourceMappingURL=clustering-service.js.map