/*
 * Libre WebUI
 * Copyright (C) 2025 Kroonen AI, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
import { v4 as uuidv4 } from 'uuid';
import storageService from '../storage.js';
import ollamaService from './ollamaService.js';
import preferencesService from './preferencesService.js';
// Utility functions for vector operations
const cosineSimilarity = (a, b) => {
    if (a.length !== b.length)
        return 0;
    let dotProduct = 0;
    let normA = 0;
    let normB = 0;
    for (let i = 0; i < a.length; i++) {
        dotProduct += a[i] * b[i];
        normA += a[i] * a[i];
        normB += b[i] * b[i];
    }
    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
};
// Lazy load pdfjs-dist legacy build for Node.js
let pdfjsLib = null;
const getPdfjsLib = async () => {
    if (!pdfjsLib) {
        try {
            // Use the legacy build for Node.js compatibility
            const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
            pdfjsLib = pdfjs;
            console.log('Successfully loaded pdfjs-dist legacy build');
        }
        catch (error) {
            console.error('Failed to load pdfjs-dist legacy:', error);
            throw new Error('PDF parsing is not available');
        }
    }
    return pdfjsLib;
};
class DocumentService {
    constructor() {
        this.documents = new Map();
        this.chunks = new Map();
        this.loadDocuments();
        this.loadChunks();
    }
    loadDocuments() {
        try {
            const documentsArray = storageService.getAllDocuments();
            this.documents = new Map(documentsArray.map(doc => [doc.id, doc]));
            console.log(`Loaded ${documentsArray.length} documents from storage`);
        }
        catch (error) {
            console.error('Failed to load documents:', error);
        }
    }
    loadChunks() {
        try {
            // Load chunks for all documents
            const documentsArray = Array.from(this.documents.values());
            for (const doc of documentsArray) {
                const chunks = storageService.getDocumentChunks(doc.id);
                if (chunks.length > 0) {
                    this.chunks.set(doc.id, chunks);
                }
            }
            console.log(`Loaded chunks for ${this.chunks.size} documents from storage`);
        }
        catch (error) {
            console.error('Failed to load document chunks:', error);
        }
    }
    saveDocuments() {
        // This method is no longer needed as we save documents individually
        // through the storage service
    }
    saveChunks() {
        // This method is no longer needed as we save chunks individually
        // through the storage service
    }
    async processDocument(fileName, fileBuffer, mimeType, sessionId) {
        const documentId = uuidv4();
        let content = '';
        let fileType;
        try {
            if (mimeType === 'application/pdf') {
                try {
                    const pdfLib = await getPdfjsLib();
                    // Convert Buffer to Uint8Array for pdfjs-dist compatibility
                    const uint8Array = new Uint8Array(fileBuffer);
                    const pdfDocument = await pdfLib.getDocument({ data: uint8Array })
                        .promise;
                    const numPages = pdfDocument.numPages;
                    let textContent = '';
                    for (let pageNum = 1; pageNum <= numPages; pageNum++) {
                        const page = await pdfDocument.getPage(pageNum);
                        const pageText = await page.getTextContent();
                        const pageContent = pageText.items
                            .map(item => ('str' in item ? item.str : ''))
                            .join(' ');
                        textContent += pageContent + '\n\n';
                    }
                    content = textContent.trim();
                    fileType = 'pdf';
                }
                catch (pdfError) {
                    console.error('PDF parsing error:', pdfError);
                    throw new Error(`Failed to parse PDF: ${pdfError instanceof Error ? pdfError.message : 'Unknown PDF error'}`);
                }
            }
            else if (mimeType === 'text/plain') {
                content = fileBuffer.toString('utf-8');
                fileType = 'txt';
            }
            else {
                throw new Error(`Unsupported file type: ${mimeType}`);
            }
            const document = {
                id: documentId,
                filename: fileName,
                content,
                fileType,
                size: fileBuffer.length,
                sessionId,
                uploadedAt: Date.now(),
            };
            // Process the document into chunks
            const chunks = this.chunkDocument(document);
            // Generate embeddings for chunks if enabled
            const chunksWithEmbeddings = await this.generateEmbeddingsForChunks(chunks);
            // Store document and chunks
            this.documents.set(documentId, document);
            this.chunks.set(documentId, chunksWithEmbeddings);
            storageService.saveDocument(document);
            storageService.saveDocumentChunks(documentId, chunksWithEmbeddings);
            console.log(`Processed ${fileType.toUpperCase()} document: ${fileName} (${chunksWithEmbeddings.length} chunks)`);
            return document;
        }
        catch (error) {
            console.error('Error processing document:', error);
            throw new Error(`Failed to process document: ${error instanceof Error ? error.message : 'Unknown error'}`);
        }
    }
    chunkDocument(document) {
        const chunks = [];
        const preferences = preferencesService.getPreferences();
        const chunkSize = preferences.embeddingSettings?.chunkSize || 1000; // characters per chunk
        const overlap = preferences.embeddingSettings?.chunkOverlap || 200; // character overlap between chunks
        const text = document.content?.trim();
        if (!text)
            return chunks;
        // Helper function to calculate overlap word count based on character overlap
        const calculateOverlapWordCount = (overlapChars, averageWordLength = 5) => {
            return Math.floor(overlapChars / averageWordLength);
        };
        // Split by paragraphs first, then by sentences if needed
        const paragraphs = text.split(/\n\s*\n/).filter((p) => p.trim());
        let currentChunk = '';
        let chunkIndex = 0;
        let currentOffset = 0; // Track character position in original text
        for (const paragraph of paragraphs) {
            const paragraphText = paragraph.trim();
            // If adding this paragraph would exceed chunk size, save current chunk
            if (currentChunk &&
                currentChunk.length + paragraphText.length > chunkSize) {
                if (currentChunk.trim()) {
                    const chunkStart = currentOffset - currentChunk.length;
                    const chunkEnd = currentOffset;
                    chunks.push({
                        id: uuidv4(),
                        documentId: document.id,
                        content: currentChunk.trim(),
                        chunkIndex: chunkIndex++,
                        startChar: Math.max(0, chunkStart),
                        endChar: chunkEnd,
                    });
                }
                // Start new chunk with overlap from previous chunk
                const words = currentChunk.split(' ');
                const overlapWords = words.slice(-calculateOverlapWordCount(overlap));
                currentChunk = overlapWords.join(' ') + '\n\n' + paragraphText;
                currentOffset += paragraphText.length + 2; // +2 for \n\n
            }
            else {
                // Add paragraph to current chunk
                if (currentChunk) {
                    currentChunk += '\n\n' + paragraphText;
                    currentOffset += paragraphText.length + 2; // +2 for \n\n
                }
                else {
                    currentChunk = paragraphText;
                    currentOffset += paragraphText.length;
                }
            }
        }
        // Add the final chunk
        if (currentChunk.trim()) {
            const chunkStart = currentOffset - currentChunk.length;
            const chunkEnd = currentOffset;
            chunks.push({
                id: uuidv4(),
                documentId: document.id,
                content: currentChunk.trim(),
                chunkIndex: chunkIndex,
                startChar: Math.max(0, chunkStart),
                endChar: chunkEnd,
            });
        }
        return chunks;
    }
    async generateEmbeddingForText(text) {
        try {
            const preferences = preferencesService.getPreferences();
            if (!preferences.embeddingSettings.enabled) {
                return null;
            }
            const response = await ollamaService.generateEmbeddings({
                model: preferences.embeddingSettings.model,
                input: text,
            });
            return response.embeddings[0] || null;
        }
        catch (error) {
            console.error('Failed to generate embedding:', error);
            return null;
        }
    }
    async generateEmbeddingsForChunks(chunks) {
        const preferences = preferencesService.getPreferences();
        if (!preferences.embeddingSettings.enabled) {
            return chunks;
        }
        console.log(`Generating embeddings for ${chunks.length} chunks...`);
        const chunksWithEmbeddings = [];
        for (const chunk of chunks) {
            const embedding = await this.generateEmbeddingForText(chunk.content);
            chunksWithEmbeddings.push({
                ...chunk,
                embedding: embedding || undefined,
            });
        }
        console.log(`Generated embeddings for ${chunksWithEmbeddings.filter(c => c.embedding).length} chunks`);
        return chunksWithEmbeddings;
    }
    // Method to regenerate embeddings for all existing documents
    async regenerateAllEmbeddings() {
        const preferences = preferencesService.getPreferences();
        if (!preferences.embeddingSettings.enabled) {
            console.log('Embeddings are disabled, skipping regeneration');
            return;
        }
        console.log('Starting to regenerate embeddings for all documents...');
        let processedChunks = 0;
        let totalChunks = 0;
        for (const [documentId, chunks] of this.chunks.entries()) {
            totalChunks += chunks.length;
            const chunksWithEmbeddings = await this.generateEmbeddingsForChunks(chunks);
            this.chunks.set(documentId, chunksWithEmbeddings);
            processedChunks += chunksWithEmbeddings.filter(c => c.embedding).length;
        }
        this.saveChunks();
        console.log(`Regenerated embeddings for ${processedChunks}/${totalChunks} chunks`);
    }
    // Method to get embedding model information
    async getEmbeddingModelInfo() {
        const preferences = preferencesService.getPreferences();
        let chunksWithEmbeddings = 0;
        let totalChunks = 0;
        for (const chunks of this.chunks.values()) {
            totalChunks += chunks.length;
            chunksWithEmbeddings += chunks.filter(c => c.embedding).length;
        }
        return {
            available: preferences.embeddingSettings.enabled,
            model: preferences.embeddingSettings.model,
            chunksWithEmbeddings,
            totalChunks,
        };
    }
    getDocument(documentId) {
        return this.documents.get(documentId);
    }
    getDocuments(sessionId) {
        const allDocs = Array.from(this.documents.values());
        if (sessionId) {
            return allDocs.filter(doc => doc.sessionId === sessionId);
        }
        return allDocs.sort((a, b) => b.uploadedAt - a.uploadedAt);
    }
    getDocumentChunks(documentId) {
        return this.chunks.get(documentId) || [];
    }
    deleteDocument(documentId) {
        const deleted = this.documents.delete(documentId);
        this.chunks.delete(documentId);
        if (deleted) {
            storageService.deleteDocument(documentId);
            storageService.deleteDocumentChunks(documentId);
        }
        return deleted;
    }
    // Enhanced search with semantic similarity using embeddings
    async searchDocuments(query, sessionId, limit = 5) {
        const preferences = preferencesService.getPreferences();
        // Use semantic search if embeddings are enabled
        if (preferences.embeddingSettings.enabled) {
            return this.semanticSearchDocuments(query, sessionId, limit);
        }
        // Fall back to keyword search
        return this.keywordSearchDocuments(query, sessionId, limit);
    }
    async semanticSearchDocuments(query, sessionId, limit = 5) {
        try {
            // Generate embedding for the query
            const queryEmbedding = await this.generateEmbeddingForText(query);
            if (!queryEmbedding) {
                console.warn('Failed to generate query embedding, falling back to keyword search');
                return this.keywordSearchDocuments(query, sessionId, limit);
            }
            const preferences = preferencesService.getPreferences();
            const results = [];
            for (const [documentId, documentChunks] of this.chunks.entries()) {
                const document = this.documents.get(documentId);
                if (!document)
                    continue;
                // Filter by session if specified
                if (sessionId && document.sessionId !== sessionId)
                    continue;
                for (const chunk of documentChunks) {
                    if (!chunk.embedding)
                        continue;
                    const similarity = cosineSimilarity(queryEmbedding, chunk.embedding);
                    // Only include chunks above similarity threshold
                    if (similarity >= preferences.embeddingSettings.similarityThreshold) {
                        results.push({ chunk, similarity, document });
                    }
                }
            }
            // Sort by similarity score and return top results
            return results
                .sort((a, b) => b.similarity - a.similarity)
                .slice(0, limit)
                .map(result => ({
                ...result.chunk,
                filename: result.document.filename, // Add filename for context
            }));
        }
        catch (error) {
            console.error('Semantic search failed, falling back to keyword search:', error);
            return this.keywordSearchDocuments(query, sessionId, limit);
        }
    }
    keywordSearchDocuments(query, sessionId, limit = 5) {
        const searchTerms = query
            .toLowerCase()
            .split(/\s+/)
            .filter(term => term.length > 2);
        const results = [];
        for (const [documentId, documentChunks] of this.chunks.entries()) {
            const document = this.documents.get(documentId);
            if (!document)
                continue;
            // Filter by session if specified
            if (sessionId && document.sessionId !== sessionId)
                continue;
            for (const chunk of documentChunks) {
                const chunkText = chunk.content.toLowerCase();
                let score = 0;
                // Simple scoring based on term frequency
                for (const term of searchTerms) {
                    // Escape special regex characters to prevent injection
                    const escapedTerm = term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
                    const matches = (chunkText.match(new RegExp(escapedTerm, 'gi')) || [])
                        .length;
                    score += matches;
                }
                if (score > 0) {
                    results.push({ chunk, score, document });
                }
            }
        }
        // Sort by score and return top results
        return results
            .sort((a, b) => b.score - a.score)
            .slice(0, limit)
            .map(result => ({
            ...result.chunk,
            filename: result.document.filename, // Add filename for context
        }));
    }
    // Get relevant context for RAG
    async getRelevantContext(query, sessionId) {
        const relevantChunks = await this.searchDocuments(query, sessionId, 3);
        return relevantChunks.map((chunk) => {
            const filename = chunk.filename || 'Unknown';
            return `[From: ${filename}]\n${chunk.content}`;
        });
    }
    // Restore a document from import (used during data import)
    restoreDocument(document) {
        try {
            // Add to memory
            this.documents.set(document.id, document);
            // Save to storage
            storageService.saveDocument(document);
            // If there are any chunks with this document, they'll be handled separately
            console.log(`Restored document: ${document.filename} (${document.id})`);
        }
        catch (error) {
            console.error('Error restoring document:', error);
            throw new Error(`Failed to restore document: ${error instanceof Error ? error.message : 'Unknown error'}`);
        }
    }
}
export default new DocumentService();
//# sourceMappingURL=documentService.js.map