/**
 * Embedding Service for UAP Memory System
 *
 * Provides text embedding generation for semantic memory retrieval.
 * Supports multiple backends: Ollama (local), OpenAI, local transformers, or simple TF-IDF fallback.
 *
 * Priority order:
 * 1. Ollama (if running with nomic-embed-text or similar)
 * 2. OpenAI (if API key available)
 * 3. Local sentence-transformers (if installed)
 * 4. TF-IDF fallback (always available)
 */
import { execFileSync } from 'child_process';
import { createHash } from 'crypto';
import { concurrentMap } from '../utils/concurrency-pool.js';
import { createLogger } from '../utils/logger.js';
import { getPerformanceMonitor } from '../utils/performance-monitor.js';
import { cosineSimilarity as sharedCosineSimilarity } from '../utils/string-similarity.js';
const log = createLogger('embeddings');
/**
 * llama.cpp Embeddings Provider (LOCAL - NO API COSTS)
 *
 * Uses llama-server with --embeddings flag to serve embedding models.
 * Recommended model: nomic-embed-text-v2-moe (GGUF)
 *   - 475M total params, 305M active (MoE: 8 experts, top-2 routing)
 *   - 768 dimensions (Matryoshka: can truncate to 256)
 *   - SoTA multilingual (100+ languages, MIRACL 65.8, BEIR 52.86)
 *   - Q4_K_M quant: ~328MB VRAM — fits alongside Qwen3.5 35B A3B
 *
 * To run alongside Qwen3.5 35B A3B, launch TWO llama-server instances:
 *
 *   # Instance 1: Main LLM (Qwen3.5 35B A3B) on port 8080
 *   llama-server \
 *     --model /path/to/Qwen3.5-35B-A3B-UD-IQ4_XS.gguf \
 *     --chat-template-file chat_template.jinja \
 *     --port 8080 --host 0.0.0.0 \
 *     --ctx-size 131072 --gpu-layers 99 \
 *     --cache-type-k q8_0 --cache-type-v q4_0 \
 *     -fa on --threads 8 --batch-size 512
 *
 *   # Instance 2: Embedding model on port 8081
 *   llama-server \
 *     --model /path/to/nomic-embed-text-v2-moe.Q4_K_M.gguf \
 *     --port 8081 --host 0.0.0.0 \
 *     --embeddings --pooling mean \
 *     --ctx-size 512 --gpu-layers 99 \
 *     --batch-size 2048 --ubatch-size 512 \
 *     --threads 4
 *
 * VRAM budget (24GB GPU):
 *   Qwen3.5 IQ4_XS: ~17GB model + ~3GB KV cache = ~20GB
 *   nomic-embed Q4_K_M: ~328MB model + ~50MB KV = ~380MB
 *   Total: ~20.4GB — fits in 24GB with headroom
 *
 * For CPU-only embedding (saves all GPU for LLM):
 *   llama-server \
 *     --model /path/to/nomic-embed-text-v2-moe.Q4_K_M.gguf \
 *     --port 8081 --host 0.0.0.0 \
 *     --embeddings --pooling mean \
 *     --ctx-size 512 --gpu-layers 0 \
 *     --batch-size 2048 --threads 8
 */
export class LlamaCppEmbeddingProvider {
    name = 'llama-cpp';
    dimensions = 768; // nomic-embed-text-v2-moe default
    endpoint;
    available = null;
    taskPrefix;
    constructor(endpoint = 'http://192.168.1.165:8081', options = {}) {
        // Support env var override
        this.endpoint = process.env.UAP_EMBEDDING_ENDPOINT || endpoint;
        // nomic-embed-text-v2 requires task prefixes for best results
        this.taskPrefix = options.taskPrefix || 'search_document: ';
    }
    async isAvailable() {
        if (this.available !== null)
            return this.available;
        try {
            // llama-server health check
            const response = await fetch(`${this.endpoint}/health`, {
                method: 'GET',
                signal: AbortSignal.timeout(2000),
            });
            if (!response.ok) {
                this.available = false;
                return false;
            }
            // Verify it supports embeddings by checking /v1/embeddings
            const testResponse = await fetch(`${this.endpoint}/v1/embeddings`, {
                method: 'POST',
                headers: { 'Content-Type': 'application/json' },
                body: JSON.stringify({ input: 'test' }),
                signal: AbortSignal.timeout(5000),
            });
            this.available = testResponse.ok;
            if (testResponse.ok) {
                // Detect actual dimensions from response
                const data = (await testResponse.json());
                if (data.data?.[0]?.embedding) {
                    this.dimensions = data.data[0].embedding.length;
                }
            }
            return this.available;
        }
        catch {
            this.available = false;
            return false;
        }
    }
    async embed(text) {
        const results = await this.embedBatch([text]);
        return results[0];
    }
    async postEmbeddings(inputs) {
        const response = await fetch(`${this.endpoint}/v1/embeddings`, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({ input: inputs }),
            signal: AbortSignal.timeout(30000),
        });
        if (!response.ok) {
            throw new Error(`llama.cpp embedding API error: ${response.status} ${response.statusText}`);
        }
        const data = (await response.json());
        if (!data?.data || !Array.isArray(data.data)) {
            throw new Error('OpenAI embedding API returned unexpected response shape');
        }
        const embeddings = data.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
        if (embeddings[0]) {
            this.dimensions = embeddings[0].length;
        }
        return embeddings;
    }
    async embedBatch(texts) {
        // nomic-embed-text-v2 task prefix; truncate so one doc stays under the
        // server's per-request token budget (long docs return 4xx/5xx).
        const MAX_CHARS = 2000;
        const SUB_BATCH = 8; // the embed server caps total tokens per request
        const prefixed = texts.map((t) => {
            const p = t.startsWith('search_query: ') || t.startsWith('search_document: ')
                ? t
                : this.taskPrefix + t;
            return p.length > MAX_CHARS ? p.slice(0, MAX_CHARS) : p;
        });
        const out = [];
        for (let i = 0; i < prefixed.length; i += SUB_BATCH) {
            const chunk = prefixed.slice(i, i + SUB_BATCH);
            try {
                out.push(...(await this.postEmbeddings(chunk)));
            }
            catch (err) {
                // A single embed (e.g. a query) fails loudly so callers can fall back.
                if (prefixed.length === 1)
                    throw err;
                // Multi-doc store: retry per item; a doc the server still refuses gets a
                // zero vector so the run never aborts mid-batch.
                for (const one of chunk) {
                    try {
                        out.push((await this.postEmbeddings([one]))[0]);
                    }
                    catch {
                        out.push(new Array(this.dimensions).fill(0));
                    }
                }
            }
        }
        return out;
    }
}
/**
 * Ollama Embeddings Provider (LOCAL - NO API COSTS)
 * Uses nomic-embed-text (768 dimensions) or other embedding models
 */
export class OllamaEmbeddingProvider {
    name = 'ollama';
    dimensions = 768; // nomic-embed-text default
    endpoint;
    model;
    available = null;
    constructor(endpoint = 'http://localhost:11434', model = 'nomic-embed-text') {
        this.endpoint = endpoint;
        this.model = model;
    }
    async isAvailable() {
        if (this.available !== null)
            return this.available;
        try {
            const response = await fetch(`${this.endpoint}/api/tags`, {
                method: 'GET',
                signal: AbortSignal.timeout(2000),
            });
            if (!response.ok) {
                this.available = false;
                return false;
            }
            const data = (await response.json());
            const hasEmbedModel = data.models?.some((m) => m.name.includes('embed') ||
                m.name.includes('nomic') ||
                m.name === this.model ||
                m.name.startsWith(this.model));
            this.available = hasEmbedModel;
            return hasEmbedModel;
        }
        catch {
            this.available = false;
            return false;
        }
    }
    async embed(text) {
        const response = await fetch(`${this.endpoint}/api/embeddings`, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({
                model: this.model,
                prompt: text,
            }),
        });
        if (!response.ok) {
            throw new Error(`Ollama API error: ${response.status} ${response.statusText}`);
        }
        const data = (await response.json());
        if (!data?.embedding || !Array.isArray(data.embedding)) {
            throw new Error('Ollama embedding API returned unexpected response shape');
        }
        this.dimensions = data.embedding.length; // Update dimensions from actual response
        return data.embedding;
    }
    async embedBatch(texts) {
        // Ollama doesn't have native batch -- use bounded concurrency
        return concurrentMap(texts, (t) => this.embed(t));
    }
}
/**
 * OpenAI Embeddings Provider
 * Uses text-embedding-3-small (1536 dimensions) or text-embedding-ada-002
 */
export class OpenAIEmbeddingProvider {
    name = 'openai';
    dimensions = 1536;
    apiKey;
    model;
    constructor(apiKey, model = 'text-embedding-3-small') {
        this.apiKey = apiKey || process.env.OPENAI_API_KEY || '';
        this.model = model;
    }
    async isAvailable() {
        return !!this.apiKey;
    }
    async embed(text) {
        const results = await this.embedBatch([text]);
        return results[0];
    }
    async embedBatch(texts) {
        if (!this.apiKey) {
            throw new Error('OpenAI API key not configured');
        }
        const response = await fetch('https://api.openai.com/v1/embeddings', {
            method: 'POST',
            headers: {
                Authorization: `Bearer ${this.apiKey}`,
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({
                model: this.model,
                input: texts,
            }),
        });
        if (!response.ok) {
            throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`);
        }
        const data = (await response.json());
        return data.data.map((d) => d.embedding);
    }
}
/**
 * Local Sentence Transformers Provider
 * Uses Python sentence-transformers library for local embedding generation
 */
export class LocalEmbeddingProvider {
    name = 'local';
    dimensions = 384;
    model;
    pythonPath;
    constructor(model = 'all-MiniLM-L6-v2', pythonPath = 'python3') {
        this.model = model;
        this.pythonPath = pythonPath;
    }
    async isAvailable() {
        try {
            execFileSync(this.pythonPath, ['-c', 'from sentence_transformers import SentenceTransformer'], {
                stdio: 'pipe',
                timeout: 10000,
            });
            return true;
        }
        catch {
            return false;
        }
    }
    async embed(text) {
        const results = await this.embedBatch([text]);
        return results[0];
    }
    async embedBatch(texts) {
        // Pass texts via stdin to avoid shell injection and Python code injection.
        // The model name is validated to contain only safe characters.
        const safeModel = this.model.replace(/[^a-zA-Z0-9_\-./]/g, '');
        const script = `
import json
import sys
from sentence_transformers import SentenceTransformer

model = SentenceTransformer(sys.argv[1], device="cpu")
texts = json.loads(sys.stdin.read())
embeddings = model.encode(texts, convert_to_numpy=True)
print(json.dumps(embeddings.tolist()))
`;
        try {
            const result = execFileSync(this.pythonPath, ['-c', script, safeModel], {
                encoding: 'utf-8',
                timeout: 60000,
                maxBuffer: 50 * 1024 * 1024,
                input: JSON.stringify(texts),
            });
            const parsed = JSON.parse(result.trim());
            if (!Array.isArray(parsed)) {
                throw new Error('Local embedding script returned non-array result');
            }
            return parsed;
        }
        catch (error) {
            throw new Error(`Local embedding generation failed: ${error instanceof Error ? error.message : String(error)}`);
        }
    }
}
/**
 * Enhanced TF-IDF Fallback Provider with Word Vectors
 * Improved keyword-based embeddings with semantic awareness
 */
export class TFIDFEmbeddingProvider {
    name = 'tfidf-enhanced';
    dimensions = 384;
    vocabulary = new Map();
    idfScores = new Map();
    documents = [];
    wordVectors = new Map();
    constructor() {
        // Initialize semantic word clusters for better similarity
        this.initializeSemanticClusters();
    }
    async isAvailable() {
        return true; // Always available as fallback
    }
    async embed(text) {
        const tokens = this.tokenize(text);
        const vector = new Array(this.dimensions).fill(0);
        // Combine TF-IDF with semantic clustering
        for (const token of tokens) {
            const idx = this.getTokenIndex(token);
            const tf = tokens.filter((t) => t === token).length / tokens.length;
            const idf = this.idfScores.get(token) || Math.log(this.documents.length + 2);
            // Base TF-IDF contribution
            vector[idx % this.dimensions] += tf * idf;
            // Add semantic cluster contribution
            const wordVec = this.wordVectors.get(token);
            if (wordVec) {
                for (let i = 0; i < wordVec.length; i++) {
                    vector[i] += wordVec[i] * tf * 0.3; // Weighted semantic contribution
                }
            }
        }
        // Add n-gram features for better phrase matching
        const bigrams = this.getBigrams(tokens);
        for (const bigram of bigrams) {
            const idx = this.hashString(bigram) % this.dimensions;
            vector[idx] += 0.5;
        }
        return this.normalize(vector);
    }
    async embedBatch(texts) {
        // Update IDF scores with new documents
        this.updateIDF(texts);
        return concurrentMap(texts, (t) => this.embed(t));
    }
    addDocument(text) {
        this.documents.push(text);
        this.updateIDF([text]);
    }
    tokenize(text) {
        // Enhanced tokenization that handles code better
        return text
            .toLowerCase()
            .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase
            .replace(/_/g, ' ') // Split snake_case
            .replace(/[^\w\s-]/g, ' ')
            .split(/\s+/)
            .filter((t) => t.length > 1 && !this.isStopWord(t));
    }
    isStopWord(token) {
        const stopWords = new Set([
            'the',
            'a',
            'an',
            'is',
            'are',
            'was',
            'were',
            'be',
            'been',
            'being',
            'have',
            'has',
            'had',
            'do',
            'does',
            'did',
            'will',
            'would',
            'could',
            'should',
            'may',
            'might',
            'must',
            'shall',
            'can',
            'need',
            'dare',
            'ought',
            'used',
            'to',
            'of',
            'in',
            'for',
            'on',
            'with',
            'at',
            'by',
            'from',
            'as',
            'into',
            'through',
            'during',
            'before',
            'after',
            'above',
            'below',
            'between',
            'under',
            'again',
            'further',
            'then',
            'once',
            'here',
            'there',
            'when',
            'where',
            'why',
            'how',
            'all',
            'each',
            'few',
            'more',
            'most',
            'other',
            'some',
            'such',
            'no',
            'nor',
            'not',
            'only',
            'own',
            'same',
            'so',
            'than',
            'too',
            'very',
            'just',
            'and',
            'but',
            'if',
            'or',
            'because',
            'until',
            'while',
            'this',
            'that',
            'these',
            'those',
            'it',
        ]);
        return stopWords.has(token);
    }
    getBigrams(tokens) {
        const bigrams = [];
        for (let i = 0; i < tokens.length - 1; i++) {
            bigrams.push(`${tokens[i]}_${tokens[i + 1]}`);
        }
        return bigrams;
    }
    hashString(str) {
        let hash = 0;
        for (let i = 0; i < str.length; i++) {
            const char = str.charCodeAt(i);
            hash = (hash << 5) - hash + char;
            hash = hash & hash;
        }
        return Math.abs(hash);
    }
    getTokenIndex(token) {
        if (!this.vocabulary.has(token)) {
            this.vocabulary.set(token, this.vocabulary.size);
        }
        return this.vocabulary.get(token);
    }
    updateIDF(_newDocs) {
        const tokenDocs = new Map();
        for (let i = 0; i < this.documents.length; i++) {
            const tokens = new Set(this.tokenize(this.documents[i]));
            for (const token of tokens) {
                if (!tokenDocs.has(token)) {
                    tokenDocs.set(token, new Set());
                }
                tokenDocs.get(token).add(i);
            }
        }
        for (const [token, docs] of tokenDocs) {
            this.idfScores.set(token, Math.log((this.documents.length + 1) / (docs.size + 1)) + 1);
        }
    }
    normalize(vector) {
        const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0));
        if (magnitude === 0)
            return vector;
        return vector.map((v) => v / magnitude);
    }
    /**
     * Initialize semantic word clusters for domain-specific terms
     * Words in the same cluster get similar vector contributions
     */
    initializeSemanticClusters() {
        const clusters = {
            // Programming concepts
            programming: [
                'code',
                'function',
                'class',
                'method',
                'variable',
                'const',
                'let',
                'var',
                'import',
                'export',
                'async',
                'await',
                'promise',
                'callback',
            ],
            types: [
                'type',
                'interface',
                'enum',
                'string',
                'number',
                'boolean',
                'array',
                'object',
                'null',
                'undefined',
            ],
            errors: [
                'error',
                'exception',
                'throw',
                'catch',
                'try',
                'finally',
                'bug',
                'fix',
                'debug',
                'trace',
                'stack',
            ],
            testing: [
                'test',
                'spec',
                'describe',
                'it',
                'expect',
                'assert',
                'mock',
                'stub',
                'spy',
                'coverage',
            ],
            git: [
                'git',
                'commit',
                'push',
                'pull',
                'merge',
                'branch',
                'checkout',
                'rebase',
                'stash',
                'diff',
            ],
            memory: [
                'memory',
                'cache',
                'store',
                'retrieve',
                'query',
                'embed',
                'vector',
                'semantic',
                'context',
            ],
            security: [
                'security',
                'auth',
                'token',
                'secret',
                'password',
                'encrypt',
                'decrypt',
                'hash',
                'salt',
            ],
            performance: [
                'performance',
                'optimize',
                'fast',
                'slow',
                'latency',
                'throughput',
                'cache',
                'batch',
            ],
            database: [
                'database',
                'sql',
                'query',
                'table',
                'index',
                'schema',
                'migration',
                'insert',
                'update',
                'delete',
            ],
            network: [
                'network',
                'http',
                'api',
                'request',
                'response',
                'endpoint',
                'url',
                'fetch',
                'axios',
            ],
        };
        // Generate pseudo-random but consistent vectors for each cluster
        for (const [clusterName, words] of Object.entries(clusters)) {
            const clusterVector = this.generateClusterVector(clusterName);
            for (const word of words) {
                // Add slight variation for each word
                const wordVector = clusterVector.map((v, i) => v + ((this.hashString(word + i.toString()) % 100) - 50) / 500);
                this.wordVectors.set(word, wordVector);
            }
        }
    }
    generateClusterVector(seed) {
        const vector = new Array(this.dimensions).fill(0);
        const hash = this.hashString(seed);
        // Create sparse activation pattern for this cluster
        for (let i = 0; i < 20; i++) {
            const idx = (hash + i * 17) % this.dimensions;
            vector[idx] = ((hash + i) % 100) / 100;
        }
        return vector;
    }
}
/**
 * Embedding Service - Main interface for UAP memory system
 * Automatically selects best available provider
 *
 * Priority: Ollama (local, free) > OpenAI > Local transformers > TF-IDF fallback
 */
export class EmbeddingService {
    provider = null;
    providers;
    cache = new Map();
    cacheMaxSize = 10000;
    /** Memory pressure threshold (fraction of heap used). Evicts aggressively above this. */
    memoryPressureThreshold = 0.85;
    constructor(ollamaEndpoint, ollamaModel, llamaCppEndpoint) {
        this.providers = [
            new LlamaCppEmbeddingProvider(llamaCppEndpoint), // First priority - llama.cpp local server
            new OllamaEmbeddingProvider(ollamaEndpoint, ollamaModel), // Second priority - Ollama local
            new OpenAIEmbeddingProvider(),
            new LocalEmbeddingProvider(),
            new TFIDFEmbeddingProvider(),
        ];
    }
    async initialize() {
        for (const provider of this.providers) {
            if (await provider.isAvailable()) {
                this.provider = provider;
                log.info(`[EmbeddingService] Using provider: ${provider.name} (${provider.dimensions} dims)`);
                return;
            }
        }
        // Fallback to TF-IDF which is always available
        this.provider = this.providers[this.providers.length - 1];
        log.info('Fallback to TF-IDF provider');
    }
    async embed(text) {
        if (!this.provider) {
            await this.initialize();
        }
        // Check cache (LRU: re-insert on hit to move to end of Map iteration order)
        const cacheKey = this.getCacheKey(text);
        if (this.cache.has(cacheKey)) {
            const cached = this.cache.get(cacheKey);
            // Move to end (most recently used) by re-inserting
            this.cache.delete(cacheKey);
            this.cache.set(cacheKey, cached);
            return cached;
        }
        const monitor = getPerformanceMonitor();
        const embedding = await monitor.measure('embedding.embed', () => this.provider.embed(text));
        // Memory-pressure-aware eviction (C2 optimization)
        const heapUsed = process.memoryUsage().heapUsed;
        const heapTotal = process.memoryUsage().heapTotal;
        const pressure = heapTotal > 0 ? heapUsed / heapTotal : 0;
        if (pressure > this.memoryPressureThreshold && this.cache.size > 100) {
            // Under memory pressure: evict 25% of cache (LRU order)
            const evictCount = Math.floor(this.cache.size * 0.25);
            const keys = this.cache.keys();
            for (let i = 0; i < evictCount; i++) {
                const key = keys.next().value;
                if (key !== undefined) {
                    this.cache.delete(key);
                }
            }
            log.debug(`[EmbeddingService] Memory pressure eviction: removed ${evictCount} entries`);
        }
        else if (this.cache.size >= this.cacheMaxSize) {
            // Normal LRU eviction: remove least recently used entry
            const firstKey = this.cache.keys().next().value;
            if (firstKey !== undefined) {
                this.cache.delete(firstKey);
            }
        }
        this.cache.set(cacheKey, embedding);
        return embedding;
    }
    async embedBatch(texts) {
        if (!this.provider) {
            await this.initialize();
        }
        const results = [];
        const uncachedTexts = [];
        const uncachedIndices = [];
        // Check cache for each text (LRU: re-insert on hit to mark as recently used)
        for (let i = 0; i < texts.length; i++) {
            const cacheKey = this.getCacheKey(texts[i]);
            if (this.cache.has(cacheKey)) {
                const cached = this.cache.get(cacheKey);
                // Move to end (most recently used) by re-inserting
                this.cache.delete(cacheKey);
                this.cache.set(cacheKey, cached);
                results[i] = cached;
            }
            else {
                uncachedTexts.push(texts[i]);
                uncachedIndices.push(i);
            }
        }
        // Generate embeddings for uncached texts
        if (uncachedTexts.length > 0) {
            const newEmbeddings = await this.provider.embedBatch(uncachedTexts);
            for (let i = 0; i < uncachedTexts.length; i++) {
                const idx = uncachedIndices[i];
                results[idx] = newEmbeddings[i];
                // Update cache
                const cacheKey = this.getCacheKey(uncachedTexts[i]);
                if (this.cache.size < this.cacheMaxSize) {
                    this.cache.set(cacheKey, newEmbeddings[i]);
                }
            }
        }
        return results;
    }
    cosineSimilarity(a, b) {
        return sharedCosineSimilarity(a, b);
    }
    getDimensions() {
        return this.provider?.dimensions || 384;
    }
    getProviderName() {
        return this.provider?.name || 'uninitialized';
    }
    getCacheKey(text) {
        // Use SHA-256 hash to avoid collisions from truncation
        return createHash('sha256').update(text.toLowerCase().trim()).digest('hex');
    }
}
// Singleton instance
let embeddingServiceInstance = null;
export function getEmbeddingService() {
    if (!embeddingServiceInstance) {
        const llamaCppEndpoint = process.env.UAP_EMBEDDING_ENDPOINT || undefined;
        embeddingServiceInstance = new EmbeddingService(undefined, undefined, llamaCppEndpoint);
    }
    return embeddingServiceInstance;
}
export async function generateEmbedding(text) {
    const service = getEmbeddingService();
    return service.embed(text);
}
export async function generateEmbeddings(texts) {
    const service = getEmbeddingService();
    return service.embedBatch(texts);
}
//# sourceMappingURL=embeddings.js.map