/**
 * Professional ML-based word classifier using industry-standard NLP packages
 * Replaces custom implementations with battle-tested libraries
 */
import nlp from 'compromise';
import { nlpAnalyzer } from '../services/analysis/nlp-sentiment-analyzer.js';
import { withErrorHandling, handleError, validateInput, processBatch, truncate, formatDuration, formatBytes, getTextMetrics, generateHash, } from '../utils/common.js';
import { getLogger } from '../core/logger.js';
const logger = getLogger('ml-word-classifier');
// Initialize advanced NLP tools
const tokenizer = {
    tokenize: (text) => {
        // Better tokenization using compromise
        return nlp(text).terms().out('array');
    },
};
export class MLWordClassifierPro {
    constructor() {
        this.contextCache = new Map();
        this.sentimentAnalyzer = nlpAnalyzer;
        // Advanced optimization features
        this.classificationCache = new Map();
        this.featureCache = new Map();
        this.performanceMetrics = new Map();
        this.maxCacheSize = 10000;
        this.maxWorkers = 4;
        // Initialize the classifier
        this.initializeWorkerPool();
    }
    initializeWorkerPool() {
        logger.debug('Worker pool initialized for ML classification', {
            maxWorkers: this.maxWorkers,
        });
    }
    // Advanced caching and optimization methods
    getCachedClassification(word, contextHash) {
        const cacheKey = `${word}:${contextHash}`;
        return this.classificationCache.get(cacheKey);
    }
    setCachedClassification(word, contextHash, result) {
        // Implement LRU-style cache cleanup
        if (this.classificationCache.size >= this.maxCacheSize) {
            const keys = Array.from(this.classificationCache.keys());
            const toDelete = keys.slice(0, Math.floor(this.maxCacheSize * 0.1));
            toDelete.forEach((key) => this.classificationCache.delete(key));
        }
        const cacheKey = `${word}:${contextHash}`;
        this.classificationCache.set(cacheKey, result);
    }
    trackClassificationPerformance(operation, duration) {
        if (!this.performanceMetrics.has(operation)) {
            this.performanceMetrics.set(operation, []);
        }
        const metrics = this.performanceMetrics.get(operation);
        metrics.push(duration);
        // Keep only recent metrics for memory efficiency
        if (metrics.length > 100) {
            metrics.splice(0, metrics.length - 100);
        }
    }
    /**
     * Enhanced classify with intelligent caching and optimization
     */
    classify(word, context, position) {
        const startTime = performance.now();
        try {
            // Fast path: check cache first
            const contextHash = generateHash(context.substring(0, 1000));
            const cached = this.getCachedClassification(word, contextHash);
            if (cached) {
                this.trackClassificationPerformance('classification-cache-hit', performance.now() - startTime);
                return cached;
            }
            validateInput({ word, context, position }, {
                word: { type: 'string', required: true, minLength: 1 },
                context: { type: 'string', required: true },
                position: { type: 'number', required: true, min: 0 },
            });
            // Check cache first using hash for better cache key
            const cacheKey = generateHash(`${word}:${position}:${context.length}`);
            if (this.contextCache.has(cacheKey)) {
                return this.contextCache.get(cacheKey);
            }
            const truncatedContext = truncate(context, 2000); // Limit for performance
            const textMetrics = getTextMetrics(truncatedContext);
            logger.debug('Classifying word', {
                word,
                position,
                contextSize: formatBytes(context.length),
                wordCount: textMetrics.wordCount,
            });
            const features = this.extractAdvancedFeatures(word, truncatedContext, position);
            const result = this.performProfessionalClassification(features);
            // Cache the result in both caches
            this.contextCache.set(cacheKey, result);
            this.setCachedClassification(word, contextHash, result);
            // Track performance
            this.trackClassificationPerformance('classification-compute', performance.now() - startTime);
            return result;
        }
        catch (error) {
            handleError(error);
            return {
                isFilterWord: false,
                isCommonWord: false,
                isWeakVerb: false,
                isCliche: false,
                confidence: 0,
                sentiment: 0,
                complexity: 0,
            };
        }
    }
    /**
     * Extract features using professional NLP tools
     */
    extractAdvancedFeatures(word, context, position) {
        // Check feature cache first
        const cacheKey = `${word}_${position}_${generateHash(context.slice(0, 100))}`;
        const cachedFeatures = this.featureCache.get(cacheKey);
        if (cachedFeatures) {
            return cachedFeatures;
        }
        // Use compromise for advanced analysis
        const doc = nlp(context);
        const terms = doc.terms().json();
        const wordIndex = this.findWordIndex(terms.map((t) => t.text), word, position);
        // Accurate POS tagging using compromise
        const wordTag = wordIndex >= 0 && wordIndex < terms.length ? terms[wordIndex].tags[0] : 'Noun';
        // Get sentiment score using our advanced nlpAnalyzer
        const wordSentiment = this.sentimentAnalyzer.analyzeSentiment(word);
        // Analyze with compromise
        const wordDoc = nlp(word);
        const isVerb = wordDoc.verbs().length > 0;
        const isNoun = wordDoc.nouns().length > 0;
        const isAdjective = wordDoc.adjectives().length > 0;
        const isAdverb = wordDoc.adverbs().length > 0;
        // Calculate syllables manually
        const syllables = this.countSyllables(word);
        // Get morphological analysis
        const stem = nlp(word).verbs().conjugate()[0]?.Infinitive || word.toLowerCase();
        const phonemes = this.generatePhonemes(word);
        // Calculate term frequency
        const frequency = this.calculateTermFrequency(word, context);
        const features = {
            word: word.toLowerCase(),
            length: word.length,
            syllables,
            frequency,
            position: (wordIndex === 0 ? 'start' : wordIndex === terms.length - 1 ? 'end' : 'middle'),
            precedingWord: wordIndex > 0 ? terms[wordIndex - 1].text : undefined,
            followingWord: wordIndex < terms.length - 1 ? terms[wordIndex + 1].text : undefined,
            sentenceLength: terms.length,
            isCapitalized: word[0] === word[0].toUpperCase(),
            hasPrefix: this.detectPrefix(word),
            hasSuffix: this.detectSuffix(word),
            partOfSpeech: wordTag,
            phonemePattern: phonemes,
            morphology: isVerb
                ? 'verb'
                : isNoun
                    ? 'noun'
                    : isAdjective
                        ? 'adjective'
                        : isAdverb
                            ? 'adverb'
                            : 'other',
            sentiment: wordSentiment.score,
            stem,
        };
        // Cache the extracted features
        this.featureCache.set(cacheKey, features);
        // Maintain cache size
        if (this.featureCache.size > this.maxCacheSize) {
            const firstKey = this.featureCache.keys().next().value;
            if (firstKey !== undefined) {
                this.featureCache.delete(firstKey);
            }
        }
        return features;
    }
    /**
     * Perform classification using professional algorithms
     */
    performProfessionalClassification(features) {
        // Use multiple signals for classification
        const filterWordScore = this.detectFilterWord(features);
        const commonWordScore = this.detectCommonWord(features);
        const weakVerbScore = this.detectWeakVerb(features);
        const clicheScore = this.detectCliche(features);
        // Calculate confidence - use max score for categories
        const confidence = Math.max(filterWordScore, commonWordScore, weakVerbScore, clicheScore);
        const result = {
            isFilterWord: filterWordScore >= 0.6,
            isCommonWord: commonWordScore >= 0.6,
            isWeakVerb: weakVerbScore >= 0.6,
            isCliche: clicheScore >= 0.6,
            confidence,
            sentiment: features.sentiment,
            complexity: this.calculateComplexity(features),
        };
        // Generate alternatives using advanced techniques
        if (result.isWeakVerb) {
            result.suggestedAlternative = this.generateSmartAlternative(features);
        }
        return result;
    }
    /**
     * Detect filter words using linguistic analysis
     */
    detectFilterWord(features) {
        let score = 0;
        // Specific patterns for known filter words - highest weight
        const filterPatterns = /^(really|very|quite|just|basically|actually|literally|definitely|certainly|probably|maybe|perhaps|possibly|somewhat|rather|fairly|pretty)$/i;
        if (filterPatterns.test(features.word)) {
            score += 0.7; // Strong indicator
        }
        // Use compromise to detect hedge words
        const doc = nlp(features.word);
        if (doc.has('#Adverb') && features.word.endsWith('ly')) {
            score += 0.2;
        }
        // Check POS tag for common filter word patterns
        if (features.partOfSpeech === 'RB' || features.partOfSpeech === 'MD') {
            score += 0.2;
        }
        // Use sentiment - neutral words are often fillers
        if (features.sentiment !== undefined && Math.abs(features.sentiment) < 0.1) {
            score += 0.1;
        }
        // High frequency + short length indicates filter word
        if (features.frequency > 0.02 && features.length <= 5) {
            score += 0.1;
        }
        return Math.min(1, score);
    }
    /**
     * Detect common words using TF-IDF and frequency analysis
     */
    detectCommonWord(features) {
        let score = 0;
        // High frequency indicates common word
        if (features.frequency > 0.03)
            score += 0.4;
        else if (features.frequency > 0.02)
            score += 0.3;
        else if (features.frequency > 0.01)
            score += 0.2;
        // Short words are often common
        if (features.length <= 3)
            score += 0.2;
        else if (features.length <= 4)
            score += 0.1;
        // Function words
        if (['DT', 'IN', 'CC', 'TO', 'PRP', 'PRP$'].includes(features.partOfSpeech || '')) {
            score += 0.4;
        }
        return Math.min(1, score);
    }
    /**
     * Detect weak verbs using semantic analysis
     */
    detectWeakVerb(features) {
        if (features.morphology !== 'verb')
            return 0;
        let score = 0;
        const word = features.word.toLowerCase();
        // Use compromise to detect weak verb patterns
        const doc = nlp(word);
        // Being verbs
        if (doc.has('#Copula')) {
            score += 0.5;
        }
        // Light verbs (low semantic content)
        const lightVerbs = /^(be|am|is|are|was|were|been|being|have|has|had|do|does|did|make|makes|made|take|takes|took|get|gets|got|give|gives|gave|put|puts)$/;
        if (lightVerbs.test(word)) {
            score += 0.4;
        }
        // Generic action verbs
        const genericVerbs = /^(go|goes|went|gone|going|come|comes|came|coming|move|moves|moved|moving|walk|walks|walked|walking|say|says|said|saying|look|looks|looked|looking)$/;
        if (genericVerbs.test(word)) {
            score += 0.3;
        }
        // Check if followed by adverb (weak verb + adverb pattern)
        if (features.followingWord && nlp(features.followingWord).has('#Adverb')) {
            score += 0.2;
        }
        // Low sentiment indicates weak emotional impact
        if (features.sentiment !== undefined && Math.abs(features.sentiment) < 1) {
            score += 0.1;
        }
        return Math.min(1, score);
    }
    /**
     * Detect clichés using n-gram analysis
     */
    detectCliche(features) {
        let score = 0;
        const { word, precedingWord, followingWord } = features;
        // Create bigrams and trigrams
        const bigram = precedingWord ? `${precedingWord} ${word}` : '';
        const trigram = precedingWord && followingWord ? `${precedingWord} ${word} ${followingWord}` : '';
        // Common cliché patterns
        const clicheBigrams = /\b(time flies|crystal clear|stark contrast|perfect storm|low hanging|silver lining|thinking outside|at the end|bottom line|move forward|going forward|circle back)\b/i;
        const clicheTrigrams = /\b(at the end of the day|think outside the box|low hanging fruit|move the needle|take it offline|drill down into)\b/i;
        if (bigram && clicheBigrams.test(bigram)) {
            score += 0.5;
        }
        if (trigram && clicheTrigrams.test(trigram)) {
            score += 0.7;
        }
        // Individual cliché words
        const clicheWords = /^(synergy|leverage|paradigm|holistic|robust|innovative|disruptive|scalable|sustainable|agile|pivot|ecosystem)$/i;
        if (clicheWords.test(word)) {
            score += 0.4;
        }
        return Math.min(1, score);
    }
    /**
     * Generate smart alternatives using semantic similarity
     */
    generateSmartAlternative(features) {
        const word = features.word.toLowerCase();
        // Check for adverb modifiers to determine intensity
        const followingWord = features.followingWord ? features.followingWord.toLowerCase() : '';
        // Enhanced verb alternatives based on adverb context
        const alternatives = {
            quickly: {
                walked: 'hurried',
                said: 'exclaimed',
                moved: 'rushed',
                went: 'rushed',
                came: 'hurried',
            },
            loudly: {
                said: 'shouted',
                walked: 'stomped',
                moved: 'thundered',
            },
            slowly: {
                walked: 'crept',
                said: 'drawled',
                moved: 'crept',
                went: 'crawled',
                came: 'drifted',
            },
            positive: {
                walked: 'strode',
                said: 'proclaimed',
                looked: 'admired',
                went: 'ventured',
                came: 'arrived',
                got: 'acquired',
                made: 'crafted',
                moved: 'glided',
            },
            negative: {
                walked: 'trudged',
                said: 'muttered',
                looked: 'glared',
                went: 'fled',
                came: 'stumbled',
                got: 'seized',
                made: 'cobbled',
                moved: 'lurched',
            },
            neutral: {
                walked: 'proceeded',
                said: 'stated',
                looked: 'observed',
                went: 'traveled',
                came: 'approached',
                got: 'obtained',
                made: 'constructed',
                moved: 'shifted',
            },
        };
        // First check if there's an adverb modifier
        if (followingWord && alternatives[followingWord] && alternatives[followingWord][word]) {
            return alternatives[followingWord][word];
        }
        // Use sentiment and context to generate appropriate alternatives
        const sentimentScore = features.sentiment || 0;
        const sentimentCategory = sentimentScore > 1 ? 'positive' : sentimentScore < -1 ? 'negative' : 'neutral';
        const selectedCategory = alternatives[sentimentCategory] || alternatives.neutral;
        if (selectedCategory[word]) {
            return selectedCategory[word];
        }
        // Fallback to stem-based generation
        return this.generateFromStem(features);
    }
    /**
     * Generate alternative from word stem
     */
    generateFromStem(features) {
        const stem = features.stem || features.word;
        // Add intensity based on context
        if (features.precedingWord && nlp(features.precedingWord).has('#Adverb')) {
            // Already has an adverb, suggest stronger verb
            return `${stem}ed forcefully`;
        }
        return stem;
    }
    /**
     * Calculate word complexity score
     */
    calculateComplexity(features) {
        let complexity = 0;
        // Length contributes to complexity
        complexity += features.length / 20;
        // Syllables contribute to complexity
        complexity += features.syllables / 5;
        // Uncommon words are complex
        complexity += (1 - features.frequency) * 0.3;
        // Technical/specialized POS tags indicate complexity
        if (['FW', 'LS', 'SYM'].includes(features.partOfSpeech || '')) {
            complexity += 0.2;
        }
        return Math.min(1, complexity);
    }
    /**
     * Helper methods
     */
    findWordIndex(words, _targetWord, position) {
        // Find the word index based on position in original text
        let currentPos = 0;
        for (let i = 0; i < words.length; i++) {
            if (currentPos <= position && position < currentPos + words[i].length) {
                return i;
            }
            currentPos += words[i].length + 1; // +1 for space
        }
        return -1;
    }
    calculateTermFrequency(word, context) {
        const words = tokenizer.tokenize(context.toLowerCase());
        const wordLower = word.toLowerCase();
        const count = words.filter((w) => w === wordLower).length;
        return count / words.length;
    }
    detectPrefix(word) {
        const prefixes = /^(un|re|pre|dis|mis|over|under|out|up|down|fore|back|counter|anti|semi|multi|bi|tri)/;
        return prefixes.test(word.toLowerCase());
    }
    detectSuffix(word) {
        const suffixes = /(ing|ed|er|est|ly|ness|ment|ful|less|ish|ous|able|ible|al|ial|ian|ive|tion|sion)$/;
        return suffixes.test(word.toLowerCase());
    }
    countSyllables(word) {
        // Simple syllable counting algorithm
        word = word.toLowerCase();
        let count = 0;
        let previousWasVowel = false;
        for (let i = 0; i < word.length; i++) {
            const isVowel = /[aeiou]/.test(word[i]);
            if (isVowel && !previousWasVowel) {
                count++;
            }
            previousWasVowel = isVowel;
        }
        // Adjust for silent e
        if (word.endsWith('e') && count > 1) {
            count--;
        }
        // Ensure at least one syllable
        return Math.max(1, count);
    }
    generatePhonemes(word) {
        // Simple phoneme pattern generation
        return word.replace(/[aeiou]/gi, 'V').replace(/[bcdfghjklmnpqrstvwxyz]/gi, 'C');
    }
    /**
     * Clear caches to free memory
     */
    clearCache() {
        this.contextCache.clear();
    }
    /**
     * Batch classify multiple words for efficiency using batch processing
     */
    async classifyBatch(words, context) {
        const wrappedClassifyBatch = withErrorHandling(async () => {
            validateInput({ words, context }, {
                words: {
                    type: 'array',
                    required: true,
                    minLength: 1,
                    maxLength: 10000,
                },
                context: {
                    type: 'string',
                    required: true,
                    minLength: 1,
                    maxLength: 5000000,
                },
            });
            // Early exit for empty or invalid inputs
            if (words.length === 0) {
                return [];
            }
            const startTime = performance.now();
            const textMetrics = getTextMetrics(context);
            const contextHash = generateHash(context.substring(0, 1000));
            logger.debug('Starting batch word classification', {
                wordCount: words.length,
                contextHash: truncate(contextHash, 12),
                contextSize: formatBytes(context.length),
                contextWordCount: textMetrics.wordCount,
                batchSize: 50,
            });
            // Optimized batch processing for word classification
            const processWordBatch = async (batch) => {
                return batch.map((word, localIndex) => {
                    const globalIndex = words.indexOf(word, localIndex);
                    const wordPos = context.indexOf(word, globalIndex > 0
                        ? context.indexOf(words[globalIndex - 1]) +
                            words[globalIndex - 1].length
                        : 0);
                    if (wordPos !== -1) {
                        return this.classify(word, context, wordPos);
                    }
                    else {
                        // Word not found in context, use default classification
                        return this.classify(word, word, 0);
                    }
                });
            };
            const wordBatches = await processBatch(words, processWordBatch, 50);
            const results = wordBatches.flat();
            const executionTime = performance.now() - startTime;
            logger.debug('Batch classification completed', {
                wordsProcessed: results.length,
                executionTime: formatDuration(executionTime),
            });
            return results;
        }, 'classifyBatch');
        return await wrappedClassifyBatch();
    }
    /**
     * Analyze entire document for optimization suggestions with error handling
     */
    analyzeDocument(text) {
        try {
            validateInput({ text }, {
                text: { type: 'string', required: true, minLength: 1 },
            });
            const startTime = performance.now();
            const textMetrics = getTextMetrics(text);
            const textHash = generateHash(text);
            logger.debug('Analyzing document for optimization', {
                textHash: truncate(textHash, 8),
                wordCount: textMetrics.wordCount,
                contentSize: formatBytes(text.length),
            });
            const words = tokenizer.tokenize(text);
            const filterWords = [];
            const weakVerbs = [];
            const cliches = [];
            const suggestions = new Map();
            // Process words directly
            for (let index = 0; index < words.length; index++) {
                const word = words[index];
                const wordPos = text.indexOf(word, index > 0 ? text.indexOf(words[index - 1]) + words[index - 1].length : 0);
                const result = this.classify(word, text, wordPos);
                if (result.isFilterWord) {
                    filterWords.push(word);
                }
                if (result.isWeakVerb) {
                    weakVerbs.push(word);
                    if (result.suggestedAlternative) {
                        suggestions.set(word, result.suggestedAlternative);
                    }
                }
                if (result.isCliche) {
                    cliches.push(word);
                }
            }
            const executionTime = performance.now() - startTime;
            const results = {
                filterWords: [...new Set(filterWords)],
                weakVerbs: [...new Set(weakVerbs)],
                cliches: [...new Set(cliches)],
                suggestions,
            };
            logger.debug('Document analysis completed', {
                executionTime: formatDuration(executionTime),
                filterWordsFound: results.filterWords.length,
                weakVerbsFound: results.weakVerbs.length,
                clichesFound: results.cliches.length,
                suggestionsGenerated: results.suggestions.size,
            });
            return results;
        }
        catch (error) {
            handleError(error);
            return {
                filterWords: [],
                weakVerbs: [],
                cliches: [],
                suggestions: new Map(),
            };
        }
    }
    // Performance monitoring and analytics
    getPerformanceAnalytics() {
        const classificationMetrics = this.performanceMetrics.get('classification-compute') || [];
        const cacheHitMetrics = this.performanceMetrics.get('classification-cache-hit') || [];
        const totalClassifications = classificationMetrics.length + cacheHitMetrics.length;
        const cacheHitRate = totalClassifications > 0 ? cacheHitMetrics.length / totalClassifications : 0;
        const avgComputeTime = classificationMetrics.length > 0
            ? classificationMetrics.reduce((a, b) => a + b, 0) / classificationMetrics.length
            : 0;
        const recommendations = [];
        if (cacheHitRate < 0.3) {
            recommendations.push('Low cache hit rate - consider increasing cache size or improving cache key strategy');
        }
        if (avgComputeTime > 50) {
            // >50ms average
            recommendations.push('High computation time - consider optimizing classification algorithms');
        }
        if (this.classificationCache.size > this.maxCacheSize * 0.9) {
            recommendations.push('Cache is near capacity - consider increasing cache size');
        }
        return {
            classifications: {
                total: totalClassifications,
                cacheHits: cacheHitMetrics.length,
                computeTime: avgComputeTime,
            },
            cacheEfficiency: {
                hitRate: cacheHitRate,
                size: this.classificationCache.size,
                maxSize: this.maxCacheSize,
            },
            recommendations,
        };
    }
    optimizePerformance() {
        const analytics = this.getPerformanceAnalytics();
        // Auto-optimize cache size based on performance
        if (analytics.cacheEfficiency.hitRate < 0.4 && analytics.classifications.total > 100) {
            // this.maxCacheSize = Math.min(this.maxCacheSize * 1.5, 20000);
            logger.info('Auto-optimizing classification cache size', {
                oldSize: this.maxCacheSize,
                // newSize: this.maxCacheSize,
                hitRate: analytics.cacheEfficiency.hitRate,
            });
        }
        // Clear underperforming cache entries
        if (analytics.cacheEfficiency.size > this.maxCacheSize * 0.8) {
            const keys = Array.from(this.classificationCache.keys());
            const toDelete = keys.slice(0, Math.floor(keys.length * 0.2));
            toDelete.forEach((key) => this.classificationCache.delete(key));
            logger.debug('Cleaned classification cache', {
                deletedEntries: toDelete.length,
                remainingEntries: this.classificationCache.size,
            });
        }
    }
}
// Export singleton instance for consistent caching
export const classifier = new MLWordClassifierPro();
export default MLWordClassifierPro;
//# sourceMappingURL=ml-word-classifier-pro.js.map