/**
 * Schema Recommendation Service
 *
 * Analyzes raw_fragments to identify fields that should be promoted to schema fields.
 * Supports auto-enhancement for high-confidence fields and manual recommendations.
 */
import { db } from "../db.js";
import { loadCodeDefinedSchemaEntry, normalizeEntityTypeForSchema, SchemaRegistryService, } from "./schema_registry.js";
import { logger } from "../utils/logger.js";
// Observation sources considered "trusted" for threshold relaxation.
// These represent agent-driven or bulk-import sessions where repeated
// observations of the same field pattern are high-signal, not noise.
const TRUSTED_OBSERVATION_SOURCES = new Set(["llm_summary", "import", "sync"]);
// Default configuration (recommended)
export const DEFAULT_AUTO_ENHANCEMENT_CONFIG = {
    enabled: true, // Enable by default
    threshold: 3, // After 3 occurrences (filters one-offs, still fast)
    min_confidence: 0.85, // 85% confidence minimum (balances speed and quality)
    auto_enhance_high_confidence: true, // Auto-enhance high-confidence fields
    user_specific_aggressive: true, // More aggressive for user-specific schemas (user's own data)
    global_conservative: true, // Conservative for global schemas (affects all users)
    trusted_source_relaxation: true, // Relax thresholds for trusted sources on user-scoped schemas
};
export class SchemaRecommendationService {
    schemaRegistry;
    constructor() {
        this.schemaRegistry = new SchemaRegistryService();
    }
    /** Active registry row, or code-defined ENTITY_SCHEMAS baseline when none. */
    async loadSchemaForAutoEnhance(entityType, userId) {
        const row = await this.schemaRegistry.loadActiveSchema(entityType, userId);
        if (row)
            return row;
        const normalized = normalizeEntityTypeForSchema(entityType);
        return ((await loadCodeDefinedSchemaEntry(entityType)) ??
            (await loadCodeDefinedSchemaEntry(normalized)) ??
            null);
    }
    /**
     * Check if a field is eligible for auto-enhancement based on confidence criteria
     */
    async checkAutoEnhancementEligibility(options) {
        const config = options.config || DEFAULT_AUTO_ENHANCEMENT_CONFIG;
        // 1. Check blacklist first
        const isBlacklisted = await this.checkBlacklist(options.entity_type, options.fragment_key, options.user_id);
        if (isBlacklisted) {
            return {
                eligible: false,
                confidence: 0,
                reasoning: "Field is blacklisted",
            };
        }
        // 2. Validate field name
        if (!this.isValidFieldName(options.fragment_key)) {
            return {
                eligible: false,
                confidence: 0,
                reasoning: "Invalid field name pattern",
            };
        }
        // 3. Load current schema to check if field exists (type mismatch detection)
        const currentSchema = await this.loadSchemaForAutoEnhance(options.entity_type, options.user_id);
        const fieldExists = currentSchema?.schema_definition.fields[options.fragment_key];
        // 4. Query raw_fragments for this field
        // Include source_id for diversity checks
        let fragmentsQuery = db
            .from("raw_fragments")
            .select("fragment_value, frequency_count, source_id, entity_type")
            .eq("entity_type", options.entity_type)
            .eq("fragment_key", options.fragment_key);
        // Handle user_id properly: check both the provided user_id and the default UUID
        // Also handle null (for global schemas)
        const defaultUserId = "00000000-0000-0000-0000-000000000000";
        if (options.user_id) {
            if (options.user_id === defaultUserId) {
                // Check both default UUID and null (legacy data might use null)
                fragmentsQuery = fragmentsQuery.or(`user_id.eq.${defaultUserId},user_id.is.null`);
            }
            else {
                fragmentsQuery = fragmentsQuery.eq("user_id", options.user_id);
            }
        }
        else {
            // No user_id provided - check both null and default UUID
            fragmentsQuery = fragmentsQuery.or(`user_id.is.null,user_id.eq.${defaultUserId}`);
        }
        const { data: fragments, error } = await fragmentsQuery;
        if (error || !fragments || fragments.length === 0) {
            return {
                eligible: false,
                confidence: 0,
                reasoning: "No fragments found for this field",
            };
        }
        // 5. If field exists in schema, check if converter is needed (type mismatch)
        if (fieldExists) {
            // Field exists but is in raw_fragments = type mismatch or missing converter
            const converterNeeded = await this.detectConverterNeeded({
                field_name: options.fragment_key,
                schema_field: fieldExists,
                fragments: fragments,
            });
            if (converterNeeded) {
                // Converter detected - eligible for auto-enhancement
                return {
                    eligible: true,
                    confidence: converterNeeded.confidence,
                    inferred_type: fieldExists.type, // Keep schema type
                    converter_suggestion: converterNeeded.converter,
                    reasoning: `Field exists but needs converter: ${converterNeeded.reasoning}`,
                };
            }
            else {
                // Field exists but converter detection failed
                // Calculate confidence from fragments for pending recommendation
                const confidenceResult = await this.calculateFieldConfidence({
                    fragment_key: options.fragment_key,
                    entity_type: options.entity_type,
                    user_id: options.user_id,
                });
                const actualType = confidenceResult.inferred_type || "unknown";
                return {
                    eligible: false,
                    confidence: confidenceResult.confidence,
                    reasoning: `Field exists in schema (type: ${fieldExists.type}) but converter detection failed. Actual data type: ${actualType}. May need manual review or custom converter.`,
                };
            }
        }
        // 6. Sum frequency counts
        const totalFrequency = fragments.reduce((sum, f) => sum + (f.frequency_count || 1), 0);
        // Derive effective thresholds — two modifiers applied in order:
        //
        // 1. global_conservative (tighten): raises threshold 2x and min_confidence +0.05
        //    for global-scope schemas, protecting all users from premature changes.
        //
        // 2. trusted_source_relaxation (loosen): lowers threshold to ceil(base * 0.5)
        //    (min 1) and min_confidence -0.05 for user-scoped schemas when the dominant
        //    observation source is trusted (llm_summary, import, sync). Does NOT apply
        //    to global schemas — global_conservative always wins.
        const isGlobalSchema = config.global_conservative &&
            currentSchema != null &&
            (currentSchema.scope === "global" || currentSchema.user_id == null);
        // Determine trusted-source eligibility: user-scoped schema + trusted observation source
        let isTrustedSource = false;
        if (config.trusted_source_relaxation && !isGlobalSchema) {
            if (options.observation_source &&
                TRUSTED_OBSERVATION_SOURCES.has(options.observation_source)) {
                isTrustedSource = true;
            }
            else {
                // Look up dominant observation_source for this entity_type + user_id
                const dominantSource = await this.lookupDominantObservationSource(options.entity_type, options.user_id);
                if (dominantSource && TRUSTED_OBSERVATION_SOURCES.has(dominantSource)) {
                    isTrustedSource = true;
                }
            }
        }
        let effectiveThreshold;
        let effectiveMinConfidence;
        if (config.threshold === "pattern") {
            effectiveThreshold = "pattern";
            effectiveMinConfidence = config.min_confidence;
        }
        else if (isGlobalSchema) {
            effectiveThreshold = config.threshold * 2;
            effectiveMinConfidence = config.min_confidence + 0.05;
        }
        else if (isTrustedSource) {
            effectiveThreshold = Math.max(1, Math.ceil(config.threshold * 0.5));
            effectiveMinConfidence = config.min_confidence - 0.05;
        }
        else {
            effectiveThreshold = config.threshold;
            effectiveMinConfidence = config.min_confidence;
        }
        const thresholdSuffix = isGlobalSchema
            ? " (global_conservative)"
            : isTrustedSource
                ? " (trusted_source)"
                : "";
        // Check threshold
        if (effectiveThreshold !== "pattern" && totalFrequency < effectiveThreshold) {
            return {
                eligible: false,
                confidence: 0,
                reasoning: `Frequency ${totalFrequency} below threshold ${effectiveThreshold}${thresholdSuffix}`,
            };
        }
        // 7. Calculate confidence
        const confidenceResult = await this.calculateFieldConfidence({
            fragment_key: options.fragment_key,
            entity_type: options.entity_type,
            user_id: options.user_id,
        });
        // 8. Check minimum confidence
        if (confidenceResult.confidence < effectiveMinConfidence) {
            return {
                eligible: false,
                confidence: confidenceResult.confidence,
                reasoning: `Confidence ${confidenceResult.confidence.toFixed(2)} below minimum ${effectiveMinConfidence.toFixed(2)}${thresholdSuffix}`,
            };
        }
        // 9. Check source diversity (2+ different sources) OR row diversity (2+ different rows/observations)
        // This allows structured files (parquet, CSV, JSON arrays) with multiple rows to trigger enhancement
        // while still requiring multiple sources for single-row file types
        const uniqueSources = new Set(fragments.map((f) => f.source_id)).size;
        // For row diversity: count unique observations per source
        // Each observation represents a row in the structured file
        let uniqueObservations = 0;
        if (uniqueSources === 1) {
            // This is likely a parquet/structured file - count observations for this field
            const sourceId = fragments[0]?.source_id;
            if (sourceId) {
                // Build query with proper null handling for user_id
                // Note: Queue items convert default UUID to null, but observations may have default UUID
                // So we need to check both null and default UUID
                let obsQuery = db
                    .from("observations")
                    .select("id", { count: "exact", head: true })
                    .eq("source_id", sourceId)
                    .eq("entity_type", options.entity_type);
                // Handle user_id properly: check both default UUID and null
                const defaultUserId = "00000000-0000-0000-0000-000000000000";
                if (options.user_id) {
                    if (options.user_id === defaultUserId) {
                        // Check both default UUID and null (observations might have either)
                        obsQuery = obsQuery.or(`user_id.eq.${defaultUserId},user_id.is.null`);
                    }
                    else {
                        obsQuery = obsQuery.eq("user_id", options.user_id);
                    }
                }
                else {
                    // No user_id provided - check both null and default UUID
                    obsQuery = obsQuery.or(`user_id.is.null,user_id.eq.${defaultUserId}`);
                }
                const { count } = await obsQuery;
                uniqueObservations = count || 0;
            }
        }
        // Require EITHER 2+ sources OR 2+ observations
        // This enables auto-enhancement for structured files (parquet, CSV) while preserving
        // source diversity requirement for single-row file types
        const hasDiversity = uniqueSources >= 2 || uniqueObservations >= 2;
        if (!hasDiversity) {
            return {
                eligible: false,
                confidence: confidenceResult.confidence,
                reasoning: uniqueSources < 2 && uniqueObservations < 2
                    ? "Field appears in only one source and one observation (no diversity)"
                    : uniqueSources < 2
                        ? "Field appears in only one source (no diversity)"
                        : "Field appears in only one observation (no diversity)",
            };
        }
        // Eligible for auto-enhancement!
        const diversityInfo = uniqueSources >= 2 ? `${uniqueSources} sources` : `${uniqueObservations} observations`;
        return {
            eligible: true,
            confidence: confidenceResult.confidence,
            inferred_type: confidenceResult.inferred_type,
            reasoning: `High confidence (${confidenceResult.confidence.toFixed(2)}) with ${diversityInfo}`,
        };
    }
    /**
     * Automatically enhance schema for high-confidence fields
     * Includes race condition handling for concurrent enhancements
     */
    async autoEnhanceSchema(options) {
        // Generate idempotency key to prevent duplicate enhancements
        const idempotencyKey = `auto_enhance_${options.entity_type}_${options.field_name}_${options.user_id || "global"}`;
        // Check if already enhanced (idempotent check)
        const { data: existing } = await db
            .from("schema_recommendations")
            .select("id, status")
            .eq("idempotency_key", idempotencyKey)
            .single();
        if (existing && ["applied", "auto_applied"].includes(existing.status)) {
            logger.error(`[AUTO_ENHANCE] Field ${options.field_name} already enhanced for ${options.entity_type}`);
            return existing;
        }
        // Check if field already exists in schema (registry or code-defined baseline)
        const currentSchema = await this.loadSchemaForAutoEnhance(options.entity_type, options.user_id);
        const fieldExists = currentSchema?.schema_definition.fields[options.field_name];
        // Determine recommendation type based on field existence and converter suggestion
        let recommendationType;
        if (fieldExists && options.converter_suggestion) {
            // Field exists - add converter to existing field
            recommendationType = "add_converters";
            logger.error(`[AUTO_ENHANCE] Field ${options.field_name} exists in schema, adding converter for ${options.entity_type}`);
        }
        else if (fieldExists && !options.converter_suggestion) {
            // Field exists but no converter suggestion - shouldn't enhance
            logger.error(`[AUTO_ENHANCE] Field ${options.field_name} already exists in schema for ${options.entity_type} and no converter needed`);
            return currentSchema;
        }
        else {
            // Field doesn't exist - add new field
            recommendationType = "add_fields";
        }
        try {
            // Handle default user ID: convert to null for foreign key constraint
            // The default UUID '00000000-0000-0000-0000-000000000000' doesn't exist in auth.users
            // So we use NULL instead, which the unique index handles via COALESCE
            const defaultUserId = "00000000-0000-0000-0000-000000000000";
            const userId = options.user_id && options.user_id !== defaultUserId ? options.user_id : null;
            // Create recommendation record
            const recommendationData = {
                entity_type: options.entity_type,
                user_id: userId,
                source: "raw_fragments",
                recommendation_type: recommendationType,
                confidence_score: 0.9, // High confidence for auto-enhancement
                status: "auto_applied",
                applied_at: new Date().toISOString(),
                idempotency_key: idempotencyKey,
                can_rollback: true,
            };
            // Add type-specific fields
            if (recommendationType === "add_fields") {
                recommendationData.fields_to_add = [
                    {
                        field_name: options.field_name,
                        field_type: options.field_type,
                        required: false,
                    },
                ];
            }
            else if (recommendationType === "add_converters") {
                recommendationData.fields_to_add = null; // Null for add_converters recommendations
                recommendationData.converters_to_add = [
                    {
                        field_name: options.field_name,
                        converter: options.converter_suggestion,
                    },
                ];
            }
            const { data: recommendation, error: recError } = await db
                .from("schema_recommendations")
                .insert(recommendationData)
                .select()
                .single();
            if (recError) {
                // Check if it's a unique constraint violation (already exists)
                if (recError.code === "23505") {
                    logger.error(`[AUTO_ENHANCE] Idempotency key collision - enhancement already in progress`);
                    // Return the existing record rather than null so callers get a usable result
                    const { data: existingRec } = await db
                        .from("schema_recommendations")
                        .select("*")
                        .eq("idempotency_key", idempotencyKey)
                        .single();
                    return existingRec ?? null;
                }
                throw recError;
            }
            if (recommendationType === "add_converters") {
                logger.error(`[AUTO_ENHANCE] Auto-enhancing ${options.entity_type}.${options.field_name} by adding converter: ${options.converter_suggestion?.function}`);
            }
            else {
                logger.error(`[AUTO_ENHANCE] Auto-enhancing ${options.entity_type}.${options.field_name} with type ${options.field_type}`);
            }
            // Note: Actual schema update would be done via SchemaRegistryService.updateSchemaIncremental()
            // This would be called by the auto-enhancement processor, not here directly
            // to avoid blocking the storage operation
            return recommendation;
        }
        catch (error) {
            logger.error(`[AUTO_ENHANCE] Failed to auto-enhance schema:`, error.message);
            throw error;
        }
    }
    /**
     * Calculate confidence score for a field based on samples
     * Enhanced with robust type inference logic
     */
    async calculateFieldConfidence(options) {
        // Get all samples for this field
        let confidenceQuery = db
            .from("raw_fragments")
            .select("fragment_value, frequency_count")
            .eq("entity_type", options.entity_type)
            .eq("fragment_key", options.fragment_key);
        // Handle user_id properly: check both the provided user_id and the default UUID
        const defaultUserId = "00000000-0000-0000-0000-000000000000";
        if (options.user_id) {
            if (options.user_id === defaultUserId) {
                // Check both default UUID and null (legacy data might use null)
                confidenceQuery = confidenceQuery.or(`user_id.eq.${defaultUserId},user_id.is.null`);
            }
            else {
                confidenceQuery = confidenceQuery.eq("user_id", options.user_id);
            }
        }
        else {
            // No user_id provided - check both null and default UUID
            confidenceQuery = confidenceQuery.or(`user_id.is.null,user_id.eq.${defaultUserId}`);
        }
        const { data: fragments } = await confidenceQuery;
        if (!fragments || fragments.length === 0) {
            return {
                confidence: 0,
                type_consistency: 0,
                naming_pattern_match: false,
                format_consistency: 0,
            };
        }
        // Collect all sample values
        const samples = fragments.map((f) => f.fragment_value);
        // Multi-pass type detection
        const typeAnalysis = this.analyzeTypes(samples);
        // Check naming patterns
        const namingPatternMatch = this.checkNamingPattern(options.fragment_key);
        // Check format consistency for specific types
        const formatConsistency = this.checkFormatConsistency(samples, typeAnalysis.dominant_type);
        // Calculate overall confidence
        const confidence = this.calculateOverallConfidence({
            type_consistency: typeAnalysis.consistency,
            naming_pattern_match: namingPatternMatch,
            format_consistency: formatConsistency,
            sample_count: samples.length,
        });
        return {
            confidence,
            type_consistency: typeAnalysis.consistency,
            naming_pattern_match: namingPatternMatch,
            format_consistency: formatConsistency,
            inferred_type: typeAnalysis.dominant_type,
        };
    }
    /**
     * Analyze raw_fragments to identify fields that should be promoted
     */
    async analyzeRawFragments(options) {
        const minFrequency = options.min_frequency || 5;
        const minConfidence = options.min_confidence || 0.8;
        // Query raw_fragments grouped by entity_type and fragment_key
        let query = db
            .from("raw_fragments")
            .select("entity_type, fragment_key, fragment_value, frequency_count, user_id");
        if (options.entity_type) {
            query = query.eq("entity_type", options.entity_type);
        }
        // Handle user_id properly: check both the provided user_id and the default UUID
        // Also handle null (for global schemas)
        const defaultUserId = "00000000-0000-0000-0000-000000000000";
        if (options.user_id) {
            if (options.user_id === defaultUserId) {
                // Check both default UUID and null (legacy data might use null)
                query = query.or(`user_id.eq.${defaultUserId},user_id.is.null`);
            }
            else {
                query = query.eq("user_id", options.user_id);
            }
        }
        else {
            // No user_id provided - check both null and default UUID
            query = query.or(`user_id.is.null,user_id.eq.${defaultUserId}`);
        }
        const { data: fragments, error } = await query;
        if (error || !fragments) {
            return [];
        }
        // Group by entity_type and fragment_key
        const grouped = new Map();
        for (const fragment of fragments) {
            if (!fragment.entity_type) {
                // Skip fragments without entity type (shouldn't happen, but defensive)
                continue;
            }
            const key = `${fragment.entity_type}::${fragment.fragment_key}::${fragment.user_id || "global"}`;
            if (!grouped.has(key)) {
                grouped.set(key, []);
            }
            grouped.get(key).push({
                fragment_value: fragment.fragment_value,
                frequency_count: fragment.frequency_count || 1,
            });
        }
        // Analyze each group
        const recommendations = [];
        for (const [key, samples] of grouped.entries()) {
            const [entity_type, fragment_key, user_id_str] = key.split("::");
            const user_id = user_id_str === "global" ? undefined : user_id_str;
            // Calculate total frequency
            const totalFrequency = samples.reduce((sum, s) => sum + s.frequency_count, 0);
            if (totalFrequency < minFrequency) {
                continue;
            }
            // Calculate confidence
            const confidenceResult = await this.calculateFieldConfidence({
                fragment_key,
                entity_type,
                user_id,
            });
            if (confidenceResult.confidence < minConfidence) {
                continue;
            }
            // Create recommendation
            const fieldRec = {
                field_name: fragment_key,
                field_type: confidenceResult.inferred_type || "string",
                required: false,
                frequency: totalFrequency,
                sample_values: samples.map((s) => s.fragment_value).slice(0, 10), // Limit sample values
                confidence: confidenceResult.confidence,
                type_consistency: confidenceResult.type_consistency,
                naming_pattern_match: confidenceResult.naming_pattern_match,
                format_consistency: confidenceResult.format_consistency,
            };
            // Check if recommendation already exists for this entity type
            const existingRec = recommendations.find((r) => r.entity_type === entity_type);
            if (existingRec) {
                existingRec.fields.push(fieldRec);
                existingRec.confidence_score = Math.max(existingRec.confidence_score, confidenceResult.confidence);
            }
            else {
                recommendations.push({
                    entity_type,
                    fields: [fieldRec],
                    source: "raw_fragments",
                    confidence_score: confidenceResult.confidence,
                    reasoning: `Analyzed ${samples.length} fragments with ${totalFrequency} occurrences`,
                    affected_entities_count: samples.length,
                });
            }
        }
        return recommendations;
    }
    /**
     * Generate LLM-based recommendations (opt-in, non-deterministic)
     */
    async generateInferenceRecommendations(options) {
        // Prepare prompt for LLM
        const fragmentsSummary = options.raw_fragments_sample
            .map((f) => `- ${f.fragment_key}: appears ${f.frequency_count} times, sample value: ${JSON.stringify(f.fragment_value).slice(0, 100)}`)
            .join("\n");
        const prompt = `Analyze these fields from raw data fragments for entity type "${options.entity_type}":

${fragmentsSummary}

Based on these fragments, recommend:
1. Field names (use snake_case)
2. Field types (string, number, date, boolean, array, object)
3. Whether each field should be required
4. Confidence level for each recommendation (0-1)

Return your recommendations in JSON format:
{
  "fields": [
    {
      "field_name": "...",
      "field_type": "...",
      "required": false,
      "confidence": 0.9,
      "reasoning": "..."
    }
  ],
  "overall_confidence": 0.85,
  "reasoning": "..."
}`;
        try {
            // Use OpenAI directly for LLM-based recommendations
            const OpenAI = (await import("openai")).default;
            const { config } = await import("../config.js");
            if (!config.openaiApiKey) {
                throw new Error("OpenAI API key not configured");
            }
            const openai = new OpenAI({ apiKey: config.openaiApiKey });
            // Call LLM with prompt
            const completion = await openai.chat.completions.create({
                model: "gpt-4o-mini",
                temperature: 0.3, // Lower temperature for more consistent results
                messages: [
                    {
                        role: "user",
                        content: prompt,
                    },
                ],
            });
            const response = completion.choices[0]?.message?.content || "{}";
            // Parse JSON response
            const parsed = JSON.parse(response);
            // Convert to SchemaRecommendation format
            const fields = parsed.fields.map((f) => ({
                field_name: f.field_name,
                field_type: f.field_type,
                required: f.required || false,
                frequency: options.raw_fragments_sample.find((frag) => frag.fragment_key === f.field_name)
                    ?.frequency_count || 0,
                sample_values: [
                    options.raw_fragments_sample.find((frag) => frag.fragment_key === f.field_name)
                        ?.fragment_value,
                ],
                confidence: f.confidence,
                type_consistency: f.confidence, // Use confidence as type_consistency
            }));
            return {
                entity_type: options.entity_type,
                fields,
                source: "inference",
                confidence_score: parsed.overall_confidence,
                reasoning: parsed.reasoning,
            };
        }
        catch (error) {
            logger.error(`[SCHEMA_INFERENCE] Failed to generate LLM recommendations:`, error.message);
            // Fallback to deterministic analysis if LLM fails
            logger.error(`[SCHEMA_INFERENCE] Falling back to deterministic analysis`);
            const fields = options.raw_fragments_sample.map((f) => ({
                field_name: f.fragment_key,
                field_type: this.inferType(f.fragment_value),
                required: false,
                frequency: f.frequency_count,
                sample_values: [f.fragment_value],
                confidence: 0.6, // Lower confidence for fallback
                type_consistency: 0.8,
            }));
            return {
                entity_type: options.entity_type,
                fields,
                source: "inference",
                confidence_score: 0.6,
                reasoning: "LLM inference failed, using fallback deterministic analysis",
            };
        }
    }
    /**
     * Store recommendation from agent or inference
     */
    async storeRecommendation(recommendation) {
        const { data, error } = await db
            .from("schema_recommendations")
            .insert({
            entity_type: recommendation.entity_type,
            user_id: recommendation.user_id || null,
            source: recommendation.source,
            recommendation_type: "add_fields",
            fields_to_add: recommendation.fields.map((f) => ({
                field_name: f.field_name,
                field_type: f.field_type,
                required: f.required || false,
            })),
            confidence_score: recommendation.confidence_score,
            reasoning: recommendation.reasoning,
            status: "pending",
        })
            .select("id")
            .single();
        if (error) {
            throw new Error(`Failed to store recommendation: ${error.message}`);
        }
        return data.id;
    }
    /**
     * Get pending recommendations for an entity type
     */
    async getRecommendations(options) {
        let query = db.from("schema_recommendations").select("*");
        if (options.entity_type) {
            query = query.eq("entity_type", options.entity_type);
        }
        if (options.user_id !== undefined) {
            query = query.eq("user_id", options.user_id || null);
        }
        if (options.source) {
            query = query.eq("source", options.source);
        }
        if (options.status) {
            query = query.eq("status", options.status);
        }
        const { data, error } = await query.order("created_at", {
            ascending: false,
        });
        if (error) {
            throw new Error(`Failed to get recommendations: ${error.message}`);
        }
        return (data?.map((rec) => ({
            id: rec.id,
            entity_type: rec.entity_type,
            fields: rec.fields_to_add,
            source: rec.source,
            confidence_score: rec.confidence_score,
            reasoning: rec.reasoning,
            status: rec.status,
            affected_entities_count: rec.sample_count,
        })) || []);
    }
    /**
     * Queue auto-enhancement check for deferred processing
     */
    async queueAutoEnhancementCheck(options) {
        try {
            // Handle default user ID: convert to null for foreign key constraint
            // The default UUID '00000000-0000-0000-0000-000000000000' doesn't exist in auth.users
            // So we use NULL instead, which the unique index handles via COALESCE
            const defaultUserId = "00000000-0000-0000-0000-000000000000";
            const userId = options.user_id && options.user_id !== defaultUserId ? options.user_id : null;
            // Check if entry already exists
            let query = db
                .from("auto_enhancement_queue")
                .select("id, status, frequency_count")
                .eq("entity_type", options.entity_type)
                .eq("fragment_key", options.fragment_key);
            if (userId) {
                query = query.eq("user_id", userId);
            }
            else {
                query = query.is("user_id", null);
            }
            const { data: existing, error: checkError } = await query.maybeSingle();
            if (checkError && checkError.code !== "PGRST116") {
                // PGRST116 is "no rows" which is fine
                throw checkError;
            }
            if (existing) {
                // Update existing entry
                const { error: updateError } = await db
                    .from("auto_enhancement_queue")
                    .update({
                    status: "pending", // Reset to pending if it was skipped/failed
                    frequency_count: options.frequency_count || existing.frequency_count,
                })
                    .eq("id", existing.id);
                if (updateError) {
                    throw updateError;
                }
            }
            else {
                // Insert new entry
                const { error: insertError } = await db.from("auto_enhancement_queue").insert({
                    entity_type: options.entity_type,
                    fragment_key: options.fragment_key,
                    user_id: userId,
                    status: "pending",
                    frequency_count: options.frequency_count,
                });
                if (insertError) {
                    throw insertError;
                }
            }
        }
        catch (error) {
            // Don't throw - queuing is best-effort
            logger.error(`[AUTO_ENHANCE] Failed to queue enhancement check:`, error.message);
        }
    }
    // --- Private helper methods ---
    /**
     * Look up the most common observation_source for a given entity_type + user_id.
     * Used to determine if the majority of observations come from trusted sources.
     */
    async lookupDominantObservationSource(entityType, userId) {
        try {
            const defaultUserId = "00000000-0000-0000-0000-000000000000";
            let query = db
                .from("observations")
                .select("observation_source")
                .eq("entity_type", entityType)
                .not("observation_source", "is", null)
                .order("observed_at", { ascending: false })
                .limit(20);
            if (userId) {
                if (userId === defaultUserId) {
                    query = query.or(`user_id.eq.${defaultUserId},user_id.is.null`);
                }
                else {
                    query = query.eq("user_id", userId);
                }
            }
            else {
                query = query.or(`user_id.is.null,user_id.eq.${defaultUserId}`);
            }
            const { data, error } = await query;
            if (error || !data || data.length === 0)
                return null;
            const counts = new Map();
            for (const row of data) {
                const src = row.observation_source;
                if (src)
                    counts.set(src, (counts.get(src) || 0) + 1);
            }
            let dominant = null;
            let maxCount = 0;
            for (const [src, count] of counts) {
                if (count > maxCount) {
                    dominant = src;
                    maxCount = count;
                }
            }
            return dominant;
        }
        catch {
            return null;
        }
    }
    /**
     * Detect if a converter is needed for an existing field with type mismatch
     * Analyzes raw_fragments data to determine appropriate converter based on data patterns
     */
    async detectConverterNeeded(options) {
        const samples = options.fragments.map((f) => f.fragment_value);
        const schemaType = options.schema_field.type;
        // Determine the ACTUAL type (JavaScript type) of the data, not semantic type
        // We need the raw type to determine if conversion is needed
        const typeCounts = new Map();
        for (const sample of samples) {
            let rawType;
            if (sample === null || sample === undefined) {
                rawType = "null";
            }
            else if (typeof sample === "boolean") {
                rawType = "boolean";
            }
            else if (typeof sample === "number") {
                rawType = "number";
            }
            else if (typeof sample === "string") {
                rawType = "string";
            }
            else if (Array.isArray(sample)) {
                rawType = "array";
            }
            else {
                rawType = "object";
            }
            typeCounts.set(rawType, (typeCounts.get(rawType) || 0) + 1);
        }
        // Find dominant raw type (excluding null)
        let actualType = "string";
        let maxCount = 0;
        for (const [type, count] of typeCounts.entries()) {
            if (type === "null")
                continue;
            if (count > maxCount) {
                maxCount = count;
                actualType = type;
            }
        }
        // If raw type matches schema type, no converter needed
        // Exception: "date" schema type accepts both string and Date objects
        if (actualType === schemaType) {
            return null;
        }
        if (schemaType === "date" && actualType === "string") {
            // Strings are valid for date type (ISO dates)
            return null;
        }
        // Detect converter based on type combination
        // Case 1: Schema expects date, data is number (timestamp)
        if (schemaType === "date" && actualType === "number") {
            // Find a numeric sample to determine timestamp format
            const numericSample = samples.find((s) => typeof s === "number");
            if (numericSample) {
                // Detect timestamp format based on value magnitude
                if (numericSample > 9999999999999) {
                    // Nanoseconds (> 9,999,999,999,999 = year 2286 in milliseconds)
                    return {
                        converter: {
                            from: "number",
                            to: "date",
                            function: "timestamp_nanos_to_iso",
                            deterministic: true,
                        },
                        confidence: 0.95,
                        reasoning: "Number values are nanosecond timestamps (>9,999,999,999,999)",
                    };
                }
                else if (numericSample > 9999999999) {
                    // Milliseconds (> 9,999,999,999 = year 2286 in seconds)
                    return {
                        converter: {
                            from: "number",
                            to: "date",
                            function: "timestamp_ms_to_iso",
                            deterministic: true,
                        },
                        confidence: 0.95,
                        reasoning: "Number values are millisecond timestamps (>9,999,999,999)",
                    };
                }
                else {
                    // Seconds
                    return {
                        converter: {
                            from: "number",
                            to: "date",
                            function: "timestamp_s_to_iso",
                            deterministic: true,
                        },
                        confidence: 0.95,
                        reasoning: "Number values are second timestamps",
                    };
                }
            }
        }
        // Case 2: Schema expects number, data is string (numeric string)
        if (schemaType === "number" && actualType === "string") {
            // Verify all strings are numeric
            const allNumeric = samples.every((s) => {
                if (typeof s !== "string")
                    return false;
                return this.isNumericString(s);
            });
            if (allNumeric) {
                return {
                    converter: {
                        from: "string",
                        to: "number",
                        function: "string_to_number",
                        deterministic: true,
                    },
                    confidence: 0.9,
                    reasoning: "String values are numeric and can be converted to numbers",
                };
            }
        }
        // Case 3: Schema expects string, data is number
        if (schemaType === "string" && actualType === "number") {
            return {
                converter: {
                    from: "number",
                    to: "string",
                    function: "number_to_string",
                    deterministic: true,
                },
                confidence: 0.9,
                reasoning: "Number values can be converted to strings",
            };
        }
        // Case 4: Schema expects boolean, data is string (boolean string)
        if (schemaType === "boolean" && actualType === "string") {
            const allBooleanStrings = samples.every((s) => {
                if (typeof s !== "string")
                    return false;
                return this.isBooleanString(s);
            });
            if (allBooleanStrings) {
                return {
                    converter: {
                        from: "string",
                        to: "boolean",
                        function: "string_to_boolean",
                        deterministic: true,
                    },
                    confidence: 0.9,
                    reasoning: "String values are boolean strings (true/false/yes/no)",
                };
            }
        }
        // Case 5: Schema expects string, data is boolean
        if (schemaType === "string" && actualType === "boolean") {
            return {
                converter: {
                    from: "boolean",
                    to: "string",
                    function: "boolean_to_string",
                    deterministic: true,
                },
                confidence: 0.9,
                reasoning: "Boolean values can be converted to strings",
            };
        }
        // No converter detected - return null
        return null;
    }
    /**
     * Check if field matches blacklist patterns
     */
    async checkBlacklist(entityType, fieldName, userId) {
        const { data: blacklist } = await db
            .from("field_blacklist")
            .select("field_pattern")
            .or(`entity_type.is.null,entity_type.eq.${entityType}`)
            .or(`user_id.is.null,user_id.eq.${userId || "00000000-0000-0000-0000-000000000000"}`);
        if (!blacklist)
            return false;
        // Check if field name matches any blacklist pattern
        for (const entry of blacklist) {
            if (this.matchesPattern(fieldName, entry.field_pattern)) {
                return true;
            }
        }
        return false;
    }
    /**
     * Check if field name matches a wildcard pattern
     */
    matchesPattern(fieldName, pattern) {
        // Convert wildcard pattern to regex
        const regexPattern = pattern
            .replace(/[.+?^${}()|[\]\\]/g, "\\$&") // Escape special chars
            .replace(/\*/g, ".*"); // Convert * to .*
        const regex = new RegExp(`^${regexPattern}$`, "i");
        return regex.test(fieldName);
    }
    /**
     * Validate field name
     */
    isValidFieldName(fieldName) {
        // Reject suspicious patterns
        if (fieldName.startsWith("_") || fieldName.endsWith("_"))
            return false;
        if (fieldName.startsWith("__"))
            return false; // Double underscore
        if (fieldName.length > 50)
            return false;
        if (fieldName.length < 2)
            return false;
        if (/[^a-zA-Z0-9_]/.test(fieldName))
            return false; // Only alphanumeric + underscore
        return true;
    }
    /**
     * Analyze types across samples (multi-pass type detection)
     * Null values are excluded from consistency calculation since they're expected for optional fields
     */
    analyzeTypes(samples) {
        const typeCounts = new Map();
        let nonNullCount = 0;
        for (const sample of samples) {
            const type = this.inferType(sample);
            typeCounts.set(type, (typeCounts.get(type) || 0) + 1);
            // Count non-null samples for consistency calculation
            if (type !== "null") {
                nonNullCount++;
            }
        }
        // Find dominant type (excluding null)
        let dominantType = "string";
        let maxCount = 0;
        for (const [type, count] of typeCounts.entries()) {
            if (type === "null")
                continue; // Skip null when finding dominant type
            if (count > maxCount) {
                maxCount = count;
                dominantType = type;
            }
        }
        // Calculate consistency based on non-null samples only
        // If all samples are null, consistency is 0
        // Otherwise, consistency = (dominant type count) / (non-null sample count)
        const consistency = nonNullCount === 0 ? 0 : maxCount / nonNullCount;
        return {
            dominant_type: dominantType,
            consistency,
        };
    }
    /**
     * Infer type from a single value (multi-pass detection)
     */
    inferType(value) {
        if (value === null || value === undefined)
            return "null";
        // Pass 1: Detect obvious types
        if (typeof value === "boolean")
            return "boolean";
        if (Array.isArray(value))
            return "array";
        if (typeof value === "object")
            return "object";
        // Pass 1.5: Number analysis - detect timestamps (BigInt or large numbers that are likely dates)
        if (typeof value === "number") {
            // Check if it's a timestamp (reasonable date range)
            // Timestamps are typically:
            // - Unix timestamp in seconds: 1000000000 to 9999999999 (1970-2099)
            // - Unix timestamp in milliseconds: 1000000000000 to 9999999999999 (1970-2099)
            // - BigInt timestamps (nanoseconds): 1000000000000000 to 9999999999999999999
            if (value > 1000000000 && value < 1e19) {
                // Check if it's in a reasonable timestamp range
                // Convert to milliseconds if needed and check if it's a valid date
                let timestampMs = value;
                if (value < 1000000000000) {
                    // Likely seconds, convert to milliseconds
                    timestampMs = value * 1000;
                }
                else if (value > 9999999999999) {
                    // Likely nanoseconds, convert to milliseconds
                    timestampMs = value / 1000000;
                }
                // Check if it's a reasonable date (between 1970 and 2100)
                const date = new Date(timestampMs);
                if (!isNaN(date.getTime()) && date.getFullYear() >= 1970 && date.getFullYear() <= 2100) {
                    return "date";
                }
            }
            return "number";
        }
        // Pass 2: String analysis - detect dates, emails, UUIDs
        if (typeof value === "string") {
            const str = value.trim();
            // ISO 8601 date detection
            if (this.isISODate(str))
                return "date";
            // Email detection (basic)
            if (this.isEmail(str))
                return "string"; // Keep as string for emails
            // UUID detection
            if (this.isUUID(str))
                return "string"; // Keep as string for UUIDs
            // Boolean string detection
            if (this.isBooleanString(str))
                return "boolean";
            // Numeric string detection - but keep as string for IDs and long numeric strings
            // Only convert to number if it's a short numeric string that's likely a numeric value
            // Long numeric strings (like IDs) should remain as strings
            if (this.isNumericString(str)) {
                // Keep as string if it's longer than 10 digits (likely an ID)
                // or if it starts with leading zeros (definitely an ID)
                if (str.length > 10 || /^0+/.test(str)) {
                    return "string";
                }
                return "number";
            }
        }
        return "string";
    }
    /**
     * Check if string is ISO 8601 date
     */
    isISODate(str) {
        if (str.length < 10)
            return false;
        const date = new Date(str);
        return !isNaN(date.getTime()) && str.includes("-"); // Simple check
    }
    /**
     * Check if string is email
     */
    isEmail(str) {
        return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(str);
    }
    /**
     * Check if string is UUID
     */
    isUUID(str) {
        return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(str);
    }
    /**
     * Check if string is numeric
     */
    isNumericString(str) {
        if (str === "")
            return false;
        const num = parseFloat(str);
        return !isNaN(num) && isFinite(num);
    }
    /**
     * Check if string is boolean
     */
    isBooleanString(str) {
        const lower = str.toLowerCase();
        return (lower === "true" ||
            lower === "false" ||
            lower === "yes" ||
            lower === "no" ||
            lower === "1" ||
            lower === "0");
    }
    /**
     * Check if field name matches common naming patterns
     */
    checkNamingPattern(fieldName) {
        const patterns = [
            /_id$/i, // singular ID
            /_ids$/i, // plural IDs (e.g., project_ids, section_ids)
            /_gid$/i, // global ID (e.g., assignee_gid, asana_source_gid)
            /_gids$/i, // plural global IDs (e.g., followers_gids)
            /_date$/i,
            /_time$/i,
            /_at$/i,
            /_amount$/i,
            /_price$/i,
            /_cost$/i,
            /_name$/i, // singular name
            /_names$/i, // plural names (e.g., follower_names, project_names)
            /_email$/i,
            /_url$/i,
            /_link$/i,
            /_address$/i,
            /_phone$/i,
            /_path$/i, // file paths (e.g., execution_plan_path)
            /_html$/i, // HTML content (e.g., description_html)
            /_workspace$/i, // workspace fields (e.g., asana_workspace)
            /^urgency$/i, // urgency/enum fields (exact match)
            /^recurrence$/i, // recurrence/enum fields (exact match)
            /^is_/i,
            /^has_/i,
            /^can_/i,
            /^should_/i,
        ];
        return patterns.some((pattern) => pattern.test(fieldName));
    }
    /**
     * Check format consistency for specific types
     * Null values are excluded from format consistency calculation since they're expected for optional fields
     */
    checkFormatConsistency(samples, type) {
        // Filter out null values - they don't affect format consistency
        const nonNullSamples = samples.filter((s) => s !== null && s !== undefined);
        if (nonNullSamples.length === 0) {
            // All samples are null - can't assess format consistency
            return 0.9; // Default to high consistency for optional fields
        }
        if (type === "date") {
            // Check if all dates follow same format
            const formats = new Set();
            for (const sample of nonNullSamples) {
                if (typeof sample === "string") {
                    if (this.isISODate(sample))
                        formats.add("iso");
                }
            }
            return formats.size === 1 ? 1.0 : 0.5;
        }
        if (type === "number") {
            // Check if all numbers have similar precision
            // Handle both actual numbers and numeric strings
            const integers = nonNullSamples.filter((s) => {
                if (typeof s === "number") {
                    return Number.isInteger(s);
                }
                if (typeof s === "string") {
                    // Check if it's a numeric string that represents an integer
                    const num = parseFloat(s);
                    return !isNaN(num) && isFinite(num) && Number.isInteger(num);
                }
                return false;
            });
            const consistency = integers.length / nonNullSamples.length;
            return consistency;
        }
        // Default: high consistency for other types
        return 0.9;
    }
    /**
     * Calculate overall confidence score
     */
    calculateOverallConfidence(params) {
        let confidence = 0;
        // Type consistency is most important (50% weight)
        confidence += params.type_consistency * 0.5;
        // Format consistency (25% weight)
        confidence += params.format_consistency * 0.25;
        // Naming pattern match (15% weight)
        confidence += (params.naming_pattern_match ? 1.0 : 0.5) * 0.15;
        // Sample count bonus (10% weight) - more samples = higher confidence
        const sampleScore = Math.min(params.sample_count / 10, 1.0);
        confidence += sampleScore * 0.1;
        return Math.min(confidence, 1.0);
    }
}
// Export singleton instance
export const schemaRecommendationService = new SchemaRecommendationService();
//# sourceMappingURL=schema_recommendation.js.map