// FU-121: Interpretation Service
// Schema validation, entity resolution, and unknown field routing
// Implements idempotence pattern: canonicalization, hash-based IDs, deduplication
import { db } from "../db.js";
import { schemaRegistry, } from "./schema_registry.js";
import { resolveEntity } from "./entity_resolution.js";
import { getCurrentAgentIdentity, getCurrentAttribution } from "./request_context.js";
import { enforceAttributionPolicy } from "./attribution_policy.js";
import { getSchemaDefinition, getRegisteredEntityTypes, resolveEntityTypeFromAlias, refineEntityTypeFromExtractedFields, } from "./schema_definitions.js";
import { observationReducer } from "../reducers/observation_reducer.js";
import { randomUUID } from "crypto";
import { canonicalizeFields, hashCanonicalFields } from "./field_canonicalization.js";
import { generateObservationId, computeCanonicalHash, checkObservationExistsByHash, } from "./observation_identity.js";
import { validateFieldsWithConverters } from "./field_validation.js";
import { prepareEntitySnapshotWithEmbedding, getEntitySnapshotUpsertPayload, } from "./entity_snapshot_embedding.js";
import { logger } from "../utils/logger.js";
import { config } from "../config.js";
import { upsertTimelineEventsForEntitySnapshot } from "./timeline_events.js";
import { emitEntitySnapshotChange, emitObservationCreated, shallowFieldsChanged, } from "../events/substrate_store_emit.js";
export const UNKNOWN_FIELDS_HINT = "Unknown fields were stored in raw_fragments. " +
    "Call update_schema_incremental to promote them to schema fields, " +
    "then set migrate_existing: true to backfill existing data.";
/**
 * Validate extracted data against schema definition with converter support
 * Returns: { validFields, unknownFields, originalValues }
 */
function validateAgainstSchema(data, schema) {
    // Use converter-aware validation
    return validateFieldsWithConverters(data, schema.fields);
}
/**
 * Write buffered raw_fragments for a single entity after entity resolution and
 * observation creation have both succeeded (issue #163: do not write fragments
 * before entity resolution).
 */
async function flushPendingFragments(opts) {
    const { sourceId, interpretationId, userId, entityType, effectiveSchemaVersion, validFields, pendingConvertedFragments, pendingUnknownFragments, } = opts;
    let insertedUnknown = 0;
    let insertedConverted = 0;
    // Write converted-value originals (preserves zero data loss for converter fields).
    for (const { key, value } of pendingConvertedFragments) {
        const { data: existing } = await db
            .from("raw_fragments")
            .select("id, frequency_count")
            .eq("source_id", sourceId)
            .eq("fragment_key", key)
            .eq("user_id", userId)
            .maybeSingle();
        if (existing) {
            await db
                .from("raw_fragments")
                .update({
                fragment_value: value,
                fragment_envelope: {
                    reason: "converted_value_original",
                    entity_type: entityType,
                    schema_version: effectiveSchemaVersion,
                    converted_to: validFields[key],
                },
                frequency_count: (existing.frequency_count || 1) + 1,
                last_seen: new Date().toISOString(),
            })
                .eq("id", existing.id);
        }
        else {
            await db.from("raw_fragments").insert({
                id: randomUUID(),
                source_id: sourceId,
                interpretation_id: interpretationId,
                user_id: userId,
                entity_type: entityType,
                fragment_key: key,
                fragment_value: value,
                fragment_envelope: {
                    reason: "converted_value_original",
                    entity_type: entityType,
                    schema_version: effectiveSchemaVersion,
                    converted_to: validFields[key],
                },
            });
            insertedConverted += 1;
        }
    }
    // Write unknown-field fragments.
    for (const { key, value } of pendingUnknownFragments) {
        const { data: existing } = await db
            .from("raw_fragments")
            .select("id, frequency_count")
            .eq("source_id", sourceId)
            .eq("fragment_key", key)
            .eq("user_id", userId)
            .maybeSingle();
        if (existing) {
            const newFreq = (existing.frequency_count || 1) + 1;
            await db
                .from("raw_fragments")
                .update({
                fragment_value: value,
                fragment_envelope: {
                    reason: "unknown_field",
                    entity_type: entityType,
                    schema_version: effectiveSchemaVersion,
                },
                frequency_count: newFreq,
                last_seen: new Date().toISOString(),
            })
                .eq("id", existing.id);
            try {
                const { schemaRecommendationService } = await import("./schema_recommendation.js");
                await schemaRecommendationService.queueAutoEnhancementCheck({
                    entity_type: entityType,
                    fragment_key: key,
                    user_id: userId,
                    frequency_count: newFreq,
                });
            }
            catch (queueError) {
                logger.warn(`[Interpretation] Failed to queue auto-enhancement for ${entityType}.${key}:`, queueError instanceof Error ? queueError.message : String(queueError));
            }
        }
        else {
            const { error: insertError } = await db.from("raw_fragments").insert({
                id: randomUUID(),
                source_id: sourceId,
                interpretation_id: interpretationId,
                user_id: userId,
                entity_type: entityType,
                fragment_key: key,
                fragment_value: value,
                fragment_envelope: {
                    reason: "unknown_field",
                    entity_type: entityType,
                    schema_version: effectiveSchemaVersion,
                },
            });
            if (insertError?.code === "23505") {
                // Race condition: another process inserted the same fragment concurrently.
                const { data: retryExisting } = await db
                    .from("raw_fragments")
                    .select("id, frequency_count")
                    .eq("source_id", sourceId)
                    .eq("fragment_key", key)
                    .eq("user_id", userId)
                    .maybeSingle();
                if (retryExisting) {
                    const retryFreq = (retryExisting.frequency_count || 1) + 1;
                    await db
                        .from("raw_fragments")
                        .update({
                        fragment_value: value,
                        fragment_envelope: {
                            reason: "unknown_field",
                            entity_type: entityType,
                            schema_version: effectiveSchemaVersion,
                        },
                        frequency_count: retryFreq,
                        last_seen: new Date().toISOString(),
                    })
                        .eq("id", retryExisting.id);
                    try {
                        const { schemaRecommendationService } = await import("./schema_recommendation.js");
                        await schemaRecommendationService.queueAutoEnhancementCheck({
                            entity_type: entityType,
                            fragment_key: key,
                            user_id: userId,
                            frequency_count: retryFreq,
                        });
                    }
                    catch (queueError) {
                        logger.warn(`[Interpretation] Failed to queue auto-enhancement for ${entityType}.${key}:`, queueError instanceof Error ? queueError.message : String(queueError));
                    }
                }
            }
            else if (!insertError) {
                insertedUnknown += 1;
                try {
                    const { schemaRecommendationService } = await import("./schema_recommendation.js");
                    await schemaRecommendationService.queueAutoEnhancementCheck({
                        entity_type: entityType,
                        fragment_key: key,
                        user_id: userId,
                        frequency_count: 1,
                    });
                }
                catch (queueError) {
                    logger.warn(`[Interpretation] Failed to queue auto-enhancement for ${entityType}.${key}:`, queueError instanceof Error ? queueError.message : String(queueError));
                }
            }
        }
    }
    return { insertedUnknown, insertedConverted };
}
/**
 * Run interpretation on extracted data
 * Creates interpretation run, validates fields, resolves entities, creates observations
 */
export async function runInterpretation(options) {
    enforceAttributionPolicy("interpretations", getCurrentAgentIdentity());
    const { userId, sourceId, extractedData, config } = options;
    // Validate that the source exists before creating an interpretation
    const { data: sourceCheck, error: sourceCheckError } = await db
        .from("sources")
        .select("id")
        .eq("id", sourceId)
        .single();
    if (sourceCheckError || !sourceCheck) {
        throw new Error(`Source not found: ${sourceId}`);
    }
    // Create interpretation
    const interpretationAttribution = getCurrentAttribution();
    const { data: run, error: runError } = await db
        .from("interpretations")
        .insert({
        user_id: userId,
        source_id: sourceId,
        interpretation_config: config,
        status: "running",
        started_at: new Date().toISOString(),
        ...(Object.keys(interpretationAttribution).length > 0
            ? { provenance: interpretationAttribution }
            : {}),
    })
        .select()
        .single();
    if (runError) {
        throw new Error(`Failed to create interpretation run: ${runError.message}`);
    }
    const interpretationId = run.id;
    let observationsCreated = 0;
    let unknownFieldsCount = 0;
    const unknownFieldNamesSet = new Set();
    const entities = [];
    const interpretationInsertedObservationIds = new Set();
    const refinementDebug = [];
    const noSchemaEntityTypes = [];
    try {
        // Build refinement candidates: DB schemas (user + global) + code-only types so dynamic schemas participate
        const dbSchemas = await schemaRegistry.listActiveSchemas(userId).catch(() => []);
        const dbTypes = new Set(dbSchemas.map((e) => e.entity_type));
        const refinementCandidates = [
            ...dbSchemas.map((e) => ({
                entity_type: e.entity_type,
                schema_definition: e.schema_definition,
            })),
            ...getRegisteredEntityTypes()
                .filter((t) => !dbTypes.has(t))
                .map((t) => getSchemaDefinition(t))
                .filter(Boolean)
                .map((s) => ({ entity_type: s.entity_type, schema_definition: s.schema_definition })),
        ];
        // Process each extracted entity
        for (const entityData of extractedData) {
            // Support both 'entity_type' and 'type' fields for entity type identification
            let entityType = entityData.entity_type || entityData.type || "generic";
            // DB-registered schemas take priority over code-defined aliases so a
            // user-registered type like `organization` is not silently remapped to
            // the built-in `company` alias.
            if (!dbTypes.has(entityType) && !getSchemaDefinition(entityType)) {
                const resolvedType = resolveEntityTypeFromAlias(entityType);
                if (resolvedType)
                    entityType = resolvedType;
            }
            // Refine type by field fit (considers dynamic + code schemas when another schema fits better)
            const extractedFieldKeys = Object.keys(entityData).filter((k) => k !== "entity_type" && k !== "type");
            const typeBeforeRefinement = entityType;
            entityType = refineEntityTypeFromExtractedFields(entityType, extractedFieldKeys, refinementCandidates);
            refinementDebug.push({
                extracted_keys: extractedFieldKeys,
                type_before: typeBeforeRefinement,
                type_after: entityType,
            });
            logger.info("[Interpretation] type refinement: extracted_keys=%s type_before=%s type_after=%s", JSON.stringify(extractedFieldKeys), typeBeforeRefinement, entityType, typeBeforeRefinement !== entityType
                ? { refined: `${typeBeforeRefinement} → ${entityType}` }
                : {});
            let currentEntityData = entityData;
            const codeSchema = getSchemaDefinition(entityType);
            // Load active entity schema (user-scoped first, then global) from database
            // NOTE: Schemas should be initialized in the database via `npm run schema:init`
            let schema = await schemaRegistry.loadActiveSchema(entityType, userId);
            // Fallback to code-defined entity schemas if database schema not found
            if (!schema) {
                if (process.env.NEOTOMA_ENV !== "production") {
                    console.warn(`[Schema Fallback] No database entity schema found for entity type "${entityType}". ` +
                        `Using code fallback. Run 'npm run schema:init' to register schemas in the database.`);
                }
                if (codeSchema &&
                    codeSchema.entity_type &&
                    codeSchema.schema_definition &&
                    codeSchema.reducer_config) {
                    schema = {
                        id: "",
                        entity_type: codeSchema.entity_type,
                        schema_version: codeSchema.schema_version || "1.0",
                        schema_definition: codeSchema.schema_definition,
                        reducer_config: codeSchema.reducer_config,
                        active: true,
                        created_at: new Date().toISOString(),
                    };
                }
            }
            // When no known schema at all (neither DB nor code): optionally create a user-scoped schema from extracted fields
            let schemaCreationAttempted = false;
            if (!schema && !codeSchema && config.create_schema_for_unknown !== false) {
                schemaCreationAttempted = true;
                schema = await schemaRegistry.ensureSchemaForExtractedEntity(entityType, entityData, userId, { create_if_missing: true });
                if (schema) {
                    entityType = schema.entity_type;
                    currentEntityData = { ...entityData, entity_type: entityType };
                }
            }
            const effectiveSchemaDefinition = {
                fields: {
                    ...(schema?.schema_definition?.fields ?? {}),
                    ...(codeSchema?.schema_definition?.fields ?? {}),
                },
            };
            const effectiveSchemaVersion = schema?.schema_version ?? codeSchema?.schema_version ?? "1.0";
            if (Object.keys(effectiveSchemaDefinition.fields).length === 0) {
                // No schema found and auto-registration failed (or was disabled): route all fields to a
                // raw_fragment so the payload is preserved. Use a distinct reason so callers can tell
                // whether creation was attempted ("no_schema_registration_failed") or never tried
                // ("no_schema" — creation disabled via config.create_schema_for_unknown=false).
                const noSchemaReason = schemaCreationAttempted
                    ? "no_schema_registration_failed"
                    : "no_schema";
                logger.warn(`[Interpretation] No schema for entity_type "${entityType}" (resolved from extracted data); ` +
                    `reason=${noSchemaReason}. Register a schema via register_schema to enable observations.`);
                const fragmentId = randomUUID();
                await db.from("raw_fragments").insert({
                    id: fragmentId,
                    source_id: sourceId,
                    interpretation_id: interpretationId,
                    user_id: userId,
                    entity_type: entityType,
                    fragment_key: "full_entity",
                    fragment_value: currentEntityData,
                    fragment_envelope: {
                        reason: noSchemaReason,
                        entity_type: entityType,
                    },
                });
                unknownFieldsCount += Object.keys(currentEntityData).length;
                // Track distinct no-schema entity types so callers can discover and fix them.
                if (!noSchemaEntityTypes.includes(entityType)) {
                    noSchemaEntityTypes.push(entityType);
                }
                for (const k of Object.keys(currentEntityData)) {
                    unknownFieldNamesSet.add(k);
                }
                continue;
            }
            // Exclude entity_type and type from field validation (they're metadata, not schema fields)
            const fieldsToValidate = { ...currentEntityData };
            delete fieldsToValidate.entity_type;
            delete fieldsToValidate.type;
            // Extract an explicit canonical_name before schema validation.
            // validateFieldsWithConverters routes it to unknownFields when the
            // schema's fields dict doesn't declare it, causing the resolver to fall
            // back to heuristic derivation (e.g. from `title`) instead of honoring
            // the caller's intent. We hoist it here so it can be passed to the
            // resolver regardless of schema declaration.
            const explicitCanonicalName = typeof fieldsToValidate.canonical_name === "string" &&
                fieldsToValidate.canonical_name.trim() !== ""
                ? fieldsToValidate.canonical_name
                : undefined;
            // Validate fields against schema (with converter support)
            const { validFields, unknownFields, originalValues } = validateAgainstSchema(fieldsToValidate, effectiveSchemaDefinition);
            // When canonical_name was explicitly supplied but is not a declared schema
            // field, remove it from unknownFields (so it doesn't inflate
            // unknown_fields_count or create a raw_fragment) — it is an identity
            // hint, not a data field the schema doesn't know about.
            if (explicitCanonicalName && !effectiveSchemaDefinition.fields["canonical_name"]) {
                delete unknownFields["canonical_name"];
            }
            // Buffer fragment writes: collect all raw_fragments for this entity so they
            // can be written AFTER entity resolution succeeds. If resolution fails the
            // fragments are discarded and no orphaned rows are created (issue #163).
            const pendingConvertedFragments = [];
            for (const [key, value] of Object.entries(originalValues)) {
                if (value === null || value === undefined)
                    continue;
                pendingConvertedFragments.push({ key, value });
            }
            const pendingUnknownFragments = [];
            for (const [key, value] of Object.entries(unknownFields)) {
                if (value === null || value === undefined)
                    continue;
                pendingUnknownFragments.push({ key, value });
                unknownFieldNamesSet.add(key);
            }
            const validFieldKeys = Object.keys(validFields).filter((key) => key !== "schema_version");
            if (validFieldKeys.length === 0) {
                logger.warn(`[Interpretation] Skipping observation creation for ${entityType}: no schema-valid fields after validation`);
                continue;
            }
            // Canonicalize valid fields for idempotence
            const canonicalFields = canonicalizeFields(validFields, effectiveSchemaDefinition);
            const canonicalHash = computeCanonicalHash(canonicalFields);
            // Build resolver fields: inject explicit canonical_name so the resolver
            // honors the caller's intent when the schema doesn't declare it as a
            // schema field (identity_opt_out schemas are the common case here).
            // The injected key is only used for derivation — canonicalFields (and
            // thus the stored observation) are unmodified.
            const fieldsForResolver = explicitCanonicalName && !effectiveSchemaDefinition.fields["canonical_name"]
                ? { canonical_name: explicitCanonicalName, ...canonicalFields }
                : canonicalFields;
            // Resolve entity (user-scoped). Interpretation runs must not fail the
            // whole pipeline when derivation can't settle — skip this observation
            // and warn so the operator can declare canonical_name_fields.
            let entityId;
            try {
                entityId = await resolveEntity({
                    entityType,
                    fields: fieldsForResolver,
                    userId,
                });
            }
            catch (err) {
                const { CanonicalNameUnresolvedError } = await import("./entity_resolution.js");
                if (err instanceof CanonicalNameUnresolvedError) {
                    logger.warn(`[Interpretation] Skipping observation for ${entityType}: ${err.message}`);
                    continue;
                }
                throw err;
            }
            // Generate deterministic observation ID
            const observationId = generateObservationId(sourceId, interpretationId, entityId, canonicalFields);
            // Check for duplicate observation (idempotence check)
            const existingObservation = await checkObservationExistsByHash(sourceId, interpretationId, entityId, canonicalHash, userId);
            if (existingObservation) {
                // Observation already exists - don't create duplicate.
                // Still flush buffered fragments so they are kept current on replay.
                const flushed = await flushPendingFragments({
                    sourceId,
                    interpretationId,
                    userId,
                    entityType,
                    effectiveSchemaVersion,
                    validFields,
                    pendingConvertedFragments,
                    pendingUnknownFragments,
                });
                // Count only newly inserted fragments (not freq-bumped existing rows),
                // otherwise idempotent replays of a dedup'd observation inflate
                // `unknown_fields_count` on the interpretations row each time. See
                // PR #224 review.
                unknownFieldsCount += flushed.insertedUnknown;
                entities.push({
                    entityId,
                    entityType,
                    observationId: existingObservation.id,
                });
                continue;
            }
            // Create observation with canonical fields and hash
            // Handle schema cache issues: if canonical_hash column not in cache, retry without it
            const observationData = {
                id: observationId,
                entity_id: entityId,
                entity_type: entityType,
                schema_version: effectiveSchemaVersion,
                source_id: sourceId,
                interpretation_id: interpretationId,
                observed_at: new Date().toISOString(),
                specificity_score: 0.5,
                source_priority: 0, // AI interpretation has priority 0
                fields: canonicalFields, // Store canonical fields, not raw
                canonical_hash: canonicalHash, // Store hash for deduplication
                user_id: userId,
            };
            let { error: obsError } = await db.from("observations").insert(observationData);
            // If error is about canonical_hash not being in schema cache, retry without it
            if (obsError &&
                (obsError.message?.includes("canonical_hash") ||
                    obsError.code === "PGRST205" ||
                    obsError.message?.includes("schema cache"))) {
                console.warn(`[WARN] canonical_hash column not in schema cache, inserting without it. ` +
                    `Migration may need to be applied or schema cache refreshed.`);
                // Retry without canonical_hash
                // eslint-disable-next-line @typescript-eslint/no-unused-vars
                const { canonical_hash: _canonical_hash, ...observationDataWithoutHash } = observationData;
                ({ error: obsError } = await db.from("observations").insert(observationDataWithoutHash));
            }
            if (obsError) {
                throw new Error(`Failed to create observation: ${obsError.message}`);
            }
            observationsCreated++;
            interpretationInsertedObservationIds.add(observationId);
            entities.push({
                entityId,
                entityType,
                observationId,
            });
            // Flush buffered raw_fragments now that entity resolution and observation
            // creation have both succeeded (issue #163).
            const flushed = await flushPendingFragments({
                sourceId,
                interpretationId,
                userId,
                entityType,
                effectiveSchemaVersion,
                validFields,
                pendingConvertedFragments,
                pendingUnknownFragments,
            });
            // Count only newly inserted fragments (see PR #224 review).
            unknownFieldsCount += flushed.insertedUnknown;
        }
        // Compute snapshots for all entities
        for (const entity of entities) {
            try {
                const { data: priorSnapRow } = await db
                    .from("entity_snapshots")
                    .select("snapshot")
                    .eq("entity_id", entity.entityId)
                    .eq("user_id", userId)
                    .maybeSingle();
                const priorSnapshot = priorSnapRow?.snapshot ?? {};
                // Get all observations for entity
                const { data: allObservations, error: fetchError } = await db
                    .from("observations")
                    .select("*")
                    .eq("entity_id", entity.entityId)
                    .order("observed_at", { ascending: false });
                if (fetchError) {
                    console.error(`Failed to fetch observations for entity ${entity.entityId}:`, fetchError.message);
                    continue;
                }
                if (allObservations && allObservations.length > 0) {
                    // Compute snapshot
                    const snapshot = await observationReducer.computeSnapshot(entity.entityId, allObservations);
                    if (!snapshot) {
                        console.error(`Failed to compute snapshot for entity ${entity.entityId}: computeSnapshot returned null`);
                        continue;
                    }
                    const rowWithEmbedding = await prepareEntitySnapshotWithEmbedding({
                        entity_id: snapshot.entity_id,
                        entity_type: snapshot.entity_type,
                        schema_version: snapshot.schema_version,
                        snapshot: snapshot.snapshot,
                        computed_at: snapshot.computed_at,
                        observation_count: snapshot.observation_count,
                        last_observation_at: snapshot.last_observation_at,
                        provenance: snapshot.provenance,
                        user_id: snapshot.user_id,
                    });
                    const toUpsert = getEntitySnapshotUpsertPayload(rowWithEmbedding);
                    await db.from("entity_snapshots").upsert(toUpsert, {
                        onConflict: "entity_id",
                    });
                    const sameTypeInBatch = entities.filter((e) => e.entityType === entity.entityType).length;
                    let timelineSchema = null;
                    try {
                        const entry = await schemaRegistry.loadActiveSchema(snapshot.entity_type, snapshot.user_id || userId);
                        timelineSchema = entry?.schema_definition ?? null;
                    }
                    catch {
                        timelineSchema = null;
                    }
                    await upsertTimelineEventsForEntitySnapshot({
                        entityType: snapshot.entity_type,
                        entityId: snapshot.entity_id,
                        sourceId,
                        userId: snapshot.user_id || userId,
                        snapshot: snapshot.snapshot || {},
                        sameTypeInSourceBatch: sameTypeInBatch,
                        schema: timelineSchema,
                    });
                    const newSnapshot = snapshot.snapshot ?? {};
                    const emitTs = snapshot.computed_at || new Date().toISOString();
                    if (interpretationInsertedObservationIds.has(entity.observationId)) {
                        emitObservationCreated({
                            user_id: userId,
                            entity_id: entity.entityId,
                            entity_type: entity.entityType,
                            observation_id: entity.observationId,
                            timestamp: emitTs,
                            source_id: sourceId,
                            observation_source: "llm_summary",
                        });
                        const isNewEntity = snapshot.observation_count === 1;
                        emitEntitySnapshotChange({
                            user_id: userId,
                            entity_id: entity.entityId,
                            entity_type: entity.entityType,
                            event_type: isNewEntity ? "entity.created" : "entity.updated",
                            timestamp: emitTs,
                            observation_id: entity.observationId,
                            fields_changed: isNewEntity
                                ? undefined
                                : shallowFieldsChanged(priorSnapshot, newSnapshot),
                            source_id: sourceId,
                            observation_source: "llm_summary",
                        });
                    }
                }
            }
            catch (error) {
                console.error(`Failed to compute snapshot for entity ${entity.entityId}:`, error instanceof Error ? error.message : String(error));
                // Continue with other entities
            }
        }
        // Mark interpretation as completed
        await db
            .from("interpretations")
            .update({
            status: "completed",
            completed_at: new Date().toISOString(),
            observations_created: observationsCreated,
            unknown_fields_count: unknownFieldsCount,
        })
            .eq("id", interpretationId);
        return {
            interpretationId,
            observationsCreated,
            unknownFieldsCount,
            unknownFieldNames: Array.from(unknownFieldNamesSet).sort(),
            hint: unknownFieldsCount > 0 ? UNKNOWN_FIELDS_HINT : undefined,
            entities,
            refinement_debug: refinementDebug.length > 0 ? refinementDebug : undefined,
            noSchemaEntityTypes: noSchemaEntityTypes.length > 0 ? noSchemaEntityTypes : undefined,
        };
    }
    catch (error) {
        // Mark interpretation as failed
        await db
            .from("interpretations")
            .update({
            status: "failed",
            completed_at: new Date().toISOString(),
            error_message: error instanceof Error ? error.message : String(error),
        })
            .eq("id", interpretationId);
        throw error;
    }
}
/**
 * Run interpretation with fixed-point guarantee
 * Retries interpretation until canonical hash stabilizes
 *
 * Per idempotence directive:
 * - Accept output only when canonical hash stabilizes
 * - hash(n) == hash(n-1) means convergence
 * - Idempotence achieved post-generation
 */
export async function runInterpretationWithFixedPoint(options, maxIterations = 5) {
    let lastHash = null;
    let iteration = 0;
    let result = null;
    while (iteration < maxIterations) {
        // Run interpretation
        result = await runInterpretation(options);
        // Compute hash of all canonical fields from result
        let allCanonicalFields = {};
        for (const entity of result.entities) {
            // Get observation to access canonical fields
            const { data: observation } = await db
                .from("observations")
                .select("fields, canonical_hash")
                .eq("id", entity.observationId)
                .single();
            if (observation?.fields) {
                allCanonicalFields = { ...allCanonicalFields, ...observation.fields };
            }
        }
        const currentHash = hashCanonicalFields(allCanonicalFields);
        // Check for fixed point
        if (lastHash === currentHash) {
            // Fixed point reached - same canonical output as previous iteration
            return {
                ...result,
                fixedPointReached: true,
                convergenceIterations: iteration + 1,
            };
        }
        lastHash = currentHash;
        iteration++;
    }
    // Fallback: return last result even if not converged
    // Log warning but don't fail (graceful degradation)
    console.warn(`Fixed-point convergence not reached after ${maxIterations} iterations for source ${options.sourceId}`);
    return {
        ...result,
        fixedPointReached: false,
        convergenceIterations: maxIterations,
    };
}
/**
 * Check interpretation quota (simple hard-coded limit for v0.2.0).
 * In local mode (user's own OPENAI key), no cap is enforced.
 */
export async function checkInterpretationQuota(userId, limit = 100) {
    if (config.storageBackend === "local") {
        return { allowed: true, current: 0, limit: 0 };
    }
    // Count interpretation runs in the last 30 days
    const thirtyDaysAgo = new Date();
    thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
    const { count, error } = await db
        .from("interpretations")
        .select("*", { count: "exact", head: true })
        .eq("user_id", userId)
        .gte("created_at", thirtyDaysAgo.toISOString());
    if (error) {
        console.warn(`Failed to check quota: ${error.message}`);
        // Allow on error (fail open)
        return { allowed: true, current: 0, limit };
    }
    const current = count || 0;
    return {
        allowed: current < limit,
        current,
        limit,
    };
}
/**
 * Create relationship observations from relationship data
 */
export async function createRelationshipObservations(relationships, sourceId, interpretationId, userId, sourcePriority = 0) {
    enforceAttributionPolicy("relationships", getCurrentAgentIdentity());
    const { relationshipReducer } = await import("../reducers/relationship_reducer.js");
    const { createHash } = await import("crypto");
    // Get relationship schema for canonicalization
    const relationshipSchema = getSchemaDefinition("relationship");
    if (!relationshipSchema) {
        throw new Error("Relationship schema not found");
    }
    let relationshipsCreated = 0;
    const uniqueRelationshipKeys = new Set();
    for (const rel of relationships) {
        try {
            // Generate relationship key
            const relationshipKey = `${rel.relationship_type}:${rel.source_entity_id}:${rel.target_entity_id}`;
            // Relationship metadata is freeform (varies by relationship type) — do not filter
            // through canonicalizeFields which would strip fields not in the fixed relationship schema.
            // Sort keys for a stable canonical hash used for deduplication.
            const rawMetadata = rel.metadata || {};
            const canonicalMetadata = Object.fromEntries(Object.entries(rawMetadata).sort(([a], [b]) => a.localeCompare(b)));
            const canonicalHash = hashCanonicalFields(canonicalMetadata);
            // Generate deterministic observation ID (UUID format)
            const hash = createHash("sha256")
                .update(JSON.stringify({
                source_id: sourceId,
                interpretation_id: interpretationId,
                relationship_key: relationshipKey,
                canonical_hash: canonicalHash,
            }))
                .digest("hex");
            // Convert hash to UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
            const observationId = [
                hash.substring(0, 8),
                hash.substring(8, 12),
                hash.substring(12, 16),
                hash.substring(16, 20),
                hash.substring(20, 32),
            ].join("-");
            // Check for duplicate observation (idempotence)
            const { data: existingObs } = await db
                .from("relationship_observations")
                .select("id")
                .eq("source_id", sourceId)
                .eq("relationship_key", relationshipKey)
                .eq("canonical_hash", canonicalHash)
                .eq("user_id", userId)
                .maybeSingle();
            if (existingObs) {
                // Observation already exists - skip
                uniqueRelationshipKeys.add(relationshipKey);
                continue;
            }
            // Create relationship observation
            // Handle schema cache issues: if canonical_hash column not in cache, retry without it
            const relAttribution = getCurrentAttribution();
            const relationshipObsData = {
                id: observationId,
                relationship_key: relationshipKey,
                relationship_type: rel.relationship_type,
                source_entity_id: rel.source_entity_id,
                target_entity_id: rel.target_entity_id,
                source_id: sourceId,
                interpretation_id: interpretationId,
                observed_at: new Date().toISOString(),
                specificity_score: 0.8,
                source_priority: sourcePriority,
                metadata: canonicalMetadata,
                canonical_hash: canonicalHash,
                user_id: userId,
                ...(Object.keys(relAttribution).length > 0 ? { provenance: relAttribution } : {}),
            };
            let { error: obsError } = await db
                .from("relationship_observations")
                .insert(relationshipObsData);
            // If error is about canonical_hash not being in schema cache, retry without it
            if (obsError &&
                (obsError.message?.includes("canonical_hash") ||
                    obsError.code === "PGRST205" ||
                    obsError.message?.includes("schema cache"))) {
                console.warn(`[WARN] canonical_hash column not in schema cache for relationship_observations, inserting without it. ` +
                    `Migration may need to be applied or schema cache refreshed.`);
                // Retry without canonical_hash
                // eslint-disable-next-line @typescript-eslint/no-unused-vars
                const { canonical_hash: _canonical_hash, ...relationshipObsDataWithoutHash } = relationshipObsData;
                ({ error: obsError } = await db
                    .from("relationship_observations")
                    .insert(relationshipObsDataWithoutHash));
            }
            if (obsError) {
                const rawDetails = typeof obsError === "object" && obsError && "details" in obsError
                    ? obsError.details
                    : undefined;
                const detailsStr = rawDetails != null
                    ? typeof rawDetails === "string"
                        ? rawDetails
                        : JSON.stringify(rawDetails)
                    : "";
                const errMsg = detailsStr ? `${obsError.message} (${detailsStr})` : obsError.message;
                const fullMsg = `Failed to create relationship observation for ${relationshipKey}: ${errMsg}`;
                console.error(fullMsg);
                throw new Error(fullMsg);
            }
            relationshipsCreated++;
            uniqueRelationshipKeys.add(relationshipKey);
        }
        catch (error) {
            console.error(`Failed to process relationship observation:`, error instanceof Error ? error.message : String(error));
        }
    }
    // Compute snapshots for all unique relationships
    for (const relationshipKey of uniqueRelationshipKeys) {
        try {
            // Get all observations for this relationship
            const { data: allObservations, error: fetchError } = await db
                .from("relationship_observations")
                .select("*")
                .eq("relationship_key", relationshipKey)
                .eq("user_id", userId)
                .order("observed_at", { ascending: false });
            if (fetchError) {
                console.error(`Failed to fetch observations for relationship ${relationshipKey}:`, fetchError.message);
                continue;
            }
            if (allObservations && allObservations.length > 0) {
                // Compute snapshot
                const snapshot = await relationshipReducer.computeSnapshot(relationshipKey, allObservations);
                // Save snapshot
                await db.from("relationship_snapshots").upsert({
                    relationship_key: snapshot.relationship_key,
                    relationship_type: snapshot.relationship_type,
                    source_entity_id: snapshot.source_entity_id,
                    target_entity_id: snapshot.target_entity_id,
                    schema_version: snapshot.schema_version,
                    snapshot: snapshot.snapshot,
                    computed_at: snapshot.computed_at,
                    observation_count: snapshot.observation_count,
                    last_observation_at: snapshot.last_observation_at,
                    provenance: snapshot.provenance,
                    user_id: snapshot.user_id,
                }, {
                    onConflict: "relationship_key",
                });
            }
        }
        catch (error) {
            console.error(`Failed to compute snapshot for relationship ${relationshipKey}:`, error instanceof Error ? error.message : String(error));
        }
    }
    return relationshipsCreated;
}
//# sourceMappingURL=interpretation.js.map