import { resolve } from 'node:path';
import { MemoryCache } from '../src/cache.js';
import { KnowledgeCompletenessEvaluator, skippedKnowledgeCompletenessReport, } from '../src/evaluation/knowledge-completeness-evaluator.js';
import { loadKnowledgeCompletenessFixture } from '../src/evaluation/fixture-loader.js';
import { IngestionService } from '../src/ingest/service.js';
import { HashModelProvider } from '../src/model/provider.js';
import { RetrievalService } from '../src/retrieval/service.js';
import { MemoryKnowledgeStore } from '../src/storage/memory-store.js';
const defaultConfig = {
    env: 'eval',
    http: {
        port: 3027,
        host: '127.0.0.1',
        requireApiKeyForNonLoopback: false,
        maxRequestBytes: 10 * 1024 * 1024,
    },
    storage: {
        databaseUrl: '',
        redisUrl: '',
        store: 'memory',
        cache: 'memory',
        autoMigrate: false,
    },
    model: {
        provider: 'hash',
        openAiTimeoutMs: 30_000,
        embeddingDimensions: 1536,
        openAiEmbeddingModel: 'text-embedding-3-small',
        llmCriticEnabled: false,
    },
    context: {
        cacheTtlSeconds: 0,
    },
    ingest: {
        maxContentBytes: 2 * 1024 * 1024,
    },
    backup: {
        dir: '.tuberosa/test-backups',
        exportBaseDir: '.tuberosa/test-exports',
        importBaseDir: '.tuberosa/test-imports',
        intervalSeconds: 0,
        startupDelaySeconds: 0,
        retentionCount: 24,
        retentionMaxAgeDays: 30,
        writeThrough: false,
        writeThroughThrottleSeconds: 600,
    },
    mirror: {
        debounceMs: 500,
    },
    errorLog: {
        dir: '.tuberosa/test-error-logs',
        maxBytes: 256 * 1024,
        autoCapture: true,
        captureClientErrors: false,
    },
    worktree: {
        enabled: true,
        maxFiles: 50,
        maxMtimeAgeHours: 72,
    },
    archival: {
        enabled: false,
        intervalHours: 24,
    },
    graphInference: {
        enabled: false,
    },
    atlas: {},
    userStyle: {},
    persistReplay: false,
};
async function main() {
    const options = parseArgs(process.argv.slice(2));
    if (options.help) {
        console.log(usage());
        return;
    }
    const fixture = await loadKnowledgeCompletenessFixture(resolve(options.fixturePath));
    if (options.mode === 'live') {
        const available = await liveApiAvailable(options.apiBase);
        if (!available) {
            const report = skippedKnowledgeCompletenessReport(fixture, 'live', `API server not reachable at ${options.apiBase}`);
            printOrJson(report, options.json);
            return;
        }
        const evaluator = new KnowledgeCompletenessEvaluator(new HttpContextSearcher(options.apiBase));
        const report = await evaluator.run(fixture, { mode: 'live' });
        printOrJson(report, options.json);
        if (shouldFail(report, options)) {
            process.exitCode = 1;
        }
        return;
    }
    const store = new MemoryKnowledgeStore();
    const cache = new MemoryCache();
    const models = new HashModelProvider(defaultConfig.model.embeddingDimensions);
    const ingestion = new IngestionService(store, models);
    const retrieval = new RetrievalService(store, cache, models, defaultConfig);
    const evaluator = new KnowledgeCompletenessEvaluator(retrieval, ingestion);
    try {
        const report = await evaluator.run(fixture, { mode: 'fixture' });
        printOrJson(report, options.json);
        if (shouldFail(report, options)) {
            process.exitCode = 1;
        }
    }
    finally {
        await Promise.allSettled([store.close(), cache.close()]);
    }
}
class HttpContextSearcher {
    apiBase;
    constructor(apiBase) {
        this.apiBase = apiBase;
    }
    async searchContext(input) {
        const response = await fetch(`${this.apiBase}/context/search`, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({ ...input, debug: true }),
            signal: AbortSignal.timeout(8_000),
        });
        if (!response.ok) {
            throw new Error(`POST /context/search failed with HTTP ${response.status}`);
        }
        return response.json();
    }
}
async function liveApiAvailable(apiBase) {
    try {
        const response = await fetch(`${apiBase}/health`, { signal: AbortSignal.timeout(2_000) });
        return response.ok;
    }
    catch {
        return false;
    }
}
function parseArgs(args) {
    const options = {
        fixturePath: 'eval/knowledge-completeness-fixtures.json',
        mode: 'fixture',
        apiBase: 'http://localhost:3027',
        json: false,
        help: false,
    };
    for (let index = 0; index < args.length; index += 1) {
        const arg = args[index];
        if (arg === '--') {
            continue;
        }
        if (arg === '--help' || arg === '-h') {
            options.help = true;
            continue;
        }
        if (arg === '--json') {
            options.json = true;
            continue;
        }
        if (arg === '--live') {
            options.mode = 'live';
            continue;
        }
        if (arg === '--fixture') {
            options.fixturePath = readOptionValue(args, index, arg);
            index += 1;
            continue;
        }
        if (arg === '--mode') {
            options.mode = readMode(readOptionValue(args, index, arg));
            index += 1;
            continue;
        }
        if (arg === '--api-base') {
            options.apiBase = readOptionValue(args, index, arg).replace(/\/+$/, '');
            index += 1;
            continue;
        }
        if (arg === '--fail-under-score') {
            options.failUnderScore = readScore(readOptionValue(args, index, arg), arg);
            index += 1;
            continue;
        }
        throw new Error(`Unknown option: ${arg}\n\n${usage()}`);
    }
    return options;
}
function readOptionValue(args, index, option) {
    const value = args[index + 1];
    if (!value || value.startsWith('--')) {
        throw new Error(`${option} requires a value.`);
    }
    return value;
}
function readMode(value) {
    if (value !== 'fixture' && value !== 'live') {
        throw new Error('--mode must be fixture or live.');
    }
    return value;
}
function readScore(value, option) {
    const parsed = Number(value);
    if (Number.isNaN(parsed) || parsed < 0 || parsed > 100) {
        throw new Error(`${option} must be a score from 0 to 100.`);
    }
    return parsed;
}
function printOrJson(report, json) {
    if (json) {
        console.log(JSON.stringify(report, null, 2));
    }
    else {
        printReport(report);
    }
}
function printReport(report) {
    console.log(`Knowledge completeness eval: ${report.fixtureName}`);
    console.log(`Project: ${report.project}`);
    console.log(`Mode: ${report.mode}`);
    if (report.skipped) {
        console.log(`Skipped: ${report.skipReason ?? 'not available'}`);
        return;
    }
    console.log(`Cases: ${report.totalCases}`);
    console.log('');
    console.log('Metrics');
    console.log(`  pass rate: ${formatRate(report.metrics.passRate)}`);
    console.log(`  completeness: ${formatRate(report.metrics.averageCompleteness)}`);
    console.log(`  source coverage: ${formatRate(report.metrics.averageSourceCoverage)}`);
    console.log(`  direct evidence placement: ${formatRate(report.metrics.averageDirectEvidencePlacement)}`);
    console.log(`  noise rate: ${formatRate(report.metrics.averageNoiseRate)}`);
    console.log(`  knowledge gain score: ${formatScore(report.metrics.averageKnowledgeGainScore)}`);
    console.log('');
    console.log('Cases');
    for (const testCase of report.cases) {
        const status = testCase.passed ? 'PASS' : 'FAIL';
        console.log(`  ${status} ${testCase.id}: completeness=${formatRate(testCase.completeness)} sources=${formatRate(testCase.sourceCoverage)} noise=${formatRate(testCase.noiseRate)} score=${formatScore(testCase.knowledgeGainScore)}`);
        if (!testCase.passed) {
            for (const fact of testCase.factResults.filter((fact) => !fact.passed)) {
                console.log(`    fact ${fact.id}: missingTerms=[${fact.missingTerms.join(', ')}] missingRefs=[${fact.missingSourceRefs.join(', ')}]`);
            }
            for (const source of testCase.sourceResults.filter((source) => !source.passed)) {
                console.log(`    source ${source.type}:${source.value} missing`);
            }
            for (const hit of testCase.forbiddenHits) {
                console.log(`    forbidden ${hit.forbiddenType}:${hit.value} in ${hit.section} item "${hit.title}"`);
            }
        }
    }
}
function shouldFail(report, options) {
    if (report.skipped) {
        return false;
    }
    const failedCase = report.cases.some((testCase) => !testCase.passed);
    const score = report.metrics.averageKnowledgeGainScore;
    const missedScore = options.failUnderScore !== undefined
        && score !== null
        && score < options.failUnderScore;
    return failedCase || missedScore;
}
function formatRate(value) {
    return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
}
function formatScore(value) {
    return value === null ? 'n/a' : value.toFixed(1);
}
function usage() {
    return [
        'Usage: pnpm run eval:knowledge-completeness -- [options]',
        '',
        'Options:',
        '  --fixture <path>              Fixture JSON path (default: eval/knowledge-completeness-fixtures.json)',
        '  --mode <fixture|live>         Run in memory fixture mode or against a live API (default: fixture)',
        '  --live                        Alias for --mode live',
        '  --api-base <url>              Live API base URL (default: http://localhost:3027)',
        '  --fail-under-score <0-100>    Exit non-zero when average knowledge gain drops below the score',
        '  --json                        Print the full JSON report',
        '  -h, --help                    Show this help',
    ].join('\n');
}
main().catch((error) => {
    console.error(error instanceof Error ? error.message : error);
    process.exitCode = 1;
});
