/**
 * Restructure VisionClaw logs into task-oriented records.
 *
 * Loads log entries, segments into wake cycles, optionally enriches with
 * SDK transcripts, optionally filters, and writes JSONL output files.
 *
 * When a transcript directory with image-archive sidecar files is provided,
 * archived images are recovered (downloaded from S3/TOS) and restored into
 * the restructured output.
 *
 * Can be invoked standalone (`restructure-logs` command) or as part of the
 * unified `process-data` pipeline.
 */
import fs from "node:fs";
import path from "node:path";
import { loadSidecarIndex, prefetchImages, emptyRecoveryStats, mergeRecoveryStats } from "./image-recovery.js";
import { filterCycles, mergeFilterStats } from "./filter.js";
import { loadAllEntries } from "./loader.js";
import { segmentWakeCycles, classifyCycles } from "./segmenter.js";
import { loadSdkTranscripts, segmentSdkSessionTurns, buildSdkTurnIndex, matchSdkTurn, } from "./sdk-transcript.js";
import { alignRecordSessionId, applyTopLevelSessionId, applyWakeCycleId, buildRecord, buildSessionSwitchRecord, } from "./record-builder.js";
function parseRestructureArgs(args) {
    let logsDir = "";
    let transcriptDir;
    let outputDir = ".";
    let samples = false;
    for (let i = 0; i < args.length; i++) {
        const arg = args[i];
        if ((arg === "-t" || arg === "--transcript-dir") && i + 1 < args.length) {
            transcriptDir = args[++i];
        }
        else if ((arg === "-o" || arg === "--output-dir") && i + 1 < args.length) {
            outputDir = args[++i];
        }
        else if (arg === "--samples") {
            samples = true;
        }
        else if (!arg.startsWith("-") && !logsDir) {
            logsDir = arg;
        }
    }
    if (!logsDir) {
        console.error("Usage: visionclaw restructure-logs <logs-dir> [-t transcript-dir] [-o output-dir] [--samples]");
        process.exit(1);
    }
    return { logsDir, transcriptDir, outputDir, samples };
}
// ---------------------------------------------------------------------------
// JSONL writer
// ---------------------------------------------------------------------------
function writeJsonl(filePath, records) {
    fs.mkdirSync(path.dirname(filePath), { recursive: true });
    const fd = fs.openSync(filePath, "w");
    try {
        for (const rec of records) {
            const line = JSON.stringify(rec);
            fs.writeSync(fd, line + "\n");
        }
    }
    finally {
        fs.closeSync(fd);
    }
}
function writeSamples(filePath, records, maxSamples = 3) {
    const samples = records.slice(0, maxSamples);
    fs.mkdirSync(path.dirname(filePath), { recursive: true });
    fs.writeFileSync(filePath, JSON.stringify(samples, null, 2));
}
// ---------------------------------------------------------------------------
// Process a list of cycles into records (with per-session-turn matching)
// ---------------------------------------------------------------------------
async function processCycles(cycles, sdkTurnIndex, hasTranscript, sidecarIndex, sessionModeMap, imageCache) {
    let matched = 0;
    let unmatched = 0;
    const records = [];
    let recoveryStats = emptyRecoveryStats();
    for (const c of cycles) {
        const sessionTurns = c.getSessionTurns();
        const wakeCycleId = c.wakeEntry?.timestamp ?? "";
        const turnCount = Math.max(sessionTurns.length, 1);
        if (hasTranscript && sdkTurnIndex && sessionTurns.length > 0) {
            const isPiCycle = c.getPiCodingInits().length > 0;
            // --- Per-session-turn matching ---
            for (let turnIdx = 0; turnIdx < sessionTurns.length; turnIdx++) {
                const { sessionId, timestamp } = sessionTurns[turnIdx];
                const sdkTurn = matchSdkTurn(timestamp, sessionId, sdkTurnIndex, {
                    crossSessionWakeFallback: isPiCycle,
                    wakeTimestamp: wakeCycleId || undefined,
                });
                const cycleStats = emptyRecoveryStats();
                let record;
                const resolvedSessionId = sdkTurn?.sessionId ?? sessionId;
                if (sdkTurn) {
                    matched++;
                    if (turnIdx === 0) {
                        record = await buildRecord(c, sdkTurn, sidecarIndex, cycleStats, imageCache);
                    }
                    else {
                        record = await buildSessionSwitchRecord(sdkTurn, sidecarIndex, cycleStats, imageCache);
                    }
                }
                else {
                    unmatched++;
                    if (turnIdx === 0) {
                        record = await buildRecord(c, undefined, sidecarIndex, cycleStats, imageCache);
                    }
                    else {
                        continue;
                    }
                }
                applyWakeCycleId(record, wakeCycleId);
                alignRecordSessionId(record, resolvedSessionId);
                record.session_mode = sessionModeMap?.get(resolvedSessionId);
                record.turn_index = turnIdx;
                record.turn_count = turnCount;
                if (!sdkTurn) {
                    applyTopLevelSessionId(record);
                }
                recoveryStats = mergeRecoveryStats(recoveryStats, cycleStats);
                records.push(record);
            }
        }
        else {
            // No transcript or no system-init entries — single record from logs
            const cycleStats = emptyRecoveryStats();
            const record = await buildRecord(c, undefined, sidecarIndex, cycleStats, imageCache);
            applyWakeCycleId(record, wakeCycleId);
            if (sessionTurns.length > 0) {
                alignRecordSessionId(record, sessionTurns[0].sessionId);
                record.session_mode = sessionModeMap?.get(sessionTurns[0].sessionId);
                record.turn_index = 0;
                record.turn_count = turnCount;
            }
            else {
                applyTopLevelSessionId(record);
            }
            recoveryStats = mergeRecoveryStats(recoveryStats, cycleStats);
            records.push(record);
        }
    }
    return { records, matched, unmatched, recoveryStats };
}
// ---------------------------------------------------------------------------
// Print stats for a set of records
// ---------------------------------------------------------------------------
function printRecordStats(label, records, matched, unmatched, hasTranscript, filterStats) {
    if (records.length === 0 && filterStats.input === 0)
        return;
    const totalTraj = records.reduce((sum, r) => sum + r.trajectory.length, 0);
    const withPrompts = records.filter((r) => r.prompt !== "heartbeat").length;
    const withInterrupts = records.filter((r) => r.has_interrupts).length;
    console.log(`\n  ${label}:`);
    console.log(`    Records:              ${records.length}`);
    if (filterStats.compactionOnly > 0) {
        console.log(`    Compaction-only (filtered): ${filterStats.compactionOnly}`);
    }
    if (filterStats.lacksRealToolCall > 0) {
        console.log(`    No real tool call (filtered): ${filterStats.lacksRealToolCall}`);
    }
    if (filterStats.apiError > 0) {
        console.log(`    API error (filtered): ${filterStats.apiError}`);
    }
    if (filterStats.heartbeat > 0) {
        console.log(`    Heartbeat (filtered): ${filterStats.heartbeat}`);
    }
    console.log(`    With user prompt:     ${withPrompts}`);
    if (withInterrupts > 0) {
        console.log(`    With interrupts:      ${withInterrupts}`);
    }
    console.log(`    Total trajectory:     ${totalTraj}`);
    if (records.length > 0) {
        console.log(`    Avg trajectory len:   ${(totalTraj / records.length).toFixed(1)}`);
    }
    if (hasTranscript) {
        console.log(`    SDK matched:          ${matched}`);
        console.log(`    Log fallback:         ${unmatched}`);
    }
}
// ---------------------------------------------------------------------------
// Pick diverse samples
// ---------------------------------------------------------------------------
const MIN_MEANINGFUL_TRAJECTORY = 5;
function pickDiverseSamples(records, maxSamples = 3) {
    const samples = [];
    // 1. With compaction summary + user prompt
    for (const r of records) {
        if (r.prompt !== "heartbeat" &&
            r.prior_context?.compaction_summary &&
            r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
            samples.push(r);
            break;
        }
    }
    // 2. Interrupted cycle (if any)
    for (const r of records) {
        if (r.has_interrupts &&
            r.interrupts &&
            r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
            samples.push(r);
            break;
        }
    }
    // 3. Compacted fresh start with real work
    for (const r of records) {
        if (samples.includes(r))
            continue;
        if (r.prior_context?.compacted_fresh_start &&
            r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
            samples.push(r);
            break;
        }
    }
    // Fill remaining — prefer records with meaningful trajectories
    for (const r of records) {
        if (samples.length >= maxSamples)
            break;
        if (!samples.includes(r) && r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
            samples.push(r);
        }
    }
    // Last resort: fill from any records if not enough yet
    for (const r of records) {
        if (samples.length >= maxSamples)
            break;
        if (!samples.includes(r))
            samples.push(r);
    }
    return samples.slice(0, maxSamples);
}
export async function restructure(opts) {
    // Load logs
    let entries = await loadAllEntries(opts.logsDir);
    // Apply --since filter
    if (opts.since) {
        const sinceMs = opts.since.getTime();
        const before = entries.length;
        entries = entries.filter((e) => new Date(e.timestamp).getTime() >= sinceMs);
        console.log(`Since filter: ${before} → ${entries.length} entries (dropped ${before - entries.length})`);
    }
    // Segment and classify
    const cycles = segmentWakeCycles(entries);
    const { clean, interrupted, incomplete } = classifyCycles(cycles);
    console.log(`\nWake cycles: ${cycles.length}`);
    console.log(`  Clean (no interrupts, complete):  ${clean.length}`);
    console.log(`  Interrupted (has injections):      ${interrupted.length}`);
    console.log(`  Incomplete (no result):            ${incomplete.length}`);
    const heartbeat = clean.filter((c) => c.trigger === "heartbeat").length;
    const message = clean.filter((c) => c.trigger === "message").length;
    console.log(`\nClean cycles breakdown:`);
    console.log(`  Heartbeat-triggered:  ${heartbeat}`);
    console.log(`  Message-triggered:    ${message}`);
    // Filter each category
    const cleanFilter = filterCycles(clean);
    const interruptedFilter = filterCycles(interrupted);
    const incompleteFilter = filterCycles(incomplete);
    const totalFilter = mergeFilterStats(mergeFilterStats(cleanFilter.stats, interruptedFilter.stats), incompleteFilter.stats);
    if (totalFilter.input !== totalFilter.output) {
        console.log(`\nFiltering:`);
        console.log(`  Compaction-only:     ${totalFilter.compactionOnly}`);
        console.log(`  No real tool call:   ${totalFilter.lacksRealToolCall}`);
        console.log(`  API error:           ${totalFilter.apiError}`);
        console.log(`  Heartbeat:           ${totalFilter.heartbeat}`);
        console.log(`  Total removed:       ${totalFilter.input - totalFilter.output}`);
    }
    // Collect pure heartbeat cycles from all categories for separate output
    const allHeartbeatCycles = [
        ...cleanFilter.heartbeatCycles,
        ...interruptedFilter.heartbeatCycles,
        ...incompleteFilter.heartbeatCycles,
    ];
    // Load SDK transcripts per-session
    let sdkTurnIndex = null;
    const hasTranscript = !!opts.transcriptDir;
    if (opts.transcriptDir) {
        console.log(`\nLoading SDK transcripts...`);
        const perSession = await loadSdkTranscripts(opts.transcriptDir, opts.sessionIds);
        // Segment each session's messages into turns
        const allTurns = [];
        for (const { sessionId, messages } of perSession) {
            const turns = segmentSdkSessionTurns(sessionId, messages);
            const wakeCount = turns.filter((t) => t.startType === "wake_event").length;
            const switchCount = turns.filter((t) => t.startType === "session_switch").length;
            const modeLabel = opts.sessionModeMap?.get(sessionId) ?? "?";
            console.log(`  Session ${sessionId.slice(0, 8)} (${modeLabel}): ${turns.length} turns (${wakeCount} wake, ${switchCount} switch)`);
            allTurns.push(...turns);
        }
        sdkTurnIndex = buildSdkTurnIndex(allTurns);
        console.log(`  Total SDK turns: ${allTurns.length}`);
    }
    // Load image archive sidecar index and prefetch images
    let sidecarIndex = null;
    let imageCache;
    if (opts.transcriptDir) {
        sidecarIndex = await loadSidecarIndex(opts.transcriptDir);
        const cacheDir = path.join(opts.outputDir, ".image-cache");
        const imagesDir = path.join(opts.outputDir, "images");
        if (sidecarIndex.size > 0) {
            console.log(`\nImage archive sidecar: ${sidecarIndex.size} entries`);
            imageCache = prefetchImages(sidecarIndex, cacheDir, imagesDir);
            console.log(`  Image prefetch complete`);
        }
        else {
            fs.mkdirSync(imagesDir, { recursive: true });
            imageCache = { cacheDir, imagesDir, written: new Set() };
        }
    }
    // Process each category
    const cleanResult = await processCycles(cleanFilter.filtered, sdkTurnIndex, hasTranscript, sidecarIndex, opts.sessionModeMap, imageCache);
    const interruptedResult = await processCycles(interruptedFilter.filtered, sdkTurnIndex, hasTranscript, sidecarIndex, opts.sessionModeMap, imageCache);
    const incompleteResult = await processCycles(incompleteFilter.filtered, sdkTurnIndex, hasTranscript, sidecarIndex, opts.sessionModeMap, imageCache);
    const heartbeatResult = await processCycles(allHeartbeatCycles, sdkTurnIndex, hasTranscript, sidecarIndex, opts.sessionModeMap, imageCache);
    // Print stats
    const totalMatched = cleanResult.matched + interruptedResult.matched + incompleteResult.matched + heartbeatResult.matched;
    const totalUnmatched = cleanResult.unmatched +
        interruptedResult.unmatched +
        incompleteResult.unmatched +
        heartbeatResult.unmatched;
    if (hasTranscript) {
        console.log(`\nSDK transcript matching:`);
        console.log(`  Matched:   ${totalMatched}`);
        console.log(`  Unmatched: ${totalUnmatched} (using log-based trajectory)`);
    }
    // Print image recovery stats
    const totalRecovery = mergeRecoveryStats(mergeRecoveryStats(mergeRecoveryStats(cleanResult.recoveryStats, interruptedResult.recoveryStats), incompleteResult.recoveryStats), heartbeatResult.recoveryStats);
    if (totalRecovery.totalPlaceholders > 0) {
        console.log(`\nImage recovery:`);
        console.log(`  Placeholders found:    ${totalRecovery.totalPlaceholders}`);
        console.log(`  Recovered:             ${totalRecovery.recovered}`);
        console.log(`  Download failed:       ${totalRecovery.failed}`);
        console.log(`  No sidecar entry:      ${totalRecovery.noSidecarEntry}`);
        if (imageCache) {
            console.log(`  Unique images written: ${imageCache.written.size}`);
        }
    }
    // Clean up temporary image cache
    if (imageCache) {
        fs.rmSync(imageCache.cacheDir, { recursive: true, force: true });
    }
    const emptyStats = { input: 0, output: 0, compactionOnly: 0, lacksRealToolCall: 0, apiError: 0, heartbeat: 0 };
    printRecordStats("Clean", cleanResult.records, cleanResult.matched, cleanResult.unmatched, hasTranscript, cleanFilter.stats);
    printRecordStats("Interrupted", interruptedResult.records, interruptedResult.matched, interruptedResult.unmatched, hasTranscript, interruptedFilter.stats);
    printRecordStats("Incomplete", incompleteResult.records, incompleteResult.matched, incompleteResult.unmatched, hasTranscript, incompleteFilter.stats);
    printRecordStats("Heartbeat", heartbeatResult.records, heartbeatResult.matched, heartbeatResult.unmatched, hasTranscript, emptyStats);
    // Write output files
    const outDir = opts.outputDir;
    const cleanPath = path.join(outDir, "restructured-clean.jsonl");
    const interruptedPath = path.join(outDir, "restructured-interrupted.jsonl");
    const incompletePath = path.join(outDir, "restructured-incomplete.jsonl");
    const heartbeatPath = path.join(outDir, "restructured-heartbeat.jsonl");
    writeJsonl(cleanPath, cleanResult.records);
    writeJsonl(interruptedPath, interruptedResult.records);
    writeJsonl(incompletePath, incompleteResult.records);
    writeJsonl(heartbeatPath, heartbeatResult.records);
    console.log(`\nWrote output files to ${outDir}:`);
    console.log(`  ${cleanPath} (${cleanResult.records.length} records)`);
    console.log(`  ${interruptedPath} (${interruptedResult.records.length} records)`);
    console.log(`  ${incompletePath} (${incompleteResult.records.length} records)`);
    console.log(`  ${heartbeatPath} (${heartbeatResult.records.length} records)`);
    // Prune images that were copied during recovery but aren't in the final output
    if (imageCache) {
        const allRecords = [
            ...cleanResult.records,
            ...interruptedResult.records,
            ...incompleteResult.records,
            ...heartbeatResult.records,
        ];
        const pruned = pruneUnreferencedImages(imageCache.imagesDir, allRecords);
        if (pruned > 0) {
            console.log(`  Pruned ${pruned} unreferenced images`);
        }
    }
    // Write report.json summarizing the output
    const reportData = buildOutputReport({
        clean: cleanResult.records,
        interrupted: interruptedResult.records,
        incomplete: incompleteResult.records,
        heartbeat: heartbeatResult.records,
    });
    const reportPath = path.join(outDir, "report.json");
    fs.writeFileSync(reportPath, JSON.stringify(reportData, null, 2));
    console.log(`  ${reportPath}`);
    // Write samples if requested
    if (opts.samples) {
        const allRecords = [
            ...cleanResult.records,
            ...interruptedResult.records,
            ...incompleteResult.records,
        ];
        if (cleanResult.records.length > 0) {
            const samplesPath = path.join(outDir, "samples-clean.json");
            writeSamples(samplesPath, pickDiverseSamples(cleanResult.records));
            console.log(`  ${samplesPath}`);
        }
        if (interruptedResult.records.length > 0) {
            const samplesPath = path.join(outDir, "samples-interrupted.json");
            writeSamples(samplesPath, pickDiverseSamples(interruptedResult.records));
            console.log(`  ${samplesPath}`);
        }
        if (incompleteResult.records.length > 0) {
            const samplesPath = path.join(outDir, "samples-incomplete.json");
            writeSamples(samplesPath, pickDiverseSamples(incompleteResult.records));
            console.log(`  ${samplesPath}`);
        }
        if (heartbeatResult.records.length > 0) {
            const samplesPath = path.join(outDir, "samples-heartbeat.json");
            writeSamples(samplesPath, pickDiverseSamples(heartbeatResult.records));
            console.log(`  ${samplesPath}`);
        }
        const combinedPath = path.join(outDir, "samples-all.json");
        writeSamples(combinedPath, pickDiverseSamples(allRecords));
        console.log(`  ${combinedPath}`);
    }
}
function countToolCalls(records) {
    let count = 0;
    for (const r of records) {
        count += r.trajectory.filter((e) => e.role === "tool_call").length;
    }
    return count;
}
function buildOutputReport(categories) {
    const files = {};
    let totalRecords = 0;
    let totalToolCalls = 0;
    let totalRecordsNoHb = 0;
    let totalToolCallsNoHb = 0;
    for (const [name, records] of Object.entries(categories)) {
        const toolCalls = countToolCalls(records);
        files[`restructured-${name}.jsonl`] = {
            records: records.length,
            tool_calls: toolCalls,
        };
        totalRecords += records.length;
        totalToolCalls += toolCalls;
        if (name !== "heartbeat") {
            totalRecordsNoHb += records.length;
            totalToolCallsNoHb += toolCalls;
        }
    }
    return {
        total_records: totalRecords,
        total_tool_calls: totalToolCalls,
        total_records_no_pure_heartbeat: totalRecordsNoHb,
        total_tool_calls_no_pure_heartbeat: totalToolCallsNoHb,
        files,
    };
}
// ---------------------------------------------------------------------------
// Prune unreferenced images from the output images directory
// ---------------------------------------------------------------------------
function collectFileRefsFromString(text) {
    const refs = new Set();
    const normalized = text.replaceAll("\\/", "/");
    // Handle plain/escaped path mentions such as:
    // - images/img_123.png
    // - {"path":"images/img_123.png"}
    const imagePathRe = /(?:^|[^A-Za-z0-9_./-])images\/([A-Za-z0-9._-]+\.(?:png|jpg|jpeg|gif|webp))/gi;
    let m = imagePathRe.exec(normalized);
    while (m) {
        if (m[1])
            refs.add(m[1]);
        m = imagePathRe.exec(normalized);
    }
    // If the string itself is JSON-encoded structured content, recurse so we
    // can also catch canonical image block objects.
    const trimmed = normalized.trim();
    if ((trimmed.startsWith("{") && trimmed.endsWith("}")) ||
        (trimmed.startsWith("[") && trimmed.endsWith("]"))) {
        try {
            const parsed = JSON.parse(trimmed);
            for (const r of collectFileRefs(parsed))
                refs.add(r);
        }
        catch {
            // Best-effort parsing only.
        }
    }
    return refs;
}
function collectFileRefs(obj) {
    const refs = new Set();
    if (typeof obj === "string") {
        for (const r of collectFileRefsFromString(obj))
            refs.add(r);
        return refs;
    }
    if (obj === null || typeof obj !== "object")
        return refs;
    if (Array.isArray(obj)) {
        for (const item of obj) {
            for (const r of collectFileRefs(item))
                refs.add(r);
        }
        return refs;
    }
    const rec = obj;
    if (rec.type === "image") {
        const src = rec.source;
        if (src?.type === "file" && typeof src.path === "string") {
            refs.add(path.basename(src.path));
        }
    }
    for (const v of Object.values(rec)) {
        for (const r of collectFileRefs(v))
            refs.add(r);
    }
    return refs;
}
function pruneUnreferencedImages(imagesDir, records) {
    const referenced = new Set();
    for (const rec of records) {
        for (const r of collectFileRefs(rec))
            referenced.add(r);
    }
    let removed = 0;
    for (const name of fs.readdirSync(imagesDir)) {
        if (!referenced.has(name)) {
            fs.unlinkSync(path.join(imagesDir, name));
            removed++;
        }
    }
    return removed;
}
// ---------------------------------------------------------------------------
// CLI entry point (standalone command)
// ---------------------------------------------------------------------------
export async function runRestructure(args) {
    const opts = parseRestructureArgs(args);
    await restructure(opts);
}
//# sourceMappingURL=restructure.js.map