import { fetchYoutubeSourceMetrics, NEGATIVE_TTL_MS } from "@steipete/summarize-core/content";
import * as urlUtils from "@steipete/summarize-core/content/url";
import { buildExtractCacheKey } from "../../../cache.js";
import { createLinkPreviewClient, } from "../../../content/index.js";
import { createFirecrawlScraper } from "../../../firecrawl.js";
import { resolveSlideSource } from "../../../slides/index.js";
import { identifySpeakersInExtractedContent, rememberSpeakerMappings, SpeakerIdentificationError, } from "../../../speaker-identification/index.js";
import { readTweetWithPreferredClient } from "../../bird.js";
import { resolveTwitterCookies } from "../../cookies/twitter.js";
import { hasBirdCli, hasXurlCli } from "../../env.js";
import { writeVerbose } from "../../logging.js";
import { resolveUrlFlowYtDlpPath } from "./external-media.js";
import { fetchLinkContentWithBirdTip } from "./extract.js";
import { resolveUrlFetchOptions } from "./fetch-options.js";
const SOURCE_METRICS_TTL_MS = 60 * 60 * 1_000;
const SOURCE_METRICS_RETRY_TTL_MS = 5 * 60 * 1_000;
const SOURCE_METRICS_REFRESH_TIMEOUT_MS = 5_000;
const YOUTUBE_TRANSCRIPT_SOURCES = new Set(["youtubei", "captionTracks", "youtube-media", "apify"]);
function resolveYoutubeVideoId(extracted, targetUrl) {
    return (extracted.sourceMetrics?.videoId ??
        (extracted.video?.kind === "youtube"
            ? urlUtils.extractYouTubeVideoId(extracted.video.url)
            : null) ??
        urlUtils.extractYouTubeVideoId(targetUrl));
}
export function createUrlExtractionSession({ ctx, markdown, onProgress, }) {
    const { io, flags, model, cache: cacheState } = ctx;
    const urlFetch = io.urlFetch ?? io.fetch;
    const cacheStore = cacheState.mode === "default" ? cacheState.store : null;
    const transcriptCache = cacheStore ? cacheStore.transcriptCache : null;
    const firecrawlApiKey = model.apiStatus.firecrawlApiKey;
    const scrapeWithFirecrawl = model.apiStatus.firecrawlConfigured && flags.firecrawlMode !== "off" && firecrawlApiKey
        ? createFirecrawlScraper({
            apiKey: firecrawlApiKey,
            fetchImpl: io.fetch,
        })
        : null;
    const readTweetWithBirdClient = hasXurlCli(io.env) || hasBirdCli(io.env)
        ? ({ url, timeoutMs }) => readTweetWithPreferredClient({ url, timeoutMs, env: io.env })
        : null;
    const client = createLinkPreviewClient({
        env: io.envForRun,
        apifyApiToken: model.apiStatus.apifyToken,
        ytDlpPath: resolveUrlFlowYtDlpPath({
            urlFetch: io.urlFetch,
            ytDlpPath: model.apiStatus.ytDlpPath,
        }),
        transcription: {
            env: io.envForRun,
            falApiKey: model.apiStatus.falApiKey,
            groqApiKey: model.apiStatus.groqApiKey,
            assemblyaiApiKey: model.apiStatus.assemblyaiApiKey,
            elevenlabsApiKey: model.apiStatus.elevenlabsApiKey,
            openaiApiKey: model.apiStatus.openaiApiKey,
            geminiApiKey: model.apiStatus.googleApiKey,
        },
        scrapeWithFirecrawl,
        convertHtmlToMarkdown: markdown.convertHtmlToMarkdown,
        readTweetWithBird: readTweetWithBirdClient,
        resolveTwitterCookies: async (_args) => {
            const res = await resolveTwitterCookies({ env: io.env });
            return {
                cookiesFromBrowser: res.cookies.cookiesFromBrowser,
                source: res.cookies.source,
                warnings: res.warnings,
            };
        },
        fetch: urlFetch,
        transcriptCache,
        mediaCache: ctx.mediaCache ?? null,
        onProgress,
    });
    const fetchWithCache = async (targetUrl, { bypassExtractCache = false } = {}) => {
        const { localFile, options } = resolveUrlFetchOptions({
            targetUrl,
            flags,
            markdown,
            cacheMode: cacheState.mode,
        });
        const cacheKey = !localFile && cacheStore && cacheState.mode === "default"
            ? buildExtractCacheKey({
                url: targetUrl,
                options: {
                    youtubeTranscript: options.youtubeTranscript,
                    mediaTranscript: options.mediaTranscript,
                    embeddedVideo: options.embeddedVideo,
                    firecrawl: options.firecrawl,
                    format: options.format,
                    markdownMode: options.markdownMode ?? null,
                    transcriptTimestamps: options.transcriptTimestamps ?? false,
                    transcriptDiarization: options.transcriptDiarization ?? null,
                    speakerIdentification: flags.speakerIdentification
                        ? {
                            sourceKey: flags.speakerIdentification.sourceKey,
                            profileName: flags.speakerIdentification.profileName,
                            host: flags.speakerIdentification.host,
                            knownSpeakers: flags.speakerIdentification.knownSpeakers,
                            context: flags.speakerIdentification.context,
                            model: flags.speakerIdentification.model,
                            minimumConfidence: flags.speakerIdentification.minimumConfidence,
                            anchors: flags.speakerIdentification.anchors,
                            remembered: flags.speakerIdentification.remembered,
                            remember: flags.speakerIdentification.remember,
                        }
                        : null,
                    throwOnAssetLikeHtmlError: options.throwOnAssetLikeHtmlError ?? false,
                    ...(typeof options.maxCharacters === "number"
                        ? { maxCharacters: options.maxCharacters }
                        : {}),
                },
            })
            : null;
        if (!bypassExtractCache && !flags.speakerIdentification?.remember && cacheKey && cacheStore) {
            const cached = cacheStore.getJson("extract", cacheKey);
            if (cached) {
                const cachedVideoId = resolveYoutubeVideoId(cached, targetUrl);
                const separatelyCachedMetrics = cacheStore.getJson("extract", `${cacheKey}:source-metrics`);
                const metricsFreshness = cacheStore.getJson("extract", `${cacheKey}:source-metrics-fresh`);
                if (metricsFreshness?.attempted === true) {
                    return {
                        ...cached,
                        sourceMetrics: separatelyCachedMetrics?.videoId === cachedVideoId
                            ? separatelyCachedMetrics
                            : cached.sourceMetrics,
                    };
                }
                const isYoutubeTranscript = (cached.transcriptSource != null &&
                    YOUTUBE_TRANSCRIPT_SOURCES.has(cached.transcriptSource)) ||
                    (cached.transcriptSource === "yt-dlp" && cached.video?.kind === "youtube");
                const needsSourceMetricsMigration = Boolean(cachedVideoId) &&
                    (Boolean(urlUtils.extractYouTubeVideoId(targetUrl)) || isYoutubeTranscript) &&
                    cached.sourceMetrics?.platform !== "youtube";
                const observedMs = cached.sourceMetrics
                    ? Date.parse(cached.sourceMetrics.observedAt)
                    : Number.NaN;
                const needsSourceMetricsRefresh = cached.sourceMetrics?.platform === "youtube" &&
                    (!Number.isFinite(observedMs) || Date.now() - observedMs >= SOURCE_METRICS_TTL_MS);
                if (!needsSourceMetricsMigration && !needsSourceMetricsRefresh) {
                    writeVerbose(io.stderr, flags.verbose, "cache hit extract", flags.verboseColor, io.envForRun);
                    return cached;
                }
                writeVerbose(io.stderr, flags.verbose, "cache refresh extract (missing source metrics)", flags.verboseColor, io.envForRun);
                if (cachedVideoId) {
                    const refreshedMetrics = await fetchYoutubeSourceMetrics({
                        fetchImpl: urlFetch,
                        ytDlpPath: resolveUrlFlowYtDlpPath({
                            urlFetch: io.urlFetch,
                            ytDlpPath: model.apiStatus.ytDlpPath,
                        }),
                        videoId: cachedVideoId,
                        timeoutMs: Math.min(flags.timeoutMs, SOURCE_METRICS_REFRESH_TIMEOUT_MS),
                    });
                    if (refreshedMetrics) {
                        const refreshedCached = { ...cached, sourceMetrics: refreshedMetrics };
                        cacheStore.setJson("extract", `${cacheKey}:source-metrics`, refreshedMetrics, cacheState.ttlMs);
                        cacheStore.setJson("extract", `${cacheKey}:source-metrics-fresh`, { attempted: true }, SOURCE_METRICS_TTL_MS);
                        return refreshedCached;
                    }
                }
                cacheStore.setJson("extract", `${cacheKey}:source-metrics-fresh`, { attempted: true }, SOURCE_METRICS_RETRY_TTL_MS);
                return separatelyCachedMetrics?.videoId === cachedVideoId
                    ? { ...cached, sourceMetrics: separatelyCachedMetrics }
                    : cached;
            }
            writeVerbose(io.stderr, flags.verbose, "cache miss extract", flags.verboseColor, io.envForRun);
        }
        try {
            let extracted = await fetchLinkContentWithBirdTip({
                client,
                url: targetUrl,
                options,
                env: io.env,
            });
            let cacheable = true;
            if (flags.speakerIdentification) {
                const identified = await identifySpeakersInExtractedContent({
                    extracted,
                    sourceUrl: targetUrl,
                    settings: flags.speakerIdentification,
                    openaiApiKey: model.apiStatus.openaiApiKey,
                    openaiBaseUrl: model.apiStatus.providerBaseUrls.openai,
                    timeoutMs: flags.timeoutMs,
                    maxContentCharacters: options.maxCharacters ?? null,
                    fetchImpl: io.fetch,
                });
                extracted = identified.extracted;
                cacheable = identified.cacheable;
                if (identified.inferenceAttempted) {
                    model.llmCalls.push({
                        provider: "openai",
                        model: flags.speakerIdentification.model,
                        usage: identified.usage,
                        purpose: "speaker-identification",
                    });
                }
                if (identified.warning) {
                    writeVerbose(io.stderr, flags.verbose, identified.warning, flags.verboseColor, io.envForRun);
                    io.stderr.write(`Warning: ${identified.warning}\n`);
                }
                if (flags.speakerIdentification.remember) {
                    if (!flags.configPath || !identified.transcriptHash) {
                        throw new SpeakerIdentificationError("Unable to resolve the config path or transcript hash for --remember-speakers.");
                    }
                    try {
                        await rememberSpeakerMappings({
                            configPath: flags.configPath,
                            settings: flags.speakerIdentification,
                            mappings: identified.mappings,
                            transcriptHash: identified.transcriptHash,
                        });
                    }
                    catch (error) {
                        throw new SpeakerIdentificationError(`Failed to remember speaker mappings: ${error instanceof Error ? error.message : String(error)}`);
                    }
                }
            }
            if (cacheable && cacheKey && cacheStore) {
                const baseExtractTtlMs = extracted.transcriptSource === "unavailable" ? NEGATIVE_TTL_MS : cacheState.ttlMs;
                cacheStore.setJson("extract", cacheKey, extracted, baseExtractTtlMs);
                if (extracted.sourceMetrics?.platform === "youtube") {
                    cacheStore.setJson("extract", `${cacheKey}:source-metrics`, extracted.sourceMetrics, cacheState.ttlMs);
                    cacheStore.setJson("extract", `${cacheKey}:source-metrics-fresh`, { attempted: true }, SOURCE_METRICS_TTL_MS);
                }
                writeVerbose(io.stderr, flags.verbose, "cache write extract", flags.verboseColor, io.envForRun);
            }
            return extracted;
        }
        catch (err) {
            if (err instanceof SpeakerIdentificationError || flags.speakerIdentification)
                throw err;
            const errorMessage = err instanceof Error
                ? [err.message, err.cause instanceof Error ? err.cause.message : null]
                    .filter(Boolean)
                    .join(": ")
                : String(err);
            const preferUrlMode = typeof urlUtils.shouldPreferUrlMode === "function"
                ? urlUtils.shouldPreferUrlMode(targetUrl)
                : false;
            const isTwitter = urlUtils.isTwitterStatusUrl?.(targetUrl) ?? false;
            const isPodcast = urlUtils.isPodcastHost?.(targetUrl) ?? false;
            if (!preferUrlMode || isTwitter || isPodcast)
                throw err;
            writeVerbose(io.stderr, flags.verbose, `extract fallback url-only (${errorMessage})`, flags.verboseColor, io.envForRun);
            return {
                content: "",
                title: null,
                description: null,
                url: targetUrl,
                siteName: null,
                wordCount: 0,
                totalCharacters: 0,
                truncated: false,
                mediaDurationSeconds: null,
                video: null,
                isVideoOnly: true,
                transcriptSource: null,
                transcriptCharacters: null,
                transcriptWordCount: null,
                transcriptLines: null,
                transcriptMetadata: null,
                transcriptSegments: null,
                transcriptTimedText: null,
                transcriptionProvider: null,
                diagnostics: {
                    strategy: "html",
                    firecrawl: {
                        attempted: false,
                        used: false,
                        cacheMode: cacheState.mode,
                        cacheStatus: "bypassed",
                        notes: `skipped (url-only fallback: ${errorMessage})`,
                    },
                    markdown: {
                        requested: false,
                        used: false,
                        provider: null,
                        notes: `skipped (url fallback: ${errorMessage})`,
                    },
                    transcript: {
                        cacheMode: cacheState.mode,
                        cacheStatus: "unknown",
                        textProvided: false,
                        provider: null,
                        attemptedProviders: [],
                    },
                },
            };
        }
    };
    const fetchInitialExtract = async (url) => {
        let extracted = await fetchWithCache(url);
        if (flags.slides && !resolveSlideSource({ url, extracted })) {
            const isTwitter = urlUtils.isTwitterStatusUrl?.(url) ?? false;
            if (isTwitter) {
                const refreshed = await fetchWithCache(url, { bypassExtractCache: true });
                if (resolveSlideSource({ url, extracted: refreshed })) {
                    writeVerbose(io.stderr, flags.verbose, "extract refresh for slides", flags.verboseColor, io.envForRun);
                    extracted = refreshed;
                }
            }
        }
        return extracted;
    };
    return {
        cacheStore,
        fetchInitialExtract,
        fetchWithCache,
    };
}
//# sourceMappingURL=extraction-session.js.map