import { DataSource, SyncInput, SyncOutput } from './source-base';
import type { ExtraFileTags } from './source-base';
import { z } from '@botpress/sdk';
import { WellKnownMetadata } from '../../constants';
import type { WorkflowStep } from '../workflow-step';
import type { Client } from '@botpress/client';
type SitemapFilterContext = {
    url: string;
    lastmod?: string;
    changefreq?: string;
    priority?: string;
};
export type FetchResult = {
    url: string;
    contentType: string;
    content: string;
    metadata?: {
        [WellKnownMetadata.knowledge.TITLE]?: string;
        [WellKnownMetadata.knowledge.DESCRIPTION]?: string;
        [WellKnownMetadata.knowledge.FAVICON]?: string;
    };
};
/**
 * Fetch strategy for retrieving page content
 *
 * - 'node:fetch': Uses Node's built-in fetch (fast, no dependencies, works for static HTML)
 * - 'integration:browser': Uses browser integration (slower, requires browser integration, handles JavaScript/SPAs)
 *
 * Note: this only controls how *page bodies* are fetched. Sitemap files (XML)
 * are always retrieved via plain HTTP — the headless browser renders XML as a
 * styled HTML tree view, which breaks XML parsing.
 */
export type FetchStrategy = 'node:fetch' | 'integration:browser';
/**
 * Fetch option can be:
 * - A strategy string: 'node:fetch' or 'integration:browser'
 * - A custom function: for special authentication, headers, or processing
 */
export type FetchOption = FetchStrategy | ((url: string) => Promise<FetchResult> | FetchResult);
type WebsiteSourceOptions = {
    id?: string;
    filter?: (context: SitemapFilterContext) => boolean;
    /**
     * Fetch method to use for retrieving web *pages* (sitemap files always use plain HTTP)
     *
     * Options:
     * - 'node:fetch': Fast, uses Node's built-in fetch (best for static HTML sites) **[DEFAULT]**
     * - 'integration:browser': Slower, uses browser integration (best for JavaScript/SPAs)
     * - Custom function: Provide your own fetch implementation (for auth, special headers, etc.)
     * - undefined: Defaults to 'node:fetch'
     *
     * @default 'node:fetch'
     *
     * @example
     * // Use Node's built-in fetch (default, can be omitted)
     * { fetch: 'node:fetch' }
     *
     * @example
     * // Use browser integration for JavaScript-heavy sites
     * { fetch: 'integration:browser' }
     *
     * @example
     * // Custom fetch with authentication
     * {
     *   fetch: async (url) => {
     *     const response = await fetch(url, {
     *       headers: { Authorization: 'Bearer token' }
     *     })
     *     return {
     *       url,
     *       contentType: 'text/html',
     *       content: await response.text()
     *     }
     *   }
     * }
     */
    fetch?: FetchOption;
    maxPages?: number;
    maxDepth?: number;
    /**
     * Extra tags applied to every file ingested by this source, on top of the
     * well-known KB/source identity tags. Either a static record or a function
     * resolved per upload. Reserved keys (`source`, `kbId`, `kbName`, `dsId`,
     * `dsType`) are ignored. Useful for slot/release labels, ownership, etc.
     *
     * @example { tags: { slot: 'draft' } }
     * @example { tags: () => ({ slot: currentSlot() }) }
     */
    tags?: ExtraFileTags;
};
type UrlsSourceOptions = {
    id?: string;
    /**
     * Fetch method to use for retrieving web pages
     *
     * See WebsiteSourceOptions.fetch for detailed documentation
     */
    fetch?: FetchOption;
    /**
     * Extra tags applied to every file ingested by this source.
     * See WebsiteSourceOptions.tags for detailed documentation.
     */
    tags?: ExtraFileTags;
};
type WebsiteSourceMode = 'website' | 'sitemap' | 'urls' | 'llms-txt';
export type SitemapUrl = {
    loc: string;
    lastmod?: string;
    changefreq?: string;
    priority?: string;
    title?: string;
};
export declare function isLikelySitemapUrl(url: string): boolean;
/**
 * Group per-URL error messages into a compact list of human-readable strings,
 * one entry per distinct error message. Caps example URLs per group so the
 * sync dialog stays readable when many URLs hit the same error (e.g. every
 * page on a site is an unrendered SPA shell, or every sub-sitemap fetch
 * times out). Without this, a 500-URL sitemap with a uniform failure mode
 * would flood the UI with 500 near-identical rows.
 */
export declare const MAX_ERROR_EXAMPLES_PER_GROUP = 3;
export declare function groupUrlErrors(errorsByUrl: Map<string, string>, prefix: string): string[];
/**
 * Heuristic: does this HTML response look like an unrendered single-page app
 * shell? When `fetch: 'node:fetch'` is used against a JS-rendered site, the
 * raw response is an empty mount point (e.g. `<div id="root"></div>`) plus
 * script tags that would have populated it in a real browser. Indexing that
 * gets the user nothing useful and is hard to debug.
 *
 * Trigger requires BOTH:
 *  - a recognized SPA mount-point div (root/app/__next/__nuxt/svelte/main)
 *  - very little visible text after stripping scripts/styles/tags
 *
 * The combined check keeps false-positives low — content-rich sites that
 * happen to use `id="app"` won't trigger because they have substantial text.
 */
export declare function looksLikeUnrenderedSpa(html: string): {
    isSpa: boolean;
    textLength: number;
    htmlLength: number;
};
export declare class WebsiteSource extends DataSource {
    protected mode: WebsiteSourceMode;
    protected baseUrl: string | undefined;
    protected sitemapUrl: string | undefined;
    protected llmsTxtUrl: string | undefined;
    protected urls: string[] | undefined;
    protected filterFn: ((context: SitemapFilterContext) => boolean) | undefined;
    protected customFetch: ((url: string) => Promise<FetchResult> | FetchResult) | undefined;
    protected fetchStrategy: FetchStrategy;
    protected maxPages: number;
    protected maxDepth: number;
    constructor(id: string, mode: WebsiteSourceMode, options: (WebsiteSourceOptions | UrlsSourceOptions) & {
        baseUrl?: string;
        sitemapUrl?: string;
        llmsTxtUrl?: string;
        urls?: string[];
    });
    /** Get serializable configuration for change detection (only user-provided options) */
    getConfig(): Record<string, unknown>;
    /**
     * Resolve a Botpress Client to use for calling deployed integrations.
     *
     * The CLI (`adk kb sync`) passes its authenticated client explicitly through
     * `syncDirect`. The runtime workflow path doesn't thread a client through
     * each call, so we fall back to the client stored in the bot context — the
     * same one used by `runtime/actions.ts` to invoke integration actions.
     */
    protected resolveActionClient(explicit?: Client): Client;
    /**
     * Default fetch implementation using Node's built-in fetch.
     */
    protected defaultFetch(url: string): Promise<FetchResult>;
    /**
     * Fetch a sitemap or llms.txt file. Always uses plain HTTP — the headless
     * browser renders XML as styled HTML, breaking sitemap parsing. A user-provided
     * `customFetch` still wins so authenticated XML hosts work, but the strategy
     * fallback never goes through `browser:browsePages`.
     *
     * Mirrors stratus's documents-lambda which uses `axios.get` for sitemap files
     * and only falls back to the scraper (with `disableRendering: true`) when
     * blocked.
     */
    protected fetchSitemapContent(url: string): Promise<FetchResult>;
    /**
     * Fetch a page body for indexing. Honors the configured `fetchStrategy`:
     * `integration:browser` routes through `browser:browsePages` for JS-rendered
     * pages; `node:fetch` (the default) hits the URL directly.
     *
     * On the `node:fetch` path, also checks whether the response looks like an
     * unrendered SPA shell — if so, throws an actionable error pointing the user
     * at `fetch: 'integration:browser'` instead of silently indexing junk.
     */
    protected fetchPageContent(url: string, explicitClient?: Client): Promise<FetchResult>;
    /**
     * Apply the user-provided filter to a list of bare URLs and return SitemapUrl entries.
     */
    private applyFilterToBareUrls;
    /**
     * Discover URLs by crawling the website with the browser integration.
     *
     * Used for `mode === 'website'` (no sitemap; we discover URLs by following
     * links). Calls `browser:discoverUrls` via the cloud actions API, so works
     * from both the runtime workflow and the CLI.
     */
    private discoverUrlsViaBrowser;
    /**
     * BFS-crawl a sitemap tree starting at `startUrl`, calling `fetchSitemap`
     * to retrieve each candidate. Skips sub-sitemap entries that don't look
     * like sitemap files (`isLikelySitemapUrl`) so malformed sitemap-indexes
     * (page URLs wrapped in `<sitemap>` tags) don't poison the queue.
     *
     * Returns the discovered page URLs plus any per-sitemap fetch errors so
     * the caller can surface them in the sync report — without these, sitemap
     * failures are silent from the user's POV.
     */
    private bfsSitemapCrawl;
    /**
     * Filter llms.txt entries by the user-provided filter (if any), capped at maxPages.
     */
    private applyFilterToLlmsTxtUrls;
    /**
     * Discover URLs for the workflow path (resumable, observable via `step()`).
     * The workflow-level `state` schema is still defined for `createSyncWorkflow`
     * — workflow persistence/resumption needs a known shape — but URL discovery
     * itself accumulates locally inside `bfsSitemapCrawl`, so this method
     * doesn't take `state` as a parameter.
     */
    protected discoverUrls(step: WorkflowStep): Promise<{
        urls: SitemapUrl[];
        errors: string[];
    }>;
    /**
     * Discover URLs for the CLI path (one-shot, no workflow steps).
     */
    protected discoverUrlsDirect(client?: Client): Promise<{
        urls: SitemapUrl[];
        errors: string[];
    }>;
    get syncWorkflow(): import("..").Workflow<`data_source_sync_${string}`, z.ZodObject<{
        dsId: z.ZodString;
        kbName: z.ZodString;
        kbId: z.ZodString;
        force: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
    }, "strip", {
        kbId: string;
        kbName: string;
        dsId: string;
        force: boolean;
    }, {
        kbId: string;
        kbName: string;
        dsId: string;
        force?: boolean | undefined;
    }>, z.ZodObject<{
        processed: z.ZodDefault<z.ZodNumber>;
        added: z.ZodDefault<z.ZodArray<z.ZodObject<{
            file: z.ZodString;
            name: z.ZodString;
            hash: z.ZodString;
            size: z.ZodNumber;
        }, "strip", {
            name: string;
            size: number;
            hash: string;
            file: string;
        }, {
            name: string;
            size: number;
            hash: string;
            file: string;
        }>, "many">>;
        updated: z.ZodDefault<z.ZodArray<z.ZodObject<{
            file: z.ZodString;
            name: z.ZodString;
            hash: z.ZodString;
            size: z.ZodNumber;
        }, "strip", {
            name: string;
            size: number;
            hash: string;
            file: string;
        }, {
            name: string;
            size: number;
            hash: string;
            file: string;
        }>, "many">>;
        deleted: z.ZodDefault<z.ZodArray<z.ZodObject<{
            file: z.ZodString;
            name: z.ZodString;
            hash: z.ZodString;
            size: z.ZodNumber;
        }, "strip", {
            name: string;
            size: number;
            hash: string;
            file: string;
        }, {
            name: string;
            size: number;
            hash: string;
            file: string;
        }>, "many">>;
        errors: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
    }, "strip", {
        processed: number;
        added: {
            name: string;
            size: number;
            hash: string;
            file: string;
        }[];
        updated: {
            name: string;
            size: number;
            hash: string;
            file: string;
        }[];
        deleted: {
            name: string;
            size: number;
            hash: string;
            file: string;
        }[];
        errors: string[];
    }, {
        processed?: number | undefined;
        added?: {
            name: string;
            size: number;
            hash: string;
            file: string;
        }[] | undefined;
        updated?: {
            name: string;
            size: number;
            hash: string;
            file: string;
        }[] | undefined;
        deleted?: {
            name: string;
            size: number;
            hash: string;
            file: string;
        }[] | undefined;
        errors?: string[] | undefined;
    }>, z.ZodObject<{
        urls: z.ZodDefault<z.ZodArray<z.ZodObject<{
            loc: z.ZodString;
            lastmod: z.ZodOptional<z.ZodString>;
            changefreq: z.ZodOptional<z.ZodString>;
            priority: z.ZodOptional<z.ZodString>;
        }, "strip", {
            loc: string;
            lastmod?: string | undefined;
            changefreq?: string | undefined;
            priority?: string | undefined;
        }, {
            loc: string;
            lastmod?: string | undefined;
            changefreq?: string | undefined;
            priority?: string | undefined;
        }>, "many">>;
        queue: z.ZodDefault<z.ZodArray<z.ZodObject<{
            url: z.ZodString;
            depth: z.ZodNumber;
        }, "strip", {
            url: string;
            depth: number;
        }, {
            url: string;
            depth: number;
        }>, "many">>;
    }, "strip", {
        urls: {
            loc: string;
            lastmod?: string | undefined;
            changefreq?: string | undefined;
            priority?: string | undefined;
        }[];
        queue: {
            url: string;
            depth: number;
        }[];
    }, {
        urls?: {
            loc: string;
            lastmod?: string | undefined;
            changefreq?: string | undefined;
            priority?: string | undefined;
        }[] | undefined;
        queue?: {
            url: string;
            depth: number;
        }[] | undefined;
    }>, Record<string, import("../../types").ZuiType>, Record<string, import("../../types").ZuiType>>;
    /**
     * Sync this website source directly without workflows.
     * Used by the CLI (KnowledgeManager) to sync website sources locally.
     *
     * Supports all four modes (`urls`, `sitemap`, `llms-txt`, `website`). Both
     * `node:fetch` and `integration:browser` work — for `integration:browser`,
     * page fetches route through `client.callAction('browser:browsePages')`,
     * and `mode === 'website'` URL discovery routes through
     * `client.callAction('browser:discoverUrls')`. Sitemap files themselves are
     * always fetched via plain HTTP (the headless browser renders XML as HTML).
     */
    syncDirect(client: Client, botId: string, input: z.infer<typeof SyncInput>): Promise<z.infer<typeof SyncOutput>>;
    static fromWebsite(baseUrl: string, options?: WebsiteSourceOptions): WebsiteSource;
    static fromSitemap(sitemapUrl: string, options?: WebsiteSourceOptions): WebsiteSource;
    static fromLlmsTxt(llmsTxtUrl: string, options?: WebsiteSourceOptions): WebsiteSource;
    static fromUrls(urls: string[], options?: UrlsSourceOptions): WebsiteSource;
}
export {};
//# sourceMappingURL=source-website.d.ts.map