import { createToken, Lexer } from "chevrotain";
/**
 * Resilient Chevrotain-based tokenizer for the SQL fragments the rewriter cares about.
 *
 * The lexer skips whitespace and comments and recognizes identifiers (quoted/unquoted),
 * SQLite-style named parameters, basic punctuation, and the subset of keywords that
 * drive lix_internal_state_vtable rewriting.
 *
 * @example
 * ```ts
 * import { tokenize, FROM, Ident } from "./tokenizer";
 * const sql = "SELECT * FROM lix_internal_state_vtable";
 * const tokens = tokenize(sql);
 * const fromIx = tokens.findIndex((t) => t.tokenType === FROM);
 * console.log(tokens[fromIx + 1].tokenType === Ident); // true
 * ```
 */
const WhiteSpace = createToken({
    name: "WhiteSpace",
    pattern: /\s+/,
    group: Lexer.SKIPPED,
    line_breaks: true,
});
const LineComment = createToken({
    name: "LineComment",
    pattern: /--[^\n]*/,
    group: Lexer.SKIPPED,
    line_breaks: true,
});
const BlockComment = createToken({
    name: "BlockComment",
    pattern: /\/\*[\s\S]*?\*\//,
    group: Lexer.SKIPPED,
    line_breaks: true,
});
export const Comma = createToken({ name: "Comma", pattern: /,/ });
export const Dot = createToken({ name: "Dot", pattern: /\./ });
export const LParen = createToken({ name: "LParen", pattern: /\(/ });
export const RParen = createToken({ name: "RParen", pattern: /\)/ });
export const Star = createToken({ name: "Star", pattern: /\*/ });
export const SQStr = createToken({
    name: "SQStr",
    pattern: /'(?:[^']|'')*'/,
    line_breaks: true,
});
export const Num = createToken({
    name: "Num",
    pattern: /\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/,
});
export const QMarkNumber = createToken({
    name: "QMarkNumber",
    pattern: /\?\d+/,
});
export const QMark = createToken({ name: "QMark", pattern: /\?/ });
export const DollarName = createToken({
    name: "DollarName",
    pattern: /\$[A-Za-z_][\w$]*/,
});
export const DollarNumber = createToken({
    name: "DollarNumber",
    pattern: /\$\d+/,
});
export const AtName = createToken({
    name: "AtName",
    pattern: /@[A-Za-z_][\w$]*/,
});
export const ColonName = createToken({
    name: "ColonName",
    pattern: /:[A-Za-z_][\w$]*/,
});
export const Equals = createToken({ name: "Equals", pattern: /=/ });
export const Minus = createToken({ name: "Minus", pattern: /-/ });
export const Plus = createToken({ name: "Plus", pattern: /\+/ });
export const Pipe = createToken({ name: "Pipe", pattern: /\|\|/ });
export const Semicolon = createToken({
    name: "Semicolon",
    pattern: /;/,
});
export const QIdent = createToken({
    name: "QIdent",
    pattern: /"(?:[^"]|"")*"/,
    line_breaks: true,
});
export const Ident = createToken({
    name: "Ident",
    pattern: /[A-Za-z_][\w$]*/,
});
function keyword(name) {
    return createToken({
        name,
        pattern: new RegExp(name, "i"),
        longer_alt: Ident,
    });
}
export const SELECT = keyword("SELECT");
export const FROM = keyword("FROM");
export const JOIN = keyword("JOIN");
export const AS = keyword("AS");
export const WHERE = keyword("WHERE");
export const LIMIT = keyword("LIMIT");
export const ORDER = keyword("ORDER");
export const BY = keyword("BY");
export const GROUP = keyword("GROUP");
export const HAVING = keyword("HAVING");
export const WINDOW = keyword("WINDOW");
export const OFFSET = keyword("OFFSET");
export const FETCH = keyword("FETCH");
export const ON = keyword("ON");
export const AND = keyword("AND");
export const OR = keyword("OR");
export const IS = keyword("IS");
export const NOT = keyword("NOT");
export const NULL = keyword("NULL");
export const CAST = keyword("CAST");
export const WITH = keyword("WITH");
export const INSERT = keyword("INSERT");
export const UPDATE = keyword("UPDATE");
export const DELETE = keyword("DELETE");
const allTokens = [
    WhiteSpace,
    LineComment,
    BlockComment,
    Comma,
    Dot,
    LParen,
    RParen,
    Star,
    Equals,
    Minus,
    Plus,
    Pipe,
    Semicolon,
    SQStr,
    Num,
    QMarkNumber,
    QMark,
    DollarName,
    DollarNumber,
    AtName,
    ColonName,
    QIdent,
    SELECT,
    FROM,
    JOIN,
    AS,
    WHERE,
    LIMIT,
    ORDER,
    BY,
    GROUP,
    HAVING,
    WINDOW,
    OFFSET,
    FETCH,
    ON,
    AND,
    OR,
    IS,
    NOT,
    NULL,
    CAST,
    WITH,
    INSERT,
    UPDATE,
    DELETE,
    Ident,
];
const lexer = new Lexer(allTokens, { ensureOptimizations: true });
/**
 * Tokenize an input SQL string into a flat sequence of Chevrotain tokens.
 *
 * Errors are tolerated; if the lexer encounters unrecognized characters the caller
 * can decide how to handle the partial output.
 *
 * @example
 * ```ts
 * const [first] = tokenize("SELECT 1");
 * console.log(first.image); // SELECT
 * ```
 */
export function tokenize(input) {
    const { tokens, errors } = lexer.tokenize(input);
    if (errors.length > 0) {
        // Preserve partial tokenization; downstream code can fall back gracefully.
    }
    return tokens;
}
//# sourceMappingURL=tokenizer.js.map