"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.TreeSitterTokenizer = void 0;
class TreeSitterTokenizer {
    constructor(parser) {
        this.parser = parser;
    }
    tokenize(content) {
        const tree = this.parser.parse(content);
        return this.collectTokens(tree, content);
    }
    collectTokens(tree, code) {
        const tokens = [];
        function visit(node) {
            if (node.childCount === 0) {
                tokens.push({
                    text: code.slice(node.startIndex, node.endIndex),
                    type: node.type,
                    startIndex: node.startIndex,
                    endIndex: node.endIndex,
                    startPosition: node.startPosition,
                });
                return;
            }
            for (let i = 0; i < node.childCount; i++) {
                const child = node.child(i);
                if (child)
                    visit(child);
            }
        }
        visit(tree.rootNode);
        return tokens;
    }
}
exports.TreeSitterTokenizer = TreeSitterTokenizer;
//# sourceMappingURL=tree-sitter-tokenizer.js.map