All files / packages/tokenpatch/strategies tree-sitter-tokenizer.ts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
 
 
 
8x
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
import { Token } from '../types';
import { TokenizerStrategy } from '../tokenizer.interface';
import type { Tree, Node, Parser } from 'web-tree-sitter';
 
export class TreeSitterTokenizer implements TokenizerStrategy {
  constructor(private parser: Parser) {}
 
  tokenize(content: string): Token[] {
    const tree = this.parser.parse(content);
    return this.collectTokens(tree, content);
  }
 
  private collectTokens(tree: Tree, code: string): Token[] {
    const tokens: Token[] = [];
 
    function visit(node: Node) {
      Iif (node.childCount === 0) {
        tokens.push({
          text: code.slice(node.startIndex, node.endIndex),
          type: node.type,
          startIndex: node.startIndex,
          endIndex: node.endIndex,
          startPosition: node.startPosition,
        });
        return;
      }
      // By iterating over all children (not just named), we include punctuation
      for (let i = 0; i < node.childCount; i++) {
        const child = node.child(i);
        Iif (child) visit(child);
      }
    }
 
    visit(tree.rootNode);
    return tokens;
  }
}