All files / packages/tokenpatch tokens.ts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
 
10x
 
 
 
2899x
2685x
 
 
 
 
214x
 
 
10x
66x
 
 
10x
83x
82x
82x
2746x
2897x
2604x
 
 
142x
 
82x
 
import type { Token } from './types';
 
export function tokensEqual(a: Token, b: Token): boolean {
  // If using Tiktoken (BPE), we relax the comparison to ignore whitespace differences
  // and newlines by trimming and stripping them. This helps when the patch might
  // have slightly different indentation or spacing than the source.
  if (a.type === 'bpe' || b.type === 'bpe') {
    return (
      a.text.replace(/\r?\n/g, '').trim() ===
      b.text.replace(/\r?\n/g, '').trim()
    );
  }
  return a.text === b.text;
}
 
export function formatAnchor(anchor: Token[]): string {
  return anchor.map((t) => t.text).join(' ');
}
 
export function findAllSequences(haystack: Token[], needle: Token[]): number[] {
  if (needle.length === 0) return [];
  const indices: number[] = [];
  outer: for (let i = 0; i <= haystack.length - needle.length; i++) {
    for (let j = 0; j < needle.length; j++) {
      if (!tokensEqual(haystack[i + j], needle[j])) {
        continue outer;
      }
    }
    indices.push(i);
  }
  return indices;
}