# All files / src/tokenizer index.js

100% Statements 49/49
100% Branches 16/16
100% Functions 10/10
100% Lines 45/45

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    27x
    27x
    27x
    27x
    27x
    27x
    27x
    27x

    237x
    237x
    237x
    237x

    19710x
    19710x
    19710x
    19710x

    19710x
    19710x

    19710x
    84711x
    84711x
    84711x
    84711x
    84711x
    84711x

    19710x
    129x

    19581x
    19581x
    19581x

    19581x
    19340x

    19581x

    464120x

    19581x
    8883x
    4429x
    8762x

    19581x

    19581x

    19710x
    29303x

    236x
    19709x

    236x

     // @flow
    import Stream from '../utils/stream';
    import punctuator from './punctuator';
    import constant from './constant';
    import identifier from './identifier';
    import keyword from './keyword';
    import string from './string';
    import comments from './comments';
    import type from './type';

    type Parsers = ((string) => any)[];

    class Tokenizer {
      stream: Stream;
      tokens: Array<any>;
      pos: number;
      parsers: Parsers;

      constructor(
        stream: Stream,
        parsers: Parsers = [
          punctuator,
          constant,
          identifier,
          keyword,
          string,
          type,
          comments,
        ]
      ) {
        this.stream = stream;
        this.tokens = [];
        this.pos = 0;
        this.parsers = parsers;
      }

      /**
       * Get next token
       */
      next() {
        let value = '';
        this.seekNonWhitespace();
        let char = '';
        let matchers = this.parsers;
        let next;
        let nextMatchers = this.match(char, matchers);
        let start = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };

        do {
          char = this.stream.peek();
          matchers = this.match(char, matchers);
          value += char;
          this.stream.next();
          next = this.stream.peek();
          nextMatchers = this.match(next, matchers);
        } while (!this.stream.eof(next) && nextMatchers.length > 0);

        // If we fell off the end then bail out
        if (this.stream.eof(value)) {
          return null;
        }

        const token = this.token(value, matchers);
        token.start = start;
        token.end = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };
        // Comments are ignored for now
        if (token.type !== comments.type) {
          this.tokens.push(token);
        }

        return this.tokens[this.pos++];
      }

      match(char: string, parsers: Parsers) {
        return parsers.map(parse => parse(char)).filter(p => p);
      }

      /**
       * Match a particular non-whitespace value to a token
       */
      token(
        value: string,
        parsers: Parsers,
        token: { type: string, value: string, start: {}, end: {} } = {
          type: 'unknown',
          value,
          start: {},
          end: {},
        }
      ) {
        // Strict parsers must end on a leaf node
        if (parsers.length > 1) {
          parsers = parsers.filter(parser => (parser.strict ? parser.leaf : true));
          if (parsers.length > 1) {
            parsers = parsers.filter(parser => parser.strict);
          }
        }

        token.type = parsers.pop().type;

        return token;
      }

      /**
       * Seek Stream until next non-whitespace character. Can end in eof/eol
       */
      seekNonWhitespace() {
        while (this.stream.peek() && Stream.whitespace(this.stream.peek())) {
          this.stream.next();
        }
      }

      parse() {
        while (!this.stream.eof(this.stream.peek())) {
          this.next();
        }
        return this.tokens;
      }
    }

    export default Tokenizer;