# All files / src/tokenizer index.js

100% Statements 49/49
100% Branches 16/16
100% Functions 10/10
100% Lines 45/45

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    26x
    26x
    26x
    26x
    26x
    26x
    26x
    26x

    122x
    122x
    122x
    122x

    7437x
    7437x
    7437x
    7437x

    7437x
    7437x

    7437x
    23627x
    23627x
    23627x
    23627x
    23627x
    23627x

    7437x
    40x

    7397x
    7397x
    7397x

    7397x
    7332x

    7397x

    160094x

    7397x
    3102x
    1539x
    2984x

    7397x

    7397x

    7437x
    7574x

    121x
    7436x

    121x

     // @flow
    import Stream from "../utils/stream";
    import punctuator from "./punctuator";
    import constant from "./constant";
    import identifier from "./identifier";
    import keyword from "./keyword";
    import string from "./string";
    import comments from "./comments";
    import type from "./type";

    type Parsers = ((string) => any)[];

    class Tokenizer {
      stream: Stream;
      tokens: Array<any>;
      pos: number;
      parsers: Parsers;

      constructor(
        stream: Stream,
        parsers: Parsers = [
          punctuator,
          constant,
          identifier,
          keyword,
          string,
          type,
          comments,
        ]
      ) {
        this.stream = stream;
        this.tokens = [];
        this.pos = 0;
        this.parsers = parsers;
      }

      /**
       * Get next token
       */
      next() {
        let value = "";
        this.seekNonWhitespace();
        let char = "";
        let matchers = this.parsers;
        let next;
        let nextMatchers = this.match(char, matchers);
        let start = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };

        do {
          char = this.stream.peek();
          matchers = this.match(char, matchers);
          value += char;
          this.stream.next();
          next = this.stream.peek();
          nextMatchers = this.match(next, matchers);
        } while (!this.stream.eof(next) && nextMatchers.length > 0);

        // If we fell off the end then bail out
        if (this.stream.eof(value)) {
          return null;
        }

        const token = this.token(value, matchers);
        token.start = start;
        token.end = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };
        // Comments are ignored for now
        if (token.type !== comments.type) {
          this.tokens.push(token);
        }

        return this.tokens[this.pos++];
      }

      match(char: string, parsers: Parsers) {
        return parsers.map(parse => parse(char)).filter(p => p);
      }

      /**
       * Match a particular non-whitespace value to a token
       */
      token(
        value: string,
        parsers: Parsers,
        token: { type: string, value: string, start: {}, end: {} } = {
          type: "unknown",
          value,
          start: {},
          end: {},
        }
      ) {
        // Strict parsers must end on a leaf node
        if (parsers.length > 1) {
          parsers = parsers.filter(parser => (parser.strict ? parser.leaf : true));
          if (parsers.length > 1) {
            parsers = parsers.filter(parser => parser.strict);
          }
        }

        token.type = parsers.pop().type;

        return token;
      }

      /**
       * Seek Stream until next non-whitespace character. Can end in eof/eol
       */
      seekNonWhitespace() {
        while (this.stream.peek() && Stream.whitespace(this.stream.peek())) {
          this.stream.next();
        }
      }

      parse() {
        while (!this.stream.eof(this.stream.peek())) {
          this.next();
        }
        return this.tokens;
      }
    }

    export default Tokenizer;