### Hierarchy

- Tokenizer

## Index

### Constructors

- constructor

### Properties

- charStack
- currentToken
- ended
- file
- hasOpenParen
- it
- state
- tokens

### Methods

- assertNotEOF
- assertNotEnded
- close
- consume
- flushToken
- state_atChar
- state_atRuleAwaitData
- state_atRuleBegin
- state_atRuleData
- state_atRuleEnd
- state_atRuleName
- state_atRuleOpenParen
- state_text

## Constructors

### constructor

- new Tokenizer(file: string): Tokenizer

- - Defined in Tokenizer.ts:22

  #### Parameters

  - ##### file: string

  #### Returns Tokenizer

## Properties

### Private charStack

charStack: string[] = []

- Defined in Tokenizer.ts:18

### Private currentToken

currentToken: TokenBuilder

- Defined in Tokenizer.ts:22

### Private ended

ended: boolean = false

- Defined in Tokenizer.ts:20

### Private file

file: string

- Defined in Tokenizer.ts:25

### Private hasOpenParen

hasOpenParen: boolean = false

- Defined in Tokenizer.ts:19

### Private it

it: TextIterator = new TextIterator()

- Defined in Tokenizer.ts:17

### Private state

state: TokenizerState = this.state_text

- Defined in Tokenizer.ts:16

### Private tokens

tokens: Token[] = []

- Defined in Tokenizer.ts:21

## Methods

### Private assertNotEOF

- assertNotEOF(eof: boolean): void

- - Defined in Tokenizer.ts:51

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private assertNotEnded

- assertNotEnded(): void

- - Defined in Tokenizer.ts:45

  #### Returns void

### close

- close(): Token[]

- - Defined in Tokenizer.ts:37

  #### Returns Token[]

### consume

- consume(buf: string): void

- - Defined in Tokenizer.ts:29

  #### Parameters

  - ##### buf: string

  #### Returns void

### Private flushToken

- flushToken(nextToken: TokenBuilder): void

- - Defined in Tokenizer.ts:275

  #### Parameters

  - ##### nextToken: TokenBuilder

  #### Returns void

### Private state_atChar

- state_atChar(eof: boolean): void

- - Defined in Tokenizer.ts:85
  @

    ^ Here

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private state_atRuleAwaitData

- state_atRuleAwaitData(eof: boolean): void

- - Defined in Tokenizer.ts:154
  rule

    (some data) ^^^ Here

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private state_atRuleBegin

- state_atRuleBegin(eof: boolean): void

- - Defined in Tokenizer.ts:97
  rule

    (some data) ^ Here

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private state_atRuleData

- state_atRuleData(eof: boolean): void

- - Defined in Tokenizer.ts:205
  rule

    (some data) ^^^^^^^^^ Here

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private state_atRuleEnd

- state_atRuleEnd(eof: boolean): void

- - Defined in Tokenizer.ts:269
  rule

    (some data) ^ Here

  simplerule

               ^
            or here

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private state_atRuleName

- state_atRuleName(eof: boolean): void

- - Defined in Tokenizer.ts:115
  rule

    (some data) ^^^^ Here

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private state_atRuleOpenParen

- state_atRuleOpenParen(eof: boolean): void

- - Defined in Tokenizer.ts:192
  rule

    (some() data) ^ Here

  #### Parameters

  - ##### eof: boolean

  #### Returns void

### Private state_text

- state_text(eof: boolean): void

- - Defined in Tokenizer.ts:61

  #### Parameters

  - ##### eof: boolean

  #### Returns void

Generated using TypeDoc