Lexer

SIMPLE_TOKENS = {'.' => "dot", '*' => "star", ']' => "rbracket", ',' => "comma", ':' => "colon", '@' => "current", '(' => "lparen", ')' => "rparen", '{' => "lbrace", '}' => "rbrace"}

START_IDENTIFIER = ((Set.new('a'..'z')) + (Set.new('A'..'Z'))) + Set {'_'}

VALID_IDENTIFIER = START_IDENTIFIER + (Set.new('0'..'9'))

VALID_NUMBER = Set.new('0'..'9')

WHITESPACE = Set {' ', '\t', '\n', '\r'}

chars=(chars : Array(Char))

current=(current : Char | Nil)

expression=(expression : String)

length=(length : Int32)

position=(position : Int32)

tokenize(expression : String) : Array(Token)