diff options
Diffstat (limited to 'tokenizer.py')
| -rw-r--r-- | tokenizer.py | 46 |
1 files changed, 23 insertions, 23 deletions
diff --git a/tokenizer.py b/tokenizer.py index cad2752..13b824a 100644 --- a/tokenizer.py +++ b/tokenizer.py @@ -15,17 +15,17 @@ class Token: return self.type == other.type and self.value == other.value def __repr__(self): - if self.type == LEFT_PARENTHESIS: + if self.type == TokenType.LEFT_PARENTHESIS: return "Token(LEFT_PARENTHESIS)" - if self.type == RIGHT_PARENTHESIS: + if self.type == TokenType.RIGHT_PARENTHESIS: return "Token(RIGHT_PARENTHESIS)" - if self.type == NUMBER: + if self.type == TokenType.NUMBER: return f"Token({self.value})" - if self.type == OPERATOR: + if self.type == TokenType.OPERATOR: return f"Token({self.subtype})" - if self.type == COMMA: + if self.type == TokenType.COMMA: return f"Token(COMMA)" - if self.type == SYMBOL: + if self.type == TokenType.SYMBOL: return f"Token(SYMBOL {self.value})" __str__ = __repr__ @@ -53,38 +53,38 @@ def parse_symbol(line, start): def tokenize(line): - state = STATE_NAME + state = State.NAME tokens = [] i = 0 while i < len(line): char = line[i] if char == "(": - tokens.append(Token(LEFT_PARENTHESIS, None)) - state = STATE_NAME + tokens.append(Token(TokenType.LEFT_PARENTHESIS, None)) + state = State.NAME elif char == ")": - tokens.append(Token(RIGHT_PARENTHESIS, None)) - state = STATE_OPERATOR + tokens.append(Token(TokenType.RIGHT_PARENTHESIS, None)) + state = State.OPERATOR elif char == ",": - tokens.append(Token(COMMA, None)) - state = STATE_NAME + tokens.append(Token(TokenType.COMMA, None)) + state = State.NAME elif char in OPERATOR_CHARS: - if state == STATE_OPERATOR: + if state == State.OPERATOR: val, i = parse_operator(line, i) - tokens.append(Token(OPERATOR, subtype=val)) - state = STATE_NAME - elif state == STATE_NAME: + tokens.append(Token(TokenType.OPERATOR, subtype=val)) + state = State.NAME + elif state == State.NAME: val, i = parse_operator(line, i) - tokens.append(Token(OPERATOR, subtype=UNARY, value=val)) - state = STATE_NAME + tokens.append(Token(TokenType.OPERATOR, subtype=UNARY, value=val)) + state = State.NAME elif char in NUMBER_CHARS: val, i = parse_number(line, i) - tokens.append(Token(NUMBER, val)) - state = STATE_OPERATOR + tokens.append(Token(TokenType.NUMBER, val)) + state = State.OPERATOR elif char in SYMBOL_CHARS: val, i = parse_symbol(line, i) - tokens.append(Token(SYMBOL, val)) - state = STATE_OPERATOR + tokens.append(Token(TokenType.SYMBOL, val)) + state = State.OPERATOR elif char != " ": raise ValueError("Line is not a valid expression") i += 1 |