summaryrefslogtreecommitdiff
path: root/tokenizer.py
diff options
context:
space:
mode:
authorAndrew <saintruler@gmail.com>2021-07-12 11:20:52 +0400
committerAndrew <saintruler@gmail.com>2021-07-12 11:20:52 +0400
commite3ae2c6c9b3a9d766b76f790ed18621765ed890d (patch)
treea923a76224991fc3fbb06b3ae83b30a832ea2477 /tokenizer.py
parenta175d139e74f52a83adb647925c3842f8ab026fb (diff)
Placed consts in classes.
Diffstat (limited to 'tokenizer.py')
-rw-r--r--tokenizer.py46
1 files changed, 23 insertions, 23 deletions
diff --git a/tokenizer.py b/tokenizer.py
index cad2752..13b824a 100644
--- a/tokenizer.py
+++ b/tokenizer.py
@@ -15,17 +15,17 @@ class Token:
return self.type == other.type and self.value == other.value
def __repr__(self):
- if self.type == LEFT_PARENTHESIS:
+ if self.type == TokenType.LEFT_PARENTHESIS:
return "Token(LEFT_PARENTHESIS)"
- if self.type == RIGHT_PARENTHESIS:
+ if self.type == TokenType.RIGHT_PARENTHESIS:
return "Token(RIGHT_PARENTHESIS)"
- if self.type == NUMBER:
+ if self.type == TokenType.NUMBER:
return f"Token({self.value})"
- if self.type == OPERATOR:
+ if self.type == TokenType.OPERATOR:
return f"Token({self.subtype})"
- if self.type == COMMA:
+ if self.type == TokenType.COMMA:
return f"Token(COMMA)"
- if self.type == SYMBOL:
+ if self.type == TokenType.SYMBOL:
return f"Token(SYMBOL {self.value})"
__str__ = __repr__
@@ -53,38 +53,38 @@ def parse_symbol(line, start):
def tokenize(line):
- state = STATE_NAME
+ state = State.NAME
tokens = []
i = 0
while i < len(line):
char = line[i]
if char == "(":
- tokens.append(Token(LEFT_PARENTHESIS, None))
- state = STATE_NAME
+ tokens.append(Token(TokenType.LEFT_PARENTHESIS, None))
+ state = State.NAME
elif char == ")":
- tokens.append(Token(RIGHT_PARENTHESIS, None))
- state = STATE_OPERATOR
+ tokens.append(Token(TokenType.RIGHT_PARENTHESIS, None))
+ state = State.OPERATOR
elif char == ",":
- tokens.append(Token(COMMA, None))
- state = STATE_NAME
+ tokens.append(Token(TokenType.COMMA, None))
+ state = State.NAME
elif char in OPERATOR_CHARS:
- if state == STATE_OPERATOR:
+ if state == State.OPERATOR:
val, i = parse_operator(line, i)
- tokens.append(Token(OPERATOR, subtype=val))
- state = STATE_NAME
- elif state == STATE_NAME:
+ tokens.append(Token(TokenType.OPERATOR, subtype=val))
+ state = State.NAME
+ elif state == State.NAME:
val, i = parse_operator(line, i)
- tokens.append(Token(OPERATOR, subtype=UNARY, value=val))
- state = STATE_NAME
+ tokens.append(Token(TokenType.OPERATOR, subtype=UNARY, value=val))
+ state = State.NAME
elif char in NUMBER_CHARS:
val, i = parse_number(line, i)
- tokens.append(Token(NUMBER, val))
- state = STATE_OPERATOR
+ tokens.append(Token(TokenType.NUMBER, val))
+ state = State.OPERATOR
elif char in SYMBOL_CHARS:
val, i = parse_symbol(line, i)
- tokens.append(Token(SYMBOL, val))
- state = STATE_OPERATOR
+ tokens.append(Token(TokenType.SYMBOL, val))
+ state = State.OPERATOR
elif char != " ":
raise ValueError("Line is not a valid expression")
i += 1