118 lines
2.2 KiB
Python
118 lines
2.2 KiB
Python
from enum import Enum
|
|
from typing import Any
|
|
|
|
|
|
class TokenType(Enum):
|
|
# Special tokens
|
|
EOF = "EOF"
|
|
ILLEGAL = "ILLEGAL"
|
|
|
|
# Data types
|
|
IDENT = "IDENT"
|
|
INT = "INT"
|
|
FLOAT = "FLOAT"
|
|
STRING = "STRING"
|
|
|
|
# Arithmetic symbols
|
|
PLUS = "PLUS"
|
|
MINUS = "MINUS"
|
|
ASTERISK = "ASTERISK"
|
|
SLASH = "SLASH"
|
|
POW = "POW"
|
|
MODULUS = "MODULUS"
|
|
|
|
# Assignment symbols
|
|
EQ = "EQ"
|
|
PLUS_EQ = "PLUS_EQ"
|
|
MINUS_EQ = "MINUS_EQ"
|
|
MUL_EQ = "MUL_EQ"
|
|
DIV_EQ = "DIV_EQ"
|
|
|
|
# Comparison symbols
|
|
LT = "<"
|
|
GT = ">"
|
|
EQ_EQ = "=="
|
|
LT_EQ = "<="
|
|
GT_EQ = ">="
|
|
NOT_EQ = "!="
|
|
|
|
# Symbols
|
|
LPAREN = "LPAREN"
|
|
RPAREN = "RPAREN"
|
|
LBRACKET = "LBRACKET"
|
|
RBRACKET = "RBRACKET"
|
|
LBRACE = "LBRACE"
|
|
RBRACE = "RBRACE"
|
|
COLON = "COLON"
|
|
SEMICOLON = "SEMICOLON"
|
|
COMMA = "COMMA"
|
|
DOLLARSIGN = "DOLLARSIGN"
|
|
|
|
# Prefix symbols
|
|
BANG = "BANG"
|
|
|
|
# Postfix symbols
|
|
PLUS_PLUS = "PLUS_PLUS"
|
|
MINUS_MINUS = "MINUS_MINUS"
|
|
|
|
# Keywords
|
|
RETURN = "RETURN"
|
|
IF = "IF"
|
|
UNLESS = "UNLESS"
|
|
TRUE = "TRUE"
|
|
FALSE = "FALSE"
|
|
WHILE = "WHILE"
|
|
CONTINUE = "CONTINUE"
|
|
BREAK = "BREAK"
|
|
FOR = "FOR"
|
|
DEPEND = "DEPEND"
|
|
|
|
# Typing
|
|
TYPE = "TYPE"
|
|
|
|
class Token:
|
|
def __init__(self, type: TokenType, literal: Any, line_no: int, position: int) -> None:
|
|
self.type = type
|
|
self.literal = literal
|
|
self.line_no = line_no
|
|
self.position = position
|
|
|
|
def __str__(self) -> str:
|
|
return f"token[{self.type} : {self.literal} : Line {self.line_no} : Position {self.position}]"
|
|
|
|
def __repr__(self) -> str:
|
|
return str(self)
|
|
|
|
|
|
KEYWORDS: dict[str, TokenType] = {
|
|
"return": TokenType.RETURN,
|
|
"if": TokenType.IF,
|
|
"unless": TokenType.UNLESS,
|
|
"true": TokenType.TRUE,
|
|
"false": TokenType.FALSE,
|
|
"while": TokenType.WHILE,
|
|
"break": TokenType.BREAK,
|
|
"continue": TokenType.CONTINUE,
|
|
"for": TokenType.FOR,
|
|
"depend": TokenType.DEPEND
|
|
}
|
|
|
|
ALT_KEYWORDS: dict[str, TokenType] = {
|
|
"else": TokenType.UNLESS
|
|
}
|
|
|
|
TYPE_KEYWORDS: list[str] = ["Int", "Float", "String", "Bool", "List", "Nil", "Func"]
|
|
|
|
def lookup_ident(ident: str) -> TokenType:
|
|
tt: TokenType | None = KEYWORDS.get(ident)
|
|
if tt is not None:
|
|
return tt
|
|
|
|
tt: TokenType | None = ALT_KEYWORDS.get(ident)
|
|
if tt is not None:
|
|
return tt
|
|
|
|
if ident in TYPE_KEYWORDS:
|
|
return TokenType.TYPE
|
|
|
|
return TokenType.IDENT |