77 lines
1.3 KiB
Python
77 lines
1.3 KiB
Python
from enum import Enum
|
|
from typing import Any
|
|
|
|
|
|
class TokenType(Enum):
|
|
# Special tokens
|
|
EOF = "EOF"
|
|
ILLEGAL = "ILLEGAL"
|
|
|
|
# Data types
|
|
IDENT = "IDENT"
|
|
INT = "INT"
|
|
FLOAT = "FLOAT"
|
|
|
|
# Arithmetic symbols
|
|
PLUS = "PLUS"
|
|
MINUS = "MINUS"
|
|
ASTERISK = "ASTERISK"
|
|
SLASH = "SLASH"
|
|
POW = "POW"
|
|
MODULUS = "MODULUS"
|
|
|
|
# Assignment symbols
|
|
EQ = "EQ"
|
|
|
|
# Symbols
|
|
LPAREN = "LPAREN"
|
|
RPAREN = "RPAREN"
|
|
LBRACKET = "LBRACKET"
|
|
RBRACKET = "RBRACKET"
|
|
LCURLY = "LCURLY"
|
|
RCURLY = "RCURLY"
|
|
COLON = "COLON"
|
|
SEMICOLON = "SEMICOLON"
|
|
|
|
# Keywords
|
|
|
|
# Typing
|
|
TYPE = "TYPE"
|
|
|
|
class Token:
|
|
def __init__(self, type: TokenType, literal: Any, line_no: int, position: int) -> None:
|
|
self.type = type
|
|
self.literal = literal
|
|
self.line_no = line_no
|
|
self.position = position
|
|
|
|
def __str__(self) -> str:
|
|
return f"token[{self.type} : {self.literal} : Line {self.line_no} : Position {self.position}]"
|
|
|
|
def __repr__(self) -> str:
|
|
return str(self)
|
|
|
|
|
|
KEYWORDS: dict[str, TokenType] = {
|
|
|
|
}
|
|
|
|
ALT_KEYWORDS: dict[str, TokenType] = {
|
|
|
|
}
|
|
|
|
TYPE_KEYWORDS: list[str] = ["Int", "Float", "String", "Bool", "List", "Nil"]
|
|
|
|
def lookup_ident(ident: str) -> TokenType:
|
|
tt: TokenType | None = KEYWORDS.get(ident)
|
|
if tt is not None:
|
|
return tt
|
|
|
|
tt: TokenType | None = ALT_KEYWORDS.get(ident)
|
|
if tt is not None:
|
|
return tt
|
|
|
|
if ident in TYPE_KEYWORDS:
|
|
return TokenType.TYPE
|
|
|
|
return TokenType.IDENT |