Initial commit

token.py

@@ -0,0 +1,95 @@
+from enum import Enum
+
+class TokenType(Enum):
+    INTEGER = 1
+    DOUBLE = 2
+    STRING = 3
+    CHAR = 4
+    BOOLEAN = 5
+
+    FUNCTION = 6
+    VARIABLE = 7
+    IF = 8
+    ELSE = 9
+    WHILE = 10
+    LET = 11
+
+    ADD = 12
+    SUBTRACT = 13
+    MULTIPLY = 14
+    DIVIDE = 15
+
+    SET = 16
+
+    EQUAL = 17
+    GREATER = 18
+    LESSER = 19
+
+    UNKNOWN = 0
+
+class Token:
+    def __init__(self, tok: str):
+        self.value = tok
+        self.type = get_type(tok)
+
+    def __repr__(self) -> str:
+        return f"Token(type={self.type.name}, value='{self.value}')"
+
+type tokenlist = list[Token]
+
+def get_type(process: str) -> TokenType:
+    # Keywords
+    match process:
+        case "let":
+            return TokenType.LET
+        case "if":
+            return TokenType.IF
+        case "else":
+            return TokenType.ELSE
+        case "while":
+            return TokenType.WHILE
+        case "true" | "false":
+            return TokenType.BOOLEAN
+        case "+":
+            return TokenType.ADD
+        case "-":
+            return TokenType.SUBTRACT
+        case "*":
+            return TokenType.MULTIPLY
+        case "/":
+            return TokenType.DIVIDE
+        case "=":
+            return TokenType.SET
+        case "==":
+            return TokenType.EQUAL
+        case ">":
+            return TokenType.GREATER
+        case "<":
+            return TokenType.LESSER
+
+    # String/Char Literals
+    if len(process) >= 2:
+        if process.startswith('"') and process.endswith('"'):
+            return TokenType.STRING
+        if process.startswith("'") and process.endswith("'") and len(process) == 3:
+            return TokenType.CHAR
+
+    # Numeric Literals
+    if '.' in process:
+        try:
+            float(process)
+            return TokenType.DOUBLE
+        except ValueError:
+            pass
+    else:
+        try:
+            int(process)
+            return TokenType.INTEGER
+        except ValueError:
+            pass
+
+    # Identifiers (Variables/Functions)
+    if process and (process[0].isalpha() or process[0] == '_') and all(c.isalnum() or c == '_' for c in process):
+        return TokenType.VARIABLE
+
+    return TokenType.UNKNOWN