started work on lexer

lexer_token.py

@@ -0,0 +1,42 @@
+from enum import Enum
+from typing import Any
+
+
+class TokenType(Enum):
+	# Special tokens
+	EOF = "EOF"
+	ILLEGAL = "ILLEGAL"
+
+	# Data types
+	INT = "INT"
+	FLOAT = "FLOAT"
+
+	# Arithmetic symbols
+	PLUS = "PLUS"
+	MINUS = "MINUS"
+	ASTERISK = "ASTERISK"
+	SLASH = "SLASH"
+	POW = "POW"
+	MODULUS = "MODULUS"
+
+	# Symbols
+	LPAREN = "LPAREN"
+	RPAREN = "RPAREN"
+	LBRACKET = "LBRACKET"
+	RBRACKET = "RBRACKET"
+	LCURLY = "LCURLY"
+	RCURLY = "RCURLY"
+	COLON = "COLON"
+
+class Token:
+	def __init__(self, type: TokenType, literal: Any, line_no: int, position: int) -> None:
+		self.type = type
+		self.literal = literal
+		self.line_no = line_no
+		self.position = position
+
+	def __str__(self) -> str:
+		return f"token[{self.type} : {self.literal} : Line {self.line_no} : Position {self.position}]"
+	
+	def __repr__(self) -> str:
+		return str(self)