v2/src/lexer/lexer.c

#include "lexer.h"
#include "SolsLiteral.h"
#include "SolsToken.h"
#include "../include/error.h"
#include "../include/estr.h"
#include "../include/ansii.h"
#include <ctype.h>

struct _SolsTokenTypeMap SolsTokenTypeMap[] = {
    {"puts", STT_KW_STRUCT},
    {"if", STT_KW_IF},
    {"while", STT_KW_WHILE},
    {"def", STT_KW_DEF},
    {"struct", STT_KW_STRUCT},
    {"{", STT_OPEN_CURLY},
    {"}", STT_CLOSE_CURLY},
    {"(", STT_OPEN_PAREN},
    {")", STT_CLOSE_PAREN},
    {"+", STT_OP_ADD},
    {"-", STT_OP_DIV},
    {"*", STT_OP_MUL},
    {"/", STT_OP_DIV},
    {"=", STT_OP_SET},
    {"+=", STT_OP_ADDTO},
    {"-=", STT_OP_SUBTO},
    {"*=", STT_OP_MULTO},
    {"/=", STT_OP_DIVTO},
    {"++", STT_OP_INCREMENT},
    {"--", STT_OP_DECREMENT},
    {"==", STT_OP_EQUAL},
    {"!=", STT_OP_INEQUAL},
    {">", STT_OP_GREATER},
    {"<", STT_OP_LESSER},
    {">=", STT_OP_EQGREATER},
    {"<=", STT_OP_EQLESSER},
    // Shh, this is our little secret
    // Your reward for actually reading the source code
    // Enable this by adding -DSUPER_SILLY_MODE to your 
    // compile flags (not recommended for production)
    #ifdef SUPER_SILLY_MODE
    {"plus", STT_OP_ADD},
    {"minus", STT_OP_SUB},
    {"times", STT_OP_MUL},
    {"dividedby", STT_OP_DIV},
    {"then", STT_OPEN_CURLY},
    {"do", STT_OPEN_CURLY},
    {"end", STT_CLOSE_CURLY},
    {"is", STT_OP_SET},
    {"equals", STT_OP_EQUAL},
    {"greaterthan", STT_OP_GREATER},
    {"lesserthan", STT_OP_LESSER},
    {"increment", STT_OP_INCREMENT},
    {"decrement", STT_OP_DECREMENT},
    {"adds", STT_OP_ADDTO},
    {"subtracts", STT_OP_SUBTO},
    {"multiplies", STT_OP_MULTO},
    {"divides", STT_OP_DIVTO},
    #endif
};

ResultType(SolsTokenType, Nothing) getTokenType(const char* input) {
    size_t mapsize = sizeof(SolsTokenTypeMap) / sizeof(struct _SolsTokenTypeMap);
    for (size_t i = 0; i < mapsize; i++) {
        if (strcmp(input, SolsTokenTypeMap[i].str) == 0) {
            return Success(SolsTokenType, Nothing, SolsTokenTypeMap[i].type);
        }
    }
    return Error(SolsTokenType, Nothing, {});
}


ResultType(SolsLexer, charptr) createLexer(char* input) {

    // Copy input into the new lexer struct
    char* inputcopy = malloc(strlen(input) + 1);
    if (inputcopy == NULL) {
        return Error(SolsLexer, charptr, "Couldn't copy string into lexer (in createLexer() function)");
    }
    strcpy(inputcopy, input);

    // Create SolsTokens
    ResultType(SolsTokens, charptr) tokens = createSolsTokens();
    if (tokens.error) {
        Estr e = CREATE_ESTR(tokens.as.error);
        APPEND_ESTR(e, " (in createLexer() function)");
        return Error(SolsLexer, charptr, e.str);
    }

    // Construct and return lexer
    SolsLexer lexer = {
        .input = inputcopy,
        .inputsize = strlen(inputcopy),
        .output = tokens.as.success,
        .current = 0,
    };
    return Success(SolsLexer, charptr, lexer);
}

ResultType(char, Nothing) lexerPeek(SolsLexer* lexer, size_t ahead) {

    // Bounds and null checking
    if (lexer->input == NULL) {
        return Error(char, Nothing, {});
    }
    if (lexer->current + ahead > lexer->inputsize) {
        return Error(char, Nothing, {});
    }

    // Char is within bounds, return it
    return Success(char, Nothing, lexer->input[lexer->current + ahead]);
}


ResultType(char, Nothing) lexerConsume(SolsLexer* lexer) {

    // Bounds and null checking
    if (lexer->input == NULL) {
        return Error(char, Nothing, {});
    }
    if (lexer->current + 1 > lexer->inputsize) {
        return Error(char, Nothing, {});
    }

    // Char is within bounds, return and increment
    return Success(char, Nothing, lexer->input[lexer->current++]);
}

ResultType(SolsToken, charptr) identifyToken(const char* token) {
    // Process strings
    if (token[0] == '"') {
        if (token[strlen(token) - 1] == '"') {
            // Cut out the quotes
            char* tokencopy = malloc(strlen(token) + 1);
            strncpy(tokencopy, token + 1, strlen(token) - 2);
            tokencopy[strlen(token) - 2] = '\0';

            // Create a literal
            ResultType(SolsLiteral, charptr) literal = createSolsLiteral(SLT_STRING, tokencopy);
            // Free our copy of the string, createSolsLiteral creates a copy
            free(tokencopy);
            if (literal.error) {
                Estr str = CREATE_ESTR(literal.as.error);
                APPEND_ESTR(str, " (in identifyToken() function)");
                return Error(SolsToken, charptr, str.str);
            }

            // Construct and return the token
            SolsToken tok = {
                .type = STT_LITERAL,
                .as.literal = literal.as.success
            };
            return Success(SolsToken, charptr, tok);
        }
        return Error(SolsToken, charptr, "Unterminated string (in identifyToken() function)");
    }

    // Process characters
    if (token[0] == '\'') {
        if (strlen(token) != 3) {
            return Error(SolsToken, charptr, "Characters can only hold one character at a time (try using \"this\" for strings?)");
        }
        if (token[2] == '\'') {
            ResultType(SolsLiteral, charptr) literal = createSolsLiteral(SLT_CHAR, token[1]);
            if (literal.error) {
                Estr str = CREATE_ESTR(literal.as.error);
                APPEND_ESTR(str, " (in identifyToken() function)");
                return Error(SolsToken, charptr, str.str);
            }
            SolsToken tok = {
                .type = STT_LITERAL,
                .as.literal = literal.as.success
            };
            return Success(SolsToken, charptr, tok);
        } else {
            return Error(SolsToken, charptr, "Unterminated character (in identifyToken() function)");
        }
    }

    // Process integers and floats
    if (isdigit(token[0]) || token[0] == '-') {
        size_t len = strlen(token);
        bool isInt = true;
        bool isDouble = false;
        for (size_t i = 1; i < len; i++) {
            if (isInt && token[i] == '.') {
                isInt = false;
                isDouble = true;
                continue;
            }
            if (!isdigit(token[i])) {
                isInt = false;
                isDouble = false;
            }
        }
        if (isInt) {
            int64_t newInt = atoll(token);
            ResultType(SolsLiteral, charptr) literal = createSolsLiteral(SLT_INT, newInt);
            if (literal.error) {
                Estr str = CREATE_ESTR(literal.as.error);
                APPEND_ESTR(str, " (in identifyToken() function)");
                return Error(SolsToken, charptr, str.str);
            }
            SolsToken tok = {
                .type = STT_LITERAL,
                .as.literal = literal.as.success
            };
            return Success(SolsToken, charptr, tok);
        }

        if (isDouble) {
            double newDouble = atof(token);
            ResultType(SolsLiteral, charptr) literal = createSolsLiteral(SLT_DOUBLE, newDouble);
            if (literal.error) {
                Estr str = CREATE_ESTR(literal.as.error);
                APPEND_ESTR(str, " (in identifyToken() function)");
                return Error(SolsToken, charptr, str.str);
            }
            SolsToken tok = {
                .type = STT_LITERAL,
                .as.literal = literal.as.success
            };
            return Success(SolsToken, charptr, tok);
        }
    }

    // Handle boolean (true/false)
    if (strcmp(token, "true") == 0) {
        ResultType(SolsLiteral, charptr) literal = createSolsLiteral(SLT_BOOL, true);
        if (literal.error) {
            Estr str = CREATE_ESTR(literal.as.error);
            APPEND_ESTR(str, " (in identifyToken() function)");
            return Error(SolsToken, charptr, str.str);
        }
        SolsToken tok = {
            .type = STT_LITERAL,
            .as.literal = literal.as.success
        };
        return Success(SolsToken, charptr, tok);
    }
    if (strcmp(token, "false") == 0) {
        ResultType(SolsLiteral, charptr) literal = createSolsLiteral(SLT_BOOL, false);
        if (literal.error) {
            Estr str = CREATE_ESTR(literal.as.error);
            APPEND_ESTR(str, " (in identifyToken() function)");
            return Error(SolsToken, charptr, str.str);
        }
        SolsToken tok = {
            .type = STT_LITERAL,
            .as.literal = literal.as.success
        };
        return Success(SolsToken, charptr, tok);
    }

    // Process base types
    if (strcmp(token, "int") == 0) {
        ResultType(SolsType, charptr) type = createSolsType(STT_INT);
        if (type.error) {
            Estr e = CREATE_ESTR(type.as.error);
            APPEND_ESTR(e, " (in identifyToken() function)");
            return Error(SolsToken, charptr, e.str);
        }
        SolsToken tok = {
            .type = STT_TYPE,
            .as.type = type.as.success
        };
        return Success(SolsToken, charptr, tok);
    }

    if (strcmp(token, "double") == 0) {
        ResultType(SolsType, charptr) type = createSolsType(STT_DOUBLE);
        if (type.error) {
            Estr e = CREATE_ESTR(type.as.error);
            APPEND_ESTR(e, " (in identifyToken() function)");
            return Error(SolsToken, charptr, e.str);
        }
        SolsToken tok = {
            .type = STT_TYPE,
            .as.type = type.as.success
        };
        return Success(SolsToken, charptr, tok);
    }

    if (strcmp(token, "string") == 0) {
        ResultType(SolsType, charptr) type = createSolsType(STT_STRING);
        if (type.error) {
            Estr e = CREATE_ESTR(type.as.error);
            APPEND_ESTR(e, " (in identifyToken() function)");
            return Error(SolsToken, charptr, e.str);
        }
        SolsToken tok = {
            .type = STT_TYPE,
            .as.type = type.as.success
        };
        return Success(SolsToken, charptr, tok);
    }

    if (strcmp(token, "char") == 0) {
        ResultType(SolsType, charptr) type = createSolsType(STT_CHAR);
        if (type.error) {
            Estr e = CREATE_ESTR(type.as.error);
            APPEND_ESTR(e, " (in identifyToken() function)");
            return Error(SolsToken, charptr, e.str);
        }
        SolsToken tok = {
            .type = STT_TYPE,
            .as.type = type.as.success
        };
        return Success(SolsToken, charptr, tok);
    }

    if (strcmp(token, "bool") == 0) {
        ResultType(SolsType, charptr) type = createSolsType(STT_BOOL);
        if (type.error) {
            Estr e = CREATE_ESTR(type.as.error);
            APPEND_ESTR(e, " (in identifyToken() function)");
            return Error(SolsToken, charptr, e.str);
        }
        SolsToken tok = {
            .type = STT_TYPE,
            .as.type = type.as.success
        };
        return Success(SolsToken, charptr, tok);
    }

    // Find if it's a reserved keyword/operator
    ResultType(SolsTokenType, Nothing) result = getTokenType(token);
    if (!result.error) {
        return Success(SolsToken, charptr, {result.as.success});
    }

    // No appropriate token found, it's an identifier (I hope)
    SolsToken id = {
        .type = STT_IDENTIFIER,
        .as.idName = malloc(strlen(token) + 1)
    };

    if (id.as.idName == NULL) {
        return Error(SolsToken, charptr, "Couldn't allocate memory to copy string (in identifyToken() function)");
    }
    strcpy(id.as.idName, token);

    return Success(SolsToken, charptr, id);

}

char* createParsingError(size_t lineNum, char* line, char* why) {
    Estr error = CREATE_ESTR(ESC_RESET ESC_BOLD ESC_RED_FG "Lexing Error " ESC_RESET ESC_YELLOW_FG "on line ");
    char buf[256];
    snprintf(buf, sizeof(buf), "%zu", lineNum);
    APPEND_ESTR(error, buf);
    APPEND_ESTR(error, ":\n\n" ESC_RESET ESC_BLUE_FG "    ");
    APPEND_ESTR(error, line);
    APPEND_ESTR(error, "\n\n");
    APPEND_ESTR(error, ESC_RESET ESC_MAGENTA_FG "-> ");
    APPEND_ESTR(error, why);
    APPEND_ESTR(error, "\n");
    return error.str;
}

ResultType(Nothing, charptr) lex(SolsLexer* lexer) {
    if (lexer->input == NULL) {
        return Error(Nothing, charptr, "Lexer is not initialised");
    }

    ResultType(SolsTokens, charptr) tokens = createSolsTokens();
    if (tokens.error) {
        Estr e = CREATE_ESTR(tokens.as.error);
        APPEND_ESTR(e, " (in createSolsTokens() function)");
        return Error(Nothing, charptr, e.str);
    }

    lexer->output = tokens.as.success;
    lexer->current = 0;

    Estr buf = CREATE_ESTR("");
    bool inString = false;

    size_t lineNum = 1;
    size_t lineStart = 0;
    Estr currentLine = CREATE_ESTR("");

    for (; lineStart < lexer->inputsize; lineStart++) {
        if (lexer->input[lineStart] == '\n') {
            break;
        }
        char tmp[] = {lexer->input[lineStart], '\0'};
        APPEND_ESTR(currentLine, tmp);
    }

    for (;;) {
        ResultType(char, Nothing) chr = lexerConsume(lexer);

        if (chr.error) {
            break;
        }

        if (chr.as.success == '/' && !inString) {
            ResultType(char, Nothing) peek = lexerPeek(lexer, 1);
            if (!peek.error && peek.as.success == '/') {
                chr = lexerConsume(lexer);
                for (;;) {
                    chr = lexerConsume(lexer);
                    if (chr.error) return Success(Nothing, charptr, {});
                    if (chr.as.success == '\n') {
                        chr = lexerConsume(lexer);
                        break;
                    }
                }
            } else if (!peek.error && peek.as.success == '*') {
                for (;;) {
                    chr = lexerConsume(lexer);
                    if (chr.error) return Success(Nothing, charptr, {});
                    if (chr.as.success == '*') {
                        peek = lexerPeek(lexer, 1);
                        if (!peek.error && peek.as.success == '/') {
                            chr = lexerConsume(lexer);
                            break;
                        }
                    }
                }
            }
        }
        if (chr.as.success == '#' && !inString) {
            for (;;) {
                chr = lexerConsume(lexer);
                if (chr.error) return Success(Nothing, charptr, {});
                if (chr.as.success == '\n') {
                    chr = lexerConsume(lexer);
                    break;
                }
            }
        }

        if (chr.as.success == '\n') {
            for (; lineStart < lexer->inputsize; lineStart++) {
                if (lexer->input[lineStart] == '\n') {
                    break;
                }
                char buf[] = {lexer->input[lineStart], '\0'};
                APPEND_ESTR(currentLine, buf);
            }
            lineNum ++;
        }

        if (inString) {
            char str[2] = { chr.as.success, '\0' };
            APPEND_ESTR(buf, str);
            if (chr.as.success == '"') {
                inString = false;
            }
            continue;
        }

        switch (chr.as.success) {
            case '"': {
                inString = true;
                APPEND_ESTR(buf, "\"");
                break;
            }

            // These characters require themselves added seperately from the previous token.
            case '{': 
            case '}':
            case '(':
            case ')':
            case ',':
            case ':':
            {
                if (strcmp(buf.str, "") != 0) {
                    ResultType(SolsToken, charptr) result = identifyToken(buf.str);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                    DESTROY_ESTR(buf);
                    buf = CREATE_ESTR("");
                }
                char tmp[] = {chr.as.success, '\0'};
                ResultType(SolsToken, charptr) result = identifyToken(tmp);
                if (result.error) {
                    return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                }
                addTokenToSolsTokens(&lexer->output, result.as.success);
                break;
            }

            // These characters may be repeated, or followed by an equals sign.
            case '+':
            case '-': {
                if (strcmp(buf.str, "") != 0) {
                    ResultType(SolsToken, charptr) result = identifyToken(buf.str);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                    DESTROY_ESTR(buf);
                    buf = CREATE_ESTR("");
                }
                ResultType(char, Nothing) next = lexerPeek(lexer, 1);
                if (next.error || next.as.success != chr.as.success || next.as.success != '=') {
                    char tmp[] = {chr.as.success, '\0'};
                    ResultType(SolsToken, charptr) result = identifyToken(tmp);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                }
                if (next.as.success == '=') {
                    char tmp[] = {chr.as.success, '=', '\0'};
                    ResultType(SolsToken, charptr) result = identifyToken(tmp);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                    lexerConsume(lexer);
                }
                if (next.as.success == chr.as.success) {
                    char tmp[] = {chr.as.success, chr.as.success, '\0'};
                    ResultType(SolsToken, charptr) result = identifyToken(tmp);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                    lexerConsume(lexer);
                }
                break;                
            }

            // These characters may be followed by an equals sign, or nothing else.
            case '=':
            case '!':
            case '>':
            case '<':
            case '*':
            case '/': {
                if (strcmp(buf.str, "") != 0) {
                    ResultType(SolsToken, charptr) result = identifyToken(buf.str);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                    DESTROY_ESTR(buf);
                    buf = CREATE_ESTR("");
                }
                ResultType(char, Nothing) next = lexerPeek(lexer, 1);
                if (next.error || next.as.success != '=') {
                    char tmp[] = {chr.as.success, '\0'};
                    ResultType(SolsToken, charptr) result = identifyToken(tmp);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                }
                if (next.as.success == '=') {
                    char tmp[] = {chr.as.success, '=', '\0'};
                    ResultType(SolsToken, charptr) result = identifyToken(tmp);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                    lexerConsume(lexer);
                }
                break;
            }


            // '.' requires checking whether it's a number or an identifier after
            case '.': {
                ResultType(char, Nothing) peek = lexerPeek(lexer, 1);
                if (peek.error) {
                    return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, "Expecting token after '.'"));
                }
                if (isdigit(peek.as.success)) {
                    char tmp[] = {peek.as.success, '\0'};
                    APPEND_ESTR(buf, tmp);
                    lexerConsume(lexer);
                } else {
                    if (strcmp(buf.str, "") != 0) {
                        ResultType(SolsToken, charptr) result = identifyToken(buf.str);
                        if (result.error) {
                            return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                        }
                        addTokenToSolsTokens(&lexer->output, result.as.success);
                        DESTROY_ESTR(buf);
                        buf = CREATE_ESTR("");
                    }
                    addTokenToSolsTokens(&lexer->output, (SolsToken) {.type = STT_DOT});
                }
                break;
            }

            // This whitespace splits the program and does not get appended as it's own token.
            case '\n':
            case ' ': {
                if (strcmp(buf.str, "") != 0) {
                    ResultType(SolsToken, charptr) result = identifyToken(buf.str);
                    if (result.error) {
                        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
                    }
                    addTokenToSolsTokens(&lexer->output, result.as.success);
                    DESTROY_ESTR(buf);
                    buf = CREATE_ESTR("");
                }
                break;
            }
            default: {
                char newchar[] = {chr.as.success, '\0'};
                APPEND_ESTR(buf, newchar);
                break;
            }
        }

        // Check whether we need to parse types
        if (strcmp(buf.str, "fun") == 0) {
            if (!lexerPeek(lexer, 1).error && lexerPeek(lexer, 1).as.success == '(') {
                // do stuff
            }
        }

        if (strcmp(buf.str, "template") == 0 ) {
            if (!lexerPeek(lexer, 1).error && lexerPeek(lexer, 1).as.success == '(') {

            }
        }

        if (strcmp(buf.str, "object") == 0 ) {
            if (!lexerPeek(lexer, 1).error && lexerPeek(lexer, 1).as.success == '(') {

            }
        }
    }

    if (strcmp(buf.str, "") != 0) {
        ResultType(SolsToken, charptr) result = identifyToken(buf.str);
        if (result.error) {
            return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, result.as.error));
        }
        DESTROY_ESTR(buf);
    }

    if (inString) {
        return Error(Nothing, charptr, createParsingError(lineNum, currentLine.str, "Unterminated string"));
    }

    return Success(Nothing, charptr, {});
}

ResultType(Nothing, charptr) processTypeSignature(SolsLexer* lexer) {
    return Error(Nothing, charptr, "WIP (in processTypeSignature() function)");
}