xiiregexbuilder

FPGA-Accelerated Regular Expression Matching Engine
git clone https://github.com/maydayv7/xiiregexbuilder

lexer.cpp (3317B)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
#include "lexer.h"
#include <stdexcept>

std::string tokenTypeToString(TokenType type) {
    switch (type) {
        case TokenType::LITERAL:   return "LITERAL";
        case TokenType::DOT:       return "DOT";
        case TokenType::STAR:      return "STAR";
        case TokenType::PLUS:      return "PLUS";
        case TokenType::QUESTION:  return "QUESTION";
        case TokenType::PIPE:      return "PIPE";
        case TokenType::LPAREN:    return "LPAREN";
        case TokenType::RPAREN:    return "RPAREN";
        case TokenType::LBRACKET:  return "LBRACKET";
        case TokenType::RBRACKET:  return "RBRACKET";
        case TokenType::END_OF_INPUT: return "END_OF_INPUT";
        default:                   return "UNKNOWN";
    }
}

Lexer::Lexer(const std::string& input, int lineNum)
    : input(input), pos(0), line(lineNum), col(1) {}

char Lexer::peek() const {
    if (isAtEnd()) return '\0';
    return input[pos];
}

char Lexer::advance() {
    char current = input[pos++];
    col++;
    return current;
}

bool Lexer::isAtEnd() const {
    return pos >= input.length();
}

std::vector<Token> Lexer::tokenize() {
    std::vector<Token> tokens;

    while (!isAtEnd()) {
        char c = peek();
        int currentCharCol = col;

        switch (c) {
            case '\\':
                advance(); // consume '\'
                if (isAtEnd()) throw std::runtime_error("Trailing backslash");
                tokens.emplace_back(TokenType::LITERAL, advance(), line, currentCharCol);
                break;
            case '.':
                tokens.emplace_back(TokenType::DOT, advance(), line, currentCharCol);
                break;
            case '*':
                tokens.emplace_back(TokenType::STAR, advance(), line, currentCharCol);
                break;
            case '+':
                tokens.emplace_back(TokenType::PLUS, advance(), line, currentCharCol);
                break;
            case '?':
                tokens.emplace_back(TokenType::QUESTION, advance(), line, currentCharCol);
                break;
            case '|':
                tokens.emplace_back(TokenType::PIPE, advance(), line, currentCharCol);
                break;
            case '(':
                tokens.emplace_back(TokenType::LPAREN, advance(), line, currentCharCol);
                break;
            case ')':
                tokens.emplace_back(TokenType::RPAREN, advance(), line, currentCharCol);
                break;
            case '[':
                tokens.emplace_back(TokenType::LBRACKET, advance(), line, currentCharCol);
                break;
            case ']':
                tokens.emplace_back(TokenType::RBRACKET, advance(), line, currentCharCol);
                break;
            default:
                if (c >= 32 && c <= 126) {
                    tokens.emplace_back(TokenType::LITERAL, advance(), line, currentCharCol);
                } else {
                    throw std::runtime_error("Invalid character at line " + std::to_string(line) + 
                                             ", column " + std::to_string(currentCharCol) + ": code " + std::to_string(static_cast<unsigned char>(c)));
                }
                break;
        }
    }

    tokens.emplace_back(TokenType::END_OF_INPUT, '\0', line, col);
    return tokens;
}