compiles! Still working on tokens

src/spider/compiler/text/Token.hpp

@@ -1,33 +1,157 @@
 #pragma once
 
-#include <spider/compiler/common.hpp>
+#include "spider/compiler/common.hpp"
 
-#include <variant>
+#include "spider/compiler/text/utf8.hpp"
 
 namespace spider {
 
-    struct LitToken {
-    public:
-        const range at;
-        const std::u32string type;
-        const std::u32string value;
+    struct TokenContext {
+
+        std::u32string_view input;
+        size_t cursor = 0;
+
+        bool has_more() const { return cursor < input.size(); }
+
+        char32_t peek() const { return input[cursor]; }
+
+        void advance(size_t n = 1) { cursor += n; }
+
     };
 
-    struct SimpleToken {
-    public:
-        const range at;
-        const std::u32string type;
+    struct TokenResult {
+        bool success;
+        std::u32string match;
     };
 
-    struct CompToken;
-
-    using Token = std::variant<LitToken, SimpleToken, LitToken>;
-
-    struct CompToken {
+    class Token {
     public:
-        const range at;
-        const std::u32string type;
-        vector<Token> inners;
+
+        virtual ~Token() = default;
+
+        virtual TokenResult parse(TokenContext& ctx) const = 0;
+
+    };
+
+    class LitToken : public Token {
+    private:
+
+        std::u32string literal;
+
+    public:
+
+        explicit LitToken(std::string_view lit) {
+            if(!utf8::toUTF32(lit, literal)) throw std::runtime_error("Illegal UTF8 literal!");
+        }
+
+        explicit LitToken(std::u32string lit) : literal(std::move(lit)) {}
+
+    public:
+
+        TokenResult parse(TokenContext& ctx) const override {
+            if (ctx.cursor + literal.size() > ctx.input.size()) return { false, {} };
+            std::u32string_view sub = ctx.input.substr(ctx.cursor, literal.size());
+            if (sub == literal) {
+                ctx.advance(literal.size());
+                return { true, std::u32string(sub) };
+            }
+            return { false, {} };
+        }
+
+    };
+
+    // AndToken references existing static tokens rather than managing their lifecycles
+    class AndToken : public Token {
+    private:
+
+        const Token& lhs;
+        const Token& rhs;
+
+    public:
+
+        AndToken(const Token& l, const Token& r) : lhs(l), rhs(r) {}
+
+    public:
+
+        TokenResult parse(TokenContext& ctx) const override {
+            size_t start_pos = ctx.cursor;
+            if (!lhs.parse(ctx).success) return { false, {} };
+            if (!rhs.parse(ctx).success) {
+                ctx.cursor = start_pos; // Backtrack
+                return { false, {} };
+            }
+            return { true, std::u32string(ctx.input.substr(start_pos, ctx.cursor - start_pos)) };
+        }
+    };
+
+    class OrToken : public Token {
+    private:
+
+        const Token& lhs;
+        const Token& rhs;
+
+    public:
+
+        OrToken(const Token& l, const Token& r) : lhs(l), rhs(r) {}
+
+    public:
+
+        TokenResult parse(TokenContext& ctx) const override {
+            size_t start_pos = ctx.cursor;
+            auto res = lhs.parse(ctx);
+            if (res.success) return res;
+            ctx.cursor = start_pos; // Backtrack
+            return rhs.parse(ctx);
+        }
+        
+    };
+
+    class OptToken : public Token {
+    private:
+
+        const Token& target;
+
+    public:
+
+        explicit OptToken(const Token& t) : target(t) {}
+
+    public:
+
+        TokenResult parse(TokenContext& ctx) const override {
+            size_t start_pos = ctx.cursor;
+            if (target.parse(ctx).success) return {
+                true,
+                std::u32string(ctx.input.substr(start_pos, ctx.cursor - start_pos))
+            };
+            ctx.cursor = start_pos;
+            return { true, std::u32string(ctx.input.substr(start_pos, 0)) };
+        }
+
+    };
+
+    class RepToken : public Token {
+    private:
+
+        const Token& target;
+
+    public:
+
+        explicit RepToken(const Token& t) : target(t) {}
+
+    public:
+
+        TokenResult parse(TokenContext& ctx) const override {
+            size_t start_pos = ctx.cursor;
+            while (ctx.has_more()) {
+                size_t loop_start = ctx.cursor;
+                if (!target.parse(ctx).success || ctx.cursor == loop_start) {
+                    ctx.cursor = loop_start;
+                    break;
+                }
+            }
+            return { true, std::u32string(ctx.input.substr(start_pos, ctx.cursor - start_pos)) };
+        }
+
     };
 
 }