time to test

src/spider/compiler/text/Token.hpp

@@ -13,7 +13,9 @@ namespace spider {
     struct TokenResult {
 
         /** @brief Indicates if the token composition successfully matched the input boundary. */
-        bool success;
+        bool success = false;
+
+        optional<std::string_view> tag;
 
         /**
          * @brief Holds the deep-copied UTF-32 matching substring upon victory.
@@ -21,13 +23,14 @@ namespace spider {
          */
         std::u32string match;
 
+        vector<TokenResult> child;
+
     };
 
-    // Forward declarations required by the abstract interface for operator returns.
-    class SeqToken;
-    class OrToken;
-    class OptToken;
-    class RepToken;
+    class Token;
+    class TokenFactory;
+
+    using FnTokenFn = std::function<bool(u32)>;
 
     /**
      * @brief Pure virtual base class defining the EBNF combinator node contract.
@@ -50,31 +53,46 @@ namespace spider {
          */
         virtual TokenResult test(TextReader& ctx) const = 0;
 
+    };
+
+    class TokenFactory {
+    private:
+        // Arena owning all created tokens
+        std::vector<std::unique_ptr<Token>> arena;
+
+        // Deduplication caches
+        std::unordered_map<std::string, const Token*> lit_cache;
+
     public:
 
-        /**
-         * @brief Chains this token and another sequentially.
-         * @return A temporary structural bridge matching both tokens sequentially.
-         */
-        virtual SeqToken operator&(const Token& tok);
+        TokenFactory() = default;
 
-        /**
-         * @brief Combines this token and another under alternation.
-         * @return A structural bridge matching either this token or the fallback selection.
-         */
-        virtual OrToken operator|(const Token& tok);
+        // Prevent copying to maintain valid internal pointers
+        TokenFactory(const TokenFactory&) = delete;
 
-        /**
-         * @brief Wraps this node in an optional layout rule.
-         * @return A structure matching zero or one instances of this current node.
-         */
-        virtual OptToken operator~();
+        TokenFactory& operator=(const TokenFactory&) = delete;
 
-        /**
-         * @brief Wraps this node in a repetitive loop match framework.
-         * @return A structure matching zero or more occurrences of this current node.
-         */
-        virtual RepToken operator*();
+    public:
+
+        // --- Primitive Constructors ---
+
+        Token* lit(std::string_view text);
+
+        Token* operator[](std::string_view text);
+
+        Token* fn(FnTokenFn predicate);
+
+        // --- Combinator Constructors ---
+
+        Token* seq(const vector<const Token*>& tokens);
+
+        Token* choice(std::string_view opts);
+
+        Token* choice(const vector<const Token*>& tokens);
+
+        Token* opt(const Token* target);
+
+        Token* rep(const Token* target);
 
     };
 
@@ -93,8 +111,6 @@ namespace spider {
          */
         LitToken(std::string_view lit);
 
-        LitToken(const char* lit);
-
         /** @brief Direct zero-conversion construction using an existing native UTF-32 literal. */
         explicit LitToken(std::u32string lit);
 
@@ -106,8 +122,6 @@ namespace spider {
         TokenResult test(TextReader& ctx) const override;
     };
 
-    using FnTokenFn = std::function<bool(u32) >;
-
     /**
      * @brief Function based token
      */
@@ -135,11 +149,10 @@ namespace spider {
          * @brief Internal contiguous layout registry storing lightweight, zero-overhead references.
          * @details Avoids heap allocation penalties by referencing static instances immutably.
          */
-        vector<ref<const Token>> tokens;
+        vector<const Token*> tokens;
 
     public:
-        /** @brief Constructs a sequence container directly out of an inline brace-enclosed listing. */
-        SeqToken(const ilist<ref<const Token>>& list);
+        SeqToken(const vector<const Token*>& tokens);
 
     public:
         /**
@@ -149,8 +162,6 @@ namespace spider {
          */
         TokenResult test(TextReader& ctx) const override;
 
-        /** @brief Intrusive override for fluid cascading sequencing (`rule1 & rule2 & rule3`). */
-        SeqToken operator&(const Token& tok) override;
     };
 
     /**
@@ -159,11 +170,10 @@ namespace spider {
     class OrToken : public Token {
     private:
         /** @brief Ordered registry of possible alternate structural paths. */
-        vector<ref<const Token>> tokens;
+        vector<const Token*> tokens;
 
     public:
-        /** @brief Constructs an alternation choice layout from brace-enclosed tokens. */
-        OrToken(const ilist<ref<const Token>>& list);
+        OrToken(const vector<const Token*>& tokens);
 
     public:
         /**
@@ -172,8 +182,6 @@ namespace spider {
          */
         TokenResult test(TextReader& ctx) const override;
 
-        /** @brief Intrusive override for cascading alternation chains (`ruleA | ruleB | ruleC`). */
-        OrToken operator|(const Token& tok) override;
     };
 
     /**
@@ -182,11 +190,11 @@ namespace spider {
     class OptToken : public Token {
     private:
         /** @brief Read-only target node reference to test optional status against. */
-        const Token& target;
+        const Token* target;
 
     public:
         /** @brief Binds the target node rule structural dependency layout wrapper. */
-        explicit OptToken(const Token& t);
+        explicit OptToken(const Token* t);
 
     public:
         /**
@@ -195,8 +203,6 @@ namespace spider {
          */
         TokenResult test(TextReader& ctx) const override;
 
-        /** @brief Stub override providing standard compliance with the base Token interface signature. */
-        OptToken operator~() override;
     };
 
     /**
@@ -205,11 +211,11 @@ namespace spider {
     class RepToken : public Token {
     private:
         /** @brief The base token node sequence layer evaluated in loops. */
-        const Token& target;
+        const Token* target;
 
     public:
         /** @brief Binds the repeated structural blueprint node wrapper configuration. */
-        explicit RepToken(const Token& t);
+        explicit RepToken(const Token* t);
 
     public:
         /**
@@ -219,8 +225,21 @@ namespace spider {
          */
         TokenResult test(TextReader& ctx) const override;
 
-        /** @brief Stub override providing standard compliance with the base Token interface signature. */
-        RepToken operator*() override;
+    };
+
+    class TagToken : public Token {
+    private:
+
+        const Token* target;
+        std::string tag_name;
+        bool flatten;
+
+    public:
+
+        TagToken(const Token* t, std::string_view tag, bool doflatten = false);
+
+        TokenResult test(TextReader& ctx) const override;
+
     };
 
 }