spider-compiler/src/spider/compiler/text/Token.hpp

#pragma once

#include <spider/compiler/common.hpp>

#include <spider/compiler/text/utf8.hpp>
#include <spider/compiler/text/TextReader.hpp>

namespace spider {

    /**
     * @brief The structural payload returned by every parsing component execution.
     */
    struct TokenResult {

        /** @brief Indicates if the token composition successfully matched the input boundary. */
        bool success = false;

        optional<std::string_view> tag = {};

        /**
         * @brief Holds the deep-copied UTF-32 matching substring upon victory.
         * @note Returns empty when success is false.
         */
        std::u32string match = U"";

        vector<TokenResult> child = {};

    };

    class Token;
    class TokenFactory;

    using FnTokenFn = std::function<bool(u32)>;

    /**
     * @brief Pure virtual base class defining the EBNF combinator node contract.
     * @details Implements structural immutability for thread-safe static allocations
     *          while enforcing fluency using intrusive member operator overloads.
     */
    class Token {
    public:

        /** @brief Virtual destructor ensuring clean polymorphic destruction of composite graphs. */
        virtual ~Token() = default;

    public:

        /**
         * @brief Evaluates this token node against the provided context.
         * @param ctx The current state tracker containing the UTF-32 viewing buffer and cursor.
         * @return TokenResult Containing verification state and the parsed copy of matching data.
         * @note Pure virtual; implementation details handle node-specific combinator semantics.
         */
        virtual TokenResult test(TextReader& ctx) const = 0;

    };

    class TokenFactory {
    private:
        // Arena owning all created tokens
        std::vector<std::unique_ptr<Token>> arena;

        // Deduplication caches
        std::unordered_map<std::string, Token*> lit_cache;

    public:

        TokenFactory() = default;

        // Prevent copying to maintain valid internal pointers
        TokenFactory(const TokenFactory&) = delete;

        TokenFactory& operator=(const TokenFactory&) = delete;

    public:

        // --- Primitive Constructors ---

        Token* lit(std::string_view text);

        Token* operator[](std::string_view text);

        Token* fn(FnTokenFn predicate);

        // --- Combinator Constructors ---

        Token* seq(const vector<const Token*>& tokens);

        Token* choice(std::string_view opts);

        Token* choice(const vector<const Token*>& tokens);

        Token* opt(const Token* target);

        Token* rep(const Token* target);

        Token* tag(const Token* target, std::string_view tagname, bool flatten = false);

    };

    /**
     * @brief Leaf terminal parser validating precise exact matching UTF-32 sequences.
     */
    class LitToken : public Token {
    private:
        /** @brief The reference literal sequence being inspected. */
        std::u32string literal;

    public:
        /**
         * @brief Constructs a literal rule by transforming a standard UTF-8 string view.
         * @throws std::runtime_error If incoming character boundaries contain invalid UTF-8 formatting.
         */
        LitToken(std::string_view lit);

        /** @brief Direct zero-conversion construction using an existing native UTF-32 literal. */
        explicit LitToken(std::u32string lit);

    public:
        /**
         * @brief Validates match of the backing u32string exactly at the context cursor pointer.
         * @details Advances the context cursor precisely by literal length on success; zero state mutation on failure.
         */
        TokenResult test(TextReader& ctx) const override;
    };

    /**
     * @brief Function based token
     */
    class FnToken : public Token {
    private:

        FnTokenFn fn;

    public:

        FnToken(FnTokenFn chfn);

    public:

        TokenResult test(TextReader& ctx) const override;

    };

    /**
     * @brief Evaluates an unrolled sequence of ordered grammatical rules sequentially (AndToken).
     */
    class SeqToken : public Token {
    private:
        /**
         * @brief Internal contiguous layout registry storing lightweight, zero-overhead references.
         * @details Avoids heap allocation penalties by referencing static instances immutably.
         */
        vector<const Token*> tokens;

    public:
        SeqToken(const vector<const Token*>& tokens);

    public:
        /**
         * @brief Iterates the internal sequence, enforcing that every child node must pass sequentially.
         * @details Implements a strict transaction boundary: if any internal element fails, the index
         *          backtracks entirely to its starting cursor value before returning failure.
         */
        TokenResult test(TextReader& ctx) const override;

    };

    /**
     * @brief Implements standard alternation selection matching rules (OrToken).
     */
    class OrToken : public Token {
    private:
        /** @brief Ordered registry of possible alternate structural paths. */
        vector<const Token*> tokens;

    public:
        OrToken(const vector<const Token*>& tokens);

    public:
        /**
         * @brief Scans through alternatives, resolving immediately on the first candidate that passes.
         * @details Safely rolls back changes to the context cursor point between failed alternative attempts.
         */
        TokenResult test(TextReader& ctx) const override;

    };

    /**
     * @brief Structurally represents an optional token condition rule layer (OptToken).
     */
    class OptToken : public Token {
    private:
        /** @brief Read-only target node reference to test optional status against. */
        const Token* target;

    public:
        /** @brief Binds the target node rule structural dependency layout wrapper. */
        explicit OptToken(const Token* t);

    public:
        /**
         * @brief Evaluates target presence. Returns success true regardless of sub-rule evaluation outcome.
         * @details If the nested rule fails, the context cursor rolls back to initial state, returning empty matches.
         */
        TokenResult test(TextReader& ctx) const override;

    };

    /**
     * @brief Evaluates zero-to-infinite loops of a single repetitive match signature (RepToken).
     */
    class RepToken : public Token {
    private:
        /** @brief The base token node sequence layer evaluated in loops. */
        const Token* target;

    public:
        /** @brief Binds the repeated structural blueprint node wrapper configuration. */
        explicit RepToken(const Token* t);

    public:
        /**
         * @brief Greedily attempts to match target repeatedly until an execution failure boundary is hit.
         * @details Includes internal safety loops checking cursor delta advancement to guarantee infinite
         *          empty-matching sub-loops do not cause thread lockups.
         */
        TokenResult test(TextReader& ctx) const override;

    };

    class TagToken : public Token {
    private:

        const Token* target;
        std::string tag_name;
        bool flatten;

    public:

        TagToken(const Token* t, std::string_view tag, bool doflatten);

        TokenResult test(TextReader& ctx) const override;

    };

}