utf-8 hexdump implmented
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "spider/compiler/common.hpp"
|
||||
#include <spider/compiler/common.hpp>
|
||||
|
||||
#include "spider/compiler/text/utf8.hpp"
|
||||
#include <spider/compiler/text/utf8.hpp>
|
||||
|
||||
namespace spider {
|
||||
|
||||
@@ -12,146 +12,203 @@ namespace spider {
|
||||
size_t cursor = 0;
|
||||
|
||||
bool has_more() const { return cursor < input.size(); }
|
||||
|
||||
char32_t peek() const { return input[cursor]; }
|
||||
|
||||
void advance(size_t n = 1) { cursor += n; }
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief The structural payload returned by every parsing component execution.
|
||||
*/
|
||||
struct TokenResult {
|
||||
|
||||
/** @brief Indicates if the token composition successfully matched the input boundary. */
|
||||
bool success;
|
||||
|
||||
/**
|
||||
* @brief Holds the deep-copied UTF-32 matching substring upon victory.
|
||||
* @note Returns empty when success is false.
|
||||
*/
|
||||
std::u32string match;
|
||||
|
||||
};
|
||||
|
||||
// Forward declarations required by the abstract interface for operator returns.
|
||||
class SeqToken;
|
||||
class OrToken;
|
||||
class OptToken;
|
||||
class RepToken;
|
||||
|
||||
/**
|
||||
* @brief Pure virtual base class defining the EBNF combinator node contract.
|
||||
* @details Implements structural immutability for thread-safe static allocations
|
||||
* while enforcing fluency using intrusive member operator overloads.
|
||||
*/
|
||||
class Token {
|
||||
public:
|
||||
|
||||
/** @brief Virtual destructor ensuring clean polymorphic destruction of composite graphs. */
|
||||
virtual ~Token() = default;
|
||||
|
||||
virtual TokenResult parse(TokenContext& ctx) const = 0;
|
||||
public:
|
||||
|
||||
/**
|
||||
* @brief Evaluates this token node against the provided context.
|
||||
* @param ctx The current state tracker containing the UTF-32 viewing buffer and cursor.
|
||||
* @return TokenResult Containing verification state and the parsed copy of matching data.
|
||||
* @note Pure virtual; implementation details handle node-specific combinator semantics.
|
||||
*/
|
||||
virtual TokenResult test(TokenContext& ctx) const = 0;
|
||||
|
||||
public:
|
||||
|
||||
/**
|
||||
* @brief Chains this token and another sequentially.
|
||||
* @return A temporary structural bridge matching both tokens sequentially.
|
||||
*/
|
||||
virtual SeqToken operator&(const Token& tok);
|
||||
|
||||
/**
|
||||
* @brief Combines this token and another under alternation.
|
||||
* @return A structural bridge matching either this token or the fallback selection.
|
||||
*/
|
||||
virtual OrToken operator|(const Token& tok);
|
||||
|
||||
/**
|
||||
* @brief Wraps this node in an optional layout rule.
|
||||
* @return A structure matching zero or one instances of this current node.
|
||||
*/
|
||||
virtual OptToken operator~();
|
||||
|
||||
/**
|
||||
* @brief Wraps this node in a repetitive loop match framework.
|
||||
* @return A structure matching zero or more occurrences of this current node.
|
||||
*/
|
||||
virtual RepToken operator*();
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Leaf terminal parser validating precise exact matching UTF-32 sequences.
|
||||
*/
|
||||
class LitToken : public Token {
|
||||
private:
|
||||
|
||||
/** @brief The reference literal sequence being inspected. */
|
||||
std::u32string literal;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Constructs a literal rule by transforming a standard UTF-8 string view.
|
||||
* @throws std::runtime_error If incoming character boundaries contain invalid UTF-8 formatting.
|
||||
*/
|
||||
explicit LitToken(std::string_view lit);
|
||||
|
||||
explicit LitToken(std::string_view lit) {
|
||||
if(!utf8::toUTF32(lit, literal)) throw std::runtime_error("Illegal UTF8 literal!");
|
||||
}
|
||||
|
||||
explicit LitToken(std::u32string lit) : literal(std::move(lit)) {}
|
||||
/** @brief Direct zero-conversion construction using an existing native UTF-32 literal. */
|
||||
explicit LitToken(std::u32string lit);
|
||||
|
||||
public:
|
||||
|
||||
TokenResult parse(TokenContext& ctx) const override {
|
||||
if (ctx.cursor + literal.size() > ctx.input.size()) return { false, {} };
|
||||
std::u32string_view sub = ctx.input.substr(ctx.cursor, literal.size());
|
||||
if (sub == literal) {
|
||||
ctx.advance(literal.size());
|
||||
return { true, std::u32string(sub) };
|
||||
}
|
||||
return { false, {} };
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Validates match of the backing u32string exactly at the context cursor pointer.
|
||||
* @details Advances the context cursor precisely by literal length on success; zero state mutation on failure.
|
||||
*/
|
||||
TokenResult test(TokenContext& ctx) const override;
|
||||
};
|
||||
|
||||
// AndToken references existing static tokens rather than managing their lifecycles
|
||||
class AndToken : public Token {
|
||||
/**
|
||||
* @brief Evaluates an unrolled sequence of ordered grammatical rules sequentially (AndToken).
|
||||
*/
|
||||
class SeqToken : public Token {
|
||||
private:
|
||||
|
||||
const Token& lhs;
|
||||
const Token& rhs;
|
||||
/**
|
||||
* @brief Internal contiguous layout registry storing lightweight, zero-overhead references.
|
||||
* @details Avoids heap allocation penalties by referencing static instances immutably.
|
||||
*/
|
||||
vector<ref<const Token>> tokens;
|
||||
|
||||
public:
|
||||
|
||||
AndToken(const Token& l, const Token& r) : lhs(l), rhs(r) {}
|
||||
/** @brief Constructs a sequence container directly out of an inline brace-enclosed listing. */
|
||||
SeqToken(const ilist<ref<const Token>>& list);
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Iterates the internal sequence, enforcing that every child node must pass sequentially.
|
||||
* @details Implements a strict transaction boundary: if any internal element fails, the index
|
||||
* backtracks entirely to its starting cursor value before returning failure.
|
||||
*/
|
||||
TokenResult test(TokenContext& ctx) const override;
|
||||
|
||||
TokenResult parse(TokenContext& ctx) const override {
|
||||
size_t start_pos = ctx.cursor;
|
||||
if (!lhs.parse(ctx).success) return { false, {} };
|
||||
if (!rhs.parse(ctx).success) {
|
||||
ctx.cursor = start_pos; // Backtrack
|
||||
return { false, {} };
|
||||
}
|
||||
return { true, std::u32string(ctx.input.substr(start_pos, ctx.cursor - start_pos)) };
|
||||
}
|
||||
/** @brief Intrusive override for fluid cascading sequencing (`rule1 & rule2 & rule3`). */
|
||||
SeqToken operator&(const Token& tok) override;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Implements standard alternation selection matching rules (OrToken).
|
||||
*/
|
||||
class OrToken : public Token {
|
||||
private:
|
||||
|
||||
const Token& lhs;
|
||||
const Token& rhs;
|
||||
/** @brief Ordered registry of possible alternate structural paths. */
|
||||
vector<ref<const Token>> tokens;
|
||||
|
||||
public:
|
||||
|
||||
OrToken(const Token& l, const Token& r) : lhs(l), rhs(r) {}
|
||||
/** @brief Constructs an alternation choice layout from brace-enclosed tokens. */
|
||||
OrToken(const ilist<ref<const Token>>& list);
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Scans through alternatives, resolving immediately on the first candidate that passes.
|
||||
* @details Safely rolls back changes to the context cursor point between failed alternative attempts.
|
||||
*/
|
||||
TokenResult test(TokenContext& ctx) const override;
|
||||
|
||||
TokenResult parse(TokenContext& ctx) const override {
|
||||
size_t start_pos = ctx.cursor;
|
||||
auto res = lhs.parse(ctx);
|
||||
if (res.success) return res;
|
||||
ctx.cursor = start_pos; // Backtrack
|
||||
return rhs.parse(ctx);
|
||||
}
|
||||
|
||||
/** @brief Intrusive override for cascading alternation chains (`ruleA | ruleB | ruleC`). */
|
||||
OrToken operator|(const Token& tok) override;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Structurally represents an optional token condition rule layer (OptToken).
|
||||
*/
|
||||
class OptToken : public Token {
|
||||
private:
|
||||
|
||||
/** @brief Read-only target node reference to test optional status against. */
|
||||
const Token& target;
|
||||
|
||||
public:
|
||||
|
||||
explicit OptToken(const Token& t) : target(t) {}
|
||||
/** @brief Binds the target node rule structural dependency layout wrapper. */
|
||||
explicit OptToken(const Token& t);
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Evaluates target presence. Returns success true regardless of sub-rule evaluation outcome.
|
||||
* @details If the nested rule fails, the context cursor rolls back to initial state, returning empty matches.
|
||||
*/
|
||||
TokenResult test(TokenContext& ctx) const override;
|
||||
|
||||
TokenResult parse(TokenContext& ctx) const override {
|
||||
size_t start_pos = ctx.cursor;
|
||||
if (target.parse(ctx).success) return {
|
||||
true,
|
||||
std::u32string(ctx.input.substr(start_pos, ctx.cursor - start_pos))
|
||||
};
|
||||
ctx.cursor = start_pos;
|
||||
return { true, std::u32string(ctx.input.substr(start_pos, 0)) };
|
||||
}
|
||||
|
||||
/** @brief Stub override providing standard compliance with the base Token interface signature. */
|
||||
OptToken operator~() override;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Evaluates zero-to-infinite loops of a single repetitive match signature (RepToken).
|
||||
*/
|
||||
class RepToken : public Token {
|
||||
private:
|
||||
|
||||
/** @brief The base token node sequence layer evaluated in loops. */
|
||||
const Token& target;
|
||||
|
||||
public:
|
||||
|
||||
explicit RepToken(const Token& t) : target(t) {}
|
||||
/** @brief Binds the repeated structural blueprint node wrapper configuration. */
|
||||
explicit RepToken(const Token& t);
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Greedily attempts to match target repeatedly until an execution failure boundary is hit.
|
||||
* @details Includes internal safety loops checking cursor delta advancement to guarantee infinite
|
||||
* empty-matching sub-loops do not cause thread lockups.
|
||||
*/
|
||||
TokenResult test(TokenContext& ctx) const override;
|
||||
|
||||
TokenResult parse(TokenContext& ctx) const override {
|
||||
size_t start_pos = ctx.cursor;
|
||||
while (ctx.has_more()) {
|
||||
size_t loop_start = ctx.cursor;
|
||||
if (!target.parse(ctx).success || ctx.cursor == loop_start) {
|
||||
ctx.cursor = loop_start;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { true, std::u32string(ctx.input.substr(start_pos, ctx.cursor - start_pos)) };
|
||||
}
|
||||
|
||||
/** @brief Stub override providing standard compliance with the base Token interface signature. */
|
||||
RepToken operator*() override;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user