From 2d9a962cf7945c77ff1d567212153eb62bbfbe3b Mon Sep 17 00:00:00 2001 From: Kittcannon Date: Sun, 27 Sep 2026 00:14:31 -0600 Subject: [PATCH] fixed tokens --- makefile | 3 ++ msys2.ps1 | 25 +++++++++++++ src/spider/compiler/Compiler.cpp | 2 +- src/spider/compiler/assembler/AsmEBNF.cpp | 10 ++--- src/spider/compiler/text/Token.cpp | 45 +++++++++++++---------- src/spider/compiler/text/Token.hpp | 16 ++++++++ 6 files changed, 75 insertions(+), 26 deletions(-) create mode 100644 msys2.ps1 diff --git a/makefile b/makefile index c42f542..529a23c 100644 --- a/makefile +++ b/makefile @@ -5,6 +5,9 @@ #Compiler and Linker CC := g++ +# Ensure POSIX utilities in MSYS2 take precedence over Windows System32 +export PATH := /usr/bin:/ucrt64/bin:$(PATH) + #The Target Binary Program TARGET := out.exe diff --git a/msys2.ps1 b/msys2.ps1 new file mode 100644 index 0000000..48db0e3 --- /dev/null +++ b/msys2.ps1 @@ -0,0 +1,25 @@ +[CmdletBinding()] +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$Command +) + +$msysRoot = if ($env:MSYS2_ROOT) { $env:MSYS2_ROOT } else { "C:\msys64" } +$bash = Join-Path $msysRoot "usr\bin\bash.exe" + +if (-not (Test-Path -LiteralPath $bash)) { + Write-Error "MSYS2 bash was not found at: $bash" + exit 1 +} + +$env:MSYSTEM = "UCRT64" +$env:CHERE_INVOKING = "1" + +if ($Command.Count -eq 0) { + & $bash --login +} else { + $cmdString = $Command -join " " + & $bash -lc $cmdString +} + +exit $LASTEXITCODE diff --git a/src/spider/compiler/Compiler.cpp b/src/spider/compiler/Compiler.cpp index b86fa33..589a1a2 100644 --- a/src/spider/compiler/Compiler.cpp +++ b/src/spider/compiler/Compiler.cpp @@ -101,7 +101,7 @@ void test_literals(TokenFactory& tf) { void test_addressing_modes(TokenFactory& tf) { std::cout << "\n--- Testing Addressing Modes ---\n"; run_test_case(asm_ebnf::register_tok, "R0", true); - run_test_case(asm_ebnf::register_tok, "R15", true); + run_test_case(asm_ebnf::register_tok, "R15", false); // R15 does not exist! run_test_case(asm_ebnf::addrm_ind, "[ 0x1000 ]", true); run_test_case(asm_ebnf::addrm_ptr, "[R1]", true); diff --git a/src/spider/compiler/assembler/AsmEBNF.cpp b/src/spider/compiler/assembler/AsmEBNF.cpp index a160364..85413eb 100644 --- a/src/spider/compiler/assembler/AsmEBNF.cpp +++ b/src/spider/compiler/assembler/AsmEBNF.cpp @@ -9,7 +9,7 @@ namespace spider::asm_ebnf { } bool isWhithespaceCharNotCrLf(u32 ch) { - return ch == u32(' '); + return ch == u32(' ') || ch == u32('\t'); } bool isUTF8CharNotCrLf(u32 ch) { @@ -17,11 +17,11 @@ namespace spider::asm_ebnf { } bool isUTF8CharLitCont(u32 ch) { - return ch != u32('\''); + return ch != u32('\'') && ch != u32('\\') && ch != u32('\r') && ch != u32('\n'); } bool isUTF8StringLitCont(u32 ch) { - return ch != u32('"'); + return ch != u32('"') && ch != u32('\\') && ch != u32('\r') && ch != u32('\n'); } const Token* letter; @@ -151,12 +151,12 @@ namespace spider::asm_ebnf { octal_lit = tf.tag(tf.seq({ tf.opt(sign), tf["0c"], octal_digit, tf.rep(octal_digit) }), "octal_lit", true); binary_lit = tf.tag(tf.seq({ tf.opt(sign), tf["0b"], binary_digit, tf.rep(binary_digit) }), "binary_lit", true); - literal = tf.tag(tf.choice({ float_lit, decimal_lit, hex_lit, octal_lit, binary_lit, string_lit, char_lit }), "literal"); + literal = tf.tag(tf.choice({ hex_lit, octal_lit, binary_lit, float_lit, decimal_lit, string_lit, char_lit }), "literal"); literal_cast = tf.tag(tf.seq({ tf.choice("BSILFD"), ws_optional, tf["("], ws_optional, literal, ws_optional, tf[")"] }), "literal_cast"); literal_decl = tf.tag(tf.choice({ literal, literal_cast }), "literal_decl"); // (* Operands *) - register_tok = tf.tag(tf.seq({ tf["R"], alpha_num_char }), "register", true); + register_tok = tf.tag(tf.seq({ tf["R"], alpha_num_char, tf.not_(tf.choice({ alpha_num_char, tf["_"] })) }), "register", true); addrm_ind = tf.tag(tf.seq({ tf["["], ws_optional, literal_decl, ws_optional, tf["]"] }), "addrm_ind", true); addrm_ptr = tf.tag(tf.seq({ tf["["], ws_optional, register_tok, ws_optional, tf["]"] }), "addrm_ptr", true); diff --git a/src/spider/compiler/text/Token.cpp b/src/spider/compiler/text/Token.cpp index 4f8d53c..80555b5 100644 --- a/src/spider/compiler/text/Token.cpp +++ b/src/spider/compiler/text/Token.cpp @@ -69,6 +69,13 @@ namespace spider { return t; } + Token* TokenFactory::not_(const Token* target) { + uptr p = std::make_unique(target); + auto t = p.get(); + arena.emplace_back(std::move(p)); + return t; + } + Token* TokenFactory::tag(const Token* target, std::string_view tagname, bool flatten) { uptr p = std::make_unique(target, tagname, flatten); auto t = p.get(); @@ -97,27 +104,15 @@ namespace spider { return { .success = false }; } - FnToken::FnToken(FnTokenFn chfn) : fn(chfn) {} + FnToken::FnToken(FnTokenFn chfn) : fn(std::move(chfn)) {} TokenResult FnToken::test(TextReader& ctx) const { - TokenResult r; - - for (;;) { - auto i = ctx.push(); - auto c = ctx.current(); - - if (!c || !fn(*c)) { - ctx.pop(i); - break; - } - - r.match += *c; + auto c = ctx.current(); + if (c && fn(*c)) { ctx.nextChar(); + return { .success = true, .match = std::u32string(1, char32_t(*c)) }; } - - r.success = !r.match.empty(); - if(r.success) std::cout << "[fn] matched: " << unicode::toUTF8(r.flatMatch()) << std::endl; - return r; + return { .success = false }; } // ============================================================================ @@ -166,7 +161,6 @@ namespace spider { // Short-circuit branch: return immediately on first valid choice match TokenResult res = token_ref->test(ctx); if (res.success) { - std::cout << "[or] matched: " << unicode::toUTF8(res.flatMatch()) << std::endl; return res; } @@ -188,7 +182,6 @@ namespace spider { TokenResult res = target->test(ctx); if (res.success) { - std::cout << "[~] matched: " << unicode::toUTF8(res.flatMatch()) << std::endl; return res; // Option matched exactly 1 instance successfully } @@ -198,6 +191,19 @@ namespace spider { return { .success = true }; } + // ============================================================================ + // NotToken Implementation + // ============================================================================ + + NotToken::NotToken(const Token* t) : target(t) {} + + TokenResult NotToken::test(TextReader& ctx) const { + auto i = ctx.push(); + TokenResult res = target->test(ctx); + ctx.pop(i); + return { .success = !res.success }; + } + // ============================================================================ // RepToken Implementation // ============================================================================ @@ -225,7 +231,6 @@ namespace spider { // Repetition rules (* token) always evaluate to successful // completion state, even with 0 matches. r.success = true; - std::cout << "[*] matched: " << unicode::toUTF8(r.flatMatch()) << std::endl; return r; } diff --git a/src/spider/compiler/text/Token.hpp b/src/spider/compiler/text/Token.hpp index 84e5f3b..b3fbdb6 100644 --- a/src/spider/compiler/text/Token.hpp +++ b/src/spider/compiler/text/Token.hpp @@ -98,6 +98,8 @@ namespace spider { Token* rep(const Token* target); + Token* not_(const Token* target); + Token* tag(const Token* target, std::string_view tagname, bool flatten = false); }; @@ -211,6 +213,20 @@ namespace spider { }; + /** + * @brief Negative lookahead combinator (NotToken). + */ + class NotToken : public Token { + private: + const Token* target; + + public: + explicit NotToken(const Token* t); + + TokenResult test(TextReader& ctx) const override; + + }; + /** * @brief Evaluates zero-to-infinite loops of a single repetitive match signature (RepToken). */