working

samples/assembly.ebnf

@@ -22,7 +22,7 @@ char_lit         = "'", char_content, "'" ;
 string_char      = char_escape | ? isUTF8StringLitCont ? ; (* Not " or \ *)
 string_lit       = '"', { string_char }, '"' ;
 
-(* Literals *)
+(* Literals | Tokenization starts here *)
 identifier       = ( letter | "_" ) , { alpha_num_char | "_" } ;
 comment          = ";" , { utf8_char } ;
 

samples/factorial.spasm

@@ -1,4 +1,7 @@
 @asm
+@include("file")
+#include "file"
+
 .data
 .code
 MOV RA, 1

src/spider/compiler/assembler/AsmParser.hpp

@@ -42,9 +42,9 @@ namespace spider {
 
         bool parse_comment();
 
-        bool parse_identifier(std::string& out_id);
+        bool parse_identifier(std::u32string& out_id);
 
-        bool parse_string_lit(std::string& out_str);
+        bool parse_string_lit(std::u32string& out_str);
 
         bool parse_char_lit(u32& out_char);
 

src/spider/compiler/assembler/Assembler.hpp

@@ -3,8 +3,7 @@
 #include <spider/compiler/common.hpp>
 
 #include <spider/compiler/text/TextReader.hpp>
-
-#include <spider/compiler/tokens/RootToken.hpp>
+#include <spider/compiler/text/Token.hpp>
 
 namespace spider {
 
@@ -22,7 +21,7 @@ namespace spider {
     public:
 
         set<fs::path> fstack;
-        RootToken root;
+        CompToken root;
 
     public:
 

src/spider/compiler/common.hpp

@@ -53,4 +53,9 @@ namespace spider {
         : byteoff(_byteoff), line(_line), col(_col) {}
     };
 
+    struct range {
+        isize line0, col0, line1, col1;
+        isize byteoff0, byteoff1;
+    };
+
 }

src/spider/compiler/text/Token.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <spider/compiler/common.hpp>
+
+#include <variant>
+
+namespace spider {
+
+    struct LitToken {
+    public:
+        const range at;
+        const std::u32string type;
+        const std::u32string value;
+    };
+
+    struct SimpleToken {
+    public:
+        const range at;
+        const std::u32string type;
+    };
+
+    struct CompToken;
+
+    using Token = std::variant<LitToken, SimpleToken, LitToken>;
+
+    struct CompToken {
+    public:
+        const range at;
+        const std::u32string type;
+        vector<Token> inners;
+    };
+
+}

src/spider/compiler/tokens/RootToken.hpp

@@ -1,25 +0,0 @@
-#pragma once
-
-#include <spider/compiler/common.hpp>
-
-namespace spider {
-
-    /**
-     * Defines the root of a token.
-     */
-    class RootToken {
-    private:
-
-    public:
-
-        RootToken();
-
-        ~RootToken();
-
-    public:
-
-        void token();
-
-    };
-
-}

src/spider/compiler/tokens/Token.cpp

@@ -1,26 +0,0 @@
-#include "Token.hpp"
-
-namespace spider {
-
-    Token::Token(pos _at, TokenType _type, std::string _str)
-    : at(_at), type(_type), str(_str) {}
-
-    Token::Token(const Token& tok)
-    : at(tok.at), type(tok.type), str(tok.str), inner(tok.inner) {}
-
-    Token::Token(Token&& tok)
-    : at(tok.at), type(tok.type), str(tok.str), inner(std::move(tok.inner)) {}
-
-    void Token::append(const Token& tok) {
-        inner.push_back(tok);
-    }
-
-    vector<Token> Token::getInner() {
-        return inner;
-    }
-
-    isize Token::innerCount() {
-        return inner.size();
-    }
-
-}

src/spider/compiler/tokens/Token.hpp

@@ -1,56 +0,0 @@
-#pragma once
-
-#include <spider/compiler/common.hpp>
-
-namespace spider {
-
-    /**
-     * Token type.
-     */
-    enum class TokenType {
-        // Assembly
-        PREPROCESSOR_TAG,
-        WHITESPACE, NEWLINE,
-        INSTRUCTION, OPCODE,
-        OPERATOR, OPERAND,
-        REGISTER, NUMBER, BIN_NUMBER, OCT_NUMBER, HEX_NUMBER, ADDR_NUMBER,
-        BRACKET, BRACKET_IND, BRACKET_PTR, BRACKET_IDX, COMMA, COMMENT,
-        SECTION, VARIABLE, ASSIGNMENT
-
-        // Classic
-        // Script
-    };
-
-    /**
-     * Defines a general token.
-     */
-    class Token {
-    public:
-
-        const pos at;
-        const TokenType type;
-        const std::string str;
-
-    private:
-
-        vector<Token> inner;
-
-    public:
-
-        Token(pos _at, TokenType _type, std::string _str);
-
-        Token(const Token& tok);
-
-        Token(Token&& tok);
-
-    public:
-
-        void append(const Token& tok);
-
-        vector<Token> getInner();
-
-        isize innerCount();
-
-    };
-
-}