diff --git a/samples/assembly.ebnf b/samples/assembly.ebnf index fb9f967..af8feec 100644 --- a/samples/assembly.ebnf +++ b/samples/assembly.ebnf @@ -22,7 +22,7 @@ char_lit = "'", char_content, "'" ; string_char = char_escape | ? isUTF8StringLitCont ? ; (* Not " or \ *) string_lit = '"', { string_char }, '"' ; -(* Literals *) +(* Literals | Tokenization starts here *) identifier = ( letter | "_" ) , { alpha_num_char | "_" } ; comment = ";" , { utf8_char } ; diff --git a/samples/factorial.spasm b/samples/factorial.spasm index be12b4d..e7e7c87 100644 --- a/samples/factorial.spasm +++ b/samples/factorial.spasm @@ -1,4 +1,7 @@ @asm +@include("file") +#include "file" + .data .code MOV RA, 1 diff --git a/src/spider/compiler/assembler/AsmParser.hpp b/src/spider/compiler/assembler/AsmParser.hpp index 3f06c67..b5b07a9 100644 --- a/src/spider/compiler/assembler/AsmParser.hpp +++ b/src/spider/compiler/assembler/AsmParser.hpp @@ -42,9 +42,9 @@ namespace spider { bool parse_comment(); - bool parse_identifier(std::string& out_id); + bool parse_identifier(std::u32string& out_id); - bool parse_string_lit(std::string& out_str); + bool parse_string_lit(std::u32string& out_str); bool parse_char_lit(u32& out_char); diff --git a/src/spider/compiler/assembler/Assembler.hpp b/src/spider/compiler/assembler/Assembler.hpp index 9a15797..6534f29 100644 --- a/src/spider/compiler/assembler/Assembler.hpp +++ b/src/spider/compiler/assembler/Assembler.hpp @@ -3,8 +3,7 @@ #include #include - -#include +#include namespace spider { @@ -22,7 +21,7 @@ namespace spider { public: set fstack; - RootToken root; + CompToken root; public: diff --git a/src/spider/compiler/common.hpp b/src/spider/compiler/common.hpp index f9bba5e..47109f3 100644 --- a/src/spider/compiler/common.hpp +++ b/src/spider/compiler/common.hpp @@ -53,4 +53,9 @@ namespace spider { : byteoff(_byteoff), line(_line), col(_col) {} }; + struct range { + isize line0, col0, line1, col1; + isize byteoff0, byteoff1; + }; + } diff --git a/src/spider/compiler/text/Token.hpp b/src/spider/compiler/text/Token.hpp new file mode 100644 index 0000000..d6f9aee --- /dev/null +++ b/src/spider/compiler/text/Token.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include + +namespace spider { + + struct LitToken { + public: + const range at; + const std::u32string type; + const std::u32string value; + }; + + struct SimpleToken { + public: + const range at; + const std::u32string type; + }; + + struct CompToken; + + using Token = std::variant; + + struct CompToken { + public: + const range at; + const std::u32string type; + vector inners; + }; + +} diff --git a/src/spider/compiler/tokens/RootToken.cpp b/src/spider/compiler/tokens/RootToken.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/src/spider/compiler/tokens/RootToken.hpp b/src/spider/compiler/tokens/RootToken.hpp deleted file mode 100644 index e74ed68..0000000 --- a/src/spider/compiler/tokens/RootToken.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include - -namespace spider { - - /** - * Defines the root of a token. - */ - class RootToken { - private: - - public: - - RootToken(); - - ~RootToken(); - - public: - - void token(); - - }; - -} diff --git a/src/spider/compiler/tokens/Token.cpp b/src/spider/compiler/tokens/Token.cpp deleted file mode 100644 index 86b759f..0000000 --- a/src/spider/compiler/tokens/Token.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "Token.hpp" - -namespace spider { - - Token::Token(pos _at, TokenType _type, std::string _str) - : at(_at), type(_type), str(_str) {} - - Token::Token(const Token& tok) - : at(tok.at), type(tok.type), str(tok.str), inner(tok.inner) {} - - Token::Token(Token&& tok) - : at(tok.at), type(tok.type), str(tok.str), inner(std::move(tok.inner)) {} - - void Token::append(const Token& tok) { - inner.push_back(tok); - } - - vector Token::getInner() { - return inner; - } - - isize Token::innerCount() { - return inner.size(); - } - -} diff --git a/src/spider/compiler/tokens/Token.hpp b/src/spider/compiler/tokens/Token.hpp deleted file mode 100644 index 27a99d4..0000000 --- a/src/spider/compiler/tokens/Token.hpp +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#include - -namespace spider { - - /** - * Token type. - */ - enum class TokenType { - // Assembly - PREPROCESSOR_TAG, - WHITESPACE, NEWLINE, - INSTRUCTION, OPCODE, - OPERATOR, OPERAND, - REGISTER, NUMBER, BIN_NUMBER, OCT_NUMBER, HEX_NUMBER, ADDR_NUMBER, - BRACKET, BRACKET_IND, BRACKET_PTR, BRACKET_IDX, COMMA, COMMENT, - SECTION, VARIABLE, ASSIGNMENT - - // Classic - // Script - }; - - /** - * Defines a general token. - */ - class Token { - public: - - const pos at; - const TokenType type; - const std::string str; - - private: - - vector inner; - - public: - - Token(pos _at, TokenType _type, std::string _str); - - Token(const Token& tok); - - Token(Token&& tok); - - public: - - void append(const Token& tok); - - vector getInner(); - - isize innerCount(); - - }; - -}