lots of improvements standby for the results
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
#include "xicu.hpp"
|
||||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
|
||||
int main() {
|
||||
printf("Testing x-icu binary tables...\n\n");
|
||||
|
||||
// Test alphanumeric
|
||||
struct AlnumTest { uint32_t cp; bool expected; const char* desc; };
|
||||
AlnumTest alnum_tests[] = {
|
||||
{ 0x0041, true, "Latin capital A" },
|
||||
{ 0x007A, true, "Latin small z" },
|
||||
{ 0x0030, true, "Digit 0" },
|
||||
{ 0x0039, true, "Digit 9" },
|
||||
{ 0x00E9, true, "Latin small e with acute" },
|
||||
{ 0x4E2D, true, "CJK ideograph" },
|
||||
{ 0x03B1, true, "Greek alpha" },
|
||||
{ 0x0040, false, "At sign" },
|
||||
{ 0x0023, false, "Hash" },
|
||||
{ 0x0020, false, "Space" },
|
||||
{ 0x0009, false, "Tab" },
|
||||
{ 0x1F600, false, "Emoji (grinning face)" },
|
||||
{ 0x0627, true, "Arabic letter alef" },
|
||||
{ 0x0915, true, "Devanagari letter ka" },
|
||||
{ 0x3042, true, "Hiragana letter a" },
|
||||
};
|
||||
|
||||
printf("=== Alphanumeric Tests ===\n");
|
||||
int passed = 0, failed = 0;
|
||||
for (const auto& t : alnum_tests) {
|
||||
bool result = xicu::isAlphanumeric(t.cp);
|
||||
if (result == t.expected) {
|
||||
printf("PASS: U+%04X (%s) -> %s\n", t.cp, t.desc, result ? "true" : "false");
|
||||
passed++;
|
||||
} else {
|
||||
printf("FAIL: U+%04X (%s) -> got %s, expected %s\n", t.cp, t.desc,
|
||||
result ? "true" : "false", t.expected ? "true" : "false");
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
// Test whitespace
|
||||
struct WsTest { uint32_t cp; bool expected; const char* desc; };
|
||||
WsTest ws_tests[] = {
|
||||
{ ' ', true, "Space" },
|
||||
{ '\t', true, "Tab" },
|
||||
{ '\n', true, "Line feed" },
|
||||
{ '\r', true, "Carriage return" },
|
||||
{ '\f', true, "Form feed" },
|
||||
{ '\v', true, "Vertical tab" },
|
||||
{ 0x1C, true, "File separator" },
|
||||
{ 0x1D, true, "Group separator" },
|
||||
{ 0x1E, true, "Record separator" },
|
||||
{ 0x1F, true, "Unit separator" },
|
||||
{ 0x85, true, "Next line (NEL)" },
|
||||
{ 0xA0, true, "No-break space" },
|
||||
{ 'A', false, "Letter A" },
|
||||
{ '0', false, "Digit 0" },
|
||||
{ '@', false, "At sign" },
|
||||
{ 0x2000, true, "En quad" },
|
||||
{ 0x2001, true, "Em quad" },
|
||||
{ 0x2002, true, "En space" },
|
||||
{ 0x2003, true, "Em space" },
|
||||
{ 0x2004, true, "Three-per-em space" },
|
||||
{ 0x2005, true, "Four-per-em space" },
|
||||
{ 0x2006, true, "Six-per-em space" },
|
||||
{ 0x2007, true, "Figure space" },
|
||||
{ 0x2008, true, "Punctuation space" },
|
||||
{ 0x2009, true, "Thin space" },
|
||||
{ 0x200A, true, "Hair space" },
|
||||
{ 0x2028, true, "Line separator" },
|
||||
{ 0x2029, true, "Paragraph separator" },
|
||||
{ 0x202F, true, "Narrow no-break space" },
|
||||
{ 0x205F, true, "Medium mathematical space" },
|
||||
{ 0x3000, true, "Ideographic space" },
|
||||
};
|
||||
|
||||
printf("\n=== Whitespace Tests ===\n");
|
||||
for (const auto& t : ws_tests) {
|
||||
bool result = xicu::isWhitespace(t.cp);
|
||||
if (result == t.expected) {
|
||||
printf("PASS: U+%04X (%s) -> %s\n", t.cp, t.desc, result ? "true" : "false");
|
||||
passed++;
|
||||
} else {
|
||||
printf("FAIL: U+%04X (%s) -> got %s, expected %s\n", t.cp, t.desc,
|
||||
result ? "true" : "false", t.expected ? "true" : "false");
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
printf("\n=== Summary ===\n");
|
||||
printf("Passed: %d\n", passed);
|
||||
printf("Failed: %d\n", failed);
|
||||
|
||||
return failed == 0 ? 0 : 1;
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,162 @@
|
||||
#include "xicu.hpp"
|
||||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
|
||||
int main() {
|
||||
printf("Testing x-icu configurable generator...\n\n");
|
||||
|
||||
// Test isLetter (bitmap)
|
||||
struct { uint32_t cp; bool expected; const char* desc; } letter_tests[] = {
|
||||
{ 'A', true, "Latin capital A" },
|
||||
{ 'z', true, "Latin small z" },
|
||||
{ '0', false, "Digit 0" },
|
||||
{ '@', false, "At sign" },
|
||||
{ ' ', false, "Space" },
|
||||
{ 0x00E9, true, "e acute" },
|
||||
{ 0x4E2D, true, "CJK ideograph" },
|
||||
{ 0x03B1, true, "Greek alpha" },
|
||||
{ 0x0627, true, "Arabic alef" },
|
||||
{ 0x1F600, false, "Emoji" },
|
||||
};
|
||||
|
||||
printf("=== isLetter (bitmap) ===\n");
|
||||
int passed = 0, failed = 0;
|
||||
for (auto& t : letter_tests) {
|
||||
bool r = xicu::isLetter(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
// Test isPunctuation (sparse_map)
|
||||
struct { uint32_t cp; bool expected; const char* desc; } punct_tests[] = {
|
||||
{ '.', true, "Period" },
|
||||
{ ',', true, "Comma" },
|
||||
{ '!', true, "Exclamation" },
|
||||
{ '?', true, "Question" },
|
||||
{ 'A', false, "Letter A" },
|
||||
{ '0', false, "Digit 0" },
|
||||
{ ' ', false, "Space" },
|
||||
{ 0x2026, true, "Ellipsis" },
|
||||
{ 0x2018, true, "Left single quote" },
|
||||
{ 0x201C, true, "Left double quote" },
|
||||
};
|
||||
|
||||
printf("\n=== isPunctuation (sparse_map) ===\n");
|
||||
for (auto& t : punct_tests) {
|
||||
bool r = xicu::isPunctuation(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
// Test isWhitespace (sparse_map)
|
||||
struct { uint32_t cp; bool expected; const char* desc; } ws_tests[] = {
|
||||
{ ' ', true, "Space" },
|
||||
{ '\t', true, "Tab" },
|
||||
{ '\n', true, "LF" },
|
||||
{ '\r', true, "CR" },
|
||||
{ 0x00A0, true, "NBSP" },
|
||||
{ 0x2000, true, "En quad" },
|
||||
{ 0x2003, true, "Em space" },
|
||||
{ 0x2028, true, "Line separator" },
|
||||
{ 0x2029, true, "Paragraph separator" },
|
||||
{ 0x3000, true, "Ideographic space" },
|
||||
{ 'A', false, "Letter A" },
|
||||
{ '0', false, "Digit 0" },
|
||||
};
|
||||
|
||||
printf("\n=== isWhitespace (sparse_map) ===\n");
|
||||
for (auto& t : ws_tests) {
|
||||
bool r = xicu::isWhitespace(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
// Test isSpace (sparse_map) - only Zs
|
||||
struct { uint32_t cp; bool expected; const char* desc; } space_tests[] = {
|
||||
{ ' ', true, "Space" },
|
||||
{ 0x00A0, true, "NBSP" },
|
||||
{ 0x2000, true, "En quad" },
|
||||
{ 0x2003, true, "Em space" },
|
||||
{ 0x3000, true, "Ideographic space" },
|
||||
{ '\t', false, "Tab (not Zs)" },
|
||||
{ '\n', false, "LF (not Zs)" },
|
||||
{ 'A', false, "Letter A" },
|
||||
};
|
||||
|
||||
printf("\n=== isSpace (sparse_map) ===\n");
|
||||
for (auto& t : space_tests) {
|
||||
bool r = xicu::isSpace(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
// Test isDecimal (sparse_map)
|
||||
struct { uint32_t cp; bool expected; const char* desc; } dec_tests[] = {
|
||||
{ '0', true, "Digit 0" },
|
||||
{ '9', true, "Digit 9" },
|
||||
{ 'A', false, "Letter A" },
|
||||
{ 0x0660, true, "Arabic-Indic 0" },
|
||||
{ 0x0966, true, "Devanagari 0" },
|
||||
{ 0xFF10, true, "Fullwidth 0" },
|
||||
};
|
||||
|
||||
printf("\n=== isDecimal (sparse_map) ===\n");
|
||||
for (auto& t : dec_tests) {
|
||||
bool r = xicu::isDecimal(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
// Test getDecimal_val (delta_ranges)
|
||||
struct { uint32_t cp; int expected; const char* desc; } decval_tests[] = {
|
||||
{ '0', 0, "Digit 0" },
|
||||
{ '5', 5, "Digit 5" },
|
||||
{ '9', 9, "Digit 9" },
|
||||
{ 0x0665, 5, "Arabic-Indic 5" },
|
||||
{ 'A', 0, "Letter A" },
|
||||
};
|
||||
|
||||
printf("\n=== getDecimal_val (delta_ranges) ===\n");
|
||||
for (auto& t : decval_tests) {
|
||||
int r = xicu::getDecimal_val(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s = %d\n", t.cp, t.desc, r); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
// Test getUppercase_map (sparse_map)
|
||||
struct { uint32_t cp; int expected; const char* desc; } up_tests[] = {
|
||||
{ 'a', 'A' - 'a', "a -> A" },
|
||||
{ 'z', 'Z' - 'z', "z -> Z" },
|
||||
{ 0x00E9, 0x00C9 - 0x00E9, "e acute -> E acute" },
|
||||
{ 'A', 0, "A (already upper)" },
|
||||
{ '0', 0, "Digit 0" },
|
||||
};
|
||||
|
||||
printf("\n=== getUppercase_map (sparse_map) ===\n");
|
||||
for (auto& t : up_tests) {
|
||||
int r = xicu::getUppercase_map(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s = %d\n", t.cp, t.desc, r); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
// Test getLowercase_map (sparse_map)
|
||||
struct { uint32_t cp; int expected; const char* desc; } low_tests[] = {
|
||||
{ 'A', 'a' - 'A', "A -> a" },
|
||||
{ 'Z', 'z' - 'Z', "Z -> z" },
|
||||
{ 0x00C9, 0x00E9 - 0x00C9, "E acute -> e acute" },
|
||||
{ 'a', 0, "a (already lower)" },
|
||||
{ '0', 0, "Digit 0" },
|
||||
};
|
||||
|
||||
printf("\n=== getLowercase_map (sparse_map) ===\n");
|
||||
for (auto& t : low_tests) {
|
||||
int r = xicu::getLowercase_map(t.cp);
|
||||
if (r == t.expected) { printf("PASS: U+%04X %s = %d\n", t.cp, t.desc, r); passed++; }
|
||||
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
|
||||
}
|
||||
|
||||
printf("\n=== Summary ===\n");
|
||||
printf("Passed: %d\n", passed);
|
||||
printf("Failed: %d\n", failed);
|
||||
return failed == 0 ? 0 : 1;
|
||||
}
|
||||
Binary file not shown.
+259
@@ -0,0 +1,259 @@
|
||||
#include "xicu.hpp"
|
||||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <string_view>
|
||||
|
||||
namespace xicu {
|
||||
|
||||
static constexpr uint32_t MAGIC = 0x58494355;
|
||||
static constexpr uint8_t VERSION = 2;
|
||||
|
||||
PropTable::PropTable(const char* path) { load(path); }
|
||||
|
||||
PropTable::~PropTable() {
|
||||
if (owns_data_ && data_) { delete[] data_; data_ = nullptr; }
|
||||
}
|
||||
|
||||
PropTable::PropTable(PropTable&& other) noexcept
|
||||
: data_(other.data_), data_size_(other.data_size_), owns_data_(other.owns_data_) {
|
||||
other.data_ = nullptr; other.owns_data_ = false;
|
||||
}
|
||||
|
||||
PropTable& PropTable::operator=(PropTable&& other) noexcept {
|
||||
if (this != &other) {
|
||||
if (owns_data_ && data_) delete[] data_;
|
||||
data_ = other.data_; data_size_ = other.data_size_; owns_data_ = other.owns_data_;
|
||||
other.data_ = nullptr; other.owns_data_ = false;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
uint32_t PropTable::readVarint(const uint8_t*& ptr) {
|
||||
uint32_t val = 0; int shift = 0;
|
||||
while (true) {
|
||||
uint8_t b = *ptr++;
|
||||
val |= (b & 0x7F) << shift;
|
||||
if (!(b & 0x80)) break;
|
||||
shift += 7;
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
bool PropTable::load(const char* path) {
|
||||
if (owns_data_ && data_) { delete[] data_; data_ = nullptr; owns_data_ = false; }
|
||||
FILE* f = std::fopen(path, "rb");
|
||||
if (!f) return false;
|
||||
std::fseek(f, 0, SEEK_END);
|
||||
long fsize = std::ftell(f);
|
||||
std::fseek(f, 0, SEEK_SET);
|
||||
if (fsize < 6) { std::fclose(f); return false; }
|
||||
data_size_ = static_cast<size_t>(fsize);
|
||||
uint8_t* buf = new uint8_t[data_size_];
|
||||
size_t read = std::fread(buf, 1, data_size_, f);
|
||||
std::fclose(f);
|
||||
if (read != data_size_) { delete[] buf; return false; }
|
||||
data_ = buf; owns_data_ = true;
|
||||
|
||||
const uint8_t* ptr = data_;
|
||||
uint32_t magic = *reinterpret_cast<const uint32_t*>(ptr); ptr += 4;
|
||||
if (magic != MAGIC) return false;
|
||||
uint8_t version = *ptr++;
|
||||
if (version != VERSION) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Find field data pointer (points to data length varint after name)
|
||||
const uint8_t* PropTable::findFieldData(const char* name, uint8_t& out_type, uint8_t& out_strat) const {
|
||||
if (!data_) return nullptr;
|
||||
const uint8_t* ptr = data_ + 5; // magic + version
|
||||
uint32_t num_fields = readVarint(ptr);
|
||||
for (uint32_t i = 0; i < num_fields; ++i) {
|
||||
uint8_t ftype = *ptr++;
|
||||
uint8_t fstrat = *ptr++;
|
||||
const char* fname = reinterpret_cast<const char*>(ptr);
|
||||
size_t name_len = std::strlen(fname);
|
||||
ptr += name_len + 1;
|
||||
if (std::strcmp(fname, name) == 0) {
|
||||
out_type = ftype;
|
||||
out_strat = fstrat;
|
||||
return ptr; // points to data length varint
|
||||
}
|
||||
// Skip data
|
||||
uint32_t data_len = readVarint(ptr);
|
||||
ptr += data_len;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Find string field (returns pointer to indices data length)
|
||||
const uint8_t* PropTable::findStringField(const char* name) const {
|
||||
if (!data_) return nullptr;
|
||||
const uint8_t* ptr = data_ + 5;
|
||||
uint32_t num_fields = readVarint(ptr);
|
||||
// Skip non-string fields (with inline data)
|
||||
for (uint32_t i = 0; i < num_fields; ++i) {
|
||||
ptr++; ptr++; // type, strat
|
||||
while (*ptr++) {} // skip name
|
||||
uint32_t data_len = readVarint(ptr);
|
||||
ptr += data_len; // skip data
|
||||
}
|
||||
uint32_t num_str_fields = readVarint(ptr);
|
||||
for (uint32_t i = 0; i < num_str_fields; ++i) {
|
||||
const char* fname = reinterpret_cast<const char*>(ptr);
|
||||
size_t name_len = std::strlen(fname);
|
||||
ptr += name_len + 1;
|
||||
if (std::strcmp(fname, name) == 0) {
|
||||
return ptr; // points to indices data length
|
||||
}
|
||||
uint32_t data_len = readVarint(ptr);
|
||||
ptr += data_len; // skip indices data
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Decode bitmap: 1 bit per codepoint
|
||||
bool PropTable::decodeBitmap(const uint8_t* data, uint32_t cp) const {
|
||||
size_t byte_idx = cp >> 3;
|
||||
uint8_t bit = cp & 7;
|
||||
return (data[byte_idx] & (1 << bit)) != 0;
|
||||
}
|
||||
|
||||
// Decode delta ranges: (start, length, value) varint-encoded
|
||||
bool PropTable::decodeDeltaRangesBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t start = readVarint(ptr);
|
||||
uint32_t length = readVarint(ptr);
|
||||
uint8_t val = *ptr++;
|
||||
uint32_t end = start + length - 1;
|
||||
if (cp >= start && cp <= end) return val != 0;
|
||||
if (cp < start) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t PropTable::decodeDeltaRangesInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t start = readVarint(ptr);
|
||||
uint32_t length = readVarint(ptr);
|
||||
uint32_t zigzag = readVarint(ptr);
|
||||
int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));
|
||||
uint32_t end = start + length - 1;
|
||||
if (cp >= start && cp <= end) return val;
|
||||
if (cp < start) return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Decode sparse map: (cp, value) pairs
|
||||
bool PropTable::decodeSparseMapBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t c = readVarint(ptr);
|
||||
if (ptr >= data_end) break;
|
||||
uint8_t val = *ptr++;
|
||||
if (c == cp) return val != 0;
|
||||
if (c > cp) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t PropTable::decodeSparseMapInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t c = readVarint(ptr);
|
||||
if (ptr >= data_end) break;
|
||||
uint32_t zigzag = readVarint(ptr);
|
||||
int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));
|
||||
if (c == cp) return val;
|
||||
if (c > cp) return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string_view PropTable::getName(uint32_t cp) const {
|
||||
// String fields use separate .str file
|
||||
const uint8_t* indices_ptr = findStringField("name");
|
||||
if (!indices_ptr) return "";
|
||||
uint32_t indices_len = readVarint(indices_ptr);
|
||||
// Find index for this codepoint (linear scan for now)
|
||||
// TODO: optimize with binary search if needed
|
||||
// For now, we need the string pool loaded separately
|
||||
return ""; // Requires string pool file
|
||||
}
|
||||
|
||||
bool PropTable::getPunctuation(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("punctuation", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getLetter(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("letter", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeBitmap(data, cp);
|
||||
}
|
||||
|
||||
int32_t PropTable::getUppercase_map(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("uppercase_map", ftype, fstrat);
|
||||
if (!data) return 0;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapInt(data, data_end, cp);
|
||||
}
|
||||
|
||||
int32_t PropTable::getLowercase_map(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("lowercase_map", ftype, fstrat);
|
||||
if (!data) return 0;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapInt(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getWhitespace(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("whitespace", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getSpace(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("space", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getDecimal(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("decimal", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
int32_t PropTable::getDecimal_val(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("decimal_val", ftype, fstrat);
|
||||
if (!data) return 0;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeDeltaRangesInt(data, data_end, cp);
|
||||
}
|
||||
|
||||
} // namespace xicu
|
||||
@@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string_view>
|
||||
|
||||
namespace xicu {
|
||||
|
||||
class PropTable {
|
||||
public:
|
||||
PropTable() = default;
|
||||
explicit PropTable(const char* path);
|
||||
~PropTable();
|
||||
|
||||
PropTable(const PropTable&) = delete;
|
||||
PropTable& operator=(const PropTable&) = delete;
|
||||
PropTable(PropTable&& other) noexcept;
|
||||
PropTable& operator=(PropTable&& other) noexcept;
|
||||
|
||||
bool load(const char* path);
|
||||
bool isLoaded() const { return data_ != nullptr; }
|
||||
std::string_view getName(uint32_t cp) const;
|
||||
bool getPunctuation(uint32_t cp) const;
|
||||
bool getLetter(uint32_t cp) const;
|
||||
int32_t getUppercase_map(uint32_t cp) const;
|
||||
int32_t getLowercase_map(uint32_t cp) const;
|
||||
bool getWhitespace(uint32_t cp) const;
|
||||
bool getSpace(uint32_t cp) const;
|
||||
bool getDecimal(uint32_t cp) const;
|
||||
int32_t getDecimal_val(uint32_t cp) const;
|
||||
|
||||
private:
|
||||
const uint8_t* data_ = nullptr;
|
||||
size_t data_size_ = 0;
|
||||
bool owns_data_ = false;
|
||||
|
||||
static uint32_t readVarint(const uint8_t*& ptr);
|
||||
const uint8_t* findFieldData(const char* name, uint8_t& out_type, uint8_t& out_strat) const;
|
||||
const uint8_t* findStringField(const char* name) const;
|
||||
|
||||
// Decoders
|
||||
bool decodeBitmap(const uint8_t* data, uint32_t cp) const;
|
||||
bool decodeDeltaRangesBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
|
||||
int32_t decodeDeltaRangesInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
|
||||
bool decodeSparseMapBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
|
||||
int32_t decodeSparseMapInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
|
||||
};
|
||||
|
||||
inline std::string_view getName(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getName(cp);
|
||||
}
|
||||
inline bool isPunctuation(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getPunctuation(cp);
|
||||
}
|
||||
inline bool isLetter(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getLetter(cp);
|
||||
}
|
||||
inline int32_t getUppercase_map(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getUppercase_map(cp);
|
||||
}
|
||||
inline int32_t getLowercase_map(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getLowercase_map(cp);
|
||||
}
|
||||
inline bool isWhitespace(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getWhitespace(cp);
|
||||
}
|
||||
inline bool isSpace(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getSpace(cp);
|
||||
}
|
||||
inline bool isDecimal(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getDecimal(cp);
|
||||
}
|
||||
inline int32_t getDecimal_val(uint32_t cp) {
|
||||
static PropTable table("../out/xicu.bin");
|
||||
return table.getDecimal_val(cp);
|
||||
}
|
||||
|
||||
} // namespace xicu
|
||||
BIN
Binary file not shown.
Reference in New Issue
Block a user