lots of improvements standby for the results

This commit is contained in:
2026-09-27 01:09:57 -06:00
parent 812ac62393
commit 150e5e8636
16 changed files with 2726 additions and 9 deletions
+96
View File
@@ -0,0 +1,96 @@
#include "xicu.hpp"
#include <cstdio>
#include <cstdint>
int main() {
printf("Testing x-icu binary tables...\n\n");
// Test alphanumeric
struct AlnumTest { uint32_t cp; bool expected; const char* desc; };
AlnumTest alnum_tests[] = {
{ 0x0041, true, "Latin capital A" },
{ 0x007A, true, "Latin small z" },
{ 0x0030, true, "Digit 0" },
{ 0x0039, true, "Digit 9" },
{ 0x00E9, true, "Latin small e with acute" },
{ 0x4E2D, true, "CJK ideograph" },
{ 0x03B1, true, "Greek alpha" },
{ 0x0040, false, "At sign" },
{ 0x0023, false, "Hash" },
{ 0x0020, false, "Space" },
{ 0x0009, false, "Tab" },
{ 0x1F600, false, "Emoji (grinning face)" },
{ 0x0627, true, "Arabic letter alef" },
{ 0x0915, true, "Devanagari letter ka" },
{ 0x3042, true, "Hiragana letter a" },
};
printf("=== Alphanumeric Tests ===\n");
int passed = 0, failed = 0;
for (const auto& t : alnum_tests) {
bool result = xicu::isAlphanumeric(t.cp);
if (result == t.expected) {
printf("PASS: U+%04X (%s) -> %s\n", t.cp, t.desc, result ? "true" : "false");
passed++;
} else {
printf("FAIL: U+%04X (%s) -> got %s, expected %s\n", t.cp, t.desc,
result ? "true" : "false", t.expected ? "true" : "false");
failed++;
}
}
// Test whitespace
struct WsTest { uint32_t cp; bool expected; const char* desc; };
WsTest ws_tests[] = {
{ ' ', true, "Space" },
{ '\t', true, "Tab" },
{ '\n', true, "Line feed" },
{ '\r', true, "Carriage return" },
{ '\f', true, "Form feed" },
{ '\v', true, "Vertical tab" },
{ 0x1C, true, "File separator" },
{ 0x1D, true, "Group separator" },
{ 0x1E, true, "Record separator" },
{ 0x1F, true, "Unit separator" },
{ 0x85, true, "Next line (NEL)" },
{ 0xA0, true, "No-break space" },
{ 'A', false, "Letter A" },
{ '0', false, "Digit 0" },
{ '@', false, "At sign" },
{ 0x2000, true, "En quad" },
{ 0x2001, true, "Em quad" },
{ 0x2002, true, "En space" },
{ 0x2003, true, "Em space" },
{ 0x2004, true, "Three-per-em space" },
{ 0x2005, true, "Four-per-em space" },
{ 0x2006, true, "Six-per-em space" },
{ 0x2007, true, "Figure space" },
{ 0x2008, true, "Punctuation space" },
{ 0x2009, true, "Thin space" },
{ 0x200A, true, "Hair space" },
{ 0x2028, true, "Line separator" },
{ 0x2029, true, "Paragraph separator" },
{ 0x202F, true, "Narrow no-break space" },
{ 0x205F, true, "Medium mathematical space" },
{ 0x3000, true, "Ideographic space" },
};
printf("\n=== Whitespace Tests ===\n");
for (const auto& t : ws_tests) {
bool result = xicu::isWhitespace(t.cp);
if (result == t.expected) {
printf("PASS: U+%04X (%s) -> %s\n", t.cp, t.desc, result ? "true" : "false");
passed++;
} else {
printf("FAIL: U+%04X (%s) -> got %s, expected %s\n", t.cp, t.desc,
result ? "true" : "false", t.expected ? "true" : "false");
failed++;
}
}
printf("\n=== Summary ===\n");
printf("Passed: %d\n", passed);
printf("Failed: %d\n", failed);
return failed == 0 ? 0 : 1;
}
BIN
View File
Binary file not shown.
+162
View File
@@ -0,0 +1,162 @@
#include "xicu.hpp"
#include <cstdio>
#include <cstdint>
int main() {
printf("Testing x-icu configurable generator...\n\n");
// Test isLetter (bitmap)
struct { uint32_t cp; bool expected; const char* desc; } letter_tests[] = {
{ 'A', true, "Latin capital A" },
{ 'z', true, "Latin small z" },
{ '0', false, "Digit 0" },
{ '@', false, "At sign" },
{ ' ', false, "Space" },
{ 0x00E9, true, "e acute" },
{ 0x4E2D, true, "CJK ideograph" },
{ 0x03B1, true, "Greek alpha" },
{ 0x0627, true, "Arabic alef" },
{ 0x1F600, false, "Emoji" },
};
printf("=== isLetter (bitmap) ===\n");
int passed = 0, failed = 0;
for (auto& t : letter_tests) {
bool r = xicu::isLetter(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
// Test isPunctuation (sparse_map)
struct { uint32_t cp; bool expected; const char* desc; } punct_tests[] = {
{ '.', true, "Period" },
{ ',', true, "Comma" },
{ '!', true, "Exclamation" },
{ '?', true, "Question" },
{ 'A', false, "Letter A" },
{ '0', false, "Digit 0" },
{ ' ', false, "Space" },
{ 0x2026, true, "Ellipsis" },
{ 0x2018, true, "Left single quote" },
{ 0x201C, true, "Left double quote" },
};
printf("\n=== isPunctuation (sparse_map) ===\n");
for (auto& t : punct_tests) {
bool r = xicu::isPunctuation(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
// Test isWhitespace (sparse_map)
struct { uint32_t cp; bool expected; const char* desc; } ws_tests[] = {
{ ' ', true, "Space" },
{ '\t', true, "Tab" },
{ '\n', true, "LF" },
{ '\r', true, "CR" },
{ 0x00A0, true, "NBSP" },
{ 0x2000, true, "En quad" },
{ 0x2003, true, "Em space" },
{ 0x2028, true, "Line separator" },
{ 0x2029, true, "Paragraph separator" },
{ 0x3000, true, "Ideographic space" },
{ 'A', false, "Letter A" },
{ '0', false, "Digit 0" },
};
printf("\n=== isWhitespace (sparse_map) ===\n");
for (auto& t : ws_tests) {
bool r = xicu::isWhitespace(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
// Test isSpace (sparse_map) - only Zs
struct { uint32_t cp; bool expected; const char* desc; } space_tests[] = {
{ ' ', true, "Space" },
{ 0x00A0, true, "NBSP" },
{ 0x2000, true, "En quad" },
{ 0x2003, true, "Em space" },
{ 0x3000, true, "Ideographic space" },
{ '\t', false, "Tab (not Zs)" },
{ '\n', false, "LF (not Zs)" },
{ 'A', false, "Letter A" },
};
printf("\n=== isSpace (sparse_map) ===\n");
for (auto& t : space_tests) {
bool r = xicu::isSpace(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
// Test isDecimal (sparse_map)
struct { uint32_t cp; bool expected; const char* desc; } dec_tests[] = {
{ '0', true, "Digit 0" },
{ '9', true, "Digit 9" },
{ 'A', false, "Letter A" },
{ 0x0660, true, "Arabic-Indic 0" },
{ 0x0966, true, "Devanagari 0" },
{ 0xFF10, true, "Fullwidth 0" },
};
printf("\n=== isDecimal (sparse_map) ===\n");
for (auto& t : dec_tests) {
bool r = xicu::isDecimal(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s\n", t.cp, t.desc); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
// Test getDecimal_val (delta_ranges)
struct { uint32_t cp; int expected; const char* desc; } decval_tests[] = {
{ '0', 0, "Digit 0" },
{ '5', 5, "Digit 5" },
{ '9', 9, "Digit 9" },
{ 0x0665, 5, "Arabic-Indic 5" },
{ 'A', 0, "Letter A" },
};
printf("\n=== getDecimal_val (delta_ranges) ===\n");
for (auto& t : decval_tests) {
int r = xicu::getDecimal_val(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s = %d\n", t.cp, t.desc, r); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
// Test getUppercase_map (sparse_map)
struct { uint32_t cp; int expected; const char* desc; } up_tests[] = {
{ 'a', 'A' - 'a', "a -> A" },
{ 'z', 'Z' - 'z', "z -> Z" },
{ 0x00E9, 0x00C9 - 0x00E9, "e acute -> E acute" },
{ 'A', 0, "A (already upper)" },
{ '0', 0, "Digit 0" },
};
printf("\n=== getUppercase_map (sparse_map) ===\n");
for (auto& t : up_tests) {
int r = xicu::getUppercase_map(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s = %d\n", t.cp, t.desc, r); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
// Test getLowercase_map (sparse_map)
struct { uint32_t cp; int expected; const char* desc; } low_tests[] = {
{ 'A', 'a' - 'A', "A -> a" },
{ 'Z', 'z' - 'Z', "Z -> z" },
{ 0x00C9, 0x00E9 - 0x00C9, "E acute -> e acute" },
{ 'a', 0, "a (already lower)" },
{ '0', 0, "Digit 0" },
};
printf("\n=== getLowercase_map (sparse_map) ===\n");
for (auto& t : low_tests) {
int r = xicu::getLowercase_map(t.cp);
if (r == t.expected) { printf("PASS: U+%04X %s = %d\n", t.cp, t.desc, r); passed++; }
else { printf("FAIL: U+%04X %s got %d exp %d\n", t.cp, t.desc, r, t.expected); failed++; }
}
printf("\n=== Summary ===\n");
printf("Passed: %d\n", passed);
printf("Failed: %d\n", failed);
return failed == 0 ? 0 : 1;
}
Binary file not shown.
+259
View File
@@ -0,0 +1,259 @@
#include "xicu.hpp"
#include <cstdio>
#include <cstdint>
#include <cstring>
#include <string_view>
namespace xicu {
static constexpr uint32_t MAGIC = 0x58494355;
static constexpr uint8_t VERSION = 2;
PropTable::PropTable(const char* path) { load(path); }
PropTable::~PropTable() {
if (owns_data_ && data_) { delete[] data_; data_ = nullptr; }
}
PropTable::PropTable(PropTable&& other) noexcept
: data_(other.data_), data_size_(other.data_size_), owns_data_(other.owns_data_) {
other.data_ = nullptr; other.owns_data_ = false;
}
PropTable& PropTable::operator=(PropTable&& other) noexcept {
if (this != &other) {
if (owns_data_ && data_) delete[] data_;
data_ = other.data_; data_size_ = other.data_size_; owns_data_ = other.owns_data_;
other.data_ = nullptr; other.owns_data_ = false;
}
return *this;
}
uint32_t PropTable::readVarint(const uint8_t*& ptr) {
uint32_t val = 0; int shift = 0;
while (true) {
uint8_t b = *ptr++;
val |= (b & 0x7F) << shift;
if (!(b & 0x80)) break;
shift += 7;
}
return val;
}
bool PropTable::load(const char* path) {
if (owns_data_ && data_) { delete[] data_; data_ = nullptr; owns_data_ = false; }
FILE* f = std::fopen(path, "rb");
if (!f) return false;
std::fseek(f, 0, SEEK_END);
long fsize = std::ftell(f);
std::fseek(f, 0, SEEK_SET);
if (fsize < 6) { std::fclose(f); return false; }
data_size_ = static_cast<size_t>(fsize);
uint8_t* buf = new uint8_t[data_size_];
size_t read = std::fread(buf, 1, data_size_, f);
std::fclose(f);
if (read != data_size_) { delete[] buf; return false; }
data_ = buf; owns_data_ = true;
const uint8_t* ptr = data_;
uint32_t magic = *reinterpret_cast<const uint32_t*>(ptr); ptr += 4;
if (magic != MAGIC) return false;
uint8_t version = *ptr++;
if (version != VERSION) return false;
return true;
}
// Find field data pointer (points to data length varint after name)
const uint8_t* PropTable::findFieldData(const char* name, uint8_t& out_type, uint8_t& out_strat) const {
if (!data_) return nullptr;
const uint8_t* ptr = data_ + 5; // magic + version
uint32_t num_fields = readVarint(ptr);
for (uint32_t i = 0; i < num_fields; ++i) {
uint8_t ftype = *ptr++;
uint8_t fstrat = *ptr++;
const char* fname = reinterpret_cast<const char*>(ptr);
size_t name_len = std::strlen(fname);
ptr += name_len + 1;
if (std::strcmp(fname, name) == 0) {
out_type = ftype;
out_strat = fstrat;
return ptr; // points to data length varint
}
// Skip data
uint32_t data_len = readVarint(ptr);
ptr += data_len;
}
return nullptr;
}
// Find string field (returns pointer to indices data length)
const uint8_t* PropTable::findStringField(const char* name) const {
if (!data_) return nullptr;
const uint8_t* ptr = data_ + 5;
uint32_t num_fields = readVarint(ptr);
// Skip non-string fields (with inline data)
for (uint32_t i = 0; i < num_fields; ++i) {
ptr++; ptr++; // type, strat
while (*ptr++) {} // skip name
uint32_t data_len = readVarint(ptr);
ptr += data_len; // skip data
}
uint32_t num_str_fields = readVarint(ptr);
for (uint32_t i = 0; i < num_str_fields; ++i) {
const char* fname = reinterpret_cast<const char*>(ptr);
size_t name_len = std::strlen(fname);
ptr += name_len + 1;
if (std::strcmp(fname, name) == 0) {
return ptr; // points to indices data length
}
uint32_t data_len = readVarint(ptr);
ptr += data_len; // skip indices data
}
return nullptr;
}
// Decode bitmap: 1 bit per codepoint
bool PropTable::decodeBitmap(const uint8_t* data, uint32_t cp) const {
size_t byte_idx = cp >> 3;
uint8_t bit = cp & 7;
return (data[byte_idx] & (1 << bit)) != 0;
}
// Decode delta ranges: (start, length, value) varint-encoded
bool PropTable::decodeDeltaRangesBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
const uint8_t* ptr = data;
while (ptr < data_end) {
uint32_t start = readVarint(ptr);
uint32_t length = readVarint(ptr);
uint8_t val = *ptr++;
uint32_t end = start + length - 1;
if (cp >= start && cp <= end) return val != 0;
if (cp < start) return false;
}
return false;
}
int32_t PropTable::decodeDeltaRangesInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
const uint8_t* ptr = data;
while (ptr < data_end) {
uint32_t start = readVarint(ptr);
uint32_t length = readVarint(ptr);
uint32_t zigzag = readVarint(ptr);
int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));
uint32_t end = start + length - 1;
if (cp >= start && cp <= end) return val;
if (cp < start) return 0;
}
return 0;
}
// Decode sparse map: (cp, value) pairs
bool PropTable::decodeSparseMapBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
const uint8_t* ptr = data;
while (ptr < data_end) {
uint32_t c = readVarint(ptr);
if (ptr >= data_end) break;
uint8_t val = *ptr++;
if (c == cp) return val != 0;
if (c > cp) return false;
}
return false;
}
int32_t PropTable::decodeSparseMapInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
const uint8_t* ptr = data;
while (ptr < data_end) {
uint32_t c = readVarint(ptr);
if (ptr >= data_end) break;
uint32_t zigzag = readVarint(ptr);
int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));
if (c == cp) return val;
if (c > cp) return 0;
}
return 0;
}
std::string_view PropTable::getName(uint32_t cp) const {
// String fields use separate .str file
const uint8_t* indices_ptr = findStringField("name");
if (!indices_ptr) return "";
uint32_t indices_len = readVarint(indices_ptr);
// Find index for this codepoint (linear scan for now)
// TODO: optimize with binary search if needed
// For now, we need the string pool loaded separately
return ""; // Requires string pool file
}
bool PropTable::getPunctuation(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("punctuation", ftype, fstrat);
if (!data) return false;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeSparseMapBool(data, data_end, cp);
}
bool PropTable::getLetter(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("letter", ftype, fstrat);
if (!data) return false;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeBitmap(data, cp);
}
int32_t PropTable::getUppercase_map(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("uppercase_map", ftype, fstrat);
if (!data) return 0;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeSparseMapInt(data, data_end, cp);
}
int32_t PropTable::getLowercase_map(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("lowercase_map", ftype, fstrat);
if (!data) return 0;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeSparseMapInt(data, data_end, cp);
}
bool PropTable::getWhitespace(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("whitespace", ftype, fstrat);
if (!data) return false;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeSparseMapBool(data, data_end, cp);
}
bool PropTable::getSpace(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("space", ftype, fstrat);
if (!data) return false;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeSparseMapBool(data, data_end, cp);
}
bool PropTable::getDecimal(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("decimal", ftype, fstrat);
if (!data) return false;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeSparseMapBool(data, data_end, cp);
}
int32_t PropTable::getDecimal_val(uint32_t cp) const {
uint8_t ftype, fstrat;
const uint8_t* data = findFieldData("decimal_val", ftype, fstrat);
if (!data) return 0;
uint32_t data_len = readVarint(data);
const uint8_t* data_end = data + data_len;
return decodeDeltaRangesInt(data, data_end, cp);
}
} // namespace xicu
+85
View File
@@ -0,0 +1,85 @@
#pragma once
#include <cstdint>
#include <string_view>
namespace xicu {
class PropTable {
public:
PropTable() = default;
explicit PropTable(const char* path);
~PropTable();
PropTable(const PropTable&) = delete;
PropTable& operator=(const PropTable&) = delete;
PropTable(PropTable&& other) noexcept;
PropTable& operator=(PropTable&& other) noexcept;
bool load(const char* path);
bool isLoaded() const { return data_ != nullptr; }
std::string_view getName(uint32_t cp) const;
bool getPunctuation(uint32_t cp) const;
bool getLetter(uint32_t cp) const;
int32_t getUppercase_map(uint32_t cp) const;
int32_t getLowercase_map(uint32_t cp) const;
bool getWhitespace(uint32_t cp) const;
bool getSpace(uint32_t cp) const;
bool getDecimal(uint32_t cp) const;
int32_t getDecimal_val(uint32_t cp) const;
private:
const uint8_t* data_ = nullptr;
size_t data_size_ = 0;
bool owns_data_ = false;
static uint32_t readVarint(const uint8_t*& ptr);
const uint8_t* findFieldData(const char* name, uint8_t& out_type, uint8_t& out_strat) const;
const uint8_t* findStringField(const char* name) const;
// Decoders
bool decodeBitmap(const uint8_t* data, uint32_t cp) const;
bool decodeDeltaRangesBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
int32_t decodeDeltaRangesInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
bool decodeSparseMapBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
int32_t decodeSparseMapInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;
};
inline std::string_view getName(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getName(cp);
}
inline bool isPunctuation(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getPunctuation(cp);
}
inline bool isLetter(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getLetter(cp);
}
inline int32_t getUppercase_map(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getUppercase_map(cp);
}
inline int32_t getLowercase_map(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getLowercase_map(cp);
}
inline bool isWhitespace(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getWhitespace(cp);
}
inline bool isSpace(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getSpace(cp);
}
inline bool isDecimal(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getDecimal(cp);
}
inline int32_t getDecimal_val(uint32_t cp) {
static PropTable table("../out/xicu.bin");
return table.getDecimal_val(cp);
}
} // namespace xicu
BIN
View File
Binary file not shown.