lots of improvements standby for the results
This commit is contained in:
+259
@@ -0,0 +1,259 @@
|
||||
#include "xicu.hpp"
|
||||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <string_view>
|
||||
|
||||
namespace xicu {
|
||||
|
||||
static constexpr uint32_t MAGIC = 0x58494355;
|
||||
static constexpr uint8_t VERSION = 2;
|
||||
|
||||
PropTable::PropTable(const char* path) { load(path); }
|
||||
|
||||
PropTable::~PropTable() {
|
||||
if (owns_data_ && data_) { delete[] data_; data_ = nullptr; }
|
||||
}
|
||||
|
||||
PropTable::PropTable(PropTable&& other) noexcept
|
||||
: data_(other.data_), data_size_(other.data_size_), owns_data_(other.owns_data_) {
|
||||
other.data_ = nullptr; other.owns_data_ = false;
|
||||
}
|
||||
|
||||
PropTable& PropTable::operator=(PropTable&& other) noexcept {
|
||||
if (this != &other) {
|
||||
if (owns_data_ && data_) delete[] data_;
|
||||
data_ = other.data_; data_size_ = other.data_size_; owns_data_ = other.owns_data_;
|
||||
other.data_ = nullptr; other.owns_data_ = false;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
uint32_t PropTable::readVarint(const uint8_t*& ptr) {
|
||||
uint32_t val = 0; int shift = 0;
|
||||
while (true) {
|
||||
uint8_t b = *ptr++;
|
||||
val |= (b & 0x7F) << shift;
|
||||
if (!(b & 0x80)) break;
|
||||
shift += 7;
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
bool PropTable::load(const char* path) {
|
||||
if (owns_data_ && data_) { delete[] data_; data_ = nullptr; owns_data_ = false; }
|
||||
FILE* f = std::fopen(path, "rb");
|
||||
if (!f) return false;
|
||||
std::fseek(f, 0, SEEK_END);
|
||||
long fsize = std::ftell(f);
|
||||
std::fseek(f, 0, SEEK_SET);
|
||||
if (fsize < 6) { std::fclose(f); return false; }
|
||||
data_size_ = static_cast<size_t>(fsize);
|
||||
uint8_t* buf = new uint8_t[data_size_];
|
||||
size_t read = std::fread(buf, 1, data_size_, f);
|
||||
std::fclose(f);
|
||||
if (read != data_size_) { delete[] buf; return false; }
|
||||
data_ = buf; owns_data_ = true;
|
||||
|
||||
const uint8_t* ptr = data_;
|
||||
uint32_t magic = *reinterpret_cast<const uint32_t*>(ptr); ptr += 4;
|
||||
if (magic != MAGIC) return false;
|
||||
uint8_t version = *ptr++;
|
||||
if (version != VERSION) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Find field data pointer (points to data length varint after name)
|
||||
const uint8_t* PropTable::findFieldData(const char* name, uint8_t& out_type, uint8_t& out_strat) const {
|
||||
if (!data_) return nullptr;
|
||||
const uint8_t* ptr = data_ + 5; // magic + version
|
||||
uint32_t num_fields = readVarint(ptr);
|
||||
for (uint32_t i = 0; i < num_fields; ++i) {
|
||||
uint8_t ftype = *ptr++;
|
||||
uint8_t fstrat = *ptr++;
|
||||
const char* fname = reinterpret_cast<const char*>(ptr);
|
||||
size_t name_len = std::strlen(fname);
|
||||
ptr += name_len + 1;
|
||||
if (std::strcmp(fname, name) == 0) {
|
||||
out_type = ftype;
|
||||
out_strat = fstrat;
|
||||
return ptr; // points to data length varint
|
||||
}
|
||||
// Skip data
|
||||
uint32_t data_len = readVarint(ptr);
|
||||
ptr += data_len;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Find string field (returns pointer to indices data length)
|
||||
const uint8_t* PropTable::findStringField(const char* name) const {
|
||||
if (!data_) return nullptr;
|
||||
const uint8_t* ptr = data_ + 5;
|
||||
uint32_t num_fields = readVarint(ptr);
|
||||
// Skip non-string fields (with inline data)
|
||||
for (uint32_t i = 0; i < num_fields; ++i) {
|
||||
ptr++; ptr++; // type, strat
|
||||
while (*ptr++) {} // skip name
|
||||
uint32_t data_len = readVarint(ptr);
|
||||
ptr += data_len; // skip data
|
||||
}
|
||||
uint32_t num_str_fields = readVarint(ptr);
|
||||
for (uint32_t i = 0; i < num_str_fields; ++i) {
|
||||
const char* fname = reinterpret_cast<const char*>(ptr);
|
||||
size_t name_len = std::strlen(fname);
|
||||
ptr += name_len + 1;
|
||||
if (std::strcmp(fname, name) == 0) {
|
||||
return ptr; // points to indices data length
|
||||
}
|
||||
uint32_t data_len = readVarint(ptr);
|
||||
ptr += data_len; // skip indices data
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Decode bitmap: 1 bit per codepoint
|
||||
bool PropTable::decodeBitmap(const uint8_t* data, uint32_t cp) const {
|
||||
size_t byte_idx = cp >> 3;
|
||||
uint8_t bit = cp & 7;
|
||||
return (data[byte_idx] & (1 << bit)) != 0;
|
||||
}
|
||||
|
||||
// Decode delta ranges: (start, length, value) varint-encoded
|
||||
bool PropTable::decodeDeltaRangesBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t start = readVarint(ptr);
|
||||
uint32_t length = readVarint(ptr);
|
||||
uint8_t val = *ptr++;
|
||||
uint32_t end = start + length - 1;
|
||||
if (cp >= start && cp <= end) return val != 0;
|
||||
if (cp < start) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t PropTable::decodeDeltaRangesInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t start = readVarint(ptr);
|
||||
uint32_t length = readVarint(ptr);
|
||||
uint32_t zigzag = readVarint(ptr);
|
||||
int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));
|
||||
uint32_t end = start + length - 1;
|
||||
if (cp >= start && cp <= end) return val;
|
||||
if (cp < start) return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Decode sparse map: (cp, value) pairs
|
||||
bool PropTable::decodeSparseMapBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t c = readVarint(ptr);
|
||||
if (ptr >= data_end) break;
|
||||
uint8_t val = *ptr++;
|
||||
if (c == cp) return val != 0;
|
||||
if (c > cp) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t PropTable::decodeSparseMapInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {
|
||||
const uint8_t* ptr = data;
|
||||
while (ptr < data_end) {
|
||||
uint32_t c = readVarint(ptr);
|
||||
if (ptr >= data_end) break;
|
||||
uint32_t zigzag = readVarint(ptr);
|
||||
int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));
|
||||
if (c == cp) return val;
|
||||
if (c > cp) return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string_view PropTable::getName(uint32_t cp) const {
|
||||
// String fields use separate .str file
|
||||
const uint8_t* indices_ptr = findStringField("name");
|
||||
if (!indices_ptr) return "";
|
||||
uint32_t indices_len = readVarint(indices_ptr);
|
||||
// Find index for this codepoint (linear scan for now)
|
||||
// TODO: optimize with binary search if needed
|
||||
// For now, we need the string pool loaded separately
|
||||
return ""; // Requires string pool file
|
||||
}
|
||||
|
||||
bool PropTable::getPunctuation(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("punctuation", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getLetter(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("letter", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeBitmap(data, cp);
|
||||
}
|
||||
|
||||
int32_t PropTable::getUppercase_map(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("uppercase_map", ftype, fstrat);
|
||||
if (!data) return 0;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapInt(data, data_end, cp);
|
||||
}
|
||||
|
||||
int32_t PropTable::getLowercase_map(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("lowercase_map", ftype, fstrat);
|
||||
if (!data) return 0;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapInt(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getWhitespace(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("whitespace", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getSpace(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("space", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
bool PropTable::getDecimal(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("decimal", ftype, fstrat);
|
||||
if (!data) return false;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeSparseMapBool(data, data_end, cp);
|
||||
}
|
||||
|
||||
int32_t PropTable::getDecimal_val(uint32_t cp) const {
|
||||
uint8_t ftype, fstrat;
|
||||
const uint8_t* data = findFieldData("decimal_val", ftype, fstrat);
|
||||
if (!data) return 0;
|
||||
uint32_t data_len = readVarint(data);
|
||||
const uint8_t* data_end = data + data_len;
|
||||
return decodeDeltaRangesInt(data, data_end, cp);
|
||||
}
|
||||
|
||||
} // namespace xicu
|
||||
Reference in New Issue
Block a user