906 lines
35 KiB
Python
906 lines
35 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
x-icu-gen: Configurable ICU data generator with optimal compression.
|
|
Generates binary tables + C++ implementation from UnicodeData.txt
|
|
"""
|
|
|
|
import os
|
|
import struct
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Dict, List, Tuple, Any, Optional, Set
|
|
from dataclasses import dataclass, field
|
|
from enum import Enum
|
|
from collections import defaultdict
|
|
|
|
# ============================================================
|
|
# CONFIGURATION (from py-gen.ipynb)
|
|
# ============================================================
|
|
|
|
CONFIG = {
|
|
'sources': {
|
|
'unicode': 'https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt',
|
|
},
|
|
'unicode': {
|
|
'useOldName': True,
|
|
'getName': True,
|
|
|
|
'getDecomposition': False,
|
|
'getDecompositionType': False,
|
|
|
|
'toLowercase': True,
|
|
'toUppercase': True,
|
|
'toTitlecase': False,
|
|
|
|
'isPunctuation': True,
|
|
'isSymbol': False,
|
|
'isCombining': False,
|
|
|
|
'isPrintable': False,
|
|
'isSpace': True,
|
|
'isWhitespace': True,
|
|
'isLetter': True,
|
|
'isUppercase': False,
|
|
'isLowercase': False,
|
|
'isTitlecase': False,
|
|
'isDeprecated': False,
|
|
|
|
'isDecimal': True,
|
|
'isDigit': False,
|
|
'isNumberLike': False,
|
|
|
|
'getDecimal': True,
|
|
'getDigit': False,
|
|
'getNumberLike': False,
|
|
}
|
|
}
|
|
|
|
# Field definitions: maps config key -> (output_name, type, category)
|
|
# category: 'bool', 'int', 'str', 'enum'
|
|
FIELD_DEFS = {
|
|
'getName': ('name', 'str', 'str'),
|
|
'getDecomposition': ('decomposition', 'str', 'str'),
|
|
'getDecompositionType': ('decomposition_type', 'str', 'str'),
|
|
'isPunctuation': ('punctuation', 'bool', 'bool'),
|
|
'isSymbol': ('symbol', 'bool', 'bool'),
|
|
'isCombining': ('combining', 'bool', 'bool'),
|
|
'isLetter': ('letter', 'bool', 'bool'),
|
|
'isUppercase': ('uppercase', 'bool', 'bool'),
|
|
'isLowercase': ('lowercase', 'bool', 'bool'),
|
|
'isTitlecase': ('titlecase', 'bool', 'bool'),
|
|
'toUppercase': ('uppercase_map', 'int', 'int'),
|
|
'toLowercase': ('lowercase_map', 'int', 'int'),
|
|
'toTitlecase': ('titlecase_map', 'int', 'int'),
|
|
'isWhitespace': ('whitespace', 'bool', 'bool'),
|
|
'isPrintable': ('printable', 'bool', 'bool'),
|
|
'isSpace': ('space', 'bool', 'bool'),
|
|
'isDecimal': ('decimal', 'bool', 'bool'),
|
|
'isDigit': ('digit', 'bool', 'bool'),
|
|
'isNumberLike': ('number_like', 'bool', 'bool'),
|
|
'getDecimal': ('decimal_val', 'int', 'int'),
|
|
'getDigit': ('digit_val', 'int', 'int'),
|
|
'getNumberLike': ('number_like_val', 'int', 'int'),
|
|
}
|
|
|
|
# ============================================================
|
|
# DATA STRUCTURES
|
|
# ============================================================
|
|
|
|
class CompStrategy(Enum):
|
|
BITMAP = "bitmap" # 1 bit per codepoint
|
|
DELTA_RANGES = "delta_ranges" # (start, end, value) runs
|
|
RLE = "rle" # Run-length encoding
|
|
SPARSE_MAP = "sparse_map" # Only store non-default values
|
|
STRING_TABLE = "string_table" # Separate string pool
|
|
|
|
@dataclass
|
|
class FieldConfig:
|
|
name: str
|
|
type: str # 'bool', 'int', 'str'
|
|
enabled: bool = False
|
|
strategy: CompStrategy = CompStrategy.DELTA_RANGES
|
|
default_value: Any = None
|
|
|
|
@dataclass
|
|
class CodePointData:
|
|
cp: int
|
|
name: str = ""
|
|
category: str = ""
|
|
decomposition: str = ""
|
|
decimal_val: str = ""
|
|
digit_val: str = ""
|
|
numeric_val: str = ""
|
|
bidi_mirrored: str = ""
|
|
unicode_1_name: str = ""
|
|
uppercase_map: str = ""
|
|
lowercase_map: str = ""
|
|
titlecase_map: str = ""
|
|
|
|
@dataclass
|
|
class ProcessedRow:
|
|
cp: int
|
|
fields: Dict[str, Any] = field(default_factory=dict)
|
|
|
|
# ============================================================
|
|
# UNICODE PARSER
|
|
# ============================================================
|
|
|
|
class UnicodeParser:
|
|
def __init__(self, data_path: str):
|
|
self.data_path = data_path
|
|
self.keys = [
|
|
"codepoint", "name", "category", "combining_class", "bidi_class",
|
|
"decomposition", "decimal_val", "digit_val", "numeric_val",
|
|
"bidi_mirrored", "unicode_1_name", "iso_comment",
|
|
"uppercase_map", "lowercase_map", "titlecase_map"
|
|
]
|
|
|
|
def parse(self) -> List[CodePointData]:
|
|
results = []
|
|
range_start = None
|
|
|
|
with open(self.data_path, 'r', encoding='utf-8') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
parts = line.split(';')
|
|
if len(parts) < 15:
|
|
continue
|
|
|
|
cp = int(parts[0], 16)
|
|
name = parts[1]
|
|
cat = parts[2]
|
|
decomp = parts[5]
|
|
dec_val = parts[6]
|
|
dig_val = parts[7]
|
|
num_val = parts[8]
|
|
bidi_mir = parts[9]
|
|
old_name = parts[10]
|
|
up_map = parts[12]
|
|
low_map = parts[13]
|
|
title_map = parts[14]
|
|
|
|
if name.endswith(', First>'):
|
|
range_start = (cp, cat, name, decomp, dec_val, dig_val, num_val,
|
|
bidi_mir, old_name, up_map, low_map, title_map)
|
|
continue
|
|
elif name.endswith(', Last>') and range_start:
|
|
sc, scat, sname, sdecomp, sdec, sdig, snum, sbidi, sold, sup, slow, stitle = range_start
|
|
base_name = sname.replace(', First>', '').replace('<', '')
|
|
for c in range(sc, cp + 1):
|
|
results.append(CodePointData(
|
|
cp=c, name=f"<{base_name}>", category=scat,
|
|
decomposition=sdecomp, decimal_val=sdec, digit_val=sdig,
|
|
numeric_val=snum, bidi_mirrored=sbidi, unicode_1_name=sold,
|
|
uppercase_map=sup, lowercase_map=slow, titlecase_map=stitle
|
|
))
|
|
range_start = None
|
|
continue
|
|
|
|
results.append(CodePointData(
|
|
cp=cp, name=name, category=cat, decomposition=decomp,
|
|
decimal_val=dec_val, digit_val=dig_val, numeric_val=num_val,
|
|
bidi_mirrored=bidi_mir, unicode_1_name=old_name,
|
|
uppercase_map=up_map, lowercase_map=low_map, titlecase_map=title_map
|
|
))
|
|
return results
|
|
|
|
# ============================================================
|
|
# FIELD PROCESSORS
|
|
# ============================================================
|
|
|
|
def process_row(row: CodePointData, cfg: Dict, use_old_name: bool) -> ProcessedRow:
|
|
"""Extract all configured fields from a parsed row."""
|
|
out = ProcessedRow(cp=row.cp)
|
|
cat = row.category
|
|
char = chr(row.cp) if row.cp <= 0x10FFFF else ''
|
|
|
|
# Name handling
|
|
name = row.name
|
|
if use_old_name and name.startswith('<') and row.unicode_1_name:
|
|
name = row.unicode_1_name
|
|
|
|
# Boolean properties from category
|
|
out.fields['punctuation'] = cat.startswith('P')
|
|
out.fields['symbol'] = cat.startswith('S')
|
|
out.fields['combining'] = cat.startswith('M')
|
|
out.fields['letter'] = cat.startswith('L')
|
|
out.fields['uppercase'] = cat == 'Lu'
|
|
out.fields['lowercase'] = cat == 'Ll'
|
|
out.fields['titlecase'] = cat == 'Lt'
|
|
out.fields['whitespace'] = cat.startswith('Z') or row.cp in (0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1C, 0x1D, 0x1E, 0x1F, 0x85, 0xA0)
|
|
out.fields['space'] = cat == 'Zs'
|
|
out.fields['printable'] = not cat.startswith('C')
|
|
|
|
# Numeric properties
|
|
is_decimal = bool(row.decimal_val)
|
|
is_digit = is_decimal or bool(row.digit_val)
|
|
is_numeric = is_digit or bool(row.numeric_val)
|
|
out.fields['decimal'] = is_decimal
|
|
out.fields['digit'] = is_digit
|
|
out.fields['number_like'] = is_numeric
|
|
out.fields['decimal_val'] = int(row.decimal_val) if is_decimal else 0
|
|
out.fields['digit_val'] = int(row.digit_val) if is_digit else 0
|
|
out.fields['number_like_val'] = row.numeric_val if is_numeric else ""
|
|
|
|
# String properties
|
|
out.fields['name'] = name
|
|
out.fields['decomposition'] = row.decomposition
|
|
decomp_type = ""
|
|
if row.decomposition and '<' in row.decomposition:
|
|
decomp_type = row.decomposition[1:row.decomposition.find('>')]
|
|
out.fields['decomposition_type'] = decomp_type
|
|
|
|
# Case mappings (relative offsets)
|
|
for k, src in [('uppercase_map', row.uppercase_map),
|
|
('lowercase_map', row.lowercase_map),
|
|
('titlecase_map', row.titlecase_map)]:
|
|
val = int(src, 16) if src else 0
|
|
out.fields[k] = val - row.cp if val != 0 else 0
|
|
|
|
return out
|
|
|
|
# ============================================================
|
|
# COMPRESSION ANALYSIS
|
|
# ============================================================
|
|
|
|
def analyze_field(rows: List[ProcessedRow], field_name: str, field_type: str) -> CompStrategy:
|
|
"""Determine optimal compression strategy for a field."""
|
|
values = [r.fields.get(field_name, None) for r in rows]
|
|
non_default = [v for v in values if v not in (False, 0, "", None)]
|
|
unique_vals = set(v for v in values if v not in (False, 0, "", None))
|
|
|
|
total = len(rows)
|
|
sparse_ratio = len(non_default) / total if total > 0 else 0
|
|
|
|
if field_type == 'str':
|
|
return CompStrategy.STRING_TABLE
|
|
|
|
if field_type == 'bool':
|
|
# If very sparse (< 1%), use sparse map
|
|
if sparse_ratio < 0.01:
|
|
return CompStrategy.SPARSE_MAP
|
|
# If dense (> 50%), bitmap is good
|
|
if sparse_ratio > 0.5:
|
|
return CompStrategy.BITMAP
|
|
# Otherwise delta ranges
|
|
return CompStrategy.DELTA_RANGES
|
|
|
|
if field_type == 'int':
|
|
# Check if values are mostly small offsets (case mappings)
|
|
if field_name.endswith('_map'):
|
|
# These are relative offsets, usually small or zero
|
|
if sparse_ratio < 0.05:
|
|
return CompStrategy.SPARSE_MAP
|
|
return CompStrategy.DELTA_RANGES
|
|
# For numeric values (decimal_val, etc.)
|
|
if len(unique_vals) < 20:
|
|
return CompStrategy.DELTA_RANGES
|
|
return CompStrategy.SPARSE_MAP
|
|
|
|
return CompStrategy.DELTA_RANGES
|
|
|
|
# ============================================================
|
|
# ENCODERS
|
|
# ============================================================
|
|
|
|
def write_varint(buf: bytearray, val: int):
|
|
while val >= 0x80:
|
|
buf.append((val & 0x7F) | 0x80)
|
|
val >>= 7
|
|
buf.append(val)
|
|
|
|
def read_varint(data: bytes, offset: int) -> Tuple[int, int]:
|
|
val = 0
|
|
shift = 0
|
|
while True:
|
|
b = data[offset]
|
|
offset += 1
|
|
val |= (b & 0x7F) << shift
|
|
if not (b & 0x80):
|
|
break
|
|
shift += 7
|
|
return val, offset
|
|
|
|
def encode_bitmap(rows: List[ProcessedRow], field: str) -> bytes:
|
|
"""Encode boolean field as bitmap."""
|
|
max_cp = max(r.cp for r in rows)
|
|
size = (max_cp + 8) // 8
|
|
bitmap = bytearray(size)
|
|
for r in rows:
|
|
if r.fields.get(field, False):
|
|
idx = r.cp >> 3
|
|
bitmap[idx] |= 1 << (r.cp & 7)
|
|
return bytes(bitmap)
|
|
|
|
def encode_delta_ranges(rows: List[ProcessedRow], field: str, field_type: str) -> bytes:
|
|
"""Encode as (start, length, value) runs with varints."""
|
|
buf = bytearray()
|
|
current_start = None
|
|
current_val = None
|
|
|
|
for r in rows:
|
|
val = r.fields.get(field)
|
|
if field_type == 'bool':
|
|
val = bool(val)
|
|
elif field_type == 'int':
|
|
val = int(val) if val else 0
|
|
else:
|
|
val = val or ""
|
|
|
|
if current_start is None:
|
|
current_start = r.cp
|
|
current_val = val
|
|
elif val != current_val:
|
|
# End previous range
|
|
length = r.cp - current_start
|
|
write_varint(buf, current_start)
|
|
write_varint(buf, length)
|
|
if field_type == 'bool':
|
|
buf.append(1 if current_val else 0)
|
|
elif field_type == 'int':
|
|
# Zigzag encode for negative values
|
|
v = current_val
|
|
write_varint(buf, (v << 1) ^ (v >> 31))
|
|
current_start = r.cp
|
|
current_val = val
|
|
|
|
# Last range
|
|
if current_start is not None:
|
|
length = 0x10FFFF - current_start + 1
|
|
write_varint(buf, current_start)
|
|
write_varint(buf, length)
|
|
if field_type == 'bool':
|
|
buf.append(1 if current_val else 0)
|
|
elif field_type == 'int':
|
|
v = current_val
|
|
write_varint(buf, (v << 1) ^ (v >> 31))
|
|
|
|
return bytes(buf)
|
|
|
|
def encode_sparse_map(rows: List[ProcessedRow], field: str, field_type: str) -> bytes:
|
|
"""Store only non-default values as (cp, value) pairs."""
|
|
buf = bytearray()
|
|
for r in rows:
|
|
val = r.fields.get(field)
|
|
if field_type == 'bool':
|
|
val = bool(val)
|
|
if not val:
|
|
continue
|
|
write_varint(buf, r.cp)
|
|
buf.append(1)
|
|
elif field_type == 'int':
|
|
val = int(val) if val else 0
|
|
if val == 0:
|
|
continue
|
|
write_varint(buf, r.cp)
|
|
v = val
|
|
write_varint(buf, (v << 1) ^ (v >> 31))
|
|
else:
|
|
val = val or ""
|
|
if not val:
|
|
continue
|
|
write_varint(buf, r.cp)
|
|
# String index will be handled separately
|
|
write_varint(buf, 0) # placeholder
|
|
return bytes(buf)
|
|
|
|
def encode_string_table(rows: List[ProcessedRow], field: str) -> Tuple[bytes, List[str]]:
|
|
"""Build string table and return (indices, string_pool)."""
|
|
strings = []
|
|
str_to_idx = {}
|
|
indices = []
|
|
|
|
for r in rows:
|
|
val = r.fields.get(field, "") or ""
|
|
if val not in str_to_idx:
|
|
str_to_idx[val] = len(strings)
|
|
strings.append(val)
|
|
indices.append(str_to_idx[val])
|
|
|
|
# Encode indices as varints per codepoint
|
|
buf = bytearray()
|
|
for idx in indices:
|
|
write_varint(buf, idx)
|
|
|
|
return bytes(buf), strings
|
|
|
|
# ============================================================
|
|
# MAIN GENERATOR
|
|
# ============================================================
|
|
|
|
class XicuGenerator:
|
|
def __init__(self, config: Dict, data_dir: str = './data', out_dir: str = './out'):
|
|
self.config = config
|
|
self.data_dir = Path(data_dir)
|
|
self.out_dir = Path(out_dir)
|
|
self.out_dir.mkdir(parents=True, exist_ok=True)
|
|
self.data_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
self.rows: List[ProcessedRow] = []
|
|
self.field_configs: Dict[str, FieldConfig] = {}
|
|
self.strategies: Dict[str, CompStrategy] = {}
|
|
self.string_tables: Dict[str, List[str]] = {}
|
|
self.encoded_data: Dict[str, bytes] = {}
|
|
|
|
def download_unicode_data(self):
|
|
import urllib.request
|
|
url = self.config['sources']['unicode']
|
|
path = self.data_dir / 'UnicodeData.txt'
|
|
if not path.exists():
|
|
print(f"Downloading {url}...")
|
|
urllib.request.urlretrieve(url, path)
|
|
print("Done.")
|
|
|
|
def parse_and_process(self):
|
|
parser = UnicodeParser(str(self.data_dir / 'UnicodeData.txt'))
|
|
raw_rows = parser.parse()
|
|
print(f"Parsed {len(raw_rows)} codepoints")
|
|
|
|
ucfg = self.config['unicode']
|
|
use_old_name = ucfg.get('useOldName', False)
|
|
|
|
for row in raw_rows:
|
|
self.rows.append(process_row(row, ucfg, use_old_name))
|
|
|
|
print(f"Processed {len(self.rows)} rows")
|
|
|
|
def analyze_and_configure(self):
|
|
ucfg = self.config['unicode']
|
|
|
|
for cfg_key, (out_name, ftype, cat) in FIELD_DEFS.items():
|
|
if not ucfg.get(cfg_key, False):
|
|
continue
|
|
|
|
strategy = analyze_field(self.rows, out_name, ftype)
|
|
default = False if ftype == 'bool' else (0 if ftype == 'int' else "")
|
|
|
|
self.field_configs[out_name] = FieldConfig(
|
|
name=out_name, type=ftype, enabled=True,
|
|
strategy=strategy, default_value=default
|
|
)
|
|
self.strategies[out_name] = strategy
|
|
|
|
print("Field strategies:")
|
|
for name, fc in self.field_configs.items():
|
|
print(f" {name} ({fc.type}): {fc.strategy.value}")
|
|
|
|
def encode_all(self):
|
|
# First pass: encode non-string fields
|
|
for name, fc in self.field_configs.items():
|
|
if fc.type == 'str':
|
|
continue
|
|
|
|
print(f"Encoding {name} with {fc.strategy.value}...")
|
|
if fc.strategy == CompStrategy.BITMAP:
|
|
self.encoded_data[name] = encode_bitmap(self.rows, name)
|
|
elif fc.strategy == CompStrategy.DELTA_RANGES:
|
|
self.encoded_data[name] = encode_delta_ranges(self.rows, name, fc.type)
|
|
elif fc.strategy == CompStrategy.SPARSE_MAP:
|
|
self.encoded_data[name] = encode_sparse_map(self.rows, name, fc.type)
|
|
|
|
# Second pass: string fields
|
|
for name, fc in self.field_configs.items():
|
|
if fc.type != 'str':
|
|
continue
|
|
print(f"Encoding string table for {name}...")
|
|
indices, strings = encode_string_table(self.rows, name)
|
|
self.encoded_data[name + '_indices'] = indices
|
|
self.string_tables[name] = strings
|
|
|
|
def write_binary(self):
|
|
"""Write main binary file with all non-string fields."""
|
|
path = self.out_dir / 'xicu.bin'
|
|
buf = bytearray()
|
|
|
|
# Header
|
|
buf.extend(struct.pack('<I', 0x58494355)) # XICU magic
|
|
buf.extend(struct.pack('<B', 2)) # version 2
|
|
non_str_fields = [n for n, fc in self.field_configs.items() if fc.type != 'str']
|
|
write_varint(buf, len(non_str_fields))
|
|
|
|
# Field descriptors + data inline
|
|
for name, fc in self.field_configs.items():
|
|
if fc.type == 'str':
|
|
continue
|
|
buf.extend(struct.pack('<B',
|
|
{'bool': 1, 'int': 2}[fc.type]))
|
|
buf.extend(struct.pack('<B',
|
|
{'bitmap': 1, 'delta_ranges': 2, 'rle': 3, 'sparse_map': 4}[fc.strategy.value]))
|
|
# Field name as null-terminated
|
|
buf.extend(name.encode('ascii'))
|
|
buf.append(0)
|
|
# Data length + data
|
|
data = self.encoded_data[name]
|
|
write_varint(buf, len(data))
|
|
buf.extend(data)
|
|
|
|
# String field descriptors
|
|
str_fields = [n for n, fc in self.field_configs.items() if fc.type == 'str']
|
|
write_varint(buf, len(str_fields))
|
|
for name in str_fields:
|
|
buf.extend(name.encode('ascii'))
|
|
buf.append(0)
|
|
|
|
# String indices
|
|
for name in str_fields:
|
|
data = self.encoded_data[name + '_indices']
|
|
write_varint(buf, len(data))
|
|
buf.extend(data)
|
|
|
|
with open(path, 'wb') as f:
|
|
f.write(buf)
|
|
|
|
print(f"Wrote {path} ({len(buf)} bytes)")
|
|
|
|
# Write string pools separately
|
|
for name, strings in self.string_tables.items():
|
|
pool_path = self.out_dir / f'xicu_{name}.str'
|
|
buf = bytearray()
|
|
write_varint(buf, len(strings))
|
|
for s in strings:
|
|
enc = s.encode('utf-8')
|
|
write_varint(buf, len(enc))
|
|
buf.extend(enc)
|
|
with open(pool_path, 'wb') as f:
|
|
f.write(buf)
|
|
print(f"Wrote {pool_path} ({len(buf)} bytes)")
|
|
|
|
def generate_cpp(self):
|
|
"""Generate C++ header and implementation."""
|
|
hpp_path = Path('cpp') / 'xicu.hpp'
|
|
cpp_path = Path('cpp') / 'xicu.cpp'
|
|
Path('cpp').mkdir(exist_ok=True)
|
|
|
|
# Generate header
|
|
hpp = self._gen_header()
|
|
with open(hpp_path, 'w') as f:
|
|
f.write(hpp)
|
|
|
|
# Generate implementation
|
|
cpp = self._gen_impl()
|
|
with open(cpp_path, 'w') as f:
|
|
f.write(cpp)
|
|
|
|
print(f"Generated {hpp_path} and {cpp_path}")
|
|
|
|
def _gen_header(self) -> str:
|
|
lines = [
|
|
'#pragma once',
|
|
'',
|
|
'#include <cstdint>',
|
|
'#include <string_view>',
|
|
'',
|
|
'namespace xicu {',
|
|
'',
|
|
'class PropTable {',
|
|
'public:',
|
|
' PropTable() = default;',
|
|
' explicit PropTable(const char* path);',
|
|
' ~PropTable();',
|
|
'',
|
|
' PropTable(const PropTable&) = delete;',
|
|
' PropTable& operator=(const PropTable&) = delete;',
|
|
' PropTable(PropTable&& other) noexcept;',
|
|
' PropTable& operator=(PropTable&& other) noexcept;',
|
|
'',
|
|
' bool load(const char* path);',
|
|
' bool isLoaded() const { return data_ != nullptr; }',
|
|
]
|
|
|
|
# Generate getter declarations
|
|
for name, fc in self.field_configs.items():
|
|
if fc.type == 'bool':
|
|
lines.append(f' bool get{name.capitalize()}(uint32_t cp) const;')
|
|
elif fc.type == 'int':
|
|
lines.append(f' int32_t get{name.capitalize()}(uint32_t cp) const;')
|
|
elif fc.type == 'str':
|
|
lines.append(f' std::string_view get{name.capitalize()}(uint32_t cp) const;')
|
|
|
|
lines.extend([
|
|
'',
|
|
'private:',
|
|
' const uint8_t* data_ = nullptr;',
|
|
' size_t data_size_ = 0;',
|
|
' bool owns_data_ = false;',
|
|
'',
|
|
' static uint32_t readVarint(const uint8_t*& ptr);',
|
|
' const uint8_t* findFieldData(const char* name, uint8_t& out_type, uint8_t& out_strat) const;',
|
|
' const uint8_t* findStringField(const char* name) const;',
|
|
'',
|
|
' // Decoders',
|
|
' bool decodeBitmap(const uint8_t* data, uint32_t cp) const;',
|
|
' bool decodeDeltaRangesBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;',
|
|
' int32_t decodeDeltaRangesInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;',
|
|
' bool decodeSparseMapBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;',
|
|
' int32_t decodeSparseMapInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const;',
|
|
'};',
|
|
'',
|
|
])
|
|
|
|
# Inline convenience functions
|
|
for name, fc in self.field_configs.items():
|
|
if fc.type == 'bool':
|
|
lines.append(f'inline bool is{name.capitalize()}(uint32_t cp) {{')
|
|
lines.append(f' static PropTable table("../out/xicu.bin");')
|
|
lines.append(f' return table.get{name.capitalize()}(cp);')
|
|
lines.append(f'}}')
|
|
elif fc.type == 'int':
|
|
lines.append(f'inline int32_t get{name.capitalize()}(uint32_t cp) {{')
|
|
lines.append(f' static PropTable table("../out/xicu.bin");')
|
|
lines.append(f' return table.get{name.capitalize()}(cp);')
|
|
lines.append(f'}}')
|
|
elif fc.type == 'str':
|
|
lines.append(f'inline std::string_view get{name.capitalize()}(uint32_t cp) {{')
|
|
lines.append(f' static PropTable table("../out/xicu.bin");')
|
|
lines.append(f' return table.get{name.capitalize()}(cp);')
|
|
lines.append(f'}}')
|
|
|
|
lines.extend([
|
|
'',
|
|
'} // namespace xicu',
|
|
])
|
|
return '\n'.join(lines)
|
|
|
|
def _gen_impl(self) -> str:
|
|
"""Generate complete C++ implementation with decoders."""
|
|
lines = [
|
|
'#include "xicu.hpp"',
|
|
'#include <cstdio>',
|
|
'#include <cstdint>',
|
|
'#include <cstring>',
|
|
'#include <string_view>',
|
|
'',
|
|
'namespace xicu {',
|
|
'',
|
|
'static constexpr uint32_t MAGIC = 0x58494355;',
|
|
'static constexpr uint8_t VERSION = 2;',
|
|
'',
|
|
'PropTable::PropTable(const char* path) { load(path); }',
|
|
'',
|
|
'PropTable::~PropTable() {',
|
|
' if (owns_data_ && data_) { delete[] data_; data_ = nullptr; }',
|
|
'}',
|
|
'',
|
|
'PropTable::PropTable(PropTable&& other) noexcept',
|
|
' : data_(other.data_), data_size_(other.data_size_), owns_data_(other.owns_data_) {',
|
|
' other.data_ = nullptr; other.owns_data_ = false;',
|
|
'}',
|
|
'',
|
|
'PropTable& PropTable::operator=(PropTable&& other) noexcept {',
|
|
' if (this != &other) {',
|
|
' if (owns_data_ && data_) delete[] data_;',
|
|
' data_ = other.data_; data_size_ = other.data_size_; owns_data_ = other.owns_data_;',
|
|
' other.data_ = nullptr; other.owns_data_ = false;',
|
|
' }',
|
|
' return *this;',
|
|
'}',
|
|
'',
|
|
'uint32_t PropTable::readVarint(const uint8_t*& ptr) {',
|
|
' uint32_t val = 0; int shift = 0;',
|
|
' while (true) {',
|
|
' uint8_t b = *ptr++;',
|
|
' val |= (b & 0x7F) << shift;',
|
|
' if (!(b & 0x80)) break;',
|
|
' shift += 7;',
|
|
' }',
|
|
' return val;',
|
|
'}',
|
|
'',
|
|
'bool PropTable::load(const char* path) {',
|
|
' if (owns_data_ && data_) { delete[] data_; data_ = nullptr; owns_data_ = false; }',
|
|
' FILE* f = std::fopen(path, "rb");',
|
|
' if (!f) return false;',
|
|
' std::fseek(f, 0, SEEK_END);',
|
|
' long fsize = std::ftell(f);',
|
|
' std::fseek(f, 0, SEEK_SET);',
|
|
' if (fsize < 6) { std::fclose(f); return false; }',
|
|
' data_size_ = static_cast<size_t>(fsize);',
|
|
' uint8_t* buf = new uint8_t[data_size_];',
|
|
' size_t read = std::fread(buf, 1, data_size_, f);',
|
|
' std::fclose(f);',
|
|
' if (read != data_size_) { delete[] buf; return false; }',
|
|
' data_ = buf; owns_data_ = true;',
|
|
' ',
|
|
' const uint8_t* ptr = data_;',
|
|
' uint32_t magic = *reinterpret_cast<const uint32_t*>(ptr); ptr += 4;',
|
|
' if (magic != MAGIC) return false;',
|
|
' uint8_t version = *ptr++;',
|
|
' if (version != VERSION) return false;',
|
|
' return true;',
|
|
'}',
|
|
'',
|
|
'// Find field data pointer (points to data length varint after name)',
|
|
'const uint8_t* PropTable::findFieldData(const char* name, uint8_t& out_type, uint8_t& out_strat) const {',
|
|
' if (!data_) return nullptr;',
|
|
' const uint8_t* ptr = data_ + 5; // magic + version',
|
|
' uint32_t num_fields = readVarint(ptr);',
|
|
' for (uint32_t i = 0; i < num_fields; ++i) {',
|
|
' uint8_t ftype = *ptr++;',
|
|
' uint8_t fstrat = *ptr++;',
|
|
' const char* fname = reinterpret_cast<const char*>(ptr);',
|
|
' size_t name_len = std::strlen(fname);',
|
|
' ptr += name_len + 1;',
|
|
' if (std::strcmp(fname, name) == 0) {',
|
|
' out_type = ftype;',
|
|
' out_strat = fstrat;',
|
|
' return ptr; // points to data length varint',
|
|
' }',
|
|
' // Skip data',
|
|
' uint32_t data_len = readVarint(ptr);',
|
|
' ptr += data_len;',
|
|
' }',
|
|
' return nullptr;',
|
|
'}',
|
|
'',
|
|
'// Find string field (returns pointer to indices data length)',
|
|
'const uint8_t* PropTable::findStringField(const char* name) const {',
|
|
' if (!data_) return nullptr;',
|
|
' const uint8_t* ptr = data_ + 5;',
|
|
' uint32_t num_fields = readVarint(ptr);',
|
|
' // Skip non-string fields (with inline data)',
|
|
' for (uint32_t i = 0; i < num_fields; ++i) {',
|
|
' ptr++; ptr++; // type, strat',
|
|
' while (*ptr++) {} // skip name',
|
|
' uint32_t data_len = readVarint(ptr);',
|
|
' ptr += data_len; // skip data',
|
|
' }',
|
|
' uint32_t num_str_fields = readVarint(ptr);',
|
|
' for (uint32_t i = 0; i < num_str_fields; ++i) {',
|
|
' const char* fname = reinterpret_cast<const char*>(ptr);',
|
|
' size_t name_len = std::strlen(fname);',
|
|
' ptr += name_len + 1;',
|
|
' if (std::strcmp(fname, name) == 0) {',
|
|
' return ptr; // points to indices data length',
|
|
' }',
|
|
' uint32_t data_len = readVarint(ptr);',
|
|
' ptr += data_len; // skip indices data',
|
|
' }',
|
|
' return nullptr;',
|
|
'}',
|
|
'',
|
|
'// Decode bitmap: 1 bit per codepoint',
|
|
'bool PropTable::decodeBitmap(const uint8_t* data, uint32_t cp) const {',
|
|
' size_t byte_idx = cp >> 3;',
|
|
' uint8_t bit = cp & 7;',
|
|
' return (data[byte_idx] & (1 << bit)) != 0;',
|
|
'}',
|
|
'',
|
|
'// Decode delta ranges: (start, length, value) varint-encoded',
|
|
'bool PropTable::decodeDeltaRangesBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {',
|
|
' const uint8_t* ptr = data;',
|
|
' while (ptr < data_end) {',
|
|
' uint32_t start = readVarint(ptr);',
|
|
' uint32_t length = readVarint(ptr);',
|
|
' uint8_t val = *ptr++;',
|
|
' uint32_t end = start + length - 1;',
|
|
' if (cp >= start && cp <= end) return val != 0;',
|
|
' if (cp < start) return false;',
|
|
' }',
|
|
' return false;',
|
|
'}',
|
|
'',
|
|
'int32_t PropTable::decodeDeltaRangesInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {',
|
|
' const uint8_t* ptr = data;',
|
|
' while (ptr < data_end) {',
|
|
' uint32_t start = readVarint(ptr);',
|
|
' uint32_t length = readVarint(ptr);',
|
|
' uint32_t zigzag = readVarint(ptr);',
|
|
' int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));',
|
|
' uint32_t end = start + length - 1;',
|
|
' if (cp >= start && cp <= end) return val;',
|
|
' if (cp < start) return 0;',
|
|
' }',
|
|
' return 0;',
|
|
'}',
|
|
'',
|
|
'// Decode sparse map: (cp, value) pairs',
|
|
'bool PropTable::decodeSparseMapBool(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {',
|
|
' const uint8_t* ptr = data;',
|
|
' while (ptr < data_end) {',
|
|
' uint32_t c = readVarint(ptr);',
|
|
' if (ptr >= data_end) break;',
|
|
' uint8_t val = *ptr++;',
|
|
' if (c == cp) return val != 0;',
|
|
' if (c > cp) return false;',
|
|
' }',
|
|
' return false;',
|
|
'}',
|
|
'',
|
|
'int32_t PropTable::decodeSparseMapInt(const uint8_t* data, const uint8_t* data_end, uint32_t cp) const {',
|
|
' const uint8_t* ptr = data;',
|
|
' while (ptr < data_end) {',
|
|
' uint32_t c = readVarint(ptr);',
|
|
' if (ptr >= data_end) break;',
|
|
' uint32_t zigzag = readVarint(ptr);',
|
|
' int32_t val = static_cast<int32_t>((zigzag >> 1) ^ -(zigzag & 1));',
|
|
' if (c == cp) return val;',
|
|
' if (c > cp) return 0;',
|
|
' }',
|
|
' return 0;',
|
|
'}',
|
|
'',
|
|
]
|
|
|
|
# Generate getters for each field
|
|
for name, fc in self.field_configs.items():
|
|
if fc.type == 'bool':
|
|
strat = fc.strategy.value
|
|
if strat == 'bitmap':
|
|
decoder = 'decodeBitmap'
|
|
call = 'decodeBitmap(data, cp)'
|
|
elif strat == 'delta_ranges':
|
|
decoder = 'decodeDeltaRangesBool'
|
|
call = 'decodeDeltaRangesBool(data, data_end, cp)'
|
|
elif strat == 'sparse_map':
|
|
decoder = 'decodeSparseMapBool'
|
|
call = 'decodeSparseMapBool(data, data_end, cp)'
|
|
else:
|
|
decoder = 'decodeDeltaRangesBool'
|
|
call = 'decodeDeltaRangesBool(data, data_end, cp)'
|
|
|
|
lines.extend([
|
|
f'bool PropTable::get{name.capitalize()}(uint32_t cp) const {{',
|
|
f' uint8_t ftype, fstrat;',
|
|
f' const uint8_t* data = findFieldData("{name}", ftype, fstrat);',
|
|
f' if (!data) return false;',
|
|
f' uint32_t data_len = readVarint(data);',
|
|
f' const uint8_t* data_end = data + data_len;',
|
|
f' return {call};',
|
|
f'}}',
|
|
'',
|
|
])
|
|
elif fc.type == 'int':
|
|
strat = fc.strategy.value
|
|
if strat == 'delta_ranges':
|
|
decoder = 'decodeDeltaRangesInt'
|
|
elif strat == 'sparse_map':
|
|
decoder = 'decodeSparseMapInt'
|
|
else:
|
|
decoder = 'decodeDeltaRangesInt'
|
|
|
|
lines.extend([
|
|
f'int32_t PropTable::get{name.capitalize()}(uint32_t cp) const {{',
|
|
f' uint8_t ftype, fstrat;',
|
|
f' const uint8_t* data = findFieldData("{name}", ftype, fstrat);',
|
|
f' if (!data) return 0;',
|
|
f' uint32_t data_len = readVarint(data);',
|
|
f' const uint8_t* data_end = data + data_len;',
|
|
f' return {decoder}(data, data_end, cp);',
|
|
f'}}',
|
|
'',
|
|
])
|
|
elif fc.type == 'str':
|
|
lines.extend([
|
|
f'std::string_view PropTable::get{name.capitalize()}(uint32_t cp) const {{',
|
|
f' // String fields use separate .str file',
|
|
f' const uint8_t* indices_ptr = findStringField("{name}");',
|
|
f' if (!indices_ptr) return "";',
|
|
f' uint32_t indices_len = readVarint(indices_ptr);',
|
|
f' // Find index for this codepoint (linear scan for now)',
|
|
f' // TODO: optimize with binary search if needed',
|
|
f' // For now, we need the string pool loaded separately',
|
|
f' return ""; // Requires string pool file',
|
|
f'}}',
|
|
'',
|
|
])
|
|
|
|
lines.append('} // namespace xicu')
|
|
return '\n'.join(lines)
|
|
|
|
def run(self):
|
|
self.download_unicode_data()
|
|
self.parse_and_process()
|
|
self.analyze_and_configure()
|
|
self.encode_all()
|
|
self.write_binary()
|
|
self.generate_cpp()
|
|
print("\nGeneration complete!")
|
|
|
|
def main():
|
|
gen = XicuGenerator(CONFIG)
|
|
gen.run()
|
|
|
|
if __name__ == '__main__':
|
|
main() |