143 lines
4.3 KiB
Python
143 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generate a single binary table with delta encoding for:
|
|
- isAlphanumeric: letter (L*) or decimal digit (Nd)
|
|
- isWhitespace: Zs, Zl, Zp categories or control whitespace chars
|
|
"""
|
|
|
|
import os
|
|
import struct
|
|
|
|
DATA_DIR = './data'
|
|
OUT_DIR = './out'
|
|
|
|
os.makedirs(DATA_DIR, exist_ok=True)
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
|
|
UNICODE_DATA_PATH = os.path.join(DATA_DIR, 'UnicodeData.txt')
|
|
|
|
def parse_unicode_data():
|
|
"""Parse UnicodeData.txt and yield (codepoint, category)"""
|
|
with open(UNICODE_DATA_PATH, 'r', encoding='utf-8') as f:
|
|
range_start = None
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
parts = line.split(';')
|
|
if len(parts) < 3:
|
|
continue
|
|
codepoint = int(parts[0], 16)
|
|
name = parts[1]
|
|
category = parts[2]
|
|
|
|
if name.endswith(', First>'):
|
|
range_start = (codepoint, category)
|
|
continue
|
|
elif name.endswith(', Last>') and range_start:
|
|
start_cp, start_cat = range_start
|
|
for cp in range(start_cp, codepoint + 1):
|
|
yield (cp, start_cat)
|
|
range_start = None
|
|
continue
|
|
|
|
yield (codepoint, category)
|
|
|
|
def get_props(category: str, codepoint: int) -> int:
|
|
"""Return 2-bit flags: bit0=alphanumeric, bit1=whitespace"""
|
|
flags = 0
|
|
# Alphanumeric: Letter (L*) or Decimal digit (Nd)
|
|
if category.startswith('L') or category == 'Nd':
|
|
flags |= 1 # bit 0
|
|
# Whitespace: Z* (Zs, Zl, Zp) or control chars
|
|
if category.startswith('Z') or codepoint in (0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1C, 0x1D, 0x1E, 0x1F, 0x85, 0xA0):
|
|
flags |= 2 # bit 1
|
|
return flags
|
|
|
|
def build_delta_ranges():
|
|
"""Build delta-encoded ranges: (start, end, flags)"""
|
|
ranges = []
|
|
current_start = None
|
|
current_flags = None
|
|
|
|
for cp, cat in parse_unicode_data():
|
|
flags = get_props(cat, cp)
|
|
|
|
if current_start is None:
|
|
current_start = cp
|
|
current_flags = flags
|
|
elif flags != current_flags:
|
|
# End previous range
|
|
ranges.append((current_start, cp - 1, current_flags))
|
|
current_start = cp
|
|
current_flags = flags
|
|
# else: continue current range
|
|
|
|
# Don't forget the last range
|
|
if current_start is not None:
|
|
ranges.append((current_start, 0x10FFFF, current_flags))
|
|
|
|
return ranges
|
|
|
|
def write_binary(path: str, ranges):
|
|
"""Write delta-encoded binary format:
|
|
Header: magic(4), version(1), num_ranges(4)
|
|
Each range: start(4), end(4), flags(1) - but we'll pack efficiently
|
|
|
|
Optimized format:
|
|
- magic: 'XICU' (0x58494355)
|
|
- version: 1
|
|
- num_ranges: uint32
|
|
- For each range: varint start, varint length, flags(1 byte)
|
|
"""
|
|
def write_varint(f, val):
|
|
while val >= 0x80:
|
|
f.write(bytes([(val & 0x7F) | 0x80]))
|
|
val >>= 7
|
|
f.write(bytes([val]))
|
|
|
|
with open(path, 'wb') as f:
|
|
f.write(struct.pack('<I', 0x58494355)) # magic 'XICU'
|
|
f.write(struct.pack('<B', 1)) # version
|
|
write_varint(f, len(ranges)) # num_ranges
|
|
|
|
for start, end, flags in ranges:
|
|
length = end - start + 1
|
|
write_varint(f, start)
|
|
write_varint(f, length)
|
|
f.write(bytes([flags]))
|
|
|
|
def read_varint(data, offset):
|
|
val = 0
|
|
shift = 0
|
|
while True:
|
|
b = data[offset]
|
|
offset += 1
|
|
val |= (b & 0x7F) << shift
|
|
if not (b & 0x80):
|
|
break
|
|
shift += 7
|
|
return val, offset
|
|
|
|
def main():
|
|
print("Building delta-encoded ranges...")
|
|
ranges = build_delta_ranges()
|
|
|
|
print(f"Total ranges: {len(ranges)}")
|
|
|
|
# Stats
|
|
alnum_count = sum(end - start + 1 for start, end, f in ranges if f & 1)
|
|
ws_count = sum(end - start + 1 for start, end, f in ranges if f & 2)
|
|
print(f"Alphanumeric codepoints: {alnum_count}")
|
|
print(f"Whitespace codepoints: {ws_count}")
|
|
|
|
out_path = os.path.join(OUT_DIR, 'props.bin')
|
|
print(f"Writing {out_path}...")
|
|
write_binary(out_path, ranges)
|
|
|
|
size = os.path.getsize(out_path)
|
|
print(f"File size: {size} bytes ({size/1024:.1f} KB)")
|
|
print("Done!")
|
|
|
|
if __name__ == '__main__':
|
|
main() |