lots of improvements standby for the results

py-gen-props.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""
+Generate a single binary table with delta encoding for:
+- isAlphanumeric: letter (L*) or decimal digit (Nd)
+- isWhitespace: Zs, Zl, Zp categories or control whitespace chars
+"""
+
+import os
+import struct
+
+DATA_DIR = './data'
+OUT_DIR = './out'
+
+os.makedirs(DATA_DIR, exist_ok=True)
+os.makedirs(OUT_DIR, exist_ok=True)
+
+UNICODE_DATA_PATH = os.path.join(DATA_DIR, 'UnicodeData.txt')
+
+def parse_unicode_data():
+    """Parse UnicodeData.txt and yield (codepoint, category)"""
+    with open(UNICODE_DATA_PATH, 'r', encoding='utf-8') as f:
+        range_start = None
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split(';')
+            if len(parts) < 3:
+                continue
+            codepoint = int(parts[0], 16)
+            name = parts[1]
+            category = parts[2]
+            
+            if name.endswith(', First>'):
+                range_start = (codepoint, category)
+                continue
+            elif name.endswith(', Last>') and range_start:
+                start_cp, start_cat = range_start
+                for cp in range(start_cp, codepoint + 1):
+                    yield (cp, start_cat)
+                range_start = None
+                continue
+            
+            yield (codepoint, category)
+
+def get_props(category: str, codepoint: int) -> int:
+    """Return 2-bit flags: bit0=alphanumeric, bit1=whitespace"""
+    flags = 0
+    # Alphanumeric: Letter (L*) or Decimal digit (Nd)
+    if category.startswith('L') or category == 'Nd':
+        flags |= 1  # bit 0
+    # Whitespace: Z* (Zs, Zl, Zp) or control chars
+    if category.startswith('Z') or codepoint in (0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1C, 0x1D, 0x1E, 0x1F, 0x85, 0xA0):
+        flags |= 2  # bit 1
+    return flags
+
+def build_delta_ranges():
+    """Build delta-encoded ranges: (start, end, flags)"""
+    ranges = []
+    current_start = None
+    current_flags = None
+    
+    for cp, cat in parse_unicode_data():
+        flags = get_props(cat, cp)
+        
+        if current_start is None:
+            current_start = cp
+            current_flags = flags
+        elif flags != current_flags:
+            # End previous range
+            ranges.append((current_start, cp - 1, current_flags))
+            current_start = cp
+            current_flags = flags
+        # else: continue current range
+    
+    # Don't forget the last range
+    if current_start is not None:
+        ranges.append((current_start, 0x10FFFF, current_flags))
+    
+    return ranges
+
+def write_binary(path: str, ranges):
+    """Write delta-encoded binary format:
+    Header: magic(4), version(1), num_ranges(4)
+    Each range: start(4), end(4), flags(1) - but we'll pack efficiently
+    
+    Optimized format:
+    - magic: 'XICU' (0x58494355)
+    - version: 1
+    - num_ranges: uint32
+    - For each range: varint start, varint length, flags(1 byte)
+    """
+    def write_varint(f, val):
+        while val >= 0x80:
+            f.write(bytes([(val & 0x7F) | 0x80]))
+            val >>= 7
+        f.write(bytes([val]))
+    
+    with open(path, 'wb') as f:
+        f.write(struct.pack('<I', 0x58494355))  # magic 'XICU'
+        f.write(struct.pack('<B', 1))            # version
+        write_varint(f, len(ranges))             # num_ranges
+        
+        for start, end, flags in ranges:
+            length = end - start + 1
+            write_varint(f, start)
+            write_varint(f, length)
+            f.write(bytes([flags]))
+
+def read_varint(data, offset):
+    val = 0
+    shift = 0
+    while True:
+        b = data[offset]
+        offset += 1
+        val |= (b & 0x7F) << shift
+        if not (b & 0x80):
+            break
+        shift += 7
+    return val, offset
+
+def main():
+    print("Building delta-encoded ranges...")
+    ranges = build_delta_ranges()
+    
+    print(f"Total ranges: {len(ranges)}")
+    
+    # Stats
+    alnum_count = sum(end - start + 1 for start, end, f in ranges if f & 1)
+    ws_count = sum(end - start + 1 for start, end, f in ranges if f & 2)
+    print(f"Alphanumeric codepoints: {alnum_count}")
+    print(f"Whitespace codepoints: {ws_count}")
+    
+    out_path = os.path.join(OUT_DIR, 'props.bin')
+    print(f"Writing {out_path}...")
+    write_binary(out_path, ranges)
+    
+    size = os.path.getsize(out_path)
+    print(f"File size: {size} bytes ({size/1024:.1f} KB)")
+    print("Done!")
+
+if __name__ == '__main__':
+    main()