#!/usr/bin/env python3
"""
Generate a single binary table with delta encoding for:
- isAlphanumeric: letter (L*) or decimal digit (Nd)
- isWhitespace: Zs, Zl, Zp categories or control whitespace chars
"""

import os
import struct

DATA_DIR = './data'
OUT_DIR = './out'

os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(OUT_DIR, exist_ok=True)

UNICODE_DATA_PATH = os.path.join(DATA_DIR, 'UnicodeData.txt')

def parse_unicode_data():
    """Parse UnicodeData.txt and yield (codepoint, category)"""
    with open(UNICODE_DATA_PATH, 'r', encoding='utf-8') as f:
        range_start = None
        for line in f:
            line = line.strip()
            if not line:
                continue
            parts = line.split(';')
            if len(parts) < 3:
                continue
            codepoint = int(parts[0], 16)
            name = parts[1]
            category = parts[2]
            
            if name.endswith(', First>'):
                range_start = (codepoint, category)
                continue
            elif name.endswith(', Last>') and range_start:
                start_cp, start_cat = range_start
                for cp in range(start_cp, codepoint + 1):
                    yield (cp, start_cat)
                range_start = None
                continue
            
            yield (codepoint, category)

def get_props(category: str, codepoint: int) -> int:
    """Return 2-bit flags: bit0=alphanumeric, bit1=whitespace"""
    flags = 0
    # Alphanumeric: Letter (L*) or Decimal digit (Nd)
    if category.startswith('L') or category == 'Nd':
        flags |= 1  # bit 0
    # Whitespace: Z* (Zs, Zl, Zp) or control chars
    if category.startswith('Z') or codepoint in (0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1C, 0x1D, 0x1E, 0x1F, 0x85, 0xA0):
        flags |= 2  # bit 1
    return flags

def build_delta_ranges():
    """Build delta-encoded ranges: (start, end, flags)"""
    ranges = []
    current_start = None
    current_flags = None
    
    for cp, cat in parse_unicode_data():
        flags = get_props(cat, cp)
        
        if current_start is None:
            current_start = cp
            current_flags = flags
        elif flags != current_flags:
            # End previous range
            ranges.append((current_start, cp - 1, current_flags))
            current_start = cp
            current_flags = flags
        # else: continue current range
    
    # Don't forget the last range
    if current_start is not None:
        ranges.append((current_start, 0x10FFFF, current_flags))
    
    return ranges

def write_binary(path: str, ranges):
    """Write delta-encoded binary format:
    Header: magic(4), version(1), num_ranges(4)
    Each range: start(4), end(4), flags(1) - but we'll pack efficiently
    
    Optimized format:
    - magic: 'XICU' (0x58494355)
    - version: 1
    - num_ranges: uint32
    - For each range: varint start, varint length, flags(1 byte)
    """
    def write_varint(f, val):
        while val >= 0x80:
            f.write(bytes([(val & 0x7F) | 0x80]))
            val >>= 7
        f.write(bytes([val]))
    
    with open(path, 'wb') as f:
        f.write(struct.pack('<I', 0x58494355))  # magic 'XICU'
        f.write(struct.pack('<B', 1))            # version
        write_varint(f, len(ranges))             # num_ranges
        
        for start, end, flags in ranges:
            length = end - start + 1
            write_varint(f, start)
            write_varint(f, length)
            f.write(bytes([flags]))

def read_varint(data, offset):
    val = 0
    shift = 0
    while True:
        b = data[offset]
        offset += 1
        val |= (b & 0x7F) << shift
        if not (b & 0x80):
            break
        shift += 7
    return val, offset

def main():
    print("Building delta-encoded ranges...")
    ranges = build_delta_ranges()
    
    print(f"Total ranges: {len(ranges)}")
    
    # Stats
    alnum_count = sum(end - start + 1 for start, end, f in ranges if f & 1)
    ws_count = sum(end - start + 1 for start, end, f in ranges if f & 2)
    print(f"Alphanumeric codepoints: {alnum_count}")
    print(f"Whitespace codepoints: {ws_count}")
    
    out_path = os.path.join(OUT_DIR, 'props.bin')
    print(f"Writing {out_path}...")
    write_binary(out_path, ranges)
    
    size = os.path.getsize(out_path)
    print(f"File size: {size} bytes ({size/1024:.1f} KB)")
    print("Done!")

if __name__ == '__main__':
    main()