{ "cells": [ { "cell_type": "markdown", "id": "4502a404", "metadata": {}, "source": [ "# x-icu\n", "> **The lightweight bridge to ICU data.**" ] }, { "cell_type": "code", "execution_count": 1, "id": "2baaf74f", "metadata": {}, "outputs": [], "source": [ "# Add libraries...\n", "import os\n", "import numpy as np\n", "import pandas as pd\n", "import urllib.request\n" ] }, { "cell_type": "code", "execution_count": 24, "id": "41361ca4", "metadata": {}, "outputs": [], "source": [ "# Configure what to get\n", "CONFIG = {\n", " 'sources':{\n", " 'unicode':'https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt',\n", " },\n", " 'unicode':{\n", " # Unicode 1.0 name, considered \"old\"\n", " # Set to None to use the old name if the\n", " # current name is not descriptive\n", " # e.g. use \"START OF TEXT\" instead of \"\"\n", " 'useOldName':True,\n", " 'getName':True, # Formal name of the codepoint\n", "\n", " # PROBABLY REMOVE # 'getCategory':False,\n", " # TODO # 'getScript': True, # e.g., \"Cyrillic\"\n", " # TODO # 'getBlock': True, # e.g., \"Mathematical Alphanumeric Symbols\"\n", "\n", " 'getDecomposition':False, # aka \"ascii-fy\"\n", " 'getDecompositionType':False, # metadata about decomposition\n", "\n", " 'toLowercase':True,\n", " 'toUppercase':True,\n", " 'toTitlecase':False,\n", "\n", " # TODO: 'isEmoji': True, # Does it have an emoji presentation?\n", " 'isPunctuation': True, # General Category starts with 'P'\n", " 'isSymbol': False, # General Category starts with 'S' (Math, Currency)\n", " 'isCombining': False, # Is it a mark/accent that needs a base letter?\n", "\n", " 'isPrintable':False,\n", " 'isSpace':True,\n", " 'isWhitespace':True,\n", " 'isLetter':True,\n", " 'isUppercase':False,\n", " 'isLowercase':False,\n", " 'isTitlecase':False,\n", " 'isDeprecated':False,\n", "\n", " # NOTE: The following fields work in a cascade,\n", " # so if one field is true, the next one is also true.\n", " # This is defined by unicode, we don't do any extra processing.\n", " 'isDecimal':True, # An actual number 0-9, in different languages\n", " 'isDigit':False, # Digits not used in standard positional notation,\n", " # like superscripts or circled numbers.\n", " # Usually used by search engines.\n", " 'isNumberLike':False, # Neither of the previous two, but number like in nature, like 3/4\n", " }\n", "}\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "94696c64", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "# Decoders the UnicodeData.txt data\n", "class UnicodeDataParser:\n", "\n", " def __init__(self):\n", " self.buffer = []\n", " self.range_start_row = None\n", " self.keys = [\n", " \"codepoint\", \"name\", \"category\", \"combining_class\", \"bidi_class\",\n", " \"decomposition\", \"decimal_val\", \"digit_val\", \"numeric_val\",\n", " \"bidi_mirrored\", \"unicode_1_name\", \"iso_comment\", \n", " \"uppercase_map\", \"lowercase_map\", \"titlecase_map\"\n", " ]\n", "\n", " def parse_line(self, text_line: str):\n", " if not text_line.strip():\n", " return\n", " \n", " parts = text_line.strip().split(';')\n", " # Ensure we have 15 columns as per spec\n", " if len(parts) < 15:\n", " return\n", " \n", " row = dict(zip(self.keys, parts))\n", " row['codepoint'] = int(row['codepoint'], 16)\n", " name = row['name']\n", "\n", " # Detect Range Start: e.g., \"\"\n", " if name.endswith(', First>'):\n", " self.range_start_row = row\n", " \n", " # Detect Range End: e.g., \"\"\n", " elif name.endswith(', Last>') and self.range_start_row:\n", " self._fill_range(row)\n", " self.range_start_row = None\n", " \n", " # Standard single codepoint\n", " else:\n", " self.buffer.append( self.normalize(row) )\n", "\n", " def _fill_range(self, end_row):\n", " # Linearly interpolates all codepoints between First and Last.\n", " start_hex = self.range_start_row['codepoint']\n", " end_hex = end_row['codepoint']\n", " \n", " # Generic name for the range (stripping the \", First>\" part)\n", " base_name = self.range_start_row['name'].replace(', First>', '').replace('<', '')\n", "\n", " # Loop this range then\n", " for cp in range(start_hex, end_hex + 1):\n", " new_row = self.range_start_row.copy()\n", " new_row['codepoint'] = cp\n", " new_row['name'] = f\"<{base_name}>\"\n", " self.buffer.append( self.normalize(new_row) )\n", "\n", " def normalize(self, in_row:dict):\n", " # Now, parse the fields individually\n", " out_row = {}\n", "\n", " # Parse codepoint\n", " out_row['codepoint'] = in_row['codepoint']\n", " char = chr(out_row['codepoint'])\n", "\n", " out_row['name'] = in_row['name']\n", " out_row['old_name'] = in_row['unicode_1_name']\n", "\n", " out_row['decomposition'] = in_row['decomposition']\n", "\n", " # Extracts text inside <> like or \n", " decomp = in_row['decomposition']\n", " out_row['decomposition_type'] = decomp[1:decomp.find('>')] if '<' in decomp else None\n", "\n", " category = in_row['category']\n", " out_row['punctuation'] = category.startswith('P') \n", " out_row['symbol'] = category.startswith('S')\n", " out_row['combining'] = category.startswith('M') # Mark category\n", " out_row['letter'] = category.startswith('L')\n", "\n", " out_row['uppercase'] = category == 'Lu'\n", " out_row['lowercase'] = category == 'Ll'\n", " out_row['titlecase'] = category == 'Lt'\n", "\n", " # Mappings\n", " for k in [\"uppercase_map\", \"lowercase_map\", \"titlecase_map\"]:\n", " out_row[k] = int(in_row[k], 16) if in_row[k] != '' else 0\n", "\n", " # Zs = Space Separator, but also check common control whitespaces\n", " out_row['whitespace'] = category == 'Zs' or char in '\\t\\n\\r\\f\\v'\n", "\n", " # Non-printable are usually Control (C) and some Separator (Z) categories\n", " out_row['printable'] = not category.startswith('C')\n", "\n", " # Decimal (0-9) -> Digit (Superscripts) -> NumberLike (Fractions/Roman)\n", " is_decimal = bool(in_row['decimal_val'])\n", " is_digit = is_decimal or bool(in_row['digit_val'])\n", " is_numeric = is_digit or bool(in_row['numeric_val'])\n", "\n", " out_row['decimal'] = is_decimal\n", " out_row['digit'] = is_digit\n", " out_row['number_like'] = is_numeric\n", "\n", " # return the data\n", " return out_row\n", "\n", " def get_dataframe(self):\n", " df = pd.DataFrame(self.buffer)\n", " self.buffer.clear()\n", " return df\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "cfb55a58", "metadata": {}, "outputs": [], "source": [ "# Download the data\n", "DATA_DIR = './data'\n", "os.makedirs(DATA_DIR, exist_ok=True)\n", "\n", "def downloadFile(key:str, url:str):\n", " urllib.request.urlretrieve(url, f'{DATA_DIR}/{key}.bin')\n", "\n", "for k in CONFIG['sources']:\n", " downloadFile(k, CONFIG['sources'][k])\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "7dac9d1f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
codepointnameold_namedecompositiondecomposition_typepunctuationsymbolcombiningletteruppercaselowercasetitlecaseuppercase_maplowercase_maptitlecase_mapwhitespaceprintabledecimaldigitnumber_like
00<control>NULLNaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
11<control>START OF HEADINGNaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
22<control>START OF TEXTNaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
33<control>END OF TEXTNaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
44<control>END OF TRANSMISSIONNaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
...............................................................
2993771114105<Plane 16 Private Use>NaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
2993781114106<Plane 16 Private Use>NaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
2993791114107<Plane 16 Private Use>NaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
2993801114108<Plane 16 Private Use>NaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
2993811114109<Plane 16 Private Use>NaNFalseFalseFalseFalseFalseFalseFalse000FalseFalseFalseFalseFalse
\n", "

299382 rows × 20 columns

\n", "
" ], "text/plain": [ " codepoint name old_name decomposition \\\n", "0 0 NULL \n", "1 1 START OF HEADING \n", "2 2 START OF TEXT \n", "3 3 END OF TEXT \n", "4 4 END OF TRANSMISSION \n", "... ... ... ... ... \n", "299377 1114105 \n", "299378 1114106 \n", "299379 1114107 \n", "299380 1114108 \n", "299381 1114109 \n", "\n", " decomposition_type punctuation symbol combining letter uppercase \\\n", "0 NaN False False False False False \n", "1 NaN False False False False False \n", "2 NaN False False False False False \n", "3 NaN False False False False False \n", "4 NaN False False False False False \n", "... ... ... ... ... ... ... \n", "299377 NaN False False False False False \n", "299378 NaN False False False False False \n", "299379 NaN False False False False False \n", "299380 NaN False False False False False \n", "299381 NaN False False False False False \n", "\n", " lowercase titlecase uppercase_map lowercase_map titlecase_map \\\n", "0 False False 0 0 0 \n", "1 False False 0 0 0 \n", "2 False False 0 0 0 \n", "3 False False 0 0 0 \n", "4 False False 0 0 0 \n", "... ... ... ... ... ... \n", "299377 False False 0 0 0 \n", "299378 False False 0 0 0 \n", "299379 False False 0 0 0 \n", "299380 False False 0 0 0 \n", "299381 False False 0 0 0 \n", "\n", " whitespace printable decimal digit number_like \n", "0 False False False False False \n", "1 False False False False False \n", "2 False False False False False \n", "3 False False False False False \n", "4 False False False False False \n", "... ... ... ... ... ... \n", "299377 False False False False False \n", "299378 False False False False False \n", "299379 False False False False False \n", "299380 False False False False False \n", "299381 False False False False False \n", "\n", "[299382 rows x 20 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Now, for each contruct of the configuration, parse the file\n", "# vvv UNICODE vvv #\n", "unicode = UnicodeDataParser()\n", "with open(f'{DATA_DIR}/unicode.bin', 'r') as file:\n", " for line in file:\n", " unicode.parse_line(line)\n", "\n", "df = unicode.get_dataframe()\n", "df\n" ] }, { "cell_type": "code", "execution_count": 6, "id": "2029f383", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
codepointnameold_namedecompositiondecomposition_typepunctuationsymbolcombiningletteruppercaselowercasetitlecaseuppercase_maplowercase_maptitlecase_mapwhitespaceprintabledecimaldigitnumber_like
160160NO-BREAK SPACENON-BREAKING SPACE<noBreak> 0020noBreakFalseFalseFalseFalseFalseFalseFalse000TrueTrueFalseFalseFalse
168168DIAERESISSPACING DIAERESIS<compat> 0020 0308compatFalseTrueFalseFalseFalseFalseFalse000FalseTrueFalseFalseFalse
170170FEMININE ORDINAL INDICATOR<super> 0061superFalseFalseFalseTrueFalseFalseFalse000FalseTrueFalseFalseFalse
175175MACRONSPACING MACRON<compat> 0020 0304compatFalseTrueFalseFalseFalseFalseFalse000FalseTrueFalseFalseFalse
178178SUPERSCRIPT TWOSUPERSCRIPT DIGIT TWO<super> 0032superFalseFalseFalseFalseFalseFalseFalse000FalseTrueFalseTrueTrue
...............................................................
93029130037SEGMENTED DIGIT FIVE<font> 0035fontFalseFalseFalseFalseFalseFalseFalse000FalseTrueTrueTrueTrue
93030130038SEGMENTED DIGIT SIX<font> 0036fontFalseFalseFalseFalseFalseFalseFalse000FalseTrueTrueTrueTrue
93031130039SEGMENTED DIGIT SEVEN<font> 0037fontFalseFalseFalseFalseFalseFalseFalse000FalseTrueTrueTrueTrue
93032130040SEGMENTED DIGIT EIGHT<font> 0038fontFalseFalseFalseFalseFalseFalseFalse000FalseTrueTrueTrueTrue
93033130041SEGMENTED DIGIT NINE<font> 0039fontFalseFalseFalseFalseFalseFalseFalse000FalseTrueTrueTrueTrue
\n", "

3833 rows × 20 columns

\n", "
" ], "text/plain": [ " codepoint name old_name \\\n", "160 160 NO-BREAK SPACE NON-BREAKING SPACE \n", "168 168 DIAERESIS SPACING DIAERESIS \n", "170 170 FEMININE ORDINAL INDICATOR \n", "175 175 MACRON SPACING MACRON \n", "178 178 SUPERSCRIPT TWO SUPERSCRIPT DIGIT TWO \n", "... ... ... ... \n", "93029 130037 SEGMENTED DIGIT FIVE \n", "93030 130038 SEGMENTED DIGIT SIX \n", "93031 130039 SEGMENTED DIGIT SEVEN \n", "93032 130040 SEGMENTED DIGIT EIGHT \n", "93033 130041 SEGMENTED DIGIT NINE \n", "\n", " decomposition decomposition_type punctuation symbol combining \\\n", "160 0020 noBreak False False False \n", "168 0020 0308 compat False True False \n", "170 0061 super False False False \n", "175 0020 0304 compat False True False \n", "178 0032 super False False False \n", "... ... ... ... ... ... \n", "93029 0035 font False False False \n", "93030 0036 font False False False \n", "93031 0037 font False False False \n", "93032 0038 font False False False \n", "93033 0039 font False False False \n", "\n", " letter uppercase lowercase titlecase uppercase_map lowercase_map \\\n", "160 False False False False 0 0 \n", "168 False False False False 0 0 \n", "170 True False False False 0 0 \n", "175 False False False False 0 0 \n", "178 False False False False 0 0 \n", "... ... ... ... ... ... ... \n", "93029 False False False False 0 0 \n", "93030 False False False False 0 0 \n", "93031 False False False False 0 0 \n", "93032 False False False False 0 0 \n", "93033 False False False False 0 0 \n", "\n", " titlecase_map whitespace printable decimal digit number_like \n", "160 0 True True False False False \n", "168 0 False True False False False \n", "170 0 False True False False False \n", "175 0 False True False False False \n", "178 0 False True False True True \n", "... ... ... ... ... ... ... \n", "93029 0 False True True True True \n", "93030 0 False True True True True \n", "93031 0 False True True True True \n", "93032 0 False True True True True \n", "93033 0 False True True True True \n", "\n", "[3833 rows x 20 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# TEST #\n", "df[~df['decomposition_type'].isna()]" ] }, { "cell_type": "code", "execution_count": 29, "id": "d29860e8", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
cp_startcp_endpunctuationletterwhitespacedecimal
0913FalseFalseTrueFalse
13232FalseFalseTrueFalse
23335TrueFalseFalseFalse
33742TrueFalseFalseFalse
44447TrueFalseFalseFalse
.....................
958183984191456FalseTrueFalseFalse
959191472192093FalseTrueFalseFalse
960194560195101FalseTrueFalseFalse
961196608201546FalseTrueFalseFalse
962201552210041FalseTrueFalseFalse
\n", "

963 rows × 6 columns

\n", "
" ], "text/plain": [ " cp_start cp_end punctuation letter whitespace decimal\n", "0 9 13 False False True False\n", "1 32 32 False False True False\n", "2 33 35 True False False False\n", "3 37 42 True False False False\n", "4 44 47 True False False False\n", ".. ... ... ... ... ... ...\n", "958 183984 191456 False True False False\n", "959 191472 192093 False True False False\n", "960 194560 195101 False True False False\n", "961 196608 201546 False True False False\n", "962 201552 210041 False True False False\n", "\n", "[963 rows x 6 columns]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
codepointuppercase_maplowercase_map
065032
166032
267032
368032
469032
............
2984125247-340
2985125248-340
2986125249-340
2987125250-340
2988125251-340
\n", "

2989 rows × 3 columns

\n", "
" ], "text/plain": [ " codepoint uppercase_map lowercase_map\n", "0 65 0 32\n", "1 66 0 32\n", "2 67 0 32\n", "3 68 0 32\n", "4 69 0 32\n", "... ... ... ...\n", "2984 125247 -34 0\n", "2985 125248 -34 0\n", "2986 125249 -34 0\n", "2987 125250 -34 0\n", "2988 125251 -34 0\n", "\n", "[2989 rows x 3 columns]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
codepointname
00NULL
11START OF HEADING
22START OF TEXT
33END OF TEXT
44END OF TRANSMISSION
.........
2993771114105<Plane 16 Private Use>
2993781114106<Plane 16 Private Use>
2993791114107<Plane 16 Private Use>
2993801114108<Plane 16 Private Use>
2993811114109<Plane 16 Private Use>
\n", "

299382 rows × 2 columns

\n", "
" ], "text/plain": [ " codepoint name\n", "0 0 NULL\n", "1 1 START OF HEADING\n", "2 2 START OF TEXT\n", "3 3 END OF TEXT\n", "4 4 END OF TRANSMISSION\n", "... ... ...\n", "299377 1114105 \n", "299378 1114106 \n", "299379 1114107 \n", "299380 1114108 \n", "299381 1114109 \n", "\n", "[299382 rows x 2 columns]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# UNICODE >> Apply config\n", "u_cfg = CONFIG['unicode']\n", "\n", "# If useOldName is True, replace or empty names with old_name\n", "if u_cfg.get('useOldName'):\n", " mask = (df['name'].str.startswith('<')) & (df['old_name'].str.len() > 0)\n", " df.loc[mask, 'name'] = df.loc[mask, 'old_name']\n", "\n", "# Map CONFIG keys to actual DataFrame column names\n", "mapping = {\n", " 'getName': 'name',\n", " 'getDecomposition': 'decomposition',\n", " 'getDecompositionType': 'decomposition_type',\n", " 'isPunctuation': 'punctuation',\n", " 'isSymbol': 'symbol',\n", " 'isCombining': 'combining',\n", " 'isLetter': 'letter',\n", " 'isUppercase': 'uppercase',\n", " 'isLowercase': 'lowercase',\n", " 'isTitlecase': 'titlecase',\n", " 'toUppercase': 'uppercase_map',\n", " 'toLowercase': 'lowercase_map',\n", " 'toTitlecase': 'titlecase_map',\n", " 'isWhitespace': 'whitespace',\n", " 'isPrintable': 'printable',\n", " 'isDecimal': 'decimal',\n", " 'isDigit': 'digit',\n", " 'isNumberLike': 'number_like'\n", "}\n", "\n", "# Categorize these keys\n", "str_props = ['name','decomposition']\n", "bool_props = [ y for x, y in mapping.items() if x.startswith('is') ]\n", "int_props = [ 'decomposition_type', 'uppercase_map', 'lowercase_map', 'titlecase_map' ]\n", "\n", "keep_cols = ['codepoint'] # Always keep codepoint\n", "keep_cols += [ col_name for cfg_key, col_name in mapping.items() if u_cfg.get(cfg_key) ]\n", "\n", "# Filter the dataframe\n", "df_keep = df[keep_cols]\n", "\n", "# Place all string properties in another df\n", "df_str = df_keep[ ['codepoint'] + [x for x in str_props if x in keep_cols] ]\n", "df_int = df_keep[ ['codepoint'] + [x for x in int_props if x in keep_cols] ]\n", "df_bool = df_keep[ ['codepoint'] + [x for x in bool_props if x in keep_cols] ]\n", "\n", "# Convert boolean-like columns to actual bools or int8 to save space\n", "for k in bool_props:\n", " if k in df_bool.columns:\n", " # Convert to bool...\n", " df_bool[k] = df_bool[k].fillna(False).astype(bool)\n", "\n", "\n", "# Remove all rows which are completely false (boolean rows)\n", "feature_cols = [col for col in df_bool.columns if col != 'codepoint']\n", "df_bool = df_bool[(~(df_bool[feature_cols].astype(bool) == False).all(axis=1))]\n", "\n", "feature_cols = [col for col in df_int.columns if col != 'codepoint']\n", "df_int = df_int[(~(df_int[feature_cols].astype(int) == 0).all(axis=1))]\n", "\n", "feature_cols = [col for col in df_str.columns if col != 'codepoint']\n", "df_str = df_str[(~(df_str[feature_cols] == \"\").all(axis=1))]\n", "\n", "# cleanup all 3\n", "df_bool = df_bool.sort_values('codepoint').reset_index(drop=True)\n", "df_int = df_int .sort_values('codepoint').reset_index(drop=True)\n", "df_str = df_str .sort_values('codepoint').reset_index(drop=True)\n", "\n", "# [BOOLEAN OPTIMIZATION]\n", "# Identify where boolean changes (comparing current row to previous)\n", "# We exclude the 'codepoint' from this comparison\n", "metadata_cols = [c for c in df_bool.columns if c != 'codepoint']\n", "metadata_changed = (df_bool[metadata_cols] != df_bool[metadata_cols].shift()).any(axis=1)\n", "# Identify where the codepoint sequence breaks (gap > 1)\n", "sequence_broken = (df_bool['codepoint'].diff() > 1)\n", "# A new range starts if metadata changed OR the sequence broke\n", "range_id = (metadata_changed | sequence_broken).cumsum()\n", "# 4. Group by the range_id and aggregate\n", "df_bool = df_bool.groupby(range_id).agg(\n", " cp_start=('codepoint', 'min'),\n", " cp_end=('codepoint', 'max'),\n", " **{col: (col, 'first') for col in metadata_cols}\n", ")\n", "df_bool = df_bool.reset_index(drop=True)\n", "\n", "# [INT OPTIMIZATION]\n", "for k in ['uppercase_map', 'lowercase_map', 'titlecase_map']:\n", " if k not in df_int.columns: continue\n", " df_int[k] = df_int.apply(lambda x : x[k] - x['codepoint'] if x[k] != 0 else 0, axis=1)\n", "\n", "display(df_bool)\n", "display(df_int)\n", "display(df_str)" ] }, { "cell_type": "code", "execution_count": 19, "id": "73cf0f32", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "False" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "False" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "True" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# TEST #\n", "display(df_str.empty)\n", "display(df_bool.empty)\n", "display(df_int.empty)" ] }, { "cell_type": "code", "execution_count": null, "id": "2a0085c0", "metadata": {}, "outputs": [], "source": [ "# Format of binaries\n", "# \"X-ICU\"\n", "# \n", "# \n", "# " ] }, { "cell_type": "code", "execution_count": null, "id": "a97b59a9", "metadata": {}, "outputs": [], "source": [ "# EXPORT UNICODE #\n", "OUT_DIR = './out'\n", "str_cols = [ 'name', 'decomposition' ]\n", "cp_size = 4\n", "same_size = CONFIG['general']['zeroPadStrings'] \\\n", " or CONFIG['general']['exportStringsInDedicatedFile'] \\\n", " or set(str_cols).isdisjoint(df.columns)\n", "\n", "\n", "\n", "# Select properties which are, and their size\n", "for cfg_key, col_name in mapping.items():\n", " if cfg_key in df.columns and cfg_key.startswith('is'):" ] }, { "cell_type": "code", "execution_count": null, "id": "e75c5b2f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1d0ba29e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.14.3" } }, "nbformat": 4, "nbformat_minor": 5 }