linux/drivers/tty/vt/gen_ucs_fallback_table.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
#
# Leverage Python's unidecode module to generate ucs_fallback_table.h
#
# The generated table maps complex characters to their simpler fallback forms
# for a terminal display when corresponding glyphs are unavailable.
#
# Usage:
#   python3 gen_ucs_fallback_table.py         # Generate fallback tables
#   python3 gen_ucs_fallback_table.py -o FILE # Specify output file

import unicodedata
from unidecode import unidecode
import sys
import argparse
from collections import defaultdict

# Try to get unidecode version
try:
    from importlib.metadata import version
    unidecode_version = version('unidecode')
except:
    unidecode_version = 'unknown'

# This script's file name
from pathlib import Path
this_file = Path(__file__).name

# Default output file name
DEFAULT_OUT_FILE = "ucs_fallback_table.h"

# Define the range marker value
RANGE_MARKER = 0x00

def generate_fallback_map():
    """Generate a fallback map using unidecode for all relevant Unicode points."""
    fallback_map = {}

    # Process BMP characters (0x0000 - 0xFFFF) to keep table size manageable
    for cp in range(0x0080, 0x10000):  # Skip ASCII range (0x00-0x7F)
        char = chr(cp)

        # Skip unassigned/control characters
        try:
            if not unicodedata.name(char, ''):
                continue
        except ValueError:
            continue

        # Get the unidecode transliteration
        ascii_version = unidecode(char)

        # Only store if it results in a single character mapping
        if len(ascii_version) == 1:
            fallback_map[cp] = ord(ascii_version)

    # Apply manual overrides for special cases
    fallback_map.update(get_special_overrides())

    return fallback_map

def get_special_overrides():
    """Get special case overrides that need different handling than unidecode
    provides... or doesn't provide at all."""

    overrides = {}

    # Multi-character unidecode output
    # These map to single chars instead of unidecode's multiple-char mappings
    # In a terminal fallback context, we need a single character rather than multiple
    overrides[0x00C6] = ord('E')  # Æ LATIN CAPITAL LETTER AE -> E (unidecode: "AE")
    overrides[0x00E6] = ord('e')  # æ LATIN SMALL LETTER AE -> e (unidecode: "ae")
    overrides[0x0152] = ord('E')  # ΠLATIN CAPITAL LIGATURE OE -> E (unidecode: "OE")
    overrides[0x0153] = ord('e')  # œ LATIN SMALL LETTER LIGATURE OE -> e (unidecode: "oe")
    overrides[0x00DF] = ord('s')  # ß LATIN SMALL LETTER SHARP S -> s (unidecode: "ss")

    # Comparison operators that unidecode renders as multiple characters
    overrides[0x2264] = ord('<')  # ≤ LESS-THAN OR EQUAL TO -> < (unidecode: "<=")
    overrides[0x2265] = ord('>')  # ≥ GREATER-THAN OR EQUAL TO -> > (unidecode: ">=")

    # Unidecode returns an empty string for these
    overrides[0x2260] = ord('#')  # ≠ NOT EQUAL TO -> # (unidecode: empty string)

    # Quadrant block characters that unidecode doesn't map
    for cp in range(0x2596, 0x259F+1):
        overrides[cp] = ord('#')  # ▖ ▗ ▘ ▙ etc. - map to # (unidecode: empty string)

    # Directional arrows
    # These provide better semantic meaning than unidecode's mappings
    overrides[0x2192] = ord('>')  # → RIGHTWARDS ARROW -> > (unidecode: "-")
    overrides[0x2190] = ord('<')  # ← LEFTWARDS ARROW -> < (unidecode: "-")
    overrides[0x2191] = ord('^')  # ↑ UPWARDS ARROW -> ^ (unidecode: "|")
    overrides[0x2193] = ord('v')  # ↓ DOWNWARDS ARROW -> v (unidecode: "|")

    # Double arrows with their directional semantic mappings
    overrides[0x21D0] = ord('<')  # ⇐ LEFTWARDS DOUBLE ARROW -> <
    overrides[0x21D1] = ord('^')  # ⇑ UPWARDS DOUBLE ARROW -> ^
    overrides[0x21D2] = ord('>')  # ⇒ RIGHTWARDS DOUBLE ARROW -> >
    overrides[0x21D3] = ord('v')  # ⇓ DOWNWARDS DOUBLE ARROW -> v

    # Halfwidth arrows
    # These need the same treatment as their normal-width counterparts
    overrides[0xFFE9] = ord('<')  # ￩ HALFWIDTH LEFTWARDS ARROW -> < (unidecode: "-")
    overrides[0xFFEA] = ord('^')  # ￪ HALFWIDTH UPWARDS ARROW -> ^ (unidecode: "|")
    overrides[0xFFEB] = ord('>')  # ￫ HALFWIDTH RIGHTWARDS ARROW -> > (unidecode: "-")
    overrides[0xFFEC] = ord('v')  # ￬ HALFWIDTH DOWNWARDS ARROW -> v (unidecode: "|")

    # Currency symbols - each mapped to a representative letter
    overrides[0x00A2] = ord('c')  # ¢ CENT SIGN -> c
    overrides[0x00A3] = ord('L')  # £ POUND SIGN -> L
    overrides[0x00A5] = ord('Y')  # ¥ YEN SIGN -> Y
    overrides[0x20AC] = ord('E')  # € EURO SIGN -> E

    # Symbols mapped to letters
    overrides[0x00A7] = ord('S')  # § SECTION SIGN -> S
    overrides[0x00A9] = ord('C')  # © COPYRIGHT SIGN -> C
    overrides[0x00AE] = ord('R')  # ® REGISTERED SIGN -> R
    overrides[0x2122] = ord('T')  # ™ TRADE MARK SIGN -> T

    # Degree-related symbols
    overrides[0x00B0] = ord('o')  # ° DEGREE SIGN -> o
    overrides[0x2103] = ord('C')  # ℃ DEGREE CELSIUS -> C
    overrides[0x2109] = ord('F')  # ℉ DEGREE FAHRENHEIT -> F

    # Angle quotation marks
    overrides[0x00AB] = ord('<')  # « LEFT-POINTING DOUBLE ANGLE QUOTATION MARK -> <
    overrides[0x00BB] = ord('>')  # » RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK -> >

    # Operators with circular shape
    overrides[0x2218] = ord('o')  # ∘ RING OPERATOR -> o
    overrides[0x2219] = ord('.')  # ∙ BULLET OPERATOR -> .

    # Negated mathematical symbols (preserving the negation semantics)
    # Negated symbols mapped to exclamation mark (semantically "not")
    for cp in (0x2204, 0x2209, 0x220C, 0x2224, 0x2226, 0x226E, 0x226F, 0x2280, 0x2281, 0x2284, 0x2285):
        overrides[cp] = ord('!')  # Negated math symbols -> ! (not)

    # Negated symbols mapped to hash sign (semantically "not equal")
    for cp in (0x2241, 0x2244, 0x2249, 0x2262, 0x2268, 0x2269, 0x226D, 0x228A, 0x228B):
        overrides[cp] = ord('#')  # Negated equality symbols -> # (not equal)

    # Negated arrows - all mapped to exclamation mark
    for cp in (0x219A, 0x219B, 0x21AE, 0x21CD, 0x21CE, 0x21CF):
        overrides[cp] = ord('!')  # Negated arrows -> ! (not)

    # Dashes and hyphens
    for cp in (0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015, 0x2043, 0x2052):
        overrides[cp] = ord('-')  # Dashes and hyphens -> -

    # Question mark punctuation
    for cp in (0x203D, 0x2047, 0x2048):
        overrides[cp] = ord('?')  # Question marks -> ?

    # Exclamation mark punctuation
    for cp in (0x203C, 0x2049):
        overrides[cp] = ord('!')  # Exclamation marks -> !

    # Asterisk-like symbols
    for cp in (0x2042, 0x2051, 0x2055):
        overrides[cp] = ord('*')

    # Other specific punctuation with unique mappings
    overrides[0x201E] = ord('"')  # „ DOUBLE LOW-9 QUOTATION MARK
    overrides[0x2023] = ord('>')  # ‣ TRIANGULAR BULLET
    overrides[0x2026] = ord('.')  # … HORIZONTAL ELLIPSIS
    overrides[0x2033] = ord('"')  # ″ DOUBLE PRIME
    overrides[0x204B] = ord('P')  # ⁋ REVERSED PILCROW SIGN
    overrides[0x204C] = ord('<')  # ⁌ BLACK LEFTWARDS BULLET
    overrides[0x204D] = ord('>')  # ⁍ BLACK RIGHTWARDS BULLET
    overrides[0x204F] = ord(';')  # ⁏ REVERSED SEMICOLON
    overrides[0x205B] = ord(':')  # ⁛ FOUR DOT MARK

    # Check marks
    overrides[0x2713] = ord('v')  # ✓ CHECK MARK
    overrides[0x2714] = ord('V')  # ✔ HEAVY CHECK MARK

    # X marks - lowercase for regular, uppercase for heavy
    for cp in (0x2715, 0x2717):
        overrides[cp] = ord('x')  # Regular X marks -> x
    for cp in (0x2716, 0x2718):
        overrides[cp] = ord('X')  # Heavy X marks -> X

    # Stars and asterisk-like symbols mapped to '*'
    for cp in (0x2605, 0x2606, 0x262A, 0x269D, 0x2698):
        overrides[cp] = ord('*')  # All star and asterisk symbols -> *
    for cp in range(0x2721, 0x2746+1):
        overrides[cp] = ord('*')  # All star and asterisk symbols -> *
    for cp in range(0x2749, 0x274B+1):
        overrides[cp] = ord('*')  # Last set of asterisk symbols -> *
    for cp in (0x229B, 0x22C6, 0x235F, 0x2363):
        overrides[cp] = ord('*')  # Star operators -> *

    # Special exclusions with fallback value of 0
    # These will be filtered out in organize_by_pages()

    # Exclude U+2028 (LINE SEPARATOR)
    overrides[0x2028] = 0  # LINE SEPARATOR (unidecode: '\n')

    # Full-width to ASCII mapping (covering all printable ASCII 33-126)
    # 0xFF01 (！) to 0xFF5E (～) -> ASCII 33 (!) to 126 (~)
    # Those are excluded here to reduce the table size.
    # It is more efficient to process them programmatically in
    # ucs.c:ucs_get_fallback().
    for cp in range(0xFF01, 0xFF5E + 1):
        overrides[cp] = 0  # Double-width ASCII characters

    return overrides

def organize_by_pages(fallback_map):
    """Organize the fallback mappings by their high byte (page)."""
    # Group by high byte (page)
    page_groups = defaultdict(list)
    for code, fallback in fallback_map.items():
        # Skip characters with fallback value of 0 (excluded characters)
        if fallback == 0:
            continue

        page = code >> 8  # Get the high byte (page)
        offset = code & 0xFF  # Get the low byte (offset within page)
        page_groups[page].append((offset, fallback))

    # Sort each page's entries by offset
    for page in page_groups:
        page_groups[page].sort()

    return page_groups

def compress_ranges(page_groups):
    """Compress consecutive entries with the same fallback character into ranges.
    A range is only compressed if it contains 3 or more consecutive entries."""

    compressed_pages = {}

    for page, entries in page_groups.items():
        compressed_entries = []
        i = 0
        while i < len(entries):
            start_offset, fallback = entries[i]

            # Look ahead to find consecutive entries with the same fallback
            j = i + 1
            while (j < len(entries) and
                   entries[j][0] == entries[j-1][0] + 1 and  # consecutive offsets
                   entries[j][1] == fallback):               # same fallback
                j += 1

            # Calculate the range end
            end_offset = entries[j-1][0]

            # If we found a range with 3 or more entries (worth compressing)
            if j - i >= 3:
                # Add a range entry
                compressed_entries.append((start_offset, RANGE_MARKER))
                compressed_entries.append((end_offset, fallback))
            else:
                # Add the individual entries as is
                for k in range(i, j):
                    compressed_entries.append(entries[k])

            i = j

        compressed_pages[page] = compressed_entries

    return compressed_pages

def cp_name(cp):
    """Get the Unicode character name for a code point."""
    try:
        return unicodedata.name(chr(cp))
    except:
        return f"U+{cp:04X}"

def generate_fallback_tables(out_file=DEFAULT_OUT_FILE):
    """Generate the fallback character tables."""
    # Generate fallback map using unidecode
    fallback_map = generate_fallback_map()
    print(f"Generated {len(fallback_map)} total fallback mappings")

    # Organize by pages
    page_groups = organize_by_pages(fallback_map)
    print(f"Organized into {len(page_groups)} pages")

    # Compress ranges
    compressed_pages = compress_ranges(page_groups)
    total_compressed_entries = sum(len(entries) for entries in compressed_pages.values())
    print(f"Total compressed entries: {total_compressed_entries}")

    # Create output file
    with open(out_file, 'w') as f:
        f.write(f"""\
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * {out_file} - Unicode character fallback table
 *
 * Auto-generated by {this_file}
 *
 * Unicode Version: {unicodedata.unidata_version}
 * Unidecode Version: {unidecode_version}
 *
 * This file contains optimized tables that map complex Unicode characters
 * to simpler fallback characters for terminal display when corresponding
 * glyphs are unavailable.
 */

static const struct ucs_page_desc ucs_fallback_pages[] = {{
""")

        # Convert compressed_pages to a sorted list of (page, entries) tuples
        sorted_pages = sorted(compressed_pages.items())

        # Track the start index for each page
        start_index = 0

        # Write page descriptors
        for page, entries in sorted_pages:
            count = len(entries)
            f.write(f"\t{{ 0x{page:02X}, {count}, {start_index} }},\n")
            start_index += count

        # Write entries array
        f.write("""\
};

/* Page entries array (referenced by page descriptors) */
static const struct ucs_page_entry ucs_fallback_entries[] = {
""")

        # Write all entries
        for page, entries in sorted_pages:
            page_hex = f"0x{page:02X}"
            f.write(f"\t/* Entries for page {page_hex} */\n")

            for i, (offset, fallback) in enumerate(entries):
                # Convert to hex for better readability
                offset_hex = f"0x{offset:02X}"
                fallback_hex = f"0x{fallback:02X}"

                # Handle comments
                codepoint = (page << 8) | offset

                if fallback == RANGE_MARKER:
                    comment = f"{cp_name(codepoint)} -> ..."
                else:
                    comment = f"{cp_name(codepoint)} -> '{chr(fallback)}'"
                f.write(f"\t{{ 0x{offset:02X}, 0x{fallback:02X} }}, /* {comment} */\n")

        f.write(f"""\
}};

#define UCS_PAGE_ENTRY_RANGE_MARKER {RANGE_MARKER}
""")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Generate Unicode fallback character tables")
    parser.add_argument("-o", "--output", dest="output_file", default=DEFAULT_OUT_FILE,
                       help=f"Output file name (default: {DEFAULT_OUT_FILE})")
    args = parser.parse_args()

    generate_fallback_tables(out_file=args.output_file)