Merge main's latest changes back onto dev

tools/fontdraw.py

@@ -0,0 +1,110 @@
+"""
+A demo script to test fontgen.py generated files
+"""
+
+import sys
+import struct
+
+INDEX_RECORD_SIZE = 64 // 8
+
+class Glyph:
+
+    def __init__(self, file, offset):
+        self.beg_x = 0
+        self.beg_y = 0
+        self.end_x = 0
+        self.end_y = 0
+        self.data = None
+        self.file = file
+        self.file.seek(offset)
+
+    def read(self):
+        # Read the glyph header
+        self.beg_x, self.beg_y, self.len_x, self.len_y = self.file.read(4)
+
+        # Compute the size to read including the last byte
+        size = self.len_x * self.len_y
+        size = (size + 7) // 8
+
+        # Optimization: do not split the row/columns yet
+        self.data = self.file.read(size)
+
+class Font:
+
+    def __init__(self, path):
+        self.file = open(path, "rb")
+
+        # Read the index header: (u32)*2
+        record = self.file.read(4 + 4)
+        self.font_height, self.index_size = struct.unpack(">II", record)
+
+        # Then read the index
+        self.font_index = []
+        n = 0
+        while n < self.index_size:
+            # Parse a record from the file
+            self.font_index.append(struct.unpack(">II", self.file.read(8)))
+            n += 8
+
+    def glyph_range(self, unicode):
+        # Start searching from the middle of the file
+        beg = 0
+        end = len(self.font_index) - 1
+        i = beg + (end - beg) // 2
+
+        # Inline implementation of binary search to find the glyph
+        while beg != end:
+
+            # Decode the u8 and u24 out of the u32
+            unicode_len = self.font_index[i][0] & 0xff
+            unicode_start = self.font_index[i][0] >> 8
+
+            # Should we search lower?
+            if unicode < unicode_start:
+                end = i
+
+            # Should we search higher?
+            elif unicode > unicode_start + unicode_len:
+                beg = i
+
+            # That's a hit!
+            else:
+                return unicode_start, self.font_index[i][1]
+
+            # Adjust the probe
+            i = beg + (end - beg) // 2
+
+        return None
+
+    def glyph(self, unicode):
+
+        # Get the range to start scanning from
+        range_start, range_address = self.glyph_range(unicode)
+
+        # Scan through the data glyph per glyph
+        glyph = Glyph(self.file, 4 + 4 + self.index_size + range_address)
+
+        # Skip several glyphs until we are there
+        for _ in range(unicode - range_start):
+            glyph.read()
+        glyph.read()
+
+        return glyph
+
+    def draw(self, glyph):
+        n = 0
+        for ch in glyph.data:
+            for i in reversed(range(8)):
+                print("#" if ch & (1 << i) else " ", end="")
+                n += 1
+                if n % glyph.len_x == 0:
+                    print("|")
+        if n % glyph.len_x != 0:
+            print('', end='\r')
+
+if __name__ == "__main__":
+    font = Font(sys.argv[1])
+    for ch in sys.argv[2]:
+        glyph = font.glyph(ord(ch))
+        print(ch)
+        font.draw(glyph)

tools/fontgen.py

@@ -0,0 +1,148 @@
+"""
+A font conversion tool taking BDF files as input and generating a custom
+compact bitmap format as output:
+
+    font_format: (u32)reserved, (u32)index_size, font_index, font_data
+
+    font_index: [ (u64)index_record ]*N
+    index_record: (u24)unicode_start, (u8)unicode_count, (u32)glyph_address
+
+    font_data: [ (u16)glyph_header, glyph_data ]*N
+    glyph_header: (u8)beg_x, (u8)beg_y, (u8)len_x, (u8)len_y
+    glyph_data: [ (u1)data_bit ]*N
+
+The font_index can be looked-up using a binary search to find a glyph block,
+and then scan the glyph data within it:
+
+If searching the unicode codepoint U+30F3:
+
+1. A binary search would allow to find the range (U+3000, U+3100),
+   which will point at the position in the glyph data
+
+2. The glyphs start with a header specifying the size of the glpyh
+   data, which allows to jump to the next header.
+
+3. Starting from U+3000, 0x3F (63) glyphs need to be skipped,
+   then the glyph data can be used to paint it.
+"""
+
+import struct
+import unicodedata
+import sys
+from bdfparser import Font
+
+font_index = bytearray()
+font_data = bytearray()
+
+def add_index_record(unicode_start, unicode_end, glyph_address):
+    len = unicode_end - unicode_start + 1
+    assert len > 0 and len <= 0xff
+
+    font_index.extend(struct.pack(">I", unicode_start)[1:])
+    font_index.extend(struct.pack(">B", unicode_end - unicode_start + 1))
+    font_index.extend(struct.pack(">I", glyph_address))
+
+def get_y_bounding_box(bitmap):
+    beg_y = None
+    end_y = 0
+    for i, row in enumerate(bitmap):
+        if '1' in row:
+            if beg_y is None:
+                beg_y = i
+            end_y = i
+    beg_y = beg_y or 0
+    beg_y = 0 if end_y == beg_y else beg_y
+    return beg_y, end_y + 1
+
+def get_x_bounding_box(bitmap):
+    beg_x = None
+    end_x = 0
+    for i in range(len(bitmap[0])):
+        if '1' in [x[i] for x in bitmap]:
+            if beg_x is None:
+                beg_x = i
+            end_x = i
+    return beg_x or 0, end_x + 1
+
+def add_glyph_header(beg_x, beg_y, end_x, end_y):
+    font_data.append(beg_x)
+    font_data.append(beg_y)
+    font_data.append(end_x - beg_x)
+    font_data.append(end_y - beg_y)
+
+def add_glyph_bitmap(bitmap):
+    i = 0
+    ch = 0
+    for row in bitmap:
+        for bit in row:
+            assert bit in ('0', '1')
+
+            ch <<= 1
+            if bit == '1':
+                ch |= 1
+            i += 1
+            if i % 8 == 0:
+                font_data.append(ch)
+                ch = 0
+
+    # There is an incomplete byte after the rest
+    if i % 8 > 0:
+        font_data.append(ch << (8 - i % 8))
+
+def add_glyph(glyph):
+    # Get a Bitmap out of the font, cropping it as small as possible
+    glyph_bitmap = glyph.draw(1)
+
+    # Format the data into the bitmap, skipping the empty rows
+    bitmap = [x.replace('2', '1') for x in glyph.draw(0).todata()]
+
+    # Crop the top and bottom margins
+    beg_y, end_y = get_y_bounding_box(bitmap)
+    bitmap = bitmap[beg_y:end_y]
+
+    # Crop the left and right margins
+    beg_x, end_x = get_x_bounding_box(bitmap)
+    bitmap = [row[beg_x:end_x] for row in bitmap]
+
+    # Add the glyph metadata
+    font_data.extend(bytes((beg_x, beg_y, end_x - beg_x, end_y - beg_y)))
+
+    # Loop over every bit of every row and append them back-to-back
+    add_glyph_bitmap(bitmap)
+
+if len(sys.argv) != 3:
+    print("usage: {} source_file.bdf output_file.bin".format(sys.argv[0]))
+    exit(1)
+
+font = Font(sys.argv[1])
+
+unicode_start = 0
+unicode_count = 0
+glyph_address = 0
+
+# Add all glyphs one by one to the font_index and font_data bytearrays()
+unicode_prev = -1
+for glyph in font.iterglyphs(order=1):
+
+    # The glyph data block is very straightforward, always the same: add the
+    # glyph, but we must keep track of the address.
+    add_glyph(glyph)
+
+    # If no more room for the current codepoint, push the record, switch to
+    # the next
+    if unicode_prev >= 0:
+        if glyph.cp() != unicode_prev + 1 or glyph.cp() >= unicode_start + 0xff:
+            add_index_record(unicode_start, unicode_prev, glyph_address)
+            glyph_address = address_prev
+            unicode_start = glyph.cp()
+
+    unicode_prev = glyph.cp()
+    address_prev = len(font_data)
+
+add_index_record(unicode_start, unicode_prev, glyph_address)
+
+with open(sys.argv[2], "wb") as f:
+	f.write(struct.pack(">I", 0))
+	f.write(struct.pack(">I", len(font_index)))
+	f.write(font_index)
+	f.write(font_data)