Discussion forum for David Beazley

Python bitsyntax

Has anyone come across a python version of bitsyntax like erlang’s http://erlang.org/doc/programming_examples/bit_syntax.html

What a coincidence, I just came across https://docs.racket-lang.org/bitsyntax/ via https://youtu.be/sy2TzZO70E4?t=360 and would also love to know if there’s anything like this for Python!

I wonder if something like that Erlang syntax could be accomplished with Python type-hints. Well, instead of a “type” you’d write a class and specify a bitsize as the hint.

1 Like

I’ve made this toy example a while back:

from types import SimpleNamespace
import struct
import sys
from contextlib import suppress

xproto_common = {
    'CARD8'    : 'B', #  8-bit unsigned integer
    'CARD16'   : 'H', # 16-bit unsigned integer
    'char'     : 's',
    'PAD_BYTE' : 'x',
    'align4'   : lambda l: (4 - (l % 4)) % 4
    }

class SetupRequest(SimpleNamespace):
    """ field_name: struct_format_fstring_expression = value_expression """
    byte_order:                      '{CARD8}'
    pad1:                            '1{PAD_BYTE}'
    protocol_major_version:          '{CARD16}'
    protocol_minor_version:          '{CARD16}'
    authorization_protocol_name_len: '{CARD16}' = 'len(authorization_protocol_name)'
    authorization_protocol_data_len: '{CARD16}' = 'len(authorization_protocol_data)'
    pad2:                            '2{PAD_BYTE}'
    authorization_protocol_name:     '{authorization_protocol_name_len}{char}'
    pad3:                            '{align4(authorization_protocol_name_len)}{PAD_BYTE}'
    authorization_protocol_data:     '{authorization_protocol_data_len}{char}'
    pad4:                            '{align4(authorization_protocol_data_len)}{PAD_BYTE}'


class SetupRequestValues:
    byte_order = ord('l' if sys.byteorder == 'little' else 'B')
    protocol_major_version = 11
    protocol_minor_version = 0
    authorization_protocol_name = b'MIT-MAGIC-COOKIE-1'
    authorization_protocol_data = b'blahsomedata1'


def filter_dunders(d):
    return {
        key: value
        for key, value in d.items()
        if not ( key.startswith('__') and key.endswith('__') )
    }


def to_bytes(spec_class, given_values):

    values_spec = {
        k : getattr(spec_class, k)
        for k in spec_class.__annotations__
        if hasattr(spec_class, k)
    }

    values_by_spec = {
        k : eval(v, {}, given_values)
        for k, v in values_spec.items()
    }

    values = { **given_values, **values_by_spec }
    
    structs_by_spec = {
        k : eval('f' + repr(v), {}, values)
        for k, v in spec_class.__annotations__.items()
    }

    fmt = ''.join(structs_by_spec[k] for k in spec_class.__annotations__)
    vals = (values[k] for k in spec_class.__annotations__ if k in values)

    return struct.pack(fmt, *vals)


b = to_bytes(SetupRequest, { **filter_dunders(SetupRequestValues.__dict__), **xproto_common })


def to_object(spec_class, buffer_):
    attrs = dict()
    offset = 0
    for field_name, fmt_exp in spec_class.__annotations__.items():
        fmt = eval('f' + repr(fmt_exp), xproto_common, attrs)
        try:
            val, *_ = struct.unpack_from(fmt, buffer_, offset) # what if format uses 'c' for char?!?
            attrs[field_name] = val
        except ValueError:
            pass # it's a padding
        offset += struct.calcsize(fmt)
    return SimpleNamespace(**attrs)

o = to_object(SetupRequest, b)

def to_object_opportunistic(spec_class, buffer_):
    attrs = dict()
    offset = 0
    nbytes_needed = 0
    for field_name, fmt_exp in spec_class.__annotations__.items():
        fmt = eval('f' + repr(fmt_exp), xproto_common, attrs)
        ofst = struct.calcsize(fmt)
        try:
            val, *_ = struct.unpack_from(fmt, buffer_, offset)
        except ValueError:
            pass # it's a padding
        except struct.error:
            val = 0
            nbytes_needed += ofst
        attrs[field_name] = val
        offset += ofst
    if nbytes_needed > 0:
        raise ValueError(f'need {nbytes_needed}')
    return SimpleNamespace(**attrs)

To specify protocol in the same declarative manner, I believe, I need pattern matching.
At this point (eval-ing tree structure based on type) it has to be my glaring gap of nonexistent CS formal education and I’m making a poor re-implementation of some well understood problem from the 70’s.
@jab Indeed, this post was made after watching the very same video. :slight_smile:

1 Like