Start porting my (better) Go library over

__init__.py

@@ -1,194 +1,197 @@
-#! /usr/bin/python3
 
-import binascii
+import typing
-import sys
+import io
-import struct
+from . import binary
-from . import ip
+
+class Error(Exception):
+    """Base class for netshovel exceptions"""
+    pass
 
 
+class ShortError(Error):
+    """Exception raised when not enough data is available.
 
-def cstring(buf):
+    Attributes:
-    "Return buf if buf were a C-style (NULL-terminate) string"
+        wanted -- how much data we wanted
-
+        available -- how much data we had
-    i = buf.index('\0')
+    """
-    return buf[:i]
+    
-
+    def __init__(self, wanted:int, available:int):
-
+        self.wanted = wanted
-def assert_equal(a, b):
+        self.available = available
-    assert a == b, ('%r != %r' % (a, b))
-
-
-def assert_in(a, *b):
-    assert a in b, ('%r not in %r' % (a, b))
-
-
-##
-## Binary and other base conversions
-##
-
-class BitVector:
-    def __init__(self, i=0, length=None):
-        try:
-            self._val = 0
-            for c in i:
-                self._val <<= 8
-                self._val += ord(c)
-            if length is not None:
-                self._len = length
-            else:
-                self._len = len(i) * 8
-        except TypeError:
-            self._val = i
-            if length is not None:
-                self._len = length
-            else:
-                self._len = 0
-                while i > 0:
-                    i >>= 1
-                    self._len += 1
-
-    def __len__(self):
-        return self._len
-
-    def __getitem__(self, idx):
-        if idx > self._len:
-            raise IndexError()
-        idx = self._len - idx
-        return int((self._val >> idx) & 1)
-
-    def __getslice__(self, a, b):
-        if b > self._len:
-            b = self._len
-        i = self._val >> (self._len - b)
-        l = b - a
-        mask = (1 << l) - 1
-        return BitVector(i & mask, length=l)
-
-    def __iter__(self):
-        """Iterate from LSB to MSB"""
-
-        v = self._val
-        for _ in range(self._len):
-            yield int(v & 1)
-            v >>= 1
 
     def __str__(self):
-        r = ''
+        return "Not enough data available: wanted %d, got %d" % (self.wanted, self.got)
-        v = self._val
-        i = self._len
-        while i > 8:
-            o = ((v >> (i - 8)) & 0xFF)
-            r += chr(o)
-            i -= 8
-        if i > 0:
-            o = v & ((1 << i) - 1)
-            r += chr(o)
-        return r
-
-    def __int__(self):
-        return self._val
-
-    def __repr__(self):
-        l = list(self)
-        l.reverse()
-        return '<BitVector ' + ''.join(str(x) for x in l) + '>'
-
-    def __add__(self, i):
-        if isinstance(i, BitVector):
-            l = len(self) + len(i)
-            v = (int(self) << len(i)) + int(i)
-            return BitVector(v, l)
-        else:
-            raise ValueError("Can't extend with this type yet")
-
-    def bitstr(self):
-        bits = [str(x) for x in self]
-        bits.reverse()
-        return ''.join(bits)
 
 
-def bin(i, bits=None):
+class MissingError(Error):
-    """Return the binary representation of i"""
+    """Exception raised when gaps were present for code that can't handle gaps.
+    """
 
-    return BitVector(i, bits).bitstr()
+    def __init__(self):
+        pass
+
+    def __str__(self):
+        return "Operation on missing bytes"
 
 
-def unhex(s):
+class namedField(typing.NamedTuple):
-    """Decode a string as hex, stripping whitespace first"""
+    key: str
+    value: str
 
-    return binascii.unhexlify(s.replace(' ', ''))
+class headerField(typing.NamedTuple):
+    name: str
+    bits: int
+    value: typing.Any
+    order: binary.ByteOrder
 
+class Packet:
+    def __init__(self, when, payload):
+        self.opcode = -1
+        self.description = "Undefined"
+        self.when = when
+        self.payload = payload
+        self.header = []
+        self.fields = []
 
-def pp(value, bits=16):
+    def describeType(self) -> str:
-    hexfmt = '%%0%dx' % (bits / 4)
+        """Returns a string with timestamp, opcode, and description of this packet"""
-    return '%6d  0x%s  %s' % (value, (hexfmt % value), bin(value, bits))
+        return "%s Opcode %d: %s"  % (self.when, self.opcode, self.description)
+    
+    def describeFields(self) -> str:
+        """Returns a multi-line string describing fields in this packet"""
+        lines = []
+        for k, v in self.fields:
+            lines.append("    %s: %s\n", k, v)
+        return "".join(lines)
 
-##
+    def describeHeader(self) -> str:
-## Codecs
+        """Returns a multi-line string describing this packet's header structure"""
-##
+        out = io.StringIO()
-import codecs
+        out.write(" 0                               1                            \n")
-import string
+        out.write(" 0 1 2 3 4 5 6 7 8 9 a b c d e f 0 1 2 3 4 5 6 7 8 9 a b c d e f\n")
+        
+        bitOffset = 0
+        for f in self.header:
+            bits = f.bits
+            while bits > 0:
+                if bitOffset == 0:
+                    out.write("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n")
+                linebits = bits
+                if linebits+bitOffset > 0x20:
+                    linebits = 0x20 - bitOffset
 
-b64alpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+                nameval = "%s (0x%x)" % (f.name, f.value)
+                out.write("|" + nameval.center(linebits*2-1))
 
-def from_b64(s, alphabet, codec='base64'):
+                bitOffset += linebits
-    tr = alphabet.maketrans(b64alpha)
+                bits -= linebits
-    t = s.translate(tr)
+                if linebits == 0x20:
-    return t.decode(codec)
+                    out.write("|\n")
+                    bitOffset = 0
+        out.write("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n")
+        return out.getvalue()
 
-class Esab64Codec(codecs.Codec):
+    def describe(self) -> str:
-    """Little-endian version of base64."""
+        """Return a multi-line string describing this packet
 
-    ## This could be made nicer by better conforming to the codecs.Codec
+        This shows the timestamp, opcode, description, and hex dump.
-    ## spec.  For instance, raising the appropriate exceptions.
+        If you set any values, those are displayed in the order they were set.
-    ##
-    ## Using BitVector makes the code very readable, but it is probably
-    ## slow.
 
-    b64_chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+        This will quickly get unweildy, especially for large conversations.
-    def decode(self, input, errors='strict'):
+        You are encouraged to implement your own describe() method.
-        r = []
+        """
-        for i in range(0, len(input), 4):
+        out = io.StringIO()
-            v = BitVector()
+        out.write(self.describeType())
-            for c in input[i:i+4]:
+        out.write("\n")
-                if c in ('=', ' ', '\n'):
+        out.write(self.describeFields())
-                    break
+        out.write(self.describeHeader())
-                v += BitVector(self.b64_chars.index(c), 6)
+        out.write(self.payload.hexdump())
+        return out.getvalue()
 
-            # Normal base64 would start at the beginning
+    def setValue(self, key:str, value:str):
-            b = (v[10:12] + v[ 0: 6] +
+        """Set a value
-                 v[14:18] + v[ 6:10] +
-                 v[18:24] + v[12:14])
 
-            r.append(str(b))
+        This is intended to be used to note debugging information
-        return ''.join(r), len(input)
+        that you'd like to see on each packet.
+        """
+        self.fields.append(namedField(key, value))
 
-    def encode(self, input, errors='strict'):
+    def setString(self, key:str, value:str):
-        raise NotImplementedError()
+        """Set a string value, displaying its Python string representation"""
+        self.setValue(key, repr(value))
 
+    def setInt(self, key:str, value:int):
+        """Set an int value, displaying its decimal and hexadecimal representations"""
+        self.setValue(key, "%d == 0x%x" % (value, value))
+    setUInt = setInt
 
-class Esab64StreamWriter(Esab64Codec, codecs.StreamWriter):
+    def setUInt32(self, key:str, value:int):
-    pass
+        """Set a Uint32 value, dispalying its decimal and 0-padded hexadecimal representations"""
+        self.setValue(key, "%d == %04x" % (value, value))
 
-class Esab64StreamReader(Esab64Codec, codecs.StreamReader):
+    def setBytes(self, key:str, value:str):
-    pass
+        """Set a bytes value, displaying the hex encoding of the bytes"""
+        self.setValue(key, binascii.hexlify(value).encode("ascii"))
 
-def _registry(encoding):
+    def peel(self, octets:int) -> bytes:
-    if encoding == 'esab64':
+        """Peel octets bytes off the Payload, returning those bytes"""
-        c = Esab64Codec()
+        pllen = len(self.payload)
-        return (c.encode, c.decode,
+        if octets > pllen:
-                Esab64StreamReader, Esab64StreamWriter)
+            raise ShortError(octets, pllen)
+        buf = self.payload[:octets]
+        if buf.missing() > 0:
+            raise MissingError()
+        self.payload = self.payload[octets:]
+        return buf.bytes()
 
-codecs.register(_registry)
+    def addHeaderField(self, order:binary.ByteOrder, name:str, bits:int, value:typing.Any):
+        """Add a field to the header field description."""
+        h = headerField(name, bits, value, order)
+        self.header.append(h)
 
-def main(session):
+    def readUint(self, order:binary.ByteOrder, bits:int, name:str):
-    s = None
+        """Peel an unsigned integer of size bits, adding it to the header field"""
-    reseq = ip.Dispatch(*sys.argv[1:])
+        if bits not in (8, 16, 32, 64):
-    for _, d in reseq:
+            raise RuntimeError("Weird number of bits: %d" % bits)
-        srv, first, chunk = d
+        octets = bits >> 3
-        if not s:
+        b = self.peel(octets)
-            s = session(first)
+        if bits == 8:
-        s.handle(srv, first, chunk, reseq.last)
+            value = b[0]
+        elif bits == 16:
+            value = order.Uint16(b)
+        elif bits == 32:
+            value = order.Uint32(b)
+        elif bits == 64:
+            value = order.Uint64(b)
+        self.addHeaderField(order, name, bits, value)
 
-Session = ip.Session
+        return value
-Packet = ip.Packet
+
+    def uint8(self, name:str) -> int:
+        "Peel off a uint8 (aka byte)"
+        return self.readUint(binary.LittleEndian, 8, name)
+    
+    def uint16le(self, name:str) -> int:
+        "Peel off a uint16, little-endian"
+        return self.readUint(binary.LittleEndian, 16, name)
+
+    def uint32le(self, name:str) -> int:
+        "Peel off a uint32, little-endian"
+        return self.readUint(binary.LittleEndian, 32, name)
+
+    def uint64le(self, name:str) -> int:
+        "Peel off a uint64, little-endian"
+        return self.readUint(binary.LittleEndian, 64, name)
+
+    def uint16be(self, name:str) -> int:
+        "Peel off a uint64, big-endian"
+        return self.readUint(binary.BigEndian, 16, name)
+
+    def uint32be(self, name:str) -> int:
+        "Peel off a uint32, big-endian"
+        return self.readUint(binary.BigEndian, 32, name)
+
+    def uint64be(self, name:str) -> int:
+        "Peel off a uint44, big-endian"
+        return self.readUint(binary.BigEndian, 64, name)

binary.py

@@ -0,0 +1,91 @@
+"""Endianness conversions.
+
+This is a blatant rip-off of the golang binary library.
+I'm not too proud to steal a nicely-thought-out API.
+
+"""
+
+def byte(v):
+    return v & 0xff
+
+class ByteOrder:
+    "A ByteOrder specifies how to convert byte sequences into 16-, 32-, or 64-bit unsigned integers."
+    pass
+
+class LittleEndian(ByteOrder):
+    "Little-Endian byte order"
+
+    def Uint16(self, b:bytes) -> int:
+        return b[0] | (b[1]<<8)
+
+    def PutUint16(self, v:int) -> bytes:
+        return bytes([
+            byte(v), 
+            byte(v>>8),
+        ])
+
+    def Uint32(self, b:bytes) -> int:
+        return b[0] | (b[1]<<8) | (b[2]<<16) | (b[3]<<24)
+
+    def PutUint16(self, v:int) -> bytes:
+        return bytes([
+            byte(v), 
+            byte(v>>8), 
+            byte(v>>16),
+            byte(v>>24),
+        ])
+
+    def Uint64(self, b:bytes) -> int:
+        return b[0] | (b[1]<<8) | (b[2]<<16) | (b[3]<<24) | \
+            (b[4]<<32) | (b[5]<<40) | (b[6]<<48) | (b[7]<<56)
+
+    def PutUint64(self, v:int) -> bytes:
+        return bytes([
+            byte(v), 
+            byte(v>>8), 
+            byte(v>>16),
+            byte(v>>24),
+            byte(v>>32),
+            byte(v>>40),
+            byte(v>>48),
+            byte(v>>56),
+        ])
+
+class BigEndian(ByteOrder):
+    "Big-Endian byte order"
+
+    def Uint16(self, b:bytes) -> int:
+        return b[1] | (b[0]<<8)
+
+    def PutUint16(self, v:int) -> bytes:
+        return bytes([
+            byte(v>>8), 
+            byte(v),
+        ])
+
+    def Uint32(self, b:bytes) -> int:
+        return b[3] | (b[2]<<8) | (b[1]<<16) | (b[0]<<24)
+
+    def PutUint16(self, v:int) -> bytes:
+        return bytes([
+            byte(v>>24), 
+            byte(v>>16), 
+            byte(v>>8),
+            byte(v),
+        ])
+
+    def Uint64(self, b:bytes) -> int:
+        return b[7] | (b[6]<<8) | (b[5]<<16) | (b[4]<<24) | \
+            (b[3]<<32) | (b[2]<<40) | (b[1]<<48) | (b[0]<<56)
+
+    def PutUint64(self, v:int) -> bytes:
+        return bytes([
+            byte(v>>56),
+            byte(v>>48),
+            byte(v>>40),
+            byte(v>>32),
+            byte(v>>24),
+            byte(v>>16),
+            byte(v>>8), 
+            byte(v), 
+        ])

ip.py

@@ -533,9 +533,6 @@ class Packet:
     def __iter__(self):
         return self.params.__iter__()
 
-    def has_key(self, k):
-        return self.params.has_key(k)
-
     def keys(self):
         return self.params.keys()
 
@@ -586,8 +583,8 @@ class Packet:
             except AttributeError:
                 print('         payload: %r' % self.payload)
 
-    def parse(self, data):
+    def decode(self, data):
-        """Parse a chunk of data (possibly a TriloBytes).
+        """Decode a chunk of data (possibly a TriloBytes).
 
         Anything returned is not part of this packet and will be passed
         in to a subsequent packet.
@@ -596,12 +593,12 @@ class Packet:
 
         self.parts = [data]
         self.payload = data
-        return None
+        return False
 
     def handle(self, data):
         """Handle data from a Session class."""
 
-        data = self.parse(data)
+        data = self.decode(data)
         if self.opcode != None:
             try:
                 f = getattr(self, 'opcode_%s' % self.opcode)

orig.py

@@ -0,0 +1,194 @@
+#! /usr/bin/python3
+
+import binascii
+import sys
+import struct
+from . import ip
+
+
+
+def cstring(buf):
+    "Return buf if buf were a C-style (NULL-terminate) string"
+
+    i = buf.index('\0')
+    return buf[:i]
+
+
+def assert_equal(a, b):
+    assert a == b, ('%r != %r' % (a, b))
+
+
+def assert_in(a, *b):
+    assert a in b, ('%r not in %r' % (a, b))
+
+
+##
+## Binary and other base conversions
+##
+
+class BitVector:
+    def __init__(self, i=0, length=None):
+        try:
+            self._val = 0
+            for c in i:
+                self._val <<= 8
+                self._val += ord(c)
+            if length is not None:
+                self._len = length
+            else:
+                self._len = len(i) * 8
+        except TypeError:
+            self._val = i
+            if length is not None:
+                self._len = length
+            else:
+                self._len = 0
+                while i > 0:
+                    i >>= 1
+                    self._len += 1
+
+    def __len__(self):
+        return self._len
+
+    def __getitem__(self, idx):
+        if idx > self._len:
+            raise IndexError()
+        idx = self._len - idx
+        return int((self._val >> idx) & 1)
+
+    def __getslice__(self, a, b):
+        if b > self._len:
+            b = self._len
+        i = self._val >> (self._len - b)
+        l = b - a
+        mask = (1 << l) - 1
+        return BitVector(i & mask, length=l)
+
+    def __iter__(self):
+        """Iterate from LSB to MSB"""
+
+        v = self._val
+        for _ in range(self._len):
+            yield int(v & 1)
+            v >>= 1
+
+    def __str__(self):
+        r = ''
+        v = self._val
+        i = self._len
+        while i > 8:
+            o = ((v >> (i - 8)) & 0xFF)
+            r += chr(o)
+            i -= 8
+        if i > 0:
+            o = v & ((1 << i) - 1)
+            r += chr(o)
+        return r
+
+    def __int__(self):
+        return self._val
+
+    def __repr__(self):
+        l = list(self)
+        l.reverse()
+        return '<BitVector ' + ''.join(str(x) for x in l) + '>'
+
+    def __add__(self, i):
+        if isinstance(i, BitVector):
+            l = len(self) + len(i)
+            v = (int(self) << len(i)) + int(i)
+            return BitVector(v, l)
+        else:
+            raise ValueError("Can't extend with this type yet")
+
+    def bitstr(self):
+        bits = [str(x) for x in self]
+        bits.reverse()
+        return ''.join(bits)
+
+
+def bin(i, bits=None):
+    """Return the binary representation of i"""
+
+    return BitVector(i, bits).bitstr()
+
+
+def unhex(s):
+    """Decode a string as hex, stripping whitespace first"""
+
+    return binascii.unhexlify(s.replace(' ', ''))
+
+
+def pp(value, bits=16):
+    hexfmt = '%%0%dx' % (bits / 4)
+    return '%6d  0x%s  %s' % (value, (hexfmt % value), bin(value, bits))
+
+##
+## Codecs
+##
+import codecs
+import string
+
+b64alpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+
+def from_b64(s, alphabet, codec='base64'):
+    tr = alphabet.maketrans(b64alpha)
+    t = s.translate(tr)
+    return t.decode(codec)
+
+class Esab64Codec(codecs.Codec):
+    """Little-endian version of base64."""
+
+    ## This could be made nicer by better conforming to the codecs.Codec
+    ## spec.  For instance, raising the appropriate exceptions.
+    ##
+    ## Using BitVector makes the code very readable, but it is probably
+    ## slow.
+
+    b64_chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+    def decode(self, input, errors='strict'):
+        r = []
+        for i in range(0, len(input), 4):
+            v = BitVector()
+            for c in input[i:i+4]:
+                if c in ('=', ' ', '\n'):
+                    break
+                v += BitVector(self.b64_chars.index(c), 6)
+
+            # Normal base64 would start at the beginning
+            b = (v[10:12] + v[ 0: 6] +
+                 v[14:18] + v[ 6:10] +
+                 v[18:24] + v[12:14])
+
+            r.append(str(b))
+        return ''.join(r), len(input)
+
+    def encode(self, input, errors='strict'):
+        raise NotImplementedError()
+
+
+class Esab64StreamWriter(Esab64Codec, codecs.StreamWriter):
+    pass
+
+class Esab64StreamReader(Esab64Codec, codecs.StreamReader):
+    pass
+
+def _registry(encoding):
+    if encoding == 'esab64':
+        c = Esab64Codec()
+        return (c.encode, c.decode,
+                Esab64StreamReader, Esab64StreamWriter)
+
+codecs.register(_registry)
+
+def main(session):
+    s = None
+    reseq = ip.Dispatch(*sys.argv[1:])
+    for _, d in reseq:
+        srv, first, chunk = d
+        if not s:
+            s = session(first)
+        s.handle(srv, first, chunk, reseq.last)
+
+Session = ip.Session
+Packet = ip.Packet

stream.py

@@ -0,0 +1,37 @@
+import typing
+from . import trilobytes
+
+class NamedFile(typing.NamedTuple):
+    """A file object and the path where it lives"""
+    File: typing.BinaryIO
+    Name: string
+
+class Utterance(typing.NamedTuple):
+    """An atomic communication within a Stream.
+
+    Streams consist of a string of Utterances.
+    Each utterance has associated data, and a time stamp.
+        
+    Typically these line up with what crosses the network,
+    but bear in mind that TCP is a streaming protocol,
+    so don't rely on Utterances alone to separate Application-layer packets.
+    """
+
+    When: float
+    Data: trilobytes.TriloBytes
+
+class Stream:
+    """A Stream is one half of a two-way conversation"""
+
+    def __init__(self, net, transport):
+        self.net = net
+        self.transport = transport
+
+    def reassembled(rs):
+        """Called by the TCP assembler when an Utterance can be built"""
+        data = trilobytes.TriloBytes()
+        for r in rs:
+            if r.Skip > 0:
+                data += [None] * r.Skip
+            data + r.Bytes
+        if len(data) > 0

unpack.py

@@ -1,73 +0,0 @@
-#! /usr/bin/python3
-
-ENDIAN_LITTLE = 1
-ENDIAN_BIG = 2
-ENDIAN_MIDDLE = 3
-ENDIAN_NETWORK = ENDIAN_BIG
-
-class Unpacker:
-    """Class that lets you peel values off
-    
-    >>> u = Unpacker(bytes((1, 0,2, 0,0,0,3, 0,0,0,0,0,0,0,4)))
-    >>> u.uint8()
-    1
-    >>> u.uint16()
-    2
-    >>> u.uint32()
-    3
-    >>> u.uint64()
-    4
-    
-    >>> u = Unpacker(bytes((1,0, 104,105)), ENDIAN_LITTLE)
-    >>> u.uint16()
-    1
-    >>> u.buf
-    b'hi'
-
-    >>> u = Unpacker(bytes((1,0, 0,2)))
-    >>> u.uint16(ENDIAN_LITTLE)
-    1
-    >>> u.uint(16, ENDIAN_BIG)
-    2
-    
-    >>> u = Unpacker(bytes((0,1,2,3)), ENDIAN_MIDDLE)
-    >>> '%08x' % u.uint32()
-    '01000302'
-    """
-    
-    def __init__(self, buf, endian=ENDIAN_NETWORK):
-        self.endian = endian
-        self.buf = buf
-
-    def uint(self, size, endian=None):
-        endian = endian or self.endian
-        if size not in (8, 16, 32, 64):
-            # XXX: I'm pretty sure this can be done, but I don't want to code it up right now.
-            raise ValueError("Can't do weird sizes")
-        noctets = size // 8
-        if endian == ENDIAN_BIG:
-            r = range(0, noctets)
-        elif endian == ENDIAN_LITTLE:
-            r = range(noctets-1, -1, -1)
-        elif endian == ENDIAN_MIDDLE:
-            r = (1, 0, 3, 2,   5, 4, 7, 6)[:noctets]
-        else:
-            raise ValueError("Unsupported byte order")
-        pull, self.buf = self.buf[:noctets], self.buf[noctets:]
-        acc = 0
-        for i in r:
-            acc = (acc << 8) | pull[i]
-        return acc
-        
-    def uint8(self):
-        return self.uint(8)
-    def uint16(self, endian=None):
-        return self.uint(16, endian)
-    def uint32(self, endian=None):
-        return self.uint(32, endian)
-    def uint64(self, endian=None):
-        return self.uint(64, endian)
-
-if __name__ == "__main__":
-    import doctest
-    doctest.testmod()