Start porting this mess to Python 3

dumbdecode.py

@@ -0,0 +1,13 @@
+#! /usr/bin/env python
+
+import sys
+from netarch import ip
+from netarch import *
+
+s = None
+reseq = ip.Dispatch(*sys.argv[1:])
+for h, d in reseq:
+    srv, first, chunk = d
+    if not s:
+        s = ip.Session(first)
+    s.handle(srv, first, chunk, reseq.last)

gapstr.py

@@ -1,246 +0,0 @@
-#! /usr/bin/python
-
-## 2008 Massive Blowout
-
-"""Functions to treat a list as a byte array with gaps.
-
-Lists should have only byte and numeric items.
-
-"""
-
-import __init__
-import sys
-
-class GapString:
-    def __init__(self, init=None, drop='?'):
-        self.contents = []
-        self.length = 0
-        self.drop = drop
-
-        if init:
-            self.append(init)
-
-    def __len__(self):
-        return int(self.length)
-
-    def loss(self):
-        ret = 0
-        for i in self.contents:
-            try:
-                ret += i
-            except TypeError:
-                pass
-        return ret
-
-    def __repr__(self):
-        return '<GapString of length %d>' % self.length
-
-    def append(self, i):
-        try:
-            self.length += len(i)
-            self.contents.append(i)
-        except TypeError:
-            self.length += i
-            self.contents.append(i)
-
-    def pop(self, idx=-1):
-        item = self.contents.pop(idx)
-        try:
-            self.length -= item
-        except TypeError:
-            self.length -= len(item)
-        return GapString(item)
-        
-
-    def __str__(self):
-        ret = []
-        for i in self.contents:
-            try:
-                ret.append(self.drop * i)
-            except TypeError:
-                ret.append(i)
-        return ''.join(ret)
-
-    def __iter__(self):
-        for i in self.contents:
-            try:
-                for c in i:
-                    yield c
-            except TypeError:
-                for j in range(i):
-                    yield self.drop
-
-    def __nonzero__(self):
-        return self.length > 0
-
-    def hasgaps(self):
-        for i in self.contents:
-            try:
-                len(i)
-            except TypeError:
-                return True
-        return False
-
-    def hexdump(self, fd=sys.stdout):
-        offset = 0
-
-        d = __init__.HexDumper(fd)
-        for i in self.contents:
-            try:
-                for j in xrange(i):
-                    d.dump_drop()
-            except TypeError:
-                for c in i:
-                    d.dump_chr(c)
-        d.finish()
-
-    def extend(self, other):
-        self.contents += other.contents
-        self.length += other.length
-
-    def __getslice__(self, start, end):
-        end = min(self.length, end)
-        start = min(self.length, start)
-
-        new = self.__class__(drop=self.drop)
-        new.length = max(end - start, 0)
-        if new.length == 0:
-            new.contents = []
-            return new
-        new.contents = self.contents[:]
-
-        l = self.length - new.length - start
-
-        # Trim off the beginning
-        while start >= 0:
-            i = new.contents.pop(0)
-            try:
-                start -= i
-                if start < 0:
-                    new.contents.insert(0, -start)
-            except TypeError:
-                start -= len(i)
-                if start < 0:
-                    new.contents.insert(0, i[start:])
-
-        # Trim off the end
-        while l >= 0:
-            i = new.contents.pop()
-            try:
-                l -= i
-                if l < 0:
-                    new.contents.append(-l)
-            except TypeError:
-                l -= len(i)
-                if l < 0:
-                    new.contents.append(i[:-l])
-
-        return new
-
-    def __getitem__(self, idx):
-        if False:
-            c = self[idx:idx+1]
-            if c.hasgaps():
-                return self.drop[0]
-            else:
-                return c.contents[0][0]
-        else:
-            l = 0
-            for i in self.contents:
-                try:
-                    l += len(i)
-                except TypeError:
-                    l += i
-                if l > idx:
-                    offs = idx - l
-                    try:
-                        return i[offs]
-                    except:
-                        return self.drop[0]
-            raise IndexError('Out of bounds')
-
-    def __add__(self, other):
-        if isinstance(other, str):
-            self.append(other)
-        else:
-            new = self.__class__(drop=self.drop)
-            new.extend(self)
-            new.extend(other)
-            return new
-
-    def __xor__(self, mask):
-        try:
-            mask = [ord(c) for c in mask]
-        except TypeError:
-            pass
-        try:
-            masklen = len(mask)
-        except TypeError:
-            masklen = 1
-            mask = [mask]
-
-        new = self.__class__(drop=self.drop)
-        for i in self.contents:
-            try:
-                r = []
-                offset = len(new) % masklen
-                for c in i:
-                    o = ord(c)
-                    r.append(chr(o ^ mask[offset]))
-                    offset = (offset + 1) % masklen
-                new.append(''.join(r))
-            except TypeError:
-                new.append(i)
-        return new
-
-    def index(self, needle):
-        pos = 0
-        for i in self.contents:
-            try:
-                return pos + i.index(needle)
-            except AttributeError:
-                pos += i
-            except ValueError:
-                pos += len(i)
-        raise ValueError('substring not found')
-
-    def split(self, pivot=' ', times=None):
-        ret = []
-        cur = self
-        while (not times) or (len(ret) < times):
-            try:
-                pos = cur.index(pivot)
-            except ValueError:
-                break
-            ret.append(cur[:pos])
-            cur = cur[pos+len(pivot):]
-        ret.append(cur)
-        return ret
-
-    def startswith(self, what):
-        return (what == str(self[:len(what)]))
-
-    def endswith(self, what):
-        return (what == str(self[-len(what):]))
-
-
-if __name__ == '__main__':
-    gs = GapString()
-    gs.append('hi')
-    assert str(gs) == 'hi'
-    assert str(gs[:40]) == 'hi'
-    gs.append(3)
-    assert str(gs) == 'hi???'
-    assert str(gs[:40]) == 'hi???'
-    assert str(gs[:3]) == 'hi?'
-    assert str(gs[-4:]) == 'i???'
-    assert str(gs + gs) == 'hi???hi???'
-    assert str(gs ^ 1) == 'ih???'
-
-    gs = GapString()
-    gs.append('123456789A')
-    assert str(gs[:4]) == '1234'
-    assert len(gs[:4]) == 4
-    assert len(gs[6:]) == 4
-    assert str(gs[:0]) == ''
-

netarch/__init__.py

@@ -1,7 +1,4 @@
-#! /usr/bin/python
+#! /usr/bin/python3
-# -*- coding: utf-8 -*-
-
-## 2008 Massive Blowout
 
 import sys
 import struct
@@ -40,24 +37,9 @@ decch = (u'␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏'
          u'················'
          u'················')
 
-cgach = (u'·☺☻♥♦♣♠•◘○◙♂♀♪♫☼'
-         u'►◄↕‼¶§▬↨↑↓→←∟↔▲▼'
-         u' !"#$%&\'()*+,-./'
-         u'0123456789:;<=>?'
-         u'@ABCDEFGHIJKLMNO'
-         u'PQRSTUVWXYZ[\]^_'
-         u'`abcdefghijklmno'
-         u'pqrstuvwxyz{|}~⌂'
-         u'ÇüéâäàåçêëèïîìÄÅ'
-         u'ÉæÆôöòûùÿÖÜ¢£¥₧ƒ'
-         u'áíóúñѪº¿⌐¬½¼¡«»'
-         u'░▒▓│┤╡╢╖╕╣║╗╝╜╛┐'
-         u'└┴┬├─┼╞╟╚╔╩╦╠═╬╧'
-         u'╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀'
-         u'αßΓπΣσµτΦΘΩδ∞φε∩'
-         u'≡±≥≤⌠⌡÷≈°∙·√ⁿ²■¤')
 
-shpch = (u'␀☺☻♥♦♣♠•◘○◙♂♀♪♫☼'
+
+cgach = (u'␀☺☻♥♦♣♠•◘○◙♂♀♪♫☼'
          u'►◄↕‼¶§▬↨↑↓→←∟↔▲▼'
          u'␣!"#$%&\'()*+,-./'
          u'0123456789:;<=>?'

netarch/ip.py

@@ -1,24 +1,22 @@
-#! /usr/bin/python
+#! /usr/bin/python3
 
 ## IP resequencing + protocol reversing skeleton
-## 2008 Massive Blowout
+## 2008, 2018 Neale Pickett
 
-import StringIO
 import struct
 import socket
 import warnings
 import heapq
-import gapstr
 import time
 try:
     import pcap
 except ImportError:
-    import py_pcap as pcap
+    import netarch.py_pcap as pcap
 import os
 import cgi
 import urllib
-import UserDict
+from netarch import *
-from __init__ import *
+from netarch.gapstr import *
 
 def unpack_nybbles(byte):
     return (byte >> 4, byte & 0x0F)
@@ -329,7 +327,7 @@ class TCP_Resequence:
             if key > seq:
                 # Dropped frame(s)
                 if key - seq > 6000:
-                    print "Gosh, bob, %d dropped octets sure is a lot!" % (key - seq)
+                    print("Gosh, %d dropped octets sure is a lot!" % (key - seq))
                 gs.append(key - seq)
                 seq = key
             if key == seq:
@@ -350,9 +348,9 @@ class TCP_Resequence:
         if seq != pkt.ack:
             # Drop at the end
             if pkt.ack - seq > 6000:
-                print 'Large drop at end of session!'
+                print('Large drop at end of session!')
-                print '    %s' % ((pkt, pkt.time),)
+                print('    %s' % ((pkt, pkt.time),))
-                print '    %x  %x' % (pkt.ack, seq)
+                print('    %x  %x' % (pkt.ack, seq))
             gs.append(pkt.ack - seq)
 
         return ret
@@ -490,7 +488,7 @@ class Dispatch:
 class NeedMoreData(Exception):
     pass
 
-class Packet(UserDict.DictMixin):
+class Packet:
     """Base class for a packet from a binary protocol.
 
     This is a base class for making protocol reverse-engineering easier.
@@ -549,16 +547,16 @@ class Packet(UserDict.DictMixin):
             assert a in b, ('%r not in %r' % (a, b))
 
     def show(self):
-        print '%s %3s: %s' % (self.__class__.__name__,
+        print('%s %3s: %s' % (self.__class__.__name__,
                               self.opcode,
-                              self.opcode_desc)
+                              self.opcode_desc))
         if self.firstframe:
-            print '    %s:%d -> %s:%d (%s.%06dZ)' % (self.firstframe.src_addr,
+            print('    %s:%d -> %s:%d (%s.%06dZ)' % (self.firstframe.src_addr,
                                                      self.firstframe.sport,
                                                      self.firstframe.dst_addr,
                                                      self.firstframe.dport,
                                                      time.strftime('%Y-%m-%dT%T', time.gmtime(self.firstframe.time)),
-                                                     self.firstframe.time_usec)
+                                                     self.firstframe.time_usec))
 
         if self.parts:
             dl = len(self.parts[-1])
@@ -568,12 +566,12 @@ class Packet(UserDict.DictMixin):
                     p.append('%3d!' % x)
                 else:
                     p.append('%3d' % x)
-            print '           parts: (%s) +%d bytes' % (','.join(p), dl)
+            print('           parts: (%s) +%d bytes' % (','.join(p), dl))
 
         keys = self.params.keys()
         keys.sort()
         for k in keys:
-            print '    %12s: %s' % (k, self.params[k])
+            print('    %12s: %s' % (k, self.params[k]))
 
         if self.subpackets:
             for p in self.subpackets:
@@ -582,7 +580,7 @@ class Packet(UserDict.DictMixin):
             try:
                 self.payload.hexdump()
             except AttributeError:
-                print '         payload: %r' % self.payload
+                print('         payload: %r' % self.payload)
 
     def parse(self, data):
         """Parse a chunk of data (possibly a GapString).
@@ -600,7 +598,7 @@ class Packet(UserDict.DictMixin):
         """Handle data from a Session class."""
 
         data = self.parse(data)
-        if self.opcode <> None:
+        if self.opcode != None:
             try:
                 f = getattr(self, 'opcode_%s' % self.opcode)
             except AttributeError:
@@ -701,7 +699,7 @@ class Session:
                                       urllib.quote(fn, ''))
         fullfn = os.path.join(self.basename, fn)
         fullfn2 = os.path.join(self.basename2, fn)
-        print '  writing %s' % (fn,)
+        print('  writing %s' % (fn,))
         fd = file(fullfn, 'w')
         try:
             os.unlink(fullfn2)

netarch/trilobytes.py

@@ -0,0 +1,104 @@
+#! /usr/bin/python3
+
+## 2008, 2018 Neale Pickett
+
+import itertools
+
+class TriloBytes:
+    """Three-level byte array (0, 1, Missing).
+
+This allows you to represent on-wire transactions with holes in the middle,
+due to eg. dropped packets.
+"""
+
+    def __init__(self, initializer=(), drop=b'?'):
+        self._drop = drop
+        self._contents = tuple(initializer)
+
+    @classmethod
+    def fromhex(cls, string):
+        return cls(bytes.fromhex(string))
+
+    @classmethod
+    def join(cls, *objects):
+        contents = []
+        for o in objects:
+            # print(o)
+            contents.extend(o._contents)
+
+        new = cls()
+        new._contents = tuple(contents)
+        return new
+
+    def __len__(self):
+        return len(self._contents)
+
+    def __nonzero__(self):
+        return len(self) > 0
+
+    def __getitem__(self, key):
+        ret = self._contents[key]
+        try:
+            return TriloBytes(ret, self._drop)
+        except:
+            return ret
+
+    def __iter__(self):
+        for val in self._contents:
+            yield val
+
+    def __bytes__(self):
+        return bytes((v or d for v,d in zip(self,itertools.cycle(self._drop))))
+
+    def __add__(self, other):
+        try:
+            contents = self._contents + other._contents
+        except AttributeError:
+            contents = self._contents + tuple(other)
+        return TriloBytes(contents, self._drop)
+
+    def __eq__(self, other):
+        try:
+            return self._contents == other._contents
+        except:
+            return False
+
+    def __hash__(self):
+        return hash(self._contents)
+        
+    def __xor__(self, mask):
+        try:
+            mask[0]
+        except TypeError:
+            mask = [mask]
+        return TriloBytes(((x^y if x else None) for x,y in zip(self._contents, itertools.cycle(mask))), drop=self._drop)
+
+    def __repr__(self):
+        return '<TriloBytes missing %d of %d>' % (self.missing(), len(self))
+
+    def missing(self):
+        return self._contents.count(None)
+
+    def map(self, func, *args):
+        return (v if v is not None else func(v, *args) for v in self)
+
+
+if __name__ == '__main__':
+    gs = TriloBytes(b'hi')
+    assert bytes(gs) == b'hi'
+    assert bytes(gs[:40]) == b'hi'
+
+    gs = gs + [None] * 3
+    assert bytes(gs) == b'hi???'
+    assert bytes(gs[:40]) == b'hi???'
+    assert bytes(gs[:3]) == b'hi?'
+    assert bytes(gs[-4:]) == b'i???'
+    assert bytes(gs + gs) == b'hi???hi???'
+    assert bytes(gs ^ 1) == b'ih???'
+    assert bytes(gs ^ [32, 1]) == b'Hh???'
+    
+    gs = TriloBytes(b'hi', drop=b'DROP')
+    assert bytes(gs) == b'hi'
+    
+    gs = gs + [None] * 7
+    assert bytes(gs) == b'hiOPDROPD'