resequencer and esab64

This features a resequencer that deals with more than one session per pcap file
(this will break all former code), and a decoder for a type of
incorrectly-implemented base64.

__init__.py

@@ -83,7 +83,7 @@ def bin(i):
     return s
 
 class bitvector:
-    def __init__(self, i, length=None):
+    def __init__(self, i=0, length=None):
         if type(i) == type(''):
             self._val = 0
             for c in i:
@@ -147,3 +147,31 @@ class bitvector:
         l.reverse()
         return '<bitvector ' + ''.join(str(x) for x in l) + '>'
 
+    def __add__(self, i):
+        if isinstance(i, bitvector):
+            l = len(self) + len(i)
+            v = (int(self) << len(i)) + int(i)
+            return bitvector(v, l)
+        else:
+            raise ValueError("Can't extend with this type yet")
+
+
+b64_chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+def esab64_decode(s):
+    """Little-endian version of base64"""
+
+    r = []
+    for i in range(0, len(s), 4):
+        v = bitvector()
+        for c in s[i:i+4]:
+            if c == '=':
+                break
+            v += bitvector(b64_chars.index(c), 6)
+
+        # Normal base64 would start at the beginning
+        b = (v[10:12] + v[ 0: 6] +
+             v[14:18] + v[ 6:10] +
+             v[18:24] + v[12:14])
+
+        r.append(str(b))
+    return ''.join(r)

resequence.py

@@ -3,6 +3,7 @@
 import StringIO
 import struct
 import socket
+import warnings
 
 def unpack(fmt, buf):
     """Unpack buf based on fmt, assuming the rest is a string."""
@@ -14,8 +15,16 @@ def unpack(fmt, buf):
 def unpack_nybbles(byte):
     return (byte >> 4, byte & 0x0F)
 
+ICMP = 1
+TCP = 6
+UDP = 17
+
 class Frame:
-    def __init__(self, frame):
+    """Turn an ethernet frame into relevant TCP parts"""
+
+    def __init__(self, pkt):
+        ((self.time, _, _), frame) = pkt
+
         # Ethernet
         (self.eth_dhost,
          self.eth_shost,
@@ -35,48 +44,67 @@ class Frame:
          self.check,
          self.saddr,
          self.daddr,
-         p) = unpack("!BBHHHBBH4s4s", p)
+         p) = unpack("!BBHHHBBHii", p)
-        if self.protocol != 6:
-            raise ValueError('Not TCP')
 
-        # TCP
+        if self.protocol == TCP:
-        (self.th_sport,
+            self.name = 'TCP'
-         self.th_dport,
+            (self.sport,
-         self.th_seq,
+             self.dport,
-         self.th_ack,
+             self.seq,
-         x2off,
+             self.ack,
-         self.th_flags,
+             x2off,
-         self.th_win,
+             self.flags,
-         self.th_sum,
+             self.win,
-         self.th_urp,
+             self.sum,
-         p) = unpack("!HHLLBBHHH", p)
+             self.urp,
-        (self.th_off, th_x2) = unpack_nybbles(x2off)
+             p) = unpack("!HHLLBBHHH", p)
-        opt_length = self.th_off * 4
+            (self.off, th_x2) = unpack_nybbles(x2off)
-
+            opt_length = self.off * 4
-        self.th_options, p = p[:opt_length - 20], p[opt_length - 20:]
+            self.options, p = p[:opt_length - 20], p[opt_length - 20:]
-        payload = p[:self.tot_len - opt_length - 20]
+            self.payload = p[:self.tot_len - opt_length - 20]
+        elif self.protocol == UDP:
+            self.name = 'UDP'
+            (self.sport,
+             self.dport,
+             self.ulen,
+             self.sum,
+             p) = unpack("!HHHH", p)
+            self.payload = p[:self.ulen - 8]
+        elif self.protocol == ICMP:
+            self.name = 'ICMP'
+            self.sport = self.dport = -1
+            (self.type,
+             self.code,
+             self.cheksum,
+             self.id,
+             self.seq,
+             p) = unpackt('!BBHHH', p)
+            self.payload = p[:self.tot-len - 8]
+        else:
+            raise ValueError('Unknown protocol')
 
         # Nice formatting
-        self.src = (self.saddr, self.th_sport)
+        self.src = (self.saddr, self.sport)
-        self.dst = (self.daddr, self.th_dport)
+        self.dst = (self.daddr, self.dport)
-        self.seq = self.th_seq
+        self.hash = (self.saddr ^ self.sport ^ self.daddr ^ self.dport)
-        self.ack = self.th_ack
-        self.payload = payload
 
     def get_src_addr(self):
-        self.src_addr = socket.inet_ntoa(self.saddr)
+        saddr = struct.pack('!i', self.saddr)
+        self.src_addr = socket.inet_ntoa(saddr)
         return self.src_addr
     src_addr = property(get_src_addr)
 
     def get_dst_addr(self):
-        self.dst_addr = socket.inet_ntoa(self.daddr)
+        daddr = struct.pack('!i', self.daddr)
+        self.dst_addr = socket.inet_ntoa(daddr)
         return self.dst_addr
     dst_addr = property(get_dst_addr)
 
     def __repr__(self):
-        return '<Frame %s:%d -> %s:%d len %d>' % (self.src_addr, self.th_sport,
+        return '<Frame %s %s:%d -> %s:%d len %d>' % (self.name,
-                                                  self.dst_addr, self.th_dport,
+                                                     self.src_addr, self.sport,
-                                                  len(self.payload))
+                                                     self.dst_addr, self.dport,
+                                                     len(self.payload))
 
 
 class DropStringIO(StringIO.StringIO):
@@ -116,99 +144,94 @@ class TCP_Session:
 
     """
 
-    def __init__(self, pc):
+    def __init__(self):
-        self.pc = pc
-
         self.cli = None
         self.srv = None
         self.seq = [None, None]
+        self.first = None
         self.pending = [{}, {}]
         self.frames = 0
+        self.closed = 0
 
-        self.read_handshake()
+        self.handle = self.handle_handshake
 
-    def read_packet(self):
-        while True:
-            p = self.pc.read()
-            if not p:
-                raise EOFError()
-            try:
-                return Frame(p[1])
-            except ValueError:
-                pass
 
-    def read_handshake(self):
+    def handle(self, pkt):
-        # Read SYN
+        """Stub.
-        pkt = self.read_packet()
-        assert (pkt.th_flags == 2) # XXX: There's got to be a better way
-        self.cli = pkt.src
-        self.srv = pkt.dst
-        self.seq[0] = pkt.seq + 1
 
-        # Read SYN-ACK
+        This function will never be called, it is immediately overridden
-        while True:
+        by __init__.  The current value of this function is the state.
-            pkt = self.read_packet()
+        """
-            if ((pkt.src == self.srv) and
-                (pkt.th_flags == 18)):
-                self.seq[1] = pkt.th_seq + 1
-                break
 
-        # Read ACK
+        pass
-        while True:
-            pkt = self.read_packet()
-            if ((pkt.src == self.cli) and
-                (pkt.th_flags == 16)):
-                assert (self.seq[0] == pkt.th_seq)
-                break
 
-        self.frames = 3
+    def handle_handshake(self, pkt):
+        self.frames += 1
 
-    def __iter__(self):
+        if not self.first:
-        while True:
+            self.first = pkt
-            try:
-                pkt = self.read_packet()
-            except EOFError:
-                return
-            self.frames += 1
 
-            # Which way is this going?
+        if pkt.flags == 2:              # SYN
-            idx = int(pkt.src == self.srv)
+            self.cli, self.srv = pkt.src, pkt.dst
-            xdi = 1 - idx
+        elif pkt.flags == 18:           # SYNACK
+            assert (pkt.src == (self.srv or pkt.src))
+            self.cli, self.srv = pkt.dst, pkt.src
+            self.seq = [pkt.ack + 1, pkt.seq + 1]
+        elif pkt.flags == 16:           # ACK
+            assert (pkt.src == (self.cli or pkt.src))
+            self.cli, self.srv = pkt.src, pkt.dst
+            self.seq = [pkt.seq, pkt.ack + 1]
+            self.handle = self.handle_packet
+        else:
+            raise ValueError('Weird flags in handshake: %d' % pkt.flags)
 
-            # Does this ACK after the last output sequence number?
+    def handle_packet(self, pkt):
-            if pkt.th_ack > self.seq[xdi]:
+        ret = None
-                pending = self.pending[xdi]
+        self.frames += 1
-                seq = self.seq[xdi]
-                ret = DropStringIO()
-                keys = pending.keys()
-                for key in keys:
-                    if key >= pkt.th_ack:
-                        continue
 
-                    pkt2 = pending[key]
+        # Which way is this going?
-                    del pending[key]
+        idx = int(pkt.src == self.srv)
+        xdi = 1 - idx
 
-                    ret.seek(pkt2.th_seq - seq)
+        # Does this ACK after the last output sequence number?
-                    ret.write(pkt2.payload)
+        if pkt.ack > self.seq[xdi]:
-                self.seq[xdi] = pkt.th_ack
+            pending = self.pending[xdi]
+            seq = self.seq[xdi]
+            ret = DropStringIO()
+            keys = pending.keys()
+            for key in keys:
+                if key >= pkt.ack:
+                    continue
 
-                yield (xdi, ret.getvalue())
+                pkt2 = pending[key]
+                del pending[key]
 
-            # If it has a payload, stick it into pending
+                ret.seek(pkt2.seq - seq)
-            if pkt.payload:
+                ret.write(pkt2.payload)
-                self.pending[idx][pkt.seq] = pkt
+            self.seq[xdi] = pkt.ack
-        self.done()
 
-    def done(self):
+            ret = (xdi, ret.getvalue())
+
+        # If it has a payload, stick it into pending
+        if pkt.payload:
+            self.pending[idx][pkt.seq] = pkt
+
+        # Is it a FIN or RST?
+        if pkt.flags & 5:
+            self.closed += 1
+            if self.closed == 2:
+                # Warn about any unhandled packets
+                if self.pending[0] or self.pending[1]:
+                    warnings.warn('Unhandled packets')
+                self.handle = self.handle_drop
+
+        return ret
+
+    def handle_drop(self, pkt):
         """Warn about any unhandled packets"""
 
-        for p in self.pending:
+        if not pkt.flags & 5:
-            k = p.keys()
+            warnings.warn('Extra packets at the end')
-            if k:
-                k.sort()
-                print 'unused packets:', k
-        return
-
 
 
 class HTTP_side:
@@ -265,6 +288,21 @@ class HTTP_side:
             self.pending_data = int(v)
 
 
+def resequence(pc):
+    sessions = {}
+    for pkt in pc:
+        f = Frame(pkt)
+        if f.protocol == TCP:
+            # compute TCP session hash
+            s = sessions.get(f.hash)
+            if not s:
+                s = TCP_Session()
+                sessions[f.hash] = s
+            r = s.handle(f)
+            if r:
+                yield (f, r)
+
+
 def process_http(filename):
     import pcap
 
@@ -285,3 +323,7 @@ def process_http(filename):
 
     return packets
 
+
+
+
+