From d1fb3439806f37d63cd77d82404fd9bd45c6bb0c Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Fri, 21 Dec 2007 17:08:00 -0700 Subject: [PATCH] resequencer and esab64 This features a resequencer that deals with more than one session per pcap file (this will break all former code), and a decoder for a type of incorrectly-implemented base64. --- __init__.py | 30 +++++- resequence.py | 246 +++++++++++++++++++++++++++++--------------------- 2 files changed, 173 insertions(+), 103 deletions(-) diff --git a/__init__.py b/__init__.py index 2dfc0bb..f70c2ee 100755 --- a/__init__.py +++ b/__init__.py @@ -83,7 +83,7 @@ def bin(i): return s class bitvector: - def __init__(self, i, length=None): + def __init__(self, i=0, length=None): if type(i) == type(''): self._val = 0 for c in i: @@ -147,3 +147,31 @@ class bitvector: l.reverse() return '' + def __add__(self, i): + if isinstance(i, bitvector): + l = len(self) + len(i) + v = (int(self) << len(i)) + int(i) + return bitvector(v, l) + else: + raise ValueError("Can't extend with this type yet") + + +b64_chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' +def esab64_decode(s): + """Little-endian version of base64""" + + r = [] + for i in range(0, len(s), 4): + v = bitvector() + for c in s[i:i+4]: + if c == '=': + break + v += bitvector(b64_chars.index(c), 6) + + # Normal base64 would start at the beginning + b = (v[10:12] + v[ 0: 6] + + v[14:18] + v[ 6:10] + + v[18:24] + v[12:14]) + + r.append(str(b)) + return ''.join(r) diff --git a/resequence.py b/resequence.py index cf68183..c7ebc20 100755 --- a/resequence.py +++ b/resequence.py @@ -3,6 +3,7 @@ import StringIO import struct import socket +import warnings def unpack(fmt, buf): """Unpack buf based on fmt, assuming the rest is a string.""" @@ -14,8 +15,16 @@ def unpack(fmt, buf): def unpack_nybbles(byte): return (byte >> 4, byte & 0x0F) +ICMP = 1 +TCP = 6 +UDP = 17 + class Frame: - def __init__(self, frame): + """Turn an ethernet frame into relevant TCP parts""" + + def __init__(self, pkt): + ((self.time, _, _), frame) = pkt + # Ethernet (self.eth_dhost, self.eth_shost, @@ -35,48 +44,67 @@ class Frame: self.check, self.saddr, self.daddr, - p) = unpack("!BBHHHBBH4s4s", p) - if self.protocol != 6: - raise ValueError('Not TCP') + p) = unpack("!BBHHHBBHii", p) - # TCP - (self.th_sport, - self.th_dport, - self.th_seq, - self.th_ack, - x2off, - self.th_flags, - self.th_win, - self.th_sum, - self.th_urp, - p) = unpack("!HHLLBBHHH", p) - (self.th_off, th_x2) = unpack_nybbles(x2off) - opt_length = self.th_off * 4 - - self.th_options, p = p[:opt_length - 20], p[opt_length - 20:] - payload = p[:self.tot_len - opt_length - 20] + if self.protocol == TCP: + self.name = 'TCP' + (self.sport, + self.dport, + self.seq, + self.ack, + x2off, + self.flags, + self.win, + self.sum, + self.urp, + p) = unpack("!HHLLBBHHH", p) + (self.off, th_x2) = unpack_nybbles(x2off) + opt_length = self.off * 4 + self.options, p = p[:opt_length - 20], p[opt_length - 20:] + self.payload = p[:self.tot_len - opt_length - 20] + elif self.protocol == UDP: + self.name = 'UDP' + (self.sport, + self.dport, + self.ulen, + self.sum, + p) = unpack("!HHHH", p) + self.payload = p[:self.ulen - 8] + elif self.protocol == ICMP: + self.name = 'ICMP' + self.sport = self.dport = -1 + (self.type, + self.code, + self.cheksum, + self.id, + self.seq, + p) = unpackt('!BBHHH', p) + self.payload = p[:self.tot-len - 8] + else: + raise ValueError('Unknown protocol') # Nice formatting - self.src = (self.saddr, self.th_sport) - self.dst = (self.daddr, self.th_dport) - self.seq = self.th_seq - self.ack = self.th_ack - self.payload = payload + self.src = (self.saddr, self.sport) + self.dst = (self.daddr, self.dport) + self.hash = (self.saddr ^ self.sport ^ self.daddr ^ self.dport) def get_src_addr(self): - self.src_addr = socket.inet_ntoa(self.saddr) + saddr = struct.pack('!i', self.saddr) + self.src_addr = socket.inet_ntoa(saddr) return self.src_addr src_addr = property(get_src_addr) def get_dst_addr(self): - self.dst_addr = socket.inet_ntoa(self.daddr) + daddr = struct.pack('!i', self.daddr) + self.dst_addr = socket.inet_ntoa(daddr) return self.dst_addr dst_addr = property(get_dst_addr) def __repr__(self): - return ' %s:%d len %d>' % (self.src_addr, self.th_sport, - self.dst_addr, self.th_dport, - len(self.payload)) + return ' %s:%d len %d>' % (self.name, + self.src_addr, self.sport, + self.dst_addr, self.dport, + len(self.payload)) class DropStringIO(StringIO.StringIO): @@ -116,99 +144,94 @@ class TCP_Session: """ - def __init__(self, pc): - self.pc = pc - + def __init__(self): self.cli = None self.srv = None self.seq = [None, None] + self.first = None self.pending = [{}, {}] self.frames = 0 + self.closed = 0 - self.read_handshake() + self.handle = self.handle_handshake - def read_packet(self): - while True: - p = self.pc.read() - if not p: - raise EOFError() - try: - return Frame(p[1]) - except ValueError: - pass - def read_handshake(self): - # Read SYN - pkt = self.read_packet() - assert (pkt.th_flags == 2) # XXX: There's got to be a better way - self.cli = pkt.src - self.srv = pkt.dst - self.seq[0] = pkt.seq + 1 + def handle(self, pkt): + """Stub. - # Read SYN-ACK - while True: - pkt = self.read_packet() - if ((pkt.src == self.srv) and - (pkt.th_flags == 18)): - self.seq[1] = pkt.th_seq + 1 - break + This function will never be called, it is immediately overridden + by __init__. The current value of this function is the state. + """ - # Read ACK - while True: - pkt = self.read_packet() - if ((pkt.src == self.cli) and - (pkt.th_flags == 16)): - assert (self.seq[0] == pkt.th_seq) - break + pass - self.frames = 3 + def handle_handshake(self, pkt): + self.frames += 1 - def __iter__(self): - while True: - try: - pkt = self.read_packet() - except EOFError: - return - self.frames += 1 + if not self.first: + self.first = pkt - # Which way is this going? - idx = int(pkt.src == self.srv) - xdi = 1 - idx + if pkt.flags == 2: # SYN + self.cli, self.srv = pkt.src, pkt.dst + elif pkt.flags == 18: # SYNACK + assert (pkt.src == (self.srv or pkt.src)) + self.cli, self.srv = pkt.dst, pkt.src + self.seq = [pkt.ack + 1, pkt.seq + 1] + elif pkt.flags == 16: # ACK + assert (pkt.src == (self.cli or pkt.src)) + self.cli, self.srv = pkt.src, pkt.dst + self.seq = [pkt.seq, pkt.ack + 1] + self.handle = self.handle_packet + else: + raise ValueError('Weird flags in handshake: %d' % pkt.flags) - # Does this ACK after the last output sequence number? - if pkt.th_ack > self.seq[xdi]: - pending = self.pending[xdi] - seq = self.seq[xdi] - ret = DropStringIO() - keys = pending.keys() - for key in keys: - if key >= pkt.th_ack: - continue + def handle_packet(self, pkt): + ret = None + self.frames += 1 - pkt2 = pending[key] - del pending[key] + # Which way is this going? + idx = int(pkt.src == self.srv) + xdi = 1 - idx - ret.seek(pkt2.th_seq - seq) - ret.write(pkt2.payload) - self.seq[xdi] = pkt.th_ack + # Does this ACK after the last output sequence number? + if pkt.ack > self.seq[xdi]: + pending = self.pending[xdi] + seq = self.seq[xdi] + ret = DropStringIO() + keys = pending.keys() + for key in keys: + if key >= pkt.ack: + continue - yield (xdi, ret.getvalue()) + pkt2 = pending[key] + del pending[key] - # If it has a payload, stick it into pending - if pkt.payload: - self.pending[idx][pkt.seq] = pkt - self.done() + ret.seek(pkt2.seq - seq) + ret.write(pkt2.payload) + self.seq[xdi] = pkt.ack - def done(self): + ret = (xdi, ret.getvalue()) + + # If it has a payload, stick it into pending + if pkt.payload: + self.pending[idx][pkt.seq] = pkt + + # Is it a FIN or RST? + if pkt.flags & 5: + self.closed += 1 + if self.closed == 2: + # Warn about any unhandled packets + if self.pending[0] or self.pending[1]: + warnings.warn('Unhandled packets') + self.handle = self.handle_drop + + return ret + + def handle_drop(self, pkt): """Warn about any unhandled packets""" - for p in self.pending: - k = p.keys() - if k: - k.sort() - print 'unused packets:', k - return - + if not pkt.flags & 5: + warnings.warn('Extra packets at the end') class HTTP_side: @@ -265,6 +288,21 @@ class HTTP_side: self.pending_data = int(v) +def resequence(pc): + sessions = {} + for pkt in pc: + f = Frame(pkt) + if f.protocol == TCP: + # compute TCP session hash + s = sessions.get(f.hash) + if not s: + s = TCP_Session() + sessions[f.hash] = s + r = s.handle(f) + if r: + yield (f, r) + + def process_http(filename): import pcap @@ -285,3 +323,7 @@ def process_http(filename): return packets + + + +