From 28868f081346b8f734847ff03dc7a74d1a1b3c31 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 24 Jul 2018 23:32:08 +0000 Subject: [PATCH] Further documentation --- gapstring/gapstring.go | 59 +++++++++++++++++++++++++++++++++++++++--- netshovel.go | 6 ++++- packet.go | 3 ++- stream.go | 39 ++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/gapstring/gapstring.go b/gapstring/gapstring.go index d8548b1..1b71a4f 100644 --- a/gapstring/gapstring.go +++ b/gapstring/gapstring.go @@ -40,33 +40,47 @@ func (c chunk) slice(a, b int) chunk { } } - +// A GapString is a string with gaps of no data in the middle +// +// Gaps are represented efficiently, +// both in memory and in computation. +// +// Several convenience functions exist +// which operate on GapString data, +// while preserving the Gaps. type GapString struct { chunks []chunk } +// Return a new zero-length GapString func New() GapString { return GapString{ chunks: []chunk{}, } } +// Return a new GapString containing a gap func OfGap(gap int) GapString { return GapString{ chunks: []chunk{{gap: gap}}, } } +// Return a new GapString containing some bytes func OfBytes(b []byte) GapString { return GapString{ chunks: []chunk{{data: b}}, } } +// Return a new GapString containing a string func OfString(s string) GapString { return OfBytes([]byte(s)) } +// Return the length of a GapString +// +// This is the number of bytes you would have if the gaps were filled with some value. func (g GapString) Length() int { n := 0 for _, c := range g.chunks { @@ -75,6 +89,7 @@ func (g GapString) Length() int { return n } +// Return the total size of all gaps func (g GapString) Missing() int { n := 0 for _, c := range g.chunks { @@ -83,6 +98,7 @@ func (g GapString) Missing() int { return n } +// Return the current GapString with another GapString appended func (g GapString) Append(h GapString) GapString { if h.Length() > 0 { return GapString{ @@ -93,18 +109,25 @@ func (g GapString) Append(h GapString) GapString { } } +// Return the current GapString with a gap appended func (g GapString) AppendGap(gap int) GapString { return g.Append(OfGap(gap)) } +// Return the current GapString with some bytes appended func (g GapString) AppendBytes(b []byte) GapString { return g.Append(OfBytes(b)) } +// Return the current GapString with a string appended func (g GapString) AppendString(s string) GapString { return g.Append(OfString(s)) } +// Return a slice of this GapString +// +// This is what you would expect from g[start:end], +// if g were a string or byte slice. func (g GapString) Slice(start, end int) GapString { outchunks := make([]chunk, 0, len(g.chunks)) @@ -141,6 +164,9 @@ func (g GapString) Slice(start, end int) GapString { return GapString{chunks: outchunks} } +// Return this GapString with the provided xor mask applied +// +// The mask is cycled for the length of the GapString. func (g GapString) Xor(mask ...byte) GapString { ret := GapString{} pos := 0 @@ -159,6 +185,7 @@ func (g GapString) Xor(mask ...byte) GapString { return ret } +// Return this GapString with gaps filled in func (g GapString) Bytes(fill ...byte) []byte { ret := make([]byte, g.Length()) pos := 0 @@ -180,7 +207,9 @@ func (g GapString) Bytes(fill ...byte) []byte { return ret } -// Returns -1 if it's a gap +// Returns the value at a specific position +// +// This returns the byte if one is present, or -1 if it's a gap func (g GapString) ValueAt(pos int) int { v := g.Slice(pos, pos+1) if v.chunks[0].gap > 0 { @@ -190,10 +219,14 @@ func (g GapString) ValueAt(pos int) int { } } +// Return a string version of the GapString, with gaps filled in func (g GapString) String(fill string) string { return string(g.Bytes([]byte(fill)...)) } +// Return a hex representation of this GapString +// +// Each octet is space-separated, and gaps are represented with "--" func (g GapString) HexString() string { out := new(strings.Builder) glen := g.Length() @@ -233,6 +266,12 @@ var fluffych = []rune{ 'α', 'ß', 'Γ', 'π', 'Σ', 'σ', 'µ', 'τ', 'Φ', 'Θ', 'Ω', 'δ', '∞', 'φ', 'ε', '∩', '≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∀', '∃', '√', 'ⁿ', '²', '■', '¤', } + +// Return a rune representation of this GapString +// +// This uses the glyph set from the Fluffy toolkit +// (https://dirtbags.github.io/fluffy/). +// Gaps are represented with the rune '�' func (g GapString) Runes() string { out := new(strings.Builder) glen := g.Length() @@ -247,6 +286,7 @@ func (g GapString) Runes() string { return out.String() } +// Return a hex dump of this GapString func (g GapString) Hexdump() string { out := new(strings.Builder) skipping := false @@ -280,14 +320,21 @@ func (g GapString) Hexdump() string { return out.String() } +// Return a uint32, little-endian, taken from the front of this GapString +// +// The rest of the GapString is returned as the second argument. func (g GapString) Uint32LE() (uint32, GapString) { - return binary.LittleEndian.Uint32(g.Slice(0, 4).Bytes()), g.Slice(4, g.Length()) + return binary.LittleEndian.Uint32(g.Slice(0, 4).Bytes(0)), g.Slice(4, g.Length()) } +// Return a uint16, little-endian, taken from the front of this GapString +// +// The rest of the GapString is returned as the second argument. func (g GapString) Uint16LE() (uint16, GapString) { - return binary.LittleEndian.Uint16(g.Slice(0, 2).Bytes()), g.Slice(2, g.Length()) + return binary.LittleEndian.Uint16(g.Slice(0, 2).Bytes(0)), g.Slice(2, g.Length()) } +// Return this GapString decoded as UTF-16 func (g GapString) Utf16(order binary.ByteOrder, fill string) string { in := g.Bytes([]byte(fill)...) ints := make([]uint16, len(in)/2) @@ -298,10 +345,14 @@ func (g GapString) Utf16(order binary.ByteOrder, fill string) string { return string(utf16.Decode(ints)) } +// Return this GapString decoded as UTF-16 Little Endian +// +// This format is used extensively in Microsoft Windows. func (g GapString) Utf16LE(gap string) string { return g.Utf16(binary.LittleEndian, gap) } +// Return this GapString decoded as UTF-16 Big Endian func (g GapString) Utf16BE(gap string) string { return g.Utf16(binary.BigEndian, gap) } diff --git a/netshovel.go b/netshovel.go index c99a8a4..3eb26bd 100644 --- a/netshovel.go +++ b/netshovel.go @@ -9,6 +9,11 @@ import ( "github.com/google/gopacket/tcpassembly" ) +// Mainloop to handle dispatching of PCAP files from command line +// +// This parses the command line arguments, +// and for each PCAP file specified on the command line, +// invokes a TCP assembler that sends streams to whatever is returned from factory. func Shovel(factory tcpassembly.StreamFactory) { //verbose := flag.Bool("verbose", false, "Write lots of information out") flag.Parse() @@ -29,7 +34,6 @@ func Shovel(factory tcpassembly.StreamFactory) { break } if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP { - log.Println("Unusable packet") continue } tcp := packet.TransportLayer().(*layers.TCP) diff --git a/packet.go b/packet.go index bed7589..b5e1622 100644 --- a/packet.go +++ b/packet.go @@ -11,7 +11,8 @@ import ( // Error returned by convenience methods that are unable to get enough data type ShortError struct { - wanted, available int + Wanted int // How many bytes you needed + Available int // How many bytes were available } func (e *ShortError) Error() string { return fmt.Sprintf("Short read: wanted %d of %d available", e.wanted, e.available) diff --git a/stream.go b/stream.go index a8094fb..5e736b1 100644 --- a/stream.go +++ b/stream.go @@ -12,22 +12,36 @@ import ( "github.com/google/gopacket/tcpassembly" ) +// A File and the path where it lives type NamedFile struct { *os.File Name string } +// An atomic communication within a Stream +// +// Streams consist of a string of Utterances. +// Each utterance has associated data, and a time stamp. +// +// Typically these line up with what crosses the network, +// but bear in mind that TCP is a streaming protocol, +// so don't rely on Utterances alone to separate Application-layer packets. type Utterance struct { When time.Time Data gapstring.GapString } +// A Stream is one half of a two-way conversation type Stream struct { Net, Transport gopacket.Flow conversation chan Utterance pending Utterance } +// Return a newly-built Stream +// +// You should embed Stream into your own Application protocol stream struct. +// Use this to initialize the internal stuff netshovel needs. func NewStream(net, transport gopacket.Flow) Stream { return Stream{ Net: net, @@ -36,6 +50,7 @@ func NewStream(net, transport gopacket.Flow) Stream { } } +// Called by the TCP assembler when an Utterance can be built func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) { ret := Utterance{ When: rs[0].Seen, @@ -53,10 +68,23 @@ func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) { } } +// Called by the TCP assemble when the Stream is closed func (stream *Stream) ReassemblyComplete() { close(stream.conversation) } +// Read an utterance of a particular size +// +// If you pass in a length of -1, +// this returns utterances as they appear in the conversation. +// +// At first, your decoder will probably want to use a length of -1: +// this will give you a sense of how the conversation works. +// When you begin to understand the structure of your protocol, +// change this to a positive integer, +// so that if you have a large application-layer packet, +// or multiple application-layer packets in a single transport-layer packet, +// your decoder handles it properly. func (stream *Stream) Read(length int) (Utterance, error) { // This probably indicates a problem, but we assume you know what you're doing if length == 0 { @@ -109,6 +137,9 @@ func (stream *Stream) Read(length int) (Utterance, error) { return ret, nil } +// Return a string description of a packet +// +// This just prefixes our source and dest IP:Port to pkt.Describe() func (stream *Stream) Describe(pkt Packet) string { out := new(strings.Builder) @@ -120,6 +151,14 @@ func (stream *Stream) Describe(pkt Packet) string { return out.String() } +// Return a newly-created, truncated file +// +// This function creates consistently-named files, +// which include a timestamp, +// and URL-escaped full path to the file. +// +// Best practice is to pass in as full a path as you can find, +// including drive letters and all parent directories. func (stream *Stream) CreateFile(when time.Time, path string) (NamedFile, error) { name := fmt.Sprintf( "xfer/%s,%sp%s,%sp%s,%s",