Further documentation

gapstring/gapstring.go

@@ -40,33 +40,47 @@ func (c chunk) slice(a, b int) chunk {
 	}
 }
 
-
+// A GapString is a string with gaps of no data in the middle
+//
+// Gaps are represented efficiently,
+// both in memory and in computation.
+//
+// Several convenience functions exist
+// which operate on GapString data,
+// while preserving the Gaps.
 type GapString struct {
 	chunks []chunk
 }
 
+// Return a new zero-length GapString
 func New() GapString {
 	return GapString{
 		chunks: []chunk{},
 	}
 }
 
+// Return a new GapString containing a gap
 func OfGap(gap int) GapString {
 	return GapString{
 		chunks: []chunk{{gap: gap}},
 	}
 }
 
+// Return a new GapString containing some bytes
 func OfBytes(b []byte) GapString {
 	return GapString{
 		chunks: []chunk{{data: b}},
 	}
 }
 
+// Return a new GapString containing a string
 func OfString(s string) GapString {
 	return OfBytes([]byte(s))
 }
 
+// Return the length of a GapString
+//
+// This is the number of bytes you would have if the gaps were filled with some value.
 func (g GapString) Length() int {
 	n := 0
 	for _, c := range g.chunks {
@@ -75,6 +89,7 @@ func (g GapString) Length() int {
 	return n
 }
 
+// Return the total size of all gaps
 func (g GapString) Missing() int {
 	n := 0
 	for _, c := range g.chunks {
@@ -83,6 +98,7 @@ func (g GapString) Missing() int {
 	return n
 }
 
+// Return the current GapString with another GapString appended
 func (g GapString) Append(h GapString) GapString {
 	if h.Length() > 0 {
 		return GapString{
@@ -93,18 +109,25 @@ func (g GapString) Append(h GapString) GapString {
 	}
 }
 
+// Return the current GapString with a gap appended
 func (g GapString) AppendGap(gap int) GapString {
 	return g.Append(OfGap(gap))
 }
 
+// Return the current GapString with some bytes appended
 func (g GapString) AppendBytes(b []byte) GapString {
 	return g.Append(OfBytes(b))
 }
 
+// Return the current GapString with a string appended
 func (g GapString) AppendString(s string) GapString {
 	return g.Append(OfString(s))
 }
 
+// Return a slice of this GapString
+//
+// This is what you would expect from g[start:end],
+// if g were a string or byte slice.
 func (g GapString) Slice(start, end int) GapString {
 	outchunks := make([]chunk, 0, len(g.chunks))
 	
@@ -141,6 +164,9 @@ func (g GapString) Slice(start, end int) GapString {
 	return GapString{chunks: outchunks}
 }
 
+// Return this GapString with the provided xor mask applied
+//
+// The mask is cycled for the length of the GapString.
 func (g GapString) Xor(mask ...byte) GapString {
 	ret := GapString{}
 	pos := 0
@@ -159,6 +185,7 @@ func (g GapString) Xor(mask ...byte) GapString {
 	return ret
 }
 
+// Return this GapString with gaps filled in
 func (g GapString) Bytes(fill ...byte) []byte {
 	ret := make([]byte, g.Length())
 	pos := 0
@@ -180,7 +207,9 @@ func (g GapString) Bytes(fill ...byte) []byte {
 	return ret
 }
 
-// Returns -1 if it's a gap
+// Returns the value at a specific position
+//
+// This returns the byte if one is present, or -1 if it's a gap
 func (g GapString) ValueAt(pos int) int {
 	v := g.Slice(pos, pos+1)
 	if v.chunks[0].gap > 0 {
@@ -190,10 +219,14 @@ func (g GapString) ValueAt(pos int) int {
 	}
 }
 
+// Return a string version of the GapString, with gaps filled in
 func (g GapString) String(fill string) string {
 	return string(g.Bytes([]byte(fill)...))
 }
 
+// Return a hex representation of this GapString
+//
+// Each octet is space-separated, and gaps are represented with "--"
 func (g GapString) HexString() string {
 	out := new(strings.Builder)
 	glen := g.Length()
@@ -233,6 +266,12 @@ var fluffych = []rune{
 	'α', 'ß', 'Γ', 'π', 'Σ', 'σ', 'µ', 'τ', 'Φ', 'Θ', 'Ω', 'δ', '∞', 'φ', 'ε', '∩',
 	'≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∀', '∃', '√', 'ⁿ', '²', '■', '¤',
 }
+
+// Return a rune representation of this GapString
+//
+// This uses the glyph set from the Fluffy toolkit
+// (https://dirtbags.github.io/fluffy/).
+// Gaps are represented with the rune '<27>'
 func (g GapString) Runes() string {
 	out := new(strings.Builder)
 	glen := g.Length()
@@ -247,6 +286,7 @@ func (g GapString) Runes() string {
 	return out.String()
 }
 
+// Return a hex dump of this GapString
 func (g GapString) Hexdump() string {
 	out := new(strings.Builder)
 	skipping := false
@@ -280,14 +320,21 @@ func (g GapString) Hexdump() string {
 	return out.String()
 }
 
+// Return a uint32, little-endian, taken from the front of this GapString
+//
+// The rest of the GapString is returned as the second argument.
 func (g GapString) Uint32LE() (uint32, GapString) {
-	return binary.LittleEndian.Uint32(g.Slice(0, 4).Bytes()), g.Slice(4, g.Length())
+	return binary.LittleEndian.Uint32(g.Slice(0, 4).Bytes(0)), g.Slice(4, g.Length())
 }
 
+// Return a uint16, little-endian, taken from the front of this GapString
+//
+// The rest of the GapString is returned as the second argument.
 func (g GapString) Uint16LE() (uint16, GapString) {
-	return binary.LittleEndian.Uint16(g.Slice(0, 2).Bytes()), g.Slice(2, g.Length())
+	return binary.LittleEndian.Uint16(g.Slice(0, 2).Bytes(0)), g.Slice(2, g.Length())
 }
 
+// Return this GapString decoded as UTF-16
 func (g GapString) Utf16(order binary.ByteOrder, fill string) string {
 	in := g.Bytes([]byte(fill)...)
 	ints := make([]uint16, len(in)/2)
@@ -298,10 +345,14 @@ func (g GapString) Utf16(order binary.ByteOrder, fill string) string {
 	return string(utf16.Decode(ints))
 }
 
+// Return this GapString decoded as UTF-16 Little Endian
+//
+// This format is used extensively in Microsoft Windows.
 func (g GapString) Utf16LE(gap string) string {
 	return g.Utf16(binary.LittleEndian, gap)
 }
 
+// Return this GapString decoded as UTF-16 Big Endian
 func (g GapString) Utf16BE(gap string) string {
 	return g.Utf16(binary.BigEndian, gap)
 }

netshovel.go

@@ -9,6 +9,11 @@ import (
 	"github.com/google/gopacket/tcpassembly"
 )
 
+// Mainloop to handle dispatching of PCAP files from command line
+//
+// This parses the command line arguments,
+// and for each PCAP file specified on the command line,
+// invokes a TCP assembler that sends streams to whatever is returned from factory.
 func Shovel(factory tcpassembly.StreamFactory) {
 	//verbose := flag.Bool("verbose", false, "Write lots of information out")
 	flag.Parse()
@@ -29,7 +34,6 @@ func Shovel(factory tcpassembly.StreamFactory) {
 				break
 			}
 			if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP {
-				log.Println("Unusable packet")
 				continue
 			}
 			tcp := packet.TransportLayer().(*layers.TCP)

packet.go

@@ -11,7 +11,8 @@ import (
 
 // Error returned by convenience methods that are unable to get enough data
 type ShortError struct {
-	wanted, available int
+	Wanted int // How many bytes you needed
+	Available int // How many bytes were available
 }
 func (e *ShortError) Error() string {
 	return fmt.Sprintf("Short read: wanted %d of %d available", e.wanted, e.available)

stream.go

@@ -12,22 +12,36 @@ import (
 	"github.com/google/gopacket/tcpassembly"
 )
 
+// A File and the path where it lives
 type NamedFile struct {
 	*os.File
 	Name string
 }
 
+// An atomic communication within a Stream
+//
+// Streams consist of a string of Utterances.
+// Each utterance has associated data, and a time stamp.
+//
+// Typically these line up with what crosses the network,
+// but bear in mind that TCP is a streaming protocol,
+// so don't rely on Utterances alone to separate Application-layer packets.
 type Utterance struct {
 	When time.Time
 	Data gapstring.GapString
 }
 
+// A Stream is one half of a two-way conversation
 type Stream struct {
 	Net, Transport gopacket.Flow
 	conversation chan Utterance
 	pending Utterance
 }
 
+// Return a newly-built Stream
+//
+// You should embed Stream into your own Application protocol stream struct.
+// Use this to initialize the internal stuff netshovel needs.
 func NewStream(net, transport gopacket.Flow) Stream {
 	return Stream{
 		Net: net,
@@ -36,6 +50,7 @@ func NewStream(net, transport gopacket.Flow) Stream {
 	}
 }
 
+// Called by the TCP assembler when an Utterance can be built
 func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) {
 	ret := Utterance{
 		When: rs[0].Seen,
@@ -53,10 +68,23 @@ func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) {
 	}
 }
 
+// Called by the TCP assemble when the Stream is closed
 func (stream *Stream) ReassemblyComplete() {
 	close(stream.conversation)
 }
 
+// Read an utterance of a particular size
+//
+// If you pass in a length of -1,
+// this returns utterances as they appear in the conversation.
+//
+// At first, your decoder will probably want to use a length of -1:
+// this will give you a sense of how the conversation works.
+// When you begin to understand the structure of your protocol,
+// change this to a positive integer,
+// so that if you have a large application-layer packet,
+// or multiple application-layer packets in a single transport-layer packet,
+// your decoder handles it properly.
 func (stream *Stream) Read(length int) (Utterance, error) {
 	// This probably indicates a problem, but we assume you know what you're doing
 	if length == 0 {
@@ -109,6 +137,9 @@ func (stream *Stream) Read(length int) (Utterance, error) {
 	return ret, nil
 }
 
+// Return a string description of a packet
+//
+// This just prefixes our source and dest IP:Port to pkt.Describe()
 func (stream *Stream) Describe(pkt Packet) string {
 	out := new(strings.Builder)
 
@@ -120,6 +151,14 @@ func (stream *Stream) Describe(pkt Packet) string {
 	return out.String()
 }
 
+// Return a newly-created, truncated file
+//
+// This function creates consistently-named files,
+// which include a timestamp,
+// and URL-escaped full path to the file.
+//
+// Best practice is to pass in as full a path as you can find,
+// including drive letters and all parent directories.
 func (stream *Stream) CreateFile(when time.Time, path string) (NamedFile, error) {
   name := fmt.Sprintf(
 		"xfer/%s,%sp%s,%sp%s,%s",