Seems to be working now

COPYING.txt

@@ -0,0 +1,13 @@
+     This software has been authored by an employee or employees of Los
+     Alamos National Security, LLC, operator of the Los Alamos National
+     Laboratory (LANL) under Contract No. DE-AC52-06NA25396 with the
+     U.S.  Department of Energy.  The U.S. Government has rights to use,
+     reproduce, and distribute this software.  The public may copy,
+     distribute, prepare derivative works and publicly display this
+     software without charge, provided that this Notice and any
+     statement of authorship are reproduced on all copies.  Neither the
+     Government nor LANS makes any warranty, express or implied, or
+     assumes any liability or responsibility for the use of this
+     software.  If software is modified to produce derivative works,
+     such modified software should be clearly marked, so as not to
+     confuse it with the version available from LANL.

README.md

@@ -0,0 +1,28 @@
+Dirtbags Netshovel Library
+==========================
+
+This is a library for advanced
+[network archaeology](https://sites.google.com/view/cyberfire/foundry/classes/network-archaeology).
+
+It provides a field-tested framework for
+exploring unknown TCP-based protocols,
+and room to grow these explorations into full-blown decoders.
+
+
+Get going
+=========
+
+Documentation sucks, sorry.
+Start with `examples/simple` and build it up into whatever you want.
+
+
+The Future
+==========
+
+This is my first real Go program,
+so it is likely to change drastically
+as I figure out how to better architect Go libraries.
+
+We strongly encourage you to bring in whatever version of this you're using
+under a [vendor folder](https://blog.gopheracademy.com/advent-2015/vendor-folder/)
+and check it in.

examples/simple/simple.go

@@ -1,26 +1,77 @@
 package main
 
 import (
+	"fmt"
+	"io"
+	"log"
+	"strings"
+	"sync"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/tcpassembly"
 	"github.com/dirtbags/netshovel"
 )
 
-struct SimpleStreamFactory {
+var wg sync.WaitGroup
-	netshovel.Factory
+
+type SimpleStreamFactory struct {
 }
 
-struct SimpleStream {
+type SimpleStream struct {
 	netshovel.Stream
 }
 
-func (f *SimpleStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
+type SimplePacket struct {
-	ret = SimpleStream{}
-	ret.Stream = f.Factory.New(net, transport)
-}
-
-struct SimplePacket {
 	netshovel.Packet
 }
 
-func main() {
+func NewSimplePacket() SimplePacket {
-	netshovel.Shovel(SimpleFactory)
+	return SimplePacket{
+		Packet: netshovel.NewPacket(),
+	}
+}
+
+func (f *SimpleStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
+	stream := &SimpleStream{
+		Stream: netshovel.NewStream(net, transport),
+	}
+	wg.Add(1)
+	go stream.Decode(wg)
+
+	return stream
+}
+
+func (stream SimpleStream) Display(pkt SimplePacket) {
+	out := new(strings.Builder)
+
+	fmt.Fprintf(out, "Simple %v:%v → %v:%v\n",
+		stream.Net.Src().String(), stream.Transport.Src().String(),
+    stream.Net.Dst().String(), stream.Transport.Dst().String(),
+  )
+  out.WriteString(pkt.Describe())
+  fmt.Println(out.String())
+}
+
+func (stream SimpleStream) Decode(wg sync.WaitGroup) {
+	defer wg.Done()
+	for {
+		pkt := NewSimplePacket()
+		
+
+		utterance, err := stream.Read(-1)
+		if err != nil {
+			if err != io.EOF {
+				log.Println(err)
+			}
+			break
+		}
+		
+		pkt.Payload = utterance.Data
+		pkt.When = utterance.When
+		stream.Display(pkt)
+	}
+}
+
+func main() {
+	netshovel.Shovel(&SimpleStreamFactory{})
+	wg.Wait()
 }

gapstring/gapstring.go

@@ -3,6 +3,7 @@ package gapstring
 import (
 	"fmt"
 	"encoding/binary"
+	"encoding/hex"
 	"strings"
 	"unicode/utf16"
 )
@@ -193,6 +194,10 @@ func (g GapString) String(fill string) string {
 	return string(g.Bytes([]byte(fill)...))
 }
 
+func (g GapString) HexString(fill ...byte) string {
+	return hex.EncodeToString(g.Bytes(fill...))
+}
+
 var fluffych = []rune{
   '·', '☺', '☻', '♥', '♦', '♣', '♠', '•', '◘', '○', '◙', '♂', '♀', '♪', '♫', '☼',
 	'►', '◄', '↕', '‼', '¶', '§', '▬', '↨', '↑', '↓', '→', '←', '∟', '↔', '▲', '▼',
@@ -212,7 +217,7 @@ var fluffych = []rune{
 	'≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∀', '∃', '√', 'ⁿ', '²', '■', '¤',
 }
 func (g GapString) Hexdump() string {
-	var out strings.Builder
+	out := new(strings.Builder)
 	skipping := false
 	glen := g.Length()
 	pos := 0
@@ -228,7 +233,7 @@ func (g GapString) Hexdump() string {
 			}
 			if repeated {
 				if ! skipping {
-					fmt.Fprintln(&out, "*")
+					fmt.Fprintln(out, "*")
 					skipping = true
 				}
 				pos += 16
@@ -239,44 +244,44 @@ func (g GapString) Hexdump() string {
 		}
 
 		// Output offset
-		fmt.Fprintf(&out, "%08x  ", pos)
+		fmt.Fprintf(out, "%08x  ", pos)
 		
 		// Output octet values
 		for i := 0; i < 16; i += 1 {
 			if pos+i < glen {
 				c := g.ValueAt(pos+i)
 				if c == -1 {
-					fmt.Fprintf(&out, "-- ")
+					fmt.Fprintf(out, "-- ")
 				} else {
-					fmt.Fprintf(&out, "%02x ", c)
+					fmt.Fprintf(out, "%02x ", c)
 				}
 			} else {
-				fmt.Fprintf(&out, "   ")
+				fmt.Fprintf(out, "   ")
 			}
 			if i == 7 {
-				fmt.Fprintf(&out, " ")
+				fmt.Fprintf(out, " ")
 			}
 		}
 		
-		fmt.Fprintf(&out, " ")
+		fmt.Fprintf(out, " ")
 		
 		
 		// Output octet glyphs
 		for i := 0; (i < 16) && (pos < glen); {
 			c := g.ValueAt(pos)
 			if c == -1 {
-				fmt.Fprintf(&out, "<22>")
+				fmt.Fprintf(out, "<22>")
 			} else {
-				fmt.Fprintf(&out, "%c", fluffych[c])
+				fmt.Fprintf(out, "%c", fluffych[c])
 			}
 			i += 1
 			pos += 1
 		}
 		
 		// Output newline
-		fmt.Fprintln(&out, "")
+		fmt.Fprintln(out, "")
 	}
-	fmt.Fprintf(&out, "%08x\n", pos)
+	fmt.Fprintf(out, "%08x\n", pos)
 	
 	return out.String()
 }

netshovel.go

@@ -24,7 +24,6 @@ func Shovel(factory tcpassembly.StreamFactory) {
 		
 		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
 		packets := packetSource.Packets()
-		npackets := 0
 		for packet := range packets {
 			if packet == nil {
 				break
@@ -35,10 +34,7 @@ func Shovel(factory tcpassembly.StreamFactory) {
 			}
 			tcp := packet.TransportLayer().(*layers.TCP)
 			assembler.AssembleWithTimestamp(packet.NetworkLayer().NetworkFlow(), tcp, packet.Metadata().Timestamp)
-			npackets += 1
 		}
-		log.Println("npackets", npackets)
 	}
 	assembler.FlushAll()
-	StreamWG.Wait()
 }

packet.go

@@ -8,7 +8,10 @@ import (
 	"github.com/dirtbags/netshovel/gapstring"
 )
 
+type PacketFactory func()Packet
+
 type Packet struct {
+	Name string
 	Opcode int
 	Description string
 	When time.Time
@@ -20,6 +23,7 @@ var never = time.Unix(0, 0)
 
 func NewPacket() Packet {
 	return Packet{
+		Name: "Generic",
 		Opcode: -1,
 		Description: "Undefined",
 		When: never,
@@ -28,21 +32,24 @@ func NewPacket() Packet {
 	}
 }
 
-func (pkt *Packet) Name() string {
-	return "Generic"
-}
-
 func (pkt *Packet) Describe() string {
 	out := new(strings.Builder)
 
-	fmt.Fprintf(out, "%s %s %d: %s\n",
+	fmt.Fprintf(out, "  %s %s %d: %s\n",
 	  pkt.When.UTC().Format(time.RFC3339Nano),
-		pkt.Name(),
+		pkt.Name,
 		pkt.Opcode,
 		pkt.Description,
 	)
-	for _, k := range pkt.Keys() {
+	keys := make([]string, len(pkt.Fields))
-		fmt.Fprintf(out, "      %s: %s\n", k, pkt.Fields[k])
+	i := 0
+	for k := range(pkt.Fields) {
+		keys[i] = k
+		i += 1
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		fmt.Fprintf(out, "    %s: %s\n", k, pkt.Fields[k])
 	}
 	fmt.Fprint(out, pkt.Payload.Hexdump())
 	return out.String()
@@ -51,14 +58,3 @@ func (pkt *Packet) Describe() string {
 func (pkt *Packet) Set(key, value string) {
 	pkt.Fields[key] = value
 }
-
-func (pkt *Packet) Keys() []string{
-	keys := make([]string, len(pkt.Fields))
-	i := 0
-	for k := range(pkt.Fields) {
-		keys[i] = k
-		i += 1
-	}
-	sort.Strings(keys)
-	return keys
-}

stream.go

@@ -5,15 +5,12 @@ import (
 	"io"
 	"log"
 	"strings"
-	"sync"
 	"time"
 	"github.com/dirtbags/netshovel/gapstring"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/tcpassembly"
 )
 
-var StreamWG sync.WaitGroup
-
 type WriteAtCloser interface {
 	io.WriterAt
 	io.WriteCloser
@@ -24,25 +21,18 @@ type Utterance struct {
 	Data gapstring.GapString
 }
 
-type StreamFactory struct {}
-
 type Stream struct {
 	Net, Transport gopacket.Flow
 	conversation chan Utterance
-	done chan bool
 	pending Utterance
 }
 
-func (f *StreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
+func NewStream(net, transport gopacket.Flow) Stream {
-	stream := &Stream{
+	return Stream{
 		Net: net,
 		Transport: transport,
 		conversation: make(chan Utterance, 100),
 	}
-	StreamWG.Add(1)
-	go stream.Run(StreamWG)
-	
-	return stream
 }
 
 func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) {
@@ -121,31 +111,3 @@ func (stream *Stream) Describe(pkt Packet) string {
   out.WriteString(pkt.Describe())
 	return out.String()
 }
-
-func (stream *Stream) Run(wg sync.WaitGroup) {
-	defer wg.Done()
-	for {
-		pkt, err := stream.BuildPacket()
-		if err == io.EOF {
-			return
-		} else if err != nil {
-			log.Println(err) // XXX: Print out 4-tuple identifying this stream
-			return
-		}
-		
-		fmt.Println(stream.Describe(pkt))
-	}
-}
-
-func (stream *Stream) BuildPacket() (Packet, error) {
-	pkt := NewPacket()
-	
-	utterance, err := stream.Read(-1)
-	if err != nil {
-		return pkt, err
-	}
-	
-	pkt.Payload = utterance.Data
-	pkt.When = utterance.When
-	return pkt, nil
-}