diff --git a/COPYING.txt b/COPYING.txt new file mode 100644 index 0000000..d650496 --- /dev/null +++ b/COPYING.txt @@ -0,0 +1,13 @@ + This software has been authored by an employee or employees of Los + Alamos National Security, LLC, operator of the Los Alamos National + Laboratory (LANL) under Contract No. DE-AC52-06NA25396 with the + U.S. Department of Energy. The U.S. Government has rights to use, + reproduce, and distribute this software. The public may copy, + distribute, prepare derivative works and publicly display this + software without charge, provided that this Notice and any + statement of authorship are reproduced on all copies. Neither the + Government nor LANS makes any warranty, express or implied, or + assumes any liability or responsibility for the use of this + software. If software is modified to produce derivative works, + such modified software should be clearly marked, so as not to + confuse it with the version available from LANL. diff --git a/README.md b/README.md new file mode 100644 index 0000000..68c2b03 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +Dirtbags Netshovel Library +========================== + +This is a library for advanced +[network archaeology](https://sites.google.com/view/cyberfire/foundry/classes/network-archaeology). + +It provides a field-tested framework for +exploring unknown TCP-based protocols, +and room to grow these explorations into full-blown decoders. + + +Get going +========= + +Documentation sucks, sorry. +Start with `examples/simple` and build it up into whatever you want. + + +The Future +========== + +This is my first real Go program, +so it is likely to change drastically +as I figure out how to better architect Go libraries. + +We strongly encourage you to bring in whatever version of this you're using +under a [vendor folder](https://blog.gopheracademy.com/advent-2015/vendor-folder/) +and check it in. diff --git a/examples/simple/simple.go b/examples/simple/simple.go index b08ffb2..65782a8 100644 --- a/examples/simple/simple.go +++ b/examples/simple/simple.go @@ -1,26 +1,77 @@ package main import ( + "fmt" + "io" + "log" + "strings" + "sync" + "github.com/google/gopacket" + "github.com/google/gopacket/tcpassembly" "github.com/dirtbags/netshovel" ) -struct SimpleStreamFactory { - netshovel.Factory +var wg sync.WaitGroup + +type SimpleStreamFactory struct { } -struct SimpleStream { +type SimpleStream struct { netshovel.Stream } -func (f *SimpleStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream { - ret = SimpleStream{} - ret.Stream = f.Factory.New(net, transport) -} - -struct SimplePacket { +type SimplePacket struct { netshovel.Packet } -func main() { - netshovel.Shovel(SimpleFactory) +func NewSimplePacket() SimplePacket { + return SimplePacket{ + Packet: netshovel.NewPacket(), + } +} + +func (f *SimpleStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream { + stream := &SimpleStream{ + Stream: netshovel.NewStream(net, transport), + } + wg.Add(1) + go stream.Decode(wg) + + return stream +} + +func (stream SimpleStream) Display(pkt SimplePacket) { + out := new(strings.Builder) + + fmt.Fprintf(out, "Simple %v:%v → %v:%v\n", + stream.Net.Src().String(), stream.Transport.Src().String(), + stream.Net.Dst().String(), stream.Transport.Dst().String(), + ) + out.WriteString(pkt.Describe()) + fmt.Println(out.String()) +} + +func (stream SimpleStream) Decode(wg sync.WaitGroup) { + defer wg.Done() + for { + pkt := NewSimplePacket() + + + utterance, err := stream.Read(-1) + if err != nil { + if err != io.EOF { + log.Println(err) + } + break + } + + pkt.Payload = utterance.Data + pkt.When = utterance.When + stream.Display(pkt) + } +} + +func main() { + netshovel.Shovel(&SimpleStreamFactory{}) + wg.Wait() } diff --git a/gapstring/gapstring.go b/gapstring/gapstring.go index 241614a..7f24df4 100644 --- a/gapstring/gapstring.go +++ b/gapstring/gapstring.go @@ -3,6 +3,7 @@ package gapstring import ( "fmt" "encoding/binary" + "encoding/hex" "strings" "unicode/utf16" ) @@ -193,6 +194,10 @@ func (g GapString) String(fill string) string { return string(g.Bytes([]byte(fill)...)) } +func (g GapString) HexString(fill ...byte) string { + return hex.EncodeToString(g.Bytes(fill...)) +} + var fluffych = []rune{ '·', '☺', '☻', '♥', '♦', '♣', '♠', '•', '◘', '○', '◙', '♂', '♀', '♪', '♫', '☼', '►', '◄', '↕', '‼', '¶', '§', '▬', '↨', '↑', '↓', '→', '←', '∟', '↔', '▲', '▼', @@ -212,7 +217,7 @@ var fluffych = []rune{ '≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∀', '∃', '√', 'ⁿ', '²', '■', '¤', } func (g GapString) Hexdump() string { - var out strings.Builder + out := new(strings.Builder) skipping := false glen := g.Length() pos := 0 @@ -228,7 +233,7 @@ func (g GapString) Hexdump() string { } if repeated { if ! skipping { - fmt.Fprintln(&out, "*") + fmt.Fprintln(out, "*") skipping = true } pos += 16 @@ -239,44 +244,44 @@ func (g GapString) Hexdump() string { } // Output offset - fmt.Fprintf(&out, "%08x ", pos) + fmt.Fprintf(out, "%08x ", pos) // Output octet values for i := 0; i < 16; i += 1 { if pos+i < glen { c := g.ValueAt(pos+i) if c == -1 { - fmt.Fprintf(&out, "-- ") + fmt.Fprintf(out, "-- ") } else { - fmt.Fprintf(&out, "%02x ", c) + fmt.Fprintf(out, "%02x ", c) } } else { - fmt.Fprintf(&out, " ") + fmt.Fprintf(out, " ") } if i == 7 { - fmt.Fprintf(&out, " ") + fmt.Fprintf(out, " ") } } - fmt.Fprintf(&out, " ") + fmt.Fprintf(out, " ") // Output octet glyphs for i := 0; (i < 16) && (pos < glen); { c := g.ValueAt(pos) if c == -1 { - fmt.Fprintf(&out, "�") + fmt.Fprintf(out, "�") } else { - fmt.Fprintf(&out, "%c", fluffych[c]) + fmt.Fprintf(out, "%c", fluffych[c]) } i += 1 pos += 1 } // Output newline - fmt.Fprintln(&out, "") + fmt.Fprintln(out, "") } - fmt.Fprintf(&out, "%08x\n", pos) + fmt.Fprintf(out, "%08x\n", pos) return out.String() } diff --git a/netshovel.go b/netshovel.go index 55ab261..c99a8a4 100644 --- a/netshovel.go +++ b/netshovel.go @@ -24,7 +24,6 @@ func Shovel(factory tcpassembly.StreamFactory) { packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) packets := packetSource.Packets() - npackets := 0 for packet := range packets { if packet == nil { break @@ -35,10 +34,7 @@ func Shovel(factory tcpassembly.StreamFactory) { } tcp := packet.TransportLayer().(*layers.TCP) assembler.AssembleWithTimestamp(packet.NetworkLayer().NetworkFlow(), tcp, packet.Metadata().Timestamp) - npackets += 1 } - log.Println("npackets", npackets) } assembler.FlushAll() - StreamWG.Wait() } diff --git a/packet.go b/packet.go index 2e0e5a0..e723cd6 100644 --- a/packet.go +++ b/packet.go @@ -8,7 +8,10 @@ import ( "github.com/dirtbags/netshovel/gapstring" ) +type PacketFactory func()Packet + type Packet struct { + Name string Opcode int Description string When time.Time @@ -20,6 +23,7 @@ var never = time.Unix(0, 0) func NewPacket() Packet { return Packet{ + Name: "Generic", Opcode: -1, Description: "Undefined", When: never, @@ -28,21 +32,24 @@ func NewPacket() Packet { } } -func (pkt *Packet) Name() string { - return "Generic" -} - func (pkt *Packet) Describe() string { out := new(strings.Builder) - fmt.Fprintf(out, "%s %s %d: %s\n", + fmt.Fprintf(out, " %s %s %d: %s\n", pkt.When.UTC().Format(time.RFC3339Nano), - pkt.Name(), + pkt.Name, pkt.Opcode, pkt.Description, ) - for _, k := range pkt.Keys() { - fmt.Fprintf(out, " %s: %s\n", k, pkt.Fields[k]) + keys := make([]string, len(pkt.Fields)) + i := 0 + for k := range(pkt.Fields) { + keys[i] = k + i += 1 + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(out, " %s: %s\n", k, pkt.Fields[k]) } fmt.Fprint(out, pkt.Payload.Hexdump()) return out.String() @@ -51,14 +58,3 @@ func (pkt *Packet) Describe() string { func (pkt *Packet) Set(key, value string) { pkt.Fields[key] = value } - -func (pkt *Packet) Keys() []string{ - keys := make([]string, len(pkt.Fields)) - i := 0 - for k := range(pkt.Fields) { - keys[i] = k - i += 1 - } - sort.Strings(keys) - return keys -} diff --git a/stream.go b/stream.go index 44d9306..72a3d36 100644 --- a/stream.go +++ b/stream.go @@ -5,15 +5,12 @@ import ( "io" "log" "strings" - "sync" "time" "github.com/dirtbags/netshovel/gapstring" "github.com/google/gopacket" "github.com/google/gopacket/tcpassembly" ) -var StreamWG sync.WaitGroup - type WriteAtCloser interface { io.WriterAt io.WriteCloser @@ -24,25 +21,18 @@ type Utterance struct { Data gapstring.GapString } -type StreamFactory struct {} - type Stream struct { Net, Transport gopacket.Flow conversation chan Utterance - done chan bool pending Utterance } -func (f *StreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream { - stream := &Stream{ +func NewStream(net, transport gopacket.Flow) Stream { + return Stream{ Net: net, Transport: transport, conversation: make(chan Utterance, 100), } - StreamWG.Add(1) - go stream.Run(StreamWG) - - return stream } func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) { @@ -121,31 +111,3 @@ func (stream *Stream) Describe(pkt Packet) string { out.WriteString(pkt.Describe()) return out.String() } - -func (stream *Stream) Run(wg sync.WaitGroup) { - defer wg.Done() - for { - pkt, err := stream.BuildPacket() - if err == io.EOF { - return - } else if err != nil { - log.Println(err) // XXX: Print out 4-tuple identifying this stream - return - } - - fmt.Println(stream.Describe(pkt)) - } -} - -func (stream *Stream) BuildPacket() (Packet, error) { - pkt := NewPacket() - - utterance, err := stream.Read(-1) - if err != nil { - return pkt, err - } - - pkt.Payload = utterance.Data - pkt.When = utterance.When - return pkt, nil -}