Seems to be working now

This commit is contained in:
Neale Pickett 2018-07-23 21:34:22 +00:00
parent 089627a480
commit 899e425e76
7 changed files with 137 additions and 86 deletions

13
COPYING.txt Normal file
View File

@ -0,0 +1,13 @@
This software has been authored by an employee or employees of Los
Alamos National Security, LLC, operator of the Los Alamos National
Laboratory (LANL) under Contract No. DE-AC52-06NA25396 with the
U.S. Department of Energy. The U.S. Government has rights to use,
reproduce, and distribute this software. The public may copy,
distribute, prepare derivative works and publicly display this
software without charge, provided that this Notice and any
statement of authorship are reproduced on all copies. Neither the
Government nor LANS makes any warranty, express or implied, or
assumes any liability or responsibility for the use of this
software. If software is modified to produce derivative works,
such modified software should be clearly marked, so as not to
confuse it with the version available from LANL.

28
README.md Normal file
View File

@ -0,0 +1,28 @@
Dirtbags Netshovel Library
==========================
This is a library for advanced
[network archaeology](https://sites.google.com/view/cyberfire/foundry/classes/network-archaeology).
It provides a field-tested framework for
exploring unknown TCP-based protocols,
and room to grow these explorations into full-blown decoders.
Get going
=========
Documentation sucks, sorry.
Start with `examples/simple` and build it up into whatever you want.
The Future
==========
This is my first real Go program,
so it is likely to change drastically
as I figure out how to better architect Go libraries.
We strongly encourage you to bring in whatever version of this you're using
under a [vendor folder](https://blog.gopheracademy.com/advent-2015/vendor-folder/)
and check it in.

View File

@ -1,26 +1,77 @@
package main
import (
"fmt"
"io"
"log"
"strings"
"sync"
"github.com/google/gopacket"
"github.com/google/gopacket/tcpassembly"
"github.com/dirtbags/netshovel"
)
struct SimpleStreamFactory {
netshovel.Factory
var wg sync.WaitGroup
type SimpleStreamFactory struct {
}
struct SimpleStream {
type SimpleStream struct {
netshovel.Stream
}
func (f *SimpleStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
ret = SimpleStream{}
ret.Stream = f.Factory.New(net, transport)
}
struct SimplePacket {
type SimplePacket struct {
netshovel.Packet
}
func main() {
netshovel.Shovel(SimpleFactory)
func NewSimplePacket() SimplePacket {
return SimplePacket{
Packet: netshovel.NewPacket(),
}
}
func (f *SimpleStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
stream := &SimpleStream{
Stream: netshovel.NewStream(net, transport),
}
wg.Add(1)
go stream.Decode(wg)
return stream
}
func (stream SimpleStream) Display(pkt SimplePacket) {
out := new(strings.Builder)
fmt.Fprintf(out, "Simple %v:%v → %v:%v\n",
stream.Net.Src().String(), stream.Transport.Src().String(),
stream.Net.Dst().String(), stream.Transport.Dst().String(),
)
out.WriteString(pkt.Describe())
fmt.Println(out.String())
}
func (stream SimpleStream) Decode(wg sync.WaitGroup) {
defer wg.Done()
for {
pkt := NewSimplePacket()
utterance, err := stream.Read(-1)
if err != nil {
if err != io.EOF {
log.Println(err)
}
break
}
pkt.Payload = utterance.Data
pkt.When = utterance.When
stream.Display(pkt)
}
}
func main() {
netshovel.Shovel(&SimpleStreamFactory{})
wg.Wait()
}

View File

@ -3,6 +3,7 @@ package gapstring
import (
"fmt"
"encoding/binary"
"encoding/hex"
"strings"
"unicode/utf16"
)
@ -193,6 +194,10 @@ func (g GapString) String(fill string) string {
return string(g.Bytes([]byte(fill)...))
}
func (g GapString) HexString(fill ...byte) string {
return hex.EncodeToString(g.Bytes(fill...))
}
var fluffych = []rune{
'·', '☺', '☻', '♥', '♦', '♣', '♠', '•', '◘', '○', '◙', '♂', '♀', '♪', '♫', '☼',
'►', '◄', '↕', '‼', '¶', '§', '▬', '↨', '↑', '↓', '→', '←', '∟', '↔', '▲', '▼',
@ -212,7 +217,7 @@ var fluffych = []rune{
'≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∀', '∃', '√', 'ⁿ', '²', '■', '¤',
}
func (g GapString) Hexdump() string {
var out strings.Builder
out := new(strings.Builder)
skipping := false
glen := g.Length()
pos := 0
@ -228,7 +233,7 @@ func (g GapString) Hexdump() string {
}
if repeated {
if ! skipping {
fmt.Fprintln(&out, "*")
fmt.Fprintln(out, "*")
skipping = true
}
pos += 16
@ -239,44 +244,44 @@ func (g GapString) Hexdump() string {
}
// Output offset
fmt.Fprintf(&out, "%08x ", pos)
fmt.Fprintf(out, "%08x ", pos)
// Output octet values
for i := 0; i < 16; i += 1 {
if pos+i < glen {
c := g.ValueAt(pos+i)
if c == -1 {
fmt.Fprintf(&out, "-- ")
fmt.Fprintf(out, "-- ")
} else {
fmt.Fprintf(&out, "%02x ", c)
fmt.Fprintf(out, "%02x ", c)
}
} else {
fmt.Fprintf(&out, " ")
fmt.Fprintf(out, " ")
}
if i == 7 {
fmt.Fprintf(&out, " ")
fmt.Fprintf(out, " ")
}
}
fmt.Fprintf(&out, " ")
fmt.Fprintf(out, " ")
// Output octet glyphs
for i := 0; (i < 16) && (pos < glen); {
c := g.ValueAt(pos)
if c == -1 {
fmt.Fprintf(&out, "<22>")
fmt.Fprintf(out, "<22>")
} else {
fmt.Fprintf(&out, "%c", fluffych[c])
fmt.Fprintf(out, "%c", fluffych[c])
}
i += 1
pos += 1
}
// Output newline
fmt.Fprintln(&out, "")
fmt.Fprintln(out, "")
}
fmt.Fprintf(&out, "%08x\n", pos)
fmt.Fprintf(out, "%08x\n", pos)
return out.String()
}

View File

@ -24,7 +24,6 @@ func Shovel(factory tcpassembly.StreamFactory) {
packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
packets := packetSource.Packets()
npackets := 0
for packet := range packets {
if packet == nil {
break
@ -35,10 +34,7 @@ func Shovel(factory tcpassembly.StreamFactory) {
}
tcp := packet.TransportLayer().(*layers.TCP)
assembler.AssembleWithTimestamp(packet.NetworkLayer().NetworkFlow(), tcp, packet.Metadata().Timestamp)
npackets += 1
}
log.Println("npackets", npackets)
}
assembler.FlushAll()
StreamWG.Wait()
}

View File

@ -8,7 +8,10 @@ import (
"github.com/dirtbags/netshovel/gapstring"
)
type PacketFactory func()Packet
type Packet struct {
Name string
Opcode int
Description string
When time.Time
@ -20,6 +23,7 @@ var never = time.Unix(0, 0)
func NewPacket() Packet {
return Packet{
Name: "Generic",
Opcode: -1,
Description: "Undefined",
When: never,
@ -28,20 +32,23 @@ func NewPacket() Packet {
}
}
func (pkt *Packet) Name() string {
return "Generic"
}
func (pkt *Packet) Describe() string {
out := new(strings.Builder)
fmt.Fprintf(out, " %s %s %d: %s\n",
pkt.When.UTC().Format(time.RFC3339Nano),
pkt.Name(),
pkt.Name,
pkt.Opcode,
pkt.Description,
)
for _, k := range pkt.Keys() {
keys := make([]string, len(pkt.Fields))
i := 0
for k := range(pkt.Fields) {
keys[i] = k
i += 1
}
sort.Strings(keys)
for _, k := range keys {
fmt.Fprintf(out, " %s: %s\n", k, pkt.Fields[k])
}
fmt.Fprint(out, pkt.Payload.Hexdump())
@ -51,14 +58,3 @@ func (pkt *Packet) Describe() string {
func (pkt *Packet) Set(key, value string) {
pkt.Fields[key] = value
}
func (pkt *Packet) Keys() []string{
keys := make([]string, len(pkt.Fields))
i := 0
for k := range(pkt.Fields) {
keys[i] = k
i += 1
}
sort.Strings(keys)
return keys
}

View File

@ -5,15 +5,12 @@ import (
"io"
"log"
"strings"
"sync"
"time"
"github.com/dirtbags/netshovel/gapstring"
"github.com/google/gopacket"
"github.com/google/gopacket/tcpassembly"
)
var StreamWG sync.WaitGroup
type WriteAtCloser interface {
io.WriterAt
io.WriteCloser
@ -24,25 +21,18 @@ type Utterance struct {
Data gapstring.GapString
}
type StreamFactory struct {}
type Stream struct {
Net, Transport gopacket.Flow
conversation chan Utterance
done chan bool
pending Utterance
}
func (f *StreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
stream := &Stream{
func NewStream(net, transport gopacket.Flow) Stream {
return Stream{
Net: net,
Transport: transport,
conversation: make(chan Utterance, 100),
}
StreamWG.Add(1)
go stream.Run(StreamWG)
return stream
}
func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) {
@ -121,31 +111,3 @@ func (stream *Stream) Describe(pkt Packet) string {
out.WriteString(pkt.Describe())
return out.String()
}
func (stream *Stream) Run(wg sync.WaitGroup) {
defer wg.Done()
for {
pkt, err := stream.BuildPacket()
if err == io.EOF {
return
} else if err != nil {
log.Println(err) // XXX: Print out 4-tuple identifying this stream
return
}
fmt.Println(stream.Describe(pkt))
}
}
func (stream *Stream) BuildPacket() (Packet, error) {
pkt := NewPacket()
utterance, err := stream.Read(-1)
if err != nil {
return pkt, err
}
pkt.Payload = utterance.Data
pkt.When = utterance.When
return pkt, nil
}