neale
/
netshovel
mirror of https://github.com/dirtbags/netshovel.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Further documentation

This commit is contained in:
Neale Pickett 2018-07-24 23:32:08 +00:00
parent a91332924a
commit 28868f0813
4 changed files with 101 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
gapstring/gapstring.go
View File

@ -40,33 +40,47 @@ func (c chunk) slice(a, b int) chunk {
}
}
// A GapString is a string with gaps of no data in the middle
//
// Gaps are represented efficiently,
// both in memory and in computation.
//
// Several convenience functions exist
// which operate on GapString data,
// while preserving the Gaps.
type GapString struct {
chunks []chunk
}
// Return a new zero-length GapString
func New() GapString {
return GapString{
chunks: []chunk{},
}
}
// Return a new GapString containing a gap
func OfGap(gap int) GapString {
return GapString{
chunks: []chunk{{gap: gap}},
}
}
// Return a new GapString containing some bytes
func OfBytes(b []byte) GapString {
return GapString{
chunks: []chunk{{data: b}},
}
}
// Return a new GapString containing a string
func OfString(s string) GapString {
return OfBytes([]byte(s))
}
// Return the length of a GapString
//
// This is the number of bytes you would have if the gaps were filled with some value.
func (g GapString) Length() int {
n := 0
for _, c := range g.chunks {
@ -75,6 +89,7 @@ func (g GapString) Length() int {
return n
}
// Return the total size of all gaps
func (g GapString) Missing() int {
n := 0
for _, c := range g.chunks {
@ -83,6 +98,7 @@ func (g GapString) Missing() int {
return n
}
// Return the current GapString with another GapString appended
func (g GapString) Append(h GapString) GapString {
if h.Length() > 0 {
return GapString{
@ -93,18 +109,25 @@ func (g GapString) Append(h GapString) GapString {
}
}
// Return the current GapString with a gap appended
func (g GapString) AppendGap(gap int) GapString {
return g.Append(OfGap(gap))
}
// Return the current GapString with some bytes appended
func (g GapString) AppendBytes(b []byte) GapString {
return g.Append(OfBytes(b))
}
// Return the current GapString with a string appended
func (g GapString) AppendString(s string) GapString {
return g.Append(OfString(s))
}
// Return a slice of this GapString
//
// This is what you would expect from g[start:end],
// if g were a string or byte slice.
func (g GapString) Slice(start, end int) GapString {
outchunks := make([]chunk, 0, len(g.chunks))
@ -141,6 +164,9 @@ func (g GapString) Slice(start, end int) GapString {
return GapString{chunks: outchunks}
}
// Return this GapString with the provided xor mask applied
//
// The mask is cycled for the length of the GapString.
func (g GapString) Xor(mask ...byte) GapString {
ret := GapString{}
pos := 0
@ -159,6 +185,7 @@ func (g GapString) Xor(mask ...byte) GapString {
return ret
}
// Return this GapString with gaps filled in
func (g GapString) Bytes(fill ...byte) []byte {
ret := make([]byte, g.Length())
pos := 0
@ -180,7 +207,9 @@ func (g GapString) Bytes(fill ...byte) []byte {
return ret
}
// Returns -1 if it's a gap
// Returns the value at a specific position
//
// This returns the byte if one is present, or -1 if it's a gap
func (g GapString) ValueAt(pos int) int {
v := g.Slice(pos, pos+1)
if v.chunks[0].gap > 0 {
@ -190,10 +219,14 @@ func (g GapString) ValueAt(pos int) int {
}
}
// Return a string version of the GapString, with gaps filled in
func (g GapString) String(fill string) string {
return string(g.Bytes([]byte(fill)...))
}
// Return a hex representation of this GapString
//
// Each octet is space-separated, and gaps are represented with "--"
func (g GapString) HexString() string {
out := new(strings.Builder)
glen := g.Length()
@ -233,6 +266,12 @@ var fluffych = []rune{
'α', 'ß', 'Γ', 'π', 'Σ', 'σ', 'µ', 'τ', 'Φ', 'Θ', 'Ω', 'δ', '∞', 'φ', 'ε', '∩',
'≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∀', '∃', '√', 'ⁿ', '²', '■', '¤',
}
// Return a rune representation of this GapString
//
// This uses the glyph set from the Fluffy toolkit
// (https://dirtbags.github.io/fluffy/).
// Gaps are represented with the rune '<27>'
func (g GapString) Runes() string {
out := new(strings.Builder)
glen := g.Length()
@ -247,6 +286,7 @@ func (g GapString) Runes() string {
return out.String()
}
// Return a hex dump of this GapString
func (g GapString) Hexdump() string {
out := new(strings.Builder)
skipping := false
@ -280,14 +320,21 @@ func (g GapString) Hexdump() string {
return out.String()
}
// Return a uint32, little-endian, taken from the front of this GapString
//
// The rest of the GapString is returned as the second argument.
func (g GapString) Uint32LE() (uint32, GapString) {
return binary.LittleEndian.Uint32(g.Slice(0, 4).Bytes()), g.Slice(4, g.Length())
return binary.LittleEndian.Uint32(g.Slice(0, 4).Bytes(0)), g.Slice(4, g.Length())
}
// Return a uint16, little-endian, taken from the front of this GapString
//
// The rest of the GapString is returned as the second argument.
func (g GapString) Uint16LE() (uint16, GapString) {
return binary.LittleEndian.Uint16(g.Slice(0, 2).Bytes()), g.Slice(2, g.Length())
return binary.LittleEndian.Uint16(g.Slice(0, 2).Bytes(0)), g.Slice(2, g.Length())
}
// Return this GapString decoded as UTF-16
func (g GapString) Utf16(order binary.ByteOrder, fill string) string {
in := g.Bytes([]byte(fill)...)
ints := make([]uint16, len(in)/2)
@ -298,10 +345,14 @@ func (g GapString) Utf16(order binary.ByteOrder, fill string) string {
return string(utf16.Decode(ints))
}
// Return this GapString decoded as UTF-16 Little Endian
//
// This format is used extensively in Microsoft Windows.
func (g GapString) Utf16LE(gap string) string {
return g.Utf16(binary.LittleEndian, gap)
}
// Return this GapString decoded as UTF-16 Big Endian
func (g GapString) Utf16BE(gap string) string {
return g.Utf16(binary.BigEndian, gap)
}

6
netshovel.go
View File

@ -9,6 +9,11 @@ import (
"github.com/google/gopacket/tcpassembly"
)
// Mainloop to handle dispatching of PCAP files from command line
//
// This parses the command line arguments,
// and for each PCAP file specified on the command line,
// invokes a TCP assembler that sends streams to whatever is returned from factory.
func Shovel(factory tcpassembly.StreamFactory) {
//verbose := flag.Bool("verbose", false, "Write lots of information out")
flag.Parse()
@ -29,7 +34,6 @@ func Shovel(factory tcpassembly.StreamFactory) {
break
}
if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP {
log.Println("Unusable packet")
continue
}
tcp := packet.TransportLayer().(*layers.TCP)

3
packet.go
View File

@ -11,7 +11,8 @@ import (
// Error returned by convenience methods that are unable to get enough data
type ShortError struct {
wanted, available int
Wanted int // How many bytes you needed
Available int // How many bytes were available
}
func (e *ShortError) Error() string {
return fmt.Sprintf("Short read: wanted %d of %d available", e.wanted, e.available)

39
stream.go
View File

@ -12,22 +12,36 @@ import (
"github.com/google/gopacket/tcpassembly"
)
// A File and the path where it lives
type NamedFile struct {
*os.File
Name string
}
// An atomic communication within a Stream
//
// Streams consist of a string of Utterances.
// Each utterance has associated data, and a time stamp.
//
// Typically these line up with what crosses the network,
// but bear in mind that TCP is a streaming protocol,
// so don't rely on Utterances alone to separate Application-layer packets.
type Utterance struct {
When time.Time
Data gapstring.GapString
}
// A Stream is one half of a two-way conversation
type Stream struct {
Net, Transport gopacket.Flow
conversation chan Utterance
pending Utterance
}
// Return a newly-built Stream
//
// You should embed Stream into your own Application protocol stream struct.
// Use this to initialize the internal stuff netshovel needs.
func NewStream(net, transport gopacket.Flow) Stream {
return Stream{
Net: net,
@ -36,6 +50,7 @@ func NewStream(net, transport gopacket.Flow) Stream {
}
}
// Called by the TCP assembler when an Utterance can be built
func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) {
ret := Utterance{
When: rs[0].Seen,
@ -53,10 +68,23 @@ func (stream *Stream) Reassembled(rs []tcpassembly.Reassembly) {
}
}
// Called by the TCP assemble when the Stream is closed
func (stream *Stream) ReassemblyComplete() {
close(stream.conversation)
}
// Read an utterance of a particular size
//
// If you pass in a length of -1,
// this returns utterances as they appear in the conversation.
//
// At first, your decoder will probably want to use a length of -1:
// this will give you a sense of how the conversation works.
// When you begin to understand the structure of your protocol,
// change this to a positive integer,
// so that if you have a large application-layer packet,
// or multiple application-layer packets in a single transport-layer packet,
// your decoder handles it properly.
func (stream *Stream) Read(length int) (Utterance, error) {
// This probably indicates a problem, but we assume you know what you're doing
if length == 0 {
@ -109,6 +137,9 @@ func (stream *Stream) Read(length int) (Utterance, error) {
return ret, nil
}
// Return a string description of a packet
//
// This just prefixes our source and dest IP:Port to pkt.Describe()
func (stream *Stream) Describe(pkt Packet) string {
out := new(strings.Builder)
@ -120,6 +151,14 @@ func (stream *Stream) Describe(pkt Packet) string {
return out.String()
}
// Return a newly-created, truncated file
//
// This function creates consistently-named files,
// which include a timestamp,
// and URL-escaped full path to the file.
//
// Best practice is to pass in as full a path as you can find,
// including drive letters and all parent directories.
func (stream *Stream) CreateFile(when time.Time, path string) (NamedFile, error) {
name := fmt.Sprintf(
"xfer/%s,%sp%s,%sp%s,%s",