general: project structure improvements (mushorg#23)

* general: project structure improvements * Only run tests inside docker. (closes mushorg#14) * Add contents to the README file. (closes mushorg#17) * Add a LPI mapping for SMTP. Signed-off-by: Nikos Filippakis <aesmade@gmail.com>
N0mansky · Jun 27, 2017 · 24ea580 · 24ea580
1 parent 71bfbc4
commit 24ea580
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 40 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@
 
 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
+vendor/
 
 # Idea IDE files
 .idea/
diff --git a/.travis.yml b/.travis.yml
@@ -1,21 +1,12 @@
-language: go
 dist: trusty
 sudo: required
 services:
     - docker
-go_import_path: github.com/mushorg/go-dpi
-before_install:
-    - echo "deb http://packages.wand.net.nz trusty main" | sudo tee -a /etc/apt/sources.list
-    - sudo apt-get update
-    - git clone --branch 2.0-stable https://github.com/ntop/nDPI/ ~/nDPI
-install:
-    - sudo apt-get -y --force-yes install libpcap-dev libtrace4 libtrace4-dev libprotoident libprotoident-dev
-    - go get golang.org/x/tools/cmd/cover
-    - go get github.com/mattn/goveralls
-    - go get github.com/Masterminds/glide
-    - glide install
-    - cd ~/nDPI && ./autogen.sh && ./configure && make && sudo make install && cd -
 script:
-    - sudo docker build -t godpi-example . && sudo docker run godpi-example
-    - go test . ./classifiers ./wrappers
-    - $HOME/gopath/bin/goveralls -ignore godpi_example/example_app.go -service=travis-ci
+    - sudo docker build -t godpi-example .
+    - >
+        sudo docker run --entrypoint=/bin/bash
+        -e TRAVIS
+        -e TRAVIS_JOB_ID
+        godpi-example -c 'go get golang.org/x/tools/cmd/cover github.com/mattn/goveralls &&
+        $GOPATH/bin/goveralls -ignore godpi_example/example_app.go -service=travis-ci'
diff --git a/README.md b/README.md
@@ -3,3 +3,76 @@
 [![](https://godoc.org/github.com/mushorg/go-dpi?status.svg)](https://godoc.org/github.com/mushorg/go-dpi)
 
 # go-dpi
+
+go-dpi is an open source Go library for application layer protocol identification of traffic flows. In addition to its own heuristic methods, it contains wrappers for other popular and well-established libraries that also perform protocol identification, such as nDPI and libprotoident. It aims to provide a simple, easy-to-use interface and the capability to be easily extended by a developer with new detection methods and protocols.
+
+It attempts to classify flows to different protocols regardless of the ports used. This makes it possible to detect protocols on non-standard ports, which is ideal for honeypots, as malware might often try and throw off detection methods by using non-standard and unregistered ports. Also, with its layered architecture, it aims to be fast in its detection, only using heavier classification methods when the simpler ones fail.
+
+It is being developed in the context of the Google Summer of Code 2017 program, under the mentorship of The Honeynet Project.
+
+Please read the project's [Wiki page](https://github.com/mushorg/go-dpi/wiki) for more information.
+
+## Example usage
+
+The library and the modules APIs aim to be very simple and straightforward to use. The library relies on the [gopacket](https://godoc.org/github.com/google/gopacket) library and its Packet structure. Once you have a Packet in your hands, it's very easy to classify it with the library.
+First you need a flow that contains the packet. There is a helper function for constructing a flow from a single packet. Simply call:
+
+```go
+flow := godpi.CreateFlowFromPacket(&packet)
+```
+
+Afterwards, classifying the flow can be done by simply calling:
+
+```go
+proto, source := classifiers.ClassifyFlow(flow)
+```
+
+This returns the guess protocol by the classifiers as well as the source (which in this case will always be go-dpi).
+
+The same thing applies for wrappers. However, for wrappers you also have to call the initialize function, and the destroy function before your program exits. All in all, the following is enough to run the wrappers:
+
+```go
+wrappers.InitializeWrappers()
+defer wrappers.DestroyWrappers()
+proto, source = wrappers.ClassifyFlow(flow)
+```
+
+A minimal example application is included below. It uses both the classifiers and wrappers to classify a simple packet capture file. Note the helpful `godpi.ReadDumpFile` function that simply returns a channel with all the packets in the file.
+
+```go
+package main
+
+import "fmt"
+import "github.com/mushorg/go-dpi"
+import "github.com/mushorg/go-dpi/classifiers"
+import "github.com/mushorg/go-dpi/wrappers"
+
+func main() {
+	packets, err := godpi.ReadDumpFile("/tmp/http.cap")
+	wrappers.InitializeWrappers()
+	defer wrappers.DestroyWrappers()
+	if err != nil {
+		fmt.Println(err)
+	} else {
+		for packet := range packets {
+			flow := godpi.CreateFlowFromPacket(&packet)
+			proto, source := classifiers.ClassifyFlow(flow)
+			if proto != godpi.Unknown {
+				fmt.Println(source, "detected protocol", proto)
+			} else {
+				fmt.Println("No detection made by classifiers")
+			}
+			proto, source = wrappers.ClassifyFlow(flow)
+			if proto != godpi.Unknown {
+				fmt.Println(source, "detected protocol", proto)
+			} else {
+				fmt.Println("No detection made by wrappers")
+			}
+		}
+	}
+}
+```
+
+## License
+
+go-dpi is available under the MIT license and distributed in source code format.
diff --git a/wrappers/LPI_wrapper.go b/wrappers/LPI_wrapper.go
@@ -12,18 +12,19 @@ import (
 
 // lpiCodeToProtocol maps the LPI protocol codes to go-dpi protocols.
 var lpiCodeToProtocol = map[uint32]godpi.Protocol{
-	0:   godpi.Http,
-	14:  godpi.Dns,
-	201: godpi.Dns,
-	8:   godpi.Ssh,
-	23:  godpi.Rpc,
-	1:   godpi.Smtp,
-	21:  godpi.Rdp,
-	24:  godpi.Smb,
-	380: godpi.Icmp,
-	27:  godpi.Ftp,
-	12:  godpi.Ssl,
-	37:  godpi.Netbios,
+	0:   godpi.Http,     // LPI_PROTO_HTTP
+	14:  godpi.Dns,      // LPI_PROTO_DNS
+	201: godpi.Dns,      // LPI_PROTO_UDP_DNS
+	8:   godpi.Ssh,      // LPI_PROTO_SSH
+	23:  godpi.Rpc,      // LPI_PROTO_RPC_SCAN
+	1:   godpi.Smtp,     // LPI_PROTO_SMTP
+	92:  godpi.Smtp,     // LPI_PROTO_INVALID_SMTP
+	21:  godpi.Rdp,      // LPI_PROTO_RDP
+	24:  godpi.Smb,      // LPI_PROTO_SMB
+	380: godpi.Icmp,     // LPI_PROTO_ICMP
+	27:  godpi.Ftp,      // LPI_PROTO_FTP_CONTROL
+	12:  godpi.Ssl,      // LPI_PROTO_SSL
+	37:  godpi.Netbios,  // LPI_PROTO_NETBIOS
 }
 
 // LPIWrapperName is the identification of the libprotoident library.

diff --git a/wrappers/nDPI_wrapper.go b/wrappers/nDPI_wrapper.go
@@ -13,18 +13,18 @@ import (
 
 // ndpiCodeToProtocol maps the nDPI protocol codes to go-dpi protocols.
 var ndpiCodeToProtocol = map[uint32]godpi.Protocol{
-	7:   godpi.Http,
-	5:   godpi.Dns,
-	92:  godpi.Ssh,
-	127: godpi.Rpc,
-	3:   godpi.Smtp,
-	88:  godpi.Rdp,
-	16:  godpi.Smb,
-	81:  godpi.Icmp,
-	1:   godpi.Ftp,
-	91:  godpi.Ssl,
-	64:  godpi.Ssl,
-	10:  godpi.Netbios,
+	7:   godpi.Http,       // NDPI_PROTOCOL_HTTP
+	5:   godpi.Dns,        // NDPI_PROTOCOL_DNS
+	92:  godpi.Ssh,        // NDPI_PROTOCOL_SSH
+	127: godpi.Rpc,        // NDPI_PROTOCOL_DCERPC
+	3:   godpi.Smtp,       // NDPI_PROTOCOL_MAIL_SMTP
+	88:  godpi.Rdp,        // NDPI_PROTOCOL_RDP
+	16:  godpi.Smb,        // NDPI_PROTOCOL_SMB
+	81:  godpi.Icmp,       // NDPI_PROTOCOL_IP_ICMP
+	1:   godpi.Ftp,        // NDPI_PROTOCOL_FTP_CONTROL
+	91:  godpi.Ssl,        // NDPI_PROTOCOL_SSL
+	64:  godpi.Ssl,        // NDPI_PROTOCOL_SSL_NO_CERT
+	10:  godpi.Netbios,    // NDPI_PROTOCOL_NETBIOS
 }
 
 // NDPIWrapperName is the identification of the nDPI library.