wip

Author: gallypette

README.md

@@ -4,4 +4,5 @@ analyzer-d4-passivessl fetch a redis feed of certificate and TLS sessions and ma
 ```bash
 go get github.com/gomodule/redigo/redis
 go get github.com/lib/pq
+sudo apt install postgresal-plpython3-10
 ```

main.go

@@ -100,12 +100,21 @@ func main() {
 			s := sessionRecord{}
 			_ = json.Unmarshal([]byte(dat), &s)
 
-			if len(s.Certificates) > 0 {
+			if isValid(&s){
+				// Create ja3* records
+				err = insertJA3(&s)
+				if err != nil {
+					log.Fatal(fmt.Sprintf("Could not insert JA3 into DB: %q", err))
+				}
 				// Insert Session
 				ids, err := insertSession(&s)
 				if err != nil {
 					log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err))
 				}
+				err = insertFuzzyHash(&s, ids)
+				if err != nil {
+					log.Fatal(fmt.Sprintf("Insert Fuzzy Hash into DB failed: %q", err))
+				}
 				// Attempt to roughly build a chain of trust
 				session := buildChain(&s)
 
@@ -119,17 +128,23 @@ func main() {
 				if err != nil {
 					log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB: %q", err))
 				}
-
-				// Create ja3* records
-				err = insertJA3(session)
-				if err != nil {
-					log.Fatal(fmt.Sprintf("Could not insert JA3 into DB: %q", err))
-				}
 			}
 		}
 	}
 }
 
+func isValid(s * sessionRecord) bool {
+	// Relationships
+	if len(s.Certificates) < 1  || s.JA3Digest == "" || s.JA3 == "" {
+		return false
+	}
+	// Basic informations
+	if s.ClientPort == "" || s.ServerPort == "" || s.ServerIP == "" || s.ClientIP == "" {
+		return false
+	}
+	return true
+}
+
 func insertJA3(s *sessionRecord) error {
 	q := `INSERT INTO "ja3" (hash, raw, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
 	_, err := db.Exec(q, s.JA3Digest, s.JA3, "ja3")
@@ -144,6 +159,15 @@ func insertJA3(s *sessionRecord) error {
 	return nil
 }
 
+func insertFuzzyHash(s *sessionRecord, ids int64) error {
+	q := `INSERT INTO "fuzzy_hash" (type, value, "id_sessionRecord") VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
+	_, err := db.Exec(q, "TLSH", s.TLSH, ids)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // insertPublicKeys insert each public key of each certificate of a session
 func insertPublicKey(c x509.Certificate) (string, error) {
 	pub, err := x509.ParsePKIXPublicKey(c.RawSubjectPublicKeyInfo)
@@ -321,9 +345,9 @@ func insertCertificates(s *sessionRecord) ([]string, error) {
 }
 
 func insertSession(s *sessionRecord) (int64, error) {
-	q := `INSERT INTO "sessionRecord" (dst_ip, src_ip, dst_port, src_port, timestamp) VALUES ($1, $2, $3, $4, $5) RETURNING id`
+	q := `INSERT INTO "sessionRecord" (dst_ip, src_ip, dst_port, src_port, timestamp, hash_ja3) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`
 	var id int64
-	err := db.QueryRow(q, s.ServerIP, s.ClientIP, s.ServerPort, s.ClientPort, s.Timestamp).Scan(&id)
+	err := db.QueryRow(q, s.ServerIP, s.ClientIP, s.ServerPort, s.ClientPort, s.Timestamp, s.JA3Digest).Scan(&id)
 	if err != nil {
 		return 0, err
 	}
@@ -339,7 +363,7 @@ func initRedis() {
 }
 
 func initDB() {
-	connStr := "user=postgres password=postgres dbname=new_database"
+	connStr := "user=postgres password=postgres dbname=passive_ssl"
 	err := errors.New("")
 	db, err = sql.Open("postgres", connStr)
 	if err != nil {

passivessl.sql

@@ -4,12 +4,15 @@
 -- Project Site: pgmodeler.com.br
 -- Model Author: ---
 
+SET check_function_bodies = false;
+-- ddl-end --
+
 
 -- Database creation must be done outside an multicommand file.
 -- These commands were put in this file only for convenience.
--- -- object: new_database | type: DATABASE --
--- -- DROP DATABASE IF EXISTS new_database;
--- CREATE DATABASE new_database
+-- -- object: passive_ssl | type: DATABASE --
+-- -- DROP DATABASE IF EXISTS passive_ssl;
+-- CREATE DATABASE passive_ssl
 -- ;
 -- -- ddl-end --
 -- 
@@ -64,7 +67,7 @@ CREATE TABLE public.certificate(
 	is_valid_chain bool NOT NULL DEFAULT false,
 	"notBefore" time,
 	"notAfter" time,
-	"isSS" bool NOT NULL DEFAULT false,
+	"is_SS" bool NOT NULL DEFAULT false,
 	CONSTRAINT certificate_pk PRIMARY KEY (hash)
 
 );
@@ -104,7 +107,7 @@ CREATE TABLE public."sessionRecord"(
 	src_ip inet NOT NULL,
 	dst_port int4 NOT NULL,
 	src_port int4 NOT NULL,
-	hash_ja3 bytea,
+	hash_ja3 bytea NOT NULL,
 	"timestamp" time(0) with time zone,
 	CONSTRAINT "sessionRecord_pk" PRIMARY KEY (id)
 
@@ -130,7 +133,7 @@ ALTER TABLE public.ja3 OWNER TO postgres;
 -- ALTER TABLE public."sessionRecord" DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
 ALTER TABLE public."sessionRecord" ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
 REFERENCES public.ja3 (hash) MATCH FULL
-ON DELETE SET NULL ON UPDATE CASCADE;
+ON DELETE RESTRICT ON UPDATE CASCADE;
 -- ddl-end --
 
 -- object: public."many_sessionRecord_has_many_certificate" | type: TABLE --
@@ -162,8 +165,8 @@ ON DELETE RESTRICT ON UPDATE CASCADE;
 CREATE TABLE public.fuzzy_hash(
 	id bigserial NOT NULL,
 	type text NOT NULL,
-	value public.hstore NOT NULL,
-	hash_ja3 bytea,
+	value text NOT NULL,
+	"id_sessionRecord" bigint,
 	CONSTRAINT fuzzy_hash_pk PRIMARY KEY (id)
 
 );
@@ -258,11 +261,62 @@ CREATE INDEX path_index ON public.certificate
 	WITH (BUFFERING = ON);
 -- ddl-end --
 
--- object: ja3_fk | type: CONSTRAINT --
--- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
-ALTER TABLE public.fuzzy_hash ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
-REFERENCES public.ja3 (hash) MATCH FULL
+-- object: "sessionRecord_fk" | type: CONSTRAINT --
+-- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE;
+ALTER TABLE public.fuzzy_hash ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord")
+REFERENCES public."sessionRecord" (id) MATCH FULL
 ON DELETE SET NULL ON UPDATE CASCADE;
 -- ddl-end --
 
+-- object: plpython3u | type: LANGUAGE --
+-- DROP LANGUAGE IF EXISTS plpython3u CASCADE;
+CREATE  LANGUAGE plpython3u;
+-- ddl-end --
+ALTER LANGUAGE plpython3u OWNER TO postgres;
+-- ddl-end --
+
+-- object: public.tlshc | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS public.tlshc(text,text) CASCADE;
+CREATE FUNCTION public.tlshc ( a text,  b text)
+	RETURNS int4
+	LANGUAGE plpython3u
+	VOLATILE 
+	CALLED ON NULL INPUT
+	SECURITY INVOKER
+	COST 1
+	AS $$
+import tlsh
+return tlsh.diff(a, b)
+$$;
+-- ddl-end --
+ALTER FUNCTION public.tlshc(text,text) OWNER TO postgres;
+-- ddl-end --
+
+-- object: public.tlsht | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS public.tlsht(IN text,IN text,IN int4,int4) CASCADE;
+CREATE FUNCTION public.tlsht (IN filter text, IN hash text, IN threshold int4,  maxrows int4)
+	RETURNS SETOF public.fuzzy_hash
+	LANGUAGE plpython3u
+	IMMUTABLE LEAKPROOF
+	RETURNS NULL ON NULL INPUT
+	SECURITY INVOKER
+	COST 1
+	ROWS 1000
+	AS $$
+import tlsh
+param = ["TLSH"]
+#param[0] = filter
+#plan = plpy.prepare("SELECT * FROM fuzzy_hash WHERE type <> $1", ["text"])
+#rv = plan.execute(param, maxrows)
+rv = plpy.execute("SELECT * FROM fuzzy_hash", 1000)
+r = []
+for x in rv:
+    if tlsh.diff(x["value"], hash) < threshold:
+        r.append(x)
+return r
+$$;
+-- ddl-end --
+ALTER FUNCTION public.tlsht(IN text,IN text,IN int4,int4) OWNER TO postgres;
+-- ddl-end --
+