wip

Author: gallypette

main.go

@@ -46,6 +46,7 @@ type sessionRecord struct {
 
 type chain struct {
 	isValid bool
+	isSS    bool
 	s       string
 }
 
@@ -77,38 +78,72 @@ func main() {
 		}
 
 	} else {
-		jsonPath = "./test.json"
-
-		// read corresponding json file
-		dat, err := ioutil.ReadFile(jsonPath)
+		var path []string
+		path = append(path, "./sessions")
+		path = append(path, "")
+		files, err := ioutil.ReadDir(path[0])
 		if err != nil {
 			log.Fatal(err)
 		}
-		// Unmarshal JSON file
-		s := sessionRecord{}
-		_ = json.Unmarshal([]byte(dat), &s)
 
-		// Insert Session
-		ids, err := insertSession(&s)
-		if err != nil {
-			log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err))
-		}
-		// Attempt to roughly build a chain of trust
-		session := buildChain(&s)
+		for _, f := range files {
 
-		// Insert Certificates
-		idc, err := insertCertificates(session)
-		if err != nil {
-			log.Fatal(fmt.Sprintf("Insert Certificate into DB failed: %q", err))
-		}
-		// Create the relationship between certificates and sessions
-		err = linkSessionCerts(ids, idc)
-		if err != nil {
-			log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB: %q", err))
+			path[1] = f.Name()
+			jsonPath = strings.Join(path, "/")
+
+			// read corresponding json file
+			dat, err := ioutil.ReadFile(jsonPath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			// Unmarshal JSON file
+			s := sessionRecord{}
+			_ = json.Unmarshal([]byte(dat), &s)
+
+			if len(s.Certificates) > 0 {
+				// Insert Session
+				ids, err := insertSession(&s)
+				if err != nil {
+					log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err))
+				}
+				// Attempt to roughly build a chain of trust
+				session := buildChain(&s)
+
+				// Insert Certificates
+				idc, err := insertCertificates(session)
+				if err != nil {
+					log.Fatal(fmt.Sprintf("Insert Certificate into DB failed: %q", err))
+				}
+				// Create the relationship between certificates and sessions
+				err = linkSessionCerts(ids, idc)
+				if err != nil {
+					log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB: %q", err))
+				}
+
+				// Create ja3* records
+				err = insertJA3(session)
+				if err != nil {
+					log.Fatal(fmt.Sprintf("Could not insert JA3 into DB: %q", err))
+				}
+			}
 		}
 	}
 }
 
+func insertJA3(s *sessionRecord) error {
+	q := `INSERT INTO "ja3" (hash, raw, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
+	_, err := db.Exec(q, s.JA3Digest, s.JA3, "ja3")
+	if err != nil {
+		return err
+	}
+	q = `INSERT INTO "ja3" (hash, raw, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
+	_, err = db.Exec(q, s.JA3SDigest, s.JA3S, "ja3s")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // insertPublicKeys insert each public key of each certificate of a session
 func insertPublicKey(c x509.Certificate) (string, error) {
 	pub, err := x509.ParsePKIXPublicKey(c.RawSubjectPublicKeyInfo)
@@ -119,22 +154,22 @@ func insertPublicKey(c x509.Certificate) (string, error) {
 
 	switch pub := pub.(type) {
 	case *rsa.PublicKey:
-		q := `INSERT INTO "public_key" (hash, type, modulus, exponent, modulus_size) VALUES ($1, $2, $3, $4, $5)`
-		_, err := db.Query(q, hash, "RSA", (*BigNumber)(pub.N), pub.E, pub.Size())
+		q := `INSERT INTO "public_key" (hash, type, modulus, exponent, modulus_size) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`
+		_, err := db.Exec(q, hash, "RSA", (*BigNumber)(pub.N), pub.E, pub.Size())
 		if err != nil {
-			return hash, nil
+			return hash, err
 		}
 	case *dsa.PublicKey:
-		q := `INSERT INTO "public_key" (hash, type, "G", "P", "Q", "Y") VALUES ($1, $2, $3, $4, $5, $6)`
-		_, err := db.Query(q, hash, "DSA", (*BigNumber)(pub.Parameters.G), (*BigNumber)(pub.Parameters.P), (*BigNumber)(pub.Parameters.Q), (*BigNumber)(pub.Y))
+		q := `INSERT INTO "public_key" (hash, type, "G", "P", "Q", "Y") VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`
+		_, err := db.Exec(q, hash, "DSA", (*BigNumber)(pub.Parameters.G), (*BigNumber)(pub.Parameters.P), (*BigNumber)(pub.Parameters.Q), (*BigNumber)(pub.Y))
 		if err != nil {
-			return hash, nil
+			return hash, err
 		}
 	case *ecdsa.PublicKey:
-		q := `INSERT INTO "public_key" (hash, type, "Y", "X", "P", "N", "B", "bitsize", "Gx", "Gy") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`
-		_, err := db.Query(q, hash, "ECDSA", pub.Y, pub.X, pub.Curve.Params().P, pub.Curve.Params().N, pub.Curve.Params().B, pub.Curve.Params().BitSize, pub.Curve.Params().Gx, pub.Curve.Params().Gy)
+		q := `INSERT INTO "public_key" (hash, type, "Y", "X", "P", "N", "B", "bitsize", "Gx", "Gy", "curve_name") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT DO NOTHING`
+		_, err := db.Exec(q, hash, "ECDSA", (*BigNumber)(pub.Y), (*BigNumber)(pub.X), (*BigNumber)(pub.Curve.Params().P), (*BigNumber)(pub.Curve.Params().N), (*BigNumber)(pub.Curve.Params().B), pub.Curve.Params().BitSize, (*BigNumber)(pub.Curve.Params().Gx), (*BigNumber)(pub.Curve.Params().Gy), pub.Params().Name)
 		if err != nil {
-			return hash, nil
+			return hash, err
 		}
 	default:
 		return hash, fmt.Errorf("PKIx: could not determine the type of key %g", pub)
@@ -149,8 +184,8 @@ func (bn *BigNumber) Value() (driver.Value, error) {
 // linkSessionCerts creates the link between a session and its certificates
 func linkSessionCerts(ids int64, idc []string) error {
 	for _, i := range idc {
-		q := `INSERT INTO "many_sessionRecord_has_many_certificate" ("id_sessionRecord", "hash_certificate") VALUES ($1, $2)`
-		_, err := db.Query(q, ids, i)
+		q := `INSERT INTO "many_sessionRecord_has_many_certificate" ("id_sessionRecord", "hash_certificate") VALUES ($1, $2) ON CONFLICT DO NOTHING `
+		_, err := db.Exec(q, ids, i)
 		if err != nil {
 			return err
 		}
@@ -164,57 +199,107 @@ func linkSessionCerts(ids int64, idc []string) error {
 // does not touch the original slice and mark the chain as invalid.
 func buildChain(s *sessionRecord) *sessionRecord {
 	certChain := make([]certMapElm, 0)
+	// First find if there are any duplicates certificate
+	dedup := unique(s.Certificates)
+	deduplen := len(dedup)
+
+	if deduplen > 1 {
+		// Then we find the leaf
+		removed := 0
+		for i, c := range dedup {
+			if !c.Certificate.IsCA {
+				certChain = append(certChain, c)
+				// Remove this element from the list
+				dedup = append(dedup[:i-removed], dedup[i+1-removed:]...)
+				removed++
+			}
+		}
 
-	// First we find the leaf
-	for _, c := range s.Certificates {
-		fmt.Println(c.Certificate.Issuer.String())
-		fmt.Println(c.Certificate.Subject.String())
-		fmt.Println(c.Certificate.Subject.String() == c.Certificate.Issuer.String())
-		if !c.Certificate.IsCA {
-			certChain = append(certChain, c)
+		cursor := len(certChain)
+		// Find the parent of each certificate
+		for i := 0; i < cursor; i++ {
+			p, success, isSS := findParent(dedup, certChain[i])
+			// if we found a root, no need to go any further
+			if isSS {
+				p.chain.isSS = true
+				certChain = append(certChain, p)
+				break
+			}
+			if success {
+				p.chain.isSS = false
+				certChain = append(certChain, p)
+			}
+			cursor = len(certChain)
 		}
-	}
-	// Find the parent of each certificate
-	for _, _ = range s.Certificates {
-		for i, _ := range s.Certificates {
-			if s.Certificates[i].Certificate.Subject.String() == certChain[len(certChain)-1].Issuer.String() {
-				certChain = append(certChain, s.Certificates[i])
+
+		// Write the new chain if it's valid
+		if len(certChain) >= deduplen-1 {
+			cstr := make([]string, 0)
+			for i := len(certChain) - 1; i >= 0; i-- {
+				certChain[i].chain.isValid = true
+				if !certChain[i].chain.isSS {
+					cstr = append(cstr, certChain[i].CertHash)
+					certChain[i].chain.s = strings.Join(cstr, ".")
+				}
 			}
+			tmp := s
+			tmp.Certificates = certChain
+			return tmp
 		}
+		// Only one cert in the chain
+	} else {
+		s.Certificates[0].chain = chain{true, s.Certificates[0].Certificate.Issuer.String() == s.Certificates[0].Certificate.Subject.String(), s.Certificates[0].CertHash}
 	}
-	// Write the new chain
-	if len(certChain) == len(s.Certificates) {
-		cstr := make([]string, 0)
-		for i := len(certChain) - 1; i >= 0; i-- {
-			certChain[i].chain.isValid = true
-			cstr = append(cstr, certChain[i].CertHash)
-			certChain[i].chain.s = strings.Join(cstr, ".")
+
+	return s
+}
+
+func findParent(dedup []certMapElm, c certMapElm) (certMapElm, bool, bool) {
+	// A Root or SSC signs itself
+	if c.Certificate.Subject.String() == c.Certificate.Issuer.String() {
+		return c, true, true
+	}
+
+	// A leaf or a node has a parent
+	for _, candidate := range dedup {
+		if candidate.Certificate.Subject.String() == c.Certificate.Issuer.String() {
+			return candidate, true, false
 		}
-		tmp := s
-		tmp.Certificates = certChain
-		return tmp
 	}
-	return s
+
+	return c, false, false
+}
+
+func unique(s []certMapElm) []certMapElm {
+	keys := make(map[string]bool)
+	var list []certMapElm
+	for _, entry := range s {
+		if _, value := keys[entry.CertHash]; !value {
+			keys[entry.CertHash] = true
+			list = append(list, entry)
+		}
+	}
+	return list
 }
 
 func insertCertificate(c certMapElm) (string, error) {
-	q := `INSERT INTO "certificate" (hash, "is_CA", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING hash`
-	var hash string
-	err := db.QueryRow(q, c.CertHash, c.Certificate.IsCA, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), c.chain.s, c.chain.isValid, getFullPath(c.CertHash)).Scan(&hash)
+	q := `INSERT INTO "certificate" (hash, "is_CA", "is_SS", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING`
+	_, err := db.Exec(q, c.CertHash, c.Certificate.IsCA, c.chain.isSS, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), c.chain.s, c.chain.isValid, getFullPath(c.CertHash))
 	if err != nil {
-		return hash, err
+		return c.CertHash, err
 	}
 	key, err := insertPublicKey(*c.Certificate)
 	if err != nil {
-		return hash, err
+		return c.CertHash, err
 	}
-
-	q = `INSERT INTO "many_certificate_has_many_public_key" ("hash_certificate", "hash_public_key") VALUES ($1, $2)`
-	_, err = db.Query(q, hash, key)
+	fmt.Println(c.CertHash)
+	fmt.Println(key)
+	q = `INSERT INTO "many_certificate_has_many_public_key" ("hash_certificate", "hash_public_key") VALUES ($1, $2) ON CONFLICT DO NOTHING `
+	_, err = db.Exec(q, c.CertHash, key)
 	if err != nil {
-		return hash, err
+		return c.CertHash, err
 	}
-	return hash, nil
+	return c.CertHash, nil
 }
 
 // getFullPath takes a certificate's hash and return the full path to

passivessl.sql

@@ -64,6 +64,7 @@ CREATE TABLE public.certificate(
 	is_valid_chain bool NOT NULL DEFAULT false,
 	"notBefore" time,
 	"notAfter" time,
+	"isSS" bool NOT NULL DEFAULT false,
 	CONSTRAINT certificate_pk PRIMARY KEY (hash)
 
 );
@@ -117,7 +118,7 @@ ALTER TABLE public."sessionRecord" OWNER TO postgres;
 CREATE TABLE public.ja3(
 	hash bytea NOT NULL,
 	raw text,
-	type smallint NOT NULL,
+	type varchar(16) NOT NULL,
 	CONSTRAINT j3a_pk PRIMARY KEY (hash)
 
 );