Skip to content

Commit 0ec06e5

Browse files
gallypettegallypette
committed
add: [importing] walk + DER + fix #4 
1 parent 33f0b33 commit 0ec06e5

File tree

1 file changed

+153
-52
lines changed

1 file changed

+153
-52
lines changed

‎main.go‎

Lines changed: 153 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ import (
55
"crypto/dsa"
66
"crypto/ecdsa"
77
"crypto/rsa"
8+
"crypto/sha1"
89
"crypto/sha256"
9-
"crypto/x509"
1010
"database/sql"
1111
"database/sql/driver"
1212
"encoding/json"
@@ -20,11 +20,14 @@ import (
2020
"net"
2121
"os"
2222
"os/signal"
23+
"path/filepath"
2324
"regexp"
2425
"strconv"
2526
"strings"
2627
"time"
2728

29+
"github.com/gallypette/certificate-transparency-go/x509"
30+
2831
"github.com/gomodule/redigo/redis"
2932
_ "github.com/lib/pq"
3033
)
@@ -41,9 +44,11 @@ type (
4144
postgresPort string
4245
postgresDB string
4346
certPath string
47+
format string
48+
recursive bool
4449
}
4550

46-
BigNumber big.Int
51+
bigNumber big.Int
4752

4853
certMapElm struct {
4954
CertHash string
@@ -73,10 +78,12 @@ type (
7378
)
7479

7580
var (
76-
db *sql.DB
77-
confdir = flag.String("c", "conf.sample", "configuration directory")
78-
connectRedis = true
79-
cr redis.Conn
81+
db *sql.DB
82+
confdir = flag.String("c", "conf.sample", "configuration directory")
83+
recursive = flag.Bool("r", false, "should it open the directory recursively")
84+
format = flag.String("f", "json", "certificate file format [json, crt, der]")
85+
pull = flag.Bool("p", true, "pull from redis?")
86+
cr redis.Conn
8087
)
8188

8289
func main() {
@@ -119,25 +126,9 @@ func main() {
119126
*confdir = strings.TrimSuffix(*confdir, "\\")
120127
}
121128

122-
// Parse Redis Config
123-
tmp := readConfFile(*confdir, "redis")
124-
ss := strings.Split(string(tmp), "/")
125-
if len(ss) <= 1 {
126-
log.Fatal("Missing Database in Redis config: should be host:port/database_name")
127-
}
128-
c.redisDB, _ = strconv.Atoi(ss[1])
129-
var ret bool
130-
ret, ss[0] = isNet(ss[0])
131-
if !ret {
132-
sss := strings.Split(string(ss[0]), ":")
133-
c.redisHost = sss[0]
134-
c.redisPort = sss[1]
135-
}
136-
c.redisQueue = string(readConfFile(*confdir, "redis_queue"))
137-
138129
// Parse DB Config
139-
tmp = readConfFile(*confdir, "postgres")
140-
ss = strings.Split(string(tmp), "/")
130+
tmp := readConfFile(*confdir, "postgres")
131+
ss := strings.Split(string(tmp), "/")
141132
if len(ss) <= 1 {
142133
log.Fatal("Missing Database in Postgres config: should be user:pwd@host:port/database_name")
143134
}
@@ -161,27 +152,40 @@ func main() {
161152

162153
// Parse Certificate Folder
163154
c.certPath = string(readConfFile(*confdir, "certfolder"))
155+
c.recursive = *recursive
156+
c.format = *format
164157

165158
// DB
166159
initDB(c.postgresUser, c.postgresPWD, c.postgresHost, c.postgresPort, c.postgresDB)
167160
defer db.Close()
168161

169162
var jsonPath string
170163

171-
// Redis
172-
if connectRedis {
164+
if *pull { // Redis
165+
// Parse Redis Config
166+
tmp := readConfFile(*confdir, "redis")
167+
ss := strings.Split(string(tmp), "/")
168+
if len(ss) <= 1 {
169+
log.Fatal("Missing Database in Redis config: should be host:port/database_name")
170+
}
171+
c.redisDB, _ = strconv.Atoi(ss[1])
172+
var ret bool
173+
ret, ss[0] = isNet(ss[0])
174+
if !ret {
175+
sss := strings.Split(string(ss[0]), ":")
176+
c.redisHost = sss[0]
177+
c.redisPort = sss[1]
178+
}
179+
c.redisQueue = string(readConfFile(*confdir, "redis_queue"))
173180
initRedis(c.redisHost, c.redisPort, c.redisDB)
174181
defer cr.Close()
175182
// pop redis queue
176183
for {
177184
err := errors.New("")
178185
jsonPath, err = redis.String(cr.Do("LPOP", "analyzer:ja3-jl:"+c.redisQueue))
179186
if err != nil {
180-
time.Sleep(30 * time.Second)
181-
} else {
182-
processFile(c.certPath, jsonPath)
187+
time.Sleep(2 * time.Second)
183188
}
184-
185189
// Exit Signal Handle
186190
select {
187191
case <-s:
@@ -191,31 +195,123 @@ func main() {
191195
continue
192196
}
193197
}
194-
195-
} else {
196-
var path []string
197-
path = append(path, "./sessions")
198-
path = append(path, "")
199-
files, err := ioutil.ReadDir(path[0])
200-
if err != nil {
201-
log.Fatal(err)
198+
} else { // Files
199+
if c.recursive {
200+
err := filepath.Walk(c.certPath,
201+
func(path string, info os.FileInfo, err error) error {
202+
if err != nil {
203+
return err
204+
}
205+
if !info.IsDir() {
206+
processFile(c.certPath, path, c.format)
207+
}
208+
return nil
209+
})
210+
if err != nil {
211+
log.Println(err)
212+
}
213+
} else {
214+
var path []string
215+
path = append(path, "./sessions")
216+
path = append(path, "")
217+
files, err := ioutil.ReadDir(path[0])
218+
if err != nil {
219+
log.Fatal(err)
220+
}
221+
for _, f := range files {
222+
path[1] = f.Name()
223+
jsonPath = strings.Join(path, "/")
224+
processFile(c.certPath, jsonPath, c.format)
225+
226+
// Exit Signal Handle
227+
select {
228+
case <-s:
229+
fmt.Println("Exiting...")
230+
os.Exit(0)
231+
}
232+
}
202233
}
203-
for _, f := range files {
204-
path[1] = f.Name()
205-
jsonPath = strings.Join(path, "/")
206-
processFile(c.certPath, jsonPath)
207234

208-
// Exit Signal Handle
209-
select {
210-
case <-s:
211-
fmt.Println("Exiting...")
212-
os.Exit(0)
235+
}
236+
}
237+
238+
func processFile(fp string, p string, f string) {
239+
switch f {
240+
case "json":
241+
processJSON(fp, p)
242+
break
243+
case "der":
244+
processDER(fp, p)
245+
break
246+
case "crt":
247+
processCRT(fp, p)
248+
break
249+
}
250+
}
251+
252+
func processDER(fp string, p string) bool {
253+
// read corresponding der file
254+
dat, err := ioutil.ReadFile(p)
255+
if err != nil {
256+
log.Fatal(err)
257+
}
258+
cert, err := x509.ParseCertificate(dat)
259+
if err != nil {
260+
// Not stopping on Non Fatal Errors
261+
switch err := err.(type) {
262+
case x509.NonFatalErrors:
263+
// Stopping on Unknown PK Algo
264+
if cert.PublicKeyAlgorithm == 0 {
265+
fmt.Println("Unknown Public Key Algorithm, skipping key (most likely GOST R 14)")
266+
break
213267
}
268+
goto I
269+
default:
270+
fmt.Println("failed to parse certificate: " + err.Error())
214271
}
215272
}
273+
274+
I:
275+
// Cert elem
276+
h := sha1.New()
277+
h.Write(cert.Raw)
278+
c := certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}
279+
// Insert Certificate
280+
err = insertLeafCertificate(fp, c)
281+
if err != nil {
282+
log.Fatal(fmt.Sprintf("Insert Certificate into DB failed: %q", err))
283+
}
284+
285+
return true
286+
}
287+
288+
func insertLeafCertificate(fp string, c certMapElm) error {
289+
q := `INSERT INTO "certificate" (hash, "is_CA", "is_SS", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING`
290+
_, err := db.Exec(q, c.CertHash, c.Certificate.IsCA, false, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), nil, false, getFullPath(fp, c.CertHash))
291+
if err != nil {
292+
return err
293+
}
294+
key, err := insertPublicKey(*c.Certificate)
295+
if err != nil {
296+
return err
297+
}
298+
//fmt.Println(c.CertHash)
299+
//fmt.Println(key)
300+
q = `INSERT INTO "many_certificate_has_many_public_key" ("hash_certificate", "hash_public_key") VALUES ($1, $2) ON CONFLICT DO NOTHING `
301+
_, err = db.Exec(q, c.CertHash, key)
302+
if err != nil {
303+
fmt.Println(c.CertHash)
304+
return err
305+
}
306+
return nil
307+
}
308+
309+
func processCRT(fp string, p string) bool {
310+
311+
return true
216312
}
217313

218-
func processFile(fp string, p string) bool {
314+
func processJSON(fp string, p string) bool {
219315
// read corresponding json file
220316
dat, err := ioutil.ReadFile(p)
221317
if err != nil {
@@ -303,19 +399,24 @@ func insertPublicKey(c x509.Certificate) (string, error) {
303399
switch pub := pub.(type) {
304400
case *rsa.PublicKey:
305401
q := `INSERT INTO "public_key" (hash, type, modulus, exponent, modulus_size) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`
306-
_, err := db.Exec(q, hash, "RSA", (*BigNumber)(pub.N), pub.E, pub.Size())
402+
_, err := db.Exec(q, hash, "RSA", (*bigNumber)(pub.N), pub.E, pub.Size())
307403
if err != nil {
308404
return hash, err
309405
}
406+
// else {
407+
// Adds the moduli into Redis for analysis
408+
// cr.Send("HMSET", hash, "moduli", (*BigNumber)(pub.N))
409+
//cr.Send("LPUSH", "albums", "1")
410+
// }
310411
case *dsa.PublicKey:
311412
q := `INSERT INTO "public_key" (hash, type, "G", "P", "Q", "Y") VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`
312-
_, err := db.Exec(q, hash, "DSA", (*BigNumber)(pub.Parameters.G), (*BigNumber)(pub.Parameters.P), (*BigNumber)(pub.Parameters.Q), (*BigNumber)(pub.Y))
413+
_, err := db.Exec(q, hash, "DSA", (*bigNumber)(pub.Parameters.G), (*bigNumber)(pub.Parameters.P), (*bigNumber)(pub.Parameters.Q), (*bigNumber)(pub.Y))
313414
if err != nil {
314415
return hash, err
315416
}
316417
case *ecdsa.PublicKey:
317418
q := `INSERT INTO "public_key" (hash, type, "Y", "X", "P", "N", "B", "bitsize", "Gx", "Gy", "curve_name") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT DO NOTHING`
318-
_, err := db.Exec(q, hash, "ECDSA", (*BigNumber)(pub.Y), (*BigNumber)(pub.X), (*BigNumber)(pub.Curve.Params().P), (*BigNumber)(pub.Curve.Params().N), (*BigNumber)(pub.Curve.Params().B), pub.Curve.Params().BitSize, (*BigNumber)(pub.Curve.Params().Gx), (*BigNumber)(pub.Curve.Params().Gy), pub.Params().Name)
419+
_, err := db.Exec(q, hash, "ECDSA", (*bigNumber)(pub.Y), (*bigNumber)(pub.X), (*bigNumber)(pub.Curve.Params().P), (*bigNumber)(pub.Curve.Params().N), (*bigNumber)(pub.Curve.Params().B), pub.Curve.Params().BitSize, (*bigNumber)(pub.Curve.Params().Gx), (*bigNumber)(pub.Curve.Params().Gy), pub.Params().Name)
319420
if err != nil {
320421
return hash, err
321422
}
@@ -325,7 +426,7 @@ func insertPublicKey(c x509.Certificate) (string, error) {
325426
return hash, nil
326427
}
327428

328-
func (bn *BigNumber) Value() (driver.Value, error) {
429+
func (bn *bigNumber) Value() (driver.Value, error) {
329430
return driver.Value((*big.Int)(bn).Text(10)), nil
330431
}
331432

0 commit comments

Comments
 (0)