@@ -300,6 +300,7 @@ func handleDirectTranslation(ctx context.Context, user shuffle.User, value shuff
300300 authConfig = ""
301301 }
302302
303+
303304 // Remapping into non-list JSON blob
304305 newMap := map [string ]interface {}{}
305306 for _ , field := range value .Fields {
@@ -330,6 +331,10 @@ func handleDirectTranslation(ctx context.Context, user shuffle.User, value shuff
330331 value .Label = "ticket"
331332 }
332333
334+ if debug {
335+ log .Printf ("[DEBUG] Translating label %s with body:\n %s\n \n " , value .Label , string (marshalledFields ))
336+ }
337+
333338 // Is there any way to ingest these as well?
334339 schemalessOutput , err := schemaless .Translate (ctx , value .Label , marshalledFields , authConfig )
335340 if err != nil {
@@ -422,15 +427,15 @@ func autoUploadSingulOutput(ctx context.Context, orgId string, curApikey string,
422427 for cnt , item := range foundArray {
423428 // Check if uid, uuid, id or similar is a valid key
424429 if _ , ok := item .(map [string ]interface {}); ! ok {
425- log .Printf ("[WARNING ] Item in list output for label %s is not a map: %#v" , value .Label , item )
430+ log .Printf ("[ERROR ] Item in Singul list output for label %s is not a map: %#v" , value .Label , item )
426431 continue
427432 }
428433
429434 generatedItem := item .(map [string ]interface {})
430435
431436 idKeys := []string {"id" , "uid" , "uuid" , "identifier" , "key" }
432-
433437 foundIdentifier := ""
438+ foundIdentifierKey := ""
434439 for _ , idKey := range idKeys {
435440 // Check if lower case exists
436441 for key , mapValue := range generatedItem {
@@ -441,18 +446,19 @@ func autoUploadSingulOutput(ctx context.Context, orgId string, curApikey string,
441446 // Found the key, set it as the actual label
442447 if val , ok := mapValue .(string ); ok {
443448 foundIdentifier = val
449+ foundIdentifierKey = key
444450 break
445451 } else {
446452 log .Printf ("[ERROR] Failed finding ID key in item for label %s: %s" , value .Label , mapValue )
447453 }
448454 }
449455
450- if foundIdentifier != "" {
456+ if foundIdentifier != "" && len ( foundIdentifier ) > 4 {
451457 break
452458 }
453459 }
454460
455- // hardcoded override for product name
461+ // hardcoded override for product name. Just a test.
456462 toolKeys := []string {"product" , "tool" , "service" , "application" , "app" }
457463 if len (secondAction .AppName ) > 0 {
458464 for _ , toolKey := range toolKeys {
@@ -471,9 +477,29 @@ func autoUploadSingulOutput(ctx context.Context, orgId string, curApikey string,
471477 }
472478 }
473479
474- if len (foundIdentifier ) == 0 {
475- log .Printf ("[ERROR] Failed finding ID key in item for label %s: %#v" , value .Label , item )
476- break
480+ // Generate an ID if we didn't find one
481+ if len (foundIdentifier ) <= 4 {
482+ log .Printf ("[ERROR] Failed finding VALID ID key in item for label %s: %#v. ID Key: %s, Value: %s" , value .Label , item , foundIdentifierKey , foundIdentifier )
483+
484+ marshalledItem , err := json .Marshal (generatedItem )
485+ if err != nil {
486+ log .Printf ("[ERROR] Failed marshalling item in schemaless output for label %s: %s" , value .Label , err )
487+ break
488+ } else {
489+ foundIdentifier = fmt .Sprintf ("%x" , md5 .Sum (marshalledItem ))
490+ log .Printf ("MARSHALLED OCSF (%s):\n %s\n " , foundIdentifier , string (marshalledItem ))
491+ }
492+ }
493+
494+ // In case it finds some weird thingy
495+ if len (foundIdentifier ) > 36 {
496+ // md5 sum it
497+ foundIdentifier = fmt .Sprintf ("%x" , md5 .Sum ([]byte (foundIdentifier )))
498+ }
499+
500+ // Overrides the key that is being ingested
501+ if len (foundIdentifierKey ) > 0 && len (foundIdentifier ) > 0 {
502+ generatedItem [foundIdentifierKey ] = foundIdentifier
477503 }
478504
479505 // Check if we have already uploaded it recently based on identifier
@@ -490,7 +516,8 @@ func autoUploadSingulOutput(ctx context.Context, orgId string, curApikey string,
490516
491517 shuffle .SetCache (ctx , cacheKey , []byte ("true" ), 60 * 60 * 24 * 3 ) // 3 days
492518
493- marshalledBody , err := json .Marshal (item )
519+ //marshalledBody, err := json.Marshal(item)
520+ marshalledBody , err := json .Marshal (generatedItem )
494521 if err != nil {
495522 log .Printf ("[ERROR] Failed marshalling item in schemaless output for label %s: %s" , value .Label , err )
496523 continue
@@ -2672,7 +2699,6 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
26722699
26732700 // Handles the actual uploading itself
26742701 if len (foundLabelSplit ) > 1 && (strings .HasPrefix (value .Label , "list_" ) || strings .HasPrefix (value .Label , "search_" )) && len (curApikey ) > 0 && len (curOrg ) > 0 {
2675-
26762702 autoUploadSingulOutput (ctx , curOrg , curApikey , curBackend , parsedTranslation , value , secondAction )
26772703 }
26782704 }
0 commit comments