Merge pull request #7 from GreyNoise-Intelligence/use-community-endpoint

Use community endpoint, cleanup auth log, fix stats script

Author: Obsecurus

Makefile

@@ -1,4 +1,4 @@
--include env
+-include .env
 
 CONTAINER := gn-fluentbit
 WORKDIR := /app

README.md

@@ -74,7 +74,7 @@ This example watches reads a log file in and watches for new lines.
 [`conf/rewrite.conf`](https://github.com/GreyNoise-Intelligence/greynoise-fluentbit-lua/tree/main/conf/rewrite.conf)
 
 This is the same as #2 except this leverages [rewrite_tag](https://docs.fluentbit.io/manual/pipeline/filters/rewrite-tag) filter to drop records.
-This config drops invalid IPv4 records, bogon address space, GreyNoise RIOT records, and GreyNoise Quick records.
+This config drops invalid IPv4 records, bogon address space, GreyNoise RIOT records, and GreyNoise Noise records.
 1. Run `make run-rewrite`
 1. Run `make stats` in another terminal to see metrics (note the drop rates)
 

conf/parsers.conf

@@ -1,7 +1,10 @@
 [PARSER]
     Name        regex
     Format      regex
-    Regex       /(?<host>(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))/
+    Regex       /^(?<original_time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+    Time_Key    original_time
+    Time_Keep   On
+    Time_Format %b %d %H:%M:%S
 
 [PARSER]
     Name        dummy

conf/rewrite.conf

@@ -23,8 +23,8 @@
 [FILTER]
     Name          rewrite_tag
     Match         rewrite.*
-    Rule          $gn_quick ^(true)$ from.rewrite.gn_quick false
-    Emitter_Name  gn_quick
+    Rule          $gn_noise ^(true)$ from.rewrite.gn_noise false
+    Emitter_Name  gn_noise
 
 [FILTER]
     Name          rewrite_tag
@@ -44,8 +44,17 @@
     Rule          $gn_invalid ^(true)$ from.rewrite.gn_invalid false
     Emitter_Name  gn_invalid
 
+[FILTER]
+    Name record_modifier
+    Match rewrite.*
+    Remove_key gn_noise
+    Remove_key gn_riot
+    Remove_key gn_bogon
+    Remove_key gn_invalid
+
 [OUTPUT]
     Name file
-    Format plain
+    Format template
+    Template {original_time} {host} {ident}[{pid}]: {message}
     Match rewrite.*
     Path /app/output

examples/auth.log

@@ -22,6 +22,7 @@ local cache = lru.new(cache_size)
 
 local useragent = string.format('GreyNoiseFluentBit/%s', greynoise._version)
 local auth = requests.HTTPBasicAuth('none', gn_api_key)
+
 local headers = {['User-Agent'] = useragent, ['Accept'] = 'application/json'}
 
 local function has_value(tab, val)
@@ -41,7 +42,7 @@ end
 -- @table record
 -- @return table
 local function convert_record_bools(record)
-  local gn_keys = {"gn_quick", "gn_riot", "gn_bogon", "gn_invalid"}
+  local gn_keys = {"gn_noise", "gn_riot", "gn_bogon", "gn_invalid"}
   for k, v in pairs(record) do
     if has_value(gn_keys, k) then
       if (v == true) then record[k] = "true" end
@@ -100,60 +101,38 @@ local function check_ip(record, ip)
   return new_record
 end
 
--- Lookup a source_ip against `/v2/riot/` endpoint
+-- Lookup a source_ip against `/v3/community` endpoint
 --
 -- @string ip
--- @return boolean
-local function gn_riot_check(ip)
-  local url = string.format('https://api.greynoise.io/v2/riot/%s', ip)
+-- @return boolean, boolean
+local function gn_community_lookup(ip)
+  local url = string.format('https://api.greynoise.io/v3/community/%s', ip)
   local response = requests.get {url, headers = headers, auth = auth}
   if (not response) then
-    log.warn('no response from /v2/riot/ endpoint')
-    return nil
+    log.warn('no response from /v3/community endpoint')
+    return nil, nil
   end
   if response.status_code == 200 then
     local body, error = response.json()
     if error ~= nil then
       log.warn('%v', error)
-      return nil
+      return nil, nil
     end
-    if body.riot == true then return true end
-    return false
-  elseif response.status_code == 404 then
-    -- RIOT uses a soft 404 to represent that resource is not in RIOT.
-    -- This interface differs from the `v2/noise/quick` endpoint that returns 200 for all requested IPs.
-    return false
+    return body.noise, body.riot
   end
-
-  log.warn(string.format('Received %d status code from %s',
-                         response.status_code, url))
-  return nil
-end
-
--- Lookup a source_ip against `/v2/noise/quick/` endpoint
---
--- @string ip
--- @return boolean
-local function gn_quick_check(ip)
-  local url = string.format('https://api.greynoise.io/v2/noise/quick/%s', ip)
-  local response = requests.get {url, headers = headers, auth = auth}
-  if (not response) then
-    log.warn('no response from /v2/noise/quick/ endpoint')
-    return nil
-  end
-  if response.status_code == 200 then
-    local body, error = response.json()
-    if error ~= nil then
-      log.warn('%v', error)
-      return nil
-    end
-    if body.noise == true then return true end
-    return false
+  if response.status_code == 404 then
+      local body, error = response.json()
+      if error ~= nil then
+        log.warn('%v', error)
+        return nil, nil
+      end
+      log.debug(string.format('%s, %s',url, body.message))
+      return false, false
   end
 
   log.warn(string.format('Received %d status code from %s',
                          response.status_code, url))
-  return nil
+  return nil, nil
 end
 
 -- Main filter handler
@@ -163,18 +142,19 @@ end
 -- @table  record
 -- @return number, number, table
 function gn_filter(_, timestamp, record)
-  local ip = record[ip_field]
+  -- Extract IPv4 from message
+  local ip = record[ip_field]:match("(%d+%.%d+%.%d+%.%d+)")
   local new_record = record
   new_record.gn_riot = nil
-  new_record.gn_quick = nil
+  new_record.gn_noise = nil
   new_record.gn_invalid = nil
   new_record.gn_bogon = nil
   if ip then
     local cache_record = cache:get(ip)
     if cache_record then
       log.debug(string.format('cache hit: %s', ip))
       new_record.gn_riot = cache_record['r']
-      new_record.gn_quick = cache_record['q']
+      new_record.gn_noise = cache_record['q']
       new_record.gn_invalid = cache_record['i']
       new_record.gn_bogon = cache_record['b']
       local final_record = convert_record_bools(new_record)
@@ -184,11 +164,13 @@ function gn_filter(_, timestamp, record)
       log.debug(string.format('lookup: %s', ip))
       if (not validated_record.gn_invalid and not validated_record.gn_bogon) then
         -- Make GN API calls for valid non-bogon IPv4 records
-        validated_record.gn_riot = gn_riot_check(ip)
-        validated_record.gn_quick = gn_quick_check(ip)
+        validated_record.gn_noise, validated_record.gn_riot = gn_community_lookup(ip)
+        if not validated_record.gn_noise == nil or not validated_record.gn_riot == nil then
+          return -1, timestamp, record
+        end
         cache:set(ip, {
           r = validated_record.gn_riot,
-          q = validated_record.gn_quick,
+          q = validated_record.gn_noise,
           i = validated_record.gn_invalid,
           b = validated_record.gn_bogon
         })

greynoise/src/greynoise.lua

@@ -11,39 +11,25 @@ then
     exit 1;
 fi
 
-process_raw_log () {
-    log=$1
-    echo "## Stats for: $log"
-    total=$(cat $log | wc -l)
-    quick=$(jq -c '. | select(.gn_quick == true)' $log | wc -l)
-    riot=$(jq -c '. | select(.gn_riot == true)' $log | wc -l)
-    bogon=$(jq -c '. | select(.gn_bogon == true)' $log | wc -l)
-    invalid=$(jq -c '. | select(.gn_invalid == true)' $log | wc -l)
-    noise_percent=$(echo "scale=4; $quick/$total" | bc -l)
-    riot_percent=$(echo "scale=4; $riot/$total" | bc -l)
-    bogon_percent=$(echo "scale=4; $bogon/$total" | bc -l)
-    invalid_percent=$(echo "scale=4; $invalid/$total" | bc -l)
-
-    echo "--Line Counts--"
-    echo "Total:    $total"
-    echo "Noise:    $quick"
-    echo "RIOT:     $riot"
-    echo "Bogon:    $bogon"
-    echo "Invalid:  $invalid"
-    echo "--Percentages--"
-    echo "Noise:    $noise_percent"
-    echo "RIOT:     $riot_percent"
-    echo "Bogon:    $bogon_percent"
-    echo "Invalid:  $invalid_percent"
-    printf "\n"
-}
-
 get_fb_stats () {
     curl -s http://127.0.0.1:2020/api/v1/metrics | jq '.input'
 }
 
+res=$(get_fb_stats)
+records=$(echo $res | jq '.rewrite_input.records')
+noise=$(echo $res | jq '.gn_noise.records')
+riot=$(echo $res | jq '.gn_riot.records')
+bogon=$(echo $res | jq '.gn_bogon.records')
+invalid=$(echo $res | jq '.gn_invalid.records')
+
+total_invalid=$(echo "scale=4; $bogon + $invalid" | bc -l)
+total_noise=$(echo "scale=4; $noise + $riot" | bc -l)
+noise_percentage=$(echo "scale=2; $total_noise/$records" | bc -l | cut -c 2-)
 
-get_fb_stats
-# for log in output/*; do
-#     process_log $log
-# done
+echo "GreyNoise Fluentbit Filter Stats"
+echo "================================"
+echo ""
+echo "Total Records:       ${records}"
+echo "All Noise Records:   ${total_noise} (Noise, RIOT)"
+echo "All Invalid Records: ${total_invalid}    (Bogon, Invalid)"
+echo "Noise Percentage:    ${noise_percentage}%"