v1.1.15

- updated GoAccess to v1.7.2
- updated MaxMind databases to 4/30/2023 
- added fallback and dead log files to the redirection instance
- syncing main with dev branch
- performance changes
- updated readme file with more file types for NPM+R

Author: xavier-hernandez

Dockerfile

@@ -9,7 +9,7 @@ RUN apk add --no-cache \
 
 # download goaccess
 WORKDIR /goaccess-temp
-COPY /assests/goaccess/goaccess-1.7.1.tar.gz goaccess.tar.gz
+COPY /assests/goaccess/goaccess-1.7.2.tar.gz goaccess.tar.gz
 
 # set up goacess-debug
 WORKDIR /goaccess-debug
@@ -23,7 +23,7 @@ RUN make install
 WORKDIR /goaccess
 RUN cp /goaccess-temp/goaccess.tar.gz .
 RUN tar --strip-components=1  -xzvf goaccess.tar.gz
-RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.14<\/span> )/" /goaccess/resources/tpls.html
+RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.15<\/span> )/" /goaccess/resources/tpls.html
 RUN sed -i "s/bottom: 190px/bottom: 260px/" /goaccess/resources/css/app.css
 RUN ./configure --enable-utf8 --enable-geoip=mmdb --with-getline
 RUN make

README.md

@@ -14,10 +14,10 @@ Still in development... You might need to wait a bit if you have a large amount
 ![Alt text](https://i.ibb.co/fNj9Dcy/goaccess1.jpg "GoAccess Dashboard")
 
 **Dependencies:**
-- GoAccess version: 1.7.1
-- GeoLite2-City.mmdb  (2023-03-21)
-- GeoLite2-Country.mmdb  (2023-03-21)
-- GeoLite2-ASN.mmdb  (2023-03-21)
+- GoAccess version: 1.7.2
+- GeoLite2-City.mmdb  (2023-04-30)
+- GeoLite2-Country.mmdb  (2023-04-30)
+- GeoLite2-ASN.mmdb  (2023-04-30)
 
 ---
 
@@ -124,9 +124,13 @@ services:
   - NPM+R
     - a second instance of GOACCESS is created
     - append "/redirection" to the url to access the instance, for example http://localhost:7880/redirection/
-    - the following file(s) are read and parsed.
-      - redirection\*host-*.log*.gz
-      - redirection\*host-*.log
+    - the following file(s) are read and parsed: 
+      - redirection\*host-\*.log*.gz
+      - redirection\*host-\*.log
+      - fallback_access.log\*.gz
+      - fallback_access.log
+      - dead-host\*.log*.gz
+      - dead-host*.log
   - NPM+ALL
     - a second and third instance of GOACCESS are created
       - append "/redirection" to the url to access the instance, for example http://localhost:7880/redirection/

assests/goaccess/goaccess-1.7.1.tar.gz

@@ -1,2 +1,18 @@
 #!/bin/bash
 
+function checkFile(){
+    if [ -z "$1" ]; then
+        return 1
+    else
+        if [ -e "$file" ] && [ -r "$file" ]; then
+            echo -e "\tFile $file exists and is readable"
+            return 0
+        elif [ -e "$file_path" ]; then
+            echo -e "\tFile $file exists but is not readable"
+            return 1
+        else
+            echo -e "\tFile $file does not exist"
+            return 1
+        fi
+    fi
+}

assests/goaccess/goaccess-1.7.2.tar.gz

@@ -60,7 +60,7 @@ function npm(){
 
         echo -e "\n\tAdding proxy logs..."
         IFS=$'\n'
-        for file in $(find "${goan_log_path}" -name 'proxy*host-*.log' ! -name "*_error.log");
+        for file in $(find "${goan_log_path}" -name 'proxy*host-*_access.log' ! -name "*_error.log");
         do
             pressOn=0
 
@@ -79,14 +79,10 @@ function npm(){
             fi
 
             if [[ $pressOn == 1 ]]; then
-                if [ -e "$file" ] && [ -r "$file" ]; then
+                checkFile "$file"
+                if [ $? -eq 0 ]; then
                     echo "log-file ${file}" >> ${goan_config}
-                    goan_log_count=$((goan_log_count+1))
-                    echo -e "\tFile $file exists and is readable"
-                elif [ -e "$file_path" ]; then
-                    echo -e "\tFile $file exists but is not readable"
-                else
-                    echo -e "\tFile $file does not exist"
+                    ((goan_log_count++))
                 fi
             fi
         done
@@ -127,14 +123,10 @@ function npm(){
                     fi
 
                     if [[ $pressOn == 1 ]]; then
-                        if [ -e "$file" ] && [ -r "$file" ]; then
-                            echo -e "\tFile $file exists and is readable"
-                            ((goan_archive_detail_log_count++))
+                        checkFile "$file"
+                        if [ $? -eq 0 ]; then
                             zcat -f ${file} >> ${archive_log}
-                        elif [ -e "$file_path" ]; then
-                            echo -e "\tFile $file exists but is not readable"
-                        else
-                            echo -e "\tFile $file does not exist"
+                            ((goan_archive_detail_log_count++))
                         fi
                     fi
                 done

assests/maxmind/GeoLite2-ASN.mmdb

@@ -30,7 +30,7 @@ function npm_redirect_goaccess_config(){
     echo -e "\n\n\n" >> ${goan_config}
     echo "######################################" >> ${goan_config}
     echo "# ${goan_version}" >> ${goan_config}
-    echo "# GOAN_NPM_REDIRECTION_CONFIG" >> ${goan_config}
+    echo "# GOAN_NPM_REDIRECTION_FALLBACK_DEAD_CONFIG" >> ${goan_config}
     echo "######################################" >> ${goan_config}
     echo "time-format %T" >> ${goan_config}
     echo "date-format %d/%b/%Y" >> ${goan_config}
@@ -44,44 +44,33 @@ function npm_redirect(){
     npm_redirect_init
     npm_redirect_goaccess_config
 
-    echo -e "\nLOADING REDIRECTION LOGS"
-    echo "-------------------------------"
+    echo -e "\nLOADING REDIRECTION/FALLBACK/DEAD LOGS"
+    echo "---------------------------------------"
 
     echo $'\n' >> ${goan_config}
-    echo "#GOAN_REDIRECTION_LOG_FILES" >> ${goan_config}
+    echo "#GOAN_REDIRECTION_FALLBACK_DEAD_LOG_FILES" >> ${goan_config}
     echo "log-file ${archive_log}" >> ${goan_config}
     echo "log-file ${active_log}" >> ${goan_config}
 
     goan_log_count=0
     goan_archive_log_count=0
 
-    echo -e "\n#GOAN_NPM_REDIRECT_FILES" >> ${goan_config}
+    echo -e "\n#GOAN_NPM_REDIRECT_FALLBACK_DEAD_FILES" >> ${goan_config}
     if [[ -d "${goan_log_path}" ]]; then
 
-        echo -e "\n\tAdding redirection logs..."
+        echo -e "\n\tAdding redirection/fallback/dead logs..."
         IFS=$'\n'
-        for file in $(find "${goan_log_path}" -name 'redirection*host-*.log' ! -name "*_error.log");
+        for file in $(find "${goan_log_path}" \( -name 'redirection*host-*.log' -o -name 'fallback_access.log' -o -name 'dead-host*.log' \) ! -name "*_error.log*");
         do
-            if [ -f $file ]
-            then
-                if [ -r $file ] && R="Read = yes" || R="Read = No"
-                then
-                    echo "log-file ${file}" >> ${goan_config}
-                    goan_log_count=$((goan_log_count+1))
-                    echo -ne ' \t '
-                    echo "Filename: $file | $R"
-                else
-                    echo -ne ' \t '
-                    echo "Filename: $file | $R"
-                fi
-            else
-                echo -ne ' \t '
-                echo "Filename: $file | Not a file"
+            checkFile "$file"
+            if [ $? -eq 0 ]; then
+                echo "log-file ${file}" >> ${goan_config}
+                ((goan_log_count++))
             fi
         done
         unset IFS
 
-        echo -e "\tFound (${goan_log_count}) redirection logs..."
+        echo -e "\tFound (${goan_log_count}) redirection/fallback/dead logs..."
 
         echo -e "\n\tSKIP ARCHIVED LOGS"
         echo -e "\t-------------------------------"
@@ -90,36 +79,28 @@ function npm_redirect(){
             echo -e "\tTRUE"
         else
             echo -e "\tFALSE"
-            goan_archive_log_count=`ls -1 ${goan_log_path}/redirection*host-*.log*.gz 2> /dev/null | wc -l`
-
+            goan_archive_log_count=$(find "${goan_log_path}" -type f \( -name 'redirection*host-*.log*.gz' -o -name 'fallback_access.log*.gz' -o -name 'dead-host*.log*.gz' \) ! -name "*_error.log*" | wc -l)
+            goan_archive_detail_log_count=0
+            
             if [ $goan_archive_log_count != 0 ]
             then 
-                echo -e "\n\tAdding redirection archive logs..."
+                echo -e "\n\tAdding redirection/fallback/dead archive logs..."
 
                 IFS=$'\n'
-                for file in $(find "${goan_log_path}" -name 'redirection*host-*.log*.gz' ! -name "*_error.log");
+                for file in $(find "${goan_log_path}" \( -name 'redirection*host-*.log*.gz' -o -name 'fallback_access.log*.gz' -o -name 'dead-host*.log*.gz' \) ! -name "*_error.log*");
                 do
-                    if [ -f $file ]
-                    then
-                        if [ -r $file ] && R="Read = yes" || R="Read = No"
-                        then
-                            echo -ne ' \t '
-                            echo "Filename: $file | $R"
-                        else
-                            echo -ne ' \t '
-                            echo "Filename: $file | $R"
-                        fi
-                    else
-                        echo -ne ' \t '
-                        echo "Filename: $file | Not a file"
+                     checkFile "$file"
+                    if [ $? -eq 0 ]; then
+                        zcat -f ${file} >> ${archive_log}
+                        ((goan_archive_detail_log_count++))
                     fi
                 done
                 unset IFS
 
-                echo -e "\tAdded (${goan_archive_log_count}) redirection archived logs from ${goan_log_path}..."
-                zcat -f ${goan_log_path}/redirection*host-*.log*.gz > ${archive_log}
+                echo -e "\n\tAdded (${goan_archive_detail_log_count}) redirection/fallback/dead archived logs from ${goan_log_path}..."
+                
             else
-                echo -e "\tNo redirection archived logs found at ${goan_log_path}..."
+                echo -e "\tNo redirection/fallback archived logs found at ${goan_log_path}..."
             fi
         fi
 
@@ -141,7 +122,7 @@ function npm_redirect(){
     echo "Logs processing: $(($goan_log_count + $goan_archive_log_count)) (might take some time depending on the number of files to parse)" >> ${nginx_html}
     echo "<br/></p></body></html>" >> ${nginx_html}
 
-    echo -e "\nRUN NPM REDIRECT GOACCESS"
+    echo -e "\nRUN NPM REDIRECT/FALLBACK GOACCESS"
     if [[ "${DEBUG}" == "True" ]]; then
         /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
     else