Merge pull request #169 from xavier-hernandez/develop

xavier-hernandez · web-flow · commit a82f9eaa203e · 2023-11-12T21:41:26.000-06:00
v1.1.23
diff --git a/Dockerfile b/Dockerfile
@@ -9,7 +9,7 @@ RUN apk add --no-cache \
 
 # download goaccess
 WORKDIR /goaccess-temp
-COPY /assests/goaccess/goaccess-1.7.2.tar.gz goaccess.tar.gz
+COPY /assests/goaccess/goaccess-1.8.1.tar.gz goaccess.tar.gz
 
 # set up goacess-debug
 WORKDIR /goaccess-debug
@@ -23,7 +23,7 @@ RUN make install
 WORKDIR /goaccess
 RUN cp /goaccess-temp/goaccess.tar.gz .
 RUN tar --strip-components=1  -xzvf goaccess.tar.gz
-RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.22<\/span> )/" /goaccess/resources/tpls.html
+RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.23<\/span> )/" /goaccess/resources/tpls.html
 RUN sed -i "s/bottom: 190px/bottom: 260px/" /goaccess/resources/css/app.css
 RUN ./configure --enable-utf8 --enable-geoip=mmdb --with-getline
 RUN make
diff --git a/README.md b/README.md
@@ -14,10 +14,10 @@ Still in development... You might need to wait a bit if you have a large amount
 ![Alt text](https://i.ibb.co/fNj9Dcy/goaccess1.jpg "GoAccess Dashboard")
 
 **Dependencies:**
-- GoAccess version: 1.7.2
-- GeoLite2-City.mmdb  (2023-10-08)
-- GeoLite2-Country.mmdb  (2023-10-08)
-- GeoLite2-ASN.mmdb  (2023-10-08)
+- GoAccess version: 1.8.1
+- GeoLite2-City.mmdb  (2023-11-12)
+- GeoLite2-Country.mmdb  (2023-11-12)
+- GeoLite2-ASN.mmdb  (2023-11-12)
 
 ---
 
@@ -59,6 +59,8 @@ services:
             - LOG_TYPE=NPM #optional - more information below
             - ENABLE_BROWSERS_LIST=True #optional - more information below
             - CUSTOM_BROWSERS=Kuma:Uptime,TestBrowser:Crawler #optional - comma delimited, more information below
+            - HTML_REFRESH=5 #optional - Refresh the HTML report every X seconds. https://goaccess.io/man
+            - KEEP_LAST=30 #optional - Keep the last specified number of days in storage. https://goaccess.io/man
         volumes:
         - /path/to/host/nginx/logs:/opt/log
         - /path/to/host/custom:/opt/custom #optional, required if using log_type = CUSTOM
@@ -86,6 +88,8 @@ services:
             - LOG_TYPE=NPM #optional - more information below
             - ENABLE_BROWSERS_LIST=True #optional - more information below
             - CUSTOM_BROWSERS=Kuma:Uptime,TestBrowser:Crawler #optional - comma delimited, more information below
+            - HTML_REFRESH=5 #optional - Refresh the HTML report every X seconds. https://goaccess.io/man
+            - KEEP_LAST=30 #optional - Keep the last specified number of days in storage. https://goaccess.io/man
         volumes:
         - /path/to/host/nginx/logs:/opt/log
         - /path/to/host/custom:/opt/custom #optional, required if using log_type = CUSTOM
@@ -105,7 +109,8 @@ services:
 | `-e LANG=zh_CN.UTF-8 -e LANGUAGE=zh_CN.UTF-8`         |   (Optional) Language localization added. GoAccess only has a few translations available. Please visit https://github.com/allinurl/goaccess/tree/master/po to see the translations available. <br/><br/>**Current Translations**<br/>de - German<br/>es - Spanish<br/>fr - French<br/>it - Italian<br/>ja - Japanese<br/>ko - Korean<br/>pt_BR - Portuguese (Brazil)<br/>ru - Russian<br/>sv - Swedish<br/>uk - English (United Kingdom)<br/>zh_CN - Chinese - Simplified|
 | `-e ENABLE_BROWSERS_LIST=True/False`         |   (Optional) Defaults to False. Set to true if you would like to enable the [goaccess browsers.list](https://github.com/allinurl/goaccess/blob/master/config/browsers.list) file.     |
 | `-e CUSTOM_BROWSERS=`         |   - (Optional) Consumes the list of provided custom browsers. This is a comma separated list containing the custom browser(s) in the format `Browser:Browser_category`.<br/>- If your custom browser is already defined in the default `browsers.list` file, it will not be added. However, the `Browser_category` can be reused.<br/><br/> CUSTOM_BROWSERS list example: `Kuma:Crawlers,TestBrowser:Crawlers,Kuma:Uptime,Discordbot:Crawlers`<br/><br/>For the example above, only `Kuma:Crawlers` and `TestBrowser:Crawlers` will be appended to the `browsers.list` file. <br/><br/>`Kuma:Uptime` is ignored as the browser `Kuma` has already been defined in `Kuma:Crawlers`. `Discordbot:Crawlers` is ignored as the browser `Discordbot` is already defined in the [default browsers.list file](https://github.com/allinurl/goaccess/blob/master/config/browsers.list)<br/><br/>Note for users using CUSTOM LOG_TYPE:<br/><br/>If your `goaccess.conf` file references a browsers.list file other than the one located in the `/goaccess-config/ directory`, the CUSTOM_BROWSERS variable will be ignored.    |
-
+| `-e HTML_REFRESH=`         |   (Optional) Refresh the HTML report every X seconds. https://goaccess.io/man |
+| `-e KEEP_LAST=`         |   (Optional) Keep the last specified number of days in storage. https://goaccess.io/man|
 
 # **Additional environment information**  
 ` -e LOG_TYPE=`  
diff --git a/assests/goaccess/goaccess-1.7.2.tar.gz b/assests/goaccess/goaccess-1.7.2.tar.gz
diff --git a/assests/goaccess/goaccess-1.8.1.tar.gz b/assests/goaccess/goaccess-1.8.1.tar.gz
diff --git a/assests/maxmind/GeoLite2-ASN.mmdb b/assests/maxmind/GeoLite2-ASN.mmdb
diff --git a/assests/maxmind/GeoLite2-City.mmdb b/assests/maxmind/GeoLite2-City.mmdb
diff --git a/assests/maxmind/GeoLite2-Country.mmdb b/assests/maxmind/GeoLite2-Country.mmdb
diff --git a/resources/scripts/funcs/environment.sh b/resources/scripts/funcs/environment.sh
@@ -74,4 +74,30 @@ function set_geoip_database() {
     echo "geoip-database /goaccess-config/GeoLite2-City.mmdb" >> ${1}
     echo "geoip-database /goaccess-config/GeoLite2-ASN.mmdb" >> ${1}
     echo "geoip-database /goaccess-config/GeoLite2-Country.mmdb" >> ${1}
+}
+
+function runGoAccess(){
+    goacess_options="--num-tests=0 --no-global-config"
+
+    if [ -n "$HTML_REFRESH" ]; then
+        if is_integer "$HTML_REFRESH"; then
+            goacess_options="$goacess_options --html-refresh=${HTML_REFRESH}"
+        else
+            echo "Error: HTML_REFRESH does not contain an integer, ignoring".
+        fi
+    fi
+
+    if [ -n "$KEEP_LAST" ]; then
+        if is_integer "$KEEP_LAST"; then
+            goacess_options="$goacess_options --keep-last=${KEEP_LAST}"
+        else
+            echo "Error: KEEP_LAST does not contain an integer, ignoring".
+        fi
+    fi
+
+    if [[ "${DEBUG}" == "True" ]]; then
+        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
+    else
+        /goaccess/goaccess ${goacess_options} --config-file=${goan_config} &
+    fi
 }
diff --git a/resources/scripts/funcs/internal.sh b/resources/scripts/funcs/internal.sh
@@ -12,4 +12,12 @@ function checkFile(){
             return 1
         fi
     fi
+}
+
+function is_integer() {
+    if [[ $1 =~ ^[0-9]+$ ]]; then
+        return 0  # It's an integer
+    else
+        return 1  # It's not an integer
+    fi
 }
diff --git a/resources/scripts/logs/custom.sh b/resources/scripts/logs/custom.sh
@@ -64,10 +64,5 @@ function custom(){
     echo "<br/></p></body></html>" >> ${nginx_html}
 
     echo -e "\nRUN CUSTOM GOACCESS"
-    if [[ "${DEBUG}" == "True" ]]; then
-        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
-    else
-        /goaccess/goaccess --no-global-config --config-file=${goan_config} &
-    fi
-
+    runGoAccess
 }
diff --git a/resources/scripts/logs/ncsa_combined.sh b/resources/scripts/logs/ncsa_combined.sh
@@ -122,9 +122,5 @@ function ncsa_combined(){
     echo "<br/></p></body></html>" >> ${nginx_html}
 
     echo -e "\nRUN NCSA_COMBINED GOACCESS"
-    if [[ "${DEBUG}" == "True" ]]; then
-        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
-    else
-        /goaccess/goaccess --no-global-config --config-file=${goan_config} &
-    fi
+    runGoAccess
 }
diff --git a/resources/scripts/logs/npm.sh b/resources/scripts/logs/npm.sh
@@ -168,9 +168,5 @@ function npm(){
     echo "<br/></p></body></html>" >> ${nginx_html}
 
     echo -e "\nRUN NPM GOACCESS"
-    if [[ "${DEBUG}" == "True" ]]; then
-        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
-    else
-        /goaccess/goaccess --num-tests=0 --no-global-config --config-file=${goan_config} &
-    fi
+    runGoAccess
 }
diff --git a/resources/scripts/logs/npm_error.sh b/resources/scripts/logs/npm_error.sh
@@ -1,162 +1,3 @@
-#!/bin/bash
-function npm_error_init(){
-    goan_config="/goaccess-config/error/goaccess.conf"
-    nginx_html="/var/www/html/error/index.html"
-    html_config="/var/www/html/error/goaccess_conf.html"
-    archive_log="/goaccess-config/error/archive.log"
-    active_log="/goaccess-config/error/active.log"
-
-    if [[ -f ${goan_config} ]]; then
-        rm ${goan_config}
-        mkdir -p "/goaccess-config/error/"
-        cp /goaccess-config/goaccess.conf.bak ${goan_config}
-    fi
-    if [[ -f ${nginx_html} ]]; then
-        rm ${nginx_html}
-    else
-        mkdir -p "/var/www/html/error/"
-        touch ${nginx_html}
-    fi
-    if [[ -f ${html_config} ]]; then
-        rm ${html_config}
-    fi
-
-    echo -n "" > ${archive_log}
-    echo -n "" > ${active_log}
-}
-
-function npm_error_goaccess_config(){
-    echo -e "\n\n\n" >> ${goan_config}
-    echo "######################################" >> ${goan_config}
-    echo "# ${goan_version}" >> ${goan_config}
-    echo "# GOAN_NPM_ERROR_CONFIG" >> ${goan_config}
-    echo "######################################" >> ${goan_config}
-    echo "time-format %T" >> ${goan_config}
-    echo "date-format %Y/%m/%d" >> ${goan_config}
-    echo "log_format %d %t %^: %v, %^: %h, %^ %v %^"%r" %^" >> ${goan_config}
-    echo "port 7892" >> ${goan_config}
-    echo "real-time-html true" >> ${goan_config}
-    echo "output ${nginx_html}" >> ${goan_config}
-}
-
-function npm_error(){
-    npm_error_init
-    npm_error_goaccess_config
-
-    echo -e "\nLOADING ERROR LOGS"
-    echo "-------------------------------"
-
-    echo $'\n' >> ${goan_config}
-    echo "#GOAN_ERROR_LOG_FILES" >> ${goan_config}
-    echo "log-file ${archive_log}" >> ${goan_config}
-    echo "log-file ${active_log}" >> ${goan_config}
-
-    goan_log_count=0
-    goan_archive_log_count=0
-
-    echo -e "\n#GOAN_NPM_ERROR_FILES" >> ${goan_config}
-    if [[ -d "${goan_log_path}" ]]; then
-        
-        echo -e "\n\tAdding error logs..."
-        IFS=$'\n'
-        for file in $(find "${goan_log_path}" -name '*_error.log');
-        do
-            if [ -f $file ]
-            then
-                if [ -r $file ] && R="Read = yes" || R="Read = No"
-                then
-                    number_of_lines=`wc -l < $file`
-                    how_many_lines_contain_warn=`grep -c "\[warn\]" $file`
-                    
-                    if [ $how_many_lines_contain_warn == $number_of_lines ] && [ $number_of_lines != 0 ]
-                    then
-                        echo -e "\t${file} has inconsistent log types, skipping"
-                    else
-                        echo "log-file ${file}" >> ${goan_config}
-                        goan_log_count=$((goan_log_count+1))
-                        echo -ne ' \t '
-                        echo "Filename: $file | $R"
-                    fi
-                else
-                    echo -ne ' \t '
-                    echo "Filename: $file | $R"
-                fi
-            else
-                echo -ne ' \t '
-                echo "Filename: $file | Not a file"
-            fi
-        done
-        unset IFS
-
-        echo -e "\tFound (${goan_log_count}) error logs..."
-
-        echo -e "\n\tSKIP ARCHIVED LOGS"
-        echo -e "\t-------------------------------"
-        if [[ "${SKIP_ARCHIVED_LOGS}" == "True" ]]
-        then
-            echo -e "\tTRUE"
-        else
-            echo -e "\tFALSE"
-            goan_archive_log_count=`ls -1 ${goan_log_path}/*_error.log*.gz 2> /dev/null | wc -l`
-
-            if [ $goan_archive_log_count != 0 ]
-            then 
-                echo -e "\n\tAdding error archive logs..."
-
-                IFS=$'\n'
-                for file in $(find "${goan_log_path}" -name '*_error.log*.gz');
-                do
-                    if [ -f $file ]
-                    then
-                        if [ -r $file ] && R="Read = yes" || R="Read = No"
-                        then
-                            echo -ne ' \t '
-                            echo "Filename: $file | $R"
-                        else
-                            echo -ne ' \t '
-                            echo "Filename: $file | $R"
-                        fi
-                    else
-                        echo -ne ' \t '
-                        echo "Filename: $file | Not a file"
-                    fi
-                done
-                unset IFS
-
-                echo -e "\tAdded (${goan_archive_log_count}) error archived logs from ${goan_log_path}..."
-                zcat -f ${goan_log_path}/*_error.log*.gz > ${archive_log}
-                sed -e '/\[warn\]/d' -i ${archive_log}
-            else
-                echo -e "\tNo error archived logs found at ${goan_log_path}..."
-            fi
-        fi
-
-    else
-        echo "Problem loading directory (check directory or permissions)... ${goan_log_path}"
-    fi
-
-    #additonal config settings
-    exclude_ips             ${goan_config}
-    set_geoip_database      ${goan_config}
-    debug                   ${goan_config} ${html_config}
-
-    #write out loading page
-    echo "<!doctype html><html><head>" > ${nginx_html}
-    echo "<title>GOAN - ${goan_version}</title>" >> ${nginx_html}
-    echo "<meta http-equiv=\"refresh\" content=\"1\" >" >> ${nginx_html}
-    echo "<style>body {font-family: Arial, sans-serif;}</style>" >> ${nginx_html}
-    echo "</head><body><p><b>${goan_version}</b><br/><br/>loading... <br/><br/>" >> ${nginx_html}
-    echo "Logs processing: $(($goan_log_count + $goan_archive_log_count)) (might take some time depending on the number of files to parse)" >> ${nginx_html}
-    echo "<br/></p></body></html>" >> ${nginx_html}
-
-    echo -e "\nRUN NPM ERROR GOACCESS"
-    if [[ "${DEBUG}" == "True" ]]; then
-        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
-    else
-        /goaccess/goaccess --no-global-config --config-file=${goan_config} &
-    fi
-}
-
 #!/bin/bash
 function npm_error_init(){
     goan_config="/goaccess-config/error/goaccess.conf"
@@ -315,9 +156,5 @@ function npm_error(){
     echo "<br/></p></body></html>" >> ${nginx_html}
 
     echo -e "\nRUN NPM ERROR GOACCESS"
-    if [[ "${DEBUG}" == "True" ]]; then
-        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
-    else
-        /goaccess/goaccess --no-global-config --config-file=${goan_config} &
-    fi
+    runGoAccess
 }
diff --git a/resources/scripts/logs/npm_redirection.sh b/resources/scripts/logs/npm_redirection.sh
@@ -128,9 +128,5 @@ function npm_redirect(){
     echo "<br/></p></body></html>" >> ${nginx_html}
 
     echo -e "\nRUN NPM REDIRECT/FALLBACK GOACCESS"
-    if [[ "${DEBUG}" == "True" ]]; then
-        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
-    else
-        /goaccess/goaccess --no-global-config --config-file=${goan_config} &
-    fi
+    runGoAccess
 }
diff --git a/resources/scripts/logs/traefik.sh b/resources/scripts/logs/traefik.sh
@@ -34,7 +34,8 @@ function traefik_goaccess_config(){
     echo "######################################" >> ${goan_config}
     echo "time-format ${TIME_FORMAT:-%T}" >> ${goan_config}
     echo "date-format ${DATE_FORMAT:-%d/%b/%Y}" >> ${goan_config}
-    echo "log-format ${LOG_FORMAT:-%h %^[%d:%t %^] \"%r\" %s %b \"%R\" \"%u\" %Lm}" >> ${goan_config}
+    #echo "log-format ${LOG_FORMAT:-%h %^[%d:%t %^] \"%r\" %s %b \"%R\" \"%u\" %Lm}" >> ${goan_config}
+    echo "log-format ${LOG_FORMAT:-%h %^ %e [%d:%t %^] \"%r\" %s %b \"%R\" \"%u\" %^ \"%v\" %^ %Lms}" >> ${goan_config}
     echo "port 7890" >> ${goan_config}
     echo "real-time-html true" >> ${goan_config}
     echo "output ${nginx_html}" >> ${goan_config}
@@ -123,9 +124,5 @@ function traefik(){
     echo "<br/></p></body></html>" >> ${nginx_html}
 
     echo -e "\nRUN TRAEFIK GOACCESS"
-    if [[ "${DEBUG}" == "True" ]]; then
-        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
-    else
-        /goaccess/goaccess --num-tests=0 --no-global-config --config-file=${goan_config} &
-    fi
+    runGoAccess
 }
diff --git a/resources/scripts/start.sh b/resources/scripts/start.sh
@@ -8,7 +8,7 @@ source $(dirname "$0")/logs/traefik.sh
 source $(dirname "$0")/logs/custom.sh
 source $(dirname "$0")/logs/ncsa_combined.sh
 
-goan_version="GOAN v1.1.22"
+goan_version="GOAN v1.1.23"
 goan_log_path="/opt/log"
 
 goaccess_ping_interval=15
