Merge remote-tracking branch 'RobertD502/add_browsers_list' into develop

xavier-hernandez · xavier-hernandez · commit b06bef007660 · 2023-10-08T22:30:07.000-05:00
diff --git a/Dockerfile b/Dockerfile
@@ -23,7 +23,7 @@ RUN make install
 WORKDIR /goaccess
 RUN cp /goaccess-temp/goaccess.tar.gz .
 RUN tar --strip-components=1  -xzvf goaccess.tar.gz
-RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.20<\/span> )/" /goaccess/resources/tpls.html
+RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.21<\/span> )/" /goaccess/resources/tpls.html
 RUN sed -i "s/bottom: 190px/bottom: 260px/" /goaccess/resources/css/app.css
 RUN ./configure --enable-utf8 --enable-geoip=mmdb --with-getline
 RUN make
diff --git a/README.md b/README.md
@@ -15,9 +15,9 @@ Still in development... You might need to wait a bit if you have a large amount
 
 **Dependencies:**
 - GoAccess version: 1.7.2
-- GeoLite2-City.mmdb  (2023-08-10)
-- GeoLite2-Country.mmdb  (2023-08-10)
-- GeoLite2-ASN.mmdb  (2023-08-10)
+- GeoLite2-City.mmdb  (2023-09-09)
+- GeoLite2-Country.mmdb  (2023-09-09)
+- GeoLite2-ASN.mmdb  (2023-09-09)
 
 ---
 
@@ -57,6 +57,8 @@ services:
             - BASIC_AUTH_PASSWORD=pass #optional   
             - EXCLUDE_IPS=127.0.0.1 #optional - comma delimited 
             - LOG_TYPE=NPM #optional - more information below
+            - ENABLE_BROWSERS_LIST=True #optional - more information below
+            - CUSTOM_BROWSERS=Kuma:Uptime,TestBrowser:Crawler #optional - comma delimited, more information below
         volumes:
         - /path/to/host/nginx/logs:/opt/log
         - /path/to/host/custom:/opt/custom #optional, required if using log_type = CUSTOM
@@ -82,6 +84,8 @@ services:
             - BASIC_AUTH_PASSWORD=pass #optional   
             - EXCLUDE_IPS=127.0.0.1 #optional - comma delimited 
             - LOG_TYPE=NPM #optional - more information below
+            - ENABLE_BROWSERS_LIST=True #optional - more information below
+            - CUSTOM_BROWSERS=Kuma:Uptime,TestBrowser:Crawler #optional - comma delimited, more information below
         volumes:
         - /path/to/host/nginx/logs:/opt/log
         - /path/to/host/custom:/opt/custom #optional, required if using log_type = CUSTOM
@@ -99,6 +103,8 @@ services:
 | `-e LOG_TYPE=`         |   (Optional) By default the configuration will be set to read NPM logs. Options are: CUSTOM, NPM, NPM+R, TRAEFIK, NCSA_COMBINED. More information below.|
 | `-e LOG_TYPE_FILE_PATTERN=`         |   (Optional) Only to be used with LOG_TYPE=NCSA_COMBINED or TRAEFIK. This parameter will pass along the file type you are trying match. For example you can pass -e LOG_TYPE_FILE_PATTERN="*.log" or -e LOG_TYPE_FILE_PATTERN="access.log". The default is *.log. Please keep it simple as I have not tested this completely. Use at your own RISK! |
 | `-e LANG=zh_CN.UTF-8 -e LANGUAGE=zh_CN.UTF-8`         |   (Optional) Language localization added. GoAccess only has a few translations available. Please visit https://github.com/allinurl/goaccess/tree/master/po to see the translations available. <br/><br/>**Current Translations**<br/>de - German<br/>es - Spanish<br/>fr - French<br/>it - Italian<br/>ja - Japanese<br/>ko - Korean<br/>pt_BR - Portuguese (Brazil)<br/>ru - Russian<br/>sv - Swedish<br/>uk - English (United Kingdom)<br/>zh_CN - Chinese - Simplified|
+| `-e ENABLE_BROWSERS_LIST=True/False`         |   (Optional) Defaults to False. Set to true if you would like to enable the [goaccess browsers.list](https://github.com/allinurl/goaccess/blob/master/config/browsers.list) file.     |
+| `-e CUSTOM_BROWSERS=`         |   - (Optional) Consumes the list of provided custom browsers. This is a comma separated list containing the custom browser(s) in the format `Browser:Browser_category`.<br/>- If your custom browser is already defined in the default `browsers.list` file, it will not be added. However, the `Browser_category` can be reused.<br/><br/> CUSTOM_BROWSERS list example: `Kuma:Crawlers,TestBrowser:Crawlers,Kuma:Uptime,Discordbot:Crawlers`<br/><br/>For the example above, only `Kuma:Crawlers` and `TestBrowser:Crawlers` will be appended to the `browsers.list` file. <br/><br/>`Kuma:Uptime` is ignored as the browser `Kuma` has already been defined in `Kuma:Crawlers`. `Discordbot:Crawlers` is ignored as the browser `Discordbot` is already defined in the [default browsers.list file](https://github.com/allinurl/goaccess/blob/master/config/browsers.list)<br/><br/>Note for users using CUSTOM LOG_TYPE:<br/><br/>If your `goaccess.conf` file references a browsers.list file other than the one located in the `/goaccess-config/ directory`, the CUSTOM_BROWSERS variable will be ignored.    |
 
 
 # **Additional environment information**  
diff --git a/resources/scripts/logs/custom.sh b/resources/scripts/logs/custom.sh
@@ -29,6 +29,15 @@ function custom_init(){
         fi
         if [[ -r ${goan_config} ]]; then
             echo -e "goaccess.conf readable"
+            if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+                if grep -Fwq "browsers-file" ${goan_config}; then
+                    echo -e "\n\t\BROWSERS FILE ALREADY DEFINED IN CUSTOM GOACCESS.CONF. IGNORING CUSTOM_BROWSERS VARIABLE"
+                else
+                    echo -e "\n\tENABLING CUSTOM INSTANCE GOACCESS BROWSERS LIST"
+                    browsers_file="/goaccess-config/browsers.list"
+                    echo "browsers-file ${browsers_file}" >> ${goan_config}
+                fi
+            fi
         else
             echo -e "goaccess.conf not readable"
             #exit
@@ -61,4 +70,4 @@ function custom(){
         /goaccess/goaccess --no-global-config --config-file=${goan_config} &
     fi
 
-}
+}
diff --git a/resources/scripts/logs/ncsa_combined.sh b/resources/scripts/logs/ncsa_combined.sh
@@ -21,7 +21,7 @@ function ncsa_combined_init(){
     if [[ -f ${html_config} ]]; then
         rm ${html_config}
     fi
-    
+
     echo -n "" > ${archive_log}
     echo -n "" > ${active_log}
 }
@@ -36,9 +36,14 @@ function ncsa_combined_goaccess_config(){
     echo "date-format %d/%b/%Y" >> ${goan_config}
     #echo "log-format [%d:%t %^] %^ %s %^ %^ %m %^ %v \"%U\" [%^ %h] [%^ %b] %^\"%u\" \"%R\"" >> ${goan_config}
     echo "log-format %h %^[%d:%t %^] \"%r\" %s %b \"%R\" \"%u\"" >> ${goan_config}
-    echo "port 7890" >> ${goan_config}    
+    echo "port 7890" >> ${goan_config}
     echo "real-time-html true" >> ${goan_config}
     echo "output ${nginx_html}" >> ${goan_config}
+    if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+        echo -e "\n\tENABLING NCSA_COMBINED INSTANCE GOACCESS BROWSERS LIST"
+        browsers_file="/goaccess-config/browsers.list"
+        echo "browsers-file ${browsers_file}" >> ${goan_config}
+    fi
 }
 
 function ncsa_combined(){
@@ -58,7 +63,7 @@ function ncsa_combined(){
 
     echo -e "\n#GOAN_NCSA_COMBINED_LOG_FILES" >> ${goan_config}
     if [[ -d "${goan_log_path}" ]]; then
-        
+
         echo -e "\n\tAdding proxy logs..."
         IFS=$'\n'
 
@@ -106,7 +111,7 @@ function ncsa_combined(){
     echo -e "\nSKIP ARCHIVED LOGS"
     echo "-------------------------------"
     echo "FEATURE NOT AVAILABLE FOR NCSA_COMBINED"
-    
+
     #write out loading page
     echo "<!doctype html><html><head>" > ${nginx_html}
     echo "<title>GOAN - ${goan_version}</title>" >> ${nginx_html}
diff --git a/resources/scripts/logs/npm.sh b/resources/scripts/logs/npm.sh
@@ -38,6 +38,11 @@ function npm_goaccess_config(){
     echo "port 7890" >> ${goan_config}
     echo "real-time-html true" >> ${goan_config}
     echo "output ${nginx_html}" >> ${goan_config}
+    if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+        echo -e "\n\tENABLING NPM INSTANCE GOACCESS BROWSERS LIST"
+        browsers_file="/goaccess-config/browsers.list"
+        echo "browsers-file ${browsers_file}" >> ${goan_config}
+    fi
 }
 
 function npm(){
@@ -57,7 +62,7 @@ function npm(){
 
     echo -e "\n#GOAN_NPM_PROXY_FILES" >> ${goan_config}
     if [[ -d "${goan_log_path}" ]]; then
-        
+
         echo -e "\n\tAdding proxy logs..."
         IFS=$'\n'
         for file in $(find "${goan_log_path}" -name 'proxy*host-*_access.log' ! -name "*_error.log");
@@ -100,7 +105,7 @@ function npm(){
             goan_archive_detail_log_count=0
 
             if [ $goan_archive_log_count != 0 ]
-            then 
+            then
                 echo -e "\n\tAdding proxy archive logs..."
 
                 IFS=$'\n'
@@ -138,7 +143,7 @@ function npm(){
                 unset IFS
 
                 echo -e "\n\tAdded (${goan_archive_detail_log_count}) proxy archived logs from ${goan_log_path}..."
-                
+
             else
                 echo -e "\n\tNo archived logs found at ${goan_log_path}..."
             fi
@@ -168,4 +173,4 @@ function npm(){
     else
         /goaccess/goaccess --num-tests=0 --no-global-config --config-file=${goan_config} &
     fi
-}
+}
diff --git a/resources/scripts/logs/npm_error.sh b/resources/scripts/logs/npm_error.sh
@@ -8,7 +8,6 @@ function npm_error_init(){
 
     if [[ -f ${goan_config} ]]; then
         rm ${goan_config}
-    else
         mkdir -p "/goaccess-config/error/"
         cp /goaccess-config/goaccess.conf.bak ${goan_config}
     fi
@@ -157,3 +156,168 @@ function npm_error(){
         /goaccess/goaccess --no-global-config --config-file=${goan_config} &
     fi
 }
+
+#!/bin/bash
+function npm_error_init(){
+    goan_config="/goaccess-config/error/goaccess.conf"
+    nginx_html="/var/www/html/error/index.html"
+    html_config="/var/www/html/error/goaccess_conf.html"
+    archive_log="/goaccess-config/error/archive.log"
+    active_log="/goaccess-config/error/active.log"
+
+    if [[ -f ${goan_config} ]]; then
+        rm ${goan_config}
+    else
+        mkdir -p "/goaccess-config/error/"
+        cp /goaccess-config/goaccess.conf.bak ${goan_config}
+    fi
+    if [[ -f ${nginx_html} ]]; then
+        rm ${nginx_html}
+    else
+        mkdir -p "/var/www/html/error/"
+        touch ${nginx_html}
+    fi
+    if [[ -f ${html_config} ]]; then
+        rm ${html_config}
+    fi
+
+    echo -n "" > ${archive_log}
+    echo -n "" > ${active_log}
+}
+
+function npm_error_goaccess_config(){
+    echo -e "\n\n\n" >> ${goan_config}
+    echo "######################################" >> ${goan_config}
+    echo "# ${goan_version}" >> ${goan_config}
+    echo "# GOAN_NPM_ERROR_CONFIG" >> ${goan_config}
+    echo "######################################" >> ${goan_config}
+    echo "time-format %T" >> ${goan_config}
+    echo "date-format %Y/%m/%d" >> ${goan_config}
+    echo "log_format %d %t %^: %v, %^: %h, %^ %v %^"%r" %^" >> ${goan_config}
+    echo "port 7892" >> ${goan_config}
+    echo "real-time-html true" >> ${goan_config}
+    echo "output ${nginx_html}" >> ${goan_config}
+    if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+        echo -e "\n\tENABLING NPM ERROR INSTANCE GOACCESS BROWSERS LIST"
+        browsers_file="/goaccess-config/browsers.list"
+        echo "browsers-file ${browsers_file}" >> ${goan_config}
+    fi
+}
+
+function npm_error(){
+    npm_error_init
+    npm_error_goaccess_config
+
+    echo -e "\nLOADING ERROR LOGS"
+    echo "-------------------------------"
+
+    echo $'\n' >> ${goan_config}
+    echo "#GOAN_ERROR_LOG_FILES" >> ${goan_config}
+    echo "log-file ${archive_log}" >> ${goan_config}
+    echo "log-file ${active_log}" >> ${goan_config}
+
+    goan_log_count=0
+    goan_archive_log_count=0
+
+    echo -e "\n#GOAN_NPM_ERROR_FILES" >> ${goan_config}
+    if [[ -d "${goan_log_path}" ]]; then
+
+        echo -e "\n\tAdding error logs..."
+        IFS=$'\n'
+        for file in $(find "${goan_log_path}" -name '*_error.log');
+        do
+            if [ -f $file ]
+            then
+                if [ -r $file ] && R="Read = yes" || R="Read = No"
+                then
+                    number_of_lines=`wc -l < $file`
+                    how_many_lines_contain_warn=`grep -c "\[warn\]" $file`
+
+                    if [ $how_many_lines_contain_warn == $number_of_lines ] && [ $number_of_lines != 0 ]
+                    then
+                        echo -e "\t${file} has inconsistent log types, skipping"
+                    else
+                        echo "log-file ${file}" >> ${goan_config}
+                        goan_log_count=$((goan_log_count+1))
+                        echo -ne ' \t '
+                        echo "Filename: $file | $R"
+                    fi
+                else
+                    echo -ne ' \t '
+                    echo "Filename: $file | $R"
+                fi
+            else
+                echo -ne ' \t '
+                echo "Filename: $file | Not a file"
+            fi
+        done
+        unset IFS
+
+        echo -e "\tFound (${goan_log_count}) error logs..."
+
+        echo -e "\n\tSKIP ARCHIVED LOGS"
+        echo -e "\t-------------------------------"
+        if [[ "${SKIP_ARCHIVED_LOGS}" == "True" ]]
+        then
+            echo -e "\tTRUE"
+        else
+            echo -e "\tFALSE"
+            goan_archive_log_count=`ls -1 ${goan_log_path}/*_error.log*.gz 2> /dev/null | wc -l`
+
+            if [ $goan_archive_log_count != 0 ]
+            then
+                echo -e "\n\tAdding error archive logs..."
+
+                IFS=$'\n'
+                for file in $(find "${goan_log_path}" -name '*_error.log*.gz');
+                do
+                    if [ -f $file ]
+                    then
+                        if [ -r $file ] && R="Read = yes" || R="Read = No"
+                        then
+                            echo -ne ' \t '
+                            echo "Filename: $file | $R"
+                        else
+                            echo -ne ' \t '
+                            echo "Filename: $file | $R"
+                        fi
+                    else
+                        echo -ne ' \t '
+                        echo "Filename: $file | Not a file"
+                    fi
+                done
+                unset IFS
+
+                echo -e "\tAdded (${goan_archive_log_count}) error archived logs from ${goan_log_path}..."
+                zcat -f ${goan_log_path}/*_error.log*.gz > ${archive_log}
+                sed -e '/\[warn\]/d' -i ${archive_log}
+            else
+                echo -e "\tNo error archived logs found at ${goan_log_path}..."
+            fi
+        fi
+
+    else
+        echo "Problem loading directory (check directory or permissions)... ${goan_log_path}"
+    fi
+
+    #additonal config settings
+    exclude_ips             ${goan_config}
+    set_geoip_database      ${goan_config}
+    debug                   ${goan_config} ${html_config}
+
+    #write out loading page
+    echo "<!doctype html><html><head>" > ${nginx_html}
+    echo "<title>GOAN - ${goan_version}</title>" >> ${nginx_html}
+    echo "<meta http-equiv=\"refresh\" content=\"1\" >" >> ${nginx_html}
+    echo "<style>body {font-family: Arial, sans-serif;}</style>" >> ${nginx_html}
+    echo "</head><body><p><b>${goan_version}</b><br/><br/>loading... <br/><br/>" >> ${nginx_html}
+    echo "Logs processing: $(($goan_log_count + $goan_archive_log_count)) (might take some time depending on the number of files to parse)" >> ${nginx_html}
+    echo "<br/></p></body></html>" >> ${nginx_html}
+
+    echo -e "\nRUN NPM ERROR GOACCESS"
+    if [[ "${DEBUG}" == "True" ]]; then
+        /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
+    else
+        /goaccess/goaccess --no-global-config --config-file=${goan_config} &
+    fi
+}
diff --git a/resources/scripts/logs/npm_redirection.sh b/resources/scripts/logs/npm_redirection.sh
@@ -38,6 +38,11 @@ function npm_redirect_goaccess_config(){
     echo "port 7891" >> ${goan_config}
     echo "real-time-html true" >> ${goan_config}
     echo "output ${nginx_html}" >> ${goan_config}
+    if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+        echo -e "\n\tENABLING NPM REDIRECTION INSTANCE GOACCESS BROWSERS LIST"
+        browsers_file="/goaccess-config/browsers.list"
+        echo "browsers-file ${browsers_file}" >> ${goan_config}
+    fi
 }
 
 function npm_redirect(){
@@ -57,7 +62,7 @@ function npm_redirect(){
 
     echo -e "\n#GOAN_NPM_REDIRECT_FALLBACK_DEAD_FILES" >> ${goan_config}
     if [[ -d "${goan_log_path}" ]]; then
-        
+
         echo -e "\n\tAdding redirection/fallback/dead logs..."
         IFS=$'\n'
         for file in $(find "${goan_log_path}" \( -name 'redirection*host-*.log' -o -name 'fallback_access.log' -o -name 'dead-host*.log' \) ! -name "*_error.log*");
@@ -81,9 +86,9 @@ function npm_redirect(){
             echo -e "\tFALSE"
             goan_archive_log_count=$(find "${goan_log_path}" -type f \( -name 'redirection*host-*.log*.gz' -o -name 'fallback_access.log*.gz' -o -name 'dead-host*.log*.gz' \) ! -name "*_error.log*" | wc -l)
             goan_archive_detail_log_count=0
-            
+
             if [ $goan_archive_log_count != 0 ]
-            then 
+            then
                 echo -e "\n\tAdding redirection/fallback/dead archive logs..."
 
                 IFS=$'\n'
@@ -98,7 +103,7 @@ function npm_redirect(){
                 unset IFS
 
                 echo -e "\n\tAdded (${goan_archive_detail_log_count}) redirection/fallback/dead archived logs from ${goan_log_path}..."
-                
+
             else
                 echo -e "\tNo redirection/fallback archived logs found at ${goan_log_path}..."
             fi
diff --git a/resources/scripts/logs/traefik.sh b/resources/scripts/logs/traefik.sh
diff --git a/resources/scripts/start.sh b/resources/scripts/start.sh
