Merge branch 'develop' into main

Author: xavier-hernandez

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.17 AS builder
+FROM alpine:3.18 AS builder
 
 RUN apk add --no-cache \
         build-base \
@@ -23,13 +23,13 @@ RUN make install
 WORKDIR /goaccess
 RUN cp /goaccess-temp/goaccess.tar.gz .
 RUN tar --strip-components=1  -xzvf goaccess.tar.gz
-RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.15<\/span> )/" /goaccess/resources/tpls.html
+RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.22<\/span> )/" /goaccess/resources/tpls.html
 RUN sed -i "s/bottom: 190px/bottom: 260px/" /goaccess/resources/css/app.css
 RUN ./configure --enable-utf8 --enable-geoip=mmdb --with-getline
 RUN make
 RUN make install
 
-FROM alpine:3.17
+FROM alpine:3.18
 RUN apk add --no-cache \
         bash \
         nginx \
@@ -67,6 +67,9 @@ ADD /resources/scripts/logs logs
 COPY /resources/scripts/start.sh start.sh
 RUN chmod +x start.sh
 
+# store archives
+RUN mkdir -p /goaccess-logs/archives
+
 VOLUME ["/opt/log"]
 VOLUME ["/opt/custom"]
 EXPOSE 7880

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Xavier
+Copyright (c) 2023 Xavier
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -15,9 +15,9 @@ Still in development... You might need to wait a bit if you have a large amount
 
 **Dependencies:**
 - GoAccess version: 1.7.2
-- GeoLite2-City.mmdb  (2023-04-30)
-- GeoLite2-Country.mmdb  (2023-04-30)
-- GeoLite2-ASN.mmdb  (2023-04-30)
+- GeoLite2-City.mmdb  (2023-10-08)
+- GeoLite2-Country.mmdb  (2023-10-08)
+- GeoLite2-ASN.mmdb  (2023-10-08)
 
 ---
 
@@ -57,6 +57,8 @@ services:
             - BASIC_AUTH_PASSWORD=pass #optional   
             - EXCLUDE_IPS=127.0.0.1 #optional - comma delimited 
             - LOG_TYPE=NPM #optional - more information below
+            - ENABLE_BROWSERS_LIST=True #optional - more information below
+            - CUSTOM_BROWSERS=Kuma:Uptime,TestBrowser:Crawler #optional - comma delimited, more information below
         volumes:
         - /path/to/host/nginx/logs:/opt/log
         - /path/to/host/custom:/opt/custom #optional, required if using log_type = CUSTOM
@@ -82,6 +84,8 @@ services:
             - BASIC_AUTH_PASSWORD=pass #optional   
             - EXCLUDE_IPS=127.0.0.1 #optional - comma delimited 
             - LOG_TYPE=NPM #optional - more information below
+            - ENABLE_BROWSERS_LIST=True #optional - more information below
+            - CUSTOM_BROWSERS=Kuma:Uptime,TestBrowser:Crawler #optional - comma delimited, more information below
         volumes:
         - /path/to/host/nginx/logs:/opt/log
         - /path/to/host/custom:/opt/custom #optional, required if using log_type = CUSTOM
@@ -90,15 +94,17 @@ services:
 | Parameter | Function |
 |-----------|----------|
 | `-e SKIP_ARCHIVED_LOGS=True/False`         |   (Optional) Defaults to False. Set to True to skip archived logs, i.e. proxy-host*.gz     |
-| `-e DEBUG=True/False`         |   (Optional) HTML version of the running goaccess.conf within the container     |
+| `-e DEBUG=True/False`         |   (Optional) Displays more information in the docker logs. This mode also checks logs for parsing errors.     |
 | `-e BASIC_AUTH=True/False`         |   (Optional) Defaults to False. Set to True to enable nginx basic authentication.  Docker container needs to stopped or restarted each time this flag is modified. This allows for the .htpasswd file to be changed accordingly.   |
 | `-e BASIC_AUTH_USERNAME=user`         |   (Optional) Requires BASIC_AUTH to bet set to True.  Username for basic authentication.     |
 | `-e BASIC_AUTH_PASSWORD=pass`         |   (Optional) Requires BASIC_AUTH to bet set to True.  Password for basic authentication.     |
 | `-e EXCLUDE_IPS=`         |   (Optional) IP Addresses or range of IPs delimited by comma refer to https://goaccess.io/man. For example: 192.168.0.1-192.168.0.100 or 127.0.0.1,192.168.0.1-192.168.0.100   |
 | `-e INCLUDE_PROXY_HOSTS=`         |   (Optional) Only consume the list of provided proxy hosts. This is a comma separated list containing the proxy host number for example "11,21". This would consume proxy-host-11_access.log* and proxy-host-21_access.log* . The host number can be found right clicking on the 3 dots on the proxy host line in the GUI.  |
 | `-e LOG_TYPE=`         |   (Optional) By default the configuration will be set to read NPM logs. Options are: CUSTOM, NPM, NPM+R, TRAEFIK, NCSA_COMBINED. More information below.|
-| `-e LOG_TYPE_FILE_PATTERN=`         |   (Optional) Only to be used with LOG_TYPE=NCSA_COMBINED. This parameter will pass along the file type you are trying match. For example you can pass -e LOG_TYPE_FILE_PATTERN="*.log" or -e LOG_TYPE_FILE_PATTERN="access.log". The default is *.log. Please keep it simple as I have not tested this completely. Use at your own RISK! |
+| `-e LOG_TYPE_FILE_PATTERN=`         |   (Optional) Only to be used with LOG_TYPE=NCSA_COMBINED or TRAEFIK. This parameter will pass along the file type you are trying match. For example you can pass -e LOG_TYPE_FILE_PATTERN="*.log" or -e LOG_TYPE_FILE_PATTERN="access.log". The default is *.log. Please keep it simple as I have not tested this completely. Use at your own RISK! |
 | `-e LANG=zh_CN.UTF-8 -e LANGUAGE=zh_CN.UTF-8`         |   (Optional) Language localization added. GoAccess only has a few translations available. Please visit https://github.com/allinurl/goaccess/tree/master/po to see the translations available. <br/><br/>**Current Translations**<br/>de - German<br/>es - Spanish<br/>fr - French<br/>it - Italian<br/>ja - Japanese<br/>ko - Korean<br/>pt_BR - Portuguese (Brazil)<br/>ru - Russian<br/>sv - Swedish<br/>uk - English (United Kingdom)<br/>zh_CN - Chinese - Simplified|
+| `-e ENABLE_BROWSERS_LIST=True/False`         |   (Optional) Defaults to False. Set to true if you would like to enable the [goaccess browsers.list](https://github.com/allinurl/goaccess/blob/master/config/browsers.list) file.     |
+| `-e CUSTOM_BROWSERS=`         |   - (Optional) Consumes the list of provided custom browsers. This is a comma separated list containing the custom browser(s) in the format `Browser:Browser_category`.<br/>- If your custom browser is already defined in the default `browsers.list` file, it will not be added. However, the `Browser_category` can be reused.<br/><br/> CUSTOM_BROWSERS list example: `Kuma:Crawlers,TestBrowser:Crawlers,Kuma:Uptime,Discordbot:Crawlers`<br/><br/>For the example above, only `Kuma:Crawlers` and `TestBrowser:Crawlers` will be appended to the `browsers.list` file. <br/><br/>`Kuma:Uptime` is ignored as the browser `Kuma` has already been defined in `Kuma:Crawlers`. `Discordbot:Crawlers` is ignored as the browser `Discordbot` is already defined in the [default browsers.list file](https://github.com/allinurl/goaccess/blob/master/config/browsers.list)<br/><br/>Note for users using CUSTOM LOG_TYPE:<br/><br/>If your `goaccess.conf` file references a browsers.list file other than the one located in the `/goaccess-config/ directory`, the CUSTOM_BROWSERS variable will be ignored.    |
 
 
 # **Additional environment information**  
@@ -147,6 +153,19 @@ services:
       - SKIP_ARCHIVED_LOGS
     - the following file(s) are read and parsed.
       - access.log
+    - You can also set the following environment variables yourself to override the defaults
+      - TIME_FORMAT
+      - DATE_FORMAT
+      - LOG_FORMAT  
+
+      ```
+      FOR EXAMPLE
+      ---------------------------------
+      TIME_FORMAT=%H:%M:%S
+      DATE_FORMAT=%Y-%m-%d
+      LOG_FORMAT='{"ClientHost": "%h", "ClientUsername": "%e", "DownstreamContentSize": "%b", "Duration": "%n", "DownstreamStatus": "%s", "RequestAddr": "%v", "RequestMethod": "%m", "RequestPath": "%U", "RequestProtocol": "%H", "StartUTC": "%dT%t.%^", "request_User-Agent": "%u"}'
+      ``````
+
   - NCSA_COMBINED
     - environment parameters that will not work and will be ignored
       - SKIP_ARCHIVED_LOGS
@@ -183,6 +202,7 @@ date-format %d/%b/%Y
 log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" %Lm"
 ```
 
+
 # **Possible/Known Issues** 
 - A lot of CPU Usage and 10000 request every second in webUI
   - https://github.com/xavier-hernandez/goaccess-for-nginxproxymanager/issues/38

assests/maxmind/GeoLite2-ASN.mmdb

@@ -4,14 +4,11 @@ function checkFile(){
     if [ -z "$1" ]; then
         return 1
     else
-        if [ -e "$file" ] && [ -r "$file" ]; then
-            echo -e "\tFile $file exists and is readable"
+        if [ -e "$1" ]; then
+            echo -e "\tFile $1 exists"
             return 0
-        elif [ -e "$file_path" ]; then
-            echo -e "\tFile $file exists but is not readable"
-            return 1
         else
-            echo -e "\tFile $file does not exist"
+            echo -e "\tFile $1 does not exist"
             return 1
         fi
     fi

assests/maxmind/GeoLite2-City.mmdb

@@ -29,6 +29,15 @@ function custom_init(){
         fi
         if [[ -r ${goan_config} ]]; then
             echo -e "goaccess.conf readable"
+            if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+                if grep -Fwq "browsers-file" ${goan_config}; then
+                    echo -e "\n\t\BROWSERS FILE ALREADY DEFINED IN CUSTOM GOACCESS.CONF. IGNORING CUSTOM_BROWSERS VARIABLE"
+                else
+                    echo -e "\n\tENABLING CUSTOM INSTANCE GOACCESS BROWSERS LIST"
+                    browsers_file="/goaccess-config/browsers.list"
+                    echo "browsers-file ${browsers_file}" >> ${goan_config}
+                fi
+            fi
         else
             echo -e "goaccess.conf not readable"
             #exit
@@ -61,4 +70,4 @@ function custom(){
         /goaccess/goaccess --no-global-config --config-file=${goan_config} &
     fi
 
-}
+}

assests/maxmind/GeoLite2-Country.mmdb

@@ -21,7 +21,7 @@ function ncsa_combined_init(){
     if [[ -f ${html_config} ]]; then
         rm ${html_config}
     fi
-    
+
     echo -n "" > ${archive_log}
     echo -n "" > ${active_log}
 }
@@ -36,9 +36,14 @@ function ncsa_combined_goaccess_config(){
     echo "date-format %d/%b/%Y" >> ${goan_config}
     #echo "log-format [%d:%t %^] %^ %s %^ %^ %m %^ %v \"%U\" [%^ %h] [%^ %b] %^\"%u\" \"%R\"" >> ${goan_config}
     echo "log-format %h %^[%d:%t %^] \"%r\" %s %b \"%R\" \"%u\"" >> ${goan_config}
-    echo "port 7890" >> ${goan_config}    
+    echo "port 7890" >> ${goan_config}
     echo "real-time-html true" >> ${goan_config}
     echo "output ${nginx_html}" >> ${goan_config}
+    if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+        echo -e "\n\tENABLING NCSA_COMBINED INSTANCE GOACCESS BROWSERS LIST"
+        browsers_file="/goaccess-config/browsers.list"
+        echo "browsers-file ${browsers_file}" >> ${goan_config}
+    fi
 }
 
 function ncsa_combined(){
@@ -58,7 +63,7 @@ function ncsa_combined(){
 
     echo -e "\n#GOAN_NCSA_COMBINED_LOG_FILES" >> ${goan_config}
     if [[ -d "${goan_log_path}" ]]; then
-        
+
         echo -e "\n\tAdding proxy logs..."
         IFS=$'\n'
 
@@ -106,7 +111,7 @@ function ncsa_combined(){
     echo -e "\nSKIP ARCHIVED LOGS"
     echo "-------------------------------"
     echo "FEATURE NOT AVAILABLE FOR NCSA_COMBINED"
-    
+
     #write out loading page
     echo "<!doctype html><html><head>" > ${nginx_html}
     echo "<title>GOAN - ${goan_version}</title>" >> ${nginx_html}

resources/scripts/funcs/internal.sh

@@ -38,6 +38,11 @@ function npm_goaccess_config(){
     echo "port 7890" >> ${goan_config}
     echo "real-time-html true" >> ${goan_config}
     echo "output ${nginx_html}" >> ${goan_config}
+    if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+        echo -e "\n\tENABLING NPM INSTANCE GOACCESS BROWSERS LIST"
+        browsers_file="/goaccess-config/browsers.list"
+        echo "browsers-file ${browsers_file}" >> ${goan_config}
+    fi
 }
 
 function npm(){
@@ -57,7 +62,7 @@ function npm(){
 
     echo -e "\n#GOAN_NPM_PROXY_FILES" >> ${goan_config}
     if [[ -d "${goan_log_path}" ]]; then
-        
+
         echo -e "\n\tAdding proxy logs..."
         IFS=$'\n'
         for file in $(find "${goan_log_path}" -name 'proxy*host-*_access.log' ! -name "*_error.log");
@@ -91,7 +96,7 @@ function npm(){
         echo -e "\tFound (${goan_log_count}) proxy logs..."
         echo -e "\n\tSKIP ARCHIVED LOGS"
         echo -e "\t-------------------------------"
-        if [[ "${SKIP_ARCHIVED_LOGS}" == "True" ]]
+        if [[ "${SKIP_ARCHIVED_LOGS}" == "True" || ${SKIP_ARCHIVED_LOGS} == true ]]
         then
             echo -e "\tTRUE"
         else
@@ -100,7 +105,7 @@ function npm(){
             goan_archive_detail_log_count=0
 
             if [ $goan_archive_log_count != 0 ]
-            then 
+            then
                 echo -e "\n\tAdding proxy archive logs..."
 
                 IFS=$'\n'
@@ -125,15 +130,20 @@ function npm(){
                     if [[ $pressOn == 1 ]]; then
                         checkFile "$file"
                         if [ $? -eq 0 ]; then
-                            zcat -f ${file} >> ${archive_log}
+                            cleanFileName="${file//.gz/}"
+                            cleanFileName="${cleanFileName//\/opt\/log/}"
+                            cleanFileName="/goaccess-logs/archives${cleanFileName}"
+
+                            zcat -f ${file} > ${cleanFileName}
+                            echo "log-file ${cleanFileName}" >> ${goan_config}
                             ((goan_archive_detail_log_count++))
                         fi
                     fi
                 done
                 unset IFS
 
                 echo -e "\n\tAdded (${goan_archive_detail_log_count}) proxy archived logs from ${goan_log_path}..."
-                
+
             else
                 echo -e "\n\tNo archived logs found at ${goan_log_path}..."
             fi
@@ -161,6 +171,6 @@ function npm(){
     if [[ "${DEBUG}" == "True" ]]; then
         /goaccess-debug/goaccess --debug-file=${goaccess_debug_file} --invalid-requests=${goaccess_invalid_file} --no-global-config --config-file=${goan_config} &
     else
-        /goaccess/goaccess --no-global-config --config-file=${goan_config} &
+        /goaccess/goaccess --num-tests=0 --no-global-config --config-file=${goan_config} &
     fi
-}
+}