Added environment variable to decrease verbosity of Saved: log messages

Author: davidfraser

Dockerfile

@@ -40,6 +40,7 @@ RUN echo '#!/bin/sh' > /app/entrypoint.sh && \
     echo 'echo "CITY=$CITY" >> /app/.env' >> /app/entrypoint.sh && \
     echo 'echo "ASN=$ASN" >> /app/.env' >> /app/entrypoint.sh && \
     echo 'echo "UPDATE_TIME=$UPDATE_TIME" >> /app/.env' >> /app/entrypoint.sh && \
+    echo 'echo "LOAD_LOG_FREQ=$LOAD_LOG_FREQ" >> /app/.env' >> /app/entrypoint.sh && \
     echo 'echo "DB_TYPE=$DB_TYPE" >> /app/.env' >> /app/entrypoint.sh && \
     echo '' >> /app/entrypoint.sh && \
     echo '# Add database variables if they are set' >> /app/entrypoint.sh && \
@@ -94,6 +95,7 @@ ENV COUNTRY="geo-whois-asn-country"
 ENV CITY=""
 ENV ASN="asn"
 ENV UPDATE_TIME="01:30"
+ENV LOAD_LOG_FREQ=50000
 # DB_TYPE can be mmdb, postgres, mysql, sqlite or :memory:
 ENV DB_TYPE="mmdb"
 # Database connection variables (used when DB_TYPE is set to postgres/mysql/sqlite)

README.md

@@ -87,6 +87,8 @@ If you wish to expose the system without a reverse proxy, you may wish to update
 
 `API_KEY` allows a very basic protection of the system to be applied, a header named `API-KEY` *(hyphen not underscore!)* with a matching value must be passed if this variable is populated. If left blank, the API is open.
 
+`LOAD_LOG_FREQ` allows adjusting how frequently progress in saving records will be logged.
+
 `COUNTRY`, `CITY` and `ASN` are the databases that will be loaded. **If you don't need cities or ASNs, just leave them blank.** The values / names used should mirror the directory values found in the [ip-location-db](https://github.com/sapics/ip-location-db) project:
 
 ### Allowed `COUNTRY` values

load.go

@@ -56,6 +56,14 @@ func loadCities(dataToLoad DataToLoad) {
 	version := dbQueryMaxVersion("ip_city", dataToLoad.Version) + 1
 	cities	:= []IpCity{}
 	count	:= 0;
+	lastLog	:= 0;
+	logFS	:= os.Getenv("LOAD_LOG_FREQ")
+	logFreq, err	:= strconv.Atoi(logFS)
+	if err != nil {
+		logFreq	= 100;
+	}
+
+
 
 	fmt.Println("rebuilding: ip_city ipv", dataToLoad.Version)
 	fmt.Print("\033[s") // Save the cursor position
@@ -71,8 +79,11 @@ func loadCities(dataToLoad DataToLoad) {
 
 		if len(cities) == 100 {
 			count += len(cities);
-			fmt.Print("\033[u\033[K") // Restore the cursor position and clear the line
-			fmt.Printf("Saved: %d entries\n", count)
+			if (count >= lastLog + logFreq) {
+				fmt.Print("\033[u\033[K") // Restore the cursor position and clear the line
+				fmt.Printf("Saved: %d entries\n", count)
+				lastLog = count
+			}
 			dbSaveCities(cities)
 			cities = []IpCity{}
 		}
@@ -99,6 +110,12 @@ func loadASNs(dataToLoad DataToLoad) {
 	version := dbQueryMaxVersion("ip_asn", dataToLoad.Version) + 1
 	ASNs	:= []IpASN{}
 	count	:= 0;
+	lastLog	:= 0;
+	logFS	:= os.Getenv("LOAD_LOG_FREQ")
+	logFreq, err	:= strconv.Atoi(logFS)
+	if err != nil {
+		logFreq	= 100;
+	}
 
 	fmt.Println("rebuilding: ip_asn ipv", dataToLoad.Version)
 	fmt.Print("\033[s") // Save the cursor position
@@ -112,8 +129,11 @@ func loadASNs(dataToLoad DataToLoad) {
 
 		if len(ASNs) == 100 {
 			count += len(ASNs);
-			fmt.Print("\033[u\033[K") // Restore the cursor position and clear the line
-			fmt.Printf("Saved: %d entries\n", count)
+			if (count >= lastLog + logFreq) {
+				fmt.Print("\033[u\033[K") // Restore the cursor position and clear the line
+				fmt.Printf("Saved: %d entries\n", count)
+				lastLog = count
+			}
 			dbSaveASNs(ASNs)
 			ASNs = []IpASN{}
 		}
@@ -140,6 +160,12 @@ func loadCountries(dataToLoad DataToLoad) {
 	version		:= dbQueryMaxVersion("ip_country", dataToLoad.Version) + 1
 	countries	:= []IpCountry{}
 	count		:= 0;
+	lastLog	:= 0;
+	logFS	:= os.Getenv("LOAD_LOG_FREQ")
+	logFreq, err	:= strconv.Atoi(logFS)
+	if err != nil {
+		logFreq	= 100;
+	}
 
 	fmt.Println("rebuilding: ip_country ipv", dataToLoad.Version)
 	fmt.Print("\033[s") // Save the cursor position
@@ -152,8 +178,11 @@ func loadCountries(dataToLoad DataToLoad) {
 
 		if len(countries) == 100 {
 			count += len(countries);
-			fmt.Print("\033[u\033[K") // Restore the cursor position and clear the line
-			fmt.Printf("Saved: %d entries\n", count)
+			if (count >= lastLog + logFreq) {
+				fmt.Print("\033[u\033[K") // Restore the cursor position and clear the line
+				fmt.Printf("Saved: %d entries\n", count)
+				lastLog = count
+			}
 			dbSaveCountries(countries)
 			countries = []IpCountry{}
 		}