From 30b15f89d145c0e4742aa3c73d4b7b3d170991f2 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Wed, 6 Oct 2021 13:33:06 -0300
Subject: [PATCH 01/77] Implemented data collecting for connectivity pings.

---
 files/usr/share/check_cable_wan.sh            |   2 +-
 files/usr/share/data_collecting.sh            |  67 +++++++++--
 .../functions/data_collecting_functions.sh    |   4 +-
 .../usr/share/functions/network_functions.sh  | 111 +++++++++++++++++-
 4 files changed, 172 insertions(+), 12 deletions(-)

diff --git a/files/usr/share/check_cable_wan.sh b/files/usr/share/check_cable_wan.sh
index 755d732c..95a5bc8a 100755
--- a/files/usr/share/check_cable_wan.sh
+++ b/files/usr/share/check_cable_wan.sh
@@ -28,7 +28,7 @@ write_access_start_time 0
 while true
 do
 	# We have layer 2 connectivity, now check external access
-	if [ ! "$(check_connectivity_internet)" -eq 0 ]
+	if [ ! "$(check_connectivity_internet "" "collect")" -eq 0 ]
 	then
 		# No external access
 		log "CHECK_WAN" "No external access..."
diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 210b7b6e..5088f1eb 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -1,6 +1,5 @@
 #!/bin/sh
 . /usr/share/functions/device_functions.sh
-. /usr/share/functions/common_functions.sh
 . /usr/share/flashman_init.conf
 
 # directory where all data related to data collecting will be stored.
@@ -17,8 +16,6 @@ compressedDataDir="${dataCollectingDir}/compressed"
 collect_QoE_Monitor_data() {
 	# echo collecting data and writing to file.
 
-	# getting current unix time in seconds.
-	local timestamp=$(date +%s)
 	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
 	# ping return value.
@@ -72,7 +69,7 @@ collect_QoE_Monitor_data() {
 	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
 	# loss=${loss##* } # removes everything before first space.
 
-	local string="$timestamp $loss $transmitted $rx $tx" # data to be sent.
+	local string="loss $loss $transmitted $rx $tx" # data to be sent.
 
 	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
@@ -83,11 +80,11 @@ collect_QoE_Monitor_data() {
 		local firstLine=true
 		# for each ping line.
 		while read line; do
-			# removes 'time=' part if it exists.
+			# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
 			reached=${line%time=*}
 			# if "time=" has actually been removed, it means that line 
 			# contains it, which also means the icmp request was fulfilled.
-			# if line doesn't contain 'time=', skip this line.
+			# if line doesn't contain 'time=', we skip this line.
 			[ ${#reached} -lt ${#line} ] || continue
 
 			 # from the whole line, removes everything until, and including, "icmp_req=".
@@ -114,7 +111,46 @@ collect_QoE_Monitor_data() {
 
 	# printf "string is: '%s'\n" "$string"
 	# appending string to file.
-	echo "$string" >> "$rawDataFile";
+	echo "loss $string"
+}
+
+collect_connectivity_pings() {
+	# file where pings are stored.
+	local pingsFile="${data_collecting_dir}/pings"
+	
+	# content from pings file will be assign to this variable.
+	local pings
+	# using a lock file, in writing mode, to block code while accessing the pings file.
+	# this is to prevent writing conflict to the script that writes new values to the pings file.
+	{
+	flock -x 9
+	pings=$(cat "$pingsFile")
+	rm "$pingsFile" > /dev/null 2>&1
+	# "${pingsFile}lock" is also used by the script that writes new values to the file.
+	} 9>"${pingsFile}lock"
+
+	# counter and total for pings. we later divide them to get an average.
+	local count=0
+	local average=0
+	for rtt in $pings; do
+		count=$(($count + 1))
+		# removing floating point. which means multiplying by 1000 as 'ping' times always have 3 floating point digits.
+		# we are using integers only in our math because flashboxes don't have floating point numbers in ash.
+		average=$(($average + ${rtt//.}))
+	done
+	# using integer division isn't perfect but it's precise enough for us.
+	average=$(($average / $count))
+	# dividing back by 1000 by just manipulating the string. Putting a dot before the last 3 digits.
+	local length=${#average}
+	average="${average:0:$(($l-3))}.${average:$(($l-3))}"
+
+	# this measures output.
+	echo "pings $average $count"
+}
+
+collect_wifi_devices() {
+	local devices="$(iwinfo $(get_root_ifname 0) assoclist)"
+	echo "$devices"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -217,7 +253,21 @@ removeOldFiles() {
 # too big, compress it and move it to a directory of compressed files. If 
 # directory of compressed files grows too big delete oldest compressed files.
 collectData() {
-	collect_QoE_Monitor_data
+	# getting current unix time in seconds.
+	local timestamp=$(date +%s)
+
+	# string to be written to file.
+	local str=""
+
+	local out
+	out=$(collect_QoE_Monitor_data)
+	[ "$out" != "" ] && str="${str}${out}|"
+	out=$(collect_connectivity_pings)
+	[ "$out" != "" ] && str="${str}${out}|"
+	out=$(collect_wifi_devices)
+	[ "$out" != "" ] && str="${str}${out}|"
+
+	[ "$str" != "" ] && echo "$timestamp:$str" >> "$rawDataFile";
 
 	# $(zipFile) returns 0 only if any amount of files has been compressed 
 	# and, consequently, moved to the directory of compressed files. So
@@ -452,6 +502,7 @@ cleanFiles() {
 	rm "${dataCollectingDir}/serverState" 2> /dev/null
 	# rm "${dataCollectingDir}/backoffCounter" 2> /dev/null
 	rm "${rawDataFile}.gz" 2> /dev/null
+	flock "${dataCollectingDir}pingslock" rm "${dataCollectingDir}/pings"
 }
 
 # collects and sends data forever.
diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index b69f14a0..4cf8b89d 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -66,8 +66,8 @@ set_data_collecting_parameters() {
 	# "false" boolean value is translated as string "0" by jshn.sh
 	if [ "$data_collecting_is_active" = "1" ] && [ "$data_collecting_ping_fqdn" != "" ]; then
 		# reload won't restart data collecting service if it's already running.
-		/etc/init.d/data_collecting reload
+		service data_collecting reload
 	elif [ "$data_collecting_is_active" != "1" ]; then
-		/etc/init.d/data_collecting stop
+		service data_collecting stop > /dev/null 2>&1
 	fi
 }
diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index fec41e93..86762f42 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -120,6 +120,28 @@ check_connectivity_flashman() {
 	check_connectivity_internet "$_addrs"
 }
 
+# check_connectivity_internet() {
+# 	_addrs="www.google.com.br"$'\n'"www.facebook.com"$'\n'"www.globo.com"
+# 	if [ "$1" != "" ]
+# 	then
+# 		_addrs="$1"
+# 	fi
+# 	for _addr in $_addrs
+# 	do
+# 		if ping -q -c 1 -w 2 "$_addr"  > /dev/null 2>&1
+# 		then
+# 			# true
+# 			echo 0
+# 			return
+# 		fi
+# 	done
+# 	# No successfull pings
+
+# 	# false
+# 	echo 1
+# 	return
+# }
+
 check_connectivity_internet() {
 	_addrs="www.google.com.br"$'\n'"www.facebook.com"$'\n'"www.globo.com"
 	if [ "$1" != "" ]
@@ -128,7 +150,8 @@ check_connectivity_internet() {
 	fi
 	for _addr in $_addrs
 	do
-		if ping -q -c 1 -w 2 "$_addr"  > /dev/null 2>&1
+		# second argument is used to allow collecting data in this call.
+		if ping_maybe_collect "$_addr" "$2"
 		then
 			# true
 			echo 0
@@ -142,6 +165,92 @@ check_connectivity_internet() {
 	return
 }
 
+ping_maybe_collect() {
+	local address="$1" should_collect="$2"
+
+	local pingResult
+	if pingResult=$(ping -q -c 1 -w 2 "$address"); then
+		# won't collect ping if data collecting service is not running.
+		if [ "$should_collect" == "collect" ] && service data_collecting running; then
+			local data_collecting_dir="/tmp/data_collecting"
+			# creates data_collecting temporary directory if it doesn't exist.
+			mkdir -p "$data_collecting_dir"
+
+			# Removes everything before, and including, 'mdev = '.
+			local rtt=${pingResult##*mdev = }
+			# removes everything after, and including, first forward slash.
+			rtt=${rtt%%/*}
+
+			# file where pings are stored.
+			local pingsFile="${data_collecting_dir}/pings"
+			# appending $rtt in a new line in pings file.
+			flock "$pingsFile" # lock.
+			echo "$rtt" >> "$pingsFile"
+			flock "$pingsFile" # unlock.
+		fi
+		return 0
+	fi
+	return 1
+}
+
+check_connectivity_internet_and_collect_ping() {
+	for _addr in "www.google.com.br"$'\n'"www.facebook.com"$'\n'"www.globo.com"; do
+		local pingResult
+		if pingResult=$(ping -q -c 1 -w 2 "$1"); then
+			# won't collect ping if data collecting service is not running.
+			if service data_collecting running; then
+				local data_collecting_dir="/tmp/data_collecting"
+				# creates data_collecting temporary directory if it doesn't exist.
+				mkdir -p "$data_collecting_dir"
+
+				# Removes everything before, and including, 'mdev = '.
+				local rtt=${pingResult##*mdev = }
+				# removes everything after, and including, first forward slash.
+				rtt=${rtt%%/*}
+
+				# file where pings are stored.
+				local pingsFile="${data_collecting_dir}/pings"
+				# using a lock file, in writing mode, to block code while accessing the pings file.
+				# this is because the data_collecting service will also write to the pings file.
+				{
+				flock -x 9
+				# appending $rtt in a new line in pings file.
+				echo "$rtt" >> "$pingsFile"
+				# "${pingsFile}lock" is also used by the data_collecting service.
+				} 9>"${pingsFile}lock"
+			fi
+
+			echo 0
+			return
+		fi
+	done
+	# No successful pings.
+
+	# false.
+	echo 1
+	return
+}
+
+# expects the output of ping, that includes the pings statistics at the end, as a single string input.
+# writes the ping rtt to a file, keeping the latest values from the last minute.
+save_ping_result() {
+	local data_collecting_dir="/tmp/data_collecting"
+	# creates data_collecting temporary directory if it doesn't exist.
+	mkdir -p "$data_collecting_dir"
+	# file where pings are stored.
+
+	# Function INPUT is used here. Removes everything before, and including, 'mdev = '.
+	local rtt=${1##*mdev = }
+	# removes everything after, and including, first forward slash.
+	rtt=${rtt%%/*}
+
+	local pingsFile="${data_collecting_dir}/pings"
+	# appending $rtt in a new line in pings file.
+	flock "$pingsFile" # lock.
+	echo "$rtt" >> "$pingsFile"
+	flock "$pingsFile" # unlock.
+}
+
 renew_dhcp() {
 	local _iface
 	local _proto

From 234fd9c232b2233aeebb14845b0e99983bb8a0f4 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Thu, 7 Oct 2021 21:44:17 -0300
Subject: [PATCH 02/77] Implemented the collecting of wifi devices signal to
 noise ration. Not removing colons from MAC address anymore.

---
 files/usr/share/data_collecting.sh | 58 +++++++++++++++++++++---------
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 5088f1eb..782aa3e2 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -108,9 +108,9 @@ collect_QoE_Monitor_data() {
 		# appending latencies to string to be sent.
 		string="${string} ${latencies}"
 	fi
-
-	# printf "string is: '%s'\n" "$string"
+	
 	# appending string to file.
+	# printf "string is: '%s'\n" "$string"
 	echo "loss $string"
 }
 
@@ -144,13 +144,39 @@ collect_connectivity_pings() {
 	local length=${#average}
 	average="${average:0:$(($l-3))}.${average:$(($l-3))}"
 
-	# this measures output.
-	echo "pings $average $count"
+	# if there no pings, we won't echo.
+	[ "$count" -gt 0 ] && echo "cnpings $pings"
 }
 
 collect_wifi_devices() {
-	local devices="$(iwinfo $(get_root_ifname 0) assoclist)"
-	echo "$devices"
+	# local devices="$(iwinfo $(get_root_ifname 0) assoclist | grep ago)"
+	local devices="$(cat wifi.txt)"
+	local str="" mac snr time
+	# first iteration won't put a space before the value.
+	local first=true
+	while [ ${#devices} -gt 0 ]; do
+		# getting everything before the first space. 
+		mac=${devices%% *}
+		# getting after '(SNR '. 
+		devices=${devices#*\(SNR }
+		# getting everything before the first closing parenthesis.
+		snr=${devices%%\)*}
+		# getting everything before ' ms'.
+		time=${devices%% ms*}
+		# getting everything after the first space.
+		time=${time##* }
+		# getting after 'ago'.
+		devices=${devices#*ago}
+		# getting after '\n'. if it exists. last line won't have it, so nothing will be changed.
+		devices=${devices#*$'\n'}
+		# if time is greater than one minute, we don't use this device's info.
+		[ "$time" -gt 60000 ] && continue
+		# if it's the first data we are storing, don't add a space before appending the data string.
+		[ "$first" == true ] && first=false || str="$str "
+		str="${str}${mac}-${snr}"
+	done
+	# we won't echo if there are no devices.
+	[ ${#str} -gt 0 ] && echo "wfdvcs $str"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -259,15 +285,17 @@ collectData() {
 	# string to be written to file.
 	local str=""
 
-	local out
-	out=$(collect_QoE_Monitor_data)
-	[ "$out" != "" ] && str="${str}${out}|"
-	out=$(collect_connectivity_pings)
-	[ "$out" != "" ] && str="${str}${out}|"
-	out=$(collect_wifi_devices)
-	[ "$out" != "" ] && str="${str}${out}|"
+	local output
+	output=$(collect_QoE_Monitor_data)
+	[ "$output" != "" ] && str="${str}|${output}"
+	output=$(collect_connectivity_pings)
+	[ "$output" != "" ] && str="${str}|${output}"
+	output=$(collect_wifi_devices)
+	[ "$output" != "" ] && str="${str}|${output}"
 
-	[ "$str" != "" ] && echo "$timestamp:$str" >> "$rawDataFile";
+	# expected raw data:
+	# '213234556456|loss 0 100 12345 1234|cnpings 10.344 30|wfdvcs aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
+	[ "$str" != "" ] && echo "${timestamp}${str}" >> "$rawDataFile";
 
 	# $(zipFile) returns 0 only if any amount of files has been compressed 
 	# and, consequently, moved to the directory of compressed files. So
@@ -311,8 +339,6 @@ sendToServer() {
 
 	# defined in /usr/share/functions/device_functions.sh
 	local mac=$(get_mac);
-	# removing all colons in mac address.
-	mac=${mac//:/}
 
 	status=$(curl --write-out '%{http_code}' -s -m 20 --connect-timeout 5 --output /dev/null \
 	-XPOST "https://$alarmServerAddress:7890/data" -H 'Content-Encoding: gzip' \

From 5af6f16ff400abaa1c0747ec2c0f7a0c31bc6759 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Fri, 8 Oct 2021 14:19:07 -0300
Subject: [PATCH 03/77] Added identification names for each raw data section.
 Removed unused parameters in calls for 'sendToSever()'

---
 files/usr/share/data_collecting.sh | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 782aa3e2..0222ec0a 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -45,7 +45,7 @@ collect_QoE_Monitor_data() {
 	local tx=$(($txBytes - $last_txBytes))
 	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
 	# we skip this measure.
-	([ "$rx" -lt 0 ] || [ "$tx" -lt 0 ]) && return
+	{ [ "$rx" -lt 0 ] || [ "$tx" -lt 0 ]; } && return
 	# saves current interface bytes value as last value.
 	last_rxBytes=$rxBytes
 	# saves current interface bytes value as last value.
@@ -111,7 +111,7 @@ collect_QoE_Monitor_data() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	echo "loss $string"
+	echo "burstLoss $string"
 }
 
 collect_connectivity_pings() {
@@ -145,12 +145,11 @@ collect_connectivity_pings() {
 	average="${average:0:$(($l-3))}.${average:$(($l-3))}"
 
 	# if there no pings, we won't echo.
-	[ "$count" -gt 0 ] && echo "cnpings $pings"
+	[ "$count" -gt 0 ] && echo "connPings $pings"
 }
 
 collect_wifi_devices() {
-	# local devices="$(iwinfo $(get_root_ifname 0) assoclist | grep ago)"
-	local devices="$(cat wifi.txt)"
+	local devices="$(iwinfo $(get_root_ifname 0) assoclist | grep ago)"
 	local str="" mac snr time
 	# first iteration won't put a space before the value.
 	local first=true
@@ -176,7 +175,7 @@ collect_wifi_devices() {
 		str="${str}${mac}-${snr}"
 	done
 	# we won't echo if there are no devices.
-	[ ${#str} -gt 0 ] && echo "wfdvcs $str"
+	[ ${#str} -gt 0 ] && echo "wifiDevices $str"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -245,7 +244,7 @@ zipFile() {
 	# if sum is smaller than $capSize, do nothing.
 	# echo checking file size to zip
 	# a return of 1 means nothing will be gzipped.
-	if [ $(($size + $dirSize)) -lt $capSize ]; then return 1; fi
+	[ $(($size + $dirSize)) -lt $capSize ] && return 1
 
 	# compressing file where raw data is held.
 	gzip "$rawDataFile"
@@ -335,7 +334,7 @@ checkServerState() {
 # sends file at given path ($1) to server at given address ($2) using $(curl) 
 # and returns $(curl) exit code.
 sendToServer() {
-	local filepath="$1" oldData="$2"
+	local filepath="$1"
 
 	# defined in /usr/share/functions/device_functions.sh
 	local mac=$(get_mac);
@@ -360,7 +359,7 @@ sendCompressedData() {
 	for i in "$compressedDataDir"/*; do
 		# if file exists, sends file and if $(curl) exit code isn't equal to 0, returns $(curl) exit code 
 		# without deleting the file we tried to send. if $(curl) exit code is equal to 0, removes file
-		[ -f "$i" ] && (sendToServer "$i" "1" || return "$?") && rm "$i"
+		[ -f "$i" ] && { sendToServer "$i" || return "$?"; } && rm "$i"
 	done
 	return 0
 }
@@ -383,7 +382,7 @@ sendUncompressedData() {
 	gzip -k "$rawDataFile"
 
 	# sends compressed file.
-	sendToServer "$compressedTempFile" "0"
+	sendToServer "$compressedTempFile"
 	# storing $(curl) exit code.
 	local sentResult="$?"
 

From ac707a94229772f44c477182351b7677b58b5dc2 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Fri, 8 Oct 2021 14:19:44 -0300
Subject: [PATCH 04/77] Copied the flock usage to all implementations of pings
 being saved to file.

---
 files/usr/share/functions/network_functions.sh | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 86762f42..51f23a2d 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -183,10 +183,14 @@ ping_maybe_collect() {
 
 			# file where pings are stored.
 			local pingsFile="${data_collecting_dir}/pings"
+			# using a lock file, in writing mode, to block code while accessing the pings file.
+			# this is because the data_collecting service will also write to the pings file.
+			{
+			flock -x 9
 			# appending $rtt in a new line in pings file.
-			flock "$pingsFile" # lock.
 			echo "$rtt" >> "$pingsFile"
-			flock "$pingsFile" # unlock.
+			# "${pingsFile}lock" is also used by the data_collecting service.
+			} 9>"${pingsFile}lock"
 		fi
 		return 0
 	fi
@@ -244,11 +248,16 @@ save_ping_result() {
 	# removes everything after, and including, first forward slash.
 	rtt=${rtt%%/*}
 
+	# file where pings are stored.
 	local pingsFile="${data_collecting_dir}/pings"
+	# using a lock file, in writing mode, to block code while accessing the pings file.
+	# this is because the data_collecting service will also write to the pings file.
+	{
+	flock -x 9
 	# appending $rtt in a new line in pings file.
-	flock "$pingsFile" # lock.
 	echo "$rtt" >> "$pingsFile"
-	flock "$pingsFile" # unlock.
+	# "${pingsFile}lock" is also used by the data_collecting service.
+	} 9>"${pingsFile}lock"
 }
 
 renew_dhcp() {

From 58c12cfc9540ef13cd708ba5c9c20fcbbc804364 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Fri, 15 Oct 2021 18:20:29 -0300
Subject: [PATCH 05/77] Added variable to enable or disable data collecting of
 connectivity pings, wifi devices and burst loss. Reading data collecting
 connectivity pings enabling variable from inside check_cabe_wan service at
 start of the script. Restarting the script if flashman sends a different
 value.

---
 files/usr/share/check_cable_wan.sh            | 14 ++++-
 files/usr/share/flashman_update.sh            |  7 ++-
 .../functions/data_collecting_functions.sh    | 55 +++++++++++++++----
 files/usr/share/keepalive.sh                  |  6 +-
 files/usr/share/mqtt.sh                       |  9 ---
 5 files changed, 68 insertions(+), 23 deletions(-)

diff --git a/files/usr/share/check_cable_wan.sh b/files/usr/share/check_cable_wan.sh
index 95a5bc8a..9722b2dc 100755
--- a/files/usr/share/check_cable_wan.sh
+++ b/files/usr/share/check_cable_wan.sh
@@ -20,15 +20,27 @@ write_access_start_time() {
 	json_cleanup
 }
 
+# opens 'flashbox_config.json' and reads the connectivity pings boolean.
+read_connectivity_pings_collecting_enabled() {
+	json_cleanup
+	json_load_file "/root/flashbox_config.json"
+	# non-existing value is translated to empty string.
+	# reading to global variable.
+	json_get_var collect_pings data_collecting_conn_pings
+	json_close_object
+
+}
+
 # Bootstrap
 reset_leds
 blink_leds "0"
 write_access_start_time 0
+read_connectivity_pings_collecting_enabled
 
 while true
 do
 	# We have layer 2 connectivity, now check external access
-	if [ ! "$(check_connectivity_internet "" "collect")" -eq 0 ]
+	if [ ! "$(check_connectivity_internet '' $collect_pings)" -eq 0 ]
 	then
 		# No external access
 		log "CHECK_WAN" "No external access..."
diff --git a/files/usr/share/flashman_update.sh b/files/usr/share/flashman_update.sh
index 438c8614..75f6c3b0 100644
--- a/files/usr/share/flashman_update.sh
+++ b/files/usr/share/flashman_update.sh
@@ -238,6 +238,9 @@ bridge_fix_dns=$_local_bridge_fix_dns"
 		json_get_var _data_collecting_alarm_fqdn data_collecting_alarm_fqdn
 		json_get_var _data_collecting_ping_fqdn data_collecting_ping_fqdn
 		json_get_var _data_collecting_ping_packets data_collecting_ping_packets
+		json_get_var _data_collecting_burst_loss data_collecting_burst_loss
+		json_get_var _data_collecting_conn_pings data_collecting_conn_pings
+		json_get_var _data_collecting_wifi_devices data_collecting_wifi_devices
 		json_get_var _bridge_mode_enabled bridge_mode_enabled
 		json_get_var _bridge_mode_switch_disable bridge_mode_switch_disable
 		json_get_var _bridge_mode_ip bridge_mode_ip
@@ -441,7 +444,9 @@ bridge_fix_dns=$_local_bridge_fix_dns"
 		# updates data collecting parameters.
 		set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
 			                           "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
-			                           "$_data_collecting_ping_packets"
+			                           "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
+			                           "$_data_collecting_conn_pings" "$_data_collecting_wifi_devices"
+
 
 		# Check for updates in port forward mapping
 		# Ignore changes if in bridge mode
diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index 4cf8b89d..5a8b7240 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -10,14 +10,20 @@ set_data_collecting_parameters() {
 	local data_collecting_alarm_fqdn="${3:-$FLM_SVADDR}"
 	local data_collecting_ping_fqdn="${4:-$FLM_SVADDR}"
 	local data_collecting_ping_packets="${5:-100}"
+	local data_collecting_burst_loss="${6:-0}"
+	local data_collecting_conn_pings="${7:-0}"
+	local data_collecting_wifi_devices="${8:-0}"
 
 	json_cleanup
-	json_load_file /root/flashbox_config.json
+	json_load_file "/root/flashbox_config.json"
 	json_get_var saved_data_collecting_is_active data_collecting_is_active
 	json_get_var saved_data_collecting_has_latency data_collecting_has_latency
 	json_get_var saved_data_collecting_alarm_fqdn data_collecting_alarm_fqdn
 	json_get_var saved_data_collecting_ping_fqdn data_collecting_ping_fqdn
 	json_get_var saved_data_collecting_ping_packets data_collecting_ping_packets
+	json_get_var saved_data_collecting_burst_loss data_collecting_burst_loss
+	json_get_var saved_data_collecting_conn_pings data_collecting_conn_pings
+	json_get_var saved_data_collecting_wifi_devices data_collecting_wifi_devices
 
 	local anyChange=false
 
@@ -25,35 +31,56 @@ set_data_collecting_parameters() {
 	if [ "$saved_data_collecting_is_active" != "$data_collecting_is_active" ]; then
 		anyChange=true
 		json_add_boolean data_collecting_is_active "$data_collecting_is_active"
-		log "DATA_COLLECTING" "Updated 'data_collecting_is_active' parameter to '$data_collecting_is_active'"
+		log "DATA_COLLECTING" "Updated 'data_collecting_is_active' parameter to '$data_collecting_is_active'."
 	fi
 
 	# Updating value if $data_collecting_has_latency has changed.
 	if [ "$saved_data_collecting_has_latency" != "$data_collecting_has_latency" ]; then
 		anyChange=true
 		json_add_boolean data_collecting_has_latency "$data_collecting_has_latency"
-		log "DATA_COLLECTING" "Updated 'data_collecting_has_latency' parameter to '$data_collecting_has_latency'"
+		log "DATA_COLLECTING" "Updated 'data_collecting_has_latency' parameter to '$data_collecting_has_latency'."
 	fi
 
 	# Updating value if $data_collecting_alarm_fqdn has changed.
 	if [ "$saved_data_collecting_alarm_fqdn" != "$data_collecting_alarm_fqdn" ]; then
 		anyChange=true
 		json_add_string data_collecting_alarm_fqdn "$data_collecting_alarm_fqdn"
-		log "DATA_COLLECTING" "Updated 'data_collecting_alarm_fqdn' parameter to '$data_collecting_alarm_fqdn'"
+		log "DATA_COLLECTING" "Updated 'data_collecting_alarm_fqdn' parameter to '$data_collecting_alarm_fqdn'."
 	fi
 
 	# Updating value if $data_collecting_ping_fqdn has changed.
 	if [ "$saved_data_collecting_ping_fqdn" != "$data_collecting_ping_fqdn" ]; then
 		anyChange=true
 		json_add_string data_collecting_ping_fqdn "$data_collecting_ping_fqdn"
-		log "DATA_COLLECTING" "Updated 'data_collecting_ping_fqdn' parameter to '$data_collecting_ping_fqdn'"
+		log "DATA_COLLECTING" "Updated 'data_collecting_ping_fqdn' parameter to '$data_collecting_ping_fqdn'."
 	fi
 
 	# Updating value if $data_collecting_alarm_fqdn has changed.
 	if [ "$saved_data_collecting_ping_packets" != "$data_collecting_ping_packets" ]; then
 		anyChange=true
 		json_add_int data_collecting_ping_packets "$data_collecting_ping_packets"
-		log "DATA_COLLECTING" "Updated 'data_collecting_ping_packets' parameter to '$data_collecting_ping_packets'"
+		log "DATA_COLLECTING" "Updated 'data_collecting_ping_packets' parameter to '$data_collecting_ping_packets'."
+	fi
+
+	# Updating value if $data_collecting_alarm_fqdn has changed.
+	if [ "$saved_data_collecting_burst_loss" != "$data_collecting_burst_loss" ]; then
+		anyChange=true
+		json_add_boolean data_collecting_burst_loss "$data_collecting_burst_loss"
+		log "DATA_COLLECTING" "Updated 'data_collecting_burst_loss' parameter to '$data_collecting_burst_loss'."
+	fi
+
+	# Updating value if $data_collecting_alarm_fqdn has changed.
+	if [ "$saved_data_collecting_conn_pings" != "$data_collecting_conn_pings" ]; then
+		anyChange=true
+		json_add_boolean data_collecting_conn_pings "$data_collecting_conn_pings"
+		log "DATA_COLLECTING" "Updated 'data_collecting_conn_pings' parameter to '$data_collecting_conn_pings'."
+	fi
+
+	# Updating value if $data_collecting_alarm_fqdn has changed.
+	if [ "$saved_data_collecting_wifi_devices" != "$data_collecting_wifi_devices" ]; then
+		anyChange=true
+		json_add_boolean data_collecting_wifi_devices "$data_collecting_wifi_devices"
+		log "DATA_COLLECTING" "Updated 'data_collecting_wifi_devices' parameter to '$data_collecting_wifi_devices'."
 	fi
 
 	# saving config json if any parameter has changed.
@@ -64,10 +91,16 @@ set_data_collecting_parameters() {
 
 	# "true" boolean value is translated as string "1" by jshn.sh
 	# "false" boolean value is translated as string "0" by jshn.sh
-	if [ "$data_collecting_is_active" = "1" ] && [ "$data_collecting_ping_fqdn" != "" ]; then
-		# reload won't restart data collecting service if it's already running.
-		service data_collecting reload
-	elif [ "$data_collecting_is_active" != "1" ]; then
-		service data_collecting stop > /dev/null 2>&1
+
+	# if data collecting service is active, reloads data_collecting service (which does 
+	# nothing if it's already running), else stops service. data_collecting service reads 
+	# it's configuration variables at each iteration (which happens once every minute).
+	[ "$data_collecting_is_active" -eq 1 ] && \
+		service data_collecting reload || service data_collecting stop > /dev/null 2>&1
+
+	# if connectivity pings has changed, restarts check_cable_wan service.
+	if [ "$saved_data_collecting_conn_pings" != "$data_collecting_conn_pings" ]; then
+		service check_cable_wan stop
+		service check_cable_wan start
 	fi
 }
diff --git a/files/usr/share/keepalive.sh b/files/usr/share/keepalive.sh
index 9f3a2e31..d51cf32e 100755
--- a/files/usr/share/keepalive.sh
+++ b/files/usr/share/keepalive.sh
@@ -119,6 +119,9 @@ wpsstate=$_local_wps_state"
 			json_get_var _data_collecting_alarm_fqdn data_collecting_alarm_fqdn
 			json_get_var _data_collecting_ping_fqdn data_collecting_ping_fqdn
 			json_get_var _data_collecting_ping_packets data_collecting_ping_packets
+			json_get_var _data_collecting_burst_loss data_collecting_burst_loss
+			json_get_var _data_collecting_conn_pings data_collecting_conn_pings
+			json_get_var _data_collecting_wifi_devices data_collecting_wifi_devices
 			json_close_object
 
 			if [ "$_do_newprobe" = "1" ]
@@ -169,7 +172,8 @@ wpsstate=$_local_wps_state"
 			# updates data collecting parameters.
 			set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
 			                               "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
-			                               "$_data_collecting_ping_packets"
+			                               "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
+			                               "$_data_collecting_conn_pings" "$_data_collecting_wifi_devices"
 
 		elif [ $_retstatus -eq 2 ]
 		then
diff --git a/files/usr/share/mqtt.sh b/files/usr/share/mqtt.sh
index a4b90678..c307866d 100644
--- a/files/usr/share/mqtt.sh
+++ b/files/usr/share/mqtt.sh
@@ -5,7 +5,6 @@
 . /usr/share/functions/dhcp_functions.sh
 . /usr/share/functions/api_functions.sh
 . /usr/share/functions/wireless_functions.sh
-. /usr/share/functions/data_collecting_functions.sh
 
 case "$1" in
 1)
@@ -48,14 +47,6 @@ ping)
 	log "MQTTMSG" "Running ping test"
 	run_ping_ondemand_test
 	;;
-datacollecting)
-	log "MQTTMSG" "Changing data collecting settings"
-	set_data_collecting_on_off "$2"
-	;;
-collectlatency)
-	log "MQTTMSG" "Changing the collecting of latencies when collecting data"
-	set_collect_latency "$2"
-	;;
 status)
 	if lock -n /tmp/get_status.lock
 	then

From 1c01c710771335c3072e9ce17fc2d9707a7c9f80 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Fri, 15 Oct 2021 18:22:39 -0300
Subject: [PATCH 06/77] Sticking to changing the 'check_connectivity_internet'
 function inside network_functions.sh to collect data if second argument is
 '1'. Not reading json file every time we have to collect connectivity pings
 as it is read inside check_cable_wan service when it starts.

---
 .../usr/share/functions/network_functions.sh  | 104 ++++--------------
 1 file changed, 21 insertions(+), 83 deletions(-)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 51f23a2d..9c0bf147 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -148,11 +148,17 @@ check_connectivity_internet() {
 	then
 		_addrs="$1"
 	fi
+	local collect_enabled="$2"
 	for _addr in $_addrs
 	do
-		# second argument is used to allow collecting data in this call.
-		if ping_maybe_collect "$_addr" "$2"
+		# ping output will be use in case collecting connectivity pings is enabled.
+		local pingResult
+		if pingResult=$(ping -q -c 1 -w 2 "$_addr")
 		then
+			# won't collect connectivity ping if data collecting service is not running.
+			[ "$collect_enabled" -eq 1 ] && service data_collecting running && \
+				save_connectivity_ping "$pingResult"
+
 			# true
 			echo 0
 			return
@@ -165,99 +171,31 @@ check_connectivity_internet() {
 	return
 }
 
-ping_maybe_collect() {
-	local address="$1" should_collect="$2"
-
-	local pingResult
-	if pingResult=$(ping -q -c 1 -w 2 "$address"); then
-		# won't collect ping if data collecting service is not running.
-		if [ "$should_collect" == "collect" ] && service data_collecting running; then
-			local data_collecting_dir="/tmp/data_collecting"
-			# creates data_collecting temporary directory if it doesn't exist.
-			mkdir -p "$data_collecting_dir"
-
-			# Removes everything before, and including, 'mdev = '.
-			local rtt=${pingResult##*mdev = }
-			# removes everything after, and including, first forward slash.
-			rtt=${rtt%%/*}
-
-			# file where pings are stored.
-			local pingsFile="${data_collecting_dir}/pings"
-			# using a lock file, in writing mode, to block code while accessing the pings file.
-			# this is because the data_collecting service will also write to the pings file.
-			{
-			flock -x 9
-			# appending $rtt in a new line in pings file.
-			echo "$rtt" >> "$pingsFile"
-			# "${pingsFile}lock" is also used by the data_collecting service.
-			} 9>"${pingsFile}lock"
-		fi
-		return 0
-	fi
-	return 1
-}
-
-check_connectivity_internet_and_collect_ping() {
-	for _addr in "www.google.com.br"$'\n'"www.facebook.com"$'\n'"www.globo.com"; do
-		local pingResult
-		if pingResult=$(ping -q -c 1 -w 2 "$1"); then
-			# won't collect ping if data collecting service is not running.
-			if service data_collecting running; then
-				local data_collecting_dir="/tmp/data_collecting"
-				# creates data_collecting temporary directory if it doesn't exist.
-				mkdir -p "$data_collecting_dir"
-
-				# Removes everything before, and including, 'mdev = '.
-				local rtt=${pingResult##*mdev = }
-				# removes everything after, and including, first forward slash.
-				rtt=${rtt%%/*}
-
-				# file where pings are stored.
-				local pingsFile="${data_collecting_dir}/pings"
-				# using a lock file, in writing mode, to block code while accessing the pings file.
-				# this is because the data_collecting service will also write to the pings file.
-				{
-				flock -x 9
-				# appending $rtt in a new line in pings file.
-				echo "$rtt" >> "$pingsFile"
-				# "${pingsFile}lock" is also used by the data_collecting service.
-				} 9>"${pingsFile}lock"
-			fi
-
-			echo 0
-			return
-		fi
-	done
-	# No successful pings.
-
-	# false.
-	echo 1
-	return
-}
-
 # expects the output of ping, that includes the pings statistics at the end, as a single string input.
-# writes the ping rtt to a file, keeping the latest values from the last minute.
-save_ping_result() {
+# writes the ping rtt to a file inside the data_collecting directory.
+save_connectivity_ping() {
+	local pingResult="$1"
+
+	# data_collecting directory
 	local data_collecting_dir="/tmp/data_collecting"
-	# creates data_collecting temporary directory if it doesn't exist.
-	mkdir -p "$data_collecting_dir"
 	# file where pings are stored.
+	local connectivityPingsFile="${data_collecting_dir}/connpings"
 
-	# Function INPUT is used here. Removes everything before, and including, 'mdev = '.
-	local rtt=${1##*mdev = }
+	# Removes everything before, and including, 'mdev = '.
+	local rtt=${pingResult##*mdev = }
 	# removes everything after, and including, first forward slash.
 	rtt=${rtt%%/*}
 
-	# file where pings are stored.
-	local pingsFile="${data_collecting_dir}/pings"
 	# using a lock file, in writing mode, to block code while accessing the pings file.
 	# this is because the data_collecting service will also write to the pings file.
 	{
 	flock -x 9
+	# creates data_collecting temporary directory if it doesn't exist.
+	mkdir -p "$data_collecting_dir"
 	# appending $rtt in a new line in pings file.
-	echo "$rtt" >> "$pingsFile"
-	# "${pingsFile}lock" is also used by the data_collecting service.
-	} 9>"${pingsFile}lock"
+	echo "$rtt" >> "$connectivityPingsFile"
+	# "${connectivityPingsFile}lock" is also used by the data_collecting service.
+	} 9>"${connectivityPingsFile}lock"
 }
 
 renew_dhcp() {

From 83b9b1a006f5b9ee3d3a489a5e1e31e9fea5424b Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Fri, 15 Oct 2021 18:26:24 -0300
Subject: [PATCH 07/77] Added a loop to collect wifi devices in both 2.4GHz and
 5GHz interfaces and added the interface name to the collected data besides
 the device values. Checking if connectivity pings file is empty so we don't
 execute math on values that are zero. Added a return statement at the
 beginning of each data collecting function to skip them if they are not
 enabled.

---
 files/usr/share/data_collecting.sh | 116 +++++++++++++++++------------
 1 file changed, 69 insertions(+), 47 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 0222ec0a..a5fb8d9b 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -4,17 +4,20 @@
 
 # directory where all data related to data collecting will be stored.
 dataCollectingDir="/tmp/data_collecting"
-# file collected data will be stored before being compressed.
+# file where collected data will be stored before being compressed.
 rawDataFile="${dataCollectingDir}/raw"
 # directory where data will be stored if compressing old data is necessary.
 compressedDataDir="${dataCollectingDir}/compressed"
+# file where connectivity pings are stored.
+connectivityPingsFile="${dataCollectingDir}/connpings"
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server, gets current 
 # rx and tx bytes from wan interface and compares then with values from previous calls to calculate 
 # cross traffic. If latency collecting is enabled, extracts the individual icmp request numbers and 
 # their respective ping times. Builds a string with all this information and write them to file.
 collect_QoE_Monitor_data() {
-	# echo collecting data and writing to file.
+	# checking if this data collecting is enabled
+	[ "$burstLoss" -ne 1 ] && return
 
 	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
@@ -115,19 +118,26 @@ collect_QoE_Monitor_data() {
 }
 
 collect_connectivity_pings() {
-	# file where pings are stored.
-	local pingsFile="${data_collecting_dir}/pings"
-	
+	# checking if this data collecting is enabled.
+	[ "$connPings" -ne 1 ] && return
+
 	# content from pings file will be assign to this variable.
 	local pings
+	# boolean that will become false if connectivity pings file is not empty.
+	local empty=true
 	# using a lock file, in writing mode, to block code while accessing the pings file.
 	# this is to prevent writing conflict to the script that writes new values to the pings file.
 	{
 	flock -x 9
-	pings=$(cat "$pingsFile")
-	rm "$pingsFile" > /dev/null 2>&1
-	# "${pingsFile}lock" is also used by the script that writes new values to the file.
-	} 9>"${pingsFile}lock"
+	# reading connectivity pings file and writing errors to '/dev/null'. 
+	pings=$(cat "$connectivityPingsFile") > /dev/null 2>&1 && \
+		# if file could be read and if amount of lines is greater than zero, sets 'empty' to false and removes file. 
+		[ $(printf "%s" "$pings" | wc -l) -gt 0 ] && empty=false && rm "$connectivityPingsFile"
+	# "${connectivityPingsFile}lock" is also used by the script that writes new values to the file.
+	} 9>"${connectivityPingsFile}lock"
+
+	# if there are no pings, we do nothing.
+	[ "$empty" == true ] && return
 
 	# counter and total for pings. we later divide them to get an average.
 	local count=0
@@ -138,41 +148,52 @@ collect_connectivity_pings() {
 		# we are using integers only in our math because flashboxes don't have floating point numbers in ash.
 		average=$(($average + ${rtt//.}))
 	done
+
 	# using integer division isn't perfect but it's precise enough for us.
 	average=$(($average / $count))
 	# dividing back by 1000 by just manipulating the string. Putting a dot before the last 3 digits.
-	local length=${#average}
-	average="${average:0:$(($l-3))}.${average:$(($l-3))}"
-
-	# if there no pings, we won't echo.
-	[ "$count" -gt 0 ] && echo "connPings $pings"
+	local dotPosition=$((${#average}-3))
+	average="${average:0:$dotPosition}.${average:$dotPosition}"
+	echo "connPings $average $count"
 }
 
 collect_wifi_devices() {
-	local devices="$(iwinfo $(get_root_ifname 0) assoclist | grep ago)"
-	local str="" mac snr time
-	# first iteration won't put a space before the value.
-	local first=true
-	while [ ${#devices} -gt 0 ]; do
-		# getting everything before the first space. 
-		mac=${devices%% *}
-		# getting after '(SNR '. 
-		devices=${devices#*\(SNR }
-		# getting everything before the first closing parenthesis.
-		snr=${devices%%\)*}
-		# getting everything before ' ms'.
-		time=${devices%% ms*}
-		# getting everything after the first space.
-		time=${time##* }
-		# getting after 'ago'.
-		devices=${devices#*ago}
-		# getting after '\n'. if it exists. last line won't have it, so nothing will be changed.
-		devices=${devices#*$'\n'}
-		# if time is greater than one minute, we don't use this device's info.
-		[ "$time" -gt 60000 ] && continue
-		# if it's the first data we are storing, don't add a space before appending the data string.
-		[ "$first" == true ] && first=false || str="$str "
-		str="${str}${mac}-${snr}"
+	# checking if this data collecting is enabled
+	[ "$wifiDevices" -ne 1 ] && return
+
+	# devices and their data will be stored in this string variable.
+	local str=""
+	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
+	for i in 0 1; do
+		# getting wifi interface name
+		local lan=$(get_root_ifname "$i")
+		# getting info from each connected device on wifi. 
+		# grep returns empty when no devices are connected or if interface doesn't exist.
+		local devices="$(iwinfo "$lan" assoclist | grep ago)"
+
+		# first iteration won't put a space before the value.
+		local first=true
+		while [ ${#devices} -gt 0 ]; do
+			# getting everything before the first space.
+			local deviceMac=${devices%% *}
+			# getting after '(SNR '. 
+			devices=${devices#*\(SNR }
+			# getting everything before the first closing parenthesis.
+			local snr=${devices%%\)*}
+			# getting everything before ' ms'.
+			local time=${devices%% ms*}
+			# getting everything after the first space.
+			time=${time##* }
+			# getting after 'ago'.
+			devices=${devices#*ago}
+			# getting after '\n'. if it exists. last line won't have it, so nothing will be changed.
+			devices=${devices#*$'\n'}
+			# if time is greater than one minute, we don't use this device's info.
+			[ "$time" -gt 60000 ] && continue
+			# if it's the first data we are storing, don't add a space before appending the data string.
+			[ "$first" == true ] && first=false || str="$str "
+			str="${str}${lan}-${deviceMac}-${snr}"
+		done
 	done
 	# we won't echo if there are no devices.
 	[ ${#str} -gt 0 ] && echo "wifiDevices $str"
@@ -344,8 +365,8 @@ sendToServer() {
 	-H 'Content-Type: text/plain' -H "X-ANLIX-ID: $mac" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" \
 	-H "Send-Time: $(date +%s)" --data-binary @"$filepath")
 	curlCode="$?"
-	[ "$curlCode" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code $curlCode" && return "$curlCode"
-	log "DATA_COLLECTING" "Data sent with response status code $status."
+	[ "$curlCode" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${curlCode}'." && return "$curlCode"
+	log "DATA_COLLECTING" "Data sent with response status code '${status}'."
 	[ "$status" -ge 200 ] && [ "$status" -lt 300 ] && return 0
 	return 1
 }
@@ -527,7 +548,7 @@ cleanFiles() {
 	rm "${dataCollectingDir}/serverState" 2> /dev/null
 	# rm "${dataCollectingDir}/backoffCounter" 2> /dev/null
 	rm "${rawDataFile}.gz" 2> /dev/null
-	flock "${dataCollectingDir}pingslock" rm "${dataCollectingDir}/pings"
+	flock "${connectivityPingsFile}lock" rm "$connectivityPingsFile"
 }
 
 # collects and sends data forever.
@@ -545,16 +566,17 @@ loop() {
 		# making sure directory exists every time.
 		mkdir -p "$dataCollectingDir"
 
-		# getting FQDNs every time we need to send data, this way we don't have to 
-		# restart the service if a fqdn changes.
+		# getting parameters every time we need to send data, this way we don't have to 
+		# restart the service if a parameter changes.
 		eval $(cat /root/flashbox_config.json | jsonfilter \
 			-e "hasLatency=@.data_collecting_has_latency" \
 			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
 			-e "pingServerAddress=@.data_collecting_ping_fqdn" \
-			-e "pingPackets=@.data_collecting_ping_packets")
-		# echo json variables: \
-		# 	hasLatency="$hasLatency", alarmServerAddress="$alarmServerAddress", \
-		# 	pingServerAddress="$pingServerAddress", pingPackets="$pingPackets"
+			-e "pingPackets=@.data_collecting_ping_packets" \
+			-e "burstLoss=@.data_collecting_burst_loss" \
+			-e "connPings=@.data_collecting_conn_pings" \
+			-e "wifiDevices=@.data_collecting_wifi_devices" \
+		)
 
 		# does everything related to collecting and storing data.`
 		collectData

From 11ce3f18f5cee2bd182aa9382d54da0c6e570418 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Mon, 18 Oct 2021 19:23:20 -0300
Subject: [PATCH 08/77] Reading "is active" variable when reading connectivity
 pings enabled variable.

---
 files/usr/share/check_cable_wan.sh                     | 5 ++++-
 files/usr/share/functions/data_collecting_functions.sh | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/files/usr/share/check_cable_wan.sh b/files/usr/share/check_cable_wan.sh
index 9722b2dc..2322b981 100755
--- a/files/usr/share/check_cable_wan.sh
+++ b/files/usr/share/check_cable_wan.sh
@@ -26,10 +26,13 @@ read_connectivity_pings_collecting_enabled() {
 	json_load_file "/root/flashbox_config.json"
 	# non-existing value is translated to empty string.
 	# reading to global variable.
-	json_get_var collect_pings data_collecting_conn_pings
+	json_get_var data_collecting_conn_pings data_collecting_conn_pings
+	json_get_var data_collecting_is_active data_collecting_is_active
 	json_close_object
 
+	[ "$data_collecting_is_active" -eq 1 ] && [ "$data_collecting_conn_pings" -eq 1 ] && collect_pings=1
 }
+collect_pings=0
 
 # Bootstrap
 reset_leds
diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index 5a8b7240..d1ac3de9 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -89,8 +89,8 @@ set_data_collecting_parameters() {
 	json_cleanup
 
 
-	# "true" boolean value is translated as string "1" by jshn.sh
-	# "false" boolean value is translated as string "0" by jshn.sh
+	# "true" boolean value is translated as string "1" by jshn.sh when variable is read.
+	# "false" boolean value is translated as string "0" by jshn.sh when variable is read.
 
 	# if data collecting service is active, reloads data_collecting service (which does 
 	# nothing if it's already running), else stops service. data_collecting service reads 

From e2bf94638e260d9e1da1f30c2f43a3755be1d9c1 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Wed, 27 Oct 2021 12:13:32 -0300
Subject: [PATCH 09/77] Added default value for second argument in
 check_connectivity_internet(). Using '/etc/init.d/servicename' instead of
 'service servicename' when issuing commands to services.

---
 files/usr/share/functions/data_collecting_functions.sh | 8 ++++----
 files/usr/share/functions/network_functions.sh         | 7 ++++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index d1ac3de9..bff00d18 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -95,12 +95,12 @@ set_data_collecting_parameters() {
 	# if data collecting service is active, reloads data_collecting service (which does 
 	# nothing if it's already running), else stops service. data_collecting service reads 
 	# it's configuration variables at each iteration (which happens once every minute).
-	[ "$data_collecting_is_active" -eq 1 ] && \
-		service data_collecting reload || service data_collecting stop > /dev/null 2>&1
+	[ "$data_collecting_is_active" -eq 1 ] && /etc/init.d/data_collecting reload \
+		|| /etc/init.d/data_collecting stop 2> /dev/null 2>&1
 
 	# if connectivity pings has changed, restarts check_cable_wan service.
 	if [ "$saved_data_collecting_conn_pings" != "$data_collecting_conn_pings" ]; then
-		service check_cable_wan stop
-		service check_cable_wan start
+		/etc/init.d/check_cable_wan stop
+		/etc/init.d/check_cable_wan start
 	fi
 }
diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 9c0bf147..f560c524 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -148,15 +148,16 @@ check_connectivity_internet() {
 	then
 		_addrs="$1"
 	fi
-	local collect_enabled="$2"
+	# if second argument is undefined, use value '0'.
+	local collect_enabled="${2:-0}"
 	for _addr in $_addrs
 	do
-		# ping output will be use in case collecting connectivity pings is enabled.
+		# ping output will be used in case collecting connectivity pings is enabled.
 		local pingResult
 		if pingResult=$(ping -q -c 1 -w 2 "$_addr")
 		then
 			# won't collect connectivity ping if data collecting service is not running.
-			[ "$collect_enabled" -eq 1 ] && service data_collecting running && \
+			[ "$collect_enabled" -eq 1 ] && /etc/init.d/data_collecting running && \
 				save_connectivity_ping "$pingResult"
 
 			# true

From 2055fc1e9979919da997cbebabac3af04b91bbfb Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Wed, 27 Oct 2021 12:23:48 -0300
Subject: [PATCH 10/77] Redirecting error output to /dev/null in some function
 calls. Cleaning lock file on service startup. Checking if wifi interface
 exists before collecting devices in wifi.

---
 files/usr/share/data_collecting.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index a5fb8d9b..2ad25a6e 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 . /usr/share/functions/device_functions.sh
+. /usr/share/functions/common_functions.sh
+. /usr/share/functions/custom_wireless_driver.sh
 . /usr/share/flashman_init.conf
 
 # directory where all data related to data collecting will be stored.
@@ -130,7 +132,7 @@ collect_connectivity_pings() {
 	{
 	flock -x 9
 	# reading connectivity pings file and writing errors to '/dev/null'. 
-	pings=$(cat "$connectivityPingsFile") > /dev/null 2>&1 && \
+	pings=$(cat "$connectivityPingsFile" 2> /dev/null) && \
 		# if file could be read and if amount of lines is greater than zero, sets 'empty' to false and removes file. 
 		[ $(printf "%s" "$pings" | wc -l) -gt 0 ] && empty=false && rm "$connectivityPingsFile"
 	# "${connectivityPingsFile}lock" is also used by the script that writes new values to the file.
@@ -165,8 +167,11 @@ collect_wifi_devices() {
 	local str=""
 	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
-		# getting wifi interface name
-		local lan=$(get_root_ifname "$i")
+		# getting wifi interface name.
+		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
+		local lan=$(get_root_ifname "$i" 2> /dev/null)
+		# if interface doesn't exist, skips this iteration.
+		[ "$lan" == "" ] && continue
 		# getting info from each connected device on wifi. 
 		# grep returns empty when no devices are connected or if interface doesn't exist.
 		local devices="$(iwinfo "$lan" assoclist | grep ago)"
@@ -357,7 +362,7 @@ checkServerState() {
 sendToServer() {
 	local filepath="$1"
 
-	# defined in /usr/share/functions/device_functions.sh
+	# 'get_mac()' is defined in /usr/share/functions/device_functions.sh.
 	local mac=$(get_mac);
 
 	status=$(curl --write-out '%{http_code}' -s -m 20 --connect-timeout 5 --output /dev/null \
@@ -365,6 +370,7 @@ sendToServer() {
 	-H 'Content-Type: text/plain' -H "X-ANLIX-ID: $mac" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" \
 	-H "Send-Time: $(date +%s)" --data-binary @"$filepath")
 	curlCode="$?"
+	# 'log()' is defined in /usr/share/functions/common_functions.sh.
 	[ "$curlCode" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${curlCode}'." && return "$curlCode"
 	log "DATA_COLLECTING" "Data sent with response status code '${status}'."
 	[ "$status" -ge 200 ] && [ "$status" -lt 300 ] && return 0
@@ -548,7 +554,7 @@ cleanFiles() {
 	rm "${dataCollectingDir}/serverState" 2> /dev/null
 	# rm "${dataCollectingDir}/backoffCounter" 2> /dev/null
 	rm "${rawDataFile}.gz" 2> /dev/null
-	flock "${connectivityPingsFile}lock" rm "$connectivityPingsFile"
+	rm "${connectivityPingsFile}lock"
 }
 
 # collects and sends data forever.

From 8154562c0e8d7ac1473741c5732053529f766503 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Fri, 29 Oct 2021 15:31:35 -0300
Subject: [PATCH 11/77] Reduced function name.

---
 files/usr/share/check_cable_wan.sh | 8 +++++---
 files/usr/share/data_collecting.sh | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/files/usr/share/check_cable_wan.sh b/files/usr/share/check_cable_wan.sh
index 2322b981..62def6f8 100755
--- a/files/usr/share/check_cable_wan.sh
+++ b/files/usr/share/check_cable_wan.sh
@@ -21,7 +21,7 @@ write_access_start_time() {
 }
 
 # opens 'flashbox_config.json' and reads the connectivity pings boolean.
-read_connectivity_pings_collecting_enabled() {
+read_data_collecting_parameters() {
 	json_cleanup
 	json_load_file "/root/flashbox_config.json"
 	# non-existing value is translated to empty string.
@@ -30,7 +30,9 @@ read_connectivity_pings_collecting_enabled() {
 	json_get_var data_collecting_is_active data_collecting_is_active
 	json_close_object
 
-	[ "$data_collecting_is_active" -eq 1 ] && [ "$data_collecting_conn_pings" -eq 1 ] && collect_pings=1
+	# we collect connectivity pings if both conn_pings and is_active is enabled.
+	[ "$data_collecting_is_active" -eq 1 ] && \
+		[ "$data_collecting_conn_pings" -eq 1 ] && collect_pings=1
 }
 collect_pings=0
 
@@ -38,7 +40,7 @@ collect_pings=0
 reset_leds
 blink_leds "0"
 write_access_start_time 0
-read_connectivity_pings_collecting_enabled
+read_data_collecting_parameters
 
 while true
 do
diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 2ad25a6e..20ca2623 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -319,7 +319,7 @@ collectData() {
 	[ "$output" != "" ] && str="${str}|${output}"
 
 	# expected raw data:
-	# '213234556456|loss 0 100 12345 1234|cnpings 10.344 30|wfdvcs aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
+	# '213234556456|burstLoss 0 100 12345 1234|connPings 10.344 30|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
 	[ "$str" != "" ] && echo "${timestamp}${str}" >> "$rawDataFile";
 
 	# $(zipFile) returns 0 only if any amount of files has been compressed 

From 2d542723038745bf87fff56595a4d0d6d1577383 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Fri, 29 Oct 2021 18:00:30 -0300
Subject: [PATCH 12/77] Not invoking a sub shell when calling a measure
 collecting function, as it might need to deal with global variables.
 Collected data is stored in a global string variable before being written to
 file, then, an empty string is assigned to it.

---
 files/usr/share/data_collecting.sh | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 20ca2623..806a0468 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -13,6 +13,8 @@ compressedDataDir="${dataCollectingDir}/compressed"
 # file where connectivity pings are stored.
 connectivityPingsFile="${dataCollectingDir}/connpings"
 
+rawData=""
+
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server, gets current 
 # rx and tx bytes from wan interface and compares then with values from previous calls to calculate 
 # cross traffic. If latency collecting is enabled, extracts the individual icmp request numbers and 
@@ -74,7 +76,7 @@ collect_QoE_Monitor_data() {
 	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
 	# loss=${loss##* } # removes everything before first space.
 
-	local string="loss $loss $transmitted $rx $tx" # data to be sent.
+	local string="$loss $transmitted $rx $tx" # data to be sent.
 
 	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
@@ -116,7 +118,7 @@ collect_QoE_Monitor_data() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	echo "burstLoss $string"
+	rawData="${rawData}|burstLoss ${string}"
 }
 
 collect_connectivity_pings() {
@@ -156,7 +158,7 @@ collect_connectivity_pings() {
 	# dividing back by 1000 by just manipulating the string. Putting a dot before the last 3 digits.
 	local dotPosition=$((${#average}-3))
 	average="${average:0:$dotPosition}.${average:$dotPosition}"
-	echo "connPings $average $count"
+	rawData="${rawData}|connPings ${average} ${count}"
 }
 
 collect_wifi_devices() {
@@ -201,7 +203,7 @@ collect_wifi_devices() {
 		done
 	done
 	# we won't echo if there are no devices.
-	[ ${#str} -gt 0 ] && echo "wifiDevices $str"
+	[ ${#str} -gt 0 ] && rawData="${rawData}|wifiDevices ${str}"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -307,20 +309,16 @@ collectData() {
 	# getting current unix time in seconds.
 	local timestamp=$(date +%s)
 
-	# string to be written to file.
-	local str=""
-
-	local output
-	output=$(collect_QoE_Monitor_data)
-	[ "$output" != "" ] && str="${str}|${output}"
-	output=$(collect_connectivity_pings)
-	[ "$output" != "" ] && str="${str}|${output}"
-	output=$(collect_wifi_devices)
-	[ "$output" != "" ] && str="${str}|${output}"
+	# collecting all measures.
+	collect_QoE_Monitor_data
+	collect_connectivity_pings
+	collect_wifi_devices
 
-	# expected raw data:
+	# example of an expected raw data with all measures present:
 	# '213234556456|burstLoss 0 100 12345 1234|connPings 10.344 30|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
-	[ "$str" != "" ] && echo "${timestamp}${str}" >> "$rawDataFile";
+	[ -n "$rawData" ] && echo "${timestamp}${rawData}" >> "$rawDataFile";
+	# cleaning 'rawData' value from memory and making it ready for next minute.
+	rawData=""
 
 	# $(zipFile) returns 0 only if any amount of files has been compressed 
 	# and, consequently, moved to the directory of compressed files. So

From 613582a655e699ed77760b3a49443156f13fa4fa Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Mon, 1 Nov 2021 15:25:13 -0300
Subject: [PATCH 13/77] Redirecting all '/etc/init.d/data_collecting stop'
 output to /dev/null because that output is not important.

---
 files/usr/share/functions/data_collecting_functions.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index 719dd45d..429a2ba4 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -96,7 +96,7 @@ set_data_collecting_parameters() {
 	# nothing if it's already running), else stops service. data_collecting service reads 
 	# it's configuration variables at each iteration (which happens once every minute).
 	[ "$data_collecting_is_active" -eq 1 ] && \
-		/etc/init.d/data_collecting reload || /etc/init.d/data_collecting stop 2> /dev/null 2>&1
+		/etc/init.d/data_collecting reload || /etc/init.d/data_collecting stop > /dev/null 2>&1
 
 	# if connectivity pings has changed, restarts check_cable_wan service.
 	if [ "$saved_data_collecting_conn_pings" != "$data_collecting_conn_pings" ]; then

From a07d5f7f427954ddc37f93363518a56178799453 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Wed, 3 Nov 2021 18:14:26 -0300
Subject: [PATCH 14/77] Enhanced comments. Changed the place where 'rawData'
 variable is declared.

---
 files/usr/share/data_collecting.sh | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 806a0468..76570263 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -13,8 +13,6 @@ compressedDataDir="${dataCollectingDir}/compressed"
 # file where connectivity pings are stored.
 connectivityPingsFile="${dataCollectingDir}/connpings"
 
-rawData=""
-
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server, gets current 
 # rx and tx bytes from wan interface and compares then with values from previous calls to calculate 
 # cross traffic. If latency collecting is enabled, extracts the individual icmp request numbers and 
@@ -308,6 +306,8 @@ removeOldFiles() {
 collectData() {
 	# getting current unix time in seconds.
 	local timestamp=$(date +%s)
+	# global variable where current raw data is stored before being written to file.
+	rawData=""
 
 	# collecting all measures.
 	collect_QoE_Monitor_data
@@ -317,16 +317,15 @@ collectData() {
 	# example of an expected raw data with all measures present:
 	# '213234556456|burstLoss 0 100 12345 1234|connPings 10.344 30|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
 	[ -n "$rawData" ] && echo "${timestamp}${rawData}" >> "$rawDataFile";
-	# cleaning 'rawData' value from memory and making it ready for next minute.
+	# cleaning 'rawData' value from memory.
 	rawData=""
 
+	# creates directory of for compressed files, if it doesn't already exists.
+	mkdir -p "$compressedDataDir"
 	# $(zipFile) returns 0 only if any amount of files has been compressed 
 	# and, consequently, moved to the directory of compressed files. So
 	# $(removeOldFiles) is only executed if any new compressed file was 
 	# created.
-
-	# creates directory of for compressed files, if it doesn't already exists.
-	mkdir -p "$compressedDataDir"
 	zipFile $((32*1024)) && removeOldFiles $((24*1024))
 	# the difference between the cap size sent to $(zipFile) and 
 	# $(removeOldFiles) is the size left as a minimum amount for raw data 

From de39504f0f4a212a4c32194b1c40932337999356 Mon Sep 17 00:00:00 2001
From: cassiohg <cassiohg@gmail.com>
Date: Wed, 3 Nov 2021 18:29:38 -0300
Subject: [PATCH 15/77] Changed a variable name and enhanced comments.

---
 files/usr/share/data_collecting.sh | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 76570263..e15f54a0 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -169,12 +169,12 @@ collect_wifi_devices() {
 	for i in 0 1; do
 		# getting wifi interface name.
 		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
-		local lan=$(get_root_ifname "$i" 2> /dev/null)
+		local wlan=$(get_root_ifname "$i" 2> /dev/null)
 		# if interface doesn't exist, skips this iteration.
-		[ "$lan" == "" ] && continue
+		[ "$wlan" == "" ] && continue
 		# getting info from each connected device on wifi. 
 		# grep returns empty when no devices are connected or if interface doesn't exist.
-		local devices="$(iwinfo "$lan" assoclist | grep ago)"
+		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
 
 		# first iteration won't put a space before the value.
 		local first=true
@@ -189,15 +189,17 @@ collect_wifi_devices() {
 			local time=${devices%% ms*}
 			# getting everything after the first space.
 			time=${time##* }
-			# getting after 'ago'.
+			# getting everything after 'ago'.
 			devices=${devices#*ago}
-			# getting after '\n'. if it exists. last line won't have it, so nothing will be changed.
+			# getting everything after '\n', if it exists. last line won't have it, so nothing will be changed.
+			# we can't add the line feed along witht the previous parameter expansion because we wouldn't match
+			# the last line and so we wouldn't make $devices length become zero.
 			devices=${devices#*$'\n'}
-			# if time is greater than one minute, we don't use this device's info.
+			# if $time is greater than one minute, we don't use this device's info.
 			[ "$time" -gt 60000 ] && continue
 			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$first" == true ] && first=false || str="$str "
-			str="${str}${lan}-${deviceMac}-${snr}"
+			str="${str}${wlan}-${deviceMac}-${snr}"
 		done
 	done
 	# we won't echo if there are no devices.

From 5ac793f2ecc0d4168c58ec321c4c4c51d18ab08a Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Wed, 20 Apr 2022 19:03:03 -0300
Subject: [PATCH 16/77] removing ping metrics collection

---
 files/usr/share/check_cable_wan.sh            | 19 +-----
 files/usr/share/data_collecting.sh            | 47 +--------------
 files/usr/share/flashman_update.sh            |  3 +-
 .../functions/data_collecting_functions.sh    | 17 +-----
 .../usr/share/functions/network_functions.sh  | 59 +------------------
 files/usr/share/keepalive.sh                  |  4 +-
 6 files changed, 6 insertions(+), 143 deletions(-)

diff --git a/files/usr/share/check_cable_wan.sh b/files/usr/share/check_cable_wan.sh
index 62def6f8..755d732c 100755
--- a/files/usr/share/check_cable_wan.sh
+++ b/files/usr/share/check_cable_wan.sh
@@ -20,32 +20,15 @@ write_access_start_time() {
 	json_cleanup
 }
 
-# opens 'flashbox_config.json' and reads the connectivity pings boolean.
-read_data_collecting_parameters() {
-	json_cleanup
-	json_load_file "/root/flashbox_config.json"
-	# non-existing value is translated to empty string.
-	# reading to global variable.
-	json_get_var data_collecting_conn_pings data_collecting_conn_pings
-	json_get_var data_collecting_is_active data_collecting_is_active
-	json_close_object
-
-	# we collect connectivity pings if both conn_pings and is_active is enabled.
-	[ "$data_collecting_is_active" -eq 1 ] && \
-		[ "$data_collecting_conn_pings" -eq 1 ] && collect_pings=1
-}
-collect_pings=0
-
 # Bootstrap
 reset_leds
 blink_leds "0"
 write_access_start_time 0
-read_data_collecting_parameters
 
 while true
 do
 	# We have layer 2 connectivity, now check external access
-	if [ ! "$(check_connectivity_internet '' $collect_pings)" -eq 0 ]
+	if [ ! "$(check_connectivity_internet)" -eq 0 ]
 	then
 		# No external access
 		log "CHECK_WAN" "No external access..."
diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index e15f54a0..150a8030 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -10,8 +10,6 @@ dataCollectingDir="/tmp/data_collecting"
 rawDataFile="${dataCollectingDir}/raw"
 # directory where data will be stored if compressing old data is necessary.
 compressedDataDir="${dataCollectingDir}/compressed"
-# file where connectivity pings are stored.
-connectivityPingsFile="${dataCollectingDir}/connpings"
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server, gets current 
 # rx and tx bytes from wan interface and compares then with values from previous calls to calculate 
@@ -119,46 +117,6 @@ collect_QoE_Monitor_data() {
 	rawData="${rawData}|burstLoss ${string}"
 }
 
-collect_connectivity_pings() {
-	# checking if this data collecting is enabled.
-	[ "$connPings" -ne 1 ] && return
-
-	# content from pings file will be assign to this variable.
-	local pings
-	# boolean that will become false if connectivity pings file is not empty.
-	local empty=true
-	# using a lock file, in writing mode, to block code while accessing the pings file.
-	# this is to prevent writing conflict to the script that writes new values to the pings file.
-	{
-	flock -x 9
-	# reading connectivity pings file and writing errors to '/dev/null'. 
-	pings=$(cat "$connectivityPingsFile" 2> /dev/null) && \
-		# if file could be read and if amount of lines is greater than zero, sets 'empty' to false and removes file. 
-		[ $(printf "%s" "$pings" | wc -l) -gt 0 ] && empty=false && rm "$connectivityPingsFile"
-	# "${connectivityPingsFile}lock" is also used by the script that writes new values to the file.
-	} 9>"${connectivityPingsFile}lock"
-
-	# if there are no pings, we do nothing.
-	[ "$empty" == true ] && return
-
-	# counter and total for pings. we later divide them to get an average.
-	local count=0
-	local average=0
-	for rtt in $pings; do
-		count=$(($count + 1))
-		# removing floating point. which means multiplying by 1000 as 'ping' times always have 3 floating point digits.
-		# we are using integers only in our math because flashboxes don't have floating point numbers in ash.
-		average=$(($average + ${rtt//.}))
-	done
-
-	# using integer division isn't perfect but it's precise enough for us.
-	average=$(($average / $count))
-	# dividing back by 1000 by just manipulating the string. Putting a dot before the last 3 digits.
-	local dotPosition=$((${#average}-3))
-	average="${average:0:$dotPosition}.${average:$dotPosition}"
-	rawData="${rawData}|connPings ${average} ${count}"
-}
-
 collect_wifi_devices() {
 	# checking if this data collecting is enabled
 	[ "$wifiDevices" -ne 1 ] && return
@@ -313,11 +271,10 @@ collectData() {
 
 	# collecting all measures.
 	collect_QoE_Monitor_data
-	collect_connectivity_pings
 	collect_wifi_devices
 
 	# example of an expected raw data with all measures present:
-	# '213234556456|burstLoss 0 100 12345 1234|connPings 10.344 30|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
+	# '213234556456|burstLoss 0 100 12345 1234|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
 	[ -n "$rawData" ] && echo "${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
@@ -553,7 +510,6 @@ cleanFiles() {
 	rm "${dataCollectingDir}/serverState" 2> /dev/null
 	# rm "${dataCollectingDir}/backoffCounter" 2> /dev/null
 	rm "${rawDataFile}.gz" 2> /dev/null
-	rm "${connectivityPingsFile}lock"
 }
 
 # collects and sends data forever.
@@ -579,7 +535,6 @@ loop() {
 			-e "pingServerAddress=@.data_collecting_ping_fqdn" \
 			-e "pingPackets=@.data_collecting_ping_packets" \
 			-e "burstLoss=@.data_collecting_burst_loss" \
-			-e "connPings=@.data_collecting_conn_pings" \
 			-e "wifiDevices=@.data_collecting_wifi_devices" \
 		)
 
diff --git a/files/usr/share/flashman_update.sh b/files/usr/share/flashman_update.sh
index b4bc9840..e93e7f70 100644
--- a/files/usr/share/flashman_update.sh
+++ b/files/usr/share/flashman_update.sh
@@ -247,7 +247,6 @@ bridge_fix_dns=$_local_bridge_fix_dns"
 		json_get_var _data_collecting_ping_fqdn data_collecting_ping_fqdn
 		json_get_var _data_collecting_ping_packets data_collecting_ping_packets
 		json_get_var _data_collecting_burst_loss data_collecting_burst_loss
-		json_get_var _data_collecting_conn_pings data_collecting_conn_pings
 		json_get_var _data_collecting_wifi_devices data_collecting_wifi_devices
 		json_get_var _bridge_mode_enabled bridge_mode_enabled
 		json_get_var _bridge_mode_switch_disable bridge_mode_switch_disable
@@ -510,7 +509,7 @@ bridge_fix_dns=$_local_bridge_fix_dns"
 		set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
 			                           "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
 			                           "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
-			                           "$_data_collecting_conn_pings" "$_data_collecting_wifi_devices"
+			                           "$_data_collecting_wifi_devices"
 
 
 		# Check for updates in port forward mapping
diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index 429a2ba4..22014d6d 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -11,8 +11,7 @@ set_data_collecting_parameters() {
 	local data_collecting_ping_fqdn="${4:-$FLM_SVADDR}"
 	local data_collecting_ping_packets="${5:-100}"
 	local data_collecting_burst_loss="${6:-0}"
-	local data_collecting_conn_pings="${7:-0}"
-	local data_collecting_wifi_devices="${8:-0}"
+	local data_collecting_wifi_devices="${7:-0}"
 
 	json_cleanup
 	json_load_file "/root/flashbox_config.json"
@@ -22,7 +21,6 @@ set_data_collecting_parameters() {
 	json_get_var saved_data_collecting_ping_fqdn data_collecting_ping_fqdn
 	json_get_var saved_data_collecting_ping_packets data_collecting_ping_packets
 	json_get_var saved_data_collecting_burst_loss data_collecting_burst_loss
-	json_get_var saved_data_collecting_conn_pings data_collecting_conn_pings
 	json_get_var saved_data_collecting_wifi_devices data_collecting_wifi_devices
 
 	local anyChange=false
@@ -69,13 +67,6 @@ set_data_collecting_parameters() {
 		log "DATA_COLLECTING" "Updated 'data_collecting_burst_loss' parameter to '$data_collecting_burst_loss'."
 	fi
 
-	# Updating value if $data_collecting_alarm_fqdn has changed.
-	if [ "$saved_data_collecting_conn_pings" != "$data_collecting_conn_pings" ]; then
-		anyChange=true
-		json_add_boolean data_collecting_conn_pings "$data_collecting_conn_pings"
-		log "DATA_COLLECTING" "Updated 'data_collecting_conn_pings' parameter to '$data_collecting_conn_pings'."
-	fi
-
 	# Updating value if $data_collecting_alarm_fqdn has changed.
 	if [ "$saved_data_collecting_wifi_devices" != "$data_collecting_wifi_devices" ]; then
 		anyChange=true
@@ -97,10 +88,4 @@ set_data_collecting_parameters() {
 	# it's configuration variables at each iteration (which happens once every minute).
 	[ "$data_collecting_is_active" -eq 1 ] && \
 		/etc/init.d/data_collecting reload || /etc/init.d/data_collecting stop > /dev/null 2>&1
-
-	# if connectivity pings has changed, restarts check_cable_wan service.
-	if [ "$saved_data_collecting_conn_pings" != "$data_collecting_conn_pings" ]; then
-		/etc/init.d/check_cable_wan stop
-		/etc/init.d/check_cable_wan start
-	fi
 }
diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index d39a414e..6a533f31 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -120,46 +120,16 @@ check_connectivity_flashman() {
 	check_connectivity_internet "$_addrs"
 }
 
-# check_connectivity_internet() {
-# 	_addrs="www.google.com.br"$'\n'"www.facebook.com"$'\n'"www.globo.com"
-# 	if [ "$1" != "" ]
-# 	then
-# 		_addrs="$1"
-# 	fi
-# 	for _addr in $_addrs
-# 	do
-# 		if ping -q -c 1 -w 2 "$_addr"  > /dev/null 2>&1
-# 		then
-# 			# true
-# 			echo 0
-# 			return
-# 		fi
-# 	done
-# 	# No successfull pings
-
-# 	# false
-# 	echo 1
-# 	return
-# }
-
 check_connectivity_internet() {
 	_addrs="www.google.com.br"$'\n'"www.facebook.com"$'\n'"www.globo.com"
 	if [ "$1" != "" ]
 	then
 		_addrs="$1"
 	fi
-	# if second argument is undefined, use value '0'.
-	local collect_enabled="${2:-0}"
 	for _addr in $_addrs
 	do
-		# ping output will be used in case collecting connectivity pings is enabled.
-		local pingResult
-		if pingResult=$(ping -q -c 1 -w 2 "$_addr")
+		if ping -q -c 1 -w 2 "$_addr"  > /dev/null 2>&1
 		then
-			# won't collect connectivity ping if data collecting service is not running.
-			[ "$collect_enabled" -eq 1 ] && /etc/init.d/data_collecting running && \
-				save_connectivity_ping "$pingResult"
-
 			# true
 			echo 0
 			return
@@ -172,33 +142,6 @@ check_connectivity_internet() {
 	return
 }
 
-# expects the output of ping, that includes the pings statistics at the end, as a single string input.
-# writes the ping rtt to a file inside the data_collecting directory.
-save_connectivity_ping() {
-	local pingResult="$1"
-
-	# data_collecting directory
-	local data_collecting_dir="/tmp/data_collecting"
-	# file where pings are stored.
-	local connectivityPingsFile="${data_collecting_dir}/connpings"
-
-	# Removes everything before, and including, 'mdev = '.
-	local rtt=${pingResult##*mdev = }
-	# removes everything after, and including, first forward slash.
-	rtt=${rtt%%/*}
-
-	# using a lock file, in writing mode, to block code while accessing the pings file.
-	# this is because the data_collecting service will also write to the pings file.
-	{
-	flock -x 9
-	# creates data_collecting temporary directory if it doesn't exist.
-	mkdir -p "$data_collecting_dir"
-	# appending $rtt in a new line in pings file.
-	echo "$rtt" >> "$connectivityPingsFile"
-	# "${connectivityPingsFile}lock" is also used by the data_collecting service.
-	} 9>"${connectivityPingsFile}lock"
-}
-
 renew_dhcp() {
 	local _iface
 	local _proto
diff --git a/files/usr/share/keepalive.sh b/files/usr/share/keepalive.sh
index fa28d7d6..cc71e587 100755
--- a/files/usr/share/keepalive.sh
+++ b/files/usr/share/keepalive.sh
@@ -121,7 +121,6 @@ wpsstate=$_local_wps_state"
 			json_get_var _data_collecting_ping_fqdn data_collecting_ping_fqdn
 			json_get_var _data_collecting_ping_packets data_collecting_ping_packets
 			json_get_var _data_collecting_burst_loss data_collecting_burst_loss
-			json_get_var _data_collecting_conn_pings data_collecting_conn_pings
 			json_get_var _data_collecting_wifi_devices data_collecting_wifi_devices
 			json_close_object
 
@@ -173,8 +172,7 @@ wpsstate=$_local_wps_state"
 			# updates data collecting parameters.
 			set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
 			                               "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
-			                               "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
-			                               "$_data_collecting_conn_pings" "$_data_collecting_wifi_devices"
+			                               "$_data_collecting_ping_packets" "$_data_collecting_burst_loss"
 
 		elif [ $_retstatus -eq 2 ]
 		then

From c37ed7faf8397002878ecc5634ea102dd6c3773a Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 22 Apr 2022 11:48:04 -0300
Subject: [PATCH 17/77] adding packet up/down collection

---
 files/usr/share/data_collecting.sh            | 40 ++++++++++++++++---
 files/usr/share/functions/device_functions.sh | 16 +++++++-
 .../usr/share/functions/network_functions.sh  |  4 +-
 3 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 150a8030..058804ea 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -27,9 +27,14 @@ collect_QoE_Monitor_data() {
 	# Even if the ping could not be executed, we'll read the wan bytes to keep tracking the amount of bytes up and down.
 
 	# bytes received by the interface.
-	local rxBytes=$(get_wan_statistics RX)
+	local rxBytes=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
-	local txBytes=$(get_wan_statistics TX)
+	local txBytes=$(get_wan_bytes_statistics TX)
+
+	# packets received by the interface.
+	local rxPackets=$(get_wan_packets_statistics RX)
+	# packets sent by the interface.
+	local txPackets=$(get_wan_packets_statistics TX)
 
 	# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
 	if [ -z "$last_rxBytes" ] || [ -z "$last_txBytes" ]; then
@@ -42,18 +47,41 @@ collect_QoE_Monitor_data() {
 		return
 	fi
 
+	# if last packets are not defined. define them using the current wan interface packets value. then we skip this measure.
+	if [ -z "$last_rxPackets" ] || [ -z "$last_txPackets" ]; then
+		# echo last packets are undefined
+		# packets received by the interface. will be used next time.
+		last_rxPackets="$rxPackets"
+		# packets sent by the interface. will be used next time.
+		last_txPackets="$txPackets"
+		# don't write data this round. we need a full minute of packets to calculate cross traffic.
+		return
+	fi
+
 	# bytes received since last time.
-	local rx=$(($rxBytes - $last_rxBytes))
+	local rxBytesDiff=$(($rxBytes - $last_rxBytes))
 	# bytes transmitted since last time
-	local tx=$(($txBytes - $last_txBytes))
+	local txBytesDiff=$(($txBytes - $last_txBytes))
 	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
 	# we skip this measure.
-	{ [ "$rx" -lt 0 ] || [ "$tx" -lt 0 ]; } && return
+	{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && return
 	# saves current interface bytes value as last value.
 	last_rxBytes=$rxBytes
 	# saves current interface bytes value as last value.
 	last_txBytes=$txBytes
 
+	# packets received since last time.
+	local rxPacketsDiff=$(($rxPackets - $last_rxPackets))
+	# packets transmitted since last time
+	local txPacketsDiff=$(($txPackets - $last_txPackets))
+	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+	# we skip this measure.
+	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return
+	# saves current interface packets value as last value.
+	last_rxPackets=$rxPackets
+	# saves current interface packets value as last value.
+	last_txPackets=$txPackets
+
 	# if ping could not be executed, we skip this measure.
 	[ "$pingError" -eq 2 ] && return;
 
@@ -72,7 +100,7 @@ collect_QoE_Monitor_data() {
 	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
 	# loss=${loss##* } # removes everything before first space.
 
-	local string="$loss $transmitted $rx $tx" # data to be sent.
+	local string="$loss $transmitted $rxBytesDiff $txBytesDiff $rxPacketsDiff $txPacketsDiff" # data to be sent.
 
 	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
diff --git a/files/usr/share/functions/device_functions.sh b/files/usr/share/functions/device_functions.sh
index d5ae0cb0..d629ac41 100644
--- a/files/usr/share/functions/device_functions.sh
+++ b/files/usr/share/functions/device_functions.sh
@@ -267,7 +267,7 @@ get_lan_dev_negotiated_speed() {
 	echo "$_speed"
 }
 
-get_wan_statistics() {
+get_wan_bytes_statistics() {
 	local _param=$1
 
 	if [ "$(lsmod | grep hwnat)" ]
@@ -304,6 +304,20 @@ get_wan_statistics() {
 	fi
 }
 
+get_wan_packets_statistics() {
+	local _param=$1
+	local _wan=$(get_wan_device)
+	if [ -f /sys/class/net/$_wan/statistics/tx_packets ]
+	then
+		case "$1" in
+			"TX") echo "$(cat /sys/class/net/$_wan/statistics/rx_packets)" ;;
+			"RX") echo "$(cat /sys/class/net/$_wan/statistics/tx_packets)" ;;
+		esac
+	else
+		echo "0"
+	fi
+}
+
 get_wifi_device_signature() {
 	local _dev_mac="$1"
 	local _q=""
diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 6a533f31..790a8505 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -658,8 +658,8 @@ set_use_dns_proxy() {
 
 store_wan_bytes() {
 	local _epoch=$(date +%s)
-	local _wan_rx=$(get_wan_statistics RX)
-	local _wan_tx=$(get_wan_statistics TX)
+	local _wan_rx=$(get_wan_bytes_statistics RX)
+	local _wan_tx=$(get_wan_bytes_statistics TX)
 
 	json_init
 

From 6f4e33431f833fc271ea58c766e10c2e2359e9b0 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 22 Apr 2022 19:22:31 -0300
Subject: [PATCH 18/77] collecting latency and up/down packets stats

---
 files/usr/share/data_collecting.sh | 92 ++++++++++++++++++------------
 1 file changed, 55 insertions(+), 37 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 058804ea..315cb882 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -11,31 +11,14 @@ rawDataFile="${dataCollectingDir}/raw"
 # directory where data will be stored if compressing old data is necessary.
 compressedDataDir="${dataCollectingDir}/compressed"
 
-# takes current unix timestamp, executes ping, in burst, to $pingServerAddress server, gets current 
-# rx and tx bytes from wan interface and compares then with values from previous calls to calculate 
-# cross traffic. If latency collecting is enabled, extracts the individual icmp request numbers and 
-# their respective ping times. Builds a string with all this information and write them to file.
-collect_QoE_Monitor_data() {
-	# checking if this data collecting is enabled
-	[ "$burstLoss" -ne 1 ] && return
-
-	# burst ping with $pingPackets amount of packets.
-	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
-	# ping return value.
-	local pingError="$?"
-
-	# Even if the ping could not be executed, we'll read the wan bytes to keep tracking the amount of bytes up and down.
-
+# gets current rx and tx bytes/packets from wan interface and compares them
+# with values from previous calls to calculate cross traffic
+collect_wan() {
 	# bytes received by the interface.
 	local rxBytes=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
 	local txBytes=$(get_wan_bytes_statistics TX)
 
-	# packets received by the interface.
-	local rxPackets=$(get_wan_packets_statistics RX)
-	# packets sent by the interface.
-	local txPackets=$(get_wan_packets_statistics TX)
-
 	# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
 	if [ -z "$last_rxBytes" ] || [ -z "$last_txBytes" ]; then
 		# echo last bytes are undefined
@@ -44,18 +27,7 @@ collect_QoE_Monitor_data() {
 		# bytes sent by the interface. will be used next time.
 		last_txBytes="$txBytes"
 		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
-		return
-	fi
-
-	# if last packets are not defined. define them using the current wan interface packets value. then we skip this measure.
-	if [ -z "$last_rxPackets" ] || [ -z "$last_txPackets" ]; then
-		# echo last packets are undefined
-		# packets received by the interface. will be used next time.
-		last_rxPackets="$rxPackets"
-		# packets sent by the interface. will be used next time.
-		last_txPackets="$txPackets"
-		# don't write data this round. we need a full minute of packets to calculate cross traffic.
-		return
+		return -1
 	fi
 
 	# bytes received since last time.
@@ -64,24 +36,57 @@ collect_QoE_Monitor_data() {
 	local txBytesDiff=$(($txBytes - $last_txBytes))
 	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
 	# we skip this measure.
-	{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && return
+	{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && return -1
 	# saves current interface bytes value as last value.
 	last_rxBytes=$rxBytes
 	# saves current interface bytes value as last value.
 	last_txBytes=$txBytes
 
+	# packets received by the interface.
+	local rxPackets=$(get_wan_packets_statistics RX)
+	# packets sent by the interface.
+	local txPackets=$(get_wan_packets_statistics TX)
+
+	# if last packets are not defined. define them using the current wan interface packets value. then we skip this measure.
+	if [ -z "$last_rxPackets" ] || [ -z "$last_txPackets" ]; then
+		# echo last packets are undefined
+		# packets received by the interface. will be used next time.
+		last_rxPackets="$rxPackets"
+		# packets sent by the interface. will be used next time.
+		last_txPackets="$txPackets"
+		# don't write data this round. we need a full minute of packets to calculate cross traffic.
+		return -1
+	fi
+
 	# packets received since last time.
 	local rxPacketsDiff=$(($rxPackets - $last_rxPackets))
 	# packets transmitted since last time
 	local txPacketsDiff=$(($txPackets - $last_txPackets))
 	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
 	# we skip this measure.
-	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return
+	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return -1
 	# saves current interface packets value as last value.
 	last_rxPackets=$rxPackets
 	# saves current interface packets value as last value.
 	last_txPackets=$txPackets
 
+	# data to be sent.
+	local string="$rxBytesDiff $txBytesDiff $rxPacketsDiff $txPacketsDiff"
+	rawData="${rawData}|wan_stats ${string}"
+}
+
+# takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
+# If latency collecting is enabled, extracts the individual icmp request numbers and 
+# their respective ping times. Builds a string with all this information and write them to file.
+collect_burst() {
+	# checking if this data collecting is enabled
+	[ "$burstLoss" -ne 1 ] && return
+
+	# burst ping with $pingPackets amount of packets.
+	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
+	# ping return value.
+	local pingError="$?"
+
 	# if ping could not be executed, we skip this measure.
 	[ "$pingError" -eq 2 ] && return;
 
@@ -100,7 +105,19 @@ collect_QoE_Monitor_data() {
 	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
 	# loss=${loss##* } # removes everything before first space.
 
-	local string="$loss $transmitted $rxBytesDiff $txBytesDiff $rxPacketsDiff $txPacketsDiff" # data to be sent.
+	# removes everything before and including 'mdev = '
+	local latencyStats=${pingResult#*/mdev = }
+	# removes everything before first backslash
+	local latencyAvg=${latencyStats#*/}
+	# removes everything after first backslash
+	latencyAvg=${latencyAvg%%/*}
+	# removes everything before and including last backslash
+	local latencyStd=${latencyStats##*/}
+	# removes everything after and including first space
+	latencyStd=${latencyStd% *}
+
+	# data to be sent.
+	local string="$loss $transmitted $latencyAvg $latencyStd"
 
 	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
@@ -298,11 +315,12 @@ collectData() {
 	rawData=""
 
 	# collecting all measures.
-	collect_QoE_Monitor_data
+	# only collect burst data if we have one minute of wan measurements already
+	collect_wan && collect_burst
 	collect_wifi_devices
 
 	# example of an expected raw data with all measures present:
-	# '213234556456|burstLoss 0 100 12345 1234|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
+	# '213234556456|burstLoss 0 100 1.246 0.161|wan_stats 12345 1234 1000 100|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
 	[ -n "$rawData" ] && echo "${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""

From 2e7fc3a8068d846b3d5eaadc512c41410234cb87 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Wed, 27 Apr 2022 08:57:54 -0300
Subject: [PATCH 19/77] collecting wifi tx/rx packet count per connected device

---
 files/usr/share/data_collecting.sh | 64 +++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 315cb882..7f8cb6a6 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -168,6 +168,7 @@ collect_wifi_devices() {
 
 	# devices and their data will be stored in this string variable.
 	local str=""
+
 	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
 		# getting wifi interface name.
@@ -178,32 +179,93 @@ collect_wifi_devices() {
 		# getting info from each connected device on wifi. 
 		# grep returns empty when no devices are connected or if interface doesn't exist.
 		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
+		local devices_rx_pkts="$(iwinfo "$wlan" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
+		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
 		# first iteration won't put a space before the value.
 		local first=true
+
+		# string to be appended to devices packets file
+		local fileStr=""
+
+		local fileName="/root/devices_24_pkts.txt"
+		if [[ "$i" -eq 1 ]]; then
+			fileName="/root/devices_5_pkts.txt"
+		fi
+
 		while [ ${#devices} -gt 0 ]; do
+
 			# getting everything before the first space.
 			local deviceMac=${devices%% *}
+
 			# getting after '(SNR '. 
 			devices=${devices#*\(SNR }
+
 			# getting everything before the first closing parenthesis.
 			local snr=${devices%%\)*}
+
 			# getting everything before ' ms'.
 			local time=${devices%% ms*}
+
 			# getting everything after the first space.
 			time=${time##* }
+
 			# getting everything after 'ago'.
 			devices=${devices#*ago}
+
 			# getting everything after '\n', if it exists. last line won't have it, so nothing will be changed.
 			# we can't add the line feed along witht the previous parameter expansion because we wouldn't match
 			# the last line and so we wouldn't make $devices length become zero.
 			devices=${devices#*$'\n'}
+
 			# if $time is greater than one minute, we don't use this device's info.
 			[ "$time" -gt 60000 ] && continue
+			
+			local rx_pkts=${devices_rx_pkts%% *}
+			devices_rx_pkts=${devices_rx_pkts#*$'\n'}
+
+			local tx_pkts=${devices_tx_pkts%% *}
+			devices_tx_pkts=${devices_tx_pkts#*$'\n'}
+
+			[ "$rx_pkts" == "" ] && continue
+			[ "$tx_pkts" == "" ] && continue
+
+			[ "$first" == true ] && first=false || fileStr="$fileStr "
+			fileStr="${fileStr}${deviceMac}_${rx_pkts}_${tx_pkts}"
+
+			local rx_pkts_diff=""
+			local tx_pkts_diff=""
+
+			if [ -f "$fileName" ]; then
+                local devices_pkts=$(cat "$fileName")
+
+				local last_pkts=${devices_pkts#*"$deviceMac"_}
+				last_pkts=${last_pkts%% *}
+
+				local last_rx_pkts=${last_pkts%_*}
+				[ "$last_rx_pkts" == "" ] && continue
+				rx_pkts_diff=$(($rx_pkts - $last_rx_pkts))
+				
+				local last_tx_pkts=${last_pkts#*_}
+				[ "$last_tx_pkts" == "" ] && continue
+				tx_pkts_diff=$(($tx_pkts - $last_tx_pkts))
+
+				# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+				# we skip this measure.
+				{ [ "$rx_pkts_diff" -lt 0 ] || [ "$tx_pkts_diff" -lt 0 ]; } && continue
+			else
+				continue
+			fi
+
 			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$first" == true ] && first=false || str="$str "
-			str="${str}${wlan}-${deviceMac}-${snr}"
+			str="${str}${wlan}_${deviceMac}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
 		done
+		# empty out file (we only need last minute info)
+		> "$fileName"
+
+		# write to it if there are device infos
+		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$fileName"
 	done
 	# we won't echo if there are no devices.
 	[ ${#str} -gt 0 ] && rawData="${rawData}|wifiDevices ${str}"

From b98a85ae11cf5acf7c0156aaf707ee383a33c98e Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Thu, 28 Apr 2022 09:27:53 -0300
Subject: [PATCH 20/77] leaving auxiliary files in data collecting directory

---
 files/usr/share/data_collecting.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 7f8cb6a6..bfa8681c 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -188,9 +188,9 @@ collect_wifi_devices() {
 		# string to be appended to devices packets file
 		local fileStr=""
 
-		local fileName="/root/devices_24_pkts.txt"
+		local fileName="${dataCollectingDir}/devices_24_pkts"
 		if [[ "$i" -eq 1 ]]; then
-			fileName="/root/devices_5_pkts.txt"
+			fileName="{dataCollectingDir}/devices_24_pkts"
 		fi
 
 		while [ ${#devices} -gt 0 ]; do

From df1aef86d055291efae76e4bc134c1fcc0cd461e Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 6 May 2022 13:48:09 -0300
Subject: [PATCH 21/77] removing wan packets collection because of hwnat

---
 files/usr/share/data_collecting.sh            | 32 ++-----------------
 files/usr/share/functions/device_functions.sh | 14 --------
 2 files changed, 2 insertions(+), 44 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index bfa8681c..39dcfcf1 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -42,36 +42,8 @@ collect_wan() {
 	# saves current interface bytes value as last value.
 	last_txBytes=$txBytes
 
-	# packets received by the interface.
-	local rxPackets=$(get_wan_packets_statistics RX)
-	# packets sent by the interface.
-	local txPackets=$(get_wan_packets_statistics TX)
-
-	# if last packets are not defined. define them using the current wan interface packets value. then we skip this measure.
-	if [ -z "$last_rxPackets" ] || [ -z "$last_txPackets" ]; then
-		# echo last packets are undefined
-		# packets received by the interface. will be used next time.
-		last_rxPackets="$rxPackets"
-		# packets sent by the interface. will be used next time.
-		last_txPackets="$txPackets"
-		# don't write data this round. we need a full minute of packets to calculate cross traffic.
-		return -1
-	fi
-
-	# packets received since last time.
-	local rxPacketsDiff=$(($rxPackets - $last_rxPackets))
-	# packets transmitted since last time
-	local txPacketsDiff=$(($txPackets - $last_txPackets))
-	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-	# we skip this measure.
-	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return -1
-	# saves current interface packets value as last value.
-	last_rxPackets=$rxPackets
-	# saves current interface packets value as last value.
-	last_txPackets=$txPackets
-
 	# data to be sent.
-	local string="$rxBytesDiff $txBytesDiff $rxPacketsDiff $txPacketsDiff"
+	local string="$rxBytesDiff $txBytesDiff"
 	rawData="${rawData}|wan_stats ${string}"
 }
 
@@ -382,7 +354,7 @@ collectData() {
 	collect_wifi_devices
 
 	# example of an expected raw data with all measures present:
-	# '213234556456|burstLoss 0 100 1.246 0.161|wan_stats 12345 1234 1000 100|wifiDevices aa:bb:cc:dd:ee:ff-22 ab:bb:cc:dd:ee:ff-45'
+	# '213234556456|burstLoss 0 100 1.246 0.161|wan_stats 12345 1234|wifiDevices aa:bb:cc:dd:ee:ff_22_12345_1234 ab:bb:cc:dd:ee:ff_45_1000_100'
 	[ -n "$rawData" ] && echo "${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
diff --git a/files/usr/share/functions/device_functions.sh b/files/usr/share/functions/device_functions.sh
index d629ac41..2e644e26 100644
--- a/files/usr/share/functions/device_functions.sh
+++ b/files/usr/share/functions/device_functions.sh
@@ -304,20 +304,6 @@ get_wan_bytes_statistics() {
 	fi
 }
 
-get_wan_packets_statistics() {
-	local _param=$1
-	local _wan=$(get_wan_device)
-	if [ -f /sys/class/net/$_wan/statistics/tx_packets ]
-	then
-		case "$1" in
-			"TX") echo "$(cat /sys/class/net/$_wan/statistics/rx_packets)" ;;
-			"RX") echo "$(cat /sys/class/net/$_wan/statistics/tx_packets)" ;;
-		esac
-	else
-		echo "0"
-	fi
-}
-
 get_wifi_device_signature() {
 	local _dev_mac="$1"
 	local _q=""

From a7c31d1af823614e54bf1edea130abd70ec1d056 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 6 May 2022 18:03:57 -0300
Subject: [PATCH 22/77] reverting previous commit

---
 files/usr/share/data_collecting.sh            | 30 ++++++++++++++++++-
 files/usr/share/functions/device_functions.sh | 14 +++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 39dcfcf1..f796d276 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -42,8 +42,36 @@ collect_wan() {
 	# saves current interface bytes value as last value.
 	last_txBytes=$txBytes
 
+		# packets received by the interface.
+	local rxPackets=$(get_wan_packets_statistics RX)
+	# packets sent by the interface.
+	local txPackets=$(get_wan_packets_statistics TX)
+
+	# if last packets are not defined. define them using the current wan interface packets value. then we skip this measure.
+	if [ -z "$last_rxPackets" ] || [ -z "$last_txPackets" ]; then
+		# echo last packets are undefined
+		# packets received by the interface. will be used next time.
+		last_rxPackets="$rxPackets"
+		# packets sent by the interface. will be used next time.
+		last_txPackets="$txPackets"
+		# don't write data this round. we need a full minute of packets to calculate cross traffic.
+		return -1
+	fi
+
+	# packets received since last time.
+	local rxPacketsDiff=$(($rxPackets - $last_rxPackets))
+	# packets transmitted since last time
+	local txPacketsDiff=$(($txPackets - $last_txPackets))
+	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+	# we skip this measure.
+	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return -1
+	# saves current interface packets value as last value.
+	last_rxPackets=$rxPackets
+	# saves current interface packets value as last value.
+	last_txPackets=$txPackets
+
 	# data to be sent.
-	local string="$rxBytesDiff $txBytesDiff"
+	local string="$rxBytesDiff $txBytesDiff rxPacketsDiff $txPacketsDiff"
 	rawData="${rawData}|wan_stats ${string}"
 }
 
diff --git a/files/usr/share/functions/device_functions.sh b/files/usr/share/functions/device_functions.sh
index 2e644e26..d629ac41 100644
--- a/files/usr/share/functions/device_functions.sh
+++ b/files/usr/share/functions/device_functions.sh
@@ -304,6 +304,20 @@ get_wan_bytes_statistics() {
 	fi
 }
 
+get_wan_packets_statistics() {
+	local _param=$1
+	local _wan=$(get_wan_device)
+	if [ -f /sys/class/net/$_wan/statistics/tx_packets ]
+	then
+		case "$1" in
+			"TX") echo "$(cat /sys/class/net/$_wan/statistics/rx_packets)" ;;
+			"RX") echo "$(cat /sys/class/net/$_wan/statistics/tx_packets)" ;;
+		esac
+	else
+		echo "0"
+	fi
+}
+
 get_wifi_device_signature() {
 	local _dev_mac="$1"
 	local _q=""

From a3df8ea66147ef9e517c2e4c25aa87bfa213b32f Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 16:51:06 -0300
Subject: [PATCH 23/77] Better way to find wan name for Mediatek & Realtek

---
 custom-files/tbs.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/custom-files/tbs.sh b/custom-files/tbs.sh
index 1b1c98fa..fecee3ee 100644
--- a/custom-files/tbs.sh
+++ b/custom-files/tbs.sh
@@ -10,7 +10,9 @@ get_custom_mac() {
 	local _mac_address_tag=""
 	local _p1
 
-	_p1=$(uci get network.wan_eth0_2_dev.macaddr | awk '{print toupper($1)}')
+	local _wan_ifname=$(uci show network | sed -n "s/network.wan_\(.*\)_dev=device/\1/p")
+
+	_p1=$(uci get network.wan_${_wan_ifname}_dev.macaddr | awk '{print toupper($1)}')
 	[ ! -z "$_p1" ] && _mac_address_tag=$_p1
 
 	echo "$_mac_address_tag"

From 5b446875e63485573cca228ab1ce8c98a64f3378 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 16:52:51 -0300
Subject: [PATCH 24/77] Added function to change wan vlan configuration.

---
 .../usr/share/functions/network_functions.sh  | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 6a533f31..c25c8171 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -695,6 +695,48 @@ get_bridge_mode_status() {
 	echo "$_status"
 }
 
+# This function changes wan interfaces to use the new virtual interface
+# after changing the vlan configuration for wan
+update_wan_interfaces() {
+	# $1: New virtual interface name
+	local _vlan_id=$1
+	local _old_wan_ifname=""
+	local _wan_ifname=""
+
+	# Get the old configuration wan ifname
+	if [ -z "$(get_station_ifname 0 | grep ath)" ]
+	then
+		# This field only exists for Mediatek and Realtek
+		_old_wan_ifname=$(uci show network | sed -n "s/network.wan_\(.*\)_dev=device/\1/p")
+
+		# Remove the vlan id from the wan ifname
+		# Ex.: eth0 from eth0_2
+		_wan_ifname=${_old_wan_ifname%_*}
+	else
+		_old_wan_ifname=$(uci show network | sed -n "s/network.wan.ifname=\'\(.*\)\' /\1/p")
+
+		# Remove the vlan id from the wan ifname
+		# Ex.: eth1 from eth1.10
+		_wan_ifname=${_old_wan_ifname%.*}
+	fi
+
+	# Assign it to wan and wan6 interface
+	# If it is Atheros and vlan = 2, clear the wan vlan
+	# If the _vlan_id come empty, clear Atheros wan vlan
+	# TODO! This function is partially implemented for Atheros
+	uci set network.wan.ifname="${_wan_ifname}.${_vlan_id}"
+	uci set network.wan6.ifname="${_wan_ifname}.${_vlan_id}"
+
+	# Assign it to dev interface if it is not Atheros
+	if [ -z "$(get_station_ifname 0 | grep ath)" ]
+	then
+		uci set network.wan_${_old_wan_ifname}_dev.name="${_wan_ifname}.${_vlan_id}"
+	fi
+
+	# Commit the changes
+	uci commit network
+}
+
 update_vlan() {
 	if [ -f /root/vlan_config.json ]; then
 		local _restart_network=$1

From 0d9579e09568010531b1e39e81a375e3f807f8a2 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 16:58:13 -0300
Subject: [PATCH 25/77] Fixed a bug on creation of the vlan json file.

---
 .../usr/share/functions/network_functions.sh  | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index c25c8171..500d7491 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -755,12 +755,17 @@ update_vlan() {
 			local _vids=''
 			local _idx=0
 
+			# Loop through every vlan that already exists
 			for _vlan in $_input; do
+				
+				# Extract the vlan id from each the line
 				_vid=${_vlan#*\'}
 				_vid=${_vid%\'}
-				local _test=${_vlans#*$_vid}
-				# Indicates _vid is in _vlans
-				if [ $(( ${#_test} < ${#_vlans} )) = 1 ]; then
+
+				# Indicates that the vlan that already exists in config/network
+				# do exists in the new vlan config file (vlan_config.json)
+				if [ -n "$(echo "${_vlans} " | grep "${_vid} ")" ]
+				then
 					json_get_var _ports $_vid
 					uci set network.@switch_vlan[$_idx].ports="$_ports"
 				else # _vid isn't in _vlans
@@ -772,15 +777,19 @@ update_vlan() {
 				else
 					_vids="$_vids $_vid"
 				fi
+				# Next switch_vlan in config/network
 				_idx=$(( _idx + 1 ))
 			done
 
 			IFS=$' '
 
+			# Loop though every entry in vlan_config.json
 			for _vlan in $_vlans; do
-				_test=${_vids#*$_vlan}
-				# Indicates _vlan isn't in _vids
-				if [ $(( ${#_test} < ${#_vids} )) = 0 ]; then
+
+				# If the vlan id does not exists yet in config/network
+				if [ -z "$(echo "${_vids} " | grep "${_vlan} ")" ]; then
+
+					# Create the new entry
 					json_get_var _ports $_vlan
 					uci add network switch_vlan
 					uci set network.@switch_vlan[-1].device="$(get_switch_device)"

From b9190979d310d44dcab474c79ac6b4c07cc481b7 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:00:21 -0300
Subject: [PATCH 26/77] Added wan vlan configuration from flashman.

---
 .../usr/share/functions/network_functions.sh  | 66 +++++++++++++++++--
 1 file changed, 60 insertions(+), 6 deletions(-)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 500d7491..32197ccf 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -766,17 +766,49 @@ update_vlan() {
 				# do exists in the new vlan config file (vlan_config.json)
 				if [ -n "$(echo "${_vlans} " | grep "${_vid} ")" ]
 				then
+					
+					# Reassign the ports to the vlan
+					# If it is an Atheros and wan configuration, bypass
 					json_get_var _ports $_vid
 					uci set network.@switch_vlan[$_idx].ports="$_ports"
-				else # _vid isn't in _vlans
+
+					# Get the wan and cpu port
+					if [ "$(type -t custom_switch_ports)" ]; then
+						local _wan_port=$(custom_switch_ports 2) 
+						local _cpu_port=$(custom_switch_ports 4) 
+					else
+						local _wan_port=$(switch_ports 2) 
+						local _cpu_port=$(switch_ports 4) 
+					fi
+
+					# If the wan configuration was changed
+					# TODO! This function is not implemented yet for Atheros, check if Atheros
+					# and based on vlan, configure the wan properly
+					if [[ -n "$(echo "$_ports" | grep "${_wan_port}t ${_cpu_port}t")" ]] || \
+					   [[ -n "$(echo "$_ports" | grep "${_wan_port} ${_cpu_port}t")" ]]; then
+
+						# Change the virtual interface name of wan 
+						# interfaces
+						$(update_wan_interfaces $_vid)
+
+					fi
+
+					# Concatenate vlan ids
+					if [ "$_vids" == '' ]; then
+						_vids="$_vid"
+					else
+						_vids="$_vids $_vid"
+					fi
+
+				# Indicates that the vlan that already exists in config/network
+				# do not exists in the new vlan config file (vlan_config.json)
+				else
+
+					# Delete the switch_vlan entry
 					uci delete network.@switch_vlan[$_idx]
 					_idx=$(( _idx - 1 ))
 				fi
-				if [ "$_vids" = '' ]; then
-					_vids="$_vid"
-				else
-					_vids="$_vids $_vid"
-				fi
+
 				# Next switch_vlan in config/network
 				_idx=$(( _idx + 1 ))
 			done
@@ -795,6 +827,28 @@ update_vlan() {
 					uci set network.@switch_vlan[-1].device="$(get_switch_device)"
 					uci set network.@switch_vlan[-1].vlan="$_vlan"
 					uci set network.@switch_vlan[-1].ports="$_ports"
+
+					# Get the wan and cpu port
+					if [ "$(type -t custom_switch_ports)" ]; then
+						local _wan_port=$(custom_switch_ports 2) 
+						local _cpu_port=$(custom_switch_ports 4) 
+					else
+						local _wan_port=$(switch_ports 2) 
+						local _cpu_port=$(switch_ports 4) 
+					fi
+
+					# If the wan configuration was changed
+					# TODO! This function is not implemented yet for Atheros, check if Atheros
+					# and based on vlan, configure the wan properly
+					if [[ -n "$(echo "$_ports" | grep "${_wan_port}t ${_cpu_port}t")" ]] || \
+					   [[ -n "$(echo "$_ports" | grep "${_wan_port} ${_cpu_port}t")" ]]; then
+
+						# Change the virtual interface name of wan 
+						# interfaces
+						$(update_wan_interfaces $_vlan)
+
+					fi
+
 				fi
 			done
 

From 983836911aa8532348cc046df555565f5d0bf232 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:01:30 -0300
Subject: [PATCH 27/77] Added a function to get the ports of the router.

---
 .../usr/share/functions/network_functions.sh  | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 32197ccf..3bf8cb5f 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -1202,4 +1202,42 @@ save_bridge_mode_vlan_config() {
 	fi
 	echo "$_vlan" > /root/vlan_config.json
 	update_vlan "n"
+}
+
+
+
+# This function is intended to be used with Tecnico's app
+# It returns wan, cpu or lan ports, to check possible configurations
+get_ports() {
+	# $1: Which port
+		# wan: Wan port
+		# lan: Lan ports
+		# cpu: CPU port
+	local _port_type="$1"
+	local _ports=""
+
+	if [ "$(type -t custom_switch_ports)" ]
+	then
+		local _wan_port=$(custom_switch_ports 2) 
+		local _lan_ports=$(custom_switch_ports 3)
+		local _cpu_port=$(custom_switch_ports 4) 
+	else
+		local _wan_port=$(switch_ports 2) 
+		local _lan_ports=$(switch_ports 3)
+		local _cpu_port=$(switch_ports 4) 
+	fi
+
+	if [ "$_port_type" = "wan" ]; then
+		_ports="$_wan_port"
+
+	elif [ "$_port_type" = "lan" ]; then
+		_ports="$_lan_ports"
+
+	elif [ "$_port_type" = "cpu" ]; then
+		_ports="$_cpu_port"
+
+	fi
+
+	# If the configuration sent to it is wrong, it will return an empty string
+	echo "$_ports"
 }
\ No newline at end of file

From 2907e164c76b9e00b678ac3831416d4accd754dd Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:02:19 -0300
Subject: [PATCH 28/77] Added a function to get the current vlan config.

---
 files/usr/share/functions/network_functions.sh | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 3bf8cb5f..fa3e609d 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -1206,6 +1206,23 @@ save_bridge_mode_vlan_config() {
 
 
 
+# This function is intended to be used with Tecnico's app
+# It returns the vlan configuration as a json
+get_vlan_config() {
+
+	local _vlan_json=""
+
+	if [ -f /root/vlan_config.json ]; then
+		json_cleanup
+		json_load_file /root/vlan_config.json
+		json_close_object
+		_vlan_json=`json_dump`
+	fi
+
+	echo "$_vlan_json"
+}
+
+
 # This function is intended to be used with Tecnico's app
 # It returns wan, cpu or lan ports, to check possible configurations
 get_ports() {

From 30d4563c7b9d304424f1892433876467078dd2a8 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:03:08 -0300
Subject: [PATCH 29/77] Added function to validate and configure vlan.

---
 .../usr/share/functions/network_functions.sh  | 71 +++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index fa3e609d..b4a045be 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -1205,6 +1205,77 @@ save_bridge_mode_vlan_config() {
 }
 
 
+# This function is intended to be used with Tecnico's app
+# It saves the configuration received from config.lua
+set_vlan_config() {
+	# $1: Vlan Configuration
+	local _vlan_config=$1
+	local _result="ok"
+	local _ports=""
+
+	json_cleanup
+	json_load "$_vlan_config"
+	json_get_values _ports
+	json_close_object
+
+	# Get ports
+	if [ "$(type -t custom_switch_ports)" ]
+	then
+		local _wan_port=$(custom_switch_ports 2) 
+		local _lan_ports=$(custom_switch_ports 3)
+		local _cpu_port=$(custom_switch_ports 4) 
+	else
+		local _wan_port=$(switch_ports 2) 
+		local _lan_ports=$(switch_ports 3)
+		local _cpu_port=$(switch_ports 4) 
+	fi
+
+	# Loop every port and check if it's valid
+	for _port in $_ports; do
+
+		# CPU without tag do exist but is not included here
+		# With proprietary Atheros driver, the wan port does not exist
+		if [ "$_port" != "$_wan_port" ] && \
+		   [ "$_port" != "${_wan_port}t" ] && \
+		   [ "$_port" != "${_cpu_port}t" ]
+		then
+
+			# Loop every lan port
+			for _lan_port in $_lan_ports; do
+				if [ "$_port" != "$_lan_port" ] && \
+				   [ "$_port" != "${_lan_port}t" ]
+				then
+					# Invalid parameters
+					_result=""
+				
+				else
+					# Found a valid port, break
+					_result="ok"
+					break
+				fi
+			done
+
+		fi
+
+		# Check if result is valid, otherwise quit
+		if [[ -z "$_result" ]]
+		then
+			break
+		fi
+	done
+
+	if [[ -n "$_ports" ]] && [[ -n "$_result" ]]
+	then
+		# Change the vlan json file
+		echo "$_vlan_config" > /root/vlan_config.json
+	else
+		# If the json could not be loaded, it is an error
+		_result=""
+	fi
+
+	echo "$_result"
+}
+
 
 # This function is intended to be used with Tecnico's app
 # It returns the vlan configuration as a json

From 86f3bb934a4dece1453b61d8ae54fe13a8310f7f Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:05:22 -0300
Subject: [PATCH 30/77] Added a func to config the wan-vlan from the Tecnico's
 APP.

---
 files/usr/share/lua/config.lua | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/files/usr/share/lua/config.lua b/files/usr/share/lua/config.lua
index 153b2247..b795ede6 100644
--- a/files/usr/share/lua/config.lua
+++ b/files/usr/share/lua/config.lua
@@ -111,6 +111,21 @@ function handle_config(command, data)
 				flashman.update_bridge(disable_switch, ip, gateway, dns)
 			end
 		end
+
+	-- Create a new function with different name to 
+	-- not interfere with wan conn_type
+	elseif command == "wan-vlan" then
+		-- Change vlan and reply with ok so that
+		local status = flashman.set_vlan_config()
+
+		if status == nil then
+			web.error_handle(web.ERROR_PARAMETERS, nil)
+		else
+			web.send_json({success = true}) -- reply before changing network
+			flashman.configure_vlan()
+		end
+		
+		return
 	end
 
 end

From c74a10ea5dd23ac983c3ff40f61182f6406c1326 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:08:30 -0300
Subject: [PATCH 31/77] Added a function to get vlan config from Tecnico's APP.

---
 files/usr/share/anlix/index.lua  | 19 +++++++++++++++++++
 files/usr/share/lua/flashman.lua | 10 ++++++++++
 2 files changed, 29 insertions(+)

diff --git a/files/usr/share/anlix/index.lua b/files/usr/share/anlix/index.lua
index 5b38d957..328f1ed9 100644
--- a/files/usr/share/anlix/index.lua
+++ b/files/usr/share/anlix/index.lua
@@ -170,6 +170,25 @@ function handle_request(env)
 		resp["data"] = data
 		web.send_json(resp)
 		return
+
+	-- Get the vlan configuration
+	elseif command == "getVlanConfig" then
+		local data = {}
+		local config = flashman.get_vlan_config()
+
+		-- Check if configuration is valid
+		if config == nil then
+			data["ok"] = false
+		else
+			data["ok"] = true
+			data["vlan_config"] = config
+		end
+
+		-- Send response
+		local resp = {}
+		resp["data"] = data
+		web.send_json(resp)
+		return
 	end
 
 	if tonumber(app_protocol_ver) == 1 then
diff --git a/files/usr/share/lua/flashman.lua b/files/usr/share/lua/flashman.lua
index a41d867d..209aa958 100644
--- a/files/usr/share/lua/flashman.lua
+++ b/files/usr/share/lua/flashman.lua
@@ -177,4 +177,14 @@ function flashman.update(remote_addr, app_id, app_secret)
   end
 end
 
+-- This function returns the vlan configuration json
+function flashman.get_vlan_config()
+  local result = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_vlan_config\"")
+  
+  if result == nil or result == "" then
+    result = nil
+  end
+
+  return result
+end
 return flashman

From d63dab3cae053090564cd9612e4fdf43957db51c Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:09:31 -0300
Subject: [PATCH 32/77] Added function to get ports from Tecnico's APP

---
 files/usr/share/anlix/index.lua  | 21 +++++++++++++++++++++
 files/usr/share/lua/flashman.lua | 12 ++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/files/usr/share/anlix/index.lua b/files/usr/share/anlix/index.lua
index 328f1ed9..16bdb3ef 100644
--- a/files/usr/share/anlix/index.lua
+++ b/files/usr/share/anlix/index.lua
@@ -184,6 +184,27 @@ function handle_request(env)
 			data["vlan_config"] = config
 		end
 
+		-- Send response
+		local resp = {}
+		resp["data"] = data
+		web.send_json(resp)
+		return
+
+	-- Get the ports to be used with vlan
+	elseif command == "getPorts" then
+		local data = {}
+		-- subcommands:
+		--   wan, lan, cpu
+		local ports = flashman.get_ports(subcommand)
+
+		-- Check if port is valid
+		if ports == nil then
+			data["ok"] = false
+		else
+			data["ok"] = true
+			data["ports"] = ports
+		end
+
 		-- Send response
 		local resp = {}
 		resp["data"] = data
diff --git a/files/usr/share/lua/flashman.lua b/files/usr/share/lua/flashman.lua
index 209aa958..05f01f2c 100644
--- a/files/usr/share/lua/flashman.lua
+++ b/files/usr/share/lua/flashman.lua
@@ -187,4 +187,16 @@ function flashman.get_vlan_config()
 
   return result
 end
+
+-- This function returns the ports in use
+function flashman.get_ports(port_name)
+  local result = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_ports ".. port_name .."\"")
+  
+  if result == nil or result == "" then
+    result = nil
+  end
+
+  return result
+end
+
 return flashman

From 80032f84152420817b56dc92fa4c8590d8ca2076 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Wed, 11 May 2022 17:10:27 -0300
Subject: [PATCH 33/77] Added func to config the vlan from Tecnico's APP.

---
 files/usr/share/lua/flashman.lua | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/files/usr/share/lua/flashman.lua b/files/usr/share/lua/flashman.lua
index 05f01f2c..b0f9b4af 100644
--- a/files/usr/share/lua/flashman.lua
+++ b/files/usr/share/lua/flashman.lua
@@ -177,6 +177,25 @@ function flashman.update(remote_addr, app_id, app_secret)
   end
 end
 
+-- This function change the vlan configuration
+function flashman.set_vlan_config(vlan_config)
+  local result = run_process("sh -c \". /usr/share/functions/network_functions.sh; set_vlan_config ".. vlan_config .."\"")
+
+  if result ~= "ok" then
+    result = nil
+  end
+
+  return result
+end
+
+-- This function configures the vlan based on the 
+-- configuration already provided
+function flashman.configure_vlan()
+  run_process("sh -c \". /usr/share/functions/network_functions.sh; update_vlan \'y\'\"")
+
+  return
+end
+
 -- This function returns the vlan configuration json
 function flashman.get_vlan_config()
   local result = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_vlan_config\"")

From e18ae3134bf06d4b12c53a4b422280546c8e4221 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Thu, 12 May 2022 15:36:42 -0300
Subject: [PATCH 34/77] Moved vlan configuration after authentication.

---
 files/usr/share/anlix/index.lua | 74 +++++++++++++++------------------
 1 file changed, 34 insertions(+), 40 deletions(-)

diff --git a/files/usr/share/anlix/index.lua b/files/usr/share/anlix/index.lua
index 16bdb3ef..eb1d0255 100644
--- a/files/usr/share/anlix/index.lua
+++ b/files/usr/share/anlix/index.lua
@@ -170,46 +170,6 @@ function handle_request(env)
 		resp["data"] = data
 		web.send_json(resp)
 		return
-
-	-- Get the vlan configuration
-	elseif command == "getVlanConfig" then
-		local data = {}
-		local config = flashman.get_vlan_config()
-
-		-- Check if configuration is valid
-		if config == nil then
-			data["ok"] = false
-		else
-			data["ok"] = true
-			data["vlan_config"] = config
-		end
-
-		-- Send response
-		local resp = {}
-		resp["data"] = data
-		web.send_json(resp)
-		return
-
-	-- Get the ports to be used with vlan
-	elseif command == "getPorts" then
-		local data = {}
-		-- subcommands:
-		--   wan, lan, cpu
-		local ports = flashman.get_ports(subcommand)
-
-		-- Check if port is valid
-		if ports == nil then
-			data["ok"] = false
-		else
-			data["ok"] = true
-			data["ports"] = ports
-		end
-
-		-- Send response
-		local resp = {}
-		resp["data"] = data
-		web.send_json(resp)
-		return
 	end
 
 	if tonumber(app_protocol_ver) == 1 then
@@ -361,6 +321,40 @@ function handle_request(env)
 		local is_done = remove_from_file("/root/done_hashes", hash)
 		resp["command_done"] = is_done
 		web.send_json(resp)
+
+	-- Get the vlan configuration
+	elseif command == "getVlanConfig" then
+		local config = flashman.get_vlan_config()
+
+		-- Check if configuration is valid
+		if config == nil then
+			web.error_handle(web.ERROR_PARAMETERS, auth)
+			return
+		else
+			resp["vlan_config"] = config
+		end
+
+		-- Send response
+		web.send_json(resp)
+		return
+
+	-- Get the ports to be used with vlan
+	elseif command == "getPorts" then
+		-- subcommands:
+		--   wan, lan, cpu
+		local ports = flashman.get_ports(subcommand)
+
+		-- Check if port is valid
+		if ports == nil then
+			web.error_handle(web.ERROR_PARAMETERS, auth)
+			return
+		else
+			resp["ports"] = ports
+		end
+
+		-- Send response
+		web.send_json(resp)
+		return
 	else
 		web.error_handle(web.ERROR_CMD_UNKNOWN, auth)
 	end

From 44258829a18be4dd6d79b6d86d193449f1b2a08b Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 13 May 2022 11:22:47 -0300
Subject: [PATCH 35/77] data collection fixesg

---
 files/usr/share/data_collecting.sh | 83 +++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 30 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index f796d276..516468ac 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -10,6 +10,10 @@ dataCollectingDir="/tmp/data_collecting"
 rawDataFile="${dataCollectingDir}/raw"
 # directory where data will be stored if compressing old data is necessary.
 compressedDataDir="${dataCollectingDir}/compressed"
+# file where wan rx/tx bytes will be stored
+wanBytesFile="${dataCollectingDir}/wan_bytes"
+# file where wan rx/tx packets will be stored
+wanPacketsFile="${dataCollectingDir}/wan_pkts"
 
 # gets current rx and tx bytes/packets from wan interface and compares them
 # with values from previous calls to calculate cross traffic
@@ -19,13 +23,22 @@ collect_wan() {
 	# bytes sent by the interface.
 	local txBytes=$(get_wan_bytes_statistics TX)
 
-	# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
-	if [ -z "$last_rxBytes" ] || [ -z "$last_txBytes" ]; then
-		# echo last bytes are undefined
-		# bytes received by the interface. will be used next time.
-		last_rxBytes="$rxBytes"
-		# bytes sent by the interface. will be used next time.
-		last_txBytes="$txBytes"
+	local last_rxBytes=""
+	local last_txBytes=""
+
+	if [ -f "$wanBytesFile" ]; then
+		last_rxBytes=$(cat "$wanBytesFile")
+		# gets second value
+		last_txBytes=${last_rxBytes##* }
+		# gets first value
+		last_rxBytes=${last_rxBytes%% *}
+	fi
+
+	if [ "$last_rxBytes" == "" ] || [ "$last_txBytes" == "" ]; then
+		# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
+		# empty out file (we only need last minute info)
+		> "$wanBytesFile"
+		echo "$rxBytes $txBytes" >> "$wanBytesFile"
 		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
 		return -1
 	fi
@@ -47,13 +60,22 @@ collect_wan() {
 	# packets sent by the interface.
 	local txPackets=$(get_wan_packets_statistics TX)
 
-	# if last packets are not defined. define them using the current wan interface packets value. then we skip this measure.
-	if [ -z "$last_rxPackets" ] || [ -z "$last_txPackets" ]; then
-		# echo last packets are undefined
-		# packets received by the interface. will be used next time.
-		last_rxPackets="$rxPackets"
-		# packets sent by the interface. will be used next time.
-		last_txPackets="$txPackets"
+	local last_rxPackets=""
+	local last_txPackets=""
+
+	if [ -f "$wanPacketsFile" ]; then
+		last_rxPackets=$(cat "$wanPacketsFile")
+		# gets second value
+		last_txPackets=${last_rxPackets##* }
+		# gets first value
+		last_rxPackets=${last_rxPackets%% *}
+	fi
+
+	if [ "$last_rxPackets" == "" ] || [ "$last_txPackets" == "" ]; then
+		# if last packets are not defined. define them using the current wan interface Packets value. then we skip this measure.
+		# empty out file (we only need last minute info)
+		> "$wanPacketsFile"
+		echo "$rxPackets $txPackets" >> "$wanPacketsFile"
 		# don't write data this round. we need a full minute of packets to calculate cross traffic.
 		return -1
 	fi
@@ -71,8 +93,8 @@ collect_wan() {
 	last_txPackets=$txPackets
 
 	# data to be sent.
-	local string="$rxBytesDiff $txBytesDiff rxPacketsDiff $txPacketsDiff"
-	rawData="${rawData}|wan_stats ${string}"
+	local string="$rxBytesDiff $txBytesDiff $rxPacketsDiff $txPacketsDiff"
+	rawData="${rawData}|wanStats ${string}"
 }
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
@@ -80,7 +102,7 @@ collect_wan() {
 # their respective ping times. Builds a string with all this information and write them to file.
 collect_burst() {
 	# checking if this data collecting is enabled
-	[ "$burstLoss" -ne 1 ] && return
+	[ "$burstPing" -ne 1 ] && return
 
 	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
@@ -159,7 +181,7 @@ collect_burst() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	rawData="${rawData}|burstLoss ${string}"
+	rawData="${rawData}|burstPing ${string}"
 }
 
 collect_wifi_devices() {
@@ -183,14 +205,15 @@ collect_wifi_devices() {
 		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
 		# first iteration won't put a space before the value.
-		local first=true
+		local firstFileWrite=true
+		local firstRawWrite=true
 
 		# string to be appended to devices packets file
 		local fileStr=""
 
-		local fileName="${dataCollectingDir}/devices_24_pkts"
+		local lastPktsFile="${dataCollectingDir}/devices_24_pkts"
 		if [[ "$i" -eq 1 ]]; then
-			fileName="{dataCollectingDir}/devices_24_pkts"
+			lastPktsFile="{dataCollectingDir}/devices_5_pkts"
 		fi
 
 		while [ ${#devices} -gt 0 ]; do
@@ -230,14 +253,14 @@ collect_wifi_devices() {
 			[ "$rx_pkts" == "" ] && continue
 			[ "$tx_pkts" == "" ] && continue
 
-			[ "$first" == true ] && first=false || fileStr="$fileStr "
+			[ "$firstFileWrite" == true ] && firstFileWrite=false || fileStr="$fileStr "
 			fileStr="${fileStr}${deviceMac}_${rx_pkts}_${tx_pkts}"
 
 			local rx_pkts_diff=""
 			local tx_pkts_diff=""
 
-			if [ -f "$fileName" ]; then
-                local devices_pkts=$(cat "$fileName")
+			if [ -f "$lastPktsFile" ]; then
+                local devices_pkts=$(cat "$lastPktsFile")
 
 				local last_pkts=${devices_pkts#*"$deviceMac"_}
 				last_pkts=${last_pkts%% *}
@@ -258,14 +281,14 @@ collect_wifi_devices() {
 			fi
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
-			[ "$first" == true ] && first=false || str="$str "
-			str="${str}${wlan}_${deviceMac}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
+			[ "$firstRawWrite" == true ] && firstRawWrite=false || str="$str "
+			str="${str}${i}_${deviceMac}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
 		done
 		# empty out file (we only need last minute info)
-		> "$fileName"
+		> "$lastPktsFile"
 
 		# write to it if there are device infos
-		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$fileName"
+		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$lastPktsFile"
 	done
 	# we won't echo if there are no devices.
 	[ ${#str} -gt 0 ] && rawData="${rawData}|wifiDevices ${str}"
@@ -382,7 +405,7 @@ collectData() {
 	collect_wifi_devices
 
 	# example of an expected raw data with all measures present:
-	# '213234556456|burstLoss 0 100 1.246 0.161|wan_stats 12345 1234|wifiDevices aa:bb:cc:dd:ee:ff_22_12345_1234 ab:bb:cc:dd:ee:ff_45_1000_100'
+	# '213234556456|burstPing 0 100 1.246 0.161|wanStats 12345 1234 1234 123|wifiDevices 0_D0:9C:7A:EC:FF:FF_33_285_5136'
 	[ -n "$rawData" ] && echo "${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
@@ -642,7 +665,7 @@ loop() {
 			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
 			-e "pingServerAddress=@.data_collecting_ping_fqdn" \
 			-e "pingPackets=@.data_collecting_ping_packets" \
-			-e "burstLoss=@.data_collecting_burst_loss" \
+			-e "burstPing=@.data_collecting_burst_loss" \
 			-e "wifiDevices=@.data_collecting_wifi_devices" \
 		)
 

From 4b37cca81d8fd161aa1e4d0ad42e6f2fbeedb3fb Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 13 May 2022 18:03:26 -0300
Subject: [PATCH 36/77] changing '-1' returns for '1'

---
 files/usr/share/data_collecting.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 516468ac..237b9716 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -40,7 +40,7 @@ collect_wan() {
 		> "$wanBytesFile"
 		echo "$rxBytes $txBytes" >> "$wanBytesFile"
 		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
-		return -1
+		return 1
 	fi
 
 	# bytes received since last time.
@@ -49,7 +49,7 @@ collect_wan() {
 	local txBytesDiff=$(($txBytes - $last_txBytes))
 	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
 	# we skip this measure.
-	{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && return -1
+	{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && return 1
 	# saves current interface bytes value as last value.
 	last_rxBytes=$rxBytes
 	# saves current interface bytes value as last value.
@@ -77,7 +77,7 @@ collect_wan() {
 		> "$wanPacketsFile"
 		echo "$rxPackets $txPackets" >> "$wanPacketsFile"
 		# don't write data this round. we need a full minute of packets to calculate cross traffic.
-		return -1
+		return 1
 	fi
 
 	# packets received since last time.
@@ -86,7 +86,7 @@ collect_wan() {
 	local txPacketsDiff=$(($txPackets - $last_txPackets))
 	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
 	# we skip this measure.
-	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return -1
+	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return 1
 	# saves current interface packets value as last value.
 	last_rxPackets=$rxPackets
 	# saves current interface packets value as last value.

From 70099d7ebfa1aacb8b7c59594d872756eb72c05c Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Mon, 16 May 2022 13:54:30 -0300
Subject: [PATCH 37/77] Added boot configuration in flashman_init.conf

---
 Config.in                                     |   4 +
 Makefile                                      |   2 +-
 .../etc/uci-defaults/z002_flashbox-network.sh |   8 +-
 .../usr/share/functions/network_functions.sh  | 100 ++++++++----------
 4 files changed, 57 insertions(+), 57 deletions(-)

diff --git a/Config.in b/Config.in
index 59ac88a3..5998d257 100644
--- a/Config.in
+++ b/Config.in
@@ -143,6 +143,10 @@ config MQTT_PORT
 	string "MQTT port"
 	default "1883"
 
+config FLASHMAN_WAN_VLAN
+	string "VLAN WAN"
+	default "2"
+
 config FLASHMAN_USE_AUTH_SERVER
 	bool "Use authentication server to authorize remote commands."
 
diff --git a/Makefile b/Makefile
index 2e2c980a..5f5495b5 100644
--- a/Makefile
+++ b/Makefile
@@ -272,7 +272,7 @@ endif
 	echo 'FLM_WAN_PPPOE_USER=$(CONFIG_FLASHMAN_PPPOE_USER)' >>$(1)/usr/share/flashman_init.conf
 	echo 'FLM_WAN_PPPOE_PASSWD=$(CONFIG_FLASHMAN_PPPOE_PASSWD)' >>$(1)/usr/share/flashman_init.conf
 	echo 'FLM_WAN_PPPOE_SERVICE=$(CONFIG_FLASHMAN_PPPOE_SERVICE)' >>$(1)/usr/share/flashman_init.conf
-
+	echo 'FLM_WAN_VLAN=$(CONFIG_FLASHMAN_WAN_VLAN)' >>$(1)/usr/share/flashman_init.conf
 	echo 'FLM_USE_AUTH_SVADDR=$(CONFIG_FLASHMAN_USE_AUTH_SERVER)' >>$(1)/usr/share/flashman_init.conf
 
 ifeq ($(CONFIG_FLASHMAN_USE_AUTH_SERVER), y)
diff --git a/files/etc/uci-defaults/z002_flashbox-network.sh b/files/etc/uci-defaults/z002_flashbox-network.sh
index 39477ec0..9a40ffd8 100755
--- a/files/etc/uci-defaults/z002_flashbox-network.sh
+++ b/files/etc/uci-defaults/z002_flashbox-network.sh
@@ -83,13 +83,19 @@ uci set network.lan.netmask="$_lan_netmask"
 uci set network.lan.ip6assign="$_lan_ipv6prefix"
 uci set network.lan.igmp_snooping='1'
 uci set network.lan.stp='1'
-
+# Configure dmz
 uci set network.dmz=interface
 uci set network.dmz.proto='static'
 uci set network.dmz.netmask='24'
 uci set network.dmz.ifname='@lan'
 uci set network.dmz.ipaddr='192.168.43.1'
 uci set network.dmz.ipv6='0'
+# Configure vlan
+if [ -z "$(get_station_ifname 0 | grep ath)" ]
+then
+	configure_boot_vlan "$FLM_WAN_VLAN"
+	update_vlan "n"
+fi
 
 # Check custom wan type
 if [ "$_wan_conn_type" = "pppoe" ] || [ "$_wan_conn_type" = "dhcp" ]
diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index b4a045be..bec30ab5 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -1205,75 +1205,65 @@ save_bridge_mode_vlan_config() {
 }
 
 
-# This function is intended to be used with Tecnico's app
-# It saves the configuration received from config.lua
-set_vlan_config() {
-	# $1: Vlan Configuration
-	local _vlan_config=$1
-	local _result="ok"
-	local _ports=""
+# Create the first vlan_config.json
+configure_boot_vlan() {
+	# $1: Wan Vlan
+	local _wan_vlan=$1
 
 	json_cleanup
-	json_load "$_vlan_config"
-	json_get_values _ports
-	json_close_object
+	json_init
 
-	# Get ports
-	if [ "$(type -t custom_switch_ports)" ]
-	then
-		local _wan_port=$(custom_switch_ports 2) 
-		local _lan_ports=$(custom_switch_ports 3)
-		local _cpu_port=$(custom_switch_ports 4) 
-	else
-		local _wan_port=$(switch_ports 2) 
-		local _lan_ports=$(switch_ports 3)
-		local _cpu_port=$(switch_ports 4) 
-	fi
+	local _input=""
+	_input="$(uci show network | grep ].vlan=)"
 
-	# Loop every port and check if it's valid
-	for _port in $_ports; do
+	IFS=$'\n'
 
-		# CPU without tag do exist but is not included here
-		# With proprietary Atheros driver, the wan port does not exist
-		if [ "$_port" != "$_wan_port" ] && \
-		   [ "$_port" != "${_wan_port}t" ] && \
-		   [ "$_port" != "${_cpu_port}t" ]
-		then
+	local _idx=""
 
-			# Loop every lan port
-			for _lan_port in $_lan_ports; do
-				if [ "$_port" != "$_lan_port" ] && \
-				   [ "$_port" != "${_lan_port}t" ]
-				then
-					# Invalid parameters
-					_result=""
+	# Loop through every vlan that already exists
+	for _idx in $_input; do
 				
-				else
-					# Found a valid port, break
-					_result="ok"
-					break
-				fi
-			done
-
+		# Extract the switch-vlan idx from each line
+		_idx=${_idx#*[}
+		_idx=${_idx%]*}
+
+		# Get the ports
+		if [ "$(type -t custom_switch_ports)" ]; then
+			local _wan_port=$(custom_switch_ports 2) 
+			local _cpu_port=$(custom_switch_ports 4) 
+		else
+			local _wan_port=$(switch_ports 2) 
+			local _cpu_port=$(switch_ports 4) 
 		fi
 
-		# Check if result is valid, otherwise quit
-		if [[ -z "$_result" ]]
+		# Get the vlan and the ports
+		local _vlan="$(uci get network.@switch_vlan[${_idx}].vlan)"
+		local _ports="$(uci get network.@switch_vlan[${_idx}].ports)"
+
+		# Check if contains both wan and cpu port
+		if [ -n "$(echo "$_ports" | grep "${_wan_port}" | grep "${_cpu_port}")" ]
 		then
-			break
+			# Check if wan vlan port is not the default
+			if [ "${_wan_vlan}" != "$(get_default_vlan wan)" ]
+			then
+				_vlan="$_wan_vlan"
+				_ports="${_wan_port}t ${_cpu_port}t"
+
+			# Otherwise, no vlan in wan
+			else
+				_ports="${_wan_port} ${_cpu_port}t"
+			fi
 		fi
+
+		# Create the entry in the json "vlan": "ports"
+		json_add_string "${_vlan}" "${_ports}"
 	done
 
-	if [[ -n "$_ports" ]] && [[ -n "$_result" ]]
-	then
-		# Change the vlan json file
-		echo "$_vlan_config" > /root/vlan_config.json
-	else
-		# If the json could not be loaded, it is an error
-		_result=""
-	fi
+	IFS=$' '
 
-	echo "$_result"
+	json_dump > /root/vlan_config.json
+	json_dump >> /tmp/log.txt
+	json_close_object
 }
 
 

From edc0cc29e895247100bcba0364ae74e904a0c100 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Mon, 16 May 2022 13:56:16 -0300
Subject: [PATCH 38/77] Fixed typos and improved port checks.

---
 files/usr/share/functions/network_functions.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index bec30ab5..bd6f0f40 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -758,7 +758,7 @@ update_vlan() {
 			# Loop through every vlan that already exists
 			for _vlan in $_input; do
 				
-				# Extract the vlan id from each the line
+				# Extract the vlan id from each line
 				_vid=${_vlan#*\'}
 				_vid=${_vid%\'}
 
@@ -784,8 +784,8 @@ update_vlan() {
 					# If the wan configuration was changed
 					# TODO! This function is not implemented yet for Atheros, check if Atheros
 					# and based on vlan, configure the wan properly
-					if [[ -n "$(echo "$_ports" | grep "${_wan_port}t ${_cpu_port}t")" ]] || \
-					   [[ -n "$(echo "$_ports" | grep "${_wan_port} ${_cpu_port}t")" ]]; then
+					if [[ -n "$(echo "$_ports" | grep "${_wan_port}" | grep "${_cpu_port}")" ]]
+					then
 
 						# Change the virtual interface name of wan 
 						# interfaces
@@ -840,8 +840,8 @@ update_vlan() {
 					# If the wan configuration was changed
 					# TODO! This function is not implemented yet for Atheros, check if Atheros
 					# and based on vlan, configure the wan properly
-					if [[ -n "$(echo "$_ports" | grep "${_wan_port}t ${_cpu_port}t")" ]] || \
-					   [[ -n "$(echo "$_ports" | grep "${_wan_port} ${_cpu_port}t")" ]]; then
+					if [[ -n "$(echo "$_ports" | grep "${_wan_port}" | grep "${_cpu_port}")" ]]
+					then
 
 						# Change the virtual interface name of wan 
 						# interfaces
@@ -1305,13 +1305,13 @@ get_ports() {
 		local _cpu_port=$(switch_ports 4) 
 	fi
 
-	if [ "$_port_type" = "wan" ]; then
+	if [ "$_port_type" == "wan" ]; then
 		_ports="$_wan_port"
 
-	elif [ "$_port_type" = "lan" ]; then
+	elif [ "$_port_type" == "lan" ]; then
 		_ports="$_lan_ports"
 
-	elif [ "$_port_type" = "cpu" ]; then
+	elif [ "$_port_type" == "cpu" ]; then
 		_ports="$_cpu_port"
 
 	fi

From 76cfe8eeea6171dbc35c8095efa23d913a07d4cf Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Mon, 16 May 2022 13:58:30 -0300
Subject: [PATCH 39/77] Changed set vlan to only receive the vlan port.

---
 .../usr/share/functions/network_functions.sh  | 18 +++++++
 files/usr/share/lua/config.lua                | 11 ++---
 files/usr/share/lua/flashman.lua              | 47 +++++++++++++++++--
 files/usr/share/lua/webHandle.lua             |  6 +++
 4 files changed, 70 insertions(+), 12 deletions(-)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index bd6f0f40..0aa993ec 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -1318,4 +1318,22 @@ get_ports() {
 
 	# If the configuration sent to it is wrong, it will return an empty string
 	echo "$_ports"
+}
+
+
+# This function reports the default vlan for lan or wan
+get_default_vlan() {
+	# $1: Which port
+		# wan: Wan default vlan
+		# lan: Lan default vlan
+	local _port_type="$1"
+	local _vlan=""
+
+	if [ "$_port_type" == "lan" ]; then
+		_vlan="1"
+	else
+		_vlan="2"
+	fi
+
+	echo "$_vlan"
 }
\ No newline at end of file
diff --git a/files/usr/share/lua/config.lua b/files/usr/share/lua/config.lua
index b795ede6..5c87c1d2 100644
--- a/files/usr/share/lua/config.lua
+++ b/files/usr/share/lua/config.lua
@@ -115,17 +115,14 @@ function handle_config(command, data)
 	-- Create a new function with different name to 
 	-- not interfere with wan conn_type
 	elseif command == "wan-vlan" then
-		-- Change vlan and reply with ok so that
-		local status = flashman.set_vlan_config()
+		-- Change vlan and reply with ok
+		local result = flashman.set_vlan_wan(data.vlan)
 
-		if status == nil then
-			web.error_handle(web.ERROR_PARAMETERS, nil)
+		if result ~= "ok" then
+			web.error_handle(result, nil)
 		else
 			web.send_json({success = true}) -- reply before changing network
 			flashman.configure_vlan()
 		end
-		
-		return
 	end
-
 end
diff --git a/files/usr/share/lua/flashman.lua b/files/usr/share/lua/flashman.lua
index b0f9b4af..7c37d1ea 100644
--- a/files/usr/share/lua/flashman.lua
+++ b/files/usr/share/lua/flashman.lua
@@ -177,15 +177,52 @@ function flashman.update(remote_addr, app_id, app_secret)
   end
 end
 
+
 -- This function change the vlan configuration
-function flashman.set_vlan_config(vlan_config)
-  local result = run_process("sh -c \". /usr/share/functions/network_functions.sh; set_vlan_config ".. vlan_config .."\"")
+function flashman.set_vlan_wan(wan_vlan)
 
-  if result ~= "ok" then
-    result = nil
+  -- Open the vlan configuration file
+  local vlan_config = read_file("/root/vlan_config.json")
+
+  -- Check if the file is valid
+  if vlan_config == nil then
+    result = web.ERROR_OPEN_VLAN_CONFIG_FILE
+    return result
   end
 
-  return result
+  -- Decode the json
+  vlan_config = json.decode(vlan_config)
+
+  -- Get the wan and cpu port number
+  local wan_port = flashman.get_ports("wan")
+  local cpu_port = flashman.get_ports("cpu")
+
+  -- Loop through every key and value, ignoring the vlan of the wan
+  local new_vlan_config = {}
+
+  for key, value in pairs(vlan_config) do
+
+    -- If it can find both the cpu and wan port in the vlan, ignore
+    -- Otherwise, add to the new configuration
+    if not (string.find(value, wan_port) ~= nil and 
+        string.find(value, cpu_port) ~= nil) then
+      new_vlan_config[key] = value
+    end
+  end
+
+  -- Add the new configuration for wan
+  -- If the wan_vlan came empty or nil, assign the default configuration
+  if (wan_vlan == nil or wan_vlan == "")
+    wan_vlan = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_default_vlan \'wan\'\"")
+    new_vlan_config[wan_vlan] = wan_port .. " " .. cpu_port .. "t"
+  else
+    new_vlan_config[wan_vlan] = wan_port .. "t " .. cpu_port .. "t"
+  end
+
+  -- Write the new configuration
+  write_file("/root/vlan_config.json", json.encode(new_vlan_config))
+
+  return "ok"
 end
 
 -- This function configures the vlan based on the 
diff --git a/files/usr/share/lua/webHandle.lua b/files/usr/share/lua/webHandle.lua
index 4f377327..249601ab 100644
--- a/files/usr/share/lua/webHandle.lua
+++ b/files/usr/share/lua/webHandle.lua
@@ -16,6 +16,9 @@ web.ERROR_COMM_AUTH_PROVIDER = 22
 web.ERROR_AUTH_PROVIDER = 23
 web.ERROR_NO_CHANGE = 30
 
+-- VLAN Specific errors
+web.ERROR_OPEN_VLAN_CONFIG_FILE = 40		-- Vlan configuration file does not exist
+
 function web.error_string(errid)
 	if errid == web.ERROR_PROT_VER then return "Invalid Protocol Version"
 	elseif errid == web.ERROR_GEN_SECRET then return "Error generating secret for app"
@@ -32,6 +35,9 @@ function web.error_string(errid)
 	elseif errid == web.ERROR_COMM_AUTH_PROVIDER then return "Command need provider authorization"
 	elseif errid == web.ERROR_AUTH_PROVIDER then return "Provider Authorization Fail"
 	elseif errid == web.ERROR_NO_CHANGE then return "No changes to current configuration"
+
+	-- VLAN
+	elseif errid == web.ERROR_VLAN_CONFIG_FILE then return "Invalid Vlan configuration file"
 	else return "Unknown Error"
 	end
 end

From 82dd4c0eedd2ea6bd679eab263a7a0602380ed00 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Mon, 16 May 2022 16:47:51 -0300
Subject: [PATCH 40/77] Assure that Atheros don't change the wan vlan.

---
 files/usr/share/functions/network_functions.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index 0aa993ec..37556c54 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -720,16 +720,16 @@ update_wan_interfaces() {
 		_wan_ifname=${_old_wan_ifname%.*}
 	fi
 
-	# Assign it to wan and wan6 interface
-	# If it is Atheros and vlan = 2, clear the wan vlan
-	# If the _vlan_id come empty, clear Atheros wan vlan
-	# TODO! This function is partially implemented for Atheros
-	uci set network.wan.ifname="${_wan_ifname}.${_vlan_id}"
-	uci set network.wan6.ifname="${_wan_ifname}.${_vlan_id}"
-
 	# Assign it to dev interface if it is not Atheros
 	if [ -z "$(get_station_ifname 0 | grep ath)" ]
 	then
+		# Assign it to wan and wan6 interface
+		# If it is Atheros and vlan = 2, clear the wan vlan
+		# If the _vlan_id come empty, clear Atheros wan vlan
+		# TODO! This function is partially implemented for Atheros
+		uci set network.wan.ifname="${_wan_ifname}.${_vlan_id}"
+		uci set network.wan6.ifname="${_wan_ifname}.${_vlan_id}"
+
 		uci set network.wan_${_old_wan_ifname}_dev.name="${_wan_ifname}.${_vlan_id}"
 	fi
 

From 4ab4508f603c7380c005f3d0a151a7b98dd78e3f Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Tue, 17 May 2022 16:13:19 -0300
Subject: [PATCH 41/77] creating mechanism to pass forward to collection server
 which measurements are active

---
 files/usr/share/data_collecting.sh            | 205 +++++++++++++-----
 files/usr/share/flashman_update.sh            |   3 +-
 .../functions/data_collecting_functions.sh    |  15 +-
 files/usr/share/keepalive.sh                  |   4 +-
 4 files changed, 171 insertions(+), 56 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 237b9716..aa9dca24 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -18,6 +18,16 @@ wanPacketsFile="${dataCollectingDir}/wan_pkts"
 # gets current rx and tx bytes/packets from wan interface and compares them
 # with values from previous calls to calculate cross traffic
 collect_wan() {
+	# checking if this data collecting is enabled
+
+	local isBurstLossActive=${activeMeasures/*bl*/bl}
+
+	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
+
+	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
+
+	local sendThisRound=1
+
 	# bytes received by the interface.
 	local rxBytes=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
@@ -34,28 +44,48 @@ collect_wan() {
 		last_rxBytes=${last_rxBytes%% *}
 	fi
 
+	local rxBytesDiff=0
+	local txBytesDiff=0
+
 	if [ "$last_rxBytes" == "" ] || [ "$last_txBytes" == "" ]; then
 		# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
 		# empty out file (we only need last minute info)
 		> "$wanBytesFile"
 		echo "$rxBytes $txBytes" >> "$wanBytesFile"
 		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
-		return 1
+		sendThisRound=0
+	else
+		# bytes received since last time.
+		rxBytesDiff=$(($rxBytes - $last_rxBytes))
+		# bytes transmitted since last time
+		txBytesDiff=$(($txBytes - $last_txBytes))
+		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+		# we skip this measure.
+		{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && sendThisRound=0
 	fi
 
-	# bytes received since last time.
-	local rxBytesDiff=$(($rxBytes - $last_rxBytes))
-	# bytes transmitted since last time
-	local txBytesDiff=$(($txBytes - $last_txBytes))
-	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-	# we skip this measure.
-	{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && return 1
-	# saves current interface bytes value as last value.
-	last_rxBytes=$rxBytes
-	# saves current interface bytes value as last value.
-	last_txBytes=$txBytes
-
-		# packets received by the interface.
+	# empty out file (we only need last minute info)
+	> "$wanBytesFile"
+	echo "$rxBytes $txBytes" >> "$wanBytesFile"
+
+	if [ "$sendThisRound" -eq 1 ]; then
+		local string="$rxBytesDiff $txBytesDiff"
+		rawData="${rawData}|wanStats ${string}"
+	else
+		activeMeasures="${activeMeasures/bl /}"
+		activeMeasures="${activeMeasures/ bl/}"
+		activeMeasures="${activeMeasures/bl/}"
+		activeMeasures="${activeMeasures/p&w /}"
+		activeMeasures="${activeMeasures/ p&w/}"
+		activeMeasures="${activeMeasures/p&w/}"
+	fi
+
+	# burstLoss only gathers byte data
+	if [ "$isPingAndWanActive" != "p&w" ]; then
+		return
+	fi
+
+	# packets received by the interface.
 	local rxPackets=$(get_wan_packets_statistics RX)
 	# packets sent by the interface.
 	local txPackets=$(get_wan_packets_statistics TX)
@@ -71,30 +101,41 @@ collect_wan() {
 		last_rxPackets=${last_rxPackets%% *}
 	fi
 
+	local rxPacketsDiff=0
+	local txPacketsDiff=0
+
 	if [ "$last_rxPackets" == "" ] || [ "$last_txPackets" == "" ]; then
 		# if last packets are not defined. define them using the current wan interface Packets value. then we skip this measure.
 		# empty out file (we only need last minute info)
 		> "$wanPacketsFile"
 		echo "$rxPackets $txPackets" >> "$wanPacketsFile"
 		# don't write data this round. we need a full minute of packets to calculate cross traffic.
-		return 1
+		sendThisRound=0
+	else
+		# packets received since last time.
+		rxPacketsDiff=$(($rxPackets - $last_rxPackets))
+		# packets transmitted since last time
+		txPacketsDiff=$(($txPackets - $last_txPackets))
+		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+		# we skip this measure.
+		{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && sendThisRound=0
 	fi
 
-	# packets received since last time.
-	local rxPacketsDiff=$(($rxPackets - $last_rxPackets))
-	# packets transmitted since last time
-	local txPacketsDiff=$(($txPackets - $last_txPackets))
-	# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-	# we skip this measure.
-	{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && return 1
-	# saves current interface packets value as last value.
-	last_rxPackets=$rxPackets
-	# saves current interface packets value as last value.
-	last_txPackets=$txPackets
+	# empty out file (we only need last minute info)
+	> "$wanPacketsFile"
+	echo "$rxPackets $txPackets" >> "$wanPacketsFile"
+
+	# needs to gather one more minute
+	if [ "$sendThisRound" -ne 1 ]; then
+		activeMeasures="${activeMeasures/p&w /}"
+		activeMeasures="${activeMeasures/ p&w/}"
+		activeMeasures="${activeMeasures/p&w/}"
+		return
+	fi
 
 	# data to be sent.
-	local string="$rxBytesDiff $txBytesDiff $rxPacketsDiff $txPacketsDiff"
-	rawData="${rawData}|wanStats ${string}"
+	local string="$rxPacketsDiff $txPacketsDiff"
+	rawData="$rawData $string"
 }
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
@@ -102,7 +143,12 @@ collect_wan() {
 # their respective ping times. Builds a string with all this information and write them to file.
 collect_burst() {
 	# checking if this data collecting is enabled
-	[ "$burstPing" -ne 1 ] && return
+
+	local isBurstLossActive=${activeMeasures/*bl*/bl}
+
+	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
+
+	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
 
 	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
@@ -110,7 +156,15 @@ collect_burst() {
 	local pingError="$?"
 
 	# if ping could not be executed, we skip this measure.
-	[ "$pingError" -eq 2 ] && return;
+	if [ "$pingError" -eq 2 ]; then
+		activeMeasures="${activeMeasures/bl /}"
+		activeMeasures="${activeMeasures/ bl/}"
+		activeMeasures="${activeMeasures/bl/}"
+		activeMeasures="${activeMeasures/p&w /}"
+		activeMeasures="${activeMeasures/ p&w/}"
+		activeMeasures="${activeMeasures/p&w/}"
+		return
+	fi
 
 	# An skipped measure will become missing data, for this minute, in the server.
 
@@ -127,19 +181,25 @@ collect_burst() {
 	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
 	# loss=${loss##* } # removes everything before first space.
 
-	# removes everything before and including 'mdev = '
-	local latencyStats=${pingResult#*/mdev = }
-	# removes everything before first backslash
-	local latencyAvg=${latencyStats#*/}
-	# removes everything after first backslash
-	latencyAvg=${latencyAvg%%/*}
-	# removes everything before and including last backslash
-	local latencyStd=${latencyStats##*/}
-	# removes everything after and including first space
-	latencyStd=${latencyStd% *}
-
 	# data to be sent.
-	local string="$loss $transmitted $latencyAvg $latencyStd"
+	local string="$loss $transmitted"
+
+	# only collect latency stats if pingAndWan is activated
+	# burstLoss only collects loss data
+	if [ "$isPingAndWanActive" == "p&w" ]; then
+		# removes everything before and including 'mdev = '
+		local latencyStats=${pingResult#*/mdev = }
+		# removes everything before first backslash
+		local latencyAvg=${latencyStats#*/}
+		# removes everything after first backslash
+		latencyAvg=${latencyAvg%%/*}
+		# removes everything before and including last backslash
+		local latencyStd=${latencyStats##*/}
+		# removes everything after and including first space
+		latencyStd=${latencyStd% *}
+
+		string="$string $latencyAvg $latencyStd"
+	fi
 
 	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
@@ -181,7 +241,7 @@ collect_burst() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	rawData="${rawData}|burstPing ${string}"
+	rawData="${rawData}|pingStats ${string}"
 }
 
 collect_wifi_devices() {
@@ -213,7 +273,7 @@ collect_wifi_devices() {
 
 		local lastPktsFile="${dataCollectingDir}/devices_24_pkts"
 		if [[ "$i" -eq 1 ]]; then
-			lastPktsFile="{dataCollectingDir}/devices_5_pkts"
+			lastPktsFile="${dataCollectingDir}/devices_5_pkts"
 		fi
 
 		while [ ${#devices} -gt 0 ]; do
@@ -290,8 +350,14 @@ collect_wifi_devices() {
 		# write to it if there are device infos
 		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$lastPktsFile"
 	done
-	# we won't echo if there are no devices.
-	[ ${#str} -gt 0 ] && rawData="${rawData}|wifiDevices ${str}"
+	if [ "$str" == "" ]; then
+		# only send data if there is something to send
+		activeMeasures="${activeMeasures/wd /}"
+		activeMeasures="${activeMeasures/ wd/}"
+		activeMeasures="${activeMeasures/wd/}"
+	else
+		rawData="${rawData}|wifiDevsStats ${str}"
+	fi
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -396,17 +462,53 @@ removeOldFiles() {
 collectData() {
 	# getting current unix time in seconds.
 	local timestamp=$(date +%s)
+
 	# global variable where current raw data is stored before being written to file.
 	rawData=""
 
+	# global variable that controls which measures are active
+	activeMeasures=""
+
+	local firstMeasurement=1
+
+	if [ "$burstLoss" -eq 1 ]; then
+		if [ "$firstMeasurement" -ne 1 ]; then
+			# add space before active measurement name
+			activeMeasures="$activeMeasures "
+		else
+			firstMeasurement=0
+		fi
+		activeMeasures="${activeMeasures}bl"
+	fi
+
+	if [ "$pingAndWan" -eq 1 ]; then
+		if [ "$firstMeasurement" -ne 1 ]; then
+			# add space before active measurement name
+			activeMeasures="$activeMeasures "
+		else
+			firstMeasurement=0
+		fi
+		activeMeasures="${activeMeasures}p&w"
+	fi
+
+	if [ "$wifiDevices" -eq 1 ]; then
+		if [ "$firstMeasurement" -ne 1 ]; then
+			# add space before active measurement name
+			activeMeasures="$activeMeasures "
+		else
+			firstMeasurement=0
+		fi
+		activeMeasures="${activeMeasures}wd"
+	fi
+
 	# collecting all measures.
-	# only collect burst data if we have one minute of wan measurements already
-	collect_wan && collect_burst
+	collect_burst
+	collect_wan
 	collect_wifi_devices
 
 	# example of an expected raw data with all measures present:
-	# '213234556456|burstPing 0 100 1.246 0.161|wanStats 12345 1234 1234 123|wifiDevices 0_D0:9C:7A:EC:FF:FF_33_285_5136'
-	[ -n "$rawData" ] && echo "${timestamp}${rawData}" >> "$rawDataFile";
+	# 'bl p&w wd|ts 213234556456|pingStats 0 100 1.246 0.161|wanStats 12345 1234 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
+	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|ts ${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
 
@@ -665,8 +767,9 @@ loop() {
 			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
 			-e "pingServerAddress=@.data_collecting_ping_fqdn" \
 			-e "pingPackets=@.data_collecting_ping_packets" \
-			-e "burstPing=@.data_collecting_burst_loss" \
+			-e "burstLoss=@.data_collecting_burst_loss" \
 			-e "wifiDevices=@.data_collecting_wifi_devices" \
+			-e "pingAndWan=@.data_collecting_ping_and_wan" \
 		)
 
 		# does everything related to collecting and storing data.`
diff --git a/files/usr/share/flashman_update.sh b/files/usr/share/flashman_update.sh
index e93e7f70..fe9fa9de 100644
--- a/files/usr/share/flashman_update.sh
+++ b/files/usr/share/flashman_update.sh
@@ -248,6 +248,7 @@ bridge_fix_dns=$_local_bridge_fix_dns"
 		json_get_var _data_collecting_ping_packets data_collecting_ping_packets
 		json_get_var _data_collecting_burst_loss data_collecting_burst_loss
 		json_get_var _data_collecting_wifi_devices data_collecting_wifi_devices
+		json_get_var _data_collecting_ping_and_wan data_collecting_ping_and_wan
 		json_get_var _bridge_mode_enabled bridge_mode_enabled
 		json_get_var _bridge_mode_switch_disable bridge_mode_switch_disable
 		json_get_var _bridge_mode_ip bridge_mode_ip
@@ -509,7 +510,7 @@ bridge_fix_dns=$_local_bridge_fix_dns"
 		set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
 			                           "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
 			                           "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
-			                           "$_data_collecting_wifi_devices"
+			                           "$_data_collecting_wifi_devices" "$_data_collecting_ping_and_wan"
 
 
 		# Check for updates in port forward mapping
diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index 22014d6d..e42a8e59 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -12,6 +12,7 @@ set_data_collecting_parameters() {
 	local data_collecting_ping_packets="${5:-100}"
 	local data_collecting_burst_loss="${6:-0}"
 	local data_collecting_wifi_devices="${7:-0}"
+	local data_collecting_ping_and_wan="${8:-0}"
 
 	json_cleanup
 	json_load_file "/root/flashbox_config.json"
@@ -22,6 +23,7 @@ set_data_collecting_parameters() {
 	json_get_var saved_data_collecting_ping_packets data_collecting_ping_packets
 	json_get_var saved_data_collecting_burst_loss data_collecting_burst_loss
 	json_get_var saved_data_collecting_wifi_devices data_collecting_wifi_devices
+	json_get_var saved_data_collecting_ping_and_wan data_collecting_ping_and_wan
 
 	local anyChange=false
 
@@ -53,27 +55,34 @@ set_data_collecting_parameters() {
 		log "DATA_COLLECTING" "Updated 'data_collecting_ping_fqdn' parameter to '$data_collecting_ping_fqdn'."
 	fi
 
-	# Updating value if $data_collecting_alarm_fqdn has changed.
+	# Updating value if $data_collecting_ping_packets has changed.
 	if [ "$saved_data_collecting_ping_packets" != "$data_collecting_ping_packets" ]; then
 		anyChange=true
 		json_add_int data_collecting_ping_packets "$data_collecting_ping_packets"
 		log "DATA_COLLECTING" "Updated 'data_collecting_ping_packets' parameter to '$data_collecting_ping_packets'."
 	fi
 
-	# Updating value if $data_collecting_alarm_fqdn has changed.
+	# Updating value if $data_collecting_burst_loss has changed.
 	if [ "$saved_data_collecting_burst_loss" != "$data_collecting_burst_loss" ]; then
 		anyChange=true
 		json_add_boolean data_collecting_burst_loss "$data_collecting_burst_loss"
 		log "DATA_COLLECTING" "Updated 'data_collecting_burst_loss' parameter to '$data_collecting_burst_loss'."
 	fi
 
-	# Updating value if $data_collecting_alarm_fqdn has changed.
+	# Updating value if $data_collecting_wifi_devices has changed.
 	if [ "$saved_data_collecting_wifi_devices" != "$data_collecting_wifi_devices" ]; then
 		anyChange=true
 		json_add_boolean data_collecting_wifi_devices "$data_collecting_wifi_devices"
 		log "DATA_COLLECTING" "Updated 'data_collecting_wifi_devices' parameter to '$data_collecting_wifi_devices'."
 	fi
 
+	# Updating value if $data_collecting_ping_and_wan has changed.
+	if [ "$saved_data_collecting_ping_and_wan" != "$data_collecting_ping_and_wan" ]; then
+		anyChange=true
+		json_add_boolean data_collecting_ping_and_wan "$data_collecting_ping_and_wan"
+		log "DATA_COLLECTING" "Updated 'data_collecting_ping_and_wan' parameter to '$data_collecting_ping_and_wan'."
+	fi
+
 	# saving config json if any parameter has changed.
 	"$anyChange" && json_dump > /root/flashbox_config.json;
 	json_close_object
diff --git a/files/usr/share/keepalive.sh b/files/usr/share/keepalive.sh
index cc71e587..466ef875 100755
--- a/files/usr/share/keepalive.sh
+++ b/files/usr/share/keepalive.sh
@@ -122,6 +122,7 @@ wpsstate=$_local_wps_state"
 			json_get_var _data_collecting_ping_packets data_collecting_ping_packets
 			json_get_var _data_collecting_burst_loss data_collecting_burst_loss
 			json_get_var _data_collecting_wifi_devices data_collecting_wifi_devices
+			json_get_var _data_collecting_ping_and_wan data_collecting_ping_and_wan
 			json_close_object
 
 			if [ "$_do_newprobe" = "1" ]
@@ -172,7 +173,8 @@ wpsstate=$_local_wps_state"
 			# updates data collecting parameters.
 			set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
 			                               "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
-			                               "$_data_collecting_ping_packets" "$_data_collecting_burst_loss"
+			                               "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
+										   "$_data_collecting_ping_and_wan"
 
 		elif [ $_retstatus -eq 2 ]
 		then

From 9fe74bc24f46d98770cdaf5786d3ed4d5e81ea57 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Thu, 19 May 2022 17:50:46 -0300
Subject: [PATCH 42/77] removing 'ts' from measurement string passed to
 collection server

---
 files/usr/share/data_collecting.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index aa9dca24..2b6216f7 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -508,7 +508,7 @@ collectData() {
 
 	# example of an expected raw data with all measures present:
 	# 'bl p&w wd|ts 213234556456|pingStats 0 100 1.246 0.161|wanStats 12345 1234 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
-	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|ts ${timestamp}${rawData}" >> "$rawDataFile";
+	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
 

From a51b81ecbe972789a753d3c94db0dc256bdfb6f9 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Thu, 19 May 2022 18:47:47 -0300
Subject: [PATCH 43/77] changing raw format to pass to collection server

---
 files/usr/share/data_collecting.sh | 38 +++++++++++++-----------------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 2b6216f7..a321c381 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -69,8 +69,8 @@ collect_wan() {
 	echo "$rxBytes $txBytes" >> "$wanBytesFile"
 
 	if [ "$sendThisRound" -eq 1 ]; then
-		local string="$rxBytesDiff $txBytesDiff"
-		rawData="${rawData}|wanStats ${string}"
+		local string="|wanBytes $rxBytesDiff $txBytesDiff"
+		rawData="$rawData $string"
 	else
 		activeMeasures="${activeMeasures/bl /}"
 		activeMeasures="${activeMeasures/ bl/}"
@@ -134,7 +134,7 @@ collect_wan() {
 	fi
 
 	# data to be sent.
-	local string="$rxPacketsDiff $txPacketsDiff"
+	local string="|wanPkts $rxPacketsDiff $txPacketsDiff"
 	rawData="$rawData $string"
 }
 
@@ -184,22 +184,18 @@ collect_burst() {
 	# data to be sent.
 	local string="$loss $transmitted"
 
-	# only collect latency stats if pingAndWan is activated
-	# burstLoss only collects loss data
-	if [ "$isPingAndWanActive" == "p&w" ]; then
-		# removes everything before and including 'mdev = '
-		local latencyStats=${pingResult#*/mdev = }
-		# removes everything before first backslash
-		local latencyAvg=${latencyStats#*/}
-		# removes everything after first backslash
-		latencyAvg=${latencyAvg%%/*}
-		# removes everything before and including last backslash
-		local latencyStd=${latencyStats##*/}
-		# removes everything after and including first space
-		latencyStd=${latencyStd% *}
-
-		string="$string $latencyAvg $latencyStd"
-	fi
+	# removes everything before and including 'mdev = '
+	local latencyStats=${pingResult#*/mdev = }
+	# removes everything before first backslash
+	local latencyAvg=${latencyStats#*/}
+	# removes everything after first backslash
+	latencyAvg=${latencyAvg%%/*}
+	# removes everything before and including last backslash
+	local latencyStd=${latencyStats##*/}
+	# removes everything after and including first space
+	latencyStd=${latencyStd% *}
+
+	string="$string $latencyAvg $latencyStd"
 
 	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
@@ -241,7 +237,7 @@ collect_burst() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	rawData="${rawData}|pingStats ${string}"
+	rawData="${rawData}|burstPing $string"
 }
 
 collect_wifi_devices() {
@@ -507,7 +503,7 @@ collectData() {
 	collect_wifi_devices
 
 	# example of an expected raw data with all measures present:
-	# 'bl p&w wd|ts 213234556456|pingStats 0 100 1.246 0.161|wanStats 12345 1234 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
+	# 'bl p&w wd|213234556456|burstPing 0 100 1.246 0.161|wanBytes 12345 1234|wanPkts 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
 	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""

From 474f1c85a28d327c5f43dda306a4f5c0866bce41 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Thu, 19 May 2022 19:15:00 -0300
Subject: [PATCH 44/77] adding important comment

---
 files/usr/share/data_collecting.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index a321c381..691b828f 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -502,6 +502,11 @@ collectData() {
 	collect_wan
 	collect_wifi_devices
 
+	# mapping from measurement names to collected artifacts:
+	# bl (burstLoss) -> burstPing, wanBytes
+	# p&w (pingAndWan) -> burstPing, wanBytes, wanPkts
+	# wd (wifiDevices) -> wifiDevsStats
+
 	# example of an expected raw data with all measures present:
 	# 'bl p&w wd|213234556456|burstPing 0 100 1.246 0.161|wanBytes 12345 1234|wanPkts 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
 	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";

From f12ac90473f3d1bba20b07aa07b2d0729818b3f6 Mon Sep 17 00:00:00 2001
From: lucasperovani <lucasperovani@gmail.com>
Date: Fri, 20 May 2022 08:51:28 -0300
Subject: [PATCH 45/77] Fixed if typo.

---
 files/usr/share/lua/flashman.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/usr/share/lua/flashman.lua b/files/usr/share/lua/flashman.lua
index 7c37d1ea..770f32ba 100644
--- a/files/usr/share/lua/flashman.lua
+++ b/files/usr/share/lua/flashman.lua
@@ -212,7 +212,7 @@ function flashman.set_vlan_wan(wan_vlan)
 
   -- Add the new configuration for wan
   -- If the wan_vlan came empty or nil, assign the default configuration
-  if (wan_vlan == nil or wan_vlan == "")
+  if (wan_vlan == nil or wan_vlan == "") then
     wan_vlan = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_default_vlan \'wan\'\"")
     new_vlan_config[wan_vlan] = wan_port .. " " .. cpu_port .. "t"
   else

From e2d656e8d52c7f6f63719dc6d55a58918f1b11b6 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 9 Jun 2022 18:58:12 -0300
Subject: [PATCH 46/77] correcting raw data formatting

---
 files/usr/share/data_collecting.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 691b828f..e45c3028 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -70,7 +70,7 @@ collect_wan() {
 
 	if [ "$sendThisRound" -eq 1 ]; then
 		local string="|wanBytes $rxBytesDiff $txBytesDiff"
-		rawData="$rawData $string"
+		rawData="${rawData}${string}"
 	else
 		activeMeasures="${activeMeasures/bl /}"
 		activeMeasures="${activeMeasures/ bl/}"
@@ -135,7 +135,7 @@ collect_wan() {
 
 	# data to be sent.
 	local string="|wanPkts $rxPacketsDiff $txPacketsDiff"
-	rawData="$rawData $string"
+	rawData="${rawData}${string}"
 }
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
@@ -193,7 +193,7 @@ collect_burst() {
 	# removes everything before and including last backslash
 	local latencyStd=${latencyStats##*/}
 	# removes everything after and including first space
-	latencyStd=${latencyStd% *}
+	latencyStd=${latencyStd%% *}
 
 	string="$string $latencyAvg $latencyStd"
 
@@ -232,7 +232,7 @@ collect_burst() {
 		# prints final $string in this sub shell back to $string.
 		echo $pairs))
 		# appending latencies to string to be sent.
-		string="${string} ${latencies}"
+		[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
 	fi
 	
 	# appending string to file.

From 32b88c6f07909e0b1de62d0646d6ea2aa8e8a14a Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Sun, 12 Jun 2022 12:11:48 -0300
Subject: [PATCH 47/77] correcting wifi devices section

---
 files/usr/share/data_collecting.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index e45c3028..f6d1c0c1 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -318,6 +318,11 @@ collect_wifi_devices() {
 			if [ -f "$lastPktsFile" ]; then
                 local devices_pkts=$(cat "$lastPktsFile")
 
+				local pkts_device_removed=${devices_pkts/"$deviceMac"/""}
+
+				# if deviceMac not in recorded devices continue
+				[ ${#pkts_device_removed} -ge ${#devices_pkts} ] && continue
+
 				local last_pkts=${devices_pkts#*"$deviceMac"_}
 				last_pkts=${last_pkts%% *}
 

From 2eda491c0424553272e446d0d319fdf890917fd1 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Sun, 12 Jun 2022 15:03:17 -0300
Subject: [PATCH 48/77] correcting wireless devices raw formatting

---
 files/usr/share/data_collecting.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index f6d1c0c1..921bfceb 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -247,6 +247,8 @@ collect_wifi_devices() {
 	# devices and their data will be stored in this string variable.
 	local str=""
 
+	local firstRawWrite=true
+
 	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
 		# getting wifi interface name.
@@ -262,7 +264,6 @@ collect_wifi_devices() {
 
 		# first iteration won't put a space before the value.
 		local firstFileWrite=true
-		local firstRawWrite=true
 
 		# string to be appended to devices packets file
 		local fileStr=""

From 13dedf93a7536d76809ee04561249c0ff8486259 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 7 Jul 2022 20:27:28 -0300
Subject: [PATCH 49/77] correcting data collecting ping_and_wan update on
 keepalive

---
 files/usr/share/keepalive.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/usr/share/keepalive.sh b/files/usr/share/keepalive.sh
index 466ef875..74e3ce95 100755
--- a/files/usr/share/keepalive.sh
+++ b/files/usr/share/keepalive.sh
@@ -174,7 +174,7 @@ wpsstate=$_local_wps_state"
 			set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
 			                               "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
 			                               "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
-										   "$_data_collecting_ping_and_wan"
+										   "$_data_collecting_wifi_devices" "$_data_collecting_ping_and_wan"
 
 		elif [ $_retstatus -eq 2 ]
 		then

From 7ea7db518c51cf03ffdd1a4605e5ca115d580043 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 14 Jul 2022 15:53:55 -0300
Subject: [PATCH 50/77] adding wifi signal to collected measures

---
 files/usr/share/data_collecting.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 921bfceb..4eb3a833 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -278,6 +278,12 @@ collect_wifi_devices() {
 			# getting everything before the first space.
 			local deviceMac=${devices%% *}
 
+			# getting everything after the first two spaces.
+			local signal=${devices#*  }
+
+			# getting everything before the first occasion of ' dBm'
+			signal=${signal%% dBm*}
+
 			# getting after '(SNR '. 
 			devices=${devices#*\(SNR }
 
@@ -344,7 +350,7 @@ collect_wifi_devices() {
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$firstRawWrite" == true ] && firstRawWrite=false || str="$str "
-			str="${str}${i}_${deviceMac}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
+			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
 		done
 		# empty out file (we only need last minute info)
 		> "$lastPktsFile"

From e04d19f17f8fcde76171e8efb0da56de517e2eaf Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 14 Jul 2022 19:00:30 -0300
Subject: [PATCH 51/77] treating case where there is 100% packet loss

---
 files/usr/share/data_collecting.sh | 34 ++++++++++++++++++------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 4eb3a833..17aaaeda 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -169,11 +169,11 @@ collect_burst() {
 	# An skipped measure will become missing data, for this minute, in the server.
 
 	# removes everything behind the summary that appears in the last lines.
-	pingResult=${pingResult##* ping statistics ---[$'\r\n']}
+	local pingResultAux=${pingResult##* ping statistics ---[$'\r\n']}
 	# removes everything after, and including, ' packets transmitted'.
-	local transmitted=${pingResult% packets transmitted*}
+	local transmitted=${pingResultAux% packets transmitted*}
 	# removes everything after, and including, ' received'.
-	local received=${pingResult% received*}
+	local received=${pingResultAux% received*}
 	# removes everything before first space.
 	received=${received##* }
 	# integer representing the amount of packets not received.
@@ -186,16 +186,24 @@ collect_burst() {
 
 	# removes everything before and including 'mdev = '
 	local latencyStats=${pingResult#*/mdev = }
-	# removes everything before first backslash
-	local latencyAvg=${latencyStats#*/}
-	# removes everything after first backslash
-	latencyAvg=${latencyAvg%%/*}
-	# removes everything before and including last backslash
-	local latencyStd=${latencyStats##*/}
-	# removes everything after and including first space
-	latencyStd=${latencyStd%% *}
-
-	string="$string $latencyAvg $latencyStd"
+
+	# when there is 100% packet loss there the strings remain equal
+	# we only want to collect latency and std when there isn't 100% loss
+	# if loss is 100% we just send 0 in both cases, which will be ignored by the server
+	if [ ${#latencyStats} == ${#pingResult} ]; then
+	    string="$string 0 0"
+	else
+		# removes everything before first backslash
+		local latencyAvg=${latencyStats#*/}
+		# removes everything after first backslash
+		latencyAvg=${latencyAvg%%/*}
+		# removes everything before and including last backslash
+		local latencyStd=${latencyStats##*/}
+		# removes everything after and including first space
+		latencyStd=${latencyStd%% *}
+
+		string="$string $latencyAvg $latencyStd"
+	fi
 
 	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then

From ab9871abc39fa7d8fe99945d1e00671466888dad Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 11 Aug 2022 17:53:33 -0300
Subject: [PATCH 52/77] fixing case where signal appears as unknown

---
 files/usr/share/data_collecting.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 17aaaeda..0bf810ed 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -289,6 +289,12 @@ collect_wifi_devices() {
 			# getting everything after the first two spaces.
 			local signal=${devices#*  }
 
+			# getting everything before the first occasion of ' /'
+			signal=${signal%% /*}
+
+			# if unknown replace it with -95 dBm
+			[ "$signal" == "unknown" ] && signal="-95 dBm"
+
 			# getting everything before the first occasion of ' dBm'
 			signal=${signal%% dBm*}
 

From 7273ea3d080f39468095d468101e4daea95f18eb Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 11 Aug 2022 17:56:07 -0300
Subject: [PATCH 53/77] fixing case where signal appears as unknown, now
 discarding

---
 files/usr/share/data_collecting.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 0bf810ed..7a6ca697 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -292,8 +292,8 @@ collect_wifi_devices() {
 			# getting everything before the first occasion of ' /'
 			signal=${signal%% /*}
 
-			# if unknown replace it with -95 dBm
-			[ "$signal" == "unknown" ] && signal="-95 dBm"
+			# if unknown discard
+			[ "$signal" == "unknown" ] && continue
 
 			# getting everything before the first occasion of ' dBm'
 			signal=${signal%% dBm*}

From c052b3689a9551bcfd28578639499392aa3b18b7 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 15 Sep 2022 19:09:25 -0300
Subject: [PATCH 54/77] fixing wifi devices bug

---
 files/usr/share/data_collecting.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 7a6ca697..23993e2a 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -293,7 +293,7 @@ collect_wifi_devices() {
 			signal=${signal%% /*}
 
 			# if unknown discard
-			[ "$signal" == "unknown" ] && continue
+			[ "$signal" == "unknown" ] && devices=${devices#*$'\n'} && continue
 
 			# getting everything before the first occasion of ' dBm'
 			signal=${signal%% dBm*}

From 80e838f952664491726157892788c178958ba0c4 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 15 Sep 2022 19:33:19 -0300
Subject: [PATCH 55/77] synchronizing with dev branch

---
 Config.in                                     |   4 -
 Makefile                                      |   1 -
 custom-files/tbs.sh                           |   4 +-
 .../etc/uci-defaults/z002_flashbox-network.sh |   6 -
 files/usr/share/anlix/index.lua               |  34 ---
 .../usr/share/functions/network_functions.sh  | 262 +-----------------
 files/usr/share/lua/config.lua                |  13 -
 files/usr/share/lua/flashman.lua              |  78 ------
 files/usr/share/lua/webHandle.lua             |   6 -
 9 files changed, 13 insertions(+), 395 deletions(-)

diff --git a/Config.in b/Config.in
index 3e03c9a4..e8b30630 100644
--- a/Config.in
+++ b/Config.in
@@ -143,10 +143,6 @@ config MQTT_PORT
 	string "MQTT port"
 	default "1883"
 
-config FLASHMAN_WAN_VLAN
-	string "VLAN WAN"
-	default "2"
-
 config FLASHMAN_USE_AUTH_SERVER
 	bool "Use authentication server to authorize remote commands."
 
diff --git a/Makefile b/Makefile
index 9852d69d..2ba7e8e4 100644
--- a/Makefile
+++ b/Makefile
@@ -273,7 +273,6 @@ endif
 	echo 'FLM_WAN_PPPOE_USER=$(CONFIG_FLASHMAN_PPPOE_USER)' >>$(1)/usr/share/flashman_init.conf
 	echo 'FLM_WAN_PPPOE_PASSWD=$(CONFIG_FLASHMAN_PPPOE_PASSWD)' >>$(1)/usr/share/flashman_init.conf
 	echo 'FLM_WAN_PPPOE_SERVICE=$(CONFIG_FLASHMAN_PPPOE_SERVICE)' >>$(1)/usr/share/flashman_init.conf
-	echo 'FLM_WAN_VLAN=$(CONFIG_FLASHMAN_WAN_VLAN)' >>$(1)/usr/share/flashman_init.conf
 	echo 'FLM_USE_AUTH_SVADDR=$(CONFIG_FLASHMAN_USE_AUTH_SERVER)' >>$(1)/usr/share/flashman_init.conf
 	echo 'FLM_CONNECTIVITY_SVADDRS_LIST=$(CONFIG_CONNECTIVITY_SVADDRS_LIST)' >>$(1)/usr/share/flashman_init.conf
 
diff --git a/custom-files/tbs.sh b/custom-files/tbs.sh
index fecee3ee..1b1c98fa 100644
--- a/custom-files/tbs.sh
+++ b/custom-files/tbs.sh
@@ -10,9 +10,7 @@ get_custom_mac() {
 	local _mac_address_tag=""
 	local _p1
 
-	local _wan_ifname=$(uci show network | sed -n "s/network.wan_\(.*\)_dev=device/\1/p")
-
-	_p1=$(uci get network.wan_${_wan_ifname}_dev.macaddr | awk '{print toupper($1)}')
+	_p1=$(uci get network.wan_eth0_2_dev.macaddr | awk '{print toupper($1)}')
 	[ ! -z "$_p1" ] && _mac_address_tag=$_p1
 
 	echo "$_mac_address_tag"
diff --git a/files/etc/uci-defaults/z002_flashbox-network.sh b/files/etc/uci-defaults/z002_flashbox-network.sh
index 9a40ffd8..ca3eec71 100755
--- a/files/etc/uci-defaults/z002_flashbox-network.sh
+++ b/files/etc/uci-defaults/z002_flashbox-network.sh
@@ -90,12 +90,6 @@ uci set network.dmz.netmask='24'
 uci set network.dmz.ifname='@lan'
 uci set network.dmz.ipaddr='192.168.43.1'
 uci set network.dmz.ipv6='0'
-# Configure vlan
-if [ -z "$(get_station_ifname 0 | grep ath)" ]
-then
-	configure_boot_vlan "$FLM_WAN_VLAN"
-	update_vlan "n"
-fi
 
 # Check custom wan type
 if [ "$_wan_conn_type" = "pppoe" ] || [ "$_wan_conn_type" = "dhcp" ]
diff --git a/files/usr/share/anlix/index.lua b/files/usr/share/anlix/index.lua
index eb1d0255..5b38d957 100644
--- a/files/usr/share/anlix/index.lua
+++ b/files/usr/share/anlix/index.lua
@@ -321,40 +321,6 @@ function handle_request(env)
 		local is_done = remove_from_file("/root/done_hashes", hash)
 		resp["command_done"] = is_done
 		web.send_json(resp)
-
-	-- Get the vlan configuration
-	elseif command == "getVlanConfig" then
-		local config = flashman.get_vlan_config()
-
-		-- Check if configuration is valid
-		if config == nil then
-			web.error_handle(web.ERROR_PARAMETERS, auth)
-			return
-		else
-			resp["vlan_config"] = config
-		end
-
-		-- Send response
-		web.send_json(resp)
-		return
-
-	-- Get the ports to be used with vlan
-	elseif command == "getPorts" then
-		-- subcommands:
-		--   wan, lan, cpu
-		local ports = flashman.get_ports(subcommand)
-
-		-- Check if port is valid
-		if ports == nil then
-			web.error_handle(web.ERROR_PARAMETERS, auth)
-			return
-		else
-			resp["ports"] = ports
-		end
-
-		-- Send response
-		web.send_json(resp)
-		return
 	else
 		web.error_handle(web.ERROR_CMD_UNKNOWN, auth)
 	end
diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index e6edb93a..e6a54bcf 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -661,47 +661,6 @@ get_bridge_mode_status() {
 	echo "$_status"
 }
 
-# This function changes wan interfaces to use the new virtual interface
-# after changing the vlan configuration for wan
-update_wan_interfaces() {
-	# $1: New virtual interface name
-	local _vlan_id=$1
-	local _old_wan_ifname=""
-	local _wan_ifname=""
-
-	# Get the old configuration wan ifname
-	if [ -z "$(get_station_ifname 0 | grep ath)" ]
-	then
-		# This field only exists for Mediatek and Realtek
-		_old_wan_ifname=$(uci show network | sed -n "s/network.wan_\(.*\)_dev=device/\1/p")
-
-		# Remove the vlan id from the wan ifname
-		# Ex.: eth0 from eth0_2
-		_wan_ifname=${_old_wan_ifname%_*}
-	else
-		_old_wan_ifname=$(uci show network | sed -n "s/network.wan.ifname=\'\(.*\)\' /\1/p")
-
-		# Remove the vlan id from the wan ifname
-		# Ex.: eth1 from eth1.10
-		_wan_ifname=${_old_wan_ifname%.*}
-	fi
-
-	# Assign it to dev interface if it is not Atheros
-	if [ -z "$(get_station_ifname 0 | grep ath)" ]
-	then
-		# Assign it to wan and wan6 interface
-		# If it is Atheros and vlan = 2, clear the wan vlan
-		# If the _vlan_id come empty, clear Atheros wan vlan
-		# TODO! This function is partially implemented for Atheros
-		uci set network.wan.ifname="${_wan_ifname}.${_vlan_id}"
-		uci set network.wan6.ifname="${_wan_ifname}.${_vlan_id}"
-
-		uci set network.wan_${_old_wan_ifname}_dev.name="${_wan_ifname}.${_vlan_id}"
-	fi
-
-	# Commit the changes
-	uci commit network
-}
 
 update_vlan() {
 	if [ -f /root/vlan_config.json ]; then
@@ -721,100 +680,37 @@ update_vlan() {
 			local _vids=''
 			local _idx=0
 
-			# Loop through every vlan that already exists
 			for _vlan in $_input; do
-				
-				# Extract the vlan id from each line
 				_vid=${_vlan#*\'}
 				_vid=${_vid%\'}
-
-				# Indicates that the vlan that already exists in config/network
-				# do exists in the new vlan config file (vlan_config.json)
-				if [ -n "$(echo "${_vlans} " | grep "${_vid} ")" ]
-				then
-					
-					# Reassign the ports to the vlan
-					# If it is an Atheros and wan configuration, bypass
+				local _test=${_vlans#*$_vid}
+				# Indicates _vid is in _vlans
+				if [ $(( ${#_test} < ${#_vlans} )) = 1 ]; then
 					json_get_var _ports $_vid
 					uci set network.@switch_vlan[$_idx].ports="$_ports"
-
-					# Get the wan and cpu port
-					if [ "$(type -t custom_switch_ports)" ]; then
-						local _wan_port=$(custom_switch_ports 2) 
-						local _cpu_port=$(custom_switch_ports 4) 
-					else
-						local _wan_port=$(switch_ports 2) 
-						local _cpu_port=$(switch_ports 4) 
-					fi
-
-					# If the wan configuration was changed
-					# TODO! This function is not implemented yet for Atheros, check if Atheros
-					# and based on vlan, configure the wan properly
-					if [[ -n "$(echo "$_ports" | grep "${_wan_port}" | grep "${_cpu_port}")" ]]
-					then
-
-						# Change the virtual interface name of wan 
-						# interfaces
-						$(update_wan_interfaces $_vid)
-
-					fi
-
-					# Concatenate vlan ids
-					if [ "$_vids" == '' ]; then
-						_vids="$_vid"
-					else
-						_vids="$_vids $_vid"
-					fi
-
-				# Indicates that the vlan that already exists in config/network
-				# do not exists in the new vlan config file (vlan_config.json)
-				else
-
-					# Delete the switch_vlan entry
+				else # _vid isn't in _vlans
 					uci delete network.@switch_vlan[$_idx]
 					_idx=$(( _idx - 1 ))
 				fi
-
-				# Next switch_vlan in config/network
+				if [ "$_vids" = '' ]; then
+					_vids="$_vid"
+				else
+					_vids="$_vids $_vid"
+				fi
 				_idx=$(( _idx + 1 ))
 			done
 
 			IFS=$' '
 
-			# Loop though every entry in vlan_config.json
 			for _vlan in $_vlans; do
-
-				# If the vlan id does not exists yet in config/network
-				if [ -z "$(echo "${_vids} " | grep "${_vlan} ")" ]; then
-
-					# Create the new entry
+				_test=${_vids#*$_vlan}
+				# Indicates _vlan isn't in _vids
+				if [ $(( ${#_test} < ${#_vids} )) = 0 ]; then
 					json_get_var _ports $_vlan
 					uci add network switch_vlan
 					uci set network.@switch_vlan[-1].device="$(get_switch_device)"
 					uci set network.@switch_vlan[-1].vlan="$_vlan"
 					uci set network.@switch_vlan[-1].ports="$_ports"
-
-					# Get the wan and cpu port
-					if [ "$(type -t custom_switch_ports)" ]; then
-						local _wan_port=$(custom_switch_ports 2) 
-						local _cpu_port=$(custom_switch_ports 4) 
-					else
-						local _wan_port=$(switch_ports 2) 
-						local _cpu_port=$(switch_ports 4) 
-					fi
-
-					# If the wan configuration was changed
-					# TODO! This function is not implemented yet for Atheros, check if Atheros
-					# and based on vlan, configure the wan properly
-					if [[ -n "$(echo "$_ports" | grep "${_wan_port}" | grep "${_cpu_port}")" ]]
-					then
-
-						# Change the virtual interface name of wan 
-						# interfaces
-						$(update_wan_interfaces $_vlan)
-
-					fi
-
 				fi
 			done
 
@@ -1169,137 +1065,3 @@ save_bridge_mode_vlan_config() {
 	echo "$_vlan" > /root/vlan_config.json
 	update_vlan "n"
 }
-
-
-# Create the first vlan_config.json
-configure_boot_vlan() {
-	# $1: Wan Vlan
-	local _wan_vlan=$1
-
-	json_cleanup
-	json_init
-
-	local _input=""
-	_input="$(uci show network | grep ].vlan=)"
-
-	IFS=$'\n'
-
-	local _idx=""
-
-	# Loop through every vlan that already exists
-	for _idx in $_input; do
-				
-		# Extract the switch-vlan idx from each line
-		_idx=${_idx#*[}
-		_idx=${_idx%]*}
-
-		# Get the ports
-		if [ "$(type -t custom_switch_ports)" ]; then
-			local _wan_port=$(custom_switch_ports 2) 
-			local _cpu_port=$(custom_switch_ports 4) 
-		else
-			local _wan_port=$(switch_ports 2) 
-			local _cpu_port=$(switch_ports 4) 
-		fi
-
-		# Get the vlan and the ports
-		local _vlan="$(uci get network.@switch_vlan[${_idx}].vlan)"
-		local _ports="$(uci get network.@switch_vlan[${_idx}].ports)"
-
-		# Check if contains both wan and cpu port
-		if [ -n "$(echo "$_ports" | grep "${_wan_port}" | grep "${_cpu_port}")" ]
-		then
-			# Check if wan vlan port is not the default
-			if [ "${_wan_vlan}" != "$(get_default_vlan wan)" ]
-			then
-				_vlan="$_wan_vlan"
-				_ports="${_wan_port}t ${_cpu_port}t"
-
-			# Otherwise, no vlan in wan
-			else
-				_ports="${_wan_port} ${_cpu_port}t"
-			fi
-		fi
-
-		# Create the entry in the json "vlan": "ports"
-		json_add_string "${_vlan}" "${_ports}"
-	done
-
-	IFS=$' '
-
-	json_dump > /root/vlan_config.json
-	json_dump >> /tmp/log.txt
-	json_close_object
-}
-
-
-# This function is intended to be used with Tecnico's app
-# It returns the vlan configuration as a json
-get_vlan_config() {
-
-	local _vlan_json=""
-
-	if [ -f /root/vlan_config.json ]; then
-		json_cleanup
-		json_load_file /root/vlan_config.json
-		json_close_object
-		_vlan_json=`json_dump`
-	fi
-
-	echo "$_vlan_json"
-}
-
-
-# This function is intended to be used with Tecnico's app
-# It returns wan, cpu or lan ports, to check possible configurations
-get_ports() {
-	# $1: Which port
-		# wan: Wan port
-		# lan: Lan ports
-		# cpu: CPU port
-	local _port_type="$1"
-	local _ports=""
-
-	if [ "$(type -t custom_switch_ports)" ]
-	then
-		local _wan_port=$(custom_switch_ports 2) 
-		local _lan_ports=$(custom_switch_ports 3)
-		local _cpu_port=$(custom_switch_ports 4) 
-	else
-		local _wan_port=$(switch_ports 2) 
-		local _lan_ports=$(switch_ports 3)
-		local _cpu_port=$(switch_ports 4) 
-	fi
-
-	if [ "$_port_type" == "wan" ]; then
-		_ports="$_wan_port"
-
-	elif [ "$_port_type" == "lan" ]; then
-		_ports="$_lan_ports"
-
-	elif [ "$_port_type" == "cpu" ]; then
-		_ports="$_cpu_port"
-
-	fi
-
-	# If the configuration sent to it is wrong, it will return an empty string
-	echo "$_ports"
-}
-
-
-# This function reports the default vlan for lan or wan
-get_default_vlan() {
-	# $1: Which port
-		# wan: Wan default vlan
-		# lan: Lan default vlan
-	local _port_type="$1"
-	local _vlan=""
-
-	if [ "$_port_type" == "lan" ]; then
-		_vlan="1"
-	else
-		_vlan="2"
-	fi
-
-	echo "$_vlan"
-}
diff --git a/files/usr/share/lua/config.lua b/files/usr/share/lua/config.lua
index 5c87c1d2..53df79b3 100644
--- a/files/usr/share/lua/config.lua
+++ b/files/usr/share/lua/config.lua
@@ -111,18 +111,5 @@ function handle_config(command, data)
 				flashman.update_bridge(disable_switch, ip, gateway, dns)
 			end
 		end
-
-	-- Create a new function with different name to 
-	-- not interfere with wan conn_type
-	elseif command == "wan-vlan" then
-		-- Change vlan and reply with ok
-		local result = flashman.set_vlan_wan(data.vlan)
-
-		if result ~= "ok" then
-			web.error_handle(result, nil)
-		else
-			web.send_json({success = true}) -- reply before changing network
-			flashman.configure_vlan()
-		end
 	end
 end
diff --git a/files/usr/share/lua/flashman.lua b/files/usr/share/lua/flashman.lua
index 770f32ba..a41d867d 100644
--- a/files/usr/share/lua/flashman.lua
+++ b/files/usr/share/lua/flashman.lua
@@ -177,82 +177,4 @@ function flashman.update(remote_addr, app_id, app_secret)
   end
 end
 
-
--- This function change the vlan configuration
-function flashman.set_vlan_wan(wan_vlan)
-
-  -- Open the vlan configuration file
-  local vlan_config = read_file("/root/vlan_config.json")
-
-  -- Check if the file is valid
-  if vlan_config == nil then
-    result = web.ERROR_OPEN_VLAN_CONFIG_FILE
-    return result
-  end
-
-  -- Decode the json
-  vlan_config = json.decode(vlan_config)
-
-  -- Get the wan and cpu port number
-  local wan_port = flashman.get_ports("wan")
-  local cpu_port = flashman.get_ports("cpu")
-
-  -- Loop through every key and value, ignoring the vlan of the wan
-  local new_vlan_config = {}
-
-  for key, value in pairs(vlan_config) do
-
-    -- If it can find both the cpu and wan port in the vlan, ignore
-    -- Otherwise, add to the new configuration
-    if not (string.find(value, wan_port) ~= nil and 
-        string.find(value, cpu_port) ~= nil) then
-      new_vlan_config[key] = value
-    end
-  end
-
-  -- Add the new configuration for wan
-  -- If the wan_vlan came empty or nil, assign the default configuration
-  if (wan_vlan == nil or wan_vlan == "") then
-    wan_vlan = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_default_vlan \'wan\'\"")
-    new_vlan_config[wan_vlan] = wan_port .. " " .. cpu_port .. "t"
-  else
-    new_vlan_config[wan_vlan] = wan_port .. "t " .. cpu_port .. "t"
-  end
-
-  -- Write the new configuration
-  write_file("/root/vlan_config.json", json.encode(new_vlan_config))
-
-  return "ok"
-end
-
--- This function configures the vlan based on the 
--- configuration already provided
-function flashman.configure_vlan()
-  run_process("sh -c \". /usr/share/functions/network_functions.sh; update_vlan \'y\'\"")
-
-  return
-end
-
--- This function returns the vlan configuration json
-function flashman.get_vlan_config()
-  local result = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_vlan_config\"")
-  
-  if result == nil or result == "" then
-    result = nil
-  end
-
-  return result
-end
-
--- This function returns the ports in use
-function flashman.get_ports(port_name)
-  local result = run_process("sh -c \". /usr/share/functions/network_functions.sh; get_ports ".. port_name .."\"")
-  
-  if result == nil or result == "" then
-    result = nil
-  end
-
-  return result
-end
-
 return flashman
diff --git a/files/usr/share/lua/webHandle.lua b/files/usr/share/lua/webHandle.lua
index 249601ab..4f377327 100644
--- a/files/usr/share/lua/webHandle.lua
+++ b/files/usr/share/lua/webHandle.lua
@@ -16,9 +16,6 @@ web.ERROR_COMM_AUTH_PROVIDER = 22
 web.ERROR_AUTH_PROVIDER = 23
 web.ERROR_NO_CHANGE = 30
 
--- VLAN Specific errors
-web.ERROR_OPEN_VLAN_CONFIG_FILE = 40		-- Vlan configuration file does not exist
-
 function web.error_string(errid)
 	if errid == web.ERROR_PROT_VER then return "Invalid Protocol Version"
 	elseif errid == web.ERROR_GEN_SECRET then return "Error generating secret for app"
@@ -35,9 +32,6 @@ function web.error_string(errid)
 	elseif errid == web.ERROR_COMM_AUTH_PROVIDER then return "Command need provider authorization"
 	elseif errid == web.ERROR_AUTH_PROVIDER then return "Provider Authorization Fail"
 	elseif errid == web.ERROR_NO_CHANGE then return "No changes to current configuration"
-
-	-- VLAN
-	elseif errid == web.ERROR_VLAN_CONFIG_FILE then return "Invalid Vlan configuration file"
 	else return "Unknown Error"
 	end
 end

From 27954002dd8796f087ba124d5c5df90a1a47f71b Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 15 Sep 2022 19:36:19 -0300
Subject: [PATCH 56/77] config file read

---
 files/usr/share/functions/data_collecting_functions.sh | 2 +-
 files/usr/share/functions/network_functions.sh         | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index e42a8e59..17c1cde6 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -15,7 +15,7 @@ set_data_collecting_parameters() {
 	local data_collecting_ping_and_wan="${8:-0}"
 
 	json_cleanup
-	json_load_file "/root/flashbox_config.json"
+	json_load_file /root/flashbox_config.json
 	json_get_var saved_data_collecting_is_active data_collecting_is_active
 	json_get_var saved_data_collecting_has_latency data_collecting_has_latency
 	json_get_var saved_data_collecting_alarm_fqdn data_collecting_alarm_fqdn
diff --git a/files/usr/share/functions/network_functions.sh b/files/usr/share/functions/network_functions.sh
index e6a54bcf..aa072166 100644
--- a/files/usr/share/functions/network_functions.sh
+++ b/files/usr/share/functions/network_functions.sh
@@ -661,7 +661,6 @@ get_bridge_mode_status() {
 	echo "$_status"
 }
 
-
 update_vlan() {
 	if [ -f /root/vlan_config.json ]; then
 		local _restart_network=$1

From 221fd43fe901a32a4e6640a260ee74e88d1c2809 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Tue, 27 Sep 2022 19:19:43 -0300
Subject: [PATCH 57/77] handling case where SNR appears as equal to signal

---
 files/usr/share/data_collecting.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 23993e2a..787fd3d1 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -304,6 +304,9 @@ collect_wifi_devices() {
 			# getting everything before the first closing parenthesis.
 			local snr=${devices%%\)*}
 
+			# if SNR equals signal we assume noise of -95dBm
+			[ $signal -eq $snr ] && snr=$(($signal+95))
+
 			# getting everything before ' ms'.
 			local time=${devices%% ms*}
 

From 932dfdae274ce2fb40551df2e1e8a6fc5919df1a Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Wed, 28 Sep 2022 18:08:48 -0300
Subject: [PATCH 58/77] removing comments that don't add much information
 because of flash limitation

---
 files/usr/share/data_collecting.sh | 298 +----------------------------
 1 file changed, 3 insertions(+), 295 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 787fd3d1..10daec37 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -4,33 +4,22 @@
 . /usr/share/functions/custom_wireless_driver.sh
 . /usr/share/flashman_init.conf
 
-# directory where all data related to data collecting will be stored.
 dataCollectingDir="/tmp/data_collecting"
-# file where collected data will be stored before being compressed.
 rawDataFile="${dataCollectingDir}/raw"
-# directory where data will be stored if compressing old data is necessary.
 compressedDataDir="${dataCollectingDir}/compressed"
-# file where wan rx/tx bytes will be stored
 wanBytesFile="${dataCollectingDir}/wan_bytes"
-# file where wan rx/tx packets will be stored
 wanPacketsFile="${dataCollectingDir}/wan_pkts"
 
-# gets current rx and tx bytes/packets from wan interface and compares them
-# with values from previous calls to calculate cross traffic
 collect_wan() {
-	# checking if this data collecting is enabled
 
 	local isBurstLossActive=${activeMeasures/*bl*/bl}
-
 	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
 
 	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
 
 	local sendThisRound=1
 
-	# bytes received by the interface.
 	local rxBytes=$(get_wan_bytes_statistics RX)
-	# bytes sent by the interface.
 	local txBytes=$(get_wan_bytes_statistics TX)
 
 	local last_rxBytes=""
@@ -38,9 +27,7 @@ collect_wan() {
 
 	if [ -f "$wanBytesFile" ]; then
 		last_rxBytes=$(cat "$wanBytesFile")
-		# gets second value
 		last_txBytes=${last_rxBytes##* }
-		# gets first value
 		last_rxBytes=${last_rxBytes%% *}
 	fi
 
@@ -48,23 +35,15 @@ collect_wan() {
 	local txBytesDiff=0
 
 	if [ "$last_rxBytes" == "" ] || [ "$last_txBytes" == "" ]; then
-		# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
-		# empty out file (we only need last minute info)
 		> "$wanBytesFile"
 		echo "$rxBytes $txBytes" >> "$wanBytesFile"
-		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
 		sendThisRound=0
 	else
-		# bytes received since last time.
 		rxBytesDiff=$(($rxBytes - $last_rxBytes))
-		# bytes transmitted since last time
 		txBytesDiff=$(($txBytes - $last_txBytes))
-		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-		# we skip this measure.
 		{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && sendThisRound=0
 	fi
 
-	# empty out file (we only need last minute info)
 	> "$wanBytesFile"
 	echo "$rxBytes $txBytes" >> "$wanBytesFile"
 
@@ -80,14 +59,11 @@ collect_wan() {
 		activeMeasures="${activeMeasures/p&w/}"
 	fi
 
-	# burstLoss only gathers byte data
 	if [ "$isPingAndWanActive" != "p&w" ]; then
 		return
 	fi
 
-	# packets received by the interface.
 	local rxPackets=$(get_wan_packets_statistics RX)
-	# packets sent by the interface.
 	local txPackets=$(get_wan_packets_statistics TX)
 
 	local last_rxPackets=""
@@ -95,9 +71,7 @@ collect_wan() {
 
 	if [ -f "$wanPacketsFile" ]; then
 		last_rxPackets=$(cat "$wanPacketsFile")
-		# gets second value
 		last_txPackets=${last_rxPackets##* }
-		# gets first value
 		last_rxPackets=${last_rxPackets%% *}
 	fi
 
@@ -105,27 +79,18 @@ collect_wan() {
 	local txPacketsDiff=0
 
 	if [ "$last_rxPackets" == "" ] || [ "$last_txPackets" == "" ]; then
-		# if last packets are not defined. define them using the current wan interface Packets value. then we skip this measure.
-		# empty out file (we only need last minute info)
 		> "$wanPacketsFile"
 		echo "$rxPackets $txPackets" >> "$wanPacketsFile"
-		# don't write data this round. we need a full minute of packets to calculate cross traffic.
 		sendThisRound=0
 	else
-		# packets received since last time.
 		rxPacketsDiff=$(($rxPackets - $last_rxPackets))
-		# packets transmitted since last time
 		txPacketsDiff=$(($txPackets - $last_txPackets))
-		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-		# we skip this measure.
 		{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && sendThisRound=0
 	fi
 
-	# empty out file (we only need last minute info)
 	> "$wanPacketsFile"
 	echo "$rxPackets $txPackets" >> "$wanPacketsFile"
 
-	# needs to gather one more minute
 	if [ "$sendThisRound" -ne 1 ]; then
 		activeMeasures="${activeMeasures/p&w /}"
 		activeMeasures="${activeMeasures/ p&w/}"
@@ -133,29 +98,20 @@ collect_wan() {
 		return
 	fi
 
-	# data to be sent.
 	local string="|wanPkts $rxPacketsDiff $txPacketsDiff"
 	rawData="${rawData}${string}"
 }
 
-# takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
-# If latency collecting is enabled, extracts the individual icmp request numbers and 
-# their respective ping times. Builds a string with all this information and write them to file.
 collect_burst() {
-	# checking if this data collecting is enabled
 
 	local isBurstLossActive=${activeMeasures/*bl*/bl}
-
 	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
 
 	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
 
-	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
-	# ping return value.
 	local pingError="$?"
 
-	# if ping could not be executed, we skip this measure.
 	if [ "$pingError" -eq 2 ]; then
 		activeMeasures="${activeMeasures/bl /}"
 		activeMeasures="${activeMeasures/ bl/}"
@@ -166,25 +122,14 @@ collect_burst() {
 		return
 	fi
 
-	# An skipped measure will become missing data, for this minute, in the server.
-
-	# removes everything behind the summary that appears in the last lines.
 	local pingResultAux=${pingResult##* ping statistics ---[$'\r\n']}
-	# removes everything after, and including, ' packets transmitted'.
 	local transmitted=${pingResultAux% packets transmitted*}
-	# removes everything after, and including, ' received'.
 	local received=${pingResultAux% received*}
-	# removes everything before first space.
 	received=${received##* }
-	# integer representing the amount of packets not received.
 	local loss=$(($transmitted - $received))
-	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
-	# loss=${loss##* } # removes everything before first space.
 
-	# data to be sent.
 	local string="$loss $transmitted"
 
-	# removes everything before and including 'mdev = '
 	local latencyStats=${pingResult#*/mdev = }
 
 	# when there is 100% packet loss there the strings remain equal
@@ -193,87 +138,56 @@ collect_burst() {
 	if [ ${#latencyStats} == ${#pingResult} ]; then
 	    string="$string 0 0"
 	else
-		# removes everything before first backslash
 		local latencyAvg=${latencyStats#*/}
-		# removes everything after first backslash
 		latencyAvg=${latencyAvg%%/*}
-		# removes everything before and including last backslash
 		local latencyStd=${latencyStats##*/}
-		# removes everything after and including first space
 		latencyStd=${latencyStd%% *}
 
 		string="$string $latencyAvg $latencyStd"
 	fi
 
-	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
-		# echo collecting latencies
-		# removing the first line and the last 4 lines. only the ping lines remain.
 		local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
 		local pairs=""
 		local firstLine=true
-		# for each ping line.
 		while read line; do
-			# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
 			reached=${line%time=*}
-			# if "time=" has actually been removed, it means that line 
-			# contains it, which also means the icmp request was fulfilled.
-			# if line doesn't contain 'time=', we skip this line.
 			[ ${#reached} -lt ${#line} ] || continue
 
-			 # from the whole line, removes everything until, and including, "icmp_req=".
 			pingNumber=${line#*icmp_*eq=}
-			# removes everything after the first space.
 			pingNumber=${pingNumber%% *}
-			# from the whole line, removes everything until, and including, "time=".
 			pingTime=${line#*time=}
-			# removes everything after the first space.
 			pingTime=${pingTime%% *}
 			if [ "$firstLine" = true ]; then
 				firstLine=false
 			else
 				pairs="${pairs},"
 			fi
-			# concatenate to $string.
 			pairs="${pairs}${pingNumber}=${pingTime}"
 		done
-		# prints final $string in this sub shell back to $string.
 		echo $pairs))
-		# appending latencies to string to be sent.
 		[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
 	fi
 	
-	# appending string to file.
-	# printf "string is: '%s'\n" "$string"
 	rawData="${rawData}|burstPing $string"
 }
 
 collect_wifi_devices() {
-	# checking if this data collecting is enabled
 	[ "$wifiDevices" -ne 1 ] && return
 
-	# devices and their data will be stored in this string variable.
 	local str=""
 
 	local firstRawWrite=true
 
-	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
-		# getting wifi interface name.
-		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
 		local wlan=$(get_root_ifname "$i" 2> /dev/null)
-		# if interface doesn't exist, skips this iteration.
 		[ "$wlan" == "" ] && continue
-		# getting info from each connected device on wifi. 
-		# grep returns empty when no devices are connected or if interface doesn't exist.
 		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
 		local devices_rx_pkts="$(iwinfo "$wlan" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
 		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
-		# first iteration won't put a space before the value.
 		local firstFileWrite=true
 
-		# string to be appended to devices packets file
 		local fileStr=""
 
 		local lastPktsFile="${dataCollectingDir}/devices_24_pkts"
@@ -283,45 +197,24 @@ collect_wifi_devices() {
 
 		while [ ${#devices} -gt 0 ]; do
 
-			# getting everything before the first space.
 			local deviceMac=${devices%% *}
 
-			# getting everything after the first two spaces.
 			local signal=${devices#*  }
-
-			# getting everything before the first occasion of ' /'
 			signal=${signal%% /*}
-
-			# if unknown discard
 			[ "$signal" == "unknown" ] && devices=${devices#*$'\n'} && continue
-
-			# getting everything before the first occasion of ' dBm'
 			signal=${signal%% dBm*}
 
-			# getting after '(SNR '. 
 			devices=${devices#*\(SNR }
 
-			# getting everything before the first closing parenthesis.
 			local snr=${devices%%\)*}
-
-			# if SNR equals signal we assume noise of -95dBm
 			[ $signal -eq $snr ] && snr=$(($signal+95))
 
-			# getting everything before ' ms'.
 			local time=${devices%% ms*}
-
-			# getting everything after the first space.
 			time=${time##* }
 
-			# getting everything after 'ago'.
 			devices=${devices#*ago}
-
-			# getting everything after '\n', if it exists. last line won't have it, so nothing will be changed.
-			# we can't add the line feed along witht the previous parameter expansion because we wouldn't match
-			# the last line and so we wouldn't make $devices length become zero.
 			devices=${devices#*$'\n'}
 
-			# if $time is greater than one minute, we don't use this device's info.
 			[ "$time" -gt 60000 ] && continue
 			
 			local rx_pkts=${devices_rx_pkts%% *}
@@ -344,7 +237,6 @@ collect_wifi_devices() {
 
 				local pkts_device_removed=${devices_pkts/"$deviceMac"/""}
 
-				# if deviceMac not in recorded devices continue
 				[ ${#pkts_device_removed} -ge ${#devices_pkts} ] && continue
 
 				local last_pkts=${devices_pkts#*"$deviceMac"_}
@@ -358,25 +250,19 @@ collect_wifi_devices() {
 				[ "$last_tx_pkts" == "" ] && continue
 				tx_pkts_diff=$(($tx_pkts - $last_tx_pkts))
 
-				# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-				# we skip this measure.
 				{ [ "$rx_pkts_diff" -lt 0 ] || [ "$tx_pkts_diff" -lt 0 ]; } && continue
 			else
 				continue
 			fi
 
-			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$firstRawWrite" == true ] && firstRawWrite=false || str="$str "
 			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
 		done
-		# empty out file (we only need last minute info)
 		> "$lastPktsFile"
 
-		# write to it if there are device infos
 		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$lastPktsFile"
 	done
 	if [ "$str" == "" ]; then
-		# only send data if there is something to send
 		activeMeasures="${activeMeasures/wd /}"
 		activeMeasures="${activeMeasures/ wd/}"
 		activeMeasures="${activeMeasures/wd/}"
@@ -385,120 +271,71 @@ collect_wifi_devices() {
 	fi
 }
 
-# prints the size of a file, using 'ls', where full file path is given as 
-# first argument ($1).
 fileSize() {
-	# file size is the information at the 1st column.
 	local wcline=$(wc -c "$1")
-	#remove suffix composed of space and anything else.
 	local size=${wcline% *}
 	echo $size
 }
 
-# prints the sum of the sizes of all files inside given directory path.
 sumFileSizesInPath() {
-	# boolean that marks that at least one file exists inside given directory.
 	local anyFile=false
-	# for each file in that directory.
 	for i in "$1"/*; do
-		# if that pattern expansion exists as a file.
 		[ -f "$i" ] || continue
-		# set boolean to true.
 		anyFile=true
-		# as we have at least one file, we don't need to loop through all files.
 		break
 	done
 	if [ "$anyFile" = false ]; then
-		# if no files.
-		# prints zero. size of nothing is 0.
 		echo 0
-		# result was given, we can leave function.
 		return 0
 	fi
-	# if there is at least one file.
 
-	# prints a list of sizes and files.
 	local wcResult=$(wc -c "$1"/*)
-	# if there is 2 or more files, the last line will have a "total". remove that string.
 	local hasTotal=${wcResult% total}
-	# if it has a total, it was removed. if not, nothing was removed and both strings are the same.
 
-	 # if length of string with "total" removed is smaller than original string.
 	if [ ${#hasTotal} -lt ${#wcResult} ]; then
-		# remove everything before the last word, which is the value for total, and print what remains.
 		echo ${hasTotal##* }
 	else
-		# if there were no total, then there was only one file, in one line of output, and the first column is the size value.
-		# remove everything past, and including, the first space, and print what remains.
 		echo ${wcResult%% *}
 	fi
 }
 
-# sum the size $rawDataFile with the size of all compressed files inside the $compressedDataDir. if that 
-# sum is bigger than number given in first argument ($1), compress that file and move it to $compressedDataDir.
 zipFile() {
 	local capSize="$1"
 
-	# if file doesn't exist, we won't have to compressed anything.
-	# a return of 1 means nothing has been be gzipped.
 	[ -f "$rawDataFile" ] || return 1 
 
-	# size of file with raw data.
 	local size=$(fileSize "$rawDataFile")
-	# sum of file sizes in directory for compressed files.
 	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
 
-	# if sum is smaller than $capSize, do nothing.
-	# echo checking file size to zip
-	# a return of 1 means nothing will be gzipped.
 	[ $(($size + $dirSize)) -lt $capSize ] && return 1
 
-	# compressing file where raw data is held.
 	gzip "$rawDataFile"
-	# move newly compressed file to directory where compressed files should be.
 	mv "${rawDataFile}.gz" "$compressedDataDir/$(date +%s).gz"
 }
 
-# files are removed from $compressedDataDir until all data remaining is below number given in first 
-# argument ($1) as bytes. As files are named using a formatted date, pattern expansion of file 
-# names will always order oldest files first. This was done this way so we don't have to sort files 
-# by date ourselves. we let shell do the sorting.
 removeOldFiles() {
 	local capSize="$1"
 
-	# get the sum of sizes of all files in bytes.
 	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
 
-	# if $dirSize is more than given $capSize, remove oldest file. which is 
-	# the file that shell orders as first.
 	for i in "$compressedDataDir"/*; do
-		# if we are under $capSize. do nothing.
 		[ $dirSize -lt $capSize ] && break;
-		# removes that file.
 		rm "$i"
-		# subtract that file's size from sum.
 		dirSize=$(($dirSize - $(fileSize "$i")))
 	done
 }
 
-# collect every data and stores in '$rawDataFile'. if the size of the file is 
-# too big, compress it and move it to a directory of compressed files. If 
-# directory of compressed files grows too big delete oldest compressed files.
 collectData() {
-	# getting current unix time in seconds.
 	local timestamp=$(date +%s)
 
-	# global variable where current raw data is stored before being written to file.
 	rawData=""
 
-	# global variable that controls which measures are active
 	activeMeasures=""
 
 	local firstMeasurement=1
 
 	if [ "$burstLoss" -eq 1 ]; then
 		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
 			activeMeasures="$activeMeasures "
 		else
 			firstMeasurement=0
@@ -508,7 +345,6 @@ collectData() {
 
 	if [ "$pingAndWan" -eq 1 ]; then
 		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
 			activeMeasures="$activeMeasures "
 		else
 			firstMeasurement=0
@@ -518,7 +354,6 @@ collectData() {
 
 	if [ "$wifiDevices" -eq 1 ]; then
 		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
 			activeMeasures="$activeMeasures "
 		else
 			firstMeasurement=0
@@ -526,7 +361,6 @@ collectData() {
 		activeMeasures="${activeMeasures}wd"
 	fi
 
-	# collecting all measures.
 	collect_burst
 	collect_wan
 	collect_wifi_devices
@@ -539,10 +373,8 @@ collectData() {
 	# example of an expected raw data with all measures present:
 	# 'bl p&w wd|213234556456|burstPing 0 100 1.246 0.161|wanBytes 12345 1234|wanPkts 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
 	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
-	# cleaning 'rawData' value from memory.
 	rawData=""
 
-	# creates directory of for compressed files, if it doesn't already exists.
 	mkdir -p "$compressedDataDir"
 	# $(zipFile) returns 0 only if any amount of files has been compressed 
 	# and, consequently, moved to the directory of compressed files. So
@@ -561,27 +393,18 @@ collectData() {
 	# the uncompressed file will always have available for it's growth.
 }
 
-# if number given as first argument ($1) isn't 0, ping server at address given 
-# in second argument ($2) and returns the exit code of $(curl), but if that 
-# number is zero, return that number.
 checkServerState() {
 	local lastState="$1"
 	if [ "$lastState" -ne "0" ]; then
-		# echo pinging alarm server to check if it's alive.
 		curl -s -m 10 "https://$alarmServerAddress:7890/ping" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" > /dev/null
-		# return $(curl) exit code.
 		lastState="$?"
 	fi
-	# echo last state is $lastState
 	return $lastState
 }
 
-# sends file at given path ($1) to server at given address ($2) using $(curl) 
-# and returns $(curl) exit code.
 sendToServer() {
 	local filepath="$1"
 
-	# 'get_mac()' is defined in /usr/share/functions/device_functions.sh.
 	local mac=$(get_mac);
 
 	status=$(curl --write-out '%{http_code}' -s -m 20 --connect-timeout 5 --output /dev/null \
@@ -589,160 +412,74 @@ sendToServer() {
 	-H 'Content-Type: text/plain' -H "X-ANLIX-ID: $mac" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" \
 	-H "Send-Time: $(date +%s)" --data-binary @"$filepath")
 	curlCode="$?"
-	# 'log()' is defined in /usr/share/functions/common_functions.sh.
+
 	[ "$curlCode" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${curlCode}'." && return "$curlCode"
 	log "DATA_COLLECTING" "Data sent with response status code '${status}'."
 	[ "$status" -ge 200 ] && [ "$status" -lt 300 ] && return 0
 	return 1
 }
 
-# for each compressed file given in $compressedDataDir, send that file to a $alarmServerAddress. 
-# If any sending is unsuccessful, stops sending files and return it's exit code.
 sendCompressedData() {
-	# echo going to send compressed files
-	# echo "$compressedDataDir"/*
-	# for each compressed file in the pattern expansion.
 	for i in "$compressedDataDir"/*; do
-		# if file exists, sends file and if $(curl) exit code isn't equal to 0, returns $(curl) exit code 
-		# without deleting the file we tried to send. if $(curl) exit code is equal to 0, removes file
 		[ -f "$i" ] && { sendToServer "$i" || return "$?"; } && rm "$i"
 	done
 	return 0
 }
 
-# compresses $rawDataFile, sends it to $alarmServerAddress and deletes it. If send was 
-# successful, remove original files, if not, keeps it. Returns the return of $(curl).
 sendUncompressedData() {
-	# if no uncompressed file, nothing wrong, but there's nothing to do in this function.
 	[ -f "$rawDataFile" ] || return 0
 
-	# echo going to send uncompressed file
-	# the name the compressed file will have.
 	local compressedTempFile="${rawDataFile}.gz"
-	# remove old file if it exists. it should never be left there.
 	[ -f "$compressedTempFile" ] && rm "$compressedTempFile"
 
-	# in case the process is interrupted, delete compressed file.
 	trap "rm $compressedTempFile" SIGTERM
-	# compressing to a temporary file but keeping original, uncompressed, intact.
 	gzip -k "$rawDataFile"
 
-	# sends compressed file.
 	sendToServer "$compressedTempFile"
-	# storing $(curl) exit code.
 	local sentResult="$?"
 
-	# if send was successful, removes original file.
 	[ "$sentResult" -eq 0 ] && rm "$rawDataFile"
-	# removes temporary file. a new temporary will be created next time, with more content, 
 	rm "$compressedTempFile"
-	# cleans trap.
 	trap - SIGTERM
 
-	return $sentResult # Returns #(curl) exit code.
+	return $sentResult
 }
 
-
-# # reads number written in file, given as first argument ($1), and subtract it by 1. 
-# # If that subtraction results in any number that isn't zero, writes that results 
-# # to the same file and returns 1. returning 1 means it's not time to send data. 
-# # If file doesn't exist, use number given as second argument ($2).
-# checkBackoff() {
-# 	local backoffCounterPath=$1 defaultBackoff=$2
-# 	local counter
-# 	if [ -f  "$backoffCounterPath" ]; then # if file exists.
-# 		counter=$(cat "$backoffCounterPath") # take number written in that file.
-# 	else # if file doesn't exist.
-# 		counter=$defaultBackoff # use default value
-# 	fi
-# 	counter=$(($counter - 1)) # subtract number found on file by 1.
-# 	# echo new backoff is $counter
-# 	if [ $counter -ne 0 ]; then # if subtraction result is not zero.
-# 		echo $counter > "$backoffCounterPath" # write result to file.
-# 		return 1 # return 1 means we remain in backoff.
-# 	fi
-# 	# return 0 means we can send data.
-# }
-
-# # given an exit code as first argument ($1), if it's 0, the second argument ($2)
-# # is written to file at path given in forth argument ($4), but if it's not 0, the 
-# # third argument ($3) is written instead.
-# writeBackoff() {
-# 	local currentServerState=$1 normalBackoff=$2 changedBackoff=$3 backoffCounterPath=$4
-# 	if [ "$currentServerState" -ne 0 ]; then
-# 		normalBackoff=$changedBackoff
-# 	fi
-# 	# echo writting backoff.
-# 	echo $normalBackoff > "$backoffCounterPath"
-# }
-
-# Attempts to send data some times with 10 seconds of sleep time between tries.
 sendData() {
-	# echo going to send data
-	# amount of attempts of sending data, to alarm server, before giving up.
 	local tries=3
 
 	while true; do
-		# check if the last time, data was sent, server was alive. if it was, 
-		# send compressed data, if everything was sent, send uncompressed 
-		# data. If server wasn't alive last time, ping it and if it's alive 
-		# now, then send all data, but if it's still dead, do nothing.
 		local lastServerStateFilePath="$dataCollectingDir/serverState"
 		local lastServerState="1"
 		[ -f "$lastServerStateFilePath" ] && lastServerState=$(cat "$lastServerStateFilePath")
-		# echo lastServerState=$lastServerState
 
 		checkServerState "$lastServerState" && sendCompressedData && sendUncompressedData
 		local currentServerState="$?"
-		# echo currentServerState=$currentServerState
-		# if server stops before sending some data, current server state will differ from last server state.
-		# $currentServerState get the exit code of the first of these 3 functions above that returns anything other than 0.
 
-		# writes the $(curl) exit code if it has changed since last attempt to send data.
 		[ "$currentServerState" -ne "$lastServerState" ] && echo "$currentServerState" > "$lastServerStateFilePath"
-		# if data was sent successfully, we stop retrying.
 		[ "$currentServerState" -eq 0 ] && break
 
 		tries=$(($tries - 1))
-		# leaves retry loop when $retries reaches zero.
 		[ "$tries" -eq 0 ] && break
-		# echo retrying in 10 seconds
-		# sleeps before retrying. this time must take the 60 second interval into consideration.
 		sleep 10
 	done
 }
 
-# echoes a random number between 0 and 59 (inclusive).
 random0To59() {
 	local rand=$(head /dev/urandom | tr -dc "0123456789")
-	# taking the first 2 digits.
 	rand=${rand:0:2}
-	# "08" and "09" don't work for "$(())".
 	[ ${rand:0:1} = "0" ] && rand=${rand:1:2}
-	# $rand is a integer between 0 and 99 (inclusive), this makes it an integer between 0 and 59.
 	echo $(($rand * 6 / 10))
 	# our Ash has not been compiled to work with floats.
 }
 
-# prints the time stamp written in file at path given in first argument ($1). 
-# that time stamp is used to mark the second when data collecting has started, 
-# so we can keep collecting data always at the same interval, without care of 
-# how long the data collecting and sending procedures takes. If that files 
-# doesn't exist, we sleep for a random time, in order to distribute data 
-# collecting time, from all routers, through a minute window, take the current 
-# time and write a new file with that time. If it exists, its time stamp is 
-# probably from a long time ago, so we advance it forward to a time close to 
-# current time, maintaining the correct second the interval, given in second 
-# argument ($2), would make it fall into, and sleep for the amount of time 
-# left to that second.
 getStartTime() {
 	local startTimeFilePath="$1" interval="$2"
 
 	local startTime
-	# if file holding start time exists.
 	if [ -f "$startTimeFilePath" ]; then
 		local currentTime=$(date +%s)
-		startTime=$(cat "$startTimeFilePath") # get the time stamp inside that file.
+		startTime=$(cat "$startTimeFilePath")
 		# advance timestamp to the closest time after current time that the given interval could produce.
 		startTime=$(($startTime + (($currentTime - $startTime) / $interval) * $interval + $interval))
 		# that division does not produce a float number. ex: (7/2)*2 = 6.
@@ -755,43 +492,26 @@ getStartTime() {
 		sleep $(($startTime - $currentTime))
 		# this makes us always start at the same second, even if the process is shut down for a long time.
 	else 
-		# if file holding start time doesn't exist.
-		# sleeping for at most 59 seconds to distribute data collecting through a minute window.
 		sleep $(random0To59)
-		# use current time.
 		startTime=$(date +%s)
 	fi
-	# substitute that time in file, or create a new file.
 	echo $startTime > "$startTimeFilePath"
-	# print start time found, or current time.
 	echo $startTime
 }
 
-# deletes files, marking process state, from previous process and deletes temporary
-# files that could be hanging if process is terminated in a critical part.
 cleanFiles() {
 	rm "${dataCollectingDir}/serverState" 2> /dev/null
-	# rm "${dataCollectingDir}/backoffCounter" 2> /dev/null
 	rm "${rawDataFile}.gz" 2> /dev/null
 }
 
-# collects and sends data forever.
 loop() {
-	# interval between beginnings of data collecting.
 	local interval=60
-	# making sure directory exists.
 	mkdir -p "$dataCollectingDir"
-	# time when we will start executing.
 	local time=$(getStartTime "${dataCollectingDir}/startTime" $interval)
 
-	# infinite loop where we execute all procedures over and over again until the end of times.
 	while true; do
-		# echo startTime $time
-		# making sure directory exists every time.
 		mkdir -p "$dataCollectingDir"
 
-		# getting parameters every time we need to send data, this way we don't have to 
-		# restart the service if a parameter changes.
 		eval $(cat /root/flashbox_config.json | jsonfilter \
 			-e "hasLatency=@.data_collecting_has_latency" \
 			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
@@ -802,32 +522,20 @@ loop() {
 			-e "pingAndWan=@.data_collecting_ping_and_wan" \
 		)
 
-		# does everything related to collecting and storing data.`
 		collectData
-		# does everything related to sending data and deletes data sent.
 		sendData
 
-		# time after all procedures are finished.
 		local endTime=$(date +%s)
-		# this will hold the time left until we should run the next iteration of this loop
 		local timeLeftForNextRun="-1"
-		# while time left is negative, which could happen if $(($time - $endTime)) is bigger than $interval.
 		while [ "$timeLeftForNextRun" -lt 0 ]; do
-			# advance time, when current data collecting has started, by one interval.
 			time=$(($time + $interval))
-			# calculate time left to collect data again.
 			timeLeftForNextRun=$(($time - $endTime))
 		done
-		# echo timeLeftForNextRun=$timeLeftForNextRun
-		 # sleep for the time remaining until next data collecting.
 		sleep $timeLeftForNextRun
 
-		# writing next loop time to file that, at this line, matches current time.
 		echo $time > "${dataCollectingDir}/startTime"
 	done
 }
 
-# deletes files, marking process state, from previous process.
 cleanFiles
-# the infinite loop.
 loop

From 7881444c39eac1f4975162d75a2763518501dffe Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Wed, 28 Sep 2022 18:53:59 -0300
Subject: [PATCH 59/77] Revert "removing comments that don't add much
 information because of flash limitation"

This reverts commit 932dfdae274ce2fb40551df2e1e8a6fc5919df1a.
---
 files/usr/share/data_collecting.sh | 298 ++++++++++++++++++++++++++++-
 1 file changed, 295 insertions(+), 3 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 10daec37..787fd3d1 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -4,22 +4,33 @@
 . /usr/share/functions/custom_wireless_driver.sh
 . /usr/share/flashman_init.conf
 
+# directory where all data related to data collecting will be stored.
 dataCollectingDir="/tmp/data_collecting"
+# file where collected data will be stored before being compressed.
 rawDataFile="${dataCollectingDir}/raw"
+# directory where data will be stored if compressing old data is necessary.
 compressedDataDir="${dataCollectingDir}/compressed"
+# file where wan rx/tx bytes will be stored
 wanBytesFile="${dataCollectingDir}/wan_bytes"
+# file where wan rx/tx packets will be stored
 wanPacketsFile="${dataCollectingDir}/wan_pkts"
 
+# gets current rx and tx bytes/packets from wan interface and compares them
+# with values from previous calls to calculate cross traffic
 collect_wan() {
+	# checking if this data collecting is enabled
 
 	local isBurstLossActive=${activeMeasures/*bl*/bl}
+
 	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
 
 	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
 
 	local sendThisRound=1
 
+	# bytes received by the interface.
 	local rxBytes=$(get_wan_bytes_statistics RX)
+	# bytes sent by the interface.
 	local txBytes=$(get_wan_bytes_statistics TX)
 
 	local last_rxBytes=""
@@ -27,7 +38,9 @@ collect_wan() {
 
 	if [ -f "$wanBytesFile" ]; then
 		last_rxBytes=$(cat "$wanBytesFile")
+		# gets second value
 		last_txBytes=${last_rxBytes##* }
+		# gets first value
 		last_rxBytes=${last_rxBytes%% *}
 	fi
 
@@ -35,15 +48,23 @@ collect_wan() {
 	local txBytesDiff=0
 
 	if [ "$last_rxBytes" == "" ] || [ "$last_txBytes" == "" ]; then
+		# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
+		# empty out file (we only need last minute info)
 		> "$wanBytesFile"
 		echo "$rxBytes $txBytes" >> "$wanBytesFile"
+		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
 		sendThisRound=0
 	else
+		# bytes received since last time.
 		rxBytesDiff=$(($rxBytes - $last_rxBytes))
+		# bytes transmitted since last time
 		txBytesDiff=$(($txBytes - $last_txBytes))
+		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+		# we skip this measure.
 		{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && sendThisRound=0
 	fi
 
+	# empty out file (we only need last minute info)
 	> "$wanBytesFile"
 	echo "$rxBytes $txBytes" >> "$wanBytesFile"
 
@@ -59,11 +80,14 @@ collect_wan() {
 		activeMeasures="${activeMeasures/p&w/}"
 	fi
 
+	# burstLoss only gathers byte data
 	if [ "$isPingAndWanActive" != "p&w" ]; then
 		return
 	fi
 
+	# packets received by the interface.
 	local rxPackets=$(get_wan_packets_statistics RX)
+	# packets sent by the interface.
 	local txPackets=$(get_wan_packets_statistics TX)
 
 	local last_rxPackets=""
@@ -71,7 +95,9 @@ collect_wan() {
 
 	if [ -f "$wanPacketsFile" ]; then
 		last_rxPackets=$(cat "$wanPacketsFile")
+		# gets second value
 		last_txPackets=${last_rxPackets##* }
+		# gets first value
 		last_rxPackets=${last_rxPackets%% *}
 	fi
 
@@ -79,18 +105,27 @@ collect_wan() {
 	local txPacketsDiff=0
 
 	if [ "$last_rxPackets" == "" ] || [ "$last_txPackets" == "" ]; then
+		# if last packets are not defined. define them using the current wan interface Packets value. then we skip this measure.
+		# empty out file (we only need last minute info)
 		> "$wanPacketsFile"
 		echo "$rxPackets $txPackets" >> "$wanPacketsFile"
+		# don't write data this round. we need a full minute of packets to calculate cross traffic.
 		sendThisRound=0
 	else
+		# packets received since last time.
 		rxPacketsDiff=$(($rxPackets - $last_rxPackets))
+		# packets transmitted since last time
 		txPacketsDiff=$(($txPackets - $last_txPackets))
+		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+		# we skip this measure.
 		{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && sendThisRound=0
 	fi
 
+	# empty out file (we only need last minute info)
 	> "$wanPacketsFile"
 	echo "$rxPackets $txPackets" >> "$wanPacketsFile"
 
+	# needs to gather one more minute
 	if [ "$sendThisRound" -ne 1 ]; then
 		activeMeasures="${activeMeasures/p&w /}"
 		activeMeasures="${activeMeasures/ p&w/}"
@@ -98,20 +133,29 @@ collect_wan() {
 		return
 	fi
 
+	# data to be sent.
 	local string="|wanPkts $rxPacketsDiff $txPacketsDiff"
 	rawData="${rawData}${string}"
 }
 
+# takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
+# If latency collecting is enabled, extracts the individual icmp request numbers and 
+# their respective ping times. Builds a string with all this information and write them to file.
 collect_burst() {
+	# checking if this data collecting is enabled
 
 	local isBurstLossActive=${activeMeasures/*bl*/bl}
+
 	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
 
 	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
 
+	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
+	# ping return value.
 	local pingError="$?"
 
+	# if ping could not be executed, we skip this measure.
 	if [ "$pingError" -eq 2 ]; then
 		activeMeasures="${activeMeasures/bl /}"
 		activeMeasures="${activeMeasures/ bl/}"
@@ -122,14 +166,25 @@ collect_burst() {
 		return
 	fi
 
+	# An skipped measure will become missing data, for this minute, in the server.
+
+	# removes everything behind the summary that appears in the last lines.
 	local pingResultAux=${pingResult##* ping statistics ---[$'\r\n']}
+	# removes everything after, and including, ' packets transmitted'.
 	local transmitted=${pingResultAux% packets transmitted*}
+	# removes everything after, and including, ' received'.
 	local received=${pingResultAux% received*}
+	# removes everything before first space.
 	received=${received##* }
+	# integer representing the amount of packets not received.
 	local loss=$(($transmitted - $received))
+	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
+	# loss=${loss##* } # removes everything before first space.
 
+	# data to be sent.
 	local string="$loss $transmitted"
 
+	# removes everything before and including 'mdev = '
 	local latencyStats=${pingResult#*/mdev = }
 
 	# when there is 100% packet loss there the strings remain equal
@@ -138,56 +193,87 @@ collect_burst() {
 	if [ ${#latencyStats} == ${#pingResult} ]; then
 	    string="$string 0 0"
 	else
+		# removes everything before first backslash
 		local latencyAvg=${latencyStats#*/}
+		# removes everything after first backslash
 		latencyAvg=${latencyAvg%%/*}
+		# removes everything before and including last backslash
 		local latencyStd=${latencyStats##*/}
+		# removes everything after and including first space
 		latencyStd=${latencyStd%% *}
 
 		string="$string $latencyAvg $latencyStd"
 	fi
 
+	# if latency collecting is enabled.
 	if [ "$hasLatency" -eq 1 ]; then
+		# echo collecting latencies
+		# removing the first line and the last 4 lines. only the ping lines remain.
 		local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
 		local pairs=""
 		local firstLine=true
+		# for each ping line.
 		while read line; do
+			# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
 			reached=${line%time=*}
+			# if "time=" has actually been removed, it means that line 
+			# contains it, which also means the icmp request was fulfilled.
+			# if line doesn't contain 'time=', we skip this line.
 			[ ${#reached} -lt ${#line} ] || continue
 
+			 # from the whole line, removes everything until, and including, "icmp_req=".
 			pingNumber=${line#*icmp_*eq=}
+			# removes everything after the first space.
 			pingNumber=${pingNumber%% *}
+			# from the whole line, removes everything until, and including, "time=".
 			pingTime=${line#*time=}
+			# removes everything after the first space.
 			pingTime=${pingTime%% *}
 			if [ "$firstLine" = true ]; then
 				firstLine=false
 			else
 				pairs="${pairs},"
 			fi
+			# concatenate to $string.
 			pairs="${pairs}${pingNumber}=${pingTime}"
 		done
+		# prints final $string in this sub shell back to $string.
 		echo $pairs))
+		# appending latencies to string to be sent.
 		[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
 	fi
 	
+	# appending string to file.
+	# printf "string is: '%s'\n" "$string"
 	rawData="${rawData}|burstPing $string"
 }
 
 collect_wifi_devices() {
+	# checking if this data collecting is enabled
 	[ "$wifiDevices" -ne 1 ] && return
 
+	# devices and their data will be stored in this string variable.
 	local str=""
 
 	local firstRawWrite=true
 
+	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
+		# getting wifi interface name.
+		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
 		local wlan=$(get_root_ifname "$i" 2> /dev/null)
+		# if interface doesn't exist, skips this iteration.
 		[ "$wlan" == "" ] && continue
+		# getting info from each connected device on wifi. 
+		# grep returns empty when no devices are connected or if interface doesn't exist.
 		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
 		local devices_rx_pkts="$(iwinfo "$wlan" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
 		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
+		# first iteration won't put a space before the value.
 		local firstFileWrite=true
 
+		# string to be appended to devices packets file
 		local fileStr=""
 
 		local lastPktsFile="${dataCollectingDir}/devices_24_pkts"
@@ -197,24 +283,45 @@ collect_wifi_devices() {
 
 		while [ ${#devices} -gt 0 ]; do
 
+			# getting everything before the first space.
 			local deviceMac=${devices%% *}
 
+			# getting everything after the first two spaces.
 			local signal=${devices#*  }
+
+			# getting everything before the first occasion of ' /'
 			signal=${signal%% /*}
+
+			# if unknown discard
 			[ "$signal" == "unknown" ] && devices=${devices#*$'\n'} && continue
+
+			# getting everything before the first occasion of ' dBm'
 			signal=${signal%% dBm*}
 
+			# getting after '(SNR '. 
 			devices=${devices#*\(SNR }
 
+			# getting everything before the first closing parenthesis.
 			local snr=${devices%%\)*}
+
+			# if SNR equals signal we assume noise of -95dBm
 			[ $signal -eq $snr ] && snr=$(($signal+95))
 
+			# getting everything before ' ms'.
 			local time=${devices%% ms*}
+
+			# getting everything after the first space.
 			time=${time##* }
 
+			# getting everything after 'ago'.
 			devices=${devices#*ago}
+
+			# getting everything after '\n', if it exists. last line won't have it, so nothing will be changed.
+			# we can't add the line feed along witht the previous parameter expansion because we wouldn't match
+			# the last line and so we wouldn't make $devices length become zero.
 			devices=${devices#*$'\n'}
 
+			# if $time is greater than one minute, we don't use this device's info.
 			[ "$time" -gt 60000 ] && continue
 			
 			local rx_pkts=${devices_rx_pkts%% *}
@@ -237,6 +344,7 @@ collect_wifi_devices() {
 
 				local pkts_device_removed=${devices_pkts/"$deviceMac"/""}
 
+				# if deviceMac not in recorded devices continue
 				[ ${#pkts_device_removed} -ge ${#devices_pkts} ] && continue
 
 				local last_pkts=${devices_pkts#*"$deviceMac"_}
@@ -250,19 +358,25 @@ collect_wifi_devices() {
 				[ "$last_tx_pkts" == "" ] && continue
 				tx_pkts_diff=$(($tx_pkts - $last_tx_pkts))
 
+				# if subtraction created a negative value, it means it has overflown or interface has been restarted.
+				# we skip this measure.
 				{ [ "$rx_pkts_diff" -lt 0 ] || [ "$tx_pkts_diff" -lt 0 ]; } && continue
 			else
 				continue
 			fi
 
+			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$firstRawWrite" == true ] && firstRawWrite=false || str="$str "
 			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
 		done
+		# empty out file (we only need last minute info)
 		> "$lastPktsFile"
 
+		# write to it if there are device infos
 		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$lastPktsFile"
 	done
 	if [ "$str" == "" ]; then
+		# only send data if there is something to send
 		activeMeasures="${activeMeasures/wd /}"
 		activeMeasures="${activeMeasures/ wd/}"
 		activeMeasures="${activeMeasures/wd/}"
@@ -271,71 +385,120 @@ collect_wifi_devices() {
 	fi
 }
 
+# prints the size of a file, using 'ls', where full file path is given as 
+# first argument ($1).
 fileSize() {
+	# file size is the information at the 1st column.
 	local wcline=$(wc -c "$1")
+	#remove suffix composed of space and anything else.
 	local size=${wcline% *}
 	echo $size
 }
 
+# prints the sum of the sizes of all files inside given directory path.
 sumFileSizesInPath() {
+	# boolean that marks that at least one file exists inside given directory.
 	local anyFile=false
+	# for each file in that directory.
 	for i in "$1"/*; do
+		# if that pattern expansion exists as a file.
 		[ -f "$i" ] || continue
+		# set boolean to true.
 		anyFile=true
+		# as we have at least one file, we don't need to loop through all files.
 		break
 	done
 	if [ "$anyFile" = false ]; then
+		# if no files.
+		# prints zero. size of nothing is 0.
 		echo 0
+		# result was given, we can leave function.
 		return 0
 	fi
+	# if there is at least one file.
 
+	# prints a list of sizes and files.
 	local wcResult=$(wc -c "$1"/*)
+	# if there is 2 or more files, the last line will have a "total". remove that string.
 	local hasTotal=${wcResult% total}
+	# if it has a total, it was removed. if not, nothing was removed and both strings are the same.
 
+	 # if length of string with "total" removed is smaller than original string.
 	if [ ${#hasTotal} -lt ${#wcResult} ]; then
+		# remove everything before the last word, which is the value for total, and print what remains.
 		echo ${hasTotal##* }
 	else
+		# if there were no total, then there was only one file, in one line of output, and the first column is the size value.
+		# remove everything past, and including, the first space, and print what remains.
 		echo ${wcResult%% *}
 	fi
 }
 
+# sum the size $rawDataFile with the size of all compressed files inside the $compressedDataDir. if that 
+# sum is bigger than number given in first argument ($1), compress that file and move it to $compressedDataDir.
 zipFile() {
 	local capSize="$1"
 
+	# if file doesn't exist, we won't have to compressed anything.
+	# a return of 1 means nothing has been be gzipped.
 	[ -f "$rawDataFile" ] || return 1 
 
+	# size of file with raw data.
 	local size=$(fileSize "$rawDataFile")
+	# sum of file sizes in directory for compressed files.
 	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
 
+	# if sum is smaller than $capSize, do nothing.
+	# echo checking file size to zip
+	# a return of 1 means nothing will be gzipped.
 	[ $(($size + $dirSize)) -lt $capSize ] && return 1
 
+	# compressing file where raw data is held.
 	gzip "$rawDataFile"
+	# move newly compressed file to directory where compressed files should be.
 	mv "${rawDataFile}.gz" "$compressedDataDir/$(date +%s).gz"
 }
 
+# files are removed from $compressedDataDir until all data remaining is below number given in first 
+# argument ($1) as bytes. As files are named using a formatted date, pattern expansion of file 
+# names will always order oldest files first. This was done this way so we don't have to sort files 
+# by date ourselves. we let shell do the sorting.
 removeOldFiles() {
 	local capSize="$1"
 
+	# get the sum of sizes of all files in bytes.
 	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
 
+	# if $dirSize is more than given $capSize, remove oldest file. which is 
+	# the file that shell orders as first.
 	for i in "$compressedDataDir"/*; do
+		# if we are under $capSize. do nothing.
 		[ $dirSize -lt $capSize ] && break;
+		# removes that file.
 		rm "$i"
+		# subtract that file's size from sum.
 		dirSize=$(($dirSize - $(fileSize "$i")))
 	done
 }
 
+# collect every data and stores in '$rawDataFile'. if the size of the file is 
+# too big, compress it and move it to a directory of compressed files. If 
+# directory of compressed files grows too big delete oldest compressed files.
 collectData() {
+	# getting current unix time in seconds.
 	local timestamp=$(date +%s)
 
+	# global variable where current raw data is stored before being written to file.
 	rawData=""
 
+	# global variable that controls which measures are active
 	activeMeasures=""
 
 	local firstMeasurement=1
 
 	if [ "$burstLoss" -eq 1 ]; then
 		if [ "$firstMeasurement" -ne 1 ]; then
+			# add space before active measurement name
 			activeMeasures="$activeMeasures "
 		else
 			firstMeasurement=0
@@ -345,6 +508,7 @@ collectData() {
 
 	if [ "$pingAndWan" -eq 1 ]; then
 		if [ "$firstMeasurement" -ne 1 ]; then
+			# add space before active measurement name
 			activeMeasures="$activeMeasures "
 		else
 			firstMeasurement=0
@@ -354,6 +518,7 @@ collectData() {
 
 	if [ "$wifiDevices" -eq 1 ]; then
 		if [ "$firstMeasurement" -ne 1 ]; then
+			# add space before active measurement name
 			activeMeasures="$activeMeasures "
 		else
 			firstMeasurement=0
@@ -361,6 +526,7 @@ collectData() {
 		activeMeasures="${activeMeasures}wd"
 	fi
 
+	# collecting all measures.
 	collect_burst
 	collect_wan
 	collect_wifi_devices
@@ -373,8 +539,10 @@ collectData() {
 	# example of an expected raw data with all measures present:
 	# 'bl p&w wd|213234556456|burstPing 0 100 1.246 0.161|wanBytes 12345 1234|wanPkts 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
 	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
+	# cleaning 'rawData' value from memory.
 	rawData=""
 
+	# creates directory of for compressed files, if it doesn't already exists.
 	mkdir -p "$compressedDataDir"
 	# $(zipFile) returns 0 only if any amount of files has been compressed 
 	# and, consequently, moved to the directory of compressed files. So
@@ -393,18 +561,27 @@ collectData() {
 	# the uncompressed file will always have available for it's growth.
 }
 
+# if number given as first argument ($1) isn't 0, ping server at address given 
+# in second argument ($2) and returns the exit code of $(curl), but if that 
+# number is zero, return that number.
 checkServerState() {
 	local lastState="$1"
 	if [ "$lastState" -ne "0" ]; then
+		# echo pinging alarm server to check if it's alive.
 		curl -s -m 10 "https://$alarmServerAddress:7890/ping" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" > /dev/null
+		# return $(curl) exit code.
 		lastState="$?"
 	fi
+	# echo last state is $lastState
 	return $lastState
 }
 
+# sends file at given path ($1) to server at given address ($2) using $(curl) 
+# and returns $(curl) exit code.
 sendToServer() {
 	local filepath="$1"
 
+	# 'get_mac()' is defined in /usr/share/functions/device_functions.sh.
 	local mac=$(get_mac);
 
 	status=$(curl --write-out '%{http_code}' -s -m 20 --connect-timeout 5 --output /dev/null \
@@ -412,74 +589,160 @@ sendToServer() {
 	-H 'Content-Type: text/plain' -H "X-ANLIX-ID: $mac" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" \
 	-H "Send-Time: $(date +%s)" --data-binary @"$filepath")
 	curlCode="$?"
-
+	# 'log()' is defined in /usr/share/functions/common_functions.sh.
 	[ "$curlCode" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${curlCode}'." && return "$curlCode"
 	log "DATA_COLLECTING" "Data sent with response status code '${status}'."
 	[ "$status" -ge 200 ] && [ "$status" -lt 300 ] && return 0
 	return 1
 }
 
+# for each compressed file given in $compressedDataDir, send that file to a $alarmServerAddress. 
+# If any sending is unsuccessful, stops sending files and return it's exit code.
 sendCompressedData() {
+	# echo going to send compressed files
+	# echo "$compressedDataDir"/*
+	# for each compressed file in the pattern expansion.
 	for i in "$compressedDataDir"/*; do
+		# if file exists, sends file and if $(curl) exit code isn't equal to 0, returns $(curl) exit code 
+		# without deleting the file we tried to send. if $(curl) exit code is equal to 0, removes file
 		[ -f "$i" ] && { sendToServer "$i" || return "$?"; } && rm "$i"
 	done
 	return 0
 }
 
+# compresses $rawDataFile, sends it to $alarmServerAddress and deletes it. If send was 
+# successful, remove original files, if not, keeps it. Returns the return of $(curl).
 sendUncompressedData() {
+	# if no uncompressed file, nothing wrong, but there's nothing to do in this function.
 	[ -f "$rawDataFile" ] || return 0
 
+	# echo going to send uncompressed file
+	# the name the compressed file will have.
 	local compressedTempFile="${rawDataFile}.gz"
+	# remove old file if it exists. it should never be left there.
 	[ -f "$compressedTempFile" ] && rm "$compressedTempFile"
 
+	# in case the process is interrupted, delete compressed file.
 	trap "rm $compressedTempFile" SIGTERM
+	# compressing to a temporary file but keeping original, uncompressed, intact.
 	gzip -k "$rawDataFile"
 
+	# sends compressed file.
 	sendToServer "$compressedTempFile"
+	# storing $(curl) exit code.
 	local sentResult="$?"
 
+	# if send was successful, removes original file.
 	[ "$sentResult" -eq 0 ] && rm "$rawDataFile"
+	# removes temporary file. a new temporary will be created next time, with more content, 
 	rm "$compressedTempFile"
+	# cleans trap.
 	trap - SIGTERM
 
-	return $sentResult
+	return $sentResult # Returns #(curl) exit code.
 }
 
+
+# # reads number written in file, given as first argument ($1), and subtract it by 1. 
+# # If that subtraction results in any number that isn't zero, writes that results 
+# # to the same file and returns 1. returning 1 means it's not time to send data. 
+# # If file doesn't exist, use number given as second argument ($2).
+# checkBackoff() {
+# 	local backoffCounterPath=$1 defaultBackoff=$2
+# 	local counter
+# 	if [ -f  "$backoffCounterPath" ]; then # if file exists.
+# 		counter=$(cat "$backoffCounterPath") # take number written in that file.
+# 	else # if file doesn't exist.
+# 		counter=$defaultBackoff # use default value
+# 	fi
+# 	counter=$(($counter - 1)) # subtract number found on file by 1.
+# 	# echo new backoff is $counter
+# 	if [ $counter -ne 0 ]; then # if subtraction result is not zero.
+# 		echo $counter > "$backoffCounterPath" # write result to file.
+# 		return 1 # return 1 means we remain in backoff.
+# 	fi
+# 	# return 0 means we can send data.
+# }
+
+# # given an exit code as first argument ($1), if it's 0, the second argument ($2)
+# # is written to file at path given in forth argument ($4), but if it's not 0, the 
+# # third argument ($3) is written instead.
+# writeBackoff() {
+# 	local currentServerState=$1 normalBackoff=$2 changedBackoff=$3 backoffCounterPath=$4
+# 	if [ "$currentServerState" -ne 0 ]; then
+# 		normalBackoff=$changedBackoff
+# 	fi
+# 	# echo writting backoff.
+# 	echo $normalBackoff > "$backoffCounterPath"
+# }
+
+# Attempts to send data some times with 10 seconds of sleep time between tries.
 sendData() {
+	# echo going to send data
+	# amount of attempts of sending data, to alarm server, before giving up.
 	local tries=3
 
 	while true; do
+		# check if the last time, data was sent, server was alive. if it was, 
+		# send compressed data, if everything was sent, send uncompressed 
+		# data. If server wasn't alive last time, ping it and if it's alive 
+		# now, then send all data, but if it's still dead, do nothing.
 		local lastServerStateFilePath="$dataCollectingDir/serverState"
 		local lastServerState="1"
 		[ -f "$lastServerStateFilePath" ] && lastServerState=$(cat "$lastServerStateFilePath")
+		# echo lastServerState=$lastServerState
 
 		checkServerState "$lastServerState" && sendCompressedData && sendUncompressedData
 		local currentServerState="$?"
+		# echo currentServerState=$currentServerState
+		# if server stops before sending some data, current server state will differ from last server state.
+		# $currentServerState get the exit code of the first of these 3 functions above that returns anything other than 0.
 
+		# writes the $(curl) exit code if it has changed since last attempt to send data.
 		[ "$currentServerState" -ne "$lastServerState" ] && echo "$currentServerState" > "$lastServerStateFilePath"
+		# if data was sent successfully, we stop retrying.
 		[ "$currentServerState" -eq 0 ] && break
 
 		tries=$(($tries - 1))
+		# leaves retry loop when $retries reaches zero.
 		[ "$tries" -eq 0 ] && break
+		# echo retrying in 10 seconds
+		# sleeps before retrying. this time must take the 60 second interval into consideration.
 		sleep 10
 	done
 }
 
+# echoes a random number between 0 and 59 (inclusive).
 random0To59() {
 	local rand=$(head /dev/urandom | tr -dc "0123456789")
+	# taking the first 2 digits.
 	rand=${rand:0:2}
+	# "08" and "09" don't work for "$(())".
 	[ ${rand:0:1} = "0" ] && rand=${rand:1:2}
+	# $rand is a integer between 0 and 99 (inclusive), this makes it an integer between 0 and 59.
 	echo $(($rand * 6 / 10))
 	# our Ash has not been compiled to work with floats.
 }
 
+# prints the time stamp written in file at path given in first argument ($1). 
+# that time stamp is used to mark the second when data collecting has started, 
+# so we can keep collecting data always at the same interval, without care of 
+# how long the data collecting and sending procedures takes. If that files 
+# doesn't exist, we sleep for a random time, in order to distribute data 
+# collecting time, from all routers, through a minute window, take the current 
+# time and write a new file with that time. If it exists, its time stamp is 
+# probably from a long time ago, so we advance it forward to a time close to 
+# current time, maintaining the correct second the interval, given in second 
+# argument ($2), would make it fall into, and sleep for the amount of time 
+# left to that second.
 getStartTime() {
 	local startTimeFilePath="$1" interval="$2"
 
 	local startTime
+	# if file holding start time exists.
 	if [ -f "$startTimeFilePath" ]; then
 		local currentTime=$(date +%s)
-		startTime=$(cat "$startTimeFilePath")
+		startTime=$(cat "$startTimeFilePath") # get the time stamp inside that file.
 		# advance timestamp to the closest time after current time that the given interval could produce.
 		startTime=$(($startTime + (($currentTime - $startTime) / $interval) * $interval + $interval))
 		# that division does not produce a float number. ex: (7/2)*2 = 6.
@@ -492,26 +755,43 @@ getStartTime() {
 		sleep $(($startTime - $currentTime))
 		# this makes us always start at the same second, even if the process is shut down for a long time.
 	else 
+		# if file holding start time doesn't exist.
+		# sleeping for at most 59 seconds to distribute data collecting through a minute window.
 		sleep $(random0To59)
+		# use current time.
 		startTime=$(date +%s)
 	fi
+	# substitute that time in file, or create a new file.
 	echo $startTime > "$startTimeFilePath"
+	# print start time found, or current time.
 	echo $startTime
 }
 
+# deletes files, marking process state, from previous process and deletes temporary
+# files that could be hanging if process is terminated in a critical part.
 cleanFiles() {
 	rm "${dataCollectingDir}/serverState" 2> /dev/null
+	# rm "${dataCollectingDir}/backoffCounter" 2> /dev/null
 	rm "${rawDataFile}.gz" 2> /dev/null
 }
 
+# collects and sends data forever.
 loop() {
+	# interval between beginnings of data collecting.
 	local interval=60
+	# making sure directory exists.
 	mkdir -p "$dataCollectingDir"
+	# time when we will start executing.
 	local time=$(getStartTime "${dataCollectingDir}/startTime" $interval)
 
+	# infinite loop where we execute all procedures over and over again until the end of times.
 	while true; do
+		# echo startTime $time
+		# making sure directory exists every time.
 		mkdir -p "$dataCollectingDir"
 
+		# getting parameters every time we need to send data, this way we don't have to 
+		# restart the service if a parameter changes.
 		eval $(cat /root/flashbox_config.json | jsonfilter \
 			-e "hasLatency=@.data_collecting_has_latency" \
 			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
@@ -522,20 +802,32 @@ loop() {
 			-e "pingAndWan=@.data_collecting_ping_and_wan" \
 		)
 
+		# does everything related to collecting and storing data.`
 		collectData
+		# does everything related to sending data and deletes data sent.
 		sendData
 
+		# time after all procedures are finished.
 		local endTime=$(date +%s)
+		# this will hold the time left until we should run the next iteration of this loop
 		local timeLeftForNextRun="-1"
+		# while time left is negative, which could happen if $(($time - $endTime)) is bigger than $interval.
 		while [ "$timeLeftForNextRun" -lt 0 ]; do
+			# advance time, when current data collecting has started, by one interval.
 			time=$(($time + $interval))
+			# calculate time left to collect data again.
 			timeLeftForNextRun=$(($time - $endTime))
 		done
+		# echo timeLeftForNextRun=$timeLeftForNextRun
+		 # sleep for the time remaining until next data collecting.
 		sleep $timeLeftForNextRun
 
+		# writing next loop time to file that, at this line, matches current time.
 		echo $time > "${dataCollectingDir}/startTime"
 	done
 }
 
+# deletes files, marking process state, from previous process.
 cleanFiles
+# the infinite loop.
 loop

From ea6a2880e3d3457ebc056f0411ef4c042f0b092c Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 29 Sep 2022 16:04:18 -0300
Subject: [PATCH 60/77] sending raw packet/byte counts + removing detailed ping
 data + using more the boolean control variables

---
 files/usr/share/data_collecting.sh | 307 ++++++-----------------------
 1 file changed, 59 insertions(+), 248 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 787fd3d1..46cccc48 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -10,21 +10,13 @@ dataCollectingDir="/tmp/data_collecting"
 rawDataFile="${dataCollectingDir}/raw"
 # directory where data will be stored if compressing old data is necessary.
 compressedDataDir="${dataCollectingDir}/compressed"
-# file where wan rx/tx bytes will be stored
-wanBytesFile="${dataCollectingDir}/wan_bytes"
-# file where wan rx/tx packets will be stored
-wanPacketsFile="${dataCollectingDir}/wan_pkts"
 
 # gets current rx and tx bytes/packets from wan interface and compares them
 # with values from previous calls to calculate cross traffic
 collect_wan() {
 	# checking if this data collecting is enabled
 
-	local isBurstLossActive=${activeMeasures/*bl*/bl}
-
-	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
-
-	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
+	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
 	local sendThisRound=1
 
@@ -33,108 +25,19 @@ collect_wan() {
 	# bytes sent by the interface.
 	local txBytes=$(get_wan_bytes_statistics TX)
 
-	local last_rxBytes=""
-	local last_txBytes=""
-
-	if [ -f "$wanBytesFile" ]; then
-		last_rxBytes=$(cat "$wanBytesFile")
-		# gets second value
-		last_txBytes=${last_rxBytes##* }
-		# gets first value
-		last_rxBytes=${last_rxBytes%% *}
-	fi
-
-	local rxBytesDiff=0
-	local txBytesDiff=0
-
-	if [ "$last_rxBytes" == "" ] || [ "$last_txBytes" == "" ]; then
-		# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
-		# empty out file (we only need last minute info)
-		> "$wanBytesFile"
-		echo "$rxBytes $txBytes" >> "$wanBytesFile"
-		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
-		sendThisRound=0
-	else
-		# bytes received since last time.
-		rxBytesDiff=$(($rxBytes - $last_rxBytes))
-		# bytes transmitted since last time
-		txBytesDiff=$(($txBytes - $last_txBytes))
-		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-		# we skip this measure.
-		{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && sendThisRound=0
-	fi
-
-	# empty out file (we only need last minute info)
-	> "$wanBytesFile"
-	echo "$rxBytes $txBytes" >> "$wanBytesFile"
-
-	if [ "$sendThisRound" -eq 1 ]; then
-		local string="|wanBytes $rxBytesDiff $txBytesDiff"
-		rawData="${rawData}${string}"
-	else
-		activeMeasures="${activeMeasures/bl /}"
-		activeMeasures="${activeMeasures/ bl/}"
-		activeMeasures="${activeMeasures/bl/}"
-		activeMeasures="${activeMeasures/p&w /}"
-		activeMeasures="${activeMeasures/ p&w/}"
-		activeMeasures="${activeMeasures/p&w/}"
-	fi
+	local string="|wanBytes $rxBytes $txBytes"
+	rawData="${rawData}${string}"
 
 	# burstLoss only gathers byte data
-	if [ "$isPingAndWanActive" != "p&w" ]; then
-		return
-	fi
+	[ $pingAndWan -eq 0 ] && return
 
 	# packets received by the interface.
 	local rxPackets=$(get_wan_packets_statistics RX)
 	# packets sent by the interface.
 	local txPackets=$(get_wan_packets_statistics TX)
 
-	local last_rxPackets=""
-	local last_txPackets=""
-
-	if [ -f "$wanPacketsFile" ]; then
-		last_rxPackets=$(cat "$wanPacketsFile")
-		# gets second value
-		last_txPackets=${last_rxPackets##* }
-		# gets first value
-		last_rxPackets=${last_rxPackets%% *}
-	fi
-
-	local rxPacketsDiff=0
-	local txPacketsDiff=0
-
-	if [ "$last_rxPackets" == "" ] || [ "$last_txPackets" == "" ]; then
-		# if last packets are not defined. define them using the current wan interface Packets value. then we skip this measure.
-		# empty out file (we only need last minute info)
-		> "$wanPacketsFile"
-		echo "$rxPackets $txPackets" >> "$wanPacketsFile"
-		# don't write data this round. we need a full minute of packets to calculate cross traffic.
-		sendThisRound=0
-	else
-		# packets received since last time.
-		rxPacketsDiff=$(($rxPackets - $last_rxPackets))
-		# packets transmitted since last time
-		txPacketsDiff=$(($txPackets - $last_txPackets))
-		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-		# we skip this measure.
-		{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && sendThisRound=0
-	fi
-
-	# empty out file (we only need last minute info)
-	> "$wanPacketsFile"
-	echo "$rxPackets $txPackets" >> "$wanPacketsFile"
-
-	# needs to gather one more minute
-	if [ "$sendThisRound" -ne 1 ]; then
-		activeMeasures="${activeMeasures/p&w /}"
-		activeMeasures="${activeMeasures/ p&w/}"
-		activeMeasures="${activeMeasures/p&w/}"
-		return
-	fi
-
 	# data to be sent.
-	local string="|wanPkts $rxPacketsDiff $txPacketsDiff"
+	local string="|wanPkts $rxPackets $txPackets"
 	rawData="${rawData}${string}"
 }
 
@@ -144,11 +47,7 @@ collect_wan() {
 collect_burst() {
 	# checking if this data collecting is enabled
 
-	local isBurstLossActive=${activeMeasures/*bl*/bl}
-
-	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
-
-	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
+	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
 	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
@@ -156,15 +55,7 @@ collect_burst() {
 	local pingError="$?"
 
 	# if ping could not be executed, we skip this measure.
-	if [ "$pingError" -eq 2 ]; then
-		activeMeasures="${activeMeasures/bl /}"
-		activeMeasures="${activeMeasures/ bl/}"
-		activeMeasures="${activeMeasures/bl/}"
-		activeMeasures="${activeMeasures/p&w /}"
-		activeMeasures="${activeMeasures/ p&w/}"
-		activeMeasures="${activeMeasures/p&w/}"
-		return
-	fi
+	[ "$pingError" -eq 2 ] && burstLoss=0 && pingAndWan=0 && return
 
 	# An skipped measure will become missing data, for this minute, in the server.
 
@@ -205,43 +96,43 @@ collect_burst() {
 		string="$string $latencyAvg $latencyStd"
 	fi
 
-	# if latency collecting is enabled.
-	if [ "$hasLatency" -eq 1 ]; then
-		# echo collecting latencies
-		# removing the first line and the last 4 lines. only the ping lines remain.
-		local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
-		local pairs=""
-		local firstLine=true
-		# for each ping line.
-		while read line; do
-			# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
-			reached=${line%time=*}
-			# if "time=" has actually been removed, it means that line 
-			# contains it, which also means the icmp request was fulfilled.
-			# if line doesn't contain 'time=', we skip this line.
-			[ ${#reached} -lt ${#line} ] || continue
-
-			 # from the whole line, removes everything until, and including, "icmp_req=".
-			pingNumber=${line#*icmp_*eq=}
-			# removes everything after the first space.
-			pingNumber=${pingNumber%% *}
-			# from the whole line, removes everything until, and including, "time=".
-			pingTime=${line#*time=}
-			# removes everything after the first space.
-			pingTime=${pingTime%% *}
-			if [ "$firstLine" = true ]; then
-				firstLine=false
-			else
-				pairs="${pairs},"
-			fi
-			# concatenate to $string.
-			pairs="${pairs}${pingNumber}=${pingTime}"
-		done
-		# prints final $string in this sub shell back to $string.
-		echo $pairs))
-		# appending latencies to string to be sent.
-		[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
-	fi
+	# # if latency collecting is enabled.
+	# if [ "$hasLatency" -eq 1 ]; then
+	# 	# echo collecting latencies
+	# 	# removing the first line and the last 4 lines. only the ping lines remain.
+	# 	local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
+	# 	local pairs=""
+	# 	local firstLine=true
+	# 	# for each ping line.
+	# 	while read line; do
+	# 		# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
+	# 		reached=${line%time=*}
+	# 		# if "time=" has actually been removed, it means that line 
+	# 		# contains it, which also means the icmp request was fulfilled.
+	# 		# if line doesn't contain 'time=', we skip this line.
+	# 		[ ${#reached} -lt ${#line} ] || continue
+
+	# 		 # from the whole line, removes everything until, and including, "icmp_req=".
+	# 		pingNumber=${line#*icmp_*eq=}
+	# 		# removes everything after the first space.
+	# 		pingNumber=${pingNumber%% *}
+	# 		# from the whole line, removes everything until, and including, "time=".
+	# 		pingTime=${line#*time=}
+	# 		# removes everything after the first space.
+	# 		pingTime=${pingTime%% *}
+	# 		if [ "$firstLine" = true ]; then
+	# 			firstLine=false
+	# 		else
+	# 			pairs="${pairs},"
+	# 		fi
+	# 		# concatenate to $string.
+	# 		pairs="${pairs}${pingNumber}=${pingTime}"
+	# 	done
+	# 	# prints final $string in this sub shell back to $string.
+	# 	echo $pairs))
+	# 	# appending latencies to string to be sent.
+	# 	[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
+	# fi
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
@@ -250,7 +141,7 @@ collect_burst() {
 
 collect_wifi_devices() {
 	# checking if this data collecting is enabled
-	[ "$wifiDevices" -ne 1 ] && return
+	[ "$wifiDevices" -eq 0 ] && return
 
 	# devices and their data will be stored in this string variable.
 	local str=""
@@ -263,24 +154,13 @@ collect_wifi_devices() {
 		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
 		local wlan=$(get_root_ifname "$i" 2> /dev/null)
 		# if interface doesn't exist, skips this iteration.
-		[ "$wlan" == "" ] && continue
+		[ -z $wlan ] && continue
 		# getting info from each connected device on wifi. 
 		# grep returns empty when no devices are connected or if interface doesn't exist.
 		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
 		local devices_rx_pkts="$(iwinfo "$wlan" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
 		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
-		# first iteration won't put a space before the value.
-		local firstFileWrite=true
-
-		# string to be appended to devices packets file
-		local fileStr=""
-
-		local lastPktsFile="${dataCollectingDir}/devices_24_pkts"
-		if [[ "$i" -eq 1 ]]; then
-			lastPktsFile="${dataCollectingDir}/devices_5_pkts"
-		fi
-
 		while [ ${#devices} -gt 0 ]; do
 
 			# getting everything before the first space.
@@ -330,59 +210,17 @@ collect_wifi_devices() {
 			local tx_pkts=${devices_tx_pkts%% *}
 			devices_tx_pkts=${devices_tx_pkts#*$'\n'}
 
-			[ "$rx_pkts" == "" ] && continue
-			[ "$tx_pkts" == "" ] && continue
-
-			[ "$firstFileWrite" == true ] && firstFileWrite=false || fileStr="$fileStr "
-			fileStr="${fileStr}${deviceMac}_${rx_pkts}_${tx_pkts}"
-
-			local rx_pkts_diff=""
-			local tx_pkts_diff=""
-
-			if [ -f "$lastPktsFile" ]; then
-                local devices_pkts=$(cat "$lastPktsFile")
-
-				local pkts_device_removed=${devices_pkts/"$deviceMac"/""}
-
-				# if deviceMac not in recorded devices continue
-				[ ${#pkts_device_removed} -ge ${#devices_pkts} ] && continue
-
-				local last_pkts=${devices_pkts#*"$deviceMac"_}
-				last_pkts=${last_pkts%% *}
-
-				local last_rx_pkts=${last_pkts%_*}
-				[ "$last_rx_pkts" == "" ] && continue
-				rx_pkts_diff=$(($rx_pkts - $last_rx_pkts))
-				
-				local last_tx_pkts=${last_pkts#*_}
-				[ "$last_tx_pkts" == "" ] && continue
-				tx_pkts_diff=$(($tx_pkts - $last_tx_pkts))
-
-				# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-				# we skip this measure.
-				{ [ "$rx_pkts_diff" -lt 0 ] || [ "$tx_pkts_diff" -lt 0 ]; } && continue
-			else
-				continue
-			fi
+			[ -z $rx_pkts ] && continue
+			[ -z $tx_pkts ] && continue
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$firstRawWrite" == true ] && firstRawWrite=false || str="$str "
-			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
+			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts}_${tx_pkts}"
 		done
-		# empty out file (we only need last minute info)
-		> "$lastPktsFile"
 
-		# write to it if there are device infos
-		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$lastPktsFile"
 	done
-	if [ "$str" == "" ]; then
-		# only send data if there is something to send
-		activeMeasures="${activeMeasures/wd /}"
-		activeMeasures="${activeMeasures/ wd/}"
-		activeMeasures="${activeMeasures/wd/}"
-	else
-		rawData="${rawData}|wifiDevsStats ${str}"
-	fi
+	# only send data if there is something to send
+	[ -z $str ] && wifiDevices=0 || rawData="${rawData}|wifiDevsStats ${str}"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -491,46 +329,19 @@ collectData() {
 	# global variable where current raw data is stored before being written to file.
 	rawData=""
 
-	# global variable that controls which measures are active
-	activeMeasures=""
-
-	local firstMeasurement=1
-
-	if [ "$burstLoss" -eq 1 ]; then
-		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
-			activeMeasures="$activeMeasures "
-		else
-			firstMeasurement=0
-		fi
-		activeMeasures="${activeMeasures}bl"
-	fi
-
-	if [ "$pingAndWan" -eq 1 ]; then
-		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
-			activeMeasures="$activeMeasures "
-		else
-			firstMeasurement=0
-		fi
-		activeMeasures="${activeMeasures}p&w"
-	fi
-
-	if [ "$wifiDevices" -eq 1 ]; then
-		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
-			activeMeasures="$activeMeasures "
-		else
-			firstMeasurement=0
-		fi
-		activeMeasures="${activeMeasures}wd"
-	fi
-
 	# collecting all measures.
 	collect_burst
 	collect_wan
 	collect_wifi_devices
 
+	# global variable that controls which measures are active
+	activeMeasures=""
+
+	[ "$burstLoss" -eq 1 ] && activeMeasures="${activeMeasures}bl "
+    [ "$wifiDevices" -eq 1 ] && activeMeasures="${activeMeasures}wd "
+    [ "$pingAndWan" -eq 1 ] && activeMeasures="${activeMeasures}p&w " 
+    [ ${#activeMeasures} -gt 0 ] && activeMeasures=${activeMeasures%* }
+
 	# mapping from measurement names to collected artifacts:
 	# bl (burstLoss) -> burstPing, wanBytes
 	# p&w (pingAndWan) -> burstPing, wanBytes, wanPkts

From 2c5cfb391914d34d74a67effb88ed304af52f931 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 29 Sep 2022 19:11:57 -0300
Subject: [PATCH 61/77] shortening variable names + syntax shortening when
 possible

---
 files/usr/share/data_collecting.sh | 465 ++++++++++++++---------------
 1 file changed, 221 insertions(+), 244 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 46cccc48..1d4b9fc5 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -5,102 +5,95 @@
 . /usr/share/flashman_init.conf
 
 # directory where all data related to data collecting will be stored.
-dataCollectingDir="/tmp/data_collecting"
+dir="/tmp/data_collecting"
 # file where collected data will be stored before being compressed.
-rawDataFile="${dataCollectingDir}/raw"
+rawF="${dir}/raw"
 # directory where data will be stored if compressing old data is necessary.
-compressedDataDir="${dataCollectingDir}/compressed"
+compD="${dir}/compD"
 
 # gets current rx and tx bytes/packets from wan interface and compares them
 # with values from previous calls to calculate cross traffic
-collect_wan() {
+wan() {
 	# checking if this data collecting is enabled
 
 	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
-	local sendThisRound=1
-
 	# bytes received by the interface.
-	local rxBytes=$(get_wan_bytes_statistics RX)
+	local rx=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
-	local txBytes=$(get_wan_bytes_statistics TX)
+	local tx=$(get_wan_bytes_statistics TX)
 
-	local string="|wanBytes $rxBytes $txBytes"
-	rawData="${rawData}${string}"
+	raw="${raw}|wanBytes $rx $tx"
 
 	# burstLoss only gathers byte data
 	[ $pingAndWan -eq 0 ] && return
 
 	# packets received by the interface.
-	local rxPackets=$(get_wan_packets_statistics RX)
+	local pr=$(get_wan_packets_statistics RX)
 	# packets sent by the interface.
-	local txPackets=$(get_wan_packets_statistics TX)
+	local pt=$(get_wan_packets_statistics TX)
 
 	# data to be sent.
-	local string="|wanPkts $rxPackets $txPackets"
-	rawData="${rawData}${string}"
+	raw="${raw}|wanPkts $pr $pt"
 }
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
 # If latency collecting is enabled, extracts the individual icmp request numbers and 
 # their respective ping times. Builds a string with all this information and write them to file.
-collect_burst() {
+burst() {
 	# checking if this data collecting is enabled
 
 	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
 	# burst ping with $pingPackets amount of packets.
-	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
+	local p=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
 	# ping return value.
-	local pingError="$?"
+	local e="$?"
 
 	# if ping could not be executed, we skip this measure.
-	[ "$pingError" -eq 2 ] && burstLoss=0 && pingAndWan=0 && return
+	[ "$e" -eq 2 ] && burstLoss=0 && pingAndWan=0 && return
 
 	# An skipped measure will become missing data, for this minute, in the server.
 
 	# removes everything behind the summary that appears in the last lines.
-	local pingResultAux=${pingResult##* ping statistics ---[$'\r\n']}
+	local aux=${p##* ping statistics ---[$'\r\n']}
 	# removes everything after, and including, ' packets transmitted'.
-	local transmitted=${pingResultAux% packets transmitted*}
+	local tx=${aux% packets transmitted*}
 	# removes everything after, and including, ' received'.
-	local received=${pingResultAux% received*}
+	local rx=${aux% received*}
 	# removes everything before first space.
-	received=${received##* }
+	rx=${rx##* }
 	# integer representing the amount of packets not received.
-	local loss=$(($transmitted - $received))
-	# local loss=${pingResult%\% packet loss*} # removes everything after, and including, '% packet loss'.
-	# loss=${loss##* } # removes everything before first space.
+	local l=$(($tx - $rx))
 
 	# data to be sent.
-	local string="$loss $transmitted"
+	local str="$l $tx"
 
-	# removes everything before and including 'mdev = '
-	local latencyStats=${pingResult#*/mdev = }
+	# get latency stats
+	local lat=${p#*/mdev = }
 
 	# when there is 100% packet loss there the strings remain equal
 	# we only want to collect latency and std when there isn't 100% loss
 	# if loss is 100% we just send 0 in both cases, which will be ignored by the server
-	if [ ${#latencyStats} == ${#pingResult} ]; then
-	    string="$string 0 0"
+	if [ ${#lat} == ${#p} ]; then
+	    str="$str 0 0"
 	else
-		# removes everything before first backslash
-		local latencyAvg=${latencyStats#*/}
-		# removes everything after first backslash
-		latencyAvg=${latencyAvg%%/*}
-		# removes everything before and including last backslash
-		local latencyStd=${latencyStats##*/}
-		# removes everything after and including first space
-		latencyStd=${latencyStd%% *}
-
-		string="$string $latencyAvg $latencyStd"
+		# latency avarage
+		local la=${lat#*/}
+		la=${la%%/*}
+
+		# latency standard deviation
+		local ls=${lat##*/}
+		ls=${ls%% *}
+
+		str="$str $la $ls"
 	fi
 
 	# # if latency collecting is enabled.
 	# if [ "$hasLatency" -eq 1 ]; then
 	# 	# echo collecting latencies
 	# 	# removing the first line and the last 4 lines. only the ping lines remain.
-	# 	local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
+	# 	local latencies=$(printf "%s" "$ping" | head -n -4 | sed '1d' | (
 	# 	local pairs=""
 	# 	local firstLine=true
 	# 	# for each ping line.
@@ -136,211 +129,194 @@ collect_burst() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	rawData="${rawData}|burstPing $string"
+	raw="${raw}|burstPing $str"
 }
 
-collect_wifi_devices() {
+wifi() {
 	# checking if this data collecting is enabled
 	[ "$wifiDevices" -eq 0 ] && return
 
 	# devices and their data will be stored in this string variable.
 	local str=""
 
-	local firstRawWrite=true
+	# flag that indicates if it's the first raw file write
+	local first=1
 
 	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
 		# getting wifi interface name.
 		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
-		local wlan=$(get_root_ifname "$i" 2> /dev/null)
+		local w=$(get_root_ifname "$i" 2> /dev/null)
 		# if interface doesn't exist, skips this iteration.
-		[ -z $wlan ] && continue
+		[ -z $w ] && continue
 		# getting info from each connected device on wifi. 
 		# grep returns empty when no devices are connected or if interface doesn't exist.
-		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
-		local devices_rx_pkts="$(iwinfo "$wlan" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
-		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
+		local iw="$(iwinfo "$w" assoclist | grep ago)"
+		local pr="$(iwinfo "$w" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
+		local pt="$(iwinfo "$w" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
-		while [ ${#devices} -gt 0 ]; do
+		while [ ${#iw} -gt 0 ]; do
 
 			# getting everything before the first space.
-			local deviceMac=${devices%% *}
-
-			# getting everything after the first two spaces.
-			local signal=${devices#*  }
+			local mac=${iw%% *}
 
-			# getting everything before the first occasion of ' /'
-			signal=${signal%% /*}
-
-			# if unknown discard
-			[ "$signal" == "unknown" ] && devices=${devices#*$'\n'} && continue
-
-			# getting everything before the first occasion of ' dBm'
-			signal=${signal%% dBm*}
+			# signal.
+			local s=${iw#*  }
+			s=${s%% /*}
+			[ "$s" == "unknown" ] && iw=${iw#*$'\n'} && continue
+			s=${s%% dBm*}
 
 			# getting after '(SNR '. 
-			devices=${devices#*\(SNR }
-
-			# getting everything before the first closing parenthesis.
-			local snr=${devices%%\)*}
-
+			iw=${iw#*\(SNR }
+			local snr=${iw%%\)*}
 			# if SNR equals signal we assume noise of -95dBm
-			[ $signal -eq $snr ] && snr=$(($signal+95))
-
-			# getting everything before ' ms'.
-			local time=${devices%% ms*}
+			[ $s -eq $snr ] && snr=$(($s+95))
 
-			# getting everything after the first space.
-			time=${time##* }
+			# time
+			local t=${iw%% ms*}
+			t=${t##* }
 
 			# getting everything after 'ago'.
-			devices=${devices#*ago}
-
+			iw=${iw#*ago}
 			# getting everything after '\n', if it exists. last line won't have it, so nothing will be changed.
 			# we can't add the line feed along witht the previous parameter expansion because we wouldn't match
-			# the last line and so we wouldn't make $devices length become zero.
-			devices=${devices#*$'\n'}
+			# the last line and so we wouldn't make $iw length become zero.
+			iw=${iw#*$'\n'}
 
-			# if $time is greater than one minute, we don't use this device's info.
-			[ "$time" -gt 60000 ] && continue
+			# if $t is greater than one minute, we don't use this device's info.
+			[ "$t" -gt 60000 ] && continue
 			
-			local rx_pkts=${devices_rx_pkts%% *}
-			devices_rx_pkts=${devices_rx_pkts#*$'\n'}
+			local rx=${pr%% *}
+			pr=${pr#*$'\n'}
 
-			local tx_pkts=${devices_tx_pkts%% *}
-			devices_tx_pkts=${devices_tx_pkts#*$'\n'}
+			local tx=${pt%% *}
+			pt=${pt#*$'\n'}
 
-			[ -z $rx_pkts ] && continue
-			[ -z $tx_pkts ] && continue
+			[ -z $rx ] && continue
+			[ -z $tx ] && continue
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
-			[ "$firstRawWrite" == true ] && firstRawWrite=false || str="$str "
-			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts}_${tx_pkts}"
+			[ "$first" -eq 1 ] && first=0 || str="$str "
+			str="${str}${i}_${mac}_${s}_${snr}_${rx}_${tx}"
 		done
 
 	done
 	# only send data if there is something to send
-	[ -z $str ] && wifiDevices=0 || rawData="${rawData}|wifiDevsStats ${str}"
+	[ -z $str ] && wifiDevices=0 || raw="${raw}|wifiDevsStats ${str}"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
 # first argument ($1).
-fileSize() {
+fSize() {
 	# file size is the information at the 1st column.
-	local wcline=$(wc -c "$1")
+	local wc=$(wc -c "$1")
 	#remove suffix composed of space and anything else.
-	local size=${wcline% *}
+	local size=${wc% *}
 	echo $size
 }
 
 # prints the sum of the sizes of all files inside given directory path.
-sumFileSizesInPath() {
+sumSizes() {
 	# boolean that marks that at least one file exists inside given directory.
-	local anyFile=false
+	local any=0
 	# for each file in that directory.
 	for i in "$1"/*; do
 		# if that pattern expansion exists as a file.
 		[ -f "$i" ] || continue
 		# set boolean to true.
-		anyFile=true
+		any=1
 		# as we have at least one file, we don't need to loop through all files.
 		break
 	done
-	if [ "$anyFile" = false ]; then
-		# if no files.
-		# prints zero. size of nothing is 0.
-		echo 0
-		# result was given, we can leave function.
-		return 0
-	fi
+	# if no files.
+	# prints zero. size of nothing is 0.
+	# result was given, we can leave function.
+	[ "$any" -eq 0 ] && then && echo 0 && return 0
+
 	# if there is at least one file.
 
 	# prints a list of sizes and files.
-	local wcResult=$(wc -c "$1"/*)
+	local wc=$(wc -c "$1"/*)
 	# if there is 2 or more files, the last line will have a "total". remove that string.
-	local hasTotal=${wcResult% total}
+	local has=${wc% total}
 	# if it has a total, it was removed. if not, nothing was removed and both strings are the same.
 
-	 # if length of string with "total" removed is smaller than original string.
-	if [ ${#hasTotal} -lt ${#wcResult} ]; then
-		# remove everything before the last word, which is the value for total, and print what remains.
-		echo ${hasTotal##* }
-	else
-		# if there were no total, then there was only one file, in one line of output, and the first column is the size value.
-		# remove everything past, and including, the first space, and print what remains.
-		echo ${wcResult%% *}
-	fi
+	# if length of string with "total" removed is smaller than original string.
+	# if it has total remove everything before the last word, which is the value for total, and print what remains.
+	# if there were no total, then there was only one file, in one line of output, and the first column is the size value.
+	# remove everything past, and including, the first space, and print what remains.
+	[ ${#has} -lt ${#wc} ] && echo ${has##* } || echo ${wc%% *}
 }
 
-# sum the size $rawDataFile with the size of all compressed files inside the $compressedDataDir. if that 
-# sum is bigger than number given in first argument ($1), compress that file and move it to $compressedDataDir.
-zipFile() {
-	local capSize="$1"
+# sum the size $raw with the size of all compressed files inside the $compD. if that 
+# sum is bigger than number given in first argument ($1), compress that file and move it to $compD.
+zip() {
+	local cap="$1"
 
 	# if file doesn't exist, we won't have to compressed anything.
 	# a return of 1 means nothing has been be gzipped.
-	[ -f "$rawDataFile" ] || return 1 
+	[ -f "$rawF" ] || return 1 
 
 	# size of file with raw data.
-	local size=$(fileSize "$rawDataFile")
+	local size=$(fSize "$rawF")
 	# sum of file sizes in directory for compressed files.
-	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
+	local dSize=$(sumSizes "$compD")
 
-	# if sum is smaller than $capSize, do nothing.
+	# if sum is smaller than $cap, do nothing.
 	# echo checking file size to zip
 	# a return of 1 means nothing will be gzipped.
-	[ $(($size + $dirSize)) -lt $capSize ] && return 1
+	[ $(($size + $dSize)) -lt $cap ] && return 1
 
 	# compressing file where raw data is held.
-	gzip "$rawDataFile"
+	gzip "$rawF"
 	# move newly compressed file to directory where compressed files should be.
-	mv "${rawDataFile}.gz" "$compressedDataDir/$(date +%s).gz"
+	mv "${rawF}.gz" "$compD/$(date +%s).gz"
 }
 
-# files are removed from $compressedDataDir until all data remaining is below number given in first 
+# files are removed from $compD until all data remaining is below number given in first 
 # argument ($1) as bytes. As files are named using a formatted date, pattern expansion of file 
 # names will always order oldest files first. This was done this way so we don't have to sort files 
 # by date ourselves. we let shell do the sorting.
-removeOldFiles() {
-	local capSize="$1"
+rmOld() {
+	local cap="$1"
 
 	# get the sum of sizes of all files in bytes.
-	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
+	local dSize=$(sumSizes "$compD")
 
-	# if $dirSize is more than given $capSize, remove oldest file. which is 
+	# if $dSize is more than given $cap, remove oldest file. which is 
 	# the file that shell orders as first.
-	for i in "$compressedDataDir"/*; do
-		# if we are under $capSize. do nothing.
-		[ $dirSize -lt $capSize ] && break;
+	for i in "$compD"/*; do
+		# if we are under $cap. do nothing.
+		[ $dSize -lt $cap ] && break;
 		# removes that file.
 		rm "$i"
 		# subtract that file's size from sum.
-		dirSize=$(($dirSize - $(fileSize "$i")))
+		dSize=$(($dSize - $(fSize "$i")))
 	done
 }
 
-# collect every data and stores in '$rawDataFile'. if the size of the file is 
+# collect every data and stores in '$rawF'. if the size of the file is 
 # too big, compress it and move it to a directory of compressed files. If 
 # directory of compressed files grows too big delete oldest compressed files.
-collectData() {
+collect() {
 	# getting current unix time in seconds.
-	local timestamp=$(date +%s)
+	local ts=$(date +%s)
 
 	# global variable where current raw data is stored before being written to file.
-	rawData=""
+	raw=""
 
 	# collecting all measures.
-	collect_burst
-	collect_wan
-	collect_wifi_devices
+	burst
+	wan
+	wifi
 
 	# global variable that controls which measures are active
-	activeMeasures=""
+	active=""
 
-	[ "$burstLoss" -eq 1 ] && activeMeasures="${activeMeasures}bl "
-    [ "$wifiDevices" -eq 1 ] && activeMeasures="${activeMeasures}wd "
-    [ "$pingAndWan" -eq 1 ] && activeMeasures="${activeMeasures}p&w " 
-    [ ${#activeMeasures} -gt 0 ] && activeMeasures=${activeMeasures%* }
+	[ "$burstLoss" -eq 1 ] && active="${active}bl "
+    [ "$wifiDevices" -eq 1 ] && active="${active}wd "
+    [ "$pingAndWan" -eq 1 ] && active="${active}p&w " 
+    [ ${#active} -gt 0 ] && active=${active%* }
 
 	# mapping from measurement names to collected artifacts:
 	# bl (burstLoss) -> burstPing, wanBytes
@@ -349,24 +325,24 @@ collectData() {
 
 	# example of an expected raw data with all measures present:
 	# 'bl p&w wd|213234556456|burstPing 0 100 1.246 0.161|wanBytes 12345 1234|wanPkts 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
-	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
-	# cleaning 'rawData' value from memory.
-	rawData=""
+	[ -n "$raw" ] && [ ${#active} -gt 0 ] && echo "${active}|${ts}${raw}" >> "$rawF";
+	# cleaning 'raw' value from memory.
+	raw=""
 
 	# creates directory of for compressed files, if it doesn't already exists.
-	mkdir -p "$compressedDataDir"
-	# $(zipFile) returns 0 only if any amount of files has been compressed 
+	mkdir -p "$compD"
+	# $(zip) returns 0 only if any amount of files has been compressed 
 	# and, consequently, moved to the directory of compressed files. So
-	# $(removeOldFiles) is only executed if any new compressed file was 
+	# $(rmOld) is only executed if any new compressed file was 
 	# created.
-	zipFile $((32*1024)) && removeOldFiles $((24*1024))
-	# the difference between the cap size sent to $(zipFile) and 
-	# $(removeOldFiles) is the size left as a minimum amount for raw data 
+	zip $((32*1024)) && rmOld $((24*1024))
+	# the difference between the cap size sent to $(zip) and 
+	# $(rmOld) is the size left as a minimum amount for raw data 
 	# before compressing it. This means that, if there are no compressed files, 
 	# the uncompressed file could grow to as much as the cap size given to 
-	# $(zipFile). but in case there is any amount of compressed files, the 
-	# uncompressed file can grow to as much as the cap size given to $(zipFile) 
-	# minus the sum of all the compressed files sizes. As $(removeOldFiles) 
+	# $(zip). but in case there is any amount of compressed files, the 
+	# uncompressed file can grow to as much as the cap size given to $(zip) 
+	# minus the sum of all the compressed files sizes. As $(rmOld) 
 	# will keep the sum of all compressed files sizes to a maximum of its given 
 	# cap size, the difference between these two cap sizes is the minimum size 
 	# the uncompressed file will always have available for it's growth.
@@ -375,82 +351,83 @@ collectData() {
 # if number given as first argument ($1) isn't 0, ping server at address given 
 # in second argument ($2) and returns the exit code of $(curl), but if that 
 # number is zero, return that number.
-checkServerState() {
-	local lastState="$1"
-	if [ "$lastState" -ne "0" ]; then
+checkState() {
+	local s="$1"
+	if [ "$s" -ne "0" ]; then
 		# echo pinging alarm server to check if it's alive.
 		curl -s -m 10 "https://$alarmServerAddress:7890/ping" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" > /dev/null
 		# return $(curl) exit code.
-		lastState="$?"
+		s="$?"
 	fi
-	# echo last state is $lastState
-	return $lastState
+	# echo last state is $s
+	return $s
 }
 
 # sends file at given path ($1) to server at given address ($2) using $(curl) 
 # and returns $(curl) exit code.
-sendToServer() {
-	local filepath="$1"
+upload() {
+	local path="$1"
 
 	# 'get_mac()' is defined in /usr/share/functions/device_functions.sh.
 	local mac=$(get_mac);
 
-	status=$(curl --write-out '%{http_code}' -s -m 20 --connect-timeout 5 --output /dev/null \
+	s=$(curl --write-out '%{http_code}' -s -m 20 --connect-timeout 5 --output /dev/null \
 	-XPOST "https://$alarmServerAddress:7890/data" -H 'Content-Encoding: gzip' \
 	-H 'Content-Type: text/plain' -H "X-ANLIX-ID: $mac" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" \
-	-H "Send-Time: $(date +%s)" --data-binary @"$filepath")
-	curlCode="$?"
+	-H "Send-Time: $(date +%s)" --data-binary @"$path")
+	code="$?"
 	# 'log()' is defined in /usr/share/functions/common_functions.sh.
-	[ "$curlCode" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${curlCode}'." && return "$curlCode"
-	log "DATA_COLLECTING" "Data sent with response status code '${status}'."
-	[ "$status" -ge 200 ] && [ "$status" -lt 300 ] && return 0
+	[ "$code" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${code}'." && return "$code"
+	log "DATA_COLLECTING" "Data sent with response status code '${s}'."
+	[ "$s" -ge 200 ] && [ "$s" -lt 300 ] && return 0
 	return 1
 }
 
-# for each compressed file given in $compressedDataDir, send that file to a $alarmServerAddress. 
+# for each compressed file given in $compD, send that file to a $alarmServerAddress. 
 # If any sending is unsuccessful, stops sending files and return it's exit code.
-sendCompressedData() {
+sendComp() {
 	# echo going to send compressed files
-	# echo "$compressedDataDir"/*
+	# echo "$compD"/*
 	# for each compressed file in the pattern expansion.
-	for i in "$compressedDataDir"/*; do
+	for i in "$compD"/*; do
 		# if file exists, sends file and if $(curl) exit code isn't equal to 0, returns $(curl) exit code 
 		# without deleting the file we tried to send. if $(curl) exit code is equal to 0, removes file
-		[ -f "$i" ] && { sendToServer "$i" || return "$?"; } && rm "$i"
+		[ -f "$i" ] && { upload "$i" || return "$?"; } && rm "$i"
 	done
 	return 0
 }
 
-# compresses $rawDataFile, sends it to $alarmServerAddress and deletes it. If send was 
+# compresses $raw, sends it to $alarmServerAddress and deletes it. If send was 
 # successful, remove original files, if not, keeps it. Returns the return of $(curl).
-sendUncompressedData() {
+sendUncomp() {
 	# if no uncompressed file, nothing wrong, but there's nothing to do in this function.
-	[ -f "$rawDataFile" ] || return 0
+	[ -f "$rawF" ] || return 0
 
 	# echo going to send uncompressed file
 	# the name the compressed file will have.
-	local compressedTempFile="${rawDataFile}.gz"
+	local file="${rawF}.gz"
 	# remove old file if it exists. it should never be left there.
-	[ -f "$compressedTempFile" ] && rm "$compressedTempFile"
+	[ -f "$file" ] && rm "$file"
 
 	# in case the process is interrupted, delete compressed file.
-	trap "rm $compressedTempFile" SIGTERM
+	trap "rm $file" SIGTERM
 	# compressing to a temporary file but keeping original, uncompressed, intact.
-	gzip -k "$rawDataFile"
+	gzip -k "$rawF"
 
 	# sends compressed file.
-	sendToServer "$compressedTempFile"
+	upload "$file"
 	# storing $(curl) exit code.
-	local sentResult="$?"
+	local res="$?"
 
 	# if send was successful, removes original file.
-	[ "$sentResult" -eq 0 ] && rm "$rawDataFile"
+	[ "$res" -eq 0 ] && rm "$rawF"
 	# removes temporary file. a new temporary will be created next time, with more content, 
-	rm "$compressedTempFile"
+	rm "$file"
 	# cleans trap.
 	trap - SIGTERM
 
-	return $sentResult # Returns #(curl) exit code.
+	# Returns #(curl) exit code.
+	return $res
 }
 
 
@@ -479,8 +456,8 @@ sendUncompressedData() {
 # # is written to file at path given in forth argument ($4), but if it's not 0, the 
 # # third argument ($3) is written instead.
 # writeBackoff() {
-# 	local currentServerState=$1 normalBackoff=$2 changedBackoff=$3 backoffCounterPath=$4
-# 	if [ "$currentServerState" -ne 0 ]; then
+# 	local curState=$1 normalBackoff=$2 changedBackoff=$3 backoffCounterPath=$4
+# 	if [ "$curState" -ne 0 ]; then
 # 		normalBackoff=$changedBackoff
 # 	fi
 # 	# echo writting backoff.
@@ -488,7 +465,7 @@ sendUncompressedData() {
 # }
 
 # Attempts to send data some times with 10 seconds of sleep time between tries.
-sendData() {
+send() {
 	# echo going to send data
 	# amount of attempts of sending data, to alarm server, before giving up.
 	local tries=3
@@ -498,21 +475,21 @@ sendData() {
 		# send compressed data, if everything was sent, send uncompressed 
 		# data. If server wasn't alive last time, ping it and if it's alive 
 		# now, then send all data, but if it's still dead, do nothing.
-		local lastServerStateFilePath="$dataCollectingDir/serverState"
-		local lastServerState="1"
-		[ -f "$lastServerStateFilePath" ] && lastServerState=$(cat "$lastServerStateFilePath")
-		# echo lastServerState=$lastServerState
-
-		checkServerState "$lastServerState" && sendCompressedData && sendUncompressedData
-		local currentServerState="$?"
-		# echo currentServerState=$currentServerState
+		local stateF="$dir/state"
+		local last="1"
+		[ -f "$stateF" ] && last=$(cat "$stateF")
+		# echo state=$state
+
+		checkState "$last" && sendComp && sendUncomp
+		local cur="$?"
+		# echo cur=$cur
 		# if server stops before sending some data, current server state will differ from last server state.
-		# $currentServerState get the exit code of the first of these 3 functions above that returns anything other than 0.
+		# $cur get the exit code of the first of these 3 functions above that returns anything other than 0.
 
 		# writes the $(curl) exit code if it has changed since last attempt to send data.
-		[ "$currentServerState" -ne "$lastServerState" ] && echo "$currentServerState" > "$lastServerStateFilePath"
+		[ "$cur" -ne "$last" ] && echo "$cur" > "$stateF"
 		# if data was sent successfully, we stop retrying.
-		[ "$currentServerState" -eq 0 ] && break
+		[ "$cur" -eq 0 ] && break
 
 		tries=$(($tries - 1))
 		# leaves retry loop when $retries reaches zero.
@@ -524,14 +501,14 @@ sendData() {
 }
 
 # echoes a random number between 0 and 59 (inclusive).
-random0To59() {
-	local rand=$(head /dev/urandom | tr -dc "0123456789")
+rand0To59() {
+	local r=$(head /dev/urandom | tr -dc "0123456789")
 	# taking the first 2 digits.
-	rand=${rand:0:2}
+	r=${r:0:2}
 	# "08" and "09" don't work for "$(())".
-	[ ${rand:0:1} = "0" ] && rand=${rand:1:2}
-	# $rand is a integer between 0 and 99 (inclusive), this makes it an integer between 0 and 59.
-	echo $(($rand * 6 / 10))
+	[ ${r:0:1} = "0" ] && r=${r:1:2}
+	# $r is a integer between 0 and 99 (inclusive), this makes it an integer between 0 and 59.
+	echo $(($r * 6 / 10))
 	# our Ash has not been compiled to work with floats.
 }
 
@@ -546,60 +523,60 @@ random0To59() {
 # current time, maintaining the correct second the interval, given in second 
 # argument ($2), would make it fall into, and sleep for the amount of time 
 # left to that second.
-getStartTime() {
-	local startTimeFilePath="$1" interval="$2"
+start() {
+	local file="$1" int="$2"
 
-	local startTime
+	local start
 	# if file holding start time exists.
-	if [ -f "$startTimeFilePath" ]; then
-		local currentTime=$(date +%s)
-		startTime=$(cat "$startTimeFilePath") # get the time stamp inside that file.
+	if [ -f "$file" ]; then
+		local cur=$(date +%s)
+		start=$(cat "$file") # get the time stamp inside that file.
 		# advance timestamp to the closest time after current time that the given interval could produce.
-		startTime=$(($startTime + (($currentTime - $startTime) / $interval) * $interval + $interval))
+		start=$(($start + (($cur - $start) / $int) * $int + $int))
 		# that division does not produce a float number. ex: (7/2)*2 = 6.
-		# $startTime + (($currentTime - $startTime) / $interval) * $interval 
-		# results in the closest time to $currentTime that $interval could 
-		# produce, starting from $startTime, that is smaller than $currentTime. 
-		# By adding $interval, we get the closest time to $currentTime that 
-		# $interval could produce that is bigger than $currentTtime.
+		# $start + (($cur - $start) / $int) * $int 
+		# results in the closest time to $cur that $int could 
+		# produce, starting from $start, that is smaller than $cur. 
+		# By adding $int, we get the closest time to $cur that 
+		# $int could produce that is bigger than $currentTtime.
 		# sleep for the amount of time left to next interval.
-		sleep $(($startTime - $currentTime))
+		sleep $(($start - $cur))
 		# this makes us always start at the same second, even if the process is shut down for a long time.
 	else 
 		# if file holding start time doesn't exist.
 		# sleeping for at most 59 seconds to distribute data collecting through a minute window.
-		sleep $(random0To59)
+		sleep $(rand0To59)
 		# use current time.
-		startTime=$(date +%s)
+		start=$(date +%s)
 	fi
 	# substitute that time in file, or create a new file.
-	echo $startTime > "$startTimeFilePath"
+	echo $start > "$file"
 	# print start time found, or current time.
-	echo $startTime
+	echo $start
 }
 
 # deletes files, marking process state, from previous process and deletes temporary
 # files that could be hanging if process is terminated in a critical part.
-cleanFiles() {
-	rm "${dataCollectingDir}/serverState" 2> /dev/null
-	# rm "${dataCollectingDir}/backoffCounter" 2> /dev/null
-	rm "${rawDataFile}.gz" 2> /dev/null
+clean() {
+	rm "${dir}/state" 2> /dev/null
+	# rm "${dir}/backoffCounter" 2> /dev/null
+	rm "${rawF}.gz" 2> /dev/null
 }
 
 # collects and sends data forever.
 loop() {
 	# interval between beginnings of data collecting.
-	local interval=60
+	local int=60
 	# making sure directory exists.
-	mkdir -p "$dataCollectingDir"
+	mkdir -p "$dir"
 	# time when we will start executing.
-	local time=$(getStartTime "${dataCollectingDir}/startTime" $interval)
+	local time=$(start "${dir}/startTime" $int)
 
 	# infinite loop where we execute all procedures over and over again until the end of times.
 	while true; do
 		# echo startTime $time
 		# making sure directory exists every time.
-		mkdir -p "$dataCollectingDir"
+		mkdir -p "$dir"
 
 		# getting parameters every time we need to send data, this way we don't have to 
 		# restart the service if a parameter changes.
@@ -614,31 +591,31 @@ loop() {
 		)
 
 		# does everything related to collecting and storing data.`
-		collectData
+		collect
 		# does everything related to sending data and deletes data sent.
-		sendData
+		send
 
 		# time after all procedures are finished.
-		local endTime=$(date +%s)
+		local end=$(date +%s)
 		# this will hold the time left until we should run the next iteration of this loop
-		local timeLeftForNextRun="-1"
-		# while time left is negative, which could happen if $(($time - $endTime)) is bigger than $interval.
-		while [ "$timeLeftForNextRun" -lt 0 ]; do
+		local delta="-1"
+		# while time left is negative, which could happen if $(($time - $end)) is bigger than $int.
+		while [ "$delta" -lt 0 ]; do
 			# advance time, when current data collecting has started, by one interval.
-			time=$(($time + $interval))
+			time=$(($time + $int))
 			# calculate time left to collect data again.
-			timeLeftForNextRun=$(($time - $endTime))
+			delta=$(($time - $end))
 		done
-		# echo timeLeftForNextRun=$timeLeftForNextRun
+		# echo delta=$delta
 		 # sleep for the time remaining until next data collecting.
-		sleep $timeLeftForNextRun
+		sleep $delta
 
 		# writing next loop time to file that, at this line, matches current time.
-		echo $time > "${dataCollectingDir}/startTime"
+		echo $time > "${dir}/startTime"
 	done
 }
 
 # deletes files, marking process state, from previous process.
-cleanFiles
+clean
 # the infinite loop.
 loop

From c86d15ea140d4ac6f8a34bf04cca74182469dc22 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 29 Sep 2022 19:49:39 -0300
Subject: [PATCH 62/77] further syntax shortening

---
 files/usr/share/data_collecting.sh | 144 ++++++++++++++---------------
 1 file changed, 72 insertions(+), 72 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 1d4b9fc5..eabbfdfa 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -16,58 +16,58 @@ compD="${dir}/compD"
 wan() {
 	# checking if this data collecting is enabled
 
-	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
+	[ $bl -eq 0 ] && [ $p&w -eq 0 ] && return
 
 	# bytes received by the interface.
-	local rx=$(get_wan_bytes_statistics RX)
+	local r=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
-	local tx=$(get_wan_bytes_statistics TX)
+	local t=$(get_wan_bytes_statistics TX)
 
-	raw="${raw}|wanBytes $rx $tx"
+	raw="${raw}|wanBytes $r $t"
 
-	# burstLoss only gathers byte data
-	[ $pingAndWan -eq 0 ] && return
+	# bl only gathers byte data
+	[ $p&w -eq 0 ] && return
 
 	# packets received by the interface.
-	local pr=$(get_wan_packets_statistics RX)
+	r=$(get_wan_packets_statistics RX)
 	# packets sent by the interface.
-	local pt=$(get_wan_packets_statistics TX)
+	t=$(get_wan_packets_statistics TX)
 
 	# data to be sent.
-	raw="${raw}|wanPkts $pr $pt"
+	raw="${raw}|wanPkts $r $t"
 }
 
-# takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
+# takes current unix timestamp, executes ping, in burst, to $ping server.
 # If latency collecting is enabled, extracts the individual icmp request numbers and 
 # their respective ping times. Builds a string with all this information and write them to file.
 burst() {
 	# checking if this data collecting is enabled
 
-	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
+	[ $bl -eq 0 ] && [ $p&w -eq 0 ] && return
 
-	# burst ping with $pingPackets amount of packets.
-	local p=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
+	# burst ping with $pkts amount of packets.
+	local p=$(ping -i 0.01 -c "$pkts" "$ping")
 	# ping return value.
 	local e="$?"
 
 	# if ping could not be executed, we skip this measure.
-	[ "$e" -eq 2 ] && burstLoss=0 && pingAndWan=0 && return
+	[ "$e" -eq 2 ] && bl=0 && p&w=0 && return
 
 	# An skipped measure will become missing data, for this minute, in the server.
 
 	# removes everything behind the summary that appears in the last lines.
-	local aux=${p##* ping statistics ---[$'\r\n']}
+	local x=${p##* ping statistics ---[$'\r\n']}
 	# removes everything after, and including, ' packets transmitted'.
-	local tx=${aux% packets transmitted*}
+	local t=${x% packets transmitted*}
 	# removes everything after, and including, ' received'.
-	local rx=${aux% received*}
+	local r=${x% received*}
 	# removes everything before first space.
-	rx=${rx##* }
+	r=${r##* }
 	# integer representing the amount of packets not received.
-	local l=$(($tx - $rx))
+	local l=$(($t - $r))
 
 	# data to be sent.
-	local str="$l $tx"
+	local s="$l $t"
 
 	# get latency stats
 	local lat=${p#*/mdev = }
@@ -76,7 +76,7 @@ burst() {
 	# we only want to collect latency and std when there isn't 100% loss
 	# if loss is 100% we just send 0 in both cases, which will be ignored by the server
 	if [ ${#lat} == ${#p} ]; then
-	    str="$str 0 0"
+	    s="$s 0 0"
 	else
 		# latency avarage
 		local la=${lat#*/}
@@ -86,7 +86,7 @@ burst() {
 		local ls=${lat##*/}
 		ls=${ls%% *}
 
-		str="$str $la $ls"
+		s="$s $la $ls"
 	fi
 
 	# # if latency collecting is enabled.
@@ -129,12 +129,12 @@ burst() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	raw="${raw}|burstPing $str"
+	raw="${raw}|burstPing $s"
 }
 
 wifi() {
 	# checking if this data collecting is enabled
-	[ "$wifiDevices" -eq 0 ] && return
+	[ "$wd" -eq 0 ] && return
 
 	# devices and their data will be stored in this string variable.
 	local str=""
@@ -174,7 +174,7 @@ wifi() {
 
 			# time
 			local t=${iw%% ms*}
-			t=${t##* }
+			ts=${ts##* }
 
 			# getting everything after 'ago'.
 			iw=${iw#*ago}
@@ -184,25 +184,25 @@ wifi() {
 			iw=${iw#*$'\n'}
 
 			# if $t is greater than one minute, we don't use this device's info.
-			[ "$t" -gt 60000 ] && continue
+			[ "$ts" -gt 60000 ] && continue
 			
-			local rx=${pr%% *}
+			local r=${pr%% *}
 			pr=${pr#*$'\n'}
 
-			local tx=${pt%% *}
+			local t=${pt%% *}
 			pt=${pt#*$'\n'}
 
-			[ -z $rx ] && continue
-			[ -z $tx ] && continue
+			[ -z $r ] && continue
+			[ -z $t ] && continue
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$first" -eq 1 ] && first=0 || str="$str "
-			str="${str}${i}_${mac}_${s}_${snr}_${rx}_${tx}"
+			str="${str}${i}_${mac}_${s}_${snr}_${r}_${t}"
 		done
 
 	done
 	# only send data if there is something to send
-	[ -z $str ] && wifiDevices=0 || raw="${raw}|wifiDevsStats ${str}"
+	[ -z $str ] && wd=0 || raw="${raw}|wifiDevsStats ${str}"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -210,9 +210,9 @@ wifi() {
 fSize() {
 	# file size is the information at the 1st column.
 	local wc=$(wc -c "$1")
-	#remove suffix composed of space and anything else.
-	local size=${wc% *}
-	echo $size
+	# gets size
+	local sz=${wc% *}
+	echo $sz
 }
 
 # prints the sum of the sizes of all files inside given directory path.
@@ -258,14 +258,14 @@ zip() {
 	[ -f "$rawF" ] || return 1 
 
 	# size of file with raw data.
-	local size=$(fSize "$rawF")
+	local sz=$(fSize "$rawF")
 	# sum of file sizes in directory for compressed files.
 	local dSize=$(sumSizes "$compD")
 
 	# if sum is smaller than $cap, do nothing.
 	# echo checking file size to zip
 	# a return of 1 means nothing will be gzipped.
-	[ $(($size + $dSize)) -lt $cap ] && return 1
+	[ $(($sz + $dSize)) -lt $cap ] && return 1
 
 	# compressing file where raw data is held.
 	gzip "$rawF"
@@ -313,15 +313,15 @@ collect() {
 	# global variable that controls which measures are active
 	active=""
 
-	[ "$burstLoss" -eq 1 ] && active="${active}bl "
-    [ "$wifiDevices" -eq 1 ] && active="${active}wd "
-    [ "$pingAndWan" -eq 1 ] && active="${active}p&w " 
+	[ "$bl" -eq 1 ] && active="${active}bl "
+    [ "$wd" -eq 1 ] && active="${active}wd "
+    [ "$p&w" -eq 1 ] && active="${active}p&w " 
     [ ${#active} -gt 0 ] && active=${active%* }
 
 	# mapping from measurement names to collected artifacts:
-	# bl (burstLoss) -> burstPing, wanBytes
-	# p&w (pingAndWan) -> burstPing, wanBytes, wanPkts
-	# wd (wifiDevices) -> wifiDevsStats
+	# bl (bl) -> burstPing, wanBytes
+	# p&w (p&w) -> burstPing, wanBytes, wanPkts
+	# wd (wd) -> wifiDevsStats
 
 	# example of an expected raw data with all measures present:
 	# 'bl p&w wd|213234556456|burstPing 0 100 1.246 0.161|wanBytes 12345 1234|wanPkts 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
@@ -355,7 +355,7 @@ checkState() {
 	local s="$1"
 	if [ "$s" -ne "0" ]; then
 		# echo pinging alarm server to check if it's alive.
-		curl -s -m 10 "https://$alarmServerAddress:7890/ping" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" > /dev/null
+		curl -s -m 10 "https://$server:7890/ping" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" > /dev/null
 		# return $(curl) exit code.
 		s="$?"
 	fi
@@ -369,21 +369,21 @@ upload() {
 	local path="$1"
 
 	# 'get_mac()' is defined in /usr/share/functions/device_functions.sh.
-	local mac=$(get_mac);
+	local m=$(get_mac);
 
 	s=$(curl --write-out '%{http_code}' -s -m 20 --connect-timeout 5 --output /dev/null \
-	-XPOST "https://$alarmServerAddress:7890/data" -H 'Content-Encoding: gzip' \
-	-H 'Content-Type: text/plain' -H "X-ANLIX-ID: $mac" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" \
+	-XPOST "https://$server:7890/data" -H 'Content-Encoding: gzip' \
+	-H 'Content-Type: text/plain' -H "X-ANLIX-ID: $m" -H "X-ANLIX-SEC: $FLM_CLIENT_SECRET" \
 	-H "Send-Time: $(date +%s)" --data-binary @"$path")
-	code="$?"
+	c="$?"
 	# 'log()' is defined in /usr/share/functions/common_functions.sh.
-	[ "$code" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${code}'." && return "$code"
+	[ "$c" -ne 0 ] && log "DATA_COLLECTING" "Data sent with curl exit code '${c}'." && return "$c"
 	log "DATA_COLLECTING" "Data sent with response status code '${s}'."
 	[ "$s" -ge 200 ] && [ "$s" -lt 300 ] && return 0
 	return 1
 }
 
-# for each compressed file given in $compD, send that file to a $alarmServerAddress. 
+# for each compressed file given in $compD, send that file to a $server. 
 # If any sending is unsuccessful, stops sending files and return it's exit code.
 sendComp() {
 	# echo going to send compressed files
@@ -397,7 +397,7 @@ sendComp() {
 	return 0
 }
 
-# compresses $raw, sends it to $alarmServerAddress and deletes it. If send was 
+# compresses $raw, sends it to $server and deletes it. If send was 
 # successful, remove original files, if not, keeps it. Returns the return of $(curl).
 sendUncomp() {
 	# if no uncompressed file, nothing wrong, but there's nothing to do in this function.
@@ -417,17 +417,17 @@ sendUncomp() {
 	# sends compressed file.
 	upload "$file"
 	# storing $(curl) exit code.
-	local res="$?"
+	local r="$?"
 
 	# if send was successful, removes original file.
-	[ "$res" -eq 0 ] && rm "$rawF"
+	[ "$r" -eq 0 ] && rm "$rawF"
 	# removes temporary file. a new temporary will be created next time, with more content, 
 	rm "$file"
 	# cleans trap.
 	trap - SIGTERM
 
 	# Returns #(curl) exit code.
-	return $res
+	return $r
 }
 
 
@@ -468,7 +468,7 @@ sendUncomp() {
 send() {
 	# echo going to send data
 	# amount of attempts of sending data, to alarm server, before giving up.
-	local tries=3
+	local i=3
 
 	while true; do
 		# check if the last time, data was sent, server was alive. if it was, 
@@ -491,9 +491,9 @@ send() {
 		# if data was sent successfully, we stop retrying.
 		[ "$cur" -eq 0 ] && break
 
-		tries=$(($tries - 1))
+		i=$(($i - 1))
 		# leaves retry loop when $retries reaches zero.
-		[ "$tries" -eq 0 ] && break
+		[ "$i" -eq 0 ] && break
 		# echo retrying in 10 seconds
 		# sleeps before retrying. this time must take the 60 second interval into consideration.
 		sleep 10
@@ -570,7 +570,7 @@ loop() {
 	# making sure directory exists.
 	mkdir -p "$dir"
 	# time when we will start executing.
-	local time=$(start "${dir}/startTime" $int)
+	local ts=$(start "${dir}/startTime" $int)
 
 	# infinite loop where we execute all procedures over and over again until the end of times.
 	while true; do
@@ -581,13 +581,13 @@ loop() {
 		# getting parameters every time we need to send data, this way we don't have to 
 		# restart the service if a parameter changes.
 		eval $(cat /root/flashbox_config.json | jsonfilter \
-			-e "hasLatency=@.data_collecting_has_latency" \
-			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
-			-e "pingServerAddress=@.data_collecting_ping_fqdn" \
-			-e "pingPackets=@.data_collecting_ping_packets" \
-			-e "burstLoss=@.data_collecting_burst_loss" \
-			-e "wifiDevices=@.data_collecting_wifi_devices" \
-			-e "pingAndWan=@.data_collecting_ping_and_wan" \
+			# -e "hasLatency=@.data_collecting_has_latency" \
+			-e "server=@.data_collecting_alarm_fqdn" \
+			-e "ping=@.data_collecting_ping_fqdn" \
+			-e "pkts=@.data_collecting_ping_packets" \
+			-e "bl=@.data_collecting_burst_loss" \
+			-e "wd=@.data_collecting_wifi_devices" \
+			-e "p&w=@.data_collecting_ping_and_wan" \
 		)
 
 		# does everything related to collecting and storing data.`
@@ -598,20 +598,20 @@ loop() {
 		# time after all procedures are finished.
 		local end=$(date +%s)
 		# this will hold the time left until we should run the next iteration of this loop
-		local delta="-1"
+		local d="-1"
 		# while time left is negative, which could happen if $(($time - $end)) is bigger than $int.
-		while [ "$delta" -lt 0 ]; do
+		while [ "$d" -lt 0 ]; do
 			# advance time, when current data collecting has started, by one interval.
-			time=$(($time + $int))
+			ts=$(($ts + $int))
 			# calculate time left to collect data again.
-			delta=$(($time - $end))
+			d=$(($ts - $end))
 		done
-		# echo delta=$delta
+		# echo d=$d
 		 # sleep for the time remaining until next data collecting.
-		sleep $delta
+		sleep $d
 
 		# writing next loop time to file that, at this line, matches current time.
-		echo $time > "${dir}/startTime"
+		echo $ts > "${dir}/startTime"
 	done
 }
 

From eba10be6b7fbc78f3eff4e2d82f4dc6b1e104662 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 29 Sep 2022 20:00:58 -0300
Subject: [PATCH 63/77] removing instances of data collecting

---
 files/usr/share/flashman_update.sh            |  3 +--
 .../functions/data_collecting_functions.sh    | 21 ++++++-------------
 files/usr/share/keepalive.sh                  |  3 +--
 3 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/files/usr/share/flashman_update.sh b/files/usr/share/flashman_update.sh
index 3e825bd5..d85e3b64 100644
--- a/files/usr/share/flashman_update.sh
+++ b/files/usr/share/flashman_update.sh
@@ -245,7 +245,6 @@ mem_usage=$(get_memory_usage)"
 		json_get_var _upnp_devices_index upnp_devices_index
 		json_get_var _vlan_index vlan_index
 		json_get_var _data_collecting_is_active data_collecting_is_active
-		json_get_var _data_collecting_has_latency data_collecting_has_latency
 		json_get_var _data_collecting_alarm_fqdn data_collecting_alarm_fqdn
 		json_get_var _data_collecting_ping_fqdn data_collecting_ping_fqdn
 		json_get_var _data_collecting_ping_packets data_collecting_ping_packets
@@ -520,7 +519,7 @@ mem_usage=$(get_memory_usage)"
 		fi
 
 		# updates data collecting parameters.
-		set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
+		set_data_collecting_parameters "$_data_collecting_is_active" \
 			                           "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
 			                           "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
 			                           "$_data_collecting_wifi_devices" "$_data_collecting_ping_and_wan"
diff --git a/files/usr/share/functions/data_collecting_functions.sh b/files/usr/share/functions/data_collecting_functions.sh
index 17c1cde6..be2d1d5f 100644
--- a/files/usr/share/functions/data_collecting_functions.sh
+++ b/files/usr/share/functions/data_collecting_functions.sh
@@ -6,18 +6,16 @@
 # 'saved_data_collecting_is_active' parameter.
 set_data_collecting_parameters() {
 	local data_collecting_is_active="${1:-0}"
-	local data_collecting_has_latency="${2:-0}"
-	local data_collecting_alarm_fqdn="${3:-$FLM_SVADDR}"
-	local data_collecting_ping_fqdn="${4:-$FLM_SVADDR}"
-	local data_collecting_ping_packets="${5:-100}"
-	local data_collecting_burst_loss="${6:-0}"
-	local data_collecting_wifi_devices="${7:-0}"
-	local data_collecting_ping_and_wan="${8:-0}"
+	local data_collecting_alarm_fqdn="${2:-$FLM_SVADDR}"
+	local data_collecting_ping_fqdn="${3:-$FLM_SVADDR}"
+	local data_collecting_ping_packets="${4:-100}"
+	local data_collecting_burst_loss="${5:-0}"
+	local data_collecting_wifi_devices="${6:-0}"
+	local data_collecting_ping_and_wan="${7:-0}"
 
 	json_cleanup
 	json_load_file /root/flashbox_config.json
 	json_get_var saved_data_collecting_is_active data_collecting_is_active
-	json_get_var saved_data_collecting_has_latency data_collecting_has_latency
 	json_get_var saved_data_collecting_alarm_fqdn data_collecting_alarm_fqdn
 	json_get_var saved_data_collecting_ping_fqdn data_collecting_ping_fqdn
 	json_get_var saved_data_collecting_ping_packets data_collecting_ping_packets
@@ -34,13 +32,6 @@ set_data_collecting_parameters() {
 		log "DATA_COLLECTING" "Updated 'data_collecting_is_active' parameter to '$data_collecting_is_active'."
 	fi
 
-	# Updating value if $data_collecting_has_latency has changed.
-	if [ "$saved_data_collecting_has_latency" != "$data_collecting_has_latency" ]; then
-		anyChange=true
-		json_add_boolean data_collecting_has_latency "$data_collecting_has_latency"
-		log "DATA_COLLECTING" "Updated 'data_collecting_has_latency' parameter to '$data_collecting_has_latency'."
-	fi
-
 	# Updating value if $data_collecting_alarm_fqdn has changed.
 	if [ "$saved_data_collecting_alarm_fqdn" != "$data_collecting_alarm_fqdn" ]; then
 		anyChange=true
diff --git a/files/usr/share/keepalive.sh b/files/usr/share/keepalive.sh
index 54871588..df4cb824 100755
--- a/files/usr/share/keepalive.sh
+++ b/files/usr/share/keepalive.sh
@@ -119,7 +119,6 @@ mem_usage=$(get_memory_usage)"
 			json_get_var _do_newprobe do_newprobe
 			json_get_var _mqtt_status mqtt_status
 			json_get_var _data_collecting_is_active data_collecting_is_active
-			json_get_var _data_collecting_has_latency data_collecting_has_latency
 			json_get_var _data_collecting_alarm_fqdn data_collecting_alarm_fqdn
 			json_get_var _data_collecting_ping_fqdn data_collecting_ping_fqdn
 			json_get_var _data_collecting_ping_packets data_collecting_ping_packets
@@ -174,7 +173,7 @@ mem_usage=$(get_memory_usage)"
 			fi
 
 			# updates data collecting parameters.
-			set_data_collecting_parameters "$_data_collecting_is_active" "$_data_collecting_has_latency" \
+			set_data_collecting_parameters "$_data_collecting_is_active" \
 			                               "$_data_collecting_alarm_fqdn" "$_data_collecting_ping_fqdn" \
 			                               "$_data_collecting_ping_packets" "$_data_collecting_burst_loss" \
 										   "$_data_collecting_wifi_devices" "$_data_collecting_ping_and_wan"

From b57fb28981546f883cfe0d3ebe057d47805e5f57 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 30 Sep 2022 12:15:34 -0300
Subject: [PATCH 64/77] removing unnecessary function

---
 files/usr/share/functions/device_functions.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/files/usr/share/functions/device_functions.sh b/files/usr/share/functions/device_functions.sh
index 615b2d76..8c3d93bc 100644
--- a/files/usr/share/functions/device_functions.sh
+++ b/files/usr/share/functions/device_functions.sh
@@ -276,14 +276,6 @@ get_wan_packets_statistics() {
 	fi
 }
 
-get_wifi_device_signature() {
-	local _dev_mac="$1"
-	local _q=""
-	_q="$(ubus -S call hostapd.wlan0 get_clients | jsonfilter -e '@.clients["'"$_dev_mac"'"].signature')"
-	[ -z "$_q" ] && [ "$(is_5ghz_capable)" -eq "1" ] && _q="$(ubus -S call hostapd.wlan1 get_clients | jsonfilter -e '@.clients["'"$_dev_mac"'"].signature')"
-	echo "$_q"
-}
-
 needs_reboot_change_mode() {
 	reboot
 }

From 6f8d64242cd94eaa694efc840bca463f8a236bfa Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 30 Sep 2022 12:57:39 -0300
Subject: [PATCH 65/77] reverting back to commit before syntax changes +
 removing compressed mechanism that allows for data to be stored then sent
 only when network is available again

---
 files/usr/share/data_collecting.sh | 85 +-----------------------------
 1 file changed, 2 insertions(+), 83 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 787fd3d1..3e4d154c 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -8,8 +8,6 @@
 dataCollectingDir="/tmp/data_collecting"
 # file where collected data will be stored before being compressed.
 rawDataFile="${dataCollectingDir}/raw"
-# directory where data will be stored if compressing old data is necessary.
-compressedDataDir="${dataCollectingDir}/compressed"
 # file where wan rx/tx bytes will be stored
 wanBytesFile="${dataCollectingDir}/wan_bytes"
 # file where wan rx/tx packets will be stored
@@ -434,53 +432,6 @@ sumFileSizesInPath() {
 	fi
 }
 
-# sum the size $rawDataFile with the size of all compressed files inside the $compressedDataDir. if that 
-# sum is bigger than number given in first argument ($1), compress that file and move it to $compressedDataDir.
-zipFile() {
-	local capSize="$1"
-
-	# if file doesn't exist, we won't have to compressed anything.
-	# a return of 1 means nothing has been be gzipped.
-	[ -f "$rawDataFile" ] || return 1 
-
-	# size of file with raw data.
-	local size=$(fileSize "$rawDataFile")
-	# sum of file sizes in directory for compressed files.
-	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
-
-	# if sum is smaller than $capSize, do nothing.
-	# echo checking file size to zip
-	# a return of 1 means nothing will be gzipped.
-	[ $(($size + $dirSize)) -lt $capSize ] && return 1
-
-	# compressing file where raw data is held.
-	gzip "$rawDataFile"
-	# move newly compressed file to directory where compressed files should be.
-	mv "${rawDataFile}.gz" "$compressedDataDir/$(date +%s).gz"
-}
-
-# files are removed from $compressedDataDir until all data remaining is below number given in first 
-# argument ($1) as bytes. As files are named using a formatted date, pattern expansion of file 
-# names will always order oldest files first. This was done this way so we don't have to sort files 
-# by date ourselves. we let shell do the sorting.
-removeOldFiles() {
-	local capSize="$1"
-
-	# get the sum of sizes of all files in bytes.
-	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
-
-	# if $dirSize is more than given $capSize, remove oldest file. which is 
-	# the file that shell orders as first.
-	for i in "$compressedDataDir"/*; do
-		# if we are under $capSize. do nothing.
-		[ $dirSize -lt $capSize ] && break;
-		# removes that file.
-		rm "$i"
-		# subtract that file's size from sum.
-		dirSize=$(($dirSize - $(fileSize "$i")))
-	done
-}
-
 # collect every data and stores in '$rawDataFile'. if the size of the file is 
 # too big, compress it and move it to a directory of compressed files. If 
 # directory of compressed files grows too big delete oldest compressed files.
@@ -541,24 +492,6 @@ collectData() {
 	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
-
-	# creates directory of for compressed files, if it doesn't already exists.
-	mkdir -p "$compressedDataDir"
-	# $(zipFile) returns 0 only if any amount of files has been compressed 
-	# and, consequently, moved to the directory of compressed files. So
-	# $(removeOldFiles) is only executed if any new compressed file was 
-	# created.
-	zipFile $((32*1024)) && removeOldFiles $((24*1024))
-	# the difference between the cap size sent to $(zipFile) and 
-	# $(removeOldFiles) is the size left as a minimum amount for raw data 
-	# before compressing it. This means that, if there are no compressed files, 
-	# the uncompressed file could grow to as much as the cap size given to 
-	# $(zipFile). but in case there is any amount of compressed files, the 
-	# uncompressed file can grow to as much as the cap size given to $(zipFile) 
-	# minus the sum of all the compressed files sizes. As $(removeOldFiles) 
-	# will keep the sum of all compressed files sizes to a maximum of its given 
-	# cap size, the difference between these two cap sizes is the minimum size 
-	# the uncompressed file will always have available for it's growth.
 }
 
 # if number given as first argument ($1) isn't 0, ping server at address given 
@@ -596,23 +529,9 @@ sendToServer() {
 	return 1
 }
 
-# for each compressed file given in $compressedDataDir, send that file to a $alarmServerAddress. 
-# If any sending is unsuccessful, stops sending files and return it's exit code.
-sendCompressedData() {
-	# echo going to send compressed files
-	# echo "$compressedDataDir"/*
-	# for each compressed file in the pattern expansion.
-	for i in "$compressedDataDir"/*; do
-		# if file exists, sends file and if $(curl) exit code isn't equal to 0, returns $(curl) exit code 
-		# without deleting the file we tried to send. if $(curl) exit code is equal to 0, removes file
-		[ -f "$i" ] && { sendToServer "$i" || return "$?"; } && rm "$i"
-	done
-	return 0
-}
-
 # compresses $rawDataFile, sends it to $alarmServerAddress and deletes it. If send was 
 # successful, remove original files, if not, keeps it. Returns the return of $(curl).
-sendUncompressedData() {
+upload() {
 	# if no uncompressed file, nothing wrong, but there's nothing to do in this function.
 	[ -f "$rawDataFile" ] || return 0
 
@@ -692,7 +611,7 @@ sendData() {
 		[ -f "$lastServerStateFilePath" ] && lastServerState=$(cat "$lastServerStateFilePath")
 		# echo lastServerState=$lastServerState
 
-		checkServerState "$lastServerState" && sendCompressedData && sendUncompressedData
+		checkServerState "$lastServerState" && upload
 		local currentServerState="$?"
 		# echo currentServerState=$currentServerState
 		# if server stops before sending some data, current server state will differ from last server state.

From 0c0131acb4ed3f60975ceaa75a7f22d97582f665 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 30 Sep 2022 13:07:55 -0300
Subject: [PATCH 66/77] removing compressed file logic

---
 files/usr/share/data_collecting.sh | 85 +-----------------------------
 1 file changed, 2 insertions(+), 83 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 787fd3d1..3e4d154c 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -8,8 +8,6 @@
 dataCollectingDir="/tmp/data_collecting"
 # file where collected data will be stored before being compressed.
 rawDataFile="${dataCollectingDir}/raw"
-# directory where data will be stored if compressing old data is necessary.
-compressedDataDir="${dataCollectingDir}/compressed"
 # file where wan rx/tx bytes will be stored
 wanBytesFile="${dataCollectingDir}/wan_bytes"
 # file where wan rx/tx packets will be stored
@@ -434,53 +432,6 @@ sumFileSizesInPath() {
 	fi
 }
 
-# sum the size $rawDataFile with the size of all compressed files inside the $compressedDataDir. if that 
-# sum is bigger than number given in first argument ($1), compress that file and move it to $compressedDataDir.
-zipFile() {
-	local capSize="$1"
-
-	# if file doesn't exist, we won't have to compressed anything.
-	# a return of 1 means nothing has been be gzipped.
-	[ -f "$rawDataFile" ] || return 1 
-
-	# size of file with raw data.
-	local size=$(fileSize "$rawDataFile")
-	# sum of file sizes in directory for compressed files.
-	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
-
-	# if sum is smaller than $capSize, do nothing.
-	# echo checking file size to zip
-	# a return of 1 means nothing will be gzipped.
-	[ $(($size + $dirSize)) -lt $capSize ] && return 1
-
-	# compressing file where raw data is held.
-	gzip "$rawDataFile"
-	# move newly compressed file to directory where compressed files should be.
-	mv "${rawDataFile}.gz" "$compressedDataDir/$(date +%s).gz"
-}
-
-# files are removed from $compressedDataDir until all data remaining is below number given in first 
-# argument ($1) as bytes. As files are named using a formatted date, pattern expansion of file 
-# names will always order oldest files first. This was done this way so we don't have to sort files 
-# by date ourselves. we let shell do the sorting.
-removeOldFiles() {
-	local capSize="$1"
-
-	# get the sum of sizes of all files in bytes.
-	local dirSize=$(sumFileSizesInPath "$compressedDataDir")
-
-	# if $dirSize is more than given $capSize, remove oldest file. which is 
-	# the file that shell orders as first.
-	for i in "$compressedDataDir"/*; do
-		# if we are under $capSize. do nothing.
-		[ $dirSize -lt $capSize ] && break;
-		# removes that file.
-		rm "$i"
-		# subtract that file's size from sum.
-		dirSize=$(($dirSize - $(fileSize "$i")))
-	done
-}
-
 # collect every data and stores in '$rawDataFile'. if the size of the file is 
 # too big, compress it and move it to a directory of compressed files. If 
 # directory of compressed files grows too big delete oldest compressed files.
@@ -541,24 +492,6 @@ collectData() {
 	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
-
-	# creates directory of for compressed files, if it doesn't already exists.
-	mkdir -p "$compressedDataDir"
-	# $(zipFile) returns 0 only if any amount of files has been compressed 
-	# and, consequently, moved to the directory of compressed files. So
-	# $(removeOldFiles) is only executed if any new compressed file was 
-	# created.
-	zipFile $((32*1024)) && removeOldFiles $((24*1024))
-	# the difference between the cap size sent to $(zipFile) and 
-	# $(removeOldFiles) is the size left as a minimum amount for raw data 
-	# before compressing it. This means that, if there are no compressed files, 
-	# the uncompressed file could grow to as much as the cap size given to 
-	# $(zipFile). but in case there is any amount of compressed files, the 
-	# uncompressed file can grow to as much as the cap size given to $(zipFile) 
-	# minus the sum of all the compressed files sizes. As $(removeOldFiles) 
-	# will keep the sum of all compressed files sizes to a maximum of its given 
-	# cap size, the difference between these two cap sizes is the minimum size 
-	# the uncompressed file will always have available for it's growth.
 }
 
 # if number given as first argument ($1) isn't 0, ping server at address given 
@@ -596,23 +529,9 @@ sendToServer() {
 	return 1
 }
 
-# for each compressed file given in $compressedDataDir, send that file to a $alarmServerAddress. 
-# If any sending is unsuccessful, stops sending files and return it's exit code.
-sendCompressedData() {
-	# echo going to send compressed files
-	# echo "$compressedDataDir"/*
-	# for each compressed file in the pattern expansion.
-	for i in "$compressedDataDir"/*; do
-		# if file exists, sends file and if $(curl) exit code isn't equal to 0, returns $(curl) exit code 
-		# without deleting the file we tried to send. if $(curl) exit code is equal to 0, removes file
-		[ -f "$i" ] && { sendToServer "$i" || return "$?"; } && rm "$i"
-	done
-	return 0
-}
-
 # compresses $rawDataFile, sends it to $alarmServerAddress and deletes it. If send was 
 # successful, remove original files, if not, keeps it. Returns the return of $(curl).
-sendUncompressedData() {
+upload() {
 	# if no uncompressed file, nothing wrong, but there's nothing to do in this function.
 	[ -f "$rawDataFile" ] || return 0
 
@@ -692,7 +611,7 @@ sendData() {
 		[ -f "$lastServerStateFilePath" ] && lastServerState=$(cat "$lastServerStateFilePath")
 		# echo lastServerState=$lastServerState
 
-		checkServerState "$lastServerState" && sendCompressedData && sendUncompressedData
+		checkServerState "$lastServerState" && upload
 		local currentServerState="$?"
 		# echo currentServerState=$currentServerState
 		# if server stops before sending some data, current server state will differ from last server state.

From 3317ff93d404e4b21af52f81f91652d5b62accf0 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 30 Sep 2022 13:08:06 -0300
Subject: [PATCH 67/77] removing unnecessary function

---
 files/usr/share/functions/device_functions.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/files/usr/share/functions/device_functions.sh b/files/usr/share/functions/device_functions.sh
index 615b2d76..8c3d93bc 100644
--- a/files/usr/share/functions/device_functions.sh
+++ b/files/usr/share/functions/device_functions.sh
@@ -276,14 +276,6 @@ get_wan_packets_statistics() {
 	fi
 }
 
-get_wifi_device_signature() {
-	local _dev_mac="$1"
-	local _q=""
-	_q="$(ubus -S call hostapd.wlan0 get_clients | jsonfilter -e '@.clients["'"$_dev_mac"'"].signature')"
-	[ -z "$_q" ] && [ "$(is_5ghz_capable)" -eq "1" ] && _q="$(ubus -S call hostapd.wlan1 get_clients | jsonfilter -e '@.clients["'"$_dev_mac"'"].signature')"
-	echo "$_q"
-}
-
 needs_reboot_change_mode() {
 	reboot
 }

From 8149a11b31077d342962e52c5560df55a7b8d724 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 30 Sep 2022 17:42:11 -0300
Subject: [PATCH 68/77] removing compressed wan bytes/packets pre-processing
 and wifi devices packets pre-processing

---
 files/usr/share/data_collecting.sh | 326 ++++++-----------------------
 1 file changed, 68 insertions(+), 258 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 3e4d154c..4e495bb1 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -18,121 +18,26 @@ wanPacketsFile="${dataCollectingDir}/wan_pkts"
 collect_wan() {
 	# checking if this data collecting is enabled
 
-	local isBurstLossActive=${activeMeasures/*bl*/bl}
-
-	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
-
-	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
-
-	local sendThisRound=1
+	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
 	# bytes received by the interface.
 	local rxBytes=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
 	local txBytes=$(get_wan_bytes_statistics TX)
 
-	local last_rxBytes=""
-	local last_txBytes=""
-
-	if [ -f "$wanBytesFile" ]; then
-		last_rxBytes=$(cat "$wanBytesFile")
-		# gets second value
-		last_txBytes=${last_rxBytes##* }
-		# gets first value
-		last_rxBytes=${last_rxBytes%% *}
-	fi
-
-	local rxBytesDiff=0
-	local txBytesDiff=0
-
-	if [ "$last_rxBytes" == "" ] || [ "$last_txBytes" == "" ]; then
-		# if last bytes are not defined. define them using the current wan interface bytes value. then we skip this measure.
-		# empty out file (we only need last minute info)
-		> "$wanBytesFile"
-		echo "$rxBytes $txBytes" >> "$wanBytesFile"
-		# don't write data this round. we need a full minute of bytes to calculate cross traffic.
-		sendThisRound=0
-	else
-		# bytes received since last time.
-		rxBytesDiff=$(($rxBytes - $last_rxBytes))
-		# bytes transmitted since last time
-		txBytesDiff=$(($txBytes - $last_txBytes))
-		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-		# we skip this measure.
-		{ [ "$rxBytesDiff" -lt 0 ] || [ "$txBytesDiff" -lt 0 ]; } && sendThisRound=0
-	fi
-
-	# empty out file (we only need last minute info)
-	> "$wanBytesFile"
-	echo "$rxBytes $txBytes" >> "$wanBytesFile"
-
-	if [ "$sendThisRound" -eq 1 ]; then
-		local string="|wanBytes $rxBytesDiff $txBytesDiff"
-		rawData="${rawData}${string}"
-	else
-		activeMeasures="${activeMeasures/bl /}"
-		activeMeasures="${activeMeasures/ bl/}"
-		activeMeasures="${activeMeasures/bl/}"
-		activeMeasures="${activeMeasures/p&w /}"
-		activeMeasures="${activeMeasures/ p&w/}"
-		activeMeasures="${activeMeasures/p&w/}"
-	fi
+	local string="|wanBytes $rxBytes $txBytes"
+	rawData="${rawData}${string}"
 
 	# burstLoss only gathers byte data
-	if [ "$isPingAndWanActive" != "p&w" ]; then
-		return
-	fi
+	[ $pingAndWan -eq 0 ] && return
 
 	# packets received by the interface.
 	local rxPackets=$(get_wan_packets_statistics RX)
 	# packets sent by the interface.
 	local txPackets=$(get_wan_packets_statistics TX)
 
-	local last_rxPackets=""
-	local last_txPackets=""
-
-	if [ -f "$wanPacketsFile" ]; then
-		last_rxPackets=$(cat "$wanPacketsFile")
-		# gets second value
-		last_txPackets=${last_rxPackets##* }
-		# gets first value
-		last_rxPackets=${last_rxPackets%% *}
-	fi
-
-	local rxPacketsDiff=0
-	local txPacketsDiff=0
-
-	if [ "$last_rxPackets" == "" ] || [ "$last_txPackets" == "" ]; then
-		# if last packets are not defined. define them using the current wan interface Packets value. then we skip this measure.
-		# empty out file (we only need last minute info)
-		> "$wanPacketsFile"
-		echo "$rxPackets $txPackets" >> "$wanPacketsFile"
-		# don't write data this round. we need a full minute of packets to calculate cross traffic.
-		sendThisRound=0
-	else
-		# packets received since last time.
-		rxPacketsDiff=$(($rxPackets - $last_rxPackets))
-		# packets transmitted since last time
-		txPacketsDiff=$(($txPackets - $last_txPackets))
-		# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-		# we skip this measure.
-		{ [ "$rxPacketsDiff" -lt 0 ] || [ "$txPacketsDiff" -lt 0 ]; } && sendThisRound=0
-	fi
-
-	# empty out file (we only need last minute info)
-	> "$wanPacketsFile"
-	echo "$rxPackets $txPackets" >> "$wanPacketsFile"
-
-	# needs to gather one more minute
-	if [ "$sendThisRound" -ne 1 ]; then
-		activeMeasures="${activeMeasures/p&w /}"
-		activeMeasures="${activeMeasures/ p&w/}"
-		activeMeasures="${activeMeasures/p&w/}"
-		return
-	fi
-
 	# data to be sent.
-	local string="|wanPkts $rxPacketsDiff $txPacketsDiff"
+	local string="|wanPkts $rxPackets $txPackets"
 	rawData="${rawData}${string}"
 }
 
@@ -142,11 +47,7 @@ collect_wan() {
 collect_burst() {
 	# checking if this data collecting is enabled
 
-	local isBurstLossActive=${activeMeasures/*bl*/bl}
-
-	local isPingAndWanActive=${activeMeasures/*p&w*/p&w}
-
-	[ "$isBurstLossActive" != "bl" ] && [ "$isPingAndWanActive" != "p&w" ] && return
+	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
 	# burst ping with $pingPackets amount of packets.
 	local pingResult=$(ping -i 0.01 -c "$pingPackets" "$pingServerAddress")
@@ -154,15 +55,7 @@ collect_burst() {
 	local pingError="$?"
 
 	# if ping could not be executed, we skip this measure.
-	if [ "$pingError" -eq 2 ]; then
-		activeMeasures="${activeMeasures/bl /}"
-		activeMeasures="${activeMeasures/ bl/}"
-		activeMeasures="${activeMeasures/bl/}"
-		activeMeasures="${activeMeasures/p&w /}"
-		activeMeasures="${activeMeasures/ p&w/}"
-		activeMeasures="${activeMeasures/p&w/}"
-		return
-	fi
+	[ "$pingError" -eq 2 ] && burstLoss=0 && pingAndWan=0 && return
 
 	# An skipped measure will become missing data, for this minute, in the server.
 
@@ -203,43 +96,43 @@ collect_burst() {
 		string="$string $latencyAvg $latencyStd"
 	fi
 
-	# if latency collecting is enabled.
-	if [ "$hasLatency" -eq 1 ]; then
-		# echo collecting latencies
-		# removing the first line and the last 4 lines. only the ping lines remain.
-		local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
-		local pairs=""
-		local firstLine=true
-		# for each ping line.
-		while read line; do
-			# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
-			reached=${line%time=*}
-			# if "time=" has actually been removed, it means that line 
-			# contains it, which also means the icmp request was fulfilled.
-			# if line doesn't contain 'time=', we skip this line.
-			[ ${#reached} -lt ${#line} ] || continue
-
-			 # from the whole line, removes everything until, and including, "icmp_req=".
-			pingNumber=${line#*icmp_*eq=}
-			# removes everything after the first space.
-			pingNumber=${pingNumber%% *}
-			# from the whole line, removes everything until, and including, "time=".
-			pingTime=${line#*time=}
-			# removes everything after the first space.
-			pingTime=${pingTime%% *}
-			if [ "$firstLine" = true ]; then
-				firstLine=false
-			else
-				pairs="${pairs},"
-			fi
-			# concatenate to $string.
-			pairs="${pairs}${pingNumber}=${pingTime}"
-		done
-		# prints final $string in this sub shell back to $string.
-		echo $pairs))
-		# appending latencies to string to be sent.
-		[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
-	fi
+	# # if latency collecting is enabled.
+	# if [ "$hasLatency" -eq 1 ]; then
+	# 	# echo collecting latencies
+	# 	# removing the first line and the last 4 lines. only the ping lines remain.
+	# 	local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
+	# 	local pairs=""
+	# 	local firstLine=true
+	# 	# for each ping line.
+	# 	while read line; do
+	# 		# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
+	# 		reached=${line%time=*}
+	# 		# if "time=" has actually been removed, it means that line 
+	# 		# contains it, which also means the icmp request was fulfilled.
+	# 		# if line doesn't contain 'time=', we skip this line.
+	# 		[ ${#reached} -lt ${#line} ] || continue
+
+	# 		 # from the whole line, removes everything until, and including, "icmp_req=".
+	# 		pingNumber=${line#*icmp_*eq=}
+	# 		# removes everything after the first space.
+	# 		pingNumber=${pingNumber%% *}
+	# 		# from the whole line, removes everything until, and including, "time=".
+	# 		pingTime=${line#*time=}
+	# 		# removes everything after the first space.
+	# 		pingTime=${pingTime%% *}
+	# 		if [ "$firstLine" = true ]; then
+	# 			firstLine=false
+	# 		else
+	# 			pairs="${pairs},"
+	# 		fi
+	# 		# concatenate to $string.
+	# 		pairs="${pairs}${pingNumber}=${pingTime}"
+	# 	done
+	# 	# prints final $string in this sub shell back to $string.
+	# 	echo $pairs))
+	# 	# appending latencies to string to be sent.
+	# 	[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
+	# fi
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
@@ -248,12 +141,12 @@ collect_burst() {
 
 collect_wifi_devices() {
 	# checking if this data collecting is enabled
-	[ "$wifiDevices" -ne 1 ] && return
+	[ "$wifiDevices" -eq 0 ] && return
 
 	# devices and their data will be stored in this string variable.
 	local str=""
 
-	local firstRawWrite=true
+	local firstRawWrite=1
 
 	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
@@ -261,24 +154,13 @@ collect_wifi_devices() {
 		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
 		local wlan=$(get_root_ifname "$i" 2> /dev/null)
 		# if interface doesn't exist, skips this iteration.
-		[ "$wlan" == "" ] && continue
+		[ -z $wlan ] && continue
 		# getting info from each connected device on wifi. 
 		# grep returns empty when no devices are connected or if interface doesn't exist.
 		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
 		local devices_rx_pkts="$(iwinfo "$wlan" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
 		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
-		# first iteration won't put a space before the value.
-		local firstFileWrite=true
-
-		# string to be appended to devices packets file
-		local fileStr=""
-
-		local lastPktsFile="${dataCollectingDir}/devices_24_pkts"
-		if [[ "$i" -eq 1 ]]; then
-			lastPktsFile="${dataCollectingDir}/devices_5_pkts"
-		fi
-
 		while [ ${#devices} -gt 0 ]; do
 
 			# getting everything before the first space.
@@ -328,59 +210,16 @@ collect_wifi_devices() {
 			local tx_pkts=${devices_tx_pkts%% *}
 			devices_tx_pkts=${devices_tx_pkts#*$'\n'}
 
-			[ "$rx_pkts" == "" ] && continue
-			[ "$tx_pkts" == "" ] && continue
-
-			[ "$firstFileWrite" == true ] && firstFileWrite=false || fileStr="$fileStr "
-			fileStr="${fileStr}${deviceMac}_${rx_pkts}_${tx_pkts}"
-
-			local rx_pkts_diff=""
-			local tx_pkts_diff=""
-
-			if [ -f "$lastPktsFile" ]; then
-                local devices_pkts=$(cat "$lastPktsFile")
-
-				local pkts_device_removed=${devices_pkts/"$deviceMac"/""}
-
-				# if deviceMac not in recorded devices continue
-				[ ${#pkts_device_removed} -ge ${#devices_pkts} ] && continue
-
-				local last_pkts=${devices_pkts#*"$deviceMac"_}
-				last_pkts=${last_pkts%% *}
-
-				local last_rx_pkts=${last_pkts%_*}
-				[ "$last_rx_pkts" == "" ] && continue
-				rx_pkts_diff=$(($rx_pkts - $last_rx_pkts))
-				
-				local last_tx_pkts=${last_pkts#*_}
-				[ "$last_tx_pkts" == "" ] && continue
-				tx_pkts_diff=$(($tx_pkts - $last_tx_pkts))
-
-				# if subtraction created a negative value, it means it has overflown or interface has been restarted.
-				# we skip this measure.
-				{ [ "$rx_pkts_diff" -lt 0 ] || [ "$tx_pkts_diff" -lt 0 ]; } && continue
-			else
-				continue
-			fi
+			[ -z $rx_pkts ] && continue
+			[ -z $tx_pkts ] && continue
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
-			[ "$firstRawWrite" == true ] && firstRawWrite=false || str="$str "
-			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts_diff}_${tx_pkts_diff}"
+			[ "$firstRawWrite" -eq 1 ] && firstRawWrite=0 || str="$str "
+			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts}_${tx_pkts}"
 		done
-		# empty out file (we only need last minute info)
-		> "$lastPktsFile"
-
-		# write to it if there are device infos
-		[ ${#fileStr} -gt 0 ] && echo "$fileStr" >> "$lastPktsFile"
 	done
-	if [ "$str" == "" ]; then
-		# only send data if there is something to send
-		activeMeasures="${activeMeasures/wd /}"
-		activeMeasures="${activeMeasures/ wd/}"
-		activeMeasures="${activeMeasures/wd/}"
-	else
-		rawData="${rawData}|wifiDevsStats ${str}"
-	fi
+	# only send data if there is something to send
+	[ -z $str ] && wifiDevices=0 || rawData="${rawData}|wifiDevsStats ${str}"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 
@@ -396,23 +235,21 @@ fileSize() {
 # prints the sum of the sizes of all files inside given directory path.
 sumFileSizesInPath() {
 	# boolean that marks that at least one file exists inside given directory.
-	local anyFile=false
+	local anyFile=0
 	# for each file in that directory.
 	for i in "$1"/*; do
 		# if that pattern expansion exists as a file.
 		[ -f "$i" ] || continue
 		# set boolean to true.
-		anyFile=true
+		anyFile=1
 		# as we have at least one file, we don't need to loop through all files.
 		break
 	done
-	if [ "$anyFile" = false ]; then
-		# if no files.
-		# prints zero. size of nothing is 0.
-		echo 0
-		# result was given, we can leave function.
-		return 0
-	fi
+	# if no files.
+	# prints zero. size of nothing is 0.
+	# result was given, we can leave function.
+	[ "$anyFile" -eq 0 ] && echo 0 && return 0
+
 	# if there is at least one file.
 
 	# prints a list of sizes and files.
@@ -442,46 +279,19 @@ collectData() {
 	# global variable where current raw data is stored before being written to file.
 	rawData=""
 
-	# global variable that controls which measures are active
-	activeMeasures=""
-
-	local firstMeasurement=1
-
-	if [ "$burstLoss" -eq 1 ]; then
-		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
-			activeMeasures="$activeMeasures "
-		else
-			firstMeasurement=0
-		fi
-		activeMeasures="${activeMeasures}bl"
-	fi
-
-	if [ "$pingAndWan" -eq 1 ]; then
-		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
-			activeMeasures="$activeMeasures "
-		else
-			firstMeasurement=0
-		fi
-		activeMeasures="${activeMeasures}p&w"
-	fi
-
-	if [ "$wifiDevices" -eq 1 ]; then
-		if [ "$firstMeasurement" -ne 1 ]; then
-			# add space before active measurement name
-			activeMeasures="$activeMeasures "
-		else
-			firstMeasurement=0
-		fi
-		activeMeasures="${activeMeasures}wd"
-	fi
-
 	# collecting all measures.
 	collect_burst
 	collect_wan
 	collect_wifi_devices
 
+	# global variable that controls which measures are active
+	activeMeasures=""
+
+	[ "$burstLoss" -eq 1 ] && activeMeasures="${activeMeasures}bl "
+    [ "$wifiDevices" -eq 1 ] && activeMeasures="${activeMeasures}wd "
+    [ "$pingAndWan" -eq 1 ] && activeMeasures="${activeMeasures}p&w " 
+    [ ${#activeMeasures} -gt 0 ] && activeMeasures=${activeMeasures%* }
+
 	# mapping from measurement names to collected artifacts:
 	# bl (burstLoss) -> burstPing, wanBytes
 	# p&w (pingAndWan) -> burstPing, wanBytes, wanPkts

From 4706cf8fde322f2701f70077cd385bfa5cadd32e Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 30 Sep 2022 17:46:07 -0300
Subject: [PATCH 69/77] adding back individual ping packets data

---
 files/usr/share/data_collecting.sh | 74 +++++++++++++++---------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 4e495bb1..2d4a9f82 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -96,43 +96,43 @@ collect_burst() {
 		string="$string $latencyAvg $latencyStd"
 	fi
 
-	# # if latency collecting is enabled.
-	# if [ "$hasLatency" -eq 1 ]; then
-	# 	# echo collecting latencies
-	# 	# removing the first line and the last 4 lines. only the ping lines remain.
-	# 	local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
-	# 	local pairs=""
-	# 	local firstLine=true
-	# 	# for each ping line.
-	# 	while read line; do
-	# 		# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
-	# 		reached=${line%time=*}
-	# 		# if "time=" has actually been removed, it means that line 
-	# 		# contains it, which also means the icmp request was fulfilled.
-	# 		# if line doesn't contain 'time=', we skip this line.
-	# 		[ ${#reached} -lt ${#line} ] || continue
-
-	# 		 # from the whole line, removes everything until, and including, "icmp_req=".
-	# 		pingNumber=${line#*icmp_*eq=}
-	# 		# removes everything after the first space.
-	# 		pingNumber=${pingNumber%% *}
-	# 		# from the whole line, removes everything until, and including, "time=".
-	# 		pingTime=${line#*time=}
-	# 		# removes everything after the first space.
-	# 		pingTime=${pingTime%% *}
-	# 		if [ "$firstLine" = true ]; then
-	# 			firstLine=false
-	# 		else
-	# 			pairs="${pairs},"
-	# 		fi
-	# 		# concatenate to $string.
-	# 		pairs="${pairs}${pingNumber}=${pingTime}"
-	# 	done
-	# 	# prints final $string in this sub shell back to $string.
-	# 	echo $pairs))
-	# 	# appending latencies to string to be sent.
-	# 	[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
-	# fi
+	# if latency collecting is enabled.
+	if [ "$hasLatency" -eq 1 ]; then
+		# echo collecting latencies
+		# removing the first line and the last 4 lines. only the ping lines remain.
+		local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
+		local pairs=""
+		local firstLine=true
+		# for each ping line.
+		while read line; do
+			# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
+			reached=${line%time=*}
+			# if "time=" has actually been removed, it means that line 
+			# contains it, which also means the icmp request was fulfilled.
+			# if line doesn't contain 'time=', we skip this line.
+			[ ${#reached} -lt ${#line} ] || continue
+
+			 # from the whole line, removes everything until, and including, "icmp_req=".
+			pingNumber=${line#*icmp_*eq=}
+			# removes everything after the first space.
+			pingNumber=${pingNumber%% *}
+			# from the whole line, removes everything until, and including, "time=".
+			pingTime=${line#*time=}
+			# removes everything after the first space.
+			pingTime=${pingTime%% *}
+			if [ "$firstLine" = true ]; then
+				firstLine=false
+			else
+				pairs="${pairs},"
+			fi
+			# concatenate to $string.
+			pairs="${pairs}${pingNumber}=${pingTime}"
+		done
+		# prints final $string in this sub shell back to $string.
+		echo $pairs))
+		# appending latencies to string to be sent.
+		[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
+	fi
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"

From 300fd3bbc6629b58dbcbe205cdd70c5e606c0319 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Fri, 30 Sep 2022 18:40:09 -0300
Subject: [PATCH 70/77] removing hasLatency sections of code

---
 files/usr/share/data_collecting.sh | 84 ++++++++++++++++--------------
 1 file changed, 46 insertions(+), 38 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index d168e92d..bc86f501 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -96,43 +96,43 @@ collect_burst() {
 		string="$string $latencyAvg $latencyStd"
 	fi
 
-	# if latency collecting is enabled.
-	if [ "$hasLatency" -eq 1 ]; then
-		# echo collecting latencies
-		# removing the first line and the last 4 lines. only the ping lines remain.
-		local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
-		local pairs=""
-		local firstLine=true
-		# for each ping line.
-		while read line; do
-			# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
-			reached=${line%time=*}
-			# if "time=" has actually been removed, it means that line 
-			# contains it, which also means the icmp request was fulfilled.
-			# if line doesn't contain 'time=', we skip this line.
-			[ ${#reached} -lt ${#line} ] || continue
-
-			 # from the whole line, removes everything until, and including, "icmp_req=".
-			pingNumber=${line#*icmp_*eq=}
-			# removes everything after the first space.
-			pingNumber=${pingNumber%% *}
-			# from the whole line, removes everything until, and including, "time=".
-			pingTime=${line#*time=}
-			# removes everything after the first space.
-			pingTime=${pingTime%% *}
-			if [ "$firstLine" = true ]; then
-				firstLine=false
-			else
-				pairs="${pairs},"
-			fi
-			# concatenate to $string.
-			pairs="${pairs}${pingNumber}=${pingTime}"
-		done
-		# prints final $string in this sub shell back to $string.
-		echo $pairs))
-		# appending latencies to string to be sent.
-		[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
-	fi
+	# # if latency collecting is enabled.
+	# if [ "$hasLatency" -eq 1 ]; then
+	# 	# echo collecting latencies
+	# 	# removing the first line and the last 4 lines. only the ping lines remain.
+	# 	local latencies=$(printf "%s" "$pingResult" | head -n -4 | sed '1d' | (
+	# 	local pairs=""
+	# 	local firstLine=true
+	# 	# for each ping line.
+	# 	while read line; do
+	# 		# removes 'time=' part if it exists. if it doesn't, '$reached' will be as long as '$line'.
+	# 		reached=${line%time=*}
+	# 		# if "time=" has actually been removed, it means that line 
+	# 		# contains it, which also means the icmp request was fulfilled.
+	# 		# if line doesn't contain 'time=', we skip this line.
+	# 		[ ${#reached} -lt ${#line} ] || continue
+
+	# 		 # from the whole line, removes everything until, and including, "icmp_req=".
+	# 		pingNumber=${line#*icmp_*eq=}
+	# 		# removes everything after the first space.
+	# 		pingNumber=${pingNumber%% *}
+	# 		# from the whole line, removes everything until, and including, "time=".
+	# 		pingTime=${line#*time=}
+	# 		# removes everything after the first space.
+	# 		pingTime=${pingTime%% *}
+	# 		if [ "$firstLine" = true ]; then
+	# 			firstLine=false
+	# 		else
+	# 			pairs="${pairs},"
+	# 		fi
+	# 		# concatenate to $string.
+	# 		pairs="${pairs}${pingNumber}=${pingTime}"
+	# 	done
+	# 	# prints final $string in this sub shell back to $string.
+	# 	echo $pairs))
+	# 	# appending latencies to string to be sent.
+	# 	[ ${#latencies} -gt 0 ] && string="${string} ${latencies}"
+	# fi
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
@@ -284,6 +284,14 @@ collectData() {
 	collect_wan
 	collect_wifi_devices
 
+	# global variable that controls which measures are active
+    activeMeasures=""
+
+    [ "$burstLoss" -eq 1 ] && activeMeasures="${activeMeasures}bl "
+    [ "$wifiDevices" -eq 1 ] && activeMeasures="${activeMeasures}wd "
+    [ "$pingAndWan" -eq 1 ] && activeMeasures="${activeMeasures}p&w " 
+    [ ${#activeMeasures} -gt 0 ] && activeMeasures=${activeMeasures%* }
+
 	# mapping from measurement names to collected artifacts:
 	# bl (burstLoss) -> burstPing, wanBytes
 	# p&w (pingAndWan) -> burstPing, wanBytes, wanPkts
@@ -514,7 +522,7 @@ loop() {
 		# getting parameters every time we need to send data, this way we don't have to 
 		# restart the service if a parameter changes.
 		eval $(cat /root/flashbox_config.json | jsonfilter \
-			-e "hasLatency=@.data_collecting_has_latency" \
+			# -e "hasLatency=@.data_collecting_has_latency" \
 			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
 			-e "pingServerAddress=@.data_collecting_ping_fqdn" \
 			-e "pingPackets=@.data_collecting_ping_packets" \

From 2abe6345672582f54c68fea58df5af55feafc26d Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Sun, 2 Oct 2022 10:32:56 -0300
Subject: [PATCH 71/77] syntax shortening

---
 files/usr/share/data_collecting.sh | 76 ++++++++++++++----------------
 1 file changed, 35 insertions(+), 41 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index bc86f501..45eab510 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -8,10 +8,6 @@
 dataCollectingDir="/tmp/data_collecting"
 # file where collected data will be stored before being compressed.
 rawDataFile="${dataCollectingDir}/raw"
-# file where wan rx/tx bytes will be stored
-wanBytesFile="${dataCollectingDir}/wan_bytes"
-# file where wan rx/tx packets will be stored
-wanPacketsFile="${dataCollectingDir}/wan_pkts"
 
 # gets current rx and tx bytes/packets from wan interface and compares them
 # with values from previous calls to calculate cross traffic
@@ -21,24 +17,22 @@ collect_wan() {
 	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
 	# bytes received by the interface.
-	local rxBytes=$(get_wan_bytes_statistics RX)
+	local rx=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
-	local txBytes=$(get_wan_bytes_statistics TX)
+	local tx=$(get_wan_bytes_statistics TX)
 
-	local string="|wanBytes $rxBytes $txBytes"
-	rawData="${rawData}${string}"
+	rawData="${rawData}|wanBytes $rx $tx"
 
 	# burstLoss only gathers byte data
 	[ $pingAndWan -eq 0 ] && return
 
 	# packets received by the interface.
-	local rxPackets=$(get_wan_packets_statistics RX)
+	rx=$(get_wan_packets_statistics RX)
 	# packets sent by the interface.
-	local txPackets=$(get_wan_packets_statistics TX)
+	tx=$(get_wan_packets_statistics TX)
 
 	# data to be sent.
-	local string="|wanPkts $rxPackets $txPackets"
-	rawData="${rawData}${string}"
+	rawData="${rawData}|wanPkts $pr $pt"
 }
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
@@ -144,82 +138,82 @@ collect_wifi_devices() {
 	[ "$wifiDevices" -eq 0 ] && return
 
 	# devices and their data will be stored in this string variable.
-	local str=""
+	local s=""
 
-	local firstRawWrite=1
+	local first=1
 
 	# 0 and 1 are the indexes for wifi interfaces: wlan0 and wlan1, or phy0 and phy1.
 	for i in 0 1; do
 		# getting wifi interface name.
 		# 'get_root_ifname()' is defined in /usr/share/functions/custom_wireless_driver.sh.
-		local wlan=$(get_root_ifname "$i" 2> /dev/null)
+		local w=$(get_root_ifname "$i" 2> /dev/null)
 		# if interface doesn't exist, skips this iteration.
-		[ -z $wlan ] && continue
+		[ -z "$w" ] && continue
 		# getting info from each connected device on wifi. 
 		# grep returns empty when no devices are connected or if interface doesn't exist.
-		local devices="$(iwinfo "$wlan" assoclist | grep ago)"
-		local devices_rx_pkts="$(iwinfo "$wlan" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
-		local devices_tx_pkts="$(iwinfo "$wlan" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
+		local iw="$(iwinfo "$w" assoclist | grep ago)"
+		local pr="$(iwinfo "$w" assoclist | grep RX | grep -o '[0-9]\+ Pkts')"
+		local pt="$(iwinfo "$w" assoclist | grep TX | grep -o '[0-9]\+ Pkts')"
 
-		while [ ${#devices} -gt 0 ]; do
+		while [ ${#iw} -gt 0 ]; do
 
 			# getting everything before the first space.
-			local deviceMac=${devices%% *}
+			local mac=${iw%% *}
 
 			# getting everything after the first two spaces.
-			local signal=${devices#*  }
+			local signal=${iw#*  }
 
 			# getting everything before the first occasion of ' /'
 			signal=${signal%% /*}
 
 			# if unknown discard
-			[ "$signal" == "unknown" ] && devices=${devices#*$'\n'} && continue
+			[ "$signal" == "unknown" ] && iw=${iw#*$'\n'} && continue
 
 			# getting everything before the first occasion of ' dBm'
 			signal=${signal%% dBm*}
 
 			# getting after '(SNR '. 
-			devices=${devices#*\(SNR }
+			iw=${iw#*\(SNR }
 
 			# getting everything before the first closing parenthesis.
-			local snr=${devices%%\)*}
+			local snr=${iw%%\)*}
 
 			# if SNR equals signal we assume noise of -95dBm
 			[ $signal -eq $snr ] && snr=$(($signal+95))
 
 			# getting everything before ' ms'.
-			local time=${devices%% ms*}
+			local ts=${iw%% ms*}
 
 			# getting everything after the first space.
-			time=${time##* }
+			ts=${ts##* }
 
 			# getting everything after 'ago'.
-			devices=${devices#*ago}
+			iw=${iw#*ago}
 
 			# getting everything after '\n', if it exists. last line won't have it, so nothing will be changed.
 			# we can't add the line feed along witht the previous parameter expansion because we wouldn't match
-			# the last line and so we wouldn't make $devices length become zero.
-			devices=${devices#*$'\n'}
+			# the last line and so we wouldn't make $iw length become zero.
+			iw=${iw#*$'\n'}
 
-			# if $time is greater than one minute, we don't use this device's info.
-			[ "$time" -gt 60000 ] && continue
+			# if $ts is greater than one minute, we don't use this device's info.
+			[ "$ts" -gt 60000 ] && continue
 			
-			local rx_pkts=${devices_rx_pkts%% *}
-			devices_rx_pkts=${devices_rx_pkts#*$'\n'}
+			local rx=${pr%% *}
+			pr=${pr#*$'\n'}
 
-			local tx_pkts=${devices_tx_pkts%% *}
-			devices_tx_pkts=${devices_tx_pkts#*$'\n'}
+			local tx=${pt%% *}
+			pt=${pt#*$'\n'}
 
-			[ -z $rx_pkts ] && continue
-			[ -z $tx_pkts ] && continue
+			[ -z "$rx" ] && continue
+			[ -z "$tx" ] && continue
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
-			[ "$firstRawWrite" -eq 1 ] && firstRawWrite=0 || str="$str "
-			str="${str}${i}_${deviceMac}_${signal}_${snr}_${rx_pkts}_${tx_pkts}"
+			[ "$first" -eq 1 ] && first=0 || s="$s "
+			s="${s}${i}_${mac}_${signal}_${snr}_${rx}_${tx}"
 		done
 	done
 	# only send data if there is something to send
-	[ -z $str ] && wifiDevices=0 || rawData="${rawData}|wifiDevsStats ${str}"
+	[ -z "$s" ] && wifiDevices=0 || rawData="${rawData}|wifiDevsStats ${s}"
 }
 
 # prints the size of a file, using 'ls', where full file path is given as 

From 2cd4fa1bdaf521b34936a73e25c50dd441c230bf Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Tue, 4 Oct 2022 20:15:42 -0300
Subject: [PATCH 72/77] shortening syntax of not yet added sections of code

---
 files/usr/share/data_collecting.sh | 76 +++++++++++++++---------------
 1 file changed, 38 insertions(+), 38 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 45eab510..dd36ba41 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -17,22 +17,22 @@ collect_wan() {
 	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
 
 	# bytes received by the interface.
-	local rx=$(get_wan_bytes_statistics RX)
+	local r=$(get_wan_bytes_statistics RX)
 	# bytes sent by the interface.
-	local tx=$(get_wan_bytes_statistics TX)
+	local t=$(get_wan_bytes_statistics TX)
 
-	rawData="${rawData}|wanBytes $rx $tx"
+	rawData="${rawData}|wanBytes $r $t"
 
 	# burstLoss only gathers byte data
 	[ $pingAndWan -eq 0 ] && return
 
 	# packets received by the interface.
-	rx=$(get_wan_packets_statistics RX)
+	r=$(get_wan_packets_statistics RX)
 	# packets sent by the interface.
-	tx=$(get_wan_packets_statistics TX)
+	t=$(get_wan_packets_statistics TX)
 
 	# data to be sent.
-	rawData="${rawData}|wanPkts $pr $pt"
+	rawData="${rawData}|wanPkts $r $t"
 }
 
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
@@ -54,11 +54,11 @@ collect_burst() {
 	# An skipped measure will become missing data, for this minute, in the server.
 
 	# removes everything behind the summary that appears in the last lines.
-	local pingResultAux=${pingResult##* ping statistics ---[$'\r\n']}
+	pingResult=${pingResult##* ping statistics ---[$'\r\n']}
 	# removes everything after, and including, ' packets transmitted'.
-	local transmitted=${pingResultAux% packets transmitted*}
+	local transmitted=${pingResult% packets transmitted*}
 	# removes everything after, and including, ' received'.
-	local received=${pingResultAux% received*}
+	local received=${pingResult% received*}
 	# removes everything before first space.
 	received=${received##* }
 	# integer representing the amount of packets not received.
@@ -67,27 +67,27 @@ collect_burst() {
 	# loss=${loss##* } # removes everything before first space.
 
 	# data to be sent.
-	local string="$loss $transmitted"
+	local s="$loss $transmitted"
 
-	# removes everything before and including 'mdev = '
-	local latencyStats=${pingResult#*/mdev = }
+	# latency stats
+	local lat=${pingResult#*/mdev = }
 
 	# when there is 100% packet loss there the strings remain equal
 	# we only want to collect latency and std when there isn't 100% loss
 	# if loss is 100% we just send 0 in both cases, which will be ignored by the server
-	if [ ${#latencyStats} == ${#pingResult} ]; then
-	    string="$string 0 0"
+	if [ ${#lat} == ${#pingResult} ]; then
+	    s="$s 0 0"
 	else
 		# removes everything before first backslash
-		local latencyAvg=${latencyStats#*/}
+		local avg=${lat#*/}
 		# removes everything after first backslash
-		latencyAvg=${latencyAvg%%/*}
+		avg=${avg%%/*}
 		# removes everything before and including last backslash
-		local latencyStd=${latencyStats##*/}
+		local std=${lat##*/}
 		# removes everything after and including first space
-		latencyStd=${latencyStd%% *}
+		std=${std%% *}
 
-		string="$string $latencyAvg $latencyStd"
+		s="$s $avg $std"
 	fi
 
 	# # if latency collecting is enabled.
@@ -130,7 +130,7 @@ collect_burst() {
 	
 	# appending string to file.
 	# printf "string is: '%s'\n" "$string"
-	rawData="${rawData}|burstPing $string"
+	rawData="${rawData}|burstPing $s"
 }
 
 collect_wifi_devices() {
@@ -161,16 +161,16 @@ collect_wifi_devices() {
 			local mac=${iw%% *}
 
 			# getting everything after the first two spaces.
-			local signal=${iw#*  }
+			local sig=${iw#*  }
 
 			# getting everything before the first occasion of ' /'
-			signal=${signal%% /*}
+			sig=${sig%% /*}
 
 			# if unknown discard
-			[ "$signal" == "unknown" ] && iw=${iw#*$'\n'} && continue
+			[ "$sig" == "unknown" ] && iw=${iw#*$'\n'} && continue
 
 			# getting everything before the first occasion of ' dBm'
-			signal=${signal%% dBm*}
+			sig=${sig%% dBm*}
 
 			# getting after '(SNR '. 
 			iw=${iw#*\(SNR }
@@ -178,8 +178,8 @@ collect_wifi_devices() {
 			# getting everything before the first closing parenthesis.
 			local snr=${iw%%\)*}
 
-			# if SNR equals signal we assume noise of -95dBm
-			[ $signal -eq $snr ] && snr=$(($signal+95))
+			# if SNR equals sig we assume noise of -95dBm
+			[ $sig -eq $snr ] && snr=$(($sig+95))
 
 			# getting everything before ' ms'.
 			local ts=${iw%% ms*}
@@ -198,18 +198,18 @@ collect_wifi_devices() {
 			# if $ts is greater than one minute, we don't use this device's info.
 			[ "$ts" -gt 60000 ] && continue
 			
-			local rx=${pr%% *}
+			local r=${pr%% *}
 			pr=${pr#*$'\n'}
 
-			local tx=${pt%% *}
+			local t=${pt%% *}
 			pt=${pt#*$'\n'}
 
-			[ -z "$rx" ] && continue
-			[ -z "$tx" ] && continue
+			[ -z "$r" ] && continue
+			[ -z "$t" ] && continue
 
 			# if it's the first data we are storing, don't add a space before appending the data string.
 			[ "$first" -eq 1 ] && first=0 || s="$s "
-			s="${s}${i}_${mac}_${signal}_${snr}_${rx}_${tx}"
+			s="${s}${i}_${mac}_${sig}_${snr}_${r}_${t}"
 		done
 	done
 	# only send data if there is something to send
@@ -268,7 +268,7 @@ sumFileSizesInPath() {
 # directory of compressed files grows too big delete oldest compressed files.
 collectData() {
 	# getting current unix time in seconds.
-	local timestamp=$(date +%s)
+	local ts=$(date +%s)
 
 	# global variable where current raw data is stored before being written to file.
 	rawData=""
@@ -279,12 +279,12 @@ collectData() {
 	collect_wifi_devices
 
 	# global variable that controls which measures are active
-    activeMeasures=""
+    on=""
 
-    [ "$burstLoss" -eq 1 ] && activeMeasures="${activeMeasures}bl "
-    [ "$wifiDevices" -eq 1 ] && activeMeasures="${activeMeasures}wd "
-    [ "$pingAndWan" -eq 1 ] && activeMeasures="${activeMeasures}p&w " 
-    [ ${#activeMeasures} -gt 0 ] && activeMeasures=${activeMeasures%* }
+    [ "$burstLoss" -eq 1 ] && on="${on}bl "
+    [ "$wifiDevices" -eq 1 ] && on="${on}wd "
+    [ "$pingAndWan" -eq 1 ] && on="${on}p&w " 
+    [ ${#on} -gt 0 ] && on=${on%* }
 
 	# mapping from measurement names to collected artifacts:
 	# bl (burstLoss) -> burstPing, wanBytes
@@ -293,7 +293,7 @@ collectData() {
 
 	# example of an expected raw data with all measures present:
 	# 'bl p&w wd|213234556456|burstPing 0 100 1.246 0.161|wanBytes 12345 1234|wanPkts 1234 123|wifiDevsStats 0_D0:9C:7A:EC:FF:FF_33_285_5136'
-	[ -n "$rawData" ] && [ ${#activeMeasures} -gt 0 ] && echo "${activeMeasures}|${timestamp}${rawData}" >> "$rawDataFile";
+	[ -n "$rawData" ] && [ ${#on} -gt 0 ] && echo "${on}|${ts}${rawData}" >> "$rawDataFile";
 	# cleaning 'rawData' value from memory.
 	rawData=""
 }

From c36a053b49c6c124aeaa26969d4465b6d27885bc Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Wed, 5 Oct 2022 20:44:52 -0300
Subject: [PATCH 73/77] removing unnused function

---
 files/usr/share/data_collecting.sh | 37 ------------------------------
 1 file changed, 37 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index dd36ba41..176e3263 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -226,43 +226,6 @@ fileSize() {
 	echo $size
 }
 
-# prints the sum of the sizes of all files inside given directory path.
-sumFileSizesInPath() {
-	# boolean that marks that at least one file exists inside given directory.
-	local anyFile=0
-	# for each file in that directory.
-	for i in "$1"/*; do
-		# if that pattern expansion exists as a file.
-		[ -f "$i" ] || continue
-		# set boolean to true.
-		anyFile=1
-		# as we have at least one file, we don't need to loop through all files.
-		break
-	done
-	# if no files.
-	# prints zero. size of nothing is 0.
-	# result was given, we can leave function.
-	[ "$anyFile" -eq 0 ] && echo 0 && return 0
-
-	# if there is at least one file.
-
-	# prints a list of sizes and files.
-	local wcResult=$(wc -c "$1"/*)
-	# if there is 2 or more files, the last line will have a "total". remove that string.
-	local hasTotal=${wcResult% total}
-	# if it has a total, it was removed. if not, nothing was removed and both strings are the same.
-
-	 # if length of string with "total" removed is smaller than original string.
-	if [ ${#hasTotal} -lt ${#wcResult} ]; then
-		# remove everything before the last word, which is the value for total, and print what remains.
-		echo ${hasTotal##* }
-	else
-		# if there were no total, then there was only one file, in one line of output, and the first column is the size value.
-		# remove everything past, and including, the first space, and print what remains.
-		echo ${wcResult%% *}
-	fi
-}
-
 # collect every data and stores in '$rawDataFile'. if the size of the file is 
 # too big, compress it and move it to a directory of compressed files. If 
 # directory of compressed files grows too big delete oldest compressed files.

From 4e2893da4b063e2c0875db5454838d3873d681f0 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Wed, 5 Oct 2022 20:47:05 -0300
Subject: [PATCH 74/77] changing new function names

---
 files/usr/share/data_collecting.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 176e3263..52a4f305 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -11,7 +11,7 @@ rawDataFile="${dataCollectingDir}/raw"
 
 # gets current rx and tx bytes/packets from wan interface and compares them
 # with values from previous calls to calculate cross traffic
-collect_wan() {
+wan() {
 	# checking if this data collecting is enabled
 
 	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
@@ -38,7 +38,7 @@ collect_wan() {
 # takes current unix timestamp, executes ping, in burst, to $pingServerAddress server.
 # If latency collecting is enabled, extracts the individual icmp request numbers and 
 # their respective ping times. Builds a string with all this information and write them to file.
-collect_burst() {
+burst() {
 	# checking if this data collecting is enabled
 
 	[ $burstLoss -eq 0 ] && [ $pingAndWan -eq 0 ] && return
@@ -133,7 +133,7 @@ collect_burst() {
 	rawData="${rawData}|burstPing $s"
 }
 
-collect_wifi_devices() {
+wifi() {
 	# checking if this data collecting is enabled
 	[ "$wifiDevices" -eq 0 ] && return
 
@@ -237,9 +237,9 @@ collectData() {
 	rawData=""
 
 	# collecting all measures.
-	collect_burst
-	collect_wan
-	collect_wifi_devices
+	burst
+	wan
+	wifi
 
 	# global variable that controls which measures are active
     on=""

From 2c75ccf58c7826d74c6270d91ae529b050898c68 Mon Sep 17 00:00:00 2001
From: danielatk <danatkhoo@yahoo.com.br>
Date: Wed, 5 Oct 2022 20:48:02 -0300
Subject: [PATCH 75/77] removing other unnused function

---
 files/usr/share/data_collecting.sh | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 52a4f305..16a7e5ba 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -216,16 +216,6 @@ wifi() {
 	[ -z "$s" ] && wifiDevices=0 || rawData="${rawData}|wifiDevsStats ${s}"
 }
 
-# prints the size of a file, using 'ls', where full file path is given as 
-# first argument ($1).
-fileSize() {
-	# file size is the information at the 1st column.
-	local wcline=$(wc -c "$1")
-	#remove suffix composed of space and anything else.
-	local size=${wcline% *}
-	echo $size
-}
-
 # collect every data and stores in '$rawDataFile'. if the size of the file is 
 # too big, compress it and move it to a directory of compressed files. If 
 # directory of compressed files grows too big delete oldest compressed files.

From 321f3be3a0bf55578a19bb750531301f5a540be8 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Thu, 6 Oct 2022 10:57:08 -0300
Subject: [PATCH 76/77] final syntax corrections

---
 files/usr/share/data_collecting.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 16a7e5ba..164b8c1c 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -75,7 +75,7 @@ burst() {
 	# when there is 100% packet loss there the strings remain equal
 	# we only want to collect latency and std when there isn't 100% loss
 	# if loss is 100% we just send 0 in both cases, which will be ignored by the server
-	if [ ${#lat} == ${#pingResult} ]; then
+	if [ ${#lat} = ${#pingResult} ]; then
 	    s="$s 0 0"
 	else
 		# removes everything before first backslash
@@ -167,7 +167,7 @@ wifi() {
 			sig=${sig%% /*}
 
 			# if unknown discard
-			[ "$sig" == "unknown" ] && iw=${iw#*$'\n'} && continue
+			[ "$sig" = "unknown" ] && iw=${iw#*$'\n'} && continue
 
 			# getting everything before the first occasion of ' dBm'
 			sig=${sig%% dBm*}
@@ -469,7 +469,7 @@ loop() {
 		# getting parameters every time we need to send data, this way we don't have to 
 		# restart the service if a parameter changes.
 		eval $(cat /root/flashbox_config.json | jsonfilter \
-			# -e "hasLatency=@.data_collecting_has_latency" \
+			-e "hasLatency=@.data_collecting_has_latency" \
 			-e "alarmServerAddress=@.data_collecting_alarm_fqdn" \
 			-e "pingServerAddress=@.data_collecting_ping_fqdn" \
 			-e "pingPackets=@.data_collecting_ping_packets" \

From 373926a0e11bdb77cac0a84c6e6f3317bed41809 Mon Sep 17 00:00:00 2001
From: Daniel <danatkhoo@yahoo.com.br>
Date: Sun, 9 Oct 2022 17:30:29 -0300
Subject: [PATCH 77/77] correcting unkown signal bug

---
 files/usr/share/data_collecting.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/usr/share/data_collecting.sh b/files/usr/share/data_collecting.sh
index 164b8c1c..306a0fe4 100755
--- a/files/usr/share/data_collecting.sh
+++ b/files/usr/share/data_collecting.sh
@@ -167,7 +167,7 @@ wifi() {
 			sig=${sig%% /*}
 
 			# if unknown discard
-			[ "$sig" = "unknown" ] && iw=${iw#*$'\n'} && continue
+			[ "$sig" = "unknown" ] && iw=${iw#*ago} && iw=${iw#*$'\n'} && continue
 
 			# getting everything before the first occasion of ' dBm'
 			sig=${sig%% dBm*}