From 62a30b02600fbba046caa4ae1d7f339dd05c4a1b Mon Sep 17 00:00:00 2001
From: poka <poka@p2p.legal>
Date: Tue, 22 Sep 2020 02:36:20 +0200
Subject: [PATCH] Improve grep for large matchs fir -F option; Fix UDToday when
 bloc is null

---
 .gitignore           |  1 +
 g1-stats.sh          | 22 ++++++++++++-------
 lib/functions.sh     | 50 +++++++++++++++++++++++---------------------
 lib/scanTxWallets.sh | 36 ++++++++++++++++---------------
 4 files changed, 60 insertions(+), 49 deletions(-)

diff --git a/.gitignore b/.gitignore
index c3e070e..d573607 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ db/*
 cache
 debug.log
 bk-debug.log
+bk
diff --git a/g1-stats.sh b/g1-stats.sh
index e6287f0..b776d6d 100755
--- a/g1-stats.sh
+++ b/g1-stats.sh
@@ -7,6 +7,7 @@
 # Git: https://git.p2p.legal/axiom-team/g1-stats
 ################################################################################
 
+[[ -z $debug ]] && debug=false
 # Exit script if error
 set -e
 
@@ -18,21 +19,25 @@ if [[ -e $SCRIPTPATH/.env ]]; then source $SCRIPTPATH/.env; else echo "Veuillez
 startTime=$(date +'%H:%M')
 day=$(date +'%y-%m-%d')
 dayP=$(date +'%d-%m-%y')
-indexhtml="$WEBPATH/index.html"
 echo -e "\n############# $day à $startTime #############\n"
 [[ -z $(which jq) || -z $(which bc) || -z $(which curl) ]] && sudo apt update && sudo apt install jq bc curl
 echo "Initialisation ..."
-#TXBLOCKS=$(cat $SCRIPTPATH/db/txblocs)  #kopa
-TXBLOCKS=$(curl -s ${DUNITER}/blockchain/with/tx | jq '.result.blocks[]')
+if $debug; then
+	TXBLOCKS=$(cat $SCRIPTPATH/db/txblocs)
+	membresPubkeys=$(cat db/membresPubkeys)
+	udBlocs=$(cat db/udblocs)
+else
+	TXBLOCKS=$(curl -s ${DUNITER}/blockchain/with/tx | jq '.result.blocks[]')
+	membresPubkeys=$(curl -s ${DUNITER}/wot/members | jq -r '.results[].pubkey')
+	udBlocs=$(curl -s ${DUNITER}/blockchain/with/ud | jq '.result.blocks[]')
+fi
+
 TXBLOCKS=($(echo "$TXBLOCKS" | sort -hu | awk '{printf $1" "}'))
 unset 'TXBLOCKS[${#TXBLOCKS[@]}-1]'
 echo "TXBLOCKS OK"
-membresPubkeys=$(curl -s ${DUNITER}/wot/members | jq -r '.results[].pubkey')
-#membresPubkeys=$(cat db/membresPubkeys) #kopa
 echo "membresPubkeys OK"
 nbrMembers=$(echo "$membresPubkeys" | wc -l)
 lastBloc=${TXBLOCKS[-1]}
-udBlocs=$(curl -s ${DUNITER}/blockchain/with/ud | jq '.result.blocks[]')
 echo "udBlocs OK"
 isWeb=$1
 jsonFile="$SCRIPTPATH/db/daily.json"
@@ -61,5 +66,6 @@ displayStats
 
 ### Fin de programme
 day=$(date +'%d-%m-%y')
-echo -e " ---"
-echo "$day - Heure de fin: $(date +'%H:%M')"
+echo "
+ ---
+$day - Heure de fin: $(date +'%H:%M')"
diff --git a/lib/functions.sh b/lib/functions.sh
index 3c6dfb5..fc6653d 100755
--- a/lib/functions.sh
+++ b/lib/functions.sh
@@ -17,18 +17,18 @@ addMembers() {
 	simpleWallets="$wallets"
 	for i in $(ls x*); do
 		local walletToSup=$(cat $i)
-		simpleWallets=$(echo "$simpleWallets" | grep -v "$walletToSup")
+		simpleWallets=$(echo "$simpleWallets" | grep -vF "$walletToSup")
 	done
 	[[ -d $tmpSplit ]] && rm -r $tmpSplit || exit 1
 	popd >/dev/null
 }
 
 web() {
-
 	[ ! -d $WEBPATH/history/ ] && mkdir -p $WEBPATH/history/
 	[ ! -d $WEBPATH/graph/ ] && mkdir -p $WEBPATH/graph/
 	[ ! -d $WEBPATH/css ] && cp -r $SCRIPTPATH/tpl/css $WEBPATH/
 	[ ! -d $WEBPATH/js ] && cp -r $SCRIPTPATH/tpl/js $WEBPATH/
+	indexhtml="$WEBPATH/index.html"
 	cp $SCRIPTPATH/tpl/index.html $indexhtml
 
 	datePrevious=$(date +'%y-%m-%d' -d "$day -1 day")
@@ -52,7 +52,7 @@ web() {
 	sed -i "s/_soldeM/$soldeM/g" $indexhtml
 	sed -i "s/_pourcentSimpleWallet/$pourcentSimpleWallet/g" $indexhtml
 	sed -i "s/_nonConsumedUDT/$nonConsumedUDT/g" $indexhtml
-    sed -i "s/_monetaryMass/$monetaryMass/g" $indexhtml
+	sed -i "s/_monetaryMass/$monetaryMass/g" $indexhtml
 	sed -i "s/_sleepyG1/$sleepyG1/g" $indexhtml
 	[[ -z $(grep '"display:none;" class="previous"' $indexhtml) ]] && sed -i "s/_datePrevious/$datePrevious/g" $indexhtml && setPrevious="Oui"
 	[[ -z $(grep '"display:none;" class="next"' $indexhtml) ]] && sed -i "s/_dateNext/$dateNext/g" $indexhtml && setNext="Oui"
@@ -116,25 +116,27 @@ nbrWallets() {
 }
 
 displayStats() {
-	echo -e "\n ---\n"
-	echo "Noeud: $DUNITER"
-	echo "Nombre total de wallet: $nbrTotalWallets"
-	echo "Nombre de membres: $nbrMembers (${pourcentMbrs}%)"
-	echo "Nombre de simple portefeuille: $nbrSimpleWallets (${pourcentWallets}%)"
-	echo -e "\n ---\n"
-	echo "Reçus Wallets: $sumRW"
-	echo "Reçus Membres: $sumRM"
-	echo "Reçus Total: $sumRT"
-	echo -e " ---"
-	echo "Envoyé Wallets: $sumSW"
-	echo "Envoyé Membres: $sumSM"
-	echo "Envoyé Total: $sumST"
-	echo -e " ---"
-	echo "Solde Wallets: $soldeW"
-	echo "Solde Membres: $soldeM"
-	echo -e " ---"
-	echo -e "Masse Monétaire:\t $monetaryMass Ḡ1"
-	echo -e "\n ---\n Analyse\n ---"
-	echo -e "$sleepyG1% des Ḡ1 n'ont jamais été utilisés."
+echo "
+ ---
+Noeud: $DUNITER
+Nombre total de wallet: $nbrTotalWallets
+Nombre de membres: $nbrMembers (${pourcentMbrs}%)
+Nombre de simple portefeuille: $nbrSimpleWallets (${pourcentWallets}%)
+ ---
+Reçus Wallets: $sumRW
+Reçus Membres: $sumRM
+Reçus Total: $sumRT
+ ---
+Envoyé Wallets: $sumSW
+Envoyé Membres: $sumSM
+Envoyé Total: $sumST
+ ---
+Solde Wallets: $soldeW
+Solde Membres: $soldeM
+ ---
+Masse Monétaire:  $monetaryMass Ḡ1
+ ---
+Analyse
+ ---
+$sleepyG1% des Ḡ1 n'ont jamais été utilisés."
 }
-
diff --git a/lib/scanTxWallets.sh b/lib/scanTxWallets.sh
index 04537d3..6190f52 100755
--- a/lib/scanTxWallets.sh
+++ b/lib/scanTxWallets.sh
@@ -2,18 +2,18 @@
 
 scanTxWallets() {
 	local REGEX_PUBKEYS="[a-zA-Z0-9]{42,44}"
-
 	# On vérifie le cache existant et on démarre au dernier bloc en cache
 	if ls $SCRIPTPATH/cache/walletsUp-* > /dev/null 2>&1; then
 		local startFile=$(ls $SCRIPTPATH/cache/walletsUp-* | tail -n1)
 		local startIter=$(echo "$startFile" | awk -F '-' '{ print $NF }')
+		((startIter++))
 		local startBloc=$(head -n1 "$startFile")
 		wallets=$(tail -n +2 "$startFile" | sed '/---/Q' | sort -u)
 		local walletsAuj=$(cat "$startFile" | sed -e '1,/---/d' | sed '/---/Q' | grep .)$'\n'
 		local nbrMembersYesterday=$(cat "$startFile" | sed -e '1,/---/d' | sed -e '1,/---/d' | sed '/---/Q' | grep . | cut -d '|' -f 1)
 		local blockDateLast=$(cat "$startFile" | sed -e '1,/---/d' | sed -e '1,/---/d' | sed '/---/Q' | grep . | cut -d '|' -f 2)
 
-##kopa		[[ $lastBloc == $startBloc ]] && return 10
+		[[ $lastBloc == $startBloc ]] && return
 		rm $startFile
 	else
 		local startIter=0
@@ -22,11 +22,10 @@ scanTxWallets() {
 	fi
 
 	incExec=1
-	for i in ${!TXBLOCKS[*]}; do
+	for ((i=$startIter; i<${#TXBLOCKS[@]}; i++)); do
 		startExec=$(($(date +%s%N)/1000000))
 		bloc=${TXBLOCKS[$i]}
 		[[ -z $bloc ]] && break
-		[[ $startIter != 0 && $i -le $startIter ]] && continue
 #		[[ $i -lt 290 ]] && continue #kopa
 		sleep 0.05
 
@@ -84,23 +83,23 @@ scanTxWallets() {
 
 		# Exporte les valeurs de la journée dans le fichier JSON
 		jsonify() {
-            [[ -z $valueReceivedW ]] && valueReceivedW=0 || valueReceivedW=$(echo "scale=2; $valueReceivedW/100" | bc)
-            [[ -z $valueReceivedM ]] && valueReceivedM=0 || valueReceivedM=$(echo "scale=2; $valueReceivedM/100" | bc)
+			[[ -z $valueReceivedW ]] && valueReceivedW=0 || valueReceivedW=$(echo "scale=2; $valueReceivedW/100" | bc)
+			[[ -z $valueReceivedM ]] && valueReceivedM=0 || valueReceivedM=$(echo "scale=2; $valueReceivedM/100" | bc)
 			[[ -z $valueSentM ]] && valueSentM=0 || valueSentM=$(echo "scale=2; $valueSentM/100" | bc)
 			[[ -z $valueSentW ]] && valueSentW=0 || valueSentW=$(echo "scale=2; $valueSentW/100" | bc)
 			[[ -z $nbrMembersToday ]] && nbrMembersToday=0
 			[[ -z $nbrWallets ]] && nbrWallets=0
 			[[ -z $UDToday ]] && UDToday=0
 
-            local jsonTPL=$(sed s/_DATE/$blockDateLast/g $SCRIPTPATH/tpl/daily.json)
+			local jsonTPL=$(sed s/_DATE/$blockDateLast/g $SCRIPTPATH/tpl/daily.json)
 			local jsonTPL=$(sed s/_UDTODAY/$UDToday/g <<< $jsonTPL)
-            local jsonTPL=$(sed s/_RWALLETS/$valueReceivedW/g <<< $jsonTPL)
+			local jsonTPL=$(sed s/_RWALLETS/$valueReceivedW/g <<< $jsonTPL)
 			local jsonTPL=$(sed s/_SWALLETS/$valueSentW/g <<< $jsonTPL)
-            local jsonTPL=$(sed s/_RMEMBERS/$valueReceivedM/g <<< $jsonTPL)
+			local jsonTPL=$(sed s/_RMEMBERS/$valueReceivedM/g <<< $jsonTPL)
 			local jsonTPL=$(sed s/_SMEMBERS/$valueSentM/g <<< $jsonTPL)
 			local jsonTPL=$(sed s/_NBRMEMBERS/$nbrMembersToday/g <<< $jsonTPL)
 			local jsonTPL=$(sed s/_NBRWALLETS/$nbrWallets/g <<< $jsonTPL)
-            echo -e "$jsonTPL" | tr -d '\\' >> $jsonFile
+			echo -e "$jsonTPL" | tr -d '\\' >> $jsonFile
 		}
 
 		# Si la date du bloc courant est différente du bloc précedent, alors on stock les valeurs journalières
@@ -119,7 +118,7 @@ scanTxWallets() {
 			local i
 			for i in $(ls x*); do
 				local walletToSup=$(cat $i)
-				local walletsAujTmp=$(echo "$walletsAujTmp" | grep -v "$walletToSup")
+				local walletsAujTmp=$(echo "$walletsAujTmp" | grep -vF "$walletToSup")
 			done
 			[[ -d $tmpSplit ]] && rm -r $tmpSplit || exit 1
 			popd >/dev/null
@@ -133,9 +132,13 @@ scanTxWallets() {
 			nbrMembersToday=$(($nbrMembersTotalToday-$nbrMembersYesterday))
 
 			# Obtention du dividende à ce block
-			udBlock=$(for i in $udBlocs; do [[ $i -ge $bloc ]] && echo $lastBloc && break; lastBloc=$i; done)
-			UDToday=$(curl -s ${DUNITER}/blockchain/block/$udBlock | jq '.dividend')
-			UDToday=$(echo "scale=2; $UDToday/100" | bc)
+			local j
+			for j in $udBlocs; do
+				[[ $j -ge $bloc ]] && break
+				lastUdBloc=$j
+			done
+			[[ -z $lastUdBloc ]] && lastUdBloc=$(echo "$udBlocs" | tail -n1)
+			UDToday=$(curl -s ${DUNITER}/blockchain/block/$lastUdBloc | jq '.dividend' | xargs -i echo "scale=2; {}/100" | bc)
 
 			jsonify
 		}
@@ -202,14 +205,13 @@ scanTxWallets() {
 			clear
 			echo "Heure de début: $startTime"
 			echo
-			echo "Scan en cours: $progress% - $bloc/$lastBloc"
+			echo "Scan en cours: $progress% - $bloc/$lastBloc (Boucle $i)"
 			echo "Date: $blockDate"
 			echo "Temps d'execution: $midTimeExec"
 		else
 			echo "Scan des blocs avec transactions en cours ..."
 		fi
-
-#		[[ $i -ge 90 ]] && break #kopa
+#		[[ $i -ge 35949 ]] && break #kopa
 	done
 
 	[[ $blockDateLast == $blockDate ]] && applyToday