From bd2b6abf9998f18d7a61413eb93e1dca242d6f29 Mon Sep 17 00:00:00 2001
From: poka <poka@p2p.legal>
Date: Mon, 21 Sep 2020 06:37:52 +0200
Subject: [PATCH] Split large wallet list for grep

---
 g1-stats.sh          |  4 ++--
 lib/functions.sh     | 16 +++++++++++++++-
 lib/scanTxWallets.sh | 22 ++++++++++++++++++++--
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/g1-stats.sh b/g1-stats.sh
index 727ce9f..756f402 100755
--- a/g1-stats.sh
+++ b/g1-stats.sh
@@ -22,8 +22,8 @@ indexhtml="$WEBPATH/index.html"
 echo -e "\n############# $day à $startTime #############\n"
 [[ -z $(which jq) || -z $(which bc) || -z $(which curl) ]] && sudo apt update && sudo apt install jq bc curl
 echo "Initialisation ..."
-#TXBLOCKS=$(cat $SCRIPTPATH/db/txblocs)  #kopa
-TXBLOCKS=$(curl -s ${DUNITER}/blockchain/with/tx | jq '.result.blocks[]')
+TXBLOCKS=$(cat $SCRIPTPATH/db/txblocs)  #kopa
+#TXBLOCKS=$(curl -s ${DUNITER}/blockchain/with/tx | jq '.result.blocks[]')
 TXBLOCKS=($(echo "$TXBLOCKS" | sort -hu | awk '{printf $1" "}'))
 unset 'TXBLOCKS[${#TXBLOCKS[@]}-1]'
 echo "TXBLOCKS OK"
diff --git a/lib/functions.sh b/lib/functions.sh
index 672e54c..3c6dfb5 100755
--- a/lib/functions.sh
+++ b/lib/functions.sh
@@ -6,7 +6,21 @@ addMembers() {
 	wallets=$(echo -e "$wallets" | sort -u)
 
 	echo "Isolation des simples portefeuilles..."
-	simpleWallets=$(echo "$wallets" | grep -v "$membresPubkeys")
+
+	# On split la listes des wallets pour ne pas saturer grep
+	local id=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+	local tmpSplit="/tmp/g1stats-splits-$id"
+	mkdir -p $tmpSplit
+	pushd $tmpSplit >/dev/null || exit 1
+	split -d -l 2500 <<<"$membresPubkeys"
+
+	simpleWallets="$wallets"
+	for i in $(ls x*); do
+		local walletToSup=$(cat $i)
+		simpleWallets=$(echo "$simpleWallets" | grep -v "$walletToSup")
+	done
+	[[ -d $tmpSplit ]] && rm -r $tmpSplit || exit 1
+	popd >/dev/null
 }
 
 web() {
diff --git a/lib/scanTxWallets.sh b/lib/scanTxWallets.sh
index 4041442..f995802 100755
--- a/lib/scanTxWallets.sh
+++ b/lib/scanTxWallets.sh
@@ -107,7 +107,25 @@ scanTxWallets() {
 		applyToday() {
 			[[ -z $wallets ]] && wallets="#"
 			WALLETS=$(echo -e "$wallets" | grep .)
-			nbrWallets=$(echo -e "$walletsAuj" | sort -u | awk 'NF' | grep -v "$membresPubkeys" | grep -v "$WALLETS" | wc -l)
+
+			# On split la listes des wallets pour ne pas saturer grep
+			local id=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+			local tmpSplit="/tmp/g1stats-splits-$id"
+			mkdir -p $tmpSplit
+			pushd $tmpSplit >/dev/null || exit 1
+			echo -e "$membresPubkeys\n$WALLETS" | split -d -l 2500
+
+			walletsAujTmp=$(echo -e "$walletsAuj" | sort -u | awk 'NF')
+			local i
+			for i in $(ls x*); do
+				local walletToSup=$(cat $i)
+				local walletsAujTmp=$(echo "$walletsAujTmp" | grep -v "$walletToSup")
+			done
+			[[ -d $tmpSplit ]] && rm -r $tmpSplit || exit 1
+			popd >/dev/null
+
+			nbrWallets=$(echo "$walletsAujTmp" | wc -l)
+			unset walletsAujTmp
 
 			# Calcul du nombre de nouveaux membres à ce block
 			nbrMembersTotalToday=$(jq '.membersCount' <<<"$blocBrut")
@@ -191,7 +209,7 @@ scanTxWallets() {
 			echo "Scan des blocs avec transactions en cours ..."
 		fi
 
-#		[[ $i -ge 10 ]] && break #kopa
+		[[ $i -ge 90 ]] && break #kopa
 	done
 
 	[[ $blockDateLast == $blockDate ]] && applyToday