g1-stats/g1-stats.sh

#!/bin/bash

################################################################################
# Author: Poka (poka@p2p.legal)
# Version: 0.2.0
# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/)
# Git: https://git.p2p.legal/axiom-team/g1-stats
################################################################################

SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
if [[ -e $SCRIPTPATH/.env ]]; then source $SCRIPTPATH/.env; else echo "Veuillez créer votre fichier .env inspiré de .env.example" && exit 1; fi

### Initialisation des données ###
startTime=$(date +'%H:%M')
day=$(date +'%y-%m-%d')
dayP=$(date +'%d-%m-%y')
id=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
indexhtml="$WEBPATH/index.html"
echo -e "\n############# $day à $startTime #############\n"
[[ -z $(which jq) || -z $(which bc) ]] && apt update && apt install jq bc
echo "Initialisation ..."
outFile="/tmp/g1-stats-$day-$startTime_$id"
#TXBLOCKS=$(cat $SCRIPTPATH/db/txblocs)
TXBLOCKS=$(curl -s ${DUNITER}/blockchain/with/tx | jq '.result.blocks[]')
TXBLOCKS=($(echo "$TXBLOCKS" | sort -hu | awk '{printf $1" "}'))
unset 'TXBLOCKS[${#TXBLOCKS[@]}-1]'
MEMBERS=$(curl -s ${DUNITER}/wot/members | jq -r '.results[].pubkey')
nbrMembers=${#MEMBERS[@]}
lastBloc=${TXBLOCKS[-1]}
isWeb=$1

### Extraction des adresses Ḡ1 actives ###

loopWalletUp() {
	local REGEX_PUBKEYS="[a-zA-Z0-9]{42,44}"

	# On vérifie le cache existant et on démarre au bloc dernier bloc en cache
	if ls $SCRIPTPATH/cache/walletsUp-* > /dev/null 2>&1; then
		local startFile=$(ls $SCRIPTPATH/cache/walletsUp-* | tail -n1)
		local startIter=$(echo "$startFile" | awk -F '-' '{ print $NF }')
		local startBloc=$(head -n1 "$startFile")
		[[ $lastBloc == $startBloc ]] && return 10

		wallets=$(tail -n +2 "$startFile")$'\n'
		local iter=$startIter
		rm $startFile
	else
		local startIter=0
		local startBloc=0
		local iter=0
		echo "[" > $SCRIPTPATH/db/recus.json
	fi

	rm -f debug.log #kopa
	for i in ${!TXBLOCKS[*]}; do
		[[ -z ${TXBLOCKS[$i]} ]] && break
		[[ $startIter != 0 && $i -le $startIter ]] && continue
		sleep 0.05

		# Récupère la date et l'objet transaction du bloc courant
		WALLETS0=$(curl -s ${DUNITER}/blockchain/block/${TXBLOCKS[$i]})
		until WALLETS=$(echo $WALLETS0 | jq -r '.medianTime, .transactions[]' 2>/dev/null); do
			[[ $isWeb != "web" ]] && echo -e "iter $iter \n$WALLETS0" | tee -a debug.log
			sleep 2
			WALLETS0=$(curl -s ${DUNITER}/blockchain/block/${TXBLOCKS[$i]})
		done
		[[ -z "$WALLETS" ]] && continue
		bloc=${TXBLOCKS[$i]}

		# Récupération de la date du block
		blockDate=$(echo "$WALLETS" | head -n1)
		blockDate=$(date -d"@$blockDate" +%y-%m-%d -u)
		WALLETS=$(echo "$WALLETS" | tail -n +2)

		# Si cette itération proviens d'un cache, alors on aggrège la dernière date
		if [[ $startBloc != 0 ]]; then
			lastDate=$(cat db/recus.json | jq -r '.[].date' | tail -n1)
			if [[ "$blockDate" == "$lastDate" ]]; then
				valueBlocW=$(cat db/recus.json | jq -r '.[].rWallets' | tail -n1 | tr -d '.,')
				valueBlocM=$(cat db/recus.json | jq -r '.[].rMembres' | tail -n1 | tr -d '.,')

				local recusJson=$(head -n -6 db/recus.json)
				echo "$recusJson" > $SCRIPTPATH/db/recus.json
			else
				local recusJson=$(head -n -2 db/recus.json)
				echo -e "$recusJson\n   }," > $SCRIPTPATH/db/recus.json
			fi
			unset recusJson
			startBloc=0
		fi
		# Sélectionne les ligne SIG en retirant les issuers
		OIFS="$IFS"
		IFS='{'
		for j in $WALLETS; do
			[[ -z $j ]] && continue
			wIssuers=$(echo "{$j" | jq -r '.issuers[0]')
			WALLETS1+=$(echo "{$j" | jq -r '.outputs[]' | grep -v "$wIssuers")
			WALLETS1+="\n"
		done
		IFS="$OIFS"
		WALLETS=$(echo -e "$WALLETS1" | head -n -1)
		unset WALLETS1

		# Exporte les valeurs de la journée dans le fichier JSON
		jsonify() {
            [[ -z $valueBlocW ]] && valueBlocW=0 || valueBlocW=$(echo "scale=2; $valueBlocW/100" | bc)
            [[ -z $valueBlocM ]] && valueBlocM=0 || valueBlocM=$(echo "scale=2; $valueBlocM/100" | bc)
            local jsonTPL=$(sed s/_DATE/$blockDateLast/g $SCRIPTPATH/tpl/recus.json)
            local jsonTPL=$(sed s/_RWALLETS/$valueBlocW/g <<< $jsonTPL)
            local jsonTPL=$(sed s/_RMEMBRES/$valueBlocM/g <<< $jsonTPL)
            echo -e "$jsonTPL" | tr -d '\\' >> $SCRIPTPATH/db/recus.json
		}

		# Si le date du bloc courant est différente du bloc précedent, alors on stock les valeurs journalières
		if [[ -n $blockDateLast && $blockDateLast != $blockDate ]]; then
			jsonify
			unset valueBlocM valueBlocW
		fi

		# Sauvegarde la date de ce bloc pour l'itération suivante
		blockDateLast=$blockDate

		# Ajoute la valeur des transactions de ce bloc au reste de la journée
		for k in $WALLETS; do
		    pubkey=$(echo $k | awk -F '(' '{ print $2 }' | tr -d ')')
		    value=$(echo $k | awk -F: '{ print $1 }')

		    if [[ $(echo "$MEMBERS" | grep $pubkey) ]]; then
		        valueBlocM=$(($valueBlocM+$value))
		        pubkeyBlocM+=${pubkey}\\n
		    else
		        valueBlocW=$(($valueBlocW+$value))
		        pubkeyBlocW+=${pubkey}\\n
		    fi
		done

		# Stock les clés publiques de ce bloc dans la variable $wallets
		wallets+=$(echo "$WALLETS" | grep -Eo $REGEX_PUBKEYS)$'\n'

		# Affiche la progression de la boucle si on est pas en mode web
		progress=$(echo "scale=1; $bloc*100/$lastBloc/1" | bc)
		if [[ $isWeb != "web" ]]; then
			clear
			echo "Heure de début: $startTime"
			echo
			echo "Scan en cours: $progress% - $bloc/$lastBloc"
			echo "Date: $blockDate"
		fi

		# Debug #
		echo -e "i: $i\niter: $iter\nBloc: $bloc\n---" >> debug.log
		((iter++))
#		[[ $i -ge 10 ]] && break #kopaa

	done

	jsonify

	# Retire la dernière virgule et ajoute le crochet de fin de JSON
	sed -i '$ s/,//g' $SCRIPTPATH/db/recus.json
	echo "]" >> $SCRIPTPATH/db/recus.json

	# On supprime les doublons et les lignes vides
	wallets=$(echo -e "$wallets" | sort -u | awk 'NF')$'\n'
	# On écrit les pubkeys avec transaction dans un fichier de cache pour la prochaine itération
#	((iter--))
	[[ ! -d $SCRIPTPATH/cache ]] && mkdir $SCRIPTPATH/cache
	echo -e "$lastBloc\n$wallets" > $SCRIPTPATH/cache/walletsUp-$i
}
loopWalletUp

$SCRIPTPATH/sum.sh

endDate=$(date +'%H:%M')
echo "Heure de fin: $endDate"

exit


### Ajout des membres sans transaction au fichier tampon ###
loopMembers() {
local iter=0
for i in ${MEMBERS[*]}; do

	progress=$(echo "scale=0; $iter*100/$nbrMembers/1" | bc)
	if [[ $progress =~ ^(0|10|20|30|40|50|60|70|80|90|99)$ ]]; then
		[[ $progress == 99 ]] && progress=100
		if [[ $isWeb != "web" ]]; then
			clear
			echo "Heure de début: $startTime"
			echo
			echo "Scan en cours: 100% - $bloc/$lastBloc"
			echo "Ajouts des comptes membres ... $progress%"
		fi
	fi

    if [[ -z $(grep "$i" $outFile) ]]; then
        echo -e "$i" >> $outFile
    fi

	((iter++))
done
}
loopMembers

### Calcul du nombre de wallets ###
nbrTotalWallets=$(cat $outFile | wc -l)
nbrSimpleWallets=$(echo "$nbrTotalWallets-$nbrMembers" | bc)
pourcentMbrs=$(echo "scale=1; $nbrMembers*100/$nbrTotalWallets/1" | bc)
pourcentWallets=$(echo "scale=1; $nbrSimpleWallets*100/$nbrTotalWallets/1" | bc)

### Renseignement de l'index web et indexation de l'historique ###
web() {

	[ ! -d $WEBPATH/history/ ] && mkdir -p $WEBPATH/history/
        [ ! -d $WEBPATH/graph/ ] && mkdir -p $WEBPATH/graph/
	[ ! -d $WEBPATH/css ] && cp -r $SCRIPTPATH/tpl/css $WEBPATH/
	[ ! -d $WEBPATH/js ] && cp -r $SCRIPTPATH/tpl/js $WEBPATH/
	cp $SCRIPTPATH/tpl/index.html $indexhtml

	datePrevious=$(date +'%y-%m-%d' -d "$day -1 day")
	[[ -z $(ls -l $WEBPATH/history/ | grep $datePrevious) && -z $(grep '"display:none;" class="previous"' $indexhtml) ]] && sed -i "s/class=\"previous\"/style=\"display:none;\" class=\"previous\"/g" $indexhtml
        dateNext=$(date +'%y-%m-%d' -d "$day +1 day")
	[[ -z $(ls -l $WEBPATH/history/ | grep $dateNext) && -z $(grep '"display:none;" class="next"' $indexhtml) ]] && sed -i "s/class=\"next\"/style=\"display:none;\" class=\"next\"/g" $indexhtml

	sed -i "s/_nbrTotalWallets/$nbrTotalWallets/g" $indexhtml
	sed -i "s/_nbrSimpleWallets/$nbrSimpleWallets/g" $indexhtml
	sed -i "s/_nbrMembers/$nbrMembers/g" $indexhtml
	sed -i "s/_pourcentMbrs/$pourcentMbrs/g" $indexhtml
	sed -i "s/_pourcentWallets/$pourcentWallets/g" $indexhtml
	# sed -i "s/_node/$DUNITER/g" $indexhtml
	sed -i "s/_heure/$startTime/g" $indexhtml
        sed -i "s/_day/$dayP/g" $indexhtml
	sed -i "s/_txInSimple/$txInSimple/g" $indexhtml
	sed -i "s/_txOutSimple/$txOutSimple/g" $indexhtml
	sed -i "s/_soldeSimple/$soldeSimple/g" $indexhtml
        sed -i "s/_txInMembers/$txInMembers/g" $indexhtml
        sed -i "s/_txOutMembers/$txOutMembers/g" $indexhtml
        sed -i "s/_soldeMembers/$soldeMembers/g" $indexhtml
	sed -i "s/_pourcentSimpleWallet/$pourcentSimpleWallet/g" $indexhtml
	sed -i "s/_nonConsumedUDT/$nonConsumedUDT/g" $indexhtml
        sed -i "s/_monetaryMass/$monetaryMass/g" $indexhtml
	sed -i "s/_sleepyG1/$sleepyG1/g" $indexhtml
        [[ -z $(grep '"display:none;" class="previous"' $indexhtml) ]] && sed -i "s/_datePrevious/$datePrevious/g" $indexhtml && setPrevious="Oui"
        [[ -z $(grep '"display:none;" class="next"' $indexhtml) ]] && sed -i "s/_dateNext/$dateNext/g" $indexhtml && setNext="Oui"

	cat "$outFile" | grep . > $WEBPATH/wallets-g1.txt
	echo -e "${MEMBERS[@]}" | sed 's/ /\n/g' > $WEBPATH/wallets-g1-membres.txt
	echo -e "$simpleWallets" > $WEBPATH/wallets-g1-simple.txt

	if [[ "$startTime" == "00:00" ]]; then
		cp $indexhtml $WEBPATH/history/index_$day.html
		sed -i "s/css\/style.css/..\/css\/style.css/g" $WEBPATH/history/index_$day.html
		sed -i "s/logo-axiom-team2.svg/..\/logo-axiom-team2.svg/g" $WEBPATH/history/index_$day.html
		sed -i "s/_dateNext/$day/g" $WEBPATH/history/index_$datePrevious.html
		sed -i "s/style=\"display:none;\" class=\"next\"/class=\"next\"/g" $WEBPATH/history/index_$datePrevious.html
	fi

	# Export JSON for graph
	$SCRIPTPATH/transform_json.sh

	# chown www-data for nginx needs
	chown -R www-data:www-data $WEBPATH
}

### Affichage du nombre de wallets ###
echo -e "\n ---\n"
echo "Noeud: $DUNITER"
echo "Nombre total de wallet: $nbrTotalWallets"
echo "Nombre de membres: $nbrMembers (${pourcentMbrs}%)"
echo "Nombre de simple portefeuille: $nbrSimpleWallets (${pourcentWallets}%)"
echo -e "\n ---\n"
echo "Extraction wallets membres / Simples portefeuille"
echo -e "\n ---\n"

### Isolation des simples portefeuilles ###
simpleWallets=$(cat $outFile)
echo "Isolation des simples portefeuilles..."
for i in ${MEMBERS[@]}; do
	simpleWallets=$(echo "$simpleWallets" | grep -v "$i")
done

### Boucle d'obtention des soldes ###
getSolde(){
solde=0
txInT=0
txOutT=0
nonConsumedUDT=0
nonConsumedUD=0
for i in $pubkeys; do
	until txInL=$(curl -s "$ESNODE/g1/movement/_search?filter_path=hits.hits._source&size=10000&q=recipient:$i&pretty"); do
		echo "Erreur: $i"
		sleep 2
	done
	until txOutL=$(curl -s "$ESNODE/g1/movement/_search?filter_path=hits.hits._source&size=10000&q=issuer:$i&pretty"); do
		echo "Erreur: $i"
		sleep 2
	done

	if [[ $1 == "mbr" ]]; then
		nonConsumedUD=$(curl -s ${DUNITER}/ud/history/$i | jq -r '.history.history[].amount' | awk '{s+=$1} END {print s}') || nonConsumedUD=0
		[[ -z $nonConsumedUD ]] && nonConsumedUD=0
		nonConsumedUDT=$(echo -e "scale=2; ($nonConsumedUD/100)+$nonConsumedUDT" | bc)
	fi

        [[ $txInL != "{ }" ]] && txIn=$(echo "$txInL" | jq '.hits.hits[]._source.amount' | awk '{s+=$1} END {print s}') || txIn=0
        [[ $txOutL != "{ }" ]] && txOut=$(echo "$txOutL" | jq '.hits.hits[]._source.amount' | awk '{s+=$1} END {print s}') || txOut=0

        solde=$(echo -e "scale=2; (($txIn-$txOut+$nonConsumedUD)/100)+$solde" | bc)

        txInT=$(echo -e "scale=2; (($txIn+$nonConsumedUD)/100)+$txInT" | bc)
        txOutT=$(echo -e "scale=2; ($txOut/100)+$txOutT" | bc)

done
}

echo "Récupération du solde des simples wallets..."

pubkeys=$simpleWallets
getSolde
txInSimple=$(echo $txInT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
txOutSimple=$(echo $txOutT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
soldeSimpleBrut=$(echo $solde)
soldeSimple=$(echo $soldeSimpleBrut | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
echo -e "Reçus simples wallets:\t $txInSimple"
echo -e "Envoyé simples wallets:\t $txOutSimple"
echo -e "Soldes simples wallets:\t $soldeSimple"
echo -e "\n ---\n"
echo "Récupération du solde des membres..."

pubkeys=${MEMBERS[@]}
getSolde
txInMembers=$(echo $txInT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
txOutMembers=$(echo $txOutT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
soldeMembersBrut=$(echo $solde)
soldeMembers=$(echo $soldeMembersBrut | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
echo -e "Reçus membres:\t $txInMembers"
echo -e "Envoyé membres:\t $txOutMembers"
echo -e "Soldes membres:\t $soldeMembers"
echo -e "Total DU non consumés:\t $nonConsumedUDT"

### Calcul de la masse monétaire et du pourcentage de Ḡ1 sur les simples portefeuilles ###
monetaryMass=$(curl -s ${DUNITER}/blockchain/current | jq .monetaryMass)
monetaryMassBrut=$(echo -e "scale=2; ($monetaryMass/100)/1" | bc)
monetaryMass=$(echo "$monetaryMassBrut" | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
soldeWalletMembers=$(echo "scale=2; $soldeMembersBrut+$soldeSimpleBrut" | bc | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta')
pourcentSimpleWallet=$(echo "scale=1; $soldeSimpleBrut*100/$monetaryMassBrut/1" | bc)

echo -e "\n ---\n"
echo -e "Solde des membres (sans DU):\t $soldeWalletMembers"
echo -e "Masse Monétaire:\t $monetaryMass Ḡ1"


# Analyse
echo -e "\n ---\n Analyse\n ---\n"
sleepyG1=$(echo -e "scale=1; 100-$txOutT*100/$monetaryMassBrut" | bc | tr . ,)
echo -e "$sleepyG1% des Ḡ1 n'ont jamais été utilisés."

[[ $isWeb == "web" ]] && web

### Fin de programme ###
rm $outFile
day=$(date +'%d-%m-%y')
echo "$day - Heure de fin: $(date +'%H:%M')"

echo "Début de la journalisation ..."
$SCRIPTPATH/soldeByDays.sh