From 2d92a40895cf5add701f9638f2a736ab92bbed4c Mon Sep 17 00:00:00 2001 From: poka Date: Sun, 20 Sep 2020 22:32:15 +0200 Subject: [PATCH] Add nbrMembres, nbrWallets and UD in JSON; Improve cache mechanic --- explorer.sh | 46 +++++++++------ g1-stats.sh | 9 ++- lib/functions.sh | 8 +-- lib/scanTxWallets.sh | 118 +++++++++++++++++++++++++++++---------- lib/transform_json.sh | 4 +- scripts/export_sommes.sh | 2 +- scripts/restore-bk.sh | 11 +++- tpl/recus.json | 2 +- 8 files changed, 141 insertions(+), 59 deletions(-) diff --git a/explorer.sh b/explorer.sh index c1c331d..20e7733 100755 --- a/explorer.sh +++ b/explorer.sh @@ -1,28 +1,38 @@ #!/bin/bash -dateRange=$1 +dateRange=$2 lineNbr=$(grep -n "$dateRange" db/recus.json | cut -d : -f 1) -lineNbr=$(($lineNbr+4)) -jsonDated=$(head -n$lineNbr db/recus.json) -jsonDated=$(echo -e "$jsonDated\n }\n]") +cumulate() { + lineNbr=$(($lineNbr+6)) -sumRWBrut=$(jq -r '.[].rWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') -sumRMBrut=$(jq -r '.[].rMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') -#sumRTBrut=$(jq -r '.[] | .rWallets, .rMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') -sumSWBrut=$(jq -r '.[].sWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') -sumSMBrut=$(jq -r '.[].sMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') -#sumSTBrut=$(jq -r '.[] | .sWallets, .sMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') + jsonDated=$(head -n$lineNbr db/recus.json) + jsonDated=$(echo -e "$jsonDated\n }\n]") -#echo "$sumRWBrut" + sumRWBrut=$(jq -r '.[].rWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumRMBrut=$(jq -r '.[].rMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') + #sumRTBrut=$(jq -r '.[] | .rWallets, .rMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumSWBrut=$(jq -r '.[].sWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumSMBrut=$(jq -r '.[].sMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') + #sumSTBrut=$(jq -r '.[] | .sWallets, .sMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }') -echo "{ - \"date\": "\"$dateRange\"", - \"rWallets\": $sumRWBrut, - \"sWallets\": $sumSWBrut, - \"rMembres\": $sumRMBrut, - \"sMembres\": $sumSMBrut + #echo "$sumRWBrut" + + echo "{ + \"date\": "\"$dateRange\"", + \"rWallets\": $sumRWBrut, + \"sWallets\": $sumSWBrut, + \"rMembers\": $sumRMBrut, + \"sMembers\": $sumSMBrut + } + " | jq . } -" | jq . +selectDay() { + lineNbr=$(($lineNbr+7)) + jsonDated=$(head -n$lineNbr db/recus.json |tail -n7 | sed 's/},/}/g') + echo "$jsonDated" | jq . +} + +$1 \ No newline at end of file diff --git a/g1-stats.sh b/g1-stats.sh index d12eee9..8b1fb0e 100755 --- a/g1-stats.sh +++ b/g1-stats.sh @@ -7,6 +7,9 @@ # Git: https://git.p2p.legal/axiom-team/g1-stats ################################################################################ +# Stop if error +set -e + SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" if [[ -e $SCRIPTPATH/.env ]]; then source $SCRIPTPATH/.env; else echo "Veuillez créer votre fichier .env inspiré de .env.example" && exit 1; fi @@ -26,6 +29,7 @@ unset 'TXBLOCKS[${#TXBLOCKS[@]}-1]' membresPubkeys=$(cat db/membresPubkeys) #kopa nbrMembers=$(echo "$membresPubkeys" | wc -l) lastBloc=${TXBLOCKS[-1]} +udBlocs=$(curl -s ${DUNITER}/blockchain/with/ud | jq '.result.blocks[]') isWeb=$1 ### On source les fonctions @@ -33,7 +37,10 @@ source $SCRIPTPATH/lib/functions.sh source $SCRIPTPATH/lib/scanTxWallets.sh ### Extraction des adresses Ḡ1 actives -scanTxWallets +scanTxWallets $1 #kopa +exit + + ### Calcul la somme des soldes portefeuilles et membres sumSoldes diff --git a/lib/functions.sh b/lib/functions.sh index 157aef9..ad5eed1 100755 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -64,12 +64,12 @@ web() { sumSoldes() { sumRWBrut=$(jq -r '.[].rWallets' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') - sumRMBrut=$(jq -r '.[].rMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') - sumRTBrut=$(jq -r '.[] | .rWallets, .rMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumRMBrut=$(jq -r '.[].rMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumRTBrut=$(jq -r '.[] | .rWallets, .rMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') sumSWBrut=$(jq -r '.[].sWallets' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') #sumSWBrut=$(jq -n '[inputs | .[].sWallets] | reduce .[] as $num (0; .+$num)' db/recus.json | xargs printf %.2f | tr ',' '.') - sumSMBrut=$(jq -r '.[].sMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') - sumSTBrut=$(jq -r '.[] | .sWallets, .sMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumSMBrut=$(jq -r '.[].sMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumSTBrut=$(jq -r '.[] | .sWallets, .sMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }') soldeWBrut=$(echo "$sumRWBrut-$sumSWBrut" | bc) soldeMBrut=$(echo "$sumRMBrut-$sumSMBrut" | bc) diff --git a/lib/scanTxWallets.sh b/lib/scanTxWallets.sh index c787bc3..93ee10b 100755 --- a/lib/scanTxWallets.sh +++ b/lib/scanTxWallets.sh @@ -1,3 +1,5 @@ +#!/bin/bash + scanTxWallets() { local REGEX_PUBKEYS="[a-zA-Z0-9]{42,44}" @@ -6,10 +8,12 @@ scanTxWallets() { local startFile=$(ls $SCRIPTPATH/cache/walletsUp-* | tail -n1) local startIter=$(echo "$startFile" | awk -F '-' '{ print $NF }') local startBloc=$(head -n1 "$startFile") - wallets=$(tail -n +2 "$startFile")$'\n' - wallets=$(echo -e "$wallets" | sort -u | awk 'NF') + wallets=$(tail -n +2 "$startFile" | sed '/---/Q' | sort -u) + local walletsAuj=$(cat "$startFile" | sed -e '1,/---/d' | sed '/---/Q' | grep .)$'\n' + local nbrMembersYesterday=$(cat "$startFile" | sed -e '1,/---/d' | sed -e '1,/---/d' | sed '/---/Q' | grep . | cut -d '|' -f 1) + local blockDateLast=$(cat "$startFile" | sed -e '1,/---/d' | sed -e '1,/---/d' | sed '/---/Q' | grep . | cut -d '|' -f 2) - [[ $lastBloc == $startBloc ]] && return 10 +##kopa [[ $lastBloc == $startBloc ]] && return 10 rm $startFile else local startIter=0 @@ -28,8 +32,7 @@ scanTxWallets() { # Récupère la date et l'objet transaction du bloc courant blocFull=$(curl -s ${DUNITER}/blockchain/block/$bloc) - #blocFull=$(cat bloc-267) - until blocBrut=$(echo $blocFull | jq -r '.medianTime, .transactions[]' 2>/dev/null); do + until blocBrut=$(echo $blocFull | jq -r '.' 2>/dev/null); do [[ $isWeb != "web" ]] && echo -e "i: $i\n$blocFull" sleep 2 blocFull=$(curl -s ${DUNITER}/blockchain/block/$bloc) @@ -37,28 +40,30 @@ scanTxWallets() { [[ -z "$blocBrut" ]] && continue # Récupération de la date du block - blockDate=$(echo "$blocBrut" | head -n1) + blockDate=$(jq '.medianTime' <<<"$blocBrut") blockDate=$(date -d"@$blockDate" +%y-%m-%d -u) - local walletsReceivedBloc=$(echo "$blocBrut" | tail -n +2) + local walletsReceivedBloc=$(jq '.transactions[]' <<<"$blocBrut") - # Si cette itération proviens d'un cache, alors on aggrège la dernière date + # Si cette itération provient d'un cache, alors on aggrège la dernière date if [[ $startBloc != 0 ]]; then lastDate=$(cat db/recus.json | jq -r '.[].date' | tail -n1) + echo -e "IFCACHE\t - $bloc - $blockDate - $lastDate" >> tata #kopa if [[ "$blockDate" == "$lastDate" ]]; then - valueReceivedW=$(cat db/recus.json | jq -r '.[].rWallets' | tail -n1 | tr -d '.,') - valueReceivedM=$(cat db/recus.json | jq -r '.[].rMembres' | tail -n1 | tr -d '.,') - valueSentW=$(cat db/recus.json | jq -r '.[].sWallets' | tail -n1 | tr -d '.,') - valueSentM=$(cat db/recus.json | jq -r '.[].sMembres' | tail -n1 | tr -d '.,') + valueReceivedW=$(cat db/recus.json | jq -r '.[].rWallets' | tail -n1 | xargs echo 100* | bc) + valueReceivedM=$(cat db/recus.json | jq -r '.[].rMembers' | tail -n1 | xargs echo 100* | bc) + valueSentW=$(cat db/recus.json | jq -r '.[].sWallets' | tail -n1 | xargs echo 100* | bc) + valueSentM=$(cat db/recus.json | jq -r '.[].sMembers' | tail -n1 | xargs echo 100* | bc) - local recusJson=$(head -n -8 db/recus.json) + local recusJson=$(head -n -11 db/recus.json) echo "$recusJson" > $SCRIPTPATH/db/recus.json else - local recusJson=$(head -n -4 db/recus.json) + local recusJson=$(head -n -2 db/recus.json) echo -e "$recusJson\n }," > $SCRIPTPATH/db/recus.json fi - unset recusJson + #unset recusJson startBloc=0 fi + # Sélectionne les ligne SIG en retirant les issuers OIFS="$IFS" IFS='{' @@ -80,19 +85,67 @@ scanTxWallets() { [[ -z $valueReceivedM ]] && valueReceivedM=0 || valueReceivedM=$(echo "scale=2; $valueReceivedM/100" | bc) [[ -z $valueSentM ]] && valueSentM=0 || valueSentM=$(echo "scale=2; $valueSentM/100" | bc) [[ -z $valueSentW ]] && valueSentW=0 || valueSentW=$(echo "scale=2; $valueSentW/100" | bc) + [[ -z $nbrMembersToday ]] && nbrMembersToday=0 + [[ -z $nbrWallets ]] && nbrWallets=0 + [[ -z $UDToday ]] && UDToday=0 + local jsonTPL=$(sed s/_DATE/$blockDateLast/g $SCRIPTPATH/tpl/recus.json) + local jsonTPL=$(sed s/_UDTODAY/$UDToday/g <<< $jsonTPL) local jsonTPL=$(sed s/_RWALLETS/$valueReceivedW/g <<< $jsonTPL) local jsonTPL=$(sed s/_SWALLETS/$valueSentW/g <<< $jsonTPL) - local jsonTPL=$(sed s/_RMEMBRES/$valueReceivedM/g <<< $jsonTPL) - local jsonTPL=$(sed s/_SMEMBRES/$valueSentM/g <<< $jsonTPL) + local jsonTPL=$(sed s/_RMEMBERS/$valueReceivedM/g <<< $jsonTPL) + local jsonTPL=$(sed s/_SMEMBERS/$valueSentM/g <<< $jsonTPL) + local jsonTPL=$(sed s/_NBRMEMBERS/$nbrMembersToday/g <<< $jsonTPL) + local jsonTPL=$(sed s/_NBRWALLETS/$nbrWallets/g <<< $jsonTPL) echo -e "$jsonTPL" | tr -d '\\' >> $SCRIPTPATH/db/recus.json } + # Stock les clés publiques de ce bloc dans la variable $wallets + walletsAuj+=$(echo "$walletsReceivedBloc" | grep -Eo $REGEX_PUBKEYS)$'\n' + # Si le date du bloc courant est différente du bloc précedent, alors on stock les valeurs journalières - if [[ -n $blockDateLast && $blockDateLast != $blockDate ]]; then + applyToday() { + [[ -z $wallets ]] && wallets="#" + WALLETS=$(echo -e "$wallets" | grep .) + nbrWallets=$(echo -e "$walletsAuj" | sort -u | awk 'NF' | grep -v "$membresPubkeys" | grep -v "$WALLETS" | wc -l) + + # Calcul du nombre de nouveaux membres à ce block + nbrMembersTotalToday=$(jq '.membersCount' <<<"$blocBrut") + [[ -z $nbrMembersYesterday ]] && nbrMembersYesterday=0 + nbrMembersToday=$(($nbrMembersTotalToday-$nbrMembersYesterday)) + + # Récupération de la masse monétaire à ce block + #monetaryMassTotalToday=$(jq '.monetaryMass' <<<"$blocBrut") + #monetaryMassTotalToday=$(echo "scale=2; $monetaryMassTotalToday/100" | bc) + #[[ -z $monetaryMassYesterday ]] && monetaryMassYesterday=0 + #monetaryMassToday=$(echo "scale=2; $monetaryMassTotalToday-$monetaryMassYesterday" | bc) + #monetaryMassYesterday=$monetaryMassTotalToday + + # Obtention du dividende à ce block + udBlock=$(for i in $udBlocs; do [[ $i -ge $bloc ]] && echo $lastBloc && break; lastBloc=$i; done) + UDToday=$(curl -s ${DUNITER}/blockchain/block/$udBlock | jq '.dividend') + UDToday=$(echo "scale=2; $UDToday/100" | bc) + jsonify - unset valueReceivedM valueReceivedW valueSentM valueSentW + } + if [[ -n $blockDateLast && $blockDateLast != $blockDate ]]; then + if [[ -z $recusJson ]]; then + applyToday + else + # Calcul du nombre de nouveaux membres à ce block + nbrMembersTotalToday=$(jq '.membersCount' <<<"$blocBrut") + [[ -z $nbrMembersYesterday ]] && nbrMembersYesterday=0 + nbrMembersToday=$(($nbrMembersTotalToday-$nbrMembersYesterday)) + fi + + [[ $wallets == "#" ]] && unset wallets + wallets+=$'\n'"$walletsAuj" + wallets=$(echo -e "$wallets" | sort -u | grep .)$'\n' + nbrMembersYesterday=$nbrMembersTotalToday + unset valueReceivedM valueReceivedW valueSentM valueSentW walletsAuj + echo -e "INLOOP\t - $bloc - $blockDateLast - $nbrMembersYesterday" >> tata #kopa fi + unset recusJson # Sauvegarde la date de ce bloc pour l'itération suivante blockDateLast=$blockDate @@ -103,10 +156,14 @@ scanTxWallets() { local value=$(echo $k | awk -F '|' '{ print $2 }') if [[ $(echo "$membresPubkeys" | grep $pubkey) ]]; then - valueReceivedM=$(($valueReceivedM+$value)) + #valueReceivedM=$(($valueReceivedM+$value)) + [[ -z $valueReceivedM ]] && valueReceivedM=0 + valueReceivedM=$(echo "$valueReceivedM+$value" | bc) pubkeyReceiveM+=${pubkey}\\n else - valueReceivedW=$(($valueReceivedW+$value)) + #valueReceivedW=$(($valueReceivedW+$value)) + [[ -z $valueReceivedW ]] && valueReceivedW=0 + valueReceivedW=$(echo "$valueReceivedW+$value" | bc) pubkeyReceiveW+=${pubkey}\\n fi done @@ -117,17 +174,18 @@ scanTxWallets() { local value=$(echo $k | awk -F '|' '{ print $2 }') if [[ $(echo "$membresPubkeys" | grep $pubkey) ]]; then - valueSentM=$(($valueSentM+$value)) + #valueSentM=$(($valueSentM+$value)) + [[ -z $valueSentM ]] && valueSentM=0 + valueSentM=$(echo "$valueSentM+$value" | bc) pubkeySentM+=${pubkey}\\n else - valueSentW=$(($valueSentW+$value)) + #valueSentW=$(($valueSentW+$value)) + [[ -z $valueSentW ]] && valueSentW=0 + valueSentW=$(echo "$valueSentW+$value" | bc) pubkeySentW+=${pubkey}\\n fi done - # Stock les clés publiques de ce bloc dans la variable $wallets - wallets+=$(echo "$walletsReceivedBloc" | grep -Eo $REGEX_PUBKEYS)$'\n' - # Affiche la progression de la boucle si on est pas en mode web progress=$(echo "scale=1; $bloc*100/$lastBloc/1" | bc) endExec=$(($(date +%s%N)/1000000)) @@ -148,19 +206,19 @@ scanTxWallets() { echo "Scan des blocs avec transactions en cours ..." fi -# [[ $i -ge 10 ]] && break #kopa + [[ $i -ge $1 ]] && break #kopa done - jsonify + [[ $blockDateLast == $blockDate ]] && applyToday && echo -e "END\t\t - $bloc - $blockDate - $nbrMembersYesterday" >> tata #kopa # Retire la dernière virgule et ajoute le crochet de fin de JSON sed -i '$ s/,//g' $SCRIPTPATH/db/recus.json echo "]" >> $SCRIPTPATH/db/recus.json # On supprime les doublons et les lignes vides - wallets=$(echo -e "$wallets" | sort -u | awk 'NF')$'\n' + wallets=$(echo -e "$wallets" | sort -u | awk 'NF') # On écrit les pubkeys avec transaction dans un fichier de cache pour la prochaine itération [[ ! -d $SCRIPTPATH/cache ]] && mkdir $SCRIPTPATH/cache - echo -e "$lastBloc\n$wallets" > $SCRIPTPATH/cache/walletsUp-$i + echo -e "$lastBloc\n$wallets\n---\n$walletsAuj\n---\n$nbrMembersYesterday|$blockDateLast" > $SCRIPTPATH/cache/walletsUp-$i } diff --git a/lib/transform_json.sh b/lib/transform_json.sh index b171c4d..0278849 100755 --- a/lib/transform_json.sh +++ b/lib/transform_json.sh @@ -23,7 +23,7 @@ rWallets() { local jsonTPL=$(sed s/_DATE/$dateN/g $SCRIPTPATH/tpl/recus.json) local jsonTPL=$(sed s/_RWALLETS/$RW/g <<< $jsonTPL) - local result+=$(sed s/_RMEMBRES/$RM/g <<< $jsonTPL) + local result+=$(sed s/_RMEMBERS/$RM/g <<< $jsonTPL) done sed "s/_JSONDATA/$result/g" $SCRIPTPATH/tpl/recus.html > $WEBPATH/graph/recus.html @@ -46,7 +46,7 @@ nbrWallets() { local jsonTPL=$(sed s/_DATE/$dateN/g $SCRIPTPATH/tpl/nbr_wallets.json) local jsonTPL=$(sed s/_NBRWALLETS/$NBRW/g <<< $jsonTPL) - local result+=$(sed s/_NBRMEMBRES/$NBRM/g <<< $jsonTPL) + local result+=$(sed s/_NBRMEMBERS/$NBRM/g <<< $jsonTPL) done sed "s/_JSONDATA/$result/g" $SCRIPTPATH/tpl/nbr_wallets.html > $WEBPATH/graph/nbr_wallets.html diff --git a/scripts/export_sommes.sh b/scripts/export_sommes.sh index 7a7af7e..e5969c9 100755 --- a/scripts/export_sommes.sh +++ b/scripts/export_sommes.sh @@ -2,4 +2,4 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -jq -r '.[] | "---", .date, .rWallets + .rMembres' $SCRIPTPATH/../db/recus.json > $SCRIPTPATH/../db/somme.txt +jq -r '.[] | "---", .date, .rWallets + .rMembers' $SCRIPTPATH/../db/recus.json > $SCRIPTPATH/../db/somme.txt diff --git a/scripts/restore-bk.sh b/scripts/restore-bk.sh index 04b1444..04370cb 100755 --- a/scripts/restore-bk.sh +++ b/scripts/restore-bk.sh @@ -2,8 +2,15 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -cacheFile=$(ls $SCRIPTPATH/../cache/bk-walletsUp-* | tail -n1) -cacheID=$(echo $cacheFile | awk -F '-' '{ print $NF }') +if [[ $1 ]]; then + cacheID=$1 + cacheFile="$SCRIPTPATH/../cache/bk-walletsUp-$cacheID" +else + cacheFile=$(ls $SCRIPTPATH/../cache/bk-walletsUp-* | tail -n1) + cacheID=$(echo $cacheFile | awk -F '-' '{ print $NF }') +fi + +[[ -f $cacheFile ]] && rm $SCRIPTPATH/../cache/walletsUp-* cp $cacheFile $SCRIPTPATH/../cache/walletsUp-$cacheID cp $SCRIPTPATH/../db/bk-recus.json-$cacheID $SCRIPTPATH/../db/recus.json diff --git a/tpl/recus.json b/tpl/recus.json index 5cf5c99..464acc8 100644 --- a/tpl/recus.json +++ b/tpl/recus.json @@ -1 +1 @@ - {\n \"date\": \"_DATE\",\n \"rWallets\": _RWALLETS,\n \"sWallets\": _SWALLETS,\n \"rMembres\": _RMEMBRES,\n \"sMembres\": _SMEMBRES\n }, + {\n \"date\": \"_DATE\",\n \"UD\": _UDTODAY,\n \"rWallets\": _RWALLETS,\n \"sWallets\": _SWALLETS,\n \"rMembers\": _RMEMBERS,\n \"sMembers\": _SMEMBERS,\n \"nbrMembers\": _NBRMEMBERS,\n \"nbrWallets\": _NBRWALLETS\n }, \ No newline at end of file