From e5c68db1f1160cd1fa07dde6683ab18636df7132 Mon Sep 17 00:00:00 2001 From: poka Date: Tue, 15 Sep 2020 05:50:24 +0200 Subject: [PATCH] Improve backup/restore DB; Add sent solde to JSON --- backup-actual.sh | 3 +- functions.sh | 81 ++++++++++++++++++++---------------------------- g1-stats.sh | 32 +++---------------- restore-bk.sh | 4 +-- 4 files changed, 42 insertions(+), 78 deletions(-) diff --git a/backup-actual.sh b/backup-actual.sh index 886f654..95e56a7 100755 --- a/backup-actual.sh +++ b/backup-actual.sh @@ -3,6 +3,7 @@ cacheFile=$(ls cache/walletsUp-*) cacheID=$(echo $cacheFile | awk -F '-' '{ print $NF }') +#rm -f cache/bk-walletsUp-* cp $cacheFile cache/bk-walletsUp-$cacheID -cp db/recus.json db/bk-recus.json +cp db/recus.json db/bk-recus.json-$cacheID diff --git a/functions.sh b/functions.sh index d846340..a793088 100755 --- a/functions.sh +++ b/functions.sh @@ -24,6 +24,7 @@ loopWalletUp() { for i in ${!TXBLOCKS[*]}; do [[ -z ${TXBLOCKS[$i]} ]] && break [[ $startIter != 0 && $i -le $startIter ]] && continue +# [[ ${TXBLOCKS[$i]} -lt 5243 ]] && continue #kopa sleep 0.05 # Récupère la date et l'objet transaction du bloc courant @@ -39,7 +40,7 @@ loopWalletUp() { # Récupération de la date du block blockDate=$(echo "$blocBrut" | head -n1) blockDate=$(date -d"@$blockDate" +%y-%m-%d -u) - local walletsReceivedToday=$(echo "$blocBrut" | tail -n +2) + local walletsReceivedBloc=$(echo "$blocBrut" | tail -n +2) # Si cette itération proviens d'un cache, alors on aggrège la dernière date if [[ $startBloc != 0 ]]; then @@ -62,15 +63,16 @@ loopWalletUp() { # Sélectionne les ligne SIG en retirant les issuers OIFS="$IFS" IFS='{' - for j in $walletsReceivedToday; do + for j in $walletsReceivedBloc; do [[ -z $j ]] && continue - wIssuers=$(echo "{$j" | jq -r '.issuers[0]') - walletsSentTodayTmp+=$wIssuers$'\n' - walletsReceivedTodayTmp+=$(echo "{$j" | jq -r '.outputs[]' | grep -v "$wIssuers")$'\n' + local wIssuers=$(echo "{$j" | jq -r '.issuers[0]') + local walletsReceivedTodayTmp+=$(echo "{$j" | jq -r '.outputs[]' | grep -v "$wIssuers")$'\n' + local sumRBloc=$(echo "{$j" | jq -r '.outputs[]' | grep -v "$wIssuers" | awk -F: '{ print $1 }' | awk '{s+=$1} END {print s}') + [[ $sumRBloc ]] && local walletsSentTodayTmp+=$wIssuers'|'$sumRBloc'\n' done IFS="$OIFS" - walletsSentToday=$(echo -e "$walletsSentTodayTmp" | head -n -1) - walletsReceivedToday=$(echo -e "$walletsReceivedTodayTmp" | head -n -1) + local walletsSentToday=$(echo -e "$walletsSentTodayTmp" | head -n -1) + local walletsReceivedBloc=$(echo -e "$walletsReceivedTodayTmp" | head -n -1) unset walletsReceivedTodayTmp walletsSentTodayTmp # Exporte les valeurs de la journée dans le fichier JSON @@ -90,14 +92,14 @@ loopWalletUp() { # Si le date du bloc courant est différente du bloc précedent, alors on stock les valeurs journalières if [[ -n $blockDateLast && $blockDateLast != $blockDate ]]; then jsonify - unset valueReceivedM valueReceivedW + unset valueReceivedM valueReceivedW valueSentM valueSentW fi # Sauvegarde la date de ce bloc pour l'itération suivante blockDateLast=$blockDate # Ajoute la valeur des transactions reçus de ce bloc au reste de la journée - for k in $walletsReceivedToday; do + for k in $walletsReceivedBloc; do local pubkey=$(echo $k | awk -F '(' '{ print $2 }' | tr -d ')') local value=$(echo $k | awk -F: '{ print $1 }') @@ -112,8 +114,8 @@ loopWalletUp() { # Ajoute la valeur des transactions envoyés de ce bloc au reste de la journée for k in $walletsSentToday; do - local pubkey=$(echo $k | awk -F '(' '{ print $2 }' | tr -d ')') - local value=$(echo $k | awk -F: '{ print $1 }') + local pubkey=$(echo $k | awk -F '|' '{ print $1 }') + local value=$(echo $k | awk -F '|' '{ print $2 }') if [[ $(echo "$MEMBERS" | grep $pubkey) ]]; then valueSentM=$(($valueSentM+$value)) @@ -125,7 +127,7 @@ loopWalletUp() { done # Stock les clés publiques de ce bloc dans la variable $wallets - wallets+=$(echo "$walletsReceivedToday" | grep -Eo $REGEX_PUBKEYS)$'\n' + wallets+=$(echo "$walletsReceivedBloc" | grep -Eo $REGEX_PUBKEYS)$'\n' # Affiche la progression de la boucle si on est pas en mode web progress=$(echo "scale=1; $bloc*100/$lastBloc/1" | bc) @@ -235,43 +237,26 @@ web() { chown -R www-data:www-data $WEBPATH } -getSolde(){ - pubkey=$@ - solde=0 - txInT=0 - txOutT=0 - nonConsumedUDT=0 - nonConsumedUD=0 - for i in $pubkeys; do - until txOutL=$(curl -s "$ESNODE/g1/movement/_search?filter_path=hits.hits._source&size=10000&q=issuer:$i&pretty"); do - echo "Erreur: $i" - sleep 1 - done - - if [[ $1 == "mbr" ]]; then - nonConsumedUD=$(curl -s ${DUNITER}/ud/history/$i | jq '.history.history[].amount' | awk '{s+=$1} END {print s}') || nonConsumedUD=0 - [[ -z $nonConsumedUD ]] && nonConsumedUD=0 - nonConsumedUDT=$(echo -e "scale=2; ($nonConsumedUD/100)+$nonConsumedUDT" | bc) - fi - - [[ $txOutL != "{ }" ]] && txOut=$(echo "$txOutL" | jq '.hits.hits[]._source.amount' | awk '{s+=$1} END {print s}') || txOut=0 - - solde=$(echo -e "scale=2; (($txIn-$txOut+$nonConsumedUD)/100)+$solde" | bc) - txOutT=$(echo -e "scale=2; ($txOut/100)+$txOutT" | bc) - - done -} - sumSoldes() { - sumWBrut=$(jq -r '.[].rWallets' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') - sumMBrut=$(jq -r '.[].rMembres' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') - sumTBrut=$(jq -r '.[] | .rWallets, .rMembres' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumRWBrut=$(jq -r '.[].rWallets' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumRMBrut=$(jq -r '.[].rMembres' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumRTBrut=$(jq -r '.[] | .rWallets, .rMembres' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumSWBrut=$(jq -r '.[].sWallets' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumSMBrut=$(jq -r '.[].sMembres' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') + sumSTBrut=$(jq -r '.[] | .sWallets, .sMembres' $1 | awk '{ SUM += $1} END { printf "%.2f", SUM }') - sumW=$(echo $sumWBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') - sumM=$(echo $sumMBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') - sumT=$(echo $sumTBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') + sumRW=$(echo $sumRWBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') + sumRM=$(echo $sumRMBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') + sumRT=$(echo $sumRTBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') + sumSW=$(echo $sumSWBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') + sumSM=$(echo $sumSMBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') + sumST=$(echo $sumSTBrut | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') - echo "Wallets: $sumW" - echo "Membres: $sumM" - echo "Total: $sumT" + echo "Reçus Wallets: $sumRW" + echo "Reçus Membres: $sumRM" + echo "Reçus Total: $sumRT" + echo "" --- + echo "Envoyé Wallets: $sumRW" + echo "Envoyé Membres: $sumRM" + echo "Envoyé Total: $sumRT" } diff --git a/g1-stats.sh b/g1-stats.sh index e2426dc..cb225be 100755 --- a/g1-stats.sh +++ b/g1-stats.sh @@ -19,8 +19,8 @@ indexhtml="$WEBPATH/index.html" echo -e "\n############# $day à $startTime #############\n" [[ -z $(which jq) || -z $(which bc) ]] && apt update && apt install jq bc echo "Initialisation ..." -#TXBLOCKS=$(cat $SCRIPTPATH/db/txblocs) -TXBLOCKS=$(curl -s ${DUNITER}/blockchain/with/tx | jq '.result.blocks[]') +TXBLOCKS=$(cat $SCRIPTPATH/db/txblocs) +#TXBLOCKS=$(curl -s ${DUNITER}/blockchain/with/tx | jq '.result.blocks[]') TXBLOCKS=($(echo "$TXBLOCKS" | sort -hu | awk '{printf $1" "}')) unset 'TXBLOCKS[${#TXBLOCKS[@]}-1]' MEMBERS=$(curl -s ${DUNITER}/wot/members | jq -r '.results[].pubkey') @@ -68,34 +68,12 @@ done ##TODO: Get Solde from JSON Sum for next ... kopa -getSolde $simpleWallets - -txInSimple=$(echo $txInT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') -txOutSimple=$(echo $txOutT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') -soldeSimpleBrut=$(echo $solde) -soldeSimple=$(echo $soldeSimpleBrut | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') -echo -e "Reçus simples wallets:\t $txInSimple" -echo -e "Envoyé simples wallets:\t $txOutSimple" -echo -e "Soldes simples wallets:\t $soldeSimple" -echo -e "\n ---\n" -echo "Récupération du solde des membres..." - -getSolde ${MEMBERS[@]} - -txInMembers=$(echo $txInT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') -txOutMembers=$(echo $txOutT | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') -soldeMembersBrut=$(echo $solde) -soldeMembers=$(echo $soldeMembersBrut | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') -echo -e "Reçus membres:\t $txInMembers" -echo -e "Envoyé membres:\t $txOutMembers" -echo -e "Soldes membres:\t $soldeMembers" -echo -e "Total DU non consumés:\t $nonConsumedUDT" - ### Calcul de la masse monétaire et du pourcentage de Ḡ1 sur les simples portefeuilles ### monetaryMass=$(curl -s ${DUNITER}/blockchain/current | jq .monetaryMass) monetaryMassBrut=$(echo -e "scale=2; ($monetaryMass/100)/1" | bc) -monetaryMass=$(echo "$monetaryMassBrut" | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') -soldeWalletMembers=$(echo "scale=2; $soldeMembersBrut+$soldeSimpleBrut" | bc | tr . , | sed ':a;s/\B[0-9]\{3\}\>/.&/;ta') +monetaryMass=$(echo "$monetaryMassBrut" | tr . , | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') + +soldeWalletMembers=$(echo "scale=2; $soldeMembersBrut+$soldeSimpleBrut" | bc | tr . , | sed ':a;s/\B[0-9]\{3\}\>/ &/;ta') pourcentSimpleWallet=$(echo "scale=1; $soldeSimpleBrut*100/$monetaryMassBrut/1" | bc) echo -e "\n ---\n" diff --git a/restore-bk.sh b/restore-bk.sh index 1e68bca..6e0ed3a 100755 --- a/restore-bk.sh +++ b/restore-bk.sh @@ -1,8 +1,8 @@ #!/bin/bash -cacheFile=$(ls cache/bk-walletsUp-*) +cacheFile=$(ls cache/bk-walletsUp-* | tail -n1) cacheID=$(echo $cacheFile | awk -F '-' '{ print $NF }') cp $cacheFile cache/walletsUp-$cacheID -cp db/bk-recus.json db/recus.json +cp db/bk-recus.json-$cacheID db/recus.json