Add nbrMembres, nbrWallets and UD in JSON; Improve cache mechanic

This commit is contained in:
poka 2020-09-20 22:32:15 +02:00
parent ae17883a51
commit 2d92a40895
8 changed files with 141 additions and 59 deletions

View File

@ -1,28 +1,38 @@
#!/bin/bash
dateRange=$1
dateRange=$2
lineNbr=$(grep -n "$dateRange" db/recus.json | cut -d : -f 1)
lineNbr=$(($lineNbr+4))
jsonDated=$(head -n$lineNbr db/recus.json)
jsonDated=$(echo -e "$jsonDated\n }\n]")
cumulate() {
lineNbr=$(($lineNbr+6))
sumRWBrut=$(jq -r '.[].rWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumRMBrut=$(jq -r '.[].rMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
#sumRTBrut=$(jq -r '.[] | .rWallets, .rMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSWBrut=$(jq -r '.[].sWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSMBrut=$(jq -r '.[].sMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
#sumSTBrut=$(jq -r '.[] | .sWallets, .sMembres' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
jsonDated=$(head -n$lineNbr db/recus.json)
jsonDated=$(echo -e "$jsonDated\n }\n]")
#echo "$sumRWBrut"
sumRWBrut=$(jq -r '.[].rWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumRMBrut=$(jq -r '.[].rMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
#sumRTBrut=$(jq -r '.[] | .rWallets, .rMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSWBrut=$(jq -r '.[].sWallets' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSMBrut=$(jq -r '.[].sMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
#sumSTBrut=$(jq -r '.[] | .sWallets, .sMembers' <<<$jsonDated | awk '{ SUM += $1} END { printf "%.2f", SUM }')
echo "{
\"date\": "\"$dateRange\"",
\"rWallets\": $sumRWBrut,
\"sWallets\": $sumSWBrut,
\"rMembres\": $sumRMBrut,
\"sMembres\": $sumSMBrut
#echo "$sumRWBrut"
echo "{
\"date\": "\"$dateRange\"",
\"rWallets\": $sumRWBrut,
\"sWallets\": $sumSWBrut,
\"rMembers\": $sumRMBrut,
\"sMembers\": $sumSMBrut
}
" | jq .
}
" | jq .
selectDay() {
lineNbr=$(($lineNbr+7))
jsonDated=$(head -n$lineNbr db/recus.json |tail -n7 | sed 's/},/}/g')
echo "$jsonDated" | jq .
}
$1

View File

@ -7,6 +7,9 @@
# Git: https://git.p2p.legal/axiom-team/g1-stats
################################################################################
# Stop if error
set -e
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
if [[ -e $SCRIPTPATH/.env ]]; then source $SCRIPTPATH/.env; else echo "Veuillez créer votre fichier .env inspiré de .env.example" && exit 1; fi
@ -26,6 +29,7 @@ unset 'TXBLOCKS[${#TXBLOCKS[@]}-1]'
membresPubkeys=$(cat db/membresPubkeys) #kopa
nbrMembers=$(echo "$membresPubkeys" | wc -l)
lastBloc=${TXBLOCKS[-1]}
udBlocs=$(curl -s ${DUNITER}/blockchain/with/ud | jq '.result.blocks[]')
isWeb=$1
### On source les fonctions
@ -33,7 +37,10 @@ source $SCRIPTPATH/lib/functions.sh
source $SCRIPTPATH/lib/scanTxWallets.sh
### Extraction des adresses Ḡ1 actives
scanTxWallets
scanTxWallets $1 #kopa
exit
### Calcul la somme des soldes portefeuilles et membres
sumSoldes

View File

@ -64,12 +64,12 @@ web() {
sumSoldes() {
sumRWBrut=$(jq -r '.[].rWallets' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumRMBrut=$(jq -r '.[].rMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumRTBrut=$(jq -r '.[] | .rWallets, .rMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumRMBrut=$(jq -r '.[].rMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumRTBrut=$(jq -r '.[] | .rWallets, .rMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSWBrut=$(jq -r '.[].sWallets' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
#sumSWBrut=$(jq -n '[inputs | .[].sWallets] | reduce .[] as $num (0; .+$num)' db/recus.json | xargs printf %.2f | tr ',' '.')
sumSMBrut=$(jq -r '.[].sMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSTBrut=$(jq -r '.[] | .sWallets, .sMembres' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSMBrut=$(jq -r '.[].sMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
sumSTBrut=$(jq -r '.[] | .sWallets, .sMembers' db/recus.json | awk '{ SUM += $1} END { printf "%.2f", SUM }')
soldeWBrut=$(echo "$sumRWBrut-$sumSWBrut" | bc)
soldeMBrut=$(echo "$sumRMBrut-$sumSMBrut" | bc)

View File

@ -1,3 +1,5 @@
#!/bin/bash
scanTxWallets() {
local REGEX_PUBKEYS="[a-zA-Z0-9]{42,44}"
@ -6,10 +8,12 @@ scanTxWallets() {
local startFile=$(ls $SCRIPTPATH/cache/walletsUp-* | tail -n1)
local startIter=$(echo "$startFile" | awk -F '-' '{ print $NF }')
local startBloc=$(head -n1 "$startFile")
wallets=$(tail -n +2 "$startFile")$'\n'
wallets=$(echo -e "$wallets" | sort -u | awk 'NF')
wallets=$(tail -n +2 "$startFile" | sed '/---/Q' | sort -u)
local walletsAuj=$(cat "$startFile" | sed -e '1,/---/d' | sed '/---/Q' | grep .)$'\n'
local nbrMembersYesterday=$(cat "$startFile" | sed -e '1,/---/d' | sed -e '1,/---/d' | sed '/---/Q' | grep . | cut -d '|' -f 1)
local blockDateLast=$(cat "$startFile" | sed -e '1,/---/d' | sed -e '1,/---/d' | sed '/---/Q' | grep . | cut -d '|' -f 2)
[[ $lastBloc == $startBloc ]] && return 10
##kopa [[ $lastBloc == $startBloc ]] && return 10
rm $startFile
else
local startIter=0
@ -28,8 +32,7 @@ scanTxWallets() {
# Récupère la date et l'objet transaction du bloc courant
blocFull=$(curl -s ${DUNITER}/blockchain/block/$bloc)
#blocFull=$(cat bloc-267)
until blocBrut=$(echo $blocFull | jq -r '.medianTime, .transactions[]' 2>/dev/null); do
until blocBrut=$(echo $blocFull | jq -r '.' 2>/dev/null); do
[[ $isWeb != "web" ]] && echo -e "i: $i\n$blocFull"
sleep 2
blocFull=$(curl -s ${DUNITER}/blockchain/block/$bloc)
@ -37,28 +40,30 @@ scanTxWallets() {
[[ -z "$blocBrut" ]] && continue
# Récupération de la date du block
blockDate=$(echo "$blocBrut" | head -n1)
blockDate=$(jq '.medianTime' <<<"$blocBrut")
blockDate=$(date -d"@$blockDate" +%y-%m-%d -u)
local walletsReceivedBloc=$(echo "$blocBrut" | tail -n +2)
local walletsReceivedBloc=$(jq '.transactions[]' <<<"$blocBrut")
# Si cette itération proviens d'un cache, alors on aggrège la dernière date
# Si cette itération provient d'un cache, alors on aggrège la dernière date
if [[ $startBloc != 0 ]]; then
lastDate=$(cat db/recus.json | jq -r '.[].date' | tail -n1)
echo -e "IFCACHE\t - $bloc - $blockDate - $lastDate" >> tata #kopa
if [[ "$blockDate" == "$lastDate" ]]; then
valueReceivedW=$(cat db/recus.json | jq -r '.[].rWallets' | tail -n1 | tr -d '.,')
valueReceivedM=$(cat db/recus.json | jq -r '.[].rMembres' | tail -n1 | tr -d '.,')
valueSentW=$(cat db/recus.json | jq -r '.[].sWallets' | tail -n1 | tr -d '.,')
valueSentM=$(cat db/recus.json | jq -r '.[].sMembres' | tail -n1 | tr -d '.,')
valueReceivedW=$(cat db/recus.json | jq -r '.[].rWallets' | tail -n1 | xargs echo 100* | bc)
valueReceivedM=$(cat db/recus.json | jq -r '.[].rMembers' | tail -n1 | xargs echo 100* | bc)
valueSentW=$(cat db/recus.json | jq -r '.[].sWallets' | tail -n1 | xargs echo 100* | bc)
valueSentM=$(cat db/recus.json | jq -r '.[].sMembers' | tail -n1 | xargs echo 100* | bc)
local recusJson=$(head -n -8 db/recus.json)
local recusJson=$(head -n -11 db/recus.json)
echo "$recusJson" > $SCRIPTPATH/db/recus.json
else
local recusJson=$(head -n -4 db/recus.json)
local recusJson=$(head -n -2 db/recus.json)
echo -e "$recusJson\n }," > $SCRIPTPATH/db/recus.json
fi
unset recusJson
#unset recusJson
startBloc=0
fi
# Sélectionne les ligne SIG en retirant les issuers
OIFS="$IFS"
IFS='{'
@ -80,19 +85,67 @@ scanTxWallets() {
[[ -z $valueReceivedM ]] && valueReceivedM=0 || valueReceivedM=$(echo "scale=2; $valueReceivedM/100" | bc)
[[ -z $valueSentM ]] && valueSentM=0 || valueSentM=$(echo "scale=2; $valueSentM/100" | bc)
[[ -z $valueSentW ]] && valueSentW=0 || valueSentW=$(echo "scale=2; $valueSentW/100" | bc)
[[ -z $nbrMembersToday ]] && nbrMembersToday=0
[[ -z $nbrWallets ]] && nbrWallets=0
[[ -z $UDToday ]] && UDToday=0
local jsonTPL=$(sed s/_DATE/$blockDateLast/g $SCRIPTPATH/tpl/recus.json)
local jsonTPL=$(sed s/_UDTODAY/$UDToday/g <<< $jsonTPL)
local jsonTPL=$(sed s/_RWALLETS/$valueReceivedW/g <<< $jsonTPL)
local jsonTPL=$(sed s/_SWALLETS/$valueSentW/g <<< $jsonTPL)
local jsonTPL=$(sed s/_RMEMBRES/$valueReceivedM/g <<< $jsonTPL)
local jsonTPL=$(sed s/_SMEMBRES/$valueSentM/g <<< $jsonTPL)
local jsonTPL=$(sed s/_RMEMBERS/$valueReceivedM/g <<< $jsonTPL)
local jsonTPL=$(sed s/_SMEMBERS/$valueSentM/g <<< $jsonTPL)
local jsonTPL=$(sed s/_NBRMEMBERS/$nbrMembersToday/g <<< $jsonTPL)
local jsonTPL=$(sed s/_NBRWALLETS/$nbrWallets/g <<< $jsonTPL)
echo -e "$jsonTPL" | tr -d '\\' >> $SCRIPTPATH/db/recus.json
}
# Stock les clés publiques de ce bloc dans la variable $wallets
walletsAuj+=$(echo "$walletsReceivedBloc" | grep -Eo $REGEX_PUBKEYS)$'\n'
# Si le date du bloc courant est différente du bloc précedent, alors on stock les valeurs journalières
if [[ -n $blockDateLast && $blockDateLast != $blockDate ]]; then
applyToday() {
[[ -z $wallets ]] && wallets="#"
WALLETS=$(echo -e "$wallets" | grep .)
nbrWallets=$(echo -e "$walletsAuj" | sort -u | awk 'NF' | grep -v "$membresPubkeys" | grep -v "$WALLETS" | wc -l)
# Calcul du nombre de nouveaux membres à ce block
nbrMembersTotalToday=$(jq '.membersCount' <<<"$blocBrut")
[[ -z $nbrMembersYesterday ]] && nbrMembersYesterday=0
nbrMembersToday=$(($nbrMembersTotalToday-$nbrMembersYesterday))
# Récupération de la masse monétaire à ce block
#monetaryMassTotalToday=$(jq '.monetaryMass' <<<"$blocBrut")
#monetaryMassTotalToday=$(echo "scale=2; $monetaryMassTotalToday/100" | bc)
#[[ -z $monetaryMassYesterday ]] && monetaryMassYesterday=0
#monetaryMassToday=$(echo "scale=2; $monetaryMassTotalToday-$monetaryMassYesterday" | bc)
#monetaryMassYesterday=$monetaryMassTotalToday
# Obtention du dividende à ce block
udBlock=$(for i in $udBlocs; do [[ $i -ge $bloc ]] && echo $lastBloc && break; lastBloc=$i; done)
UDToday=$(curl -s ${DUNITER}/blockchain/block/$udBlock | jq '.dividend')
UDToday=$(echo "scale=2; $UDToday/100" | bc)
jsonify
unset valueReceivedM valueReceivedW valueSentM valueSentW
}
if [[ -n $blockDateLast && $blockDateLast != $blockDate ]]; then
if [[ -z $recusJson ]]; then
applyToday
else
# Calcul du nombre de nouveaux membres à ce block
nbrMembersTotalToday=$(jq '.membersCount' <<<"$blocBrut")
[[ -z $nbrMembersYesterday ]] && nbrMembersYesterday=0
nbrMembersToday=$(($nbrMembersTotalToday-$nbrMembersYesterday))
fi
[[ $wallets == "#" ]] && unset wallets
wallets+=$'\n'"$walletsAuj"
wallets=$(echo -e "$wallets" | sort -u | grep .)$'\n'
nbrMembersYesterday=$nbrMembersTotalToday
unset valueReceivedM valueReceivedW valueSentM valueSentW walletsAuj
echo -e "INLOOP\t - $bloc - $blockDateLast - $nbrMembersYesterday" >> tata #kopa
fi
unset recusJson
# Sauvegarde la date de ce bloc pour l'itération suivante
blockDateLast=$blockDate
@ -103,10 +156,14 @@ scanTxWallets() {
local value=$(echo $k | awk -F '|' '{ print $2 }')
if [[ $(echo "$membresPubkeys" | grep $pubkey) ]]; then
valueReceivedM=$(($valueReceivedM+$value))
#valueReceivedM=$(($valueReceivedM+$value))
[[ -z $valueReceivedM ]] && valueReceivedM=0
valueReceivedM=$(echo "$valueReceivedM+$value" | bc)
pubkeyReceiveM+=${pubkey}\\n
else
valueReceivedW=$(($valueReceivedW+$value))
#valueReceivedW=$(($valueReceivedW+$value))
[[ -z $valueReceivedW ]] && valueReceivedW=0
valueReceivedW=$(echo "$valueReceivedW+$value" | bc)
pubkeyReceiveW+=${pubkey}\\n
fi
done
@ -117,17 +174,18 @@ scanTxWallets() {
local value=$(echo $k | awk -F '|' '{ print $2 }')
if [[ $(echo "$membresPubkeys" | grep $pubkey) ]]; then
valueSentM=$(($valueSentM+$value))
#valueSentM=$(($valueSentM+$value))
[[ -z $valueSentM ]] && valueSentM=0
valueSentM=$(echo "$valueSentM+$value" | bc)
pubkeySentM+=${pubkey}\\n
else
valueSentW=$(($valueSentW+$value))
#valueSentW=$(($valueSentW+$value))
[[ -z $valueSentW ]] && valueSentW=0
valueSentW=$(echo "$valueSentW+$value" | bc)
pubkeySentW+=${pubkey}\\n
fi
done
# Stock les clés publiques de ce bloc dans la variable $wallets
wallets+=$(echo "$walletsReceivedBloc" | grep -Eo $REGEX_PUBKEYS)$'\n'
# Affiche la progression de la boucle si on est pas en mode web
progress=$(echo "scale=1; $bloc*100/$lastBloc/1" | bc)
endExec=$(($(date +%s%N)/1000000))
@ -148,19 +206,19 @@ scanTxWallets() {
echo "Scan des blocs avec transactions en cours ..."
fi
# [[ $i -ge 10 ]] && break #kopa
[[ $i -ge $1 ]] && break #kopa
done
jsonify
[[ $blockDateLast == $blockDate ]] && applyToday && echo -e "END\t\t - $bloc - $blockDate - $nbrMembersYesterday" >> tata #kopa
# Retire la dernière virgule et ajoute le crochet de fin de JSON
sed -i '$ s/,//g' $SCRIPTPATH/db/recus.json
echo "]" >> $SCRIPTPATH/db/recus.json
# On supprime les doublons et les lignes vides
wallets=$(echo -e "$wallets" | sort -u | awk 'NF')$'\n'
wallets=$(echo -e "$wallets" | sort -u | awk 'NF')
# On écrit les pubkeys avec transaction dans un fichier de cache pour la prochaine itération
[[ ! -d $SCRIPTPATH/cache ]] && mkdir $SCRIPTPATH/cache
echo -e "$lastBloc\n$wallets" > $SCRIPTPATH/cache/walletsUp-$i
echo -e "$lastBloc\n$wallets\n---\n$walletsAuj\n---\n$nbrMembersYesterday|$blockDateLast" > $SCRIPTPATH/cache/walletsUp-$i
}

View File

@ -23,7 +23,7 @@ rWallets() {
local jsonTPL=$(sed s/_DATE/$dateN/g $SCRIPTPATH/tpl/recus.json)
local jsonTPL=$(sed s/_RWALLETS/$RW/g <<< $jsonTPL)
local result+=$(sed s/_RMEMBRES/$RM/g <<< $jsonTPL)
local result+=$(sed s/_RMEMBERS/$RM/g <<< $jsonTPL)
done
sed "s/_JSONDATA/$result/g" $SCRIPTPATH/tpl/recus.html > $WEBPATH/graph/recus.html
@ -46,7 +46,7 @@ nbrWallets() {
local jsonTPL=$(sed s/_DATE/$dateN/g $SCRIPTPATH/tpl/nbr_wallets.json)
local jsonTPL=$(sed s/_NBRWALLETS/$NBRW/g <<< $jsonTPL)
local result+=$(sed s/_NBRMEMBRES/$NBRM/g <<< $jsonTPL)
local result+=$(sed s/_NBRMEMBERS/$NBRM/g <<< $jsonTPL)
done
sed "s/_JSONDATA/$result/g" $SCRIPTPATH/tpl/nbr_wallets.html > $WEBPATH/graph/nbr_wallets.html

View File

@ -2,4 +2,4 @@
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
jq -r '.[] | "---", .date, .rWallets + .rMembres' $SCRIPTPATH/../db/recus.json > $SCRIPTPATH/../db/somme.txt
jq -r '.[] | "---", .date, .rWallets + .rMembers' $SCRIPTPATH/../db/recus.json > $SCRIPTPATH/../db/somme.txt

View File

@ -2,8 +2,15 @@
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
cacheFile=$(ls $SCRIPTPATH/../cache/bk-walletsUp-* | tail -n1)
cacheID=$(echo $cacheFile | awk -F '-' '{ print $NF }')
if [[ $1 ]]; then
cacheID=$1
cacheFile="$SCRIPTPATH/../cache/bk-walletsUp-$cacheID"
else
cacheFile=$(ls $SCRIPTPATH/../cache/bk-walletsUp-* | tail -n1)
cacheID=$(echo $cacheFile | awk -F '-' '{ print $NF }')
fi
[[ -f $cacheFile ]] && rm $SCRIPTPATH/../cache/walletsUp-*
cp $cacheFile $SCRIPTPATH/../cache/walletsUp-$cacheID
cp $SCRIPTPATH/../db/bk-recus.json-$cacheID $SCRIPTPATH/../db/recus.json

View File

@ -1 +1 @@
{\n \"date\": \"_DATE\",\n \"rWallets\": _RWALLETS,\n \"sWallets\": _SWALLETS,\n \"rMembres\": _RMEMBRES,\n \"sMembres\": _SMEMBRES\n },
{\n \"date\": \"_DATE\",\n \"UD\": _UDTODAY,\n \"rWallets\": _RWALLETS,\n \"sWallets\": _SWALLETS,\n \"rMembers\": _RMEMBERS,\n \"sMembers\": _SMEMBERS,\n \"nbrMembers\": _NBRMEMBERS,\n \"nbrWallets\": _NBRWALLETS\n },