From 8c8e2a88746c47c71959555a0386b385f9ea8caf Mon Sep 17 00:00:00 2001 From: fred Date: Fri, 9 Sep 2022 13:23:11 +0200 Subject: [PATCH] Export full IPFS --- story.fr.txt | 9 +++ tools/TW.export.sh | 26 +++----- tools/duniter_getnode.sh | 132 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 18 deletions(-) mode change 100644 => 100755 tools/TW.export.sh create mode 100755 tools/duniter_getnode.sh diff --git a/story.fr.txt b/story.fr.txt index e0f24fd4..4fd06b53 100644 --- a/story.fr.txt +++ b/story.fr.txt @@ -28,3 +28,12 @@ Eau chaude Puit canadien Compost Papin Mur en canettes + + +Astroport forme une strucutre informationnelle chainées faites de "TiddlyWiki" (TW) publiés sur IPFS. +Chaque soir à 20h12, Astro se met en marche pour + +- analyser les "tiddlers" du TW "Astronaute" +- repérer les amis dans IPFS (ayant échangé leur "Dessin de Moa") et récupérer leur TW. +- enfin opérer les enrichissements informationnels selon les canaux partagés et les règles établies (CrossTagging, PIN contract, Web Scrapping, etc ...) + diff --git a/tools/TW.export.sh b/tools/TW.export.sh old mode 100644 new mode 100755 index eefb6442..d477d110 --- a/tools/TW.export.sh +++ b/tools/TW.export.sh @@ -26,7 +26,7 @@ done echo "${vlist[@]}" -PS3='Choisissez le voeux ___ ' +PS3='Choisissez le TW à exporter ___ ' voeux=($(ls ~/.zen/game/players/$PLAYER/voeux 2>/dev/null) "QUITTER") select voeu in "${vlist[@]}"; do @@ -35,29 +35,19 @@ select voeu in "${vlist[@]}"; do exit 0 ;; - *) echo "IMPRESSION $voeu" + *) echo "OK pour $voeu" voeu=$(echo $voeu | cut -d ':' -f2) ## Get G1PUB part - myIP=$(hostname -I | awk '{print $1}' | head -n 1) - VOEUXNS=$(ipfs key list -l | grep $voeu | cut -d ' ' -f1) + VOEUNS=$(ipfs key list -l | grep -w $voeu | cut -d ' ' -f1) + echo "/ipns/$VOEUNS" - qrencode -s 12 -o "$HOME/.zen/game/world/$voeu/QR.WISHLINK.png" "http://$myIP:8080/ipns/$VOEUXNS" - convert $HOME/.zen/game/world/$voeu/QR.WISHLINK.png -resize 600 ~/.zen/tmp/QRWISHLINK.png - TITLE=$(cat ~/.zen/game/world/$voeu/.pepper) ## Get Voeu title (pepper) = simple GUI form + Name collision => Voeu fusion - convert -gravity northwest -pointsize 40 -fill black -draw "text 50,2 \"$TITLE\"" ~/.zen/tmp/QRWISHLINK.png ~/.zen/tmp/g1voeu1.png - convert -gravity southeast -pointsize 40 -fill black -draw "text 50,2 \"$TITLE\"" ~/.zen/tmp/g1voeu1.png ~/.zen/tmp/g1voeu.png + ipfs --timeout 12s cat /ipns/$VOEUNS > ~/.zen/tmp/index.html + [[ ! -s ~/.zen/tmp/index.html ]] && echo "TIMEOUT ERROR" && break - echo " QR code $TITLE : http://$myIP:8080/ipns/$VOEUXNS" + tiddlywiki --load ~/.zen/tmp/index.html --output ~/.zen/tmp --render '.' 'tiddlers.json' 'text/plain' '$:/core/templates/exporters/JsonFile' 'exportFilter' '[tag[ipfs]]' ## TUBE.copy.sh Tiddlers - LP=$(ls /dev/usb/lp* | head -n1) - [[ ! $LP ]] && echo "NO PRINTER FOUND - Brother QL700 validated" && continue - - echo "IMPRESSION LIEN TW VOEU" - brother_ql_create --model QL-700 --label-size 62 ~/.zen/tmp/g1voeu.png > ~/.zen/tmp/toprint.bin 2>/dev/null - sudo brother_ql_print ~/.zen/tmp/toprint.bin $LP + cat ~/.zen/tmp/tiddlers.json | jq -r ;; esac done - -## TODO EXPORT TW (LIGHT / HEAVY) diff --git a/tools/duniter_getnode.sh b/tools/duniter_getnode.sh new file mode 100755 index 00000000..e92ab565 --- /dev/null +++ b/tools/duniter_getnode.sh @@ -0,0 +1,132 @@ +#!/bin/bash +################################################################################ +# Authors: @jytou (https://git.duniter.org/jytou) +# Modified by Fred (support@qo-op.com) +# Version: 0.1 +# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/) +################################################################################ +# Checks the current block number of $DIR/duniter_nodes.txt (is run in parallel) +# and output random (from last synchronized) node + +checkonenode() +{ + # Timeout in seconds for https nodes + httpsTimeout=1 + # Timeout in seconds for http nodes + httpTimeout=1 + + node=$1 + watched=$2 + outfile=$3 + # Curl: -m timeout, -k ignore SSL certificate errors + cur=`echo "$( { curl -m $httpsTimeout -k https://$node/blockchain/current; } 2>&1 )"` + n=$node + if [[ "$cur" != *issuersFrameVar* ]] + then + # It failed in https, maybe try http? + cur=`echo "$( { curl -m $httpTimeout http://$node/blockchain/current; } 2>&1 )"` + if [[ "$cur" == *issuersFrameVar* ]] + then + # Indicate that the node is http + n="$n-(http)" + fi + fi + if [[ "$cur" != *issuersFrameVar* ]] + then + # The node didn't respond on time + cur="ERROR" + else + # The node did respond - grab the block number and hash of the block as key + cur="`echo "$cur"|grep '^ "number": '|awk '{print $2}'|awk -F, '{print $1}'`-`echo "$cur"|grep '^ "hash": '|awk '{print $2}'|awk '{print substr($1,2,13)}'`" + fi + if [[ $watched =~ .*#$node#.* ]] + then + # The node is a watched node, add some bold + n="\e[1m$n\e[0m" + fi + # Put the result into the file + echo "$cur $n">$outfile + # Notify that we're done here + touch $outfile.done +} + +# Temp dir where results are stored +rm -Rf /tmp/zen/gnodewatch +DIR=/tmp/zen/gnodewatch +export DIR +mkdir -p $DIR/chains +# TODO: REMOVE 777 PATCH, ACTIVATE ramdisk +# sudo mkdir /mnt/ramdisk +# sudo mount -t tmpfs -o size=50m tmpfs /mnt/ramdisk +chmod -R 777 /tmp/zen/ + +# KEEP /tmp/zen/current.duniter for 5 mn +find /tmp/zen/ -mmin +5 -type f -name "current.duniter" -exec rm -f '{}' \; +[[ -f /tmp/zen/current.duniter ]] && cat /tmp/zen/current.duniter && exit 0 + +##### $DIR/duniter_nodes.txt REFRESH after 20 minutes ##### +find $DIR/ -mmin +20 -type f -name "duniter_*" -exec rm -f '{}' \; +if [[ ! -f $DIR/duniter_nodes.txt ]]; then + # Get New BMAS known Nodes list from shuffle one $DIR/good.nodes.txt + [[ -f $DIR/good.nodes.txt ]] && DUNITER=$(shuf -n 1 $DIR/good.nodes.txt) || DUNITER="duniter-g1.p2p.legal:443" + curl -s https://$DUNITER/network/peers | jq '.peers[] | .endpoints' | grep BMAS | awk '{print $2,$3}' | sed s/\"//g | sed s/\,//g | sed s/\ /:/g | sort -u > $DIR/duniter_nodes.txt +fi + +# Grab the nodes we are actively watching - they will be in bold in the final output +watched=`grep -v "#" $DIR/duniter_nodes.txt|egrep "\!$"|awk '{print "#" $1 "#"}'` +# All nodes we are watching +nodes=`grep -v "#" $DIR/duniter_nodes.txt|awk '{print $1}'` +# The index to generate separate file names +index=0 +# Wipe out the output directory +rm $DIR/*out $DIR/*done $DIR/chains/* $DIR/NODE.* 2>/dev/null + +# Query all nodes in parallel +for node in $nodes +do + checkonenode $node "$watched" $DIR/$index.out & + ((index++)) +done + +# Wait a little for the first files to be created +sleep 1s +# Wait for all the threads to report they are done +while [ `ls $DIR/*done|wc -l` -lt $index ] +do + sleep 1s +done + +# Grab all results +curs=`cat $DIR/*out|sort` +# Extract all forks, excluding all errors +chains="`echo "$curs"|grep -v ERROR|awk '{print $1}'|sort -r|uniq`" + +# Count the number of chains and output most recent consensus to "good.nodes.txt" +nb=0 +for chain in $chains +do + echo "$curs" | egrep "^$chain " | awk '{print $2}' >> $DIR/chains/$nb; + ((nb++)) +done + +longchain=$(ls -S $DIR/chains/ | head -n 1) +# WRITE OUT shuffle Duniter Node Sync with longest chain +cp $DIR/chains/$longchain $DIR/good.nodes.txt + +## TEST if server is really running Duniter +Dtest=""; IDtest=""; lastresult=""; loop=0 +while [[ $Dtest != "duniter" ]]; do + while [[ $lastresult == $result && $loop -lt 7 ]]; do result=$(shuf -n 1 $DIR/good.nodes.txt); ((loop++)); done + lastresult=$result + Dtest=$(curl -s https://$lastresult | jq -r .duniter.software) + ## CHECK if server is not too slow + [[ $Dtest == "duniter" ]] && IDtest=$(silkaj -p $lastresult id Fred) + [[ $IDtest == "" ]] && Dtest="" + [[ $loop -eq 8 ]] && result="duniter-g1.p2p.legal:443" && break + ((loop++)) +done + + +echo "$result" > /tmp/zen/current.duniter + +echo $result