astrXbian_UX-et-webUI/actions/p4n.swarmscraper.sh

170 lines
9.7 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
########################################################################
# Author: Astrocrew
# Version: 0.1
# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/)
########################################################################
MY_PATH="`dirname \"$0\"`" # relative
MY_PATH="`( cd \"$MY_PATH\" && pwd )`" # absolutized and normalized
ME="${0##*/}"
countMErunning=$(ps auxf --sort=+utime | grep -w $ME | grep -v -E 'color=auto|grep' | wc -l)
[[ $countMErunning -gt 2 ]] && echo "$ME already running $countMErunning time" && exit 0
YOU=$(ps auxf --sort=+utime | grep -w ipfs | grep -v -E 'color=auto|grep' | tail -n 1 | cut -d " " -f 1) && [[ ! $YOU ]] && echo "ipfs NOT RUNNING. EXIT" && exit 1
G1PUB=$(cat ~/.zen/secret.dunikey | grep 'pub:' | cut -d ' ' -f 2) && [[ ! $G1PUB ]] && echo "NO ~/.zen/secret.dunikey EXIT" && exit 1
IPFSNODEID=$(ipfs --timeout=20s id -f='<id>\n' 2>/dev/null) && [[ $IPFSNODEID == "" ]] && echo 'NO IPFS RUNNING. EXIT' && exit 1
[ ! -d ~/.zen/ipfs/.$IPFSNODEID/astroport ] && echo 'ASTROPORT NOT READY' && exit 1
## INSTALL MISSING PACKAGES
if [[ -f /etc/sudoers.d/apt ]]; then
for i in chromium nmap httrack imagemagick; do
if [ $(dpkg-query -W -f='${Status}' $i 2>/dev/null | grep -c "ok installed") -eq 0 ]; then
sudo apt install -y $i
fi
done
else
echo "No sudo access for apt install... Please correct situation"
echo "PUT : $USER ALL=(ALL) NOPASSWD:/usr/bin/apt INTO /etc/sudoers.d/apt (use sudo visudo)"
exit 1
fi
##############################################################"
## Backing up europa p4n collected spots from collective intelligence ##
# SWARM is collectively PLAYing bataille navale
# Randomly get geaodata in south europe
# FOCUS on SOUTH EUROPE
# LON = -10 to 30
# LAT = 35 to 60
# https://fr.mapsofworld.com/lat_long/europe.html
##############################################################"
# initial first spot.
lon=0
lat=40
try=1
# Randomly choose spot from in RANGE $lat.$lon
while [[ $(ls ~/.zen/ipfs_swarm/.*/astroport/map/$lat.$lon/p4n/spot.json 2>/dev/null) ]]; do
echo "astroport/$lat.$lon geodata already swarm captured. CONTINUE"
FLOOR=-5; #-10;
CEILING=10; #30
RANGE=$(($CEILING-$FLOOR+1));
RESULT=$RANDOM;
let "RESULT %= $RANGE";
lon=$(($RESULT+$FLOOR));
#echo $lon
FLOOR=40; #35;
CEILING=50; #60;
RANGE=$(($CEILING-$FLOOR+1));
RESULT=$RANDOM;
let "RESULT %= $RANGE";
lat=$(($RESULT+$FLOOR));
#echo $lat
try=$(($try+1));
[ $try -eq 99 ] && break ## tirages aléatoire maximum
done
##############################################################"
## $lat.$lon GEODATA NOT SCRAPED YET ?
##############################################################"
if [ ! -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/ ]; then
### ADD Spot randomness
# Fuzzle geoloc
one=$(echo "000$RANDOM$RANDOM$RANDOM" | cut -b1-12)
latitude="$lat.$one"
two=$(echo "000$RANDOM$RANDOM$RANDOM" | cut -b1-12)
longitude="$lon.$two"
### SCRAPE P4N
## curl spot.json into /astroport/map/$lat.$lon/p4n/spot.json
echo "GETTING /astroport/map/$lat.$lon/p4n/spot.json"
mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/
echo "https://www.park4night.com/services/V3/getLieuxAroundMeLite.php?latitude=$latitude&longitude=$longitude"
curl -s "https://www.park4night.com/services/V3/getLieuxAroundMeLite.php?latitude=$latitude&longitude=$longitude" -o ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json
echo "SAVED ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json"
## TODO REFRESH OLD spot.json
fi
##############################################################"
## GEODATA. continue. Let's extract lieu pdf from my spot
##############################################################"
geosrc=$(ls ~/.zen/{ipfs,ipfs_swarm}/.*/astroport/map/$lat.$lon/p4n/spot.json 2>/dev/null)
geodata=$(echo "$geosrc" | tail -n 1)
ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2)
if [[ "$ipfsnodeid" != "$IPFSNODEID" ]]; then
echo "This spot.json is managed by
$geosrc"
echo "getting geodata from one of mine"
geodata=$(ls ~/.zen/ipfs/.*/astroport/map/*.*/p4n/spot.json | shuf | tail -n 1)
fi
[[ $geodata ]] && ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2) || exit 1
## SUB-SCRAPE SOME of my p4n/spot.json RANDOM "lieu"
echo "SUB-SCRAPING lieu FROM $geodata"
loop=1
while [ $loop -lt 21 ]; do
#{"id":"208264","latitude":"40.730727","longitude":"0.367303","hauteur_limite":"0.00","publique":"1","top_liste":"0","code":"APN","validation_admin":"1","distance":"97.341"}
# flatten json
lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
## GET RANDOM LIEU ID AND TRY MORE TO GET ( prefered APN)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$code" != "APN" ]] && [ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$code" != "APN" ]] && [ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$code" != "APN" ]] && [ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$lid" == "" ]] && echo "ERROR LIEU NO ID. CONTINUE" && loop=$(($loop+1)) && continue
mkdir -p /tmp/$lid && cd /tmp/$lid
echo "Downloading https://www.park4night.com/?page=lieu&id=$lid&bulle=YES"
mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code
echo "id,latitude,longitude,hauteur_limite,publique,top_liste,code,validation_admin,distance" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt
echo "$lieu" >> ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt
cat ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt
# Get 1st copy bulle.pdf... Now converting to index.html redirect to $lid.jpg
[ -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ] && mv ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ./output.pdf && echo "ancien fichier bulle.pdf"
[ ! -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html ] && echo "running chromium... " && chromium --timeout=10000 --headless --no-sandbox --disable-gpu --print-to-pdf=output.pdf "https://www.park4night.com/?page=lieu&id=$lid&bulle=YES" 2>/dev/null
[ ! -f ./output.pdf ] && rm -Rf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid && echo "## CHROMIUM TIMEOUT, CANCEL MISSION" && loop=$(($loop+1)) && continue
## PDF convert and crop to JPG / ipfs add / index.html for redirect
if [ -f ./output.pdf ]; then
convert -density 200 -quality 80 ./output.pdf ./bulle.jpg
convert -crop 1260x420+55+30 ./bulle.jpg ./$lid.jpg
IPFS=$(ipfs add -wq "./$lid.jpg") && echo "## Added $lid.pdf to IPFS: $IPFS"
IPFSREPFILEID=$(echo $IPFS | cut -d ' ' -f 2) && echo "## Repertory $IPFSREPFILEID"
[[ "$IPFSREPFILEID" == "" ]] && echo "IPFS ADD TROUBLE. CONTINUE" && rm -Rf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid && rm -Rf /tmp/$lid && loop=$(($loop+1)) && continue
## Create index.html with Redirect to /ipfs/$IPFSREPFILEID/$lid.jpg
echo "<html><head><meta http-equiv=\"Refresh\" content=\"0; URL=/ipfs/$IPFSREPFILEID/$lid.jpg\"></head></html>" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html
# mv ./output.pdf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf
rm ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf 2>/dev/null
echo "NOW LIEU redirect to http://127.0.0.1:8181/ipfs/$IPFSREPFILEID/$lid.jpg"
rm ./$lid.jpg
rm ./bulle.jpg
fi
cd -
sleep $((1 + RANDOM % 2))
rm -Rf /tmp/$lid
loop=$(($loop+1));
done
## CHECK SWARM GEODATA MAP
# ls -d ~/.zen/ipfs_swarm/.*/astroport/map/*/ | rev | cut -d '/' -f 2 | rev | sort
## NB OF lieu: ls -d ~/.zen/ipfs_swarm/.*/astroport/map/p4n/lieu/*/ | rev | cut -d '/' -f 2 | rev | sort | wc
# ipfs cat /ipns/$IPFSNODEID/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json | jq
exit 0