astrXbian/actions/p4n.swarmscraper.sh

163 lines
9.1 KiB
Bash
Executable File
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
########################################################################
# Author: Astrocrew
# Version: 0.1
# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/)
########################################################################
MY_PATH="`dirname \"$0\"`" # relative
MY_PATH="`( cd \"$MY_PATH\" && pwd )`" # absolutized and normalized
ME="${0##*/}"
countMErunning=$(ps auxf --sort=+utime | grep -w $ME | grep -v -E 'color=auto|grep' | wc -l)
[[ $countMErunning -gt 2 ]] && echo "$ME already running $countMErunning time" && exit 0
YOU=$(ps auxf --sort=+utime | grep -w ipfs | grep -v -E 'color=auto|grep' | tail -n 1 | cut -d " " -f 1) && [[ ! $YOU ]] && echo "ipfs NOT RUNNING. EXIT" && exit 1
G1PUB=$(cat ~/.zen/secret.dunikey | grep 'pub:' | cut -d ' ' -f 2) && [[ ! $G1PUB ]] && echo "NO ~/.zen/secret.dunikey EXIT" && exit 1
IPFSNODEID=$(ipfs --timeout=20s id -f='<id>\n' 2>/dev/null) && [[ $IPFSNODEID == "" ]] && echo 'NO IPFS RUNNING. EXIT' && exit 1
[ ! -d ~/.zen/ipfs/.$IPFSNODEID/astroport ] && echo 'ASTROPORT NOT READY' && exit 1
exit 0
## INSTALL MISSING PACKAGES
if [[ -f /etc/sudoers.d/apt ]]; then
for i in chromium nmap httrack imagemagick; do
if [ $(dpkg-query -W -f='${Status}' $i 2>/dev/null | grep -c "ok installed") -eq 0 ]; then
sudo apt install -y $i
fi
done
else
echo "No sudo access for apt install... Please correct situation"
echo "PUT : $USER ALL=(ALL) NOPASSWD:/usr/bin/apt INTO /etc/sudoers.d/apt (use sudo visudo)"
exit 1
fi
##############################################################"
## Backing up europa p4n collected spots from collective intelligence ##
# SWARM is collectively PLAYing bataille navale
# Randomly get geaodata in south europe
# FOCUS on SOUTH EUROPE
# LON = -10 to 30
# LAT = 35 to 60
# https://fr.mapsofworld.com/lat_long/europe.html
##############################################################"
# initial first spot.
lon=0
lat=40
try=1
# Randomly choose spot from in RANGE $lat.$lon
while [[ $(ls ~/.zen/ipfs_swarm/.*/astroport/map/$lat.$lon/p4n/spot.json 2>/dev/null) ]]; do
echo "astroport/$lat.$lon geodata already swarm captured. CONTINUE"
FLOOR=-5; #-10;
CEILING=10; #30
RANGE=$(($CEILING-$FLOOR+1));
RESULT=$RANDOM;
let "RESULT %= $RANGE";
lon=$(($RESULT+$FLOOR));
#echo $lon
FLOOR=40; #35;
CEILING=50; #60;
RANGE=$(($CEILING-$FLOOR+1));
RESULT=$RANDOM;
let "RESULT %= $RANGE";
lat=$(($RESULT+$FLOOR));
#echo $lat
try=$(($try+1));
[ $try -eq 99 ] && break ## tirages aléatoire maximum
done
##############################################################"
## $lat.$lon GEODATA NOT SCRAPED YET ?
##############################################################"
if [ ! -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/ ]; then
### ADD Spot randomness
# Fuzzle geoloc
one=$(echo "000$RANDOM$RANDOM$RANDOM" | cut -b1-12)
latitude="$lat.$one"
two=$(echo "000$RANDOM$RANDOM$RANDOM" | cut -b1-12)
longitude="$lon.$two"
### SCRAPE P4N
## curl spot.json into /astroport/map/$lat.$lon/p4n/spot.json
echo "GETTING /astroport/map/$lat.$lon/p4n/spot.json"
mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/
echo "https://www.park4night.com/services/V3/getLieuxAroundMeLite.php?latitude=$latitude&longitude=$longitude"
curl -s "https://www.park4night.com/services/V3/getLieuxAroundMeLite.php?latitude=$latitude&longitude=$longitude" -o ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json
echo "SAVED ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json"
## TODO REFRESH OLD spot.json
fi
##############################################################"
## GEODATA. continue. Let's extract lieu pdf from my spot
##############################################################"
geodata=$(ls ~/.zen/{ipfs,ipfs_swarm}/.*/astroport/map/$lat.$lon/p4n/spot.json 2>/dev/null | tail -n 1)
ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2)
if [[ "$ipfsnodeid" != "$IPFSNODEID" ]]; then
echo "This spot.json is already managed by $ipfsnodeid"
echo "getting geodata from one of mine"
geodata=$(ls ~/.zen/ipfs/.*/astroport/map/*.*/p4n/spot.json | shuf | tail -n 1)
fi
[[ $geodata ]] && ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2) || exit 1
## SUB-SCRAPE SOME of my p4n/spot.json RANDOM "lieu"
echo "SUB-SCRAPING lieu FROM $geodata"
loop=1
while [ $loop -lt 21 ]; do
#{"id":"208264","latitude":"40.730727","longitude":"0.367303","hauteur_limite":"0.00","publique":"1","top_liste":"0","code":"APN","validation_admin":"1","distance":"97.341"}
# flatten json
lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
## GET RANDOM LIEU ID AND TRY MORE TO GET ( prefered APN)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$code" != "APN" ]] && [ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$code" != "APN" ]] && [ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$code" != "APN" ]] && [ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[ -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
[[ "$lid" == "" ]] && continue
mkdir -p /tmp/$lid && cd /tmp/$lid
echo "Downloading https://www.park4night.com/?page=lieu&id=$lid&bulle=YES"
mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code
echo "id,latitude,longitude,hauteur_limite,publique,top_liste,code,validation_admin,distance" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt
echo "$lieu" >> ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt
# Get 1st copy bulle.pdf... Now converting to index.html redirect to $lid.jpg
[ -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ] && mv ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ./output.pdf
[ ! -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html ] && chromium --timeout=30000 --headless --no-sandbox --disable-gpu --print-to-pdf=output.pdf "https://www.park4night.com/?page=lieu&id=$lid&bulle=YES" 2>/dev/null
[ ! -f ./output.pdf ] && rm -Rf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ## TIMEOUT, CANCEL MISSION
## PDF convert and crop to JPG / ipfs add / index.html for redirect
if [ -f ./output.pdf ]; then
convert -density 300 -quality 80 ./bulle.pdf ./bulle.jpg
convert -crop 1260x420+55+30 ./bulle.jpg ./$lid.jpg
IPFS=$(ipfs add -wq "./$lid.jpg") ## Add $lid.pdf to IPFS
IPFSREPFILEID=$(echo $IPFS | cut -d ' ' -f 2) ## Get repertory ipfs link
## Create index.html with Redirect to /ipfs/$IPFSREPFILEID/$lid.jpg
echo "<html><head><meta http-equiv=\"Refresh\" content=\"0; URL=/ipfs/$IPFSREPFILEID/$lid.jpg\"></head></html>" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html
# mv ./output.pdf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf
rm "~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf" 2>/dev/null
echo "NOW LIEU redirect to /ipfs/$IPFSREPFILEID/$lid.jpg"
fi
sleep $((1 + RANDOM % 2))
cd - && rm -Rf /tmp/$lid
loop=$(($loop+1));
done
## CHECK SWARM GEODATA MAP
# ls -d ~/.zen/ipfs_swarm/.*/astroport/map/*/ | rev | cut -d '/' -f 2 | rev | sort
## NB OF lieu: ls -d ~/.zen/ipfs_swarm/.*/astroport/map/p4n/lieu/*/ | rev | cut -d '/' -f 2 | rev | sort | wc
# ipfs cat /ipns/$IPFSNODEID/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json | jq
exit 0