astrXbian/actions/p4n.swarmscraper.sh

#!/bin/bash
########################################################################
# Author: Astrocrew
# Version: 0.1
# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/)
########################################################################
MY_PATH="`dirname \"$0\"`"              # relative
MY_PATH="`( cd \"$MY_PATH\" && pwd )`"  # absolutized and normalized
ME="${0##*/}"
countMErunning=$(ps auxf --sort=+utime | grep -w $ME | grep -v -E 'color=auto|grep' | wc -l)
[[ $countMErunning -gt 2 ]] && echo "$ME already running $countMErunning time" && exit 0

YOU=$(ps auxf --sort=+utime | grep -w ipfs | grep -v -E 'color=auto|grep' | tail -n 1 | cut -d " " -f 1) && [[ ! $YOU ]] && echo "ipfs NOT RUNNING. EXIT" && exit 1
G1PUB=$(cat ~/.zen/secret.dunikey | grep 'pub:' | cut -d ' ' -f 2) && [[ ! $G1PUB ]] && echo "NO ~/.zen/secret.dunikey EXIT" && exit 1
IPFSNODEID=$(ipfs --timeout=20s id -f='<id>\n' 2>/dev/null) && [[ $IPFSNODEID == "" ]] && echo 'NO IPFS RUNNING. EXIT' && exit 1
[ ! -d ~/.zen/ipfs/.$IPFSNODEID/astroport ] &&  echo 'ASTROPORT NOT READY' && exit 1


## INSTALL MISSING PACKAGES
if [[ -f /etc/sudoers.d/apt ]]; then
    for i in chromium nmap httrack imagemagick; do
        if [ $(dpkg-query -W -f='${Status}' $i 2>/dev/null | grep -c "ok installed") -eq 0 ]; then
            sudo apt install -y $i
        fi
    done
else
    echo "No sudo access for apt install... Please correct situation"
    echo "PUT : $USER ALL=(ALL) NOPASSWD:/usr/bin/apt INTO /etc/sudoers.d/apt (use sudo visudo)"
    exit 1
fi
##############################################################"
## Backing up europa p4n collected spots from collective intelligence ##
# SWARM is collectively PLAYing bataille navale
# Randomly get geaodata in south europe
# FOCUS on SOUTH EUROPE
# LON = -10 to 30
# LAT = 35 to 60
# https://fr.mapsofworld.com/lat_long/europe.html
##############################################################"
# initial first spot.
lon=0
lat=40
try=1

# Randomly choose spot from in RANGE $lat.$lon
while [[ $(ls ~/.zen/ipfs_swarm/.*/astroport/map/$lat.$lon/p4n/spot.json 2>/dev/null) ]]; do
    echo "astroport/$lat.$lon geodata already swarm captured. CONTINUE"
    FLOOR=-5; #-10;
    CEILING=10; #30
    RANGE=$(($CEILING-$FLOOR+1));
    RESULT=$RANDOM;
    let "RESULT %= $RANGE";
    lon=$(($RESULT+$FLOOR));
#echo $lon

    FLOOR=40; #35;
    CEILING=50; #60;
    RANGE=$(($CEILING-$FLOOR+1));
    RESULT=$RANDOM;
    let "RESULT %= $RANGE";
    lat=$(($RESULT+$FLOOR));
#echo $lat
    try=$(($try+1));
    [ $try -eq 99 ] && break ## tirages aléatoire maximum
done

##############################################################"
## $lat.$lon GEODATA NOT SCRAPED YET ?
##############################################################"
if [ ! -d ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/ ]; then
### ADD Spot randomness
    # Fuzzle geoloc
    one=$(echo "000$RANDOM$RANDOM$RANDOM" | cut -b1-12)
    latitude="$lat.$one"
    two=$(echo "000$RANDOM$RANDOM$RANDOM" | cut -b1-12)
    longitude="$lon.$two"

### SCRAPE P4N
    ## curl spot.json into /astroport/map/$lat.$lon/p4n/spot.json
    echo "GETTING /astroport/map/$lat.$lon/p4n/spot.json"
    mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/
    echo "https://www.park4night.com/services/V3/getLieuxAroundMeLite.php?latitude=$latitude&longitude=$longitude"
    curl -s "https://www.park4night.com/services/V3/getLieuxAroundMeLite.php?latitude=$latitude&longitude=$longitude" -o ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json
    echo "SAVED ~/.zen/ipfs/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json"
    ## TODO REFRESH OLD spot.json
fi

##############################################################"
## GEODATA. continue. Let's extract lieu pdf from my spot
##############################################################"
geosrc=$(ls ~/.zen/{ipfs,ipfs_swarm}/.*/astroport/map/$lat.$lon/p4n/spot.json 2>/dev/null)
geodata=$(echo "$geosrc" | tail -n 1)
ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2)

if [[ "$ipfsnodeid" != "$IPFSNODEID" ]]; then
    echo "This spot.json is managed by
     $geosrc"
    echo "getting geodata from one of mine"
    geodata=$(ls ~/.zen/ipfs/.*/astroport/map/*.*/p4n/spot.json | shuf | tail -n 1)
fi

[[ $geodata ]] && ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2) || exit 1

## SUB-SCRAPE SOME of my p4n/spot.json RANDOM "lieu"
echo "SUB-SCRAPING lieu FROM $geodata"

    loop=1
    while [ $loop -lt 21 ]; do
        #{"id":"208264","latitude":"40.730727","longitude":"0.367303","hauteur_limite":"0.00","publique":"1","top_liste":"0","code":"APN","validation_admin":"1","distance":"97.341"}
        # flatten json

        lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
        ## GET RANDOM LIEU ID AND TRY MORE TO GET ( prefered APN)
        lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
        [[ "$code" != "APN" ]] && [ -d  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
        lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
        [[ "$code" != "APN" ]] && [ -d  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
        lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
        [[ "$code" != "APN" ]] && [ -d  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
        lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
        [ -d  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
        lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)
        [ -d  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ] && lieu=$(cat $geodata | jq -r '.lieux[] | "\(.id),\(.latitude),\(.longitude),\(.hauteur_limite),\(.publique),\(.top_liste),\(.code),\(.validation_admin),\(.distance)"' | shuf | tail -n 1)
        lid=$(echo "$lieu" | cut -d ',' -f 1); code=$(echo "$lieu" | cut -d ',' -f 7)

        [[ "$lid" == "" ]] && echo "ERROR LIEU NO ID. CONTINUE" && loop=$(($loop+1)) && continue


        mkdir -p /tmp/$lid && cd /tmp/$lid
        echo "Downloading https://www.park4night.com/?page=lieu&id=$lid&bulle=YES"
        mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code
        echo "id,latitude,longitude,hauteur_limite,publique,top_liste,code,validation_admin,distance" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt
        echo "$lieu" >> ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt
        cat ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt

        # Get 1st copy bulle.pdf... Now converting to index.html redirect to $lid.jpg
        [ -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ] && mv ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ./output.pdf && echo "ancien fichier bulle.pdf"
        [ ! -f  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html ] && echo "running chromium... " && chromium --timeout=10000 --headless --no-sandbox --disable-gpu --print-to-pdf=output.pdf "https://www.park4night.com/?page=lieu&id=$lid&bulle=YES" 2>/dev/null
        [ ! -f ./output.pdf ] && rm -Rf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid && echo "## CHROMIUM TIMEOUT, CANCEL MISSION" && loop=$(($loop+1)) && continue

        ## PDF convert and crop to JPG / ipfs add / index.html for redirect
        if [ -f ./output.pdf ]; then
            convert -density 200 -quality 80 ./output.pdf ./bulle.jpg
            convert -crop 1260x420+55+30 ./bulle.jpg ./$lid.jpg

            IPFS=$(ipfs add -wq "./$lid.jpg") && echo "## Added $lid.pdf to IPFS: $IPFS"
            IPFSREPFILEID=$(echo $IPFS | cut -d ' ' -f 2) && echo "## Repertory $IPFSREPFILEID"
            [[ "$IPFSREPFILEID" == "" ]] && echo "IPFS ADD TROUBLE. CONTINUE" && rm -Rf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid && rm -Rf /tmp/$lid && loop=$(($loop+1)) && continue
            ## Create index.html with Redirect to /ipfs/$IPFSREPFILEID/$lid.jpg
            echo "<html><head><meta http-equiv=\"Refresh\" content=\"0; URL=/ipfs/$IPFSREPFILEID/$lid.jpg\"></head></html>" >  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html
            # mv ./output.pdf  ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf
            rm ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf 2>/dev/null
            echo "NOW LIEU redirect to http://127.0.0.1:8080/ipfs/$IPFSREPFILEID/$lid.jpg"
            rm ./$lid.jpg
            rm ./bulle.jpg
        fi

        cd -
        sleep $((1 + RANDOM % 2))
        rm -Rf  /tmp/$lid

        loop=$(($loop+1));
    done

## CHECK SWARM GEODATA MAP
# ls -d ~/.zen/ipfs_swarm/.*/astroport/map/*/ | rev | cut -d '/' -f 2 | rev | sort
## NB OF lieu: ls -d ~/.zen/ipfs_swarm/.*/astroport/map/p4n/lieu/*/ | rev | cut -d '/' -f 2 | rev | sort | wc
# ipfs cat /ipns/$IPFSNODEID/.$IPFSNODEID/astroport/map/$lat.$lon/p4n/spot.json | jq
exit 0