diff --git a/actions/p4n.swarmscraper.sh b/actions/p4n.swarmscraper.sh index 0a8df8a..85c68bb 100755 --- a/actions/p4n.swarmscraper.sh +++ b/actions/p4n.swarmscraper.sh @@ -15,6 +15,18 @@ G1PUB=$(cat ~/.zen/secret.dunikey | grep 'pub:' | cut -d ' ' -f 2) && [[ ! $G1PU IPFSNODEID=$(ipfs --timeout=20s id -f='\n' 2>/dev/null) && [[ $IPFSNODEID == "" ]] && echo 'NO IPFS RUNNING. EXIT' && exit 1 [ ! -d ~/.zen/ipfs/.$IPFSNODEID/astroport ] && echo 'ASTROPORT NOT READY' && exit 1 +## INSTALL MISSING PACKAGES +[[ -f /etc/sudoers.d/apt ]]; then + for i in chromium nmap httrack imagemagick; do + if [ $(dpkg-query -W -f='${Status}' $i 2>/dev/null | grep -c "ok installed") -eq 0 ]; then + sudo apt install -y $i + fi + done +else + echo "No sudo access for apt install... Please correct situation" + echo "PUT : $USER ALL=(ALL) NOPASSWD:/usr/bin/apt INTO /etc/sudoers.d/apt (use sudo visudo)" + exit 1 +fi ##############################################################" ## Backing up europa p4n collected spots from collective intelligence ## # SWARM is collectively PLAYing bataille navale @@ -80,14 +92,14 @@ ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2) if [[ "$ipfsnodeid" != "$IPFSNODEID" ]]; then echo "This spot.json is already managed by $ipfsnodeid" - echo "getting one for mine" + echo "getting geodata from one of mine" geodata=$(ls ~/.zen/ipfs/.*/astroport/map/*.*/p4n/spot.json | shuf | tail -n 1) fi [[ $geodata ]] && ipfsnodeid=$(echo "$geodata" | cut -d '/' -f 6 | cut -d '.' -f 2) || exit 1 ## SUB-SCRAPE SOME of my p4n/spot.json RANDOM "lieu" -echo "SUB-SCRAPING lieu FROM /astroport/map/$lat.$lon/p4n/spot.json" +echo "SUB-SCRAPING lieu FROM $geodata" loop=1 while [ $loop -lt 21 ]; do @@ -110,26 +122,35 @@ echo "SUB-SCRAPING lieu FROM /astroport/map/$lat.$lon/p4n/spot.json" [[ "$lid" == "" ]] && continue - if [[ $(which chromium) ]]; then - mkdir -p /tmp/$lid && cd /tmp/$lid - echo "Downloading https://www.park4night.com/?page=lieu&id=$lid&bulle=YES" - mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code - echo "id,latitude,longitude,hauteur_limite,publique,top_liste,code,validation_admin,distance" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt - echo "$lieu" >> ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt - [ ! -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ] && chromium --timeout=30000 --headless --no-sandbox --disable-gpu --print-to-pdf=output.pdf "https://www.park4night.com/?page=lieu&id=$lid&bulle=YES" 2>/dev/null - [ ! -f ./output.pdf ] && rm -Rf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid - [ -f ./output.pdf ] && mv ./output.pdf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf + mkdir -p /tmp/$lid && cd /tmp/$lid + echo "Downloading https://www.park4night.com/?page=lieu&id=$lid&bulle=YES" + mkdir -p ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code + echo "id,latitude,longitude,hauteur_limite,publique,top_liste,code,validation_admin,distance" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt + echo "$lieu" >> ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/$code/lieu.txt - echo "~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.png" - sleep $((1 + RANDOM % 2)) - cd - && rm -Rf /tmp/$lid + # Get 1st copy bulle.pdf... Now converting to index.html redirect to $lid.jpg + [ -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ] && mv ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf ./output.pdf + [ ! -f ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html ] && chromium --timeout=30000 --headless --no-sandbox --disable-gpu --print-to-pdf=output.pdf "https://www.park4night.com/?page=lieu&id=$lid&bulle=YES" 2>/dev/null + [ ! -f ./output.pdf ] && rm -Rf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid ## TIMEOUT, CANCEL MISSION - # TODO FUTURE BATCH to convert & crop to jpg image (imagemagick) - # convert -density 300 -depth 8 -quality 90 -trim ./output.pdf ./output.jpg - else - sudo apt-get install chromium -y + ## PDF convert and crop to JPG / ipfs add / index.html for redirect + if [ -f ./output.pdf ]; then + convert -density 300 -quality 80 ./bulle.pdf ./bulle.jpg + convert -crop 1205x250+55+30 ./bulle.jpg ./$lid.jpg + + IPFS=$(ipfs add -wq "./$lid.jpg") ## Add $lid.pdf to IPFS + IPFSREPFILEID=$(echo $IPFS | cut -d ' ' -f 2) ## Get repertory ipfs link + ## Create index.html with Redirect to /ipfs/$IPFSREPFILEID/$lid.jpg + echo "" > ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/index.html + # mv ./output.pdf ~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf + rm "~/.zen/ipfs/.$IPFSNODEID/astroport/map/p4n/lieu/$lid/bulle.pdf" 2>/dev/null + echo "NOW LIEU redirect to /ipfs/$IPFSREPFILEID/$lid.jpg" fi + + sleep $((1 + RANDOM % 2)) + cd - && rm -Rf /tmp/$lid + loop=$(($loop+1)); done diff --git a/zen/ipfs_SWARM_refresh.sh b/zen/ipfs_SWARM_refresh.sh index 5cf93f5..a5bf481 100755 --- a/zen/ipfs_SWARM_refresh.sh +++ b/zen/ipfs_SWARM_refresh.sh @@ -125,7 +125,7 @@ do find /home/$YOU/.zen/ipfs_swarm/.$ipfsnodeid/ -empty -type d -delete; ################################################################ echo "OK Friend SWARM REFRESH /ipns/$ipfsnodeid INTO ~/.zen/ipfs_swarm/" - ipfs --timeout=60s get --output=/home/$YOU/.zen/ipfs_swarm/ /ipns/$ipfsnodeid + ipfs --timeout=120s get --output=/home/$YOU/.zen/ipfs_swarm/ /ipns/$ipfsnodeid ################################################################ if [ $? != 0 ]; then