diff --git a/ASTROBOT/G1CopierYoutube.sh b/ASTROBOT/G1CopierYoutube.sh index 13632bcd..f3a5e108 100755 --- a/ASTROBOT/G1CopierYoutube.sh +++ b/ASTROBOT/G1CopierYoutube.sh @@ -13,9 +13,11 @@ echo "$ME RUNNING" # Download video, add to ipfs and import new tiddler # Publish !! +## TODO USE API 1234 & new_file_in_astroport.sh FOR TW + # ASTROBOT FIRST PROCESS # "Copier youtube" + (voeu) => CopierYoutube (G1Voeu G1CopierYoutube) = ASTROBOT copy Ŋ1 "(G1CopierYoutube)" - +IPFSNODEID=$(ipfs id -f='\n') INDEX="$1" [[ ! $INDEX ]] && echo "ERROR - Please provide path to source TW index.html" && exit 1 @@ -26,53 +28,78 @@ WISHKEY="$2" ## IPNS KEY NAME - G1PUB - PLAYER ... TWNS=$(ipfs key list -l | grep -w $WISHKEY | cut -d ' ' -f1) [[ ! $TWNS ]] && echo "ERROR - Clef IPNS $WISHKEY introuvable!" && exit 1 -# Extract tag=tube from TW into ~/.zen/tmp/$WISHKEY/CopierYoutube.json +# Extract tag=tube from TW MOATS=$(date -u +"%Y%m%d%H%M%S%4N") -mkdir -p ~/.zen/tmp/$WISHKEY +mkdir -p ~/.zen/tmp/$IPFSNODEID ################################################################### ## TODO BOUCLER SUR LES INDEX des G1Astronautes G1Ami ################################################################### -rm -f ~/.zen/tmp/$WISHKEY/CopierYoutube.json +rm -f ~/.zen/tmp/CopierYoutube.json tiddlywiki --load ${INDEX} \ - --output ~/.zen/tmp/$WISHKEY \ + --output ~/.zen/tmp \ --render '.' 'CopierYoutube.json' 'text/plain' '$:/core/templates/exporters/JsonFile' 'exportFilter' '[tag[CopierYoutube]]' -echo "cat ~/.zen/tmp/$WISHKEY/CopierYoutube.json" +echo "cat ~/.zen/tmp/CopierYoutube.json | jq -r" + +### MAKE CACHE REFRESH FUNCTION +#~ ############################# +#~ ## Refresh _12345.sh IPNS Memories +#~ # ipfs name publish --key "MySwarm_$IPFSNODEID" --allow-offline /ipfs/$SWARMH +#~ MySwarm=$(ipfs key list -l | grep "MySwarm_$IPFSNODEID" | cut -d ' ' -f 1) +#~ if [[ $MySwarm ]]; then +#~ ############################# + #~ mkdir -p ~/.zen/tmp/$MySwarm && rm -Rf ~/.zen/tmp/$MySwarm/*.* + + #~ echo "## Getting Station MySwarm /ipns/$MySwarm" + #~ ipfs --timeout 12s get -o ~/.zen/tmp/$MySwarm /ipns/$MySwarm + + #~ echo "## Rebuild ~/.zen/tmp/$IPFSNODEID/yt-dlp.command & yt-dlp.cache" + #~ cat ~/.zen/tmp/$MySwarm/*/yt-dlp.command ~/.zen/tmp/$IPFSNODEID/yt-dlp.command | sort | uniq > ~/.zen/tmp/$IPFSNODEID/yt-dlp.command + #~ cat ~/.zen/tmp/$MySwarm/*/yt-dlp.cache ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache | sort | uniq > ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache + +#~ ############################# +#~ fi ################################################################### ## TEXT TREATMENT ## For this TAG, specific extract URL from text field and copy all video from links into tid.json -for YURL in $(cat ~/.zen/tmp/$WISHKEY/CopierYoutube.json | jq -r '.[].text' | grep 'http'); do +for YURL in $(cat ~/.zen/tmp/CopierYoutube.json | jq -r '.[].text' | grep 'http'); do echo "Detected $YURL" echo "Extracting video playlist" - ### GETTING ALL VIDEO IDs (for playlist copy) - yt-dlp --print "%(id)s" "${YURL}" >> ~/.zen/tmp/$WISHKEY/ytids.$MOATS + ### yt-dlp.command + [[ ! $(cat ~/.zen/tmp/$IPFSNODEID/yt-dlp.command 2>/dev/null | grep "$YURL") ]] \ + && echo "$WISHKEY&$YURL" >> ~/.zen/tmp/$IPFSNODEID/yt-dlp.command + + yt-dlp --print "%(id)s&%(webpage_url)s" "${YURL}" >> ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY done # FINISH YURL loop -echo "cat ~/.zen/tmp/$WISHKEY/ytids.$MOATS" +## SORT UNIQ CACHE +cat ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY | sort | uniq > ~/.zen/tmp/yt-dlp.cache +cp ~/.zen/tmp/yt-dlp.cache ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY ################################################################### -[[ ! -s ~/.zen/tmp/$WISHKEY/ytids.$MOATS ]] && echo "AUCUN YOUTUBEID pour CopierYoutube" && exit 0 +[[ ! -s ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY ]] && echo "AUCUN YOUTUBEID pour CopierYoutube" && exit 0 ################################################################### ################################################################### # PROCESS YOUTUBEID VIDEO DOWNLOAD AND CREATE TIDDLER in TW ################################################################### -while read YID; +while read LINE; do - [[ -f ~/.zen/tmp/$WISHKEY/$YID.TW.json ]] && echo "Tiddler json already existing : ~/.zen/tmp/$WISHKEY/$YID.TW.json" && continue + YID=$(echo "$LINE" | cut -d '&' -f 2) + [[ -s ~/.zen/tmp/$IPFSNODEID/$YID.TW.json ]] && echo "Tiddler json already existing : ~/.zen/tmp/$IPFSNODEID/$YID.TW.json" && continue ## TODO :: CHECK IF ALREADY YOURS OR NOT :: THEN ADD2TW / SEND MESSAGE ? # SINGLE VIDEO YURL - ZYURL="https://www.youtube.com/watch?v=$YID"; - echo "YOUTUBE : $ZYURL" + ZYURL=$(echo "$LINE" | cut -d '&' -f 3-) + echo "COPIE : $ZYURL" TITLE="$(yt-dlp --print "%(title)s" "${ZYURL}")" TITLE=${TITLE//[^A-zÀ-ÿ0-9 ]/} - [[ ! $TITLE ]] && continue + [[ ! $TITLE ]] && echo "NO TITLE" && continue echo ".... Downloading $TITLE.mp4" @@ -80,14 +107,18 @@ while read YID; # SUBS ? --write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs # (bv*[height<=720][vcodec~='^((he|a)vc|h26[45])']+ba) # TODO : DELAY COPY OPERATION... Astro can download quicker at 03:00 AM - echo "yt-dlp -f \"(bv*[ext=mp4][height<=720]+ba/b[height<=720])\" --no-mtime --embed-thumbnail --add-metadata -o \"$HOME/.zen/tmp/$WISHKEY/$TITLE.%(ext)s\" ${ZYURL}" + echo "yt-dlp -f \"(bv*[ext=mp4][height<=720]+ba/b[height<=720])\" --no-mtime --embed-thumbnail --add-metadata -o \"$HOME/.zen/tmp/yt-dlp/$TITLE.%(ext)s\" ${ZYURL}" ############################################################################# ## COPY FROM YOUTUBE (TODO DOUBLE COPY & MKV to MP4 OPTIMISATION) + ## EXTRA PARAM TO TRY + # --cookies-from-browser browser (xdg-settings get default-web-browser) allow member copies + # --dateafter DATE --download-archive myarchive.txt yt-dlp -f "(bv*[ext=mp4][height<=720]+ba/b[height<=720])" \ + --download-archive $HOME/.zen/.yt-dlp.list \ -S "filesize:700M" --no-mtime --embed-thumbnail --add-metadata \ --write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs \ - -o "$HOME/.zen/tmp/$WISHKEY/$TITLE.%(ext)s" ${ZYURL} + -o "$HOME/.zen/tmp/yt-dlp/$TITLE.%(ext)s" ${ZYURL} ################################################################################ ### ADAPT TO TW RYTHM (DELAY COPY?) echo @@ -96,21 +127,21 @@ while read YID; ############################################################################ ### CHECK RESULT CONVERT MKV TO MP4 - [[ ! -f "$HOME/.zen/tmp/$WISHKEY/$ZFILE" ]] && ffmpeg -loglevel quiet -i "$HOME/.zen/tmp/$WISHKEY/$TITLE.mkv" -c:v libx264 -c:a aac "$HOME/.zen/tmp/$WISHKEY/$TITLE.mp4" # TRY TO CONVERT MKV TO MP4 - [[ ! -f "$HOME/.zen/tmp/$WISHKEY/$ZFILE" ]] && echo "No FILE -- CONTINUE --" && continue + [[ ! -f "$HOME/.zen/tmp/yt-dlp/$ZFILE" ]] && ffmpeg -loglevel quiet -i "$HOME/.zen/tmp/yt-dlp/$TITLE.mkv" -c:v libx264 -c:a aac "$HOME/.zen/tmp/yt-dlp/$TITLE.mp4" # TRY TO CONVERT MKV TO MP4 + [[ ! -f "$HOME/.zen/tmp/yt-dlp/$ZFILE" ]] && echo "No FILE -- CONTINUE --" && continue echo #################################################### - echo "FOUND : ~/.zen/tmp/$WISHKEY/$ZFILE" - FILE_BSIZE=$(du -b "$HOME/.zen/tmp/$WISHKEY/$ZFILE" | awk '{print $1}') + echo "FOUND : ~/.zen/tmp/yt-dlp/$ZFILE" + FILE_BSIZE=$(du -b "$HOME/.zen/tmp/yt-dlp/$ZFILE" | awk '{print $1}') FILE_SIZE=$(echo "${FILE_BSIZE}" | awk '{ split( "B KB MB GB TB PB" , v ); s=1; while( $1>1024 ){ $1/=1024; s++ } printf "%.2f %s", $1, v[s] }') echo "FILE SIZE = $FILE_SIZE ($FILE_BSIZE octets)" echo "Adding to IPFS" - ILINK=$(ipfs add -q "$HOME/.zen/tmp/$WISHKEY/$ZFILE" | tail -n 1) + ILINK=$(ipfs add -q "$HOME/.zen/tmp/yt-dlp/$ZFILE" | tail -n 1) echo "/ipfs/$ILINK <=> $ZFILE" - MIME=$(file --mime-type -b "$HOME/.zen/tmp/$WISHKEY/$ZFILE") + MIME=$(file --mime-type -b "$HOME/.zen/tmp/yt-dlp/$ZFILE") ## ADD TAGS SEC=$(yt-dlp --print "%(duration)s" "${ZYURL}") @@ -138,7 +169,7 @@ while read YID; "tags": "'ipfs G1CopierYoutube ${WISHKEY} ${EXTRATAG} ${MIME}'" } ] -' > "$HOME/.zen/tmp/$WISHKEY/$YID.TW.json" +' > "$HOME/.zen/tmp/$IPFSNODEID/$YID.TW.json" ################################################################# @@ -147,22 +178,23 @@ while read YID; echo "==========================" echo "Adding $YID tiddler to TW /ipns/$TWNS " - rm -f ~/.zen/tmp/$WISHKEY/newindex.html + rm -f ~/.zen/tmp/$IPFSNODEID/newindex.html - echo ">>> Importing $HOME/.zen/tmp/$WISHKEY/$YID.TW.json" + echo ">>> Importing $HOME/.zen/tmp/$IPFSNODEID/$YID.TW.json" tiddlywiki --load $INDEX \ - --import "$HOME/.zen/tmp/$WISHKEY/$YID.TW.json" "application/json" \ - --deletetiddlers '[tag[CopierYoutube]]' \ - --output ~/.zen/tmp/$WISHKEY --render "$:/core/save/all" "newindex.html" "text/plain" + --import "$HOME/.zen/tmp/$IPFSNODEID/$YID.TW.json" "application/json" \ + --output ~/.zen/tmp/$IPFSNODEID --render "$:/core/save/all" "newindex.html" "text/plain" - if [[ -s ~/.zen/tmp/$WISHKEY/newindex.html ]]; then +# --deletetiddlers '[tag[CopierYoutube]]' ### REFRESH CHANNEL COPY + + if [[ -s ~/.zen/tmp/$IPFSNODEID/newindex.html ]]; then echo "$$$ Mise à jour $INDEX" - cp ~/.zen/tmp/$WISHKEY/newindex.html $INDEX + cp ~/.zen/tmp/$IPFSNODEID/newindex.html $INDEX else - echo "Problem with tiddlywiki command. Missing ~/.zen/tmp/$WISHKEY/newindex.html" + echo "Problem with tiddlywiki command. Missing ~/.zen/tmp/$IPFSNODEID/newindex.html" echo "XXXXXXXXXXXXXXXXXXXXXXX" fi -done < ~/.zen/tmp/$WISHKEY/ytids.$MOATS # FINISH YID loop 1 +done < ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache # FINISH YID loop 1 exit 0