2022-08-25 17:21:39 +02:00
#!/bin/bash
########################################################################
2022-10-07 22:39:01 +02:00
# Version: 0.4
2022-08-25 17:21:39 +02:00
# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/)
########################################################################
MY_PATH = " `dirname \" $0 \"` " # relative
MY_PATH = " `( cd \" $MY_PATH \" && pwd )` " # absolutized and normalized
ME = " ${ 0 ##*/ } "
2022-10-12 00:44:24 +02:00
echo "-----"
2022-10-11 17:28:15 +02:00
echo " $ME RUNNING "
2022-08-27 14:01:40 +02:00
# Need TW index.html path + IPNS publication Key (available in IPFS keystore)
2022-08-26 16:02:56 +02:00
# Search for "tube" tagged tiddlers to get URL
# Download video, add to ipfs and import new tiddler
# Publish !!
2022-11-27 18:06:24 +01:00
## TODO USE API 1234 & new_file_in_astroport.sh FOR TW
2022-10-10 16:21:33 +02:00
# ASTROBOT FIRST PROCESS
# "Copier youtube" + (voeu) => CopierYoutube (G1Voeu G1CopierYoutube) = ASTROBOT copy Ŋ1 "(G1CopierYoutube)"
2022-11-29 16:39:58 +01:00
# IPFSNODEID=$(ipfs id -f='<id>\n')
IPFSNODEID = $( cat ~/.ipfs/config | jq -r .Identity.PeerID)
2022-08-26 16:02:56 +02:00
2022-08-25 17:21:39 +02:00
INDEX = " $1 "
2022-10-11 17:28:15 +02:00
[ [ ! $INDEX ] ] && echo "ERROR - Please provide path to source TW index.html" && exit 1
[ [ ! -f $INDEX ] ] && echo " ERROR - Fichier TW absent. $INDEX " && exit 1
2022-08-25 17:21:39 +02:00
2022-10-07 22:39:01 +02:00
WISHKEY = " $2 " ## IPNS KEY NAME - G1PUB - PLAYER ...
2022-10-11 17:28:15 +02:00
[ [ ! $WISHKEY ] ] && echo "ERROR - Please provide IPFS publish key" && exit 1
2022-10-07 22:39:01 +02:00
TWNS = $( ipfs key list -l | grep -w $WISHKEY | cut -d ' ' -f1)
2022-10-11 17:28:15 +02:00
[ [ ! $TWNS ] ] && echo " ERROR - Clef IPNS $WISHKEY introuvable! " && exit 1
2022-08-25 17:21:39 +02:00
2022-11-27 18:06:24 +01:00
# Extract tag=tube from TW
2022-10-10 00:39:07 +02:00
MOATS = $( date -u +"%Y%m%d%H%M%S%4N" )
2022-11-27 18:06:24 +01:00
mkdir -p ~/.zen/tmp/$IPFSNODEID
2022-10-10 00:39:07 +02:00
2022-10-10 01:46:04 +02:00
###################################################################
2022-10-10 16:21:33 +02:00
## TODO BOUCLER SUR LES INDEX des G1Astronautes G1Ami
###################################################################
2022-11-27 18:06:24 +01:00
rm -f ~/.zen/tmp/CopierYoutube.json
2022-10-10 01:46:04 +02:00
tiddlywiki --load ${ INDEX } \
2022-11-27 18:06:24 +01:00
--output ~/.zen/tmp \
2022-10-10 16:21:33 +02:00
--render '.' 'CopierYoutube.json' 'text/plain' '$:/core/templates/exporters/JsonFile' 'exportFilter' '[tag[CopierYoutube]]'
2022-10-10 01:46:04 +02:00
2022-11-27 18:06:24 +01:00
echo "cat ~/.zen/tmp/CopierYoutube.json | jq -r"
### MAKE CACHE REFRESH FUNCTION
#~ #############################
#~ ## Refresh _12345.sh IPNS Memories
#~ # ipfs name publish --key "MySwarm_$IPFSNODEID" --allow-offline /ipfs/$SWARMH
#~ MySwarm=$(ipfs key list -l | grep "MySwarm_$IPFSNODEID" | cut -d ' ' -f 1)
#~ if [[ $MySwarm ]]; then
#~ #############################
#~ mkdir -p ~/.zen/tmp/$MySwarm && rm -Rf ~/.zen/tmp/$MySwarm/*.*
#~ echo "## Getting Station MySwarm /ipns/$MySwarm"
#~ ipfs --timeout 12s get -o ~/.zen/tmp/$MySwarm /ipns/$MySwarm
#~ echo "## Rebuild ~/.zen/tmp/$IPFSNODEID/yt-dlp.command & yt-dlp.cache"
#~ cat ~/.zen/tmp/$MySwarm/*/yt-dlp.command ~/.zen/tmp/$IPFSNODEID/yt-dlp.command | sort | uniq > ~/.zen/tmp/$IPFSNODEID/yt-dlp.command
#~ cat ~/.zen/tmp/$MySwarm/*/yt-dlp.cache ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache | sort | uniq > ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache
#~ #############################
#~ fi
2022-11-16 00:29:08 +01:00
2022-10-10 01:46:04 +02:00
###################################################################
2022-10-10 16:21:33 +02:00
## TEXT TREATMENT
## For this TAG, specific extract URL from text field and copy all video from links into tid.json
2022-11-27 18:06:24 +01:00
for YURL in $( cat ~/.zen/tmp/CopierYoutube.json | jq -r '.[].text' | grep 'http' ) ; do
2022-10-07 22:39:01 +02:00
echo " Detected $YURL "
2022-10-10 00:39:07 +02:00
echo "Extracting video playlist"
2022-08-25 17:21:39 +02:00
2022-11-27 18:06:24 +01:00
### yt-dlp.command
[ [ ! $( cat ~/.zen/tmp/$IPFSNODEID /yt-dlp.command 2>/dev/null | grep " $YURL " ) ] ] \
&& echo " $WISHKEY & $YURL " >> ~/.zen/tmp/$IPFSNODEID /yt-dlp.command
yt-dlp --print "%(id)s&%(webpage_url)s" " ${ YURL } " >> ~/.zen/tmp/$IPFSNODEID /yt-dlp.cache.$WISHKEY
2022-10-10 00:39:07 +02:00
done # FINISH YURL loop
2022-09-15 21:46:37 +02:00
2022-11-27 18:06:24 +01:00
## SORT UNIQ CACHE
cat ~/.zen/tmp/$IPFSNODEID /yt-dlp.cache.$WISHKEY | sort | uniq > ~/.zen/tmp/yt-dlp.cache
cp ~/.zen/tmp/yt-dlp.cache ~/.zen/tmp/$IPFSNODEID /yt-dlp.cache.$WISHKEY
2022-11-16 00:29:08 +01:00
2022-10-12 00:44:24 +02:00
###################################################################
2022-11-27 18:06:24 +01:00
[ [ ! -s ~/.zen/tmp/$IPFSNODEID /yt-dlp.cache.$WISHKEY ] ] && echo "AUCUN YOUTUBEID pour CopierYoutube" && exit 0
2022-10-10 00:39:07 +02:00
###################################################################
2022-10-10 01:43:02 +02:00
2022-10-10 01:46:04 +02:00
###################################################################
2022-10-10 16:21:33 +02:00
# PROCESS YOUTUBEID VIDEO DOWNLOAD AND CREATE TIDDLER in TW
2022-10-10 00:39:07 +02:00
###################################################################
2022-11-27 18:06:24 +01:00
while read LINE;
2022-10-06 15:57:00 +02:00
do
2022-10-10 00:39:07 +02:00
2022-11-29 16:39:58 +01:00
YID = $( echo " $LINE " | cut -d '&' -f 1)
2022-12-04 17:02:29 +01:00
## SEARCH IN 12345 MySwarm !!
[ [ ls ~/.zen/tmp/$IPFSNODEID /G1CopierYoutube/*/$YID .TW.json ] ] && echo " NODE existing : ~/.zen/tmp/ $IPFSNODEID /G1CopierYoutube/ $PLAYER / $YID .TW.json " && continue
[ [ ls ~/.zen/tmp/swarm/*/G1CopierYoutube/*/$YID .TW.json ] ] && echo " SWARM existing : ls ~/.zen/tmp/swarm/*/G1CopierYoutube/*/ $YID .TW.json " && continue ## TODO :: CHECK IF ALREADY YOURS OR NOT :: THEN ADD2TW / SEND MESSAGE ?
2022-10-10 00:39:07 +02:00
2022-10-07 22:39:01 +02:00
# SINGLE VIDEO YURL
2022-11-29 16:39:58 +01:00
ZYURL = $( echo " $LINE " | cut -d '&' -f 2-)
2022-11-27 18:06:24 +01:00
echo " COPIE : $ZYURL "
2022-10-10 00:39:07 +02:00
2022-10-07 22:39:01 +02:00
TITLE = " $( yt-dlp --print "%(title)s" " ${ ZYURL } " ) "
2022-09-16 11:52:04 +02:00
TITLE = ${ TITLE //[^A-zÀ-ÿ0-9 ]/ }
2022-11-27 18:06:24 +01:00
[ [ ! $TITLE ] ] && echo "NO TITLE" && continue
2022-09-15 21:46:37 +02:00
2022-10-10 00:39:07 +02:00
echo " .... Downloading $TITLE .mp4 "
2022-09-09 18:20:18 +02:00
# https://github.com/yt-dlp/yt-dlp#format-selection-examples
2022-09-16 11:52:04 +02:00
# SUBS ? --write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs
2022-10-10 00:39:07 +02:00
# (bv*[height<=720][vcodec~='^((he|a)vc|h26[45])']+ba)
2022-09-10 16:39:12 +02:00
# TODO : DELAY COPY OPERATION... Astro can download quicker at 03:00 AM
2022-11-27 18:06:24 +01:00
echo " yt-dlp -f \"(bv*[ext=mp4][height<=720]+ba/b[height<=720])\" --no-mtime --embed-thumbnail --add-metadata -o \" $HOME /.zen/tmp/yt-dlp/ $TITLE .%(ext)s\" ${ ZYURL } "
2022-10-07 22:39:01 +02:00
#############################################################################
2022-10-08 01:38:33 +02:00
## COPY FROM YOUTUBE (TODO DOUBLE COPY & MKV to MP4 OPTIMISATION)
2022-11-27 18:06:24 +01:00
## EXTRA PARAM TO TRY
# --cookies-from-browser browser (xdg-settings get default-web-browser) allow member copies
# --dateafter DATE --download-archive myarchive.txt
2022-10-10 00:39:07 +02:00
yt-dlp -f "(bv*[ext=mp4][height<=720]+ba/b[height<=720])" \
2022-11-27 18:06:24 +01:00
--download-archive $HOME /.zen/.yt-dlp.list \
2022-10-07 17:07:51 +02:00
-S "filesize:700M" --no-mtime --embed-thumbnail --add-metadata \
2022-09-16 11:52:04 +02:00
--write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs \
2022-11-27 18:06:24 +01:00
-o " $HOME /.zen/tmp/yt-dlp/ $TITLE .%(ext)s " ${ ZYURL }
2022-10-07 22:39:01 +02:00
################################################################################
### ADAPT TO TW RYTHM (DELAY COPY?)
2022-09-15 14:23:29 +02:00
echo
2022-09-15 21:17:48 +02:00
ZFILE = " $TITLE .mp4 "
2022-09-15 14:55:11 +02:00
echo " $ZFILE "
2022-09-15 14:39:48 +02:00
2022-10-07 22:39:01 +02:00
############################################################################
### CHECK RESULT CONVERT MKV TO MP4
2022-11-27 18:06:24 +01:00
[ [ ! -f " $HOME /.zen/tmp/yt-dlp/ $ZFILE " ] ] && ffmpeg -loglevel quiet -i " $HOME /.zen/tmp/yt-dlp/ $TITLE .mkv " -c:v libx264 -c:a aac " $HOME /.zen/tmp/yt-dlp/ $TITLE .mp4 " # TRY TO CONVERT MKV TO MP4
[ [ ! -f " $HOME /.zen/tmp/yt-dlp/ $ZFILE " ] ] && echo "No FILE -- CONTINUE --" && continue
2022-09-15 14:23:29 +02:00
echo
2022-08-25 17:21:39 +02:00
2022-10-10 00:39:07 +02:00
####################################################
2022-11-27 18:06:24 +01:00
echo " FOUND : ~/.zen/tmp/yt-dlp/ $ZFILE "
FILE_BSIZE = $( du -b " $HOME /.zen/tmp/yt-dlp/ $ZFILE " | awk '{print $1}' )
2022-10-10 00:39:07 +02:00
FILE_SIZE = $( echo " ${ FILE_BSIZE } " | awk '{ split( "B KB MB GB TB PB" , v ); s=1; while( $1>1024 ){ $1/=1024; s++ } printf "%.2f %s", $1, v[s] }' )
echo " FILE SIZE = $FILE_SIZE ( $FILE_BSIZE octets) "
2022-09-15 12:35:42 +02:00
2022-11-29 16:39:58 +01:00
#~ ## PREPARE FOR new_file_in_astroport.sh
#~ mkdir -p "$HOME/Astroport/youtube/$YID"
#~ REVSOURCE="$(echo "$ZYURL" | awk -F/ '{print $3}' | rev)_"
#~ MEDIAID="$REVSOURCE${YID}"
#~ URLENCODE_FILE_NAME=$(echo ${ZFILE} | jq -Rr @uri)
#~ echo "youtube;${MEDIAID};$(date -u +%s%N | cut -b1-13);${TITLE};${SAISON};${GENRES};_IPNSKEY_;${RES};/ipfs/_IPFSREPFILEID_/$URLENCODE_FILE_NAME" > ~/Astroport/youtube/$YID/ajouter_video.txt
#~ mv "$HOME/.zen/tmp/yt-dlp/$ZFILE" "$HOME/Astroport/youtube/$YID/"
###
#~ ${MY_PATH}/../tools/new_file_in_astroport.sh "$HOME/Astroport/youtube/$YID" "${ZFILE}" "$G1PUB"
path = " $HOME /.zen/tmp/yt-dlp "
file = " $ZFILE "
$( ${ MY_PATH } /../tools/make_video_gifanim_ipfs.sh " $path " " $file " | tail -n 1)
echo " /ipfs/ $ANIMH "
## Create gifanime ## TODO Search for similarities BEFORE ADD
2022-08-25 17:21:39 +02:00
echo "Adding to IPFS"
2022-11-27 18:06:24 +01:00
ILINK = $( ipfs add -q " $HOME /.zen/tmp/yt-dlp/ $ZFILE " | tail -n 1)
2022-10-07 17:07:51 +02:00
echo " /ipfs/ $ILINK <=> $ZFILE "
2022-08-25 17:21:39 +02:00
2022-11-27 18:06:24 +01:00
MIME = $( file --mime-type -b " $HOME /.zen/tmp/yt-dlp/ $ZFILE " )
2022-08-25 17:21:39 +02:00
2022-10-07 17:07:51 +02:00
## ADD TAGS
2022-10-10 00:39:07 +02:00
SEC = $( yt-dlp --print "%(duration)s" " ${ ZYURL } " )
2022-10-10 01:39:22 +02:00
CHANNEL = $( yt-dlp --print "%(channel)s" " ${ ZYURL } " | sed -r 's/\<./\U&/g' | sed 's/ //g' ) # CapitalGluedWords
PLAYLIST = $( yt-dlp --print "%(playlist)s" " ${ ZYURL } " | sed -r 's/\<./\U&/g' | sed 's/ //g' )
EXTRATAG = " $CHANNEL $PLAYLIST "
2022-10-07 17:07:51 +02:00
## PREPARE VIDEO HTML5 CODE
2022-12-01 18:26:21 +01:00
TEXT = "<video controls width=100% poster='/ipfs/" ${ ANIMH } "'><source src='/ipfs/" ${ ILINK } "' type='" ${ MIME } "'></video><br>{{!!duree}}<br><h1><a href='" ${ ZYURL } "'>" ${ TITLE } "</a></h1>"
2022-08-25 17:21:39 +02:00
2022-12-04 17:02:29 +01:00
mkdir -p $HOME /.zen/tmp/$IPFSNODEID /G1CopierYoutube/$PLAYER /
2022-10-10 16:21:33 +02:00
echo " Creating Youtube ${ YID } tiddler : G1CopierYoutube ! "
2022-09-15 13:41:53 +02:00
echo $TEXT
2022-08-25 17:21:39 +02:00
echo ' [
{
2022-11-13 04:07:47 +01:00
"created" : " ' ${ MOATS } ' " ,
2022-11-29 16:39:58 +01:00
"resolution" : " ' ${ RES } ' " ,
"duree" : " ' ${ DUREE } ' " ,
"duration" : " ' ${ DURATION } ' " ,
"giftime" : " ' ${ PROBETIME } ' " ,
"gifanime" : " '/ipfs/ ${ ANIMH } ' " ,
2022-11-16 07:39:45 +01:00
"modified" : " ' ${ MOATS } ' " ,
2022-09-15 13:45:52 +02:00
"title" : " ' $ZFILE ' " ,
2022-08-25 19:33:10 +02:00
"type" : "'text/vnd.tiddlywiki'" ,
"text" : " ' $TEXT ' " ,
2022-11-29 16:39:58 +01:00
"mime" : " ' ${ MIME } ' " ,
2022-10-10 00:39:07 +02:00
"size" : " ' ${ FILE_BSIZE } ' " ,
"sec" : " ' ${ SEC } ' " ,
2022-10-28 02:18:22 +02:00
"ipfs" : " '/ipfs/ ${ ILINK } ' " ,
2022-10-07 22:39:01 +02:00
"youtubeid" : " ' ${ YID } ' " ,
2022-11-16 07:11:14 +01:00
"tags" : " 'ipfs G1CopierYoutube ${ WISHKEY } ${ EXTRATAG } ${ MIME } ' "
2022-08-25 17:21:39 +02:00
}
]
2022-12-04 17:02:29 +01:00
' > " $HOME /.zen/tmp/ $IPFSNODEID /G1CopierYoutube/ $PLAYER / $YID .TW.json "
2022-10-07 22:39:01 +02:00
2022-08-25 17:21:39 +02:00
2022-10-07 22:39:01 +02:00
#################################################################
### ADDING $YID.TW.json to TWNS INDEX.html
#################################################################
2022-08-25 19:33:10 +02:00
echo "=========================="
2022-10-07 22:39:01 +02:00
echo " Adding $YID tiddler to TW /ipns/ $TWNS "
2022-08-25 17:21:39 +02:00
2022-11-27 18:06:24 +01:00
rm -f ~/.zen/tmp/$IPFSNODEID /newindex.html
2022-08-25 19:33:10 +02:00
2022-12-04 17:02:29 +01:00
echo " >>> Importing $HOME /.zen/tmp/ $IPFSNODEID /G1CopierYoutube/ $PLAYER / $YID .TW.json "
2022-09-15 13:41:53 +02:00
tiddlywiki --load $INDEX \
2022-12-04 17:02:29 +01:00
--import " $HOME /.zen/tmp/ $IPFSNODEID /G1CopierYoutube/ $PLAYER / $YID .TW.json " "application/json" \
2022-11-27 18:06:24 +01:00
--output ~/.zen/tmp/$IPFSNODEID --render " $:/core/save/all " "newindex.html" "text/plain"
# --deletetiddlers '[tag[CopierYoutube]]' ### REFRESH CHANNEL COPY
2022-08-25 17:21:39 +02:00
2022-11-27 18:06:24 +01:00
if [ [ -s ~/.zen/tmp/$IPFSNODEID /newindex.html ] ] ; then
2022-11-07 01:07:33 +01:00
echo " $$ $ Mise à jour $INDEX "
2022-11-27 18:06:24 +01:00
cp ~/.zen/tmp/$IPFSNODEID /newindex.html $INDEX
2022-08-25 19:33:10 +02:00
else
2022-11-27 18:06:24 +01:00
echo " Problem with tiddlywiki command. Missing ~/.zen/tmp/ $IPFSNODEID /newindex.html "
2022-08-25 19:33:10 +02:00
echo "XXXXXXXXXXXXXXXXXXXXXXX"
fi
2022-10-06 15:57:00 +02:00
2022-11-27 21:54:40 +01:00
done < ~/.zen/tmp/$IPFSNODEID /yt-dlp.cache.$WISHKEY # FINISH YID loop 1
2022-10-12 00:44:24 +02:00
exit 0