--download-archive $HOME/.zen/.yt-dlp.list \
This commit is contained in:
parent
32f6824e4b
commit
1ba3b338f5
|
@ -13,9 +13,11 @@ echo "$ME RUNNING"
|
||||||
# Download video, add to ipfs and import new tiddler
|
# Download video, add to ipfs and import new tiddler
|
||||||
# Publish !!
|
# Publish !!
|
||||||
|
|
||||||
|
## TODO USE API 1234 & new_file_in_astroport.sh FOR TW
|
||||||
|
|
||||||
# ASTROBOT FIRST PROCESS
|
# ASTROBOT FIRST PROCESS
|
||||||
# "Copier youtube" + (voeu) => CopierYoutube (G1Voeu G1CopierYoutube) = ASTROBOT copy Ŋ1 "(G1CopierYoutube)"
|
# "Copier youtube" + (voeu) => CopierYoutube (G1Voeu G1CopierYoutube) = ASTROBOT copy Ŋ1 "(G1CopierYoutube)"
|
||||||
|
IPFSNODEID=$(ipfs id -f='<id>\n')
|
||||||
|
|
||||||
INDEX="$1"
|
INDEX="$1"
|
||||||
[[ ! $INDEX ]] && echo "ERROR - Please provide path to source TW index.html" && exit 1
|
[[ ! $INDEX ]] && echo "ERROR - Please provide path to source TW index.html" && exit 1
|
||||||
|
@ -26,53 +28,78 @@ WISHKEY="$2" ## IPNS KEY NAME - G1PUB - PLAYER ...
|
||||||
TWNS=$(ipfs key list -l | grep -w $WISHKEY | cut -d ' ' -f1)
|
TWNS=$(ipfs key list -l | grep -w $WISHKEY | cut -d ' ' -f1)
|
||||||
[[ ! $TWNS ]] && echo "ERROR - Clef IPNS $WISHKEY introuvable!" && exit 1
|
[[ ! $TWNS ]] && echo "ERROR - Clef IPNS $WISHKEY introuvable!" && exit 1
|
||||||
|
|
||||||
# Extract tag=tube from TW into ~/.zen/tmp/$WISHKEY/CopierYoutube.json
|
# Extract tag=tube from TW
|
||||||
MOATS=$(date -u +"%Y%m%d%H%M%S%4N")
|
MOATS=$(date -u +"%Y%m%d%H%M%S%4N")
|
||||||
mkdir -p ~/.zen/tmp/$WISHKEY
|
mkdir -p ~/.zen/tmp/$IPFSNODEID
|
||||||
|
|
||||||
###################################################################
|
###################################################################
|
||||||
## TODO BOUCLER SUR LES INDEX des G1Astronautes G1Ami
|
## TODO BOUCLER SUR LES INDEX des G1Astronautes G1Ami
|
||||||
###################################################################
|
###################################################################
|
||||||
rm -f ~/.zen/tmp/$WISHKEY/CopierYoutube.json
|
rm -f ~/.zen/tmp/CopierYoutube.json
|
||||||
tiddlywiki --load ${INDEX} \
|
tiddlywiki --load ${INDEX} \
|
||||||
--output ~/.zen/tmp/$WISHKEY \
|
--output ~/.zen/tmp \
|
||||||
--render '.' 'CopierYoutube.json' 'text/plain' '$:/core/templates/exporters/JsonFile' 'exportFilter' '[tag[CopierYoutube]]'
|
--render '.' 'CopierYoutube.json' 'text/plain' '$:/core/templates/exporters/JsonFile' 'exportFilter' '[tag[CopierYoutube]]'
|
||||||
|
|
||||||
echo "cat ~/.zen/tmp/$WISHKEY/CopierYoutube.json"
|
echo "cat ~/.zen/tmp/CopierYoutube.json | jq -r"
|
||||||
|
|
||||||
|
### MAKE CACHE REFRESH FUNCTION
|
||||||
|
#~ #############################
|
||||||
|
#~ ## Refresh _12345.sh IPNS Memories
|
||||||
|
#~ # ipfs name publish --key "MySwarm_$IPFSNODEID" --allow-offline /ipfs/$SWARMH
|
||||||
|
#~ MySwarm=$(ipfs key list -l | grep "MySwarm_$IPFSNODEID" | cut -d ' ' -f 1)
|
||||||
|
#~ if [[ $MySwarm ]]; then
|
||||||
|
#~ #############################
|
||||||
|
#~ mkdir -p ~/.zen/tmp/$MySwarm && rm -Rf ~/.zen/tmp/$MySwarm/*.*
|
||||||
|
|
||||||
|
#~ echo "## Getting Station MySwarm /ipns/$MySwarm"
|
||||||
|
#~ ipfs --timeout 12s get -o ~/.zen/tmp/$MySwarm /ipns/$MySwarm
|
||||||
|
|
||||||
|
#~ echo "## Rebuild ~/.zen/tmp/$IPFSNODEID/yt-dlp.command & yt-dlp.cache"
|
||||||
|
#~ cat ~/.zen/tmp/$MySwarm/*/yt-dlp.command ~/.zen/tmp/$IPFSNODEID/yt-dlp.command | sort | uniq > ~/.zen/tmp/$IPFSNODEID/yt-dlp.command
|
||||||
|
#~ cat ~/.zen/tmp/$MySwarm/*/yt-dlp.cache ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache | sort | uniq > ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache
|
||||||
|
|
||||||
|
#~ #############################
|
||||||
|
#~ fi
|
||||||
|
|
||||||
###################################################################
|
###################################################################
|
||||||
## TEXT TREATMENT
|
## TEXT TREATMENT
|
||||||
## For this TAG, specific extract URL from text field and copy all video from links into tid.json
|
## For this TAG, specific extract URL from text field and copy all video from links into tid.json
|
||||||
for YURL in $(cat ~/.zen/tmp/$WISHKEY/CopierYoutube.json | jq -r '.[].text' | grep 'http'); do
|
for YURL in $(cat ~/.zen/tmp/CopierYoutube.json | jq -r '.[].text' | grep 'http'); do
|
||||||
echo "Detected $YURL"
|
echo "Detected $YURL"
|
||||||
echo "Extracting video playlist"
|
echo "Extracting video playlist"
|
||||||
|
|
||||||
### GETTING ALL VIDEO IDs (for playlist copy)
|
### yt-dlp.command
|
||||||
yt-dlp --print "%(id)s" "${YURL}" >> ~/.zen/tmp/$WISHKEY/ytids.$MOATS
|
[[ ! $(cat ~/.zen/tmp/$IPFSNODEID/yt-dlp.command 2>/dev/null | grep "$YURL") ]] \
|
||||||
|
&& echo "$WISHKEY&$YURL" >> ~/.zen/tmp/$IPFSNODEID/yt-dlp.command
|
||||||
|
|
||||||
|
yt-dlp --print "%(id)s&%(webpage_url)s" "${YURL}" >> ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY
|
||||||
|
|
||||||
done # FINISH YURL loop
|
done # FINISH YURL loop
|
||||||
|
|
||||||
echo "cat ~/.zen/tmp/$WISHKEY/ytids.$MOATS"
|
## SORT UNIQ CACHE
|
||||||
|
cat ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY | sort | uniq > ~/.zen/tmp/yt-dlp.cache
|
||||||
|
cp ~/.zen/tmp/yt-dlp.cache ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY
|
||||||
|
|
||||||
###################################################################
|
###################################################################
|
||||||
[[ ! -s ~/.zen/tmp/$WISHKEY/ytids.$MOATS ]] && echo "AUCUN YOUTUBEID pour CopierYoutube" && exit 0
|
[[ ! -s ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache.$WISHKEY ]] && echo "AUCUN YOUTUBEID pour CopierYoutube" && exit 0
|
||||||
###################################################################
|
###################################################################
|
||||||
|
|
||||||
###################################################################
|
###################################################################
|
||||||
# PROCESS YOUTUBEID VIDEO DOWNLOAD AND CREATE TIDDLER in TW
|
# PROCESS YOUTUBEID VIDEO DOWNLOAD AND CREATE TIDDLER in TW
|
||||||
###################################################################
|
###################################################################
|
||||||
while read YID;
|
while read LINE;
|
||||||
do
|
do
|
||||||
|
|
||||||
[[ -f ~/.zen/tmp/$WISHKEY/$YID.TW.json ]] && echo "Tiddler json already existing : ~/.zen/tmp/$WISHKEY/$YID.TW.json" && continue
|
YID=$(echo "$LINE" | cut -d '&' -f 2)
|
||||||
|
[[ -s ~/.zen/tmp/$IPFSNODEID/$YID.TW.json ]] && echo "Tiddler json already existing : ~/.zen/tmp/$IPFSNODEID/$YID.TW.json" && continue ## TODO :: CHECK IF ALREADY YOURS OR NOT :: THEN ADD2TW / SEND MESSAGE ?
|
||||||
|
|
||||||
# SINGLE VIDEO YURL
|
# SINGLE VIDEO YURL
|
||||||
ZYURL="https://www.youtube.com/watch?v=$YID";
|
ZYURL=$(echo "$LINE" | cut -d '&' -f 3-)
|
||||||
echo "YOUTUBE : $ZYURL"
|
echo "COPIE : $ZYURL"
|
||||||
|
|
||||||
TITLE="$(yt-dlp --print "%(title)s" "${ZYURL}")"
|
TITLE="$(yt-dlp --print "%(title)s" "${ZYURL}")"
|
||||||
TITLE=${TITLE//[^A-zÀ-ÿ0-9 ]/}
|
TITLE=${TITLE//[^A-zÀ-ÿ0-9 ]/}
|
||||||
[[ ! $TITLE ]] && continue
|
[[ ! $TITLE ]] && echo "NO TITLE" && continue
|
||||||
|
|
||||||
echo ".... Downloading $TITLE.mp4"
|
echo ".... Downloading $TITLE.mp4"
|
||||||
|
|
||||||
|
@ -80,14 +107,18 @@ while read YID;
|
||||||
# SUBS ? --write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs
|
# SUBS ? --write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs
|
||||||
# (bv*[height<=720][vcodec~='^((he|a)vc|h26[45])']+ba)
|
# (bv*[height<=720][vcodec~='^((he|a)vc|h26[45])']+ba)
|
||||||
# TODO : DELAY COPY OPERATION... Astro can download quicker at 03:00 AM
|
# TODO : DELAY COPY OPERATION... Astro can download quicker at 03:00 AM
|
||||||
echo "yt-dlp -f \"(bv*[ext=mp4][height<=720]+ba/b[height<=720])\" --no-mtime --embed-thumbnail --add-metadata -o \"$HOME/.zen/tmp/$WISHKEY/$TITLE.%(ext)s\" ${ZYURL}"
|
echo "yt-dlp -f \"(bv*[ext=mp4][height<=720]+ba/b[height<=720])\" --no-mtime --embed-thumbnail --add-metadata -o \"$HOME/.zen/tmp/yt-dlp/$TITLE.%(ext)s\" ${ZYURL}"
|
||||||
|
|
||||||
#############################################################################
|
#############################################################################
|
||||||
## COPY FROM YOUTUBE (TODO DOUBLE COPY & MKV to MP4 OPTIMISATION)
|
## COPY FROM YOUTUBE (TODO DOUBLE COPY & MKV to MP4 OPTIMISATION)
|
||||||
|
## EXTRA PARAM TO TRY
|
||||||
|
# --cookies-from-browser browser (xdg-settings get default-web-browser) allow member copies
|
||||||
|
# --dateafter DATE --download-archive myarchive.txt
|
||||||
yt-dlp -f "(bv*[ext=mp4][height<=720]+ba/b[height<=720])" \
|
yt-dlp -f "(bv*[ext=mp4][height<=720]+ba/b[height<=720])" \
|
||||||
|
--download-archive $HOME/.zen/.yt-dlp.list \
|
||||||
-S "filesize:700M" --no-mtime --embed-thumbnail --add-metadata \
|
-S "filesize:700M" --no-mtime --embed-thumbnail --add-metadata \
|
||||||
--write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs \
|
--write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs \
|
||||||
-o "$HOME/.zen/tmp/$WISHKEY/$TITLE.%(ext)s" ${ZYURL}
|
-o "$HOME/.zen/tmp/yt-dlp/$TITLE.%(ext)s" ${ZYURL}
|
||||||
################################################################################
|
################################################################################
|
||||||
### ADAPT TO TW RYTHM (DELAY COPY?)
|
### ADAPT TO TW RYTHM (DELAY COPY?)
|
||||||
echo
|
echo
|
||||||
|
@ -96,21 +127,21 @@ while read YID;
|
||||||
|
|
||||||
############################################################################
|
############################################################################
|
||||||
### CHECK RESULT CONVERT MKV TO MP4
|
### CHECK RESULT CONVERT MKV TO MP4
|
||||||
[[ ! -f "$HOME/.zen/tmp/$WISHKEY/$ZFILE" ]] && ffmpeg -loglevel quiet -i "$HOME/.zen/tmp/$WISHKEY/$TITLE.mkv" -c:v libx264 -c:a aac "$HOME/.zen/tmp/$WISHKEY/$TITLE.mp4" # TRY TO CONVERT MKV TO MP4
|
[[ ! -f "$HOME/.zen/tmp/yt-dlp/$ZFILE" ]] && ffmpeg -loglevel quiet -i "$HOME/.zen/tmp/yt-dlp/$TITLE.mkv" -c:v libx264 -c:a aac "$HOME/.zen/tmp/yt-dlp/$TITLE.mp4" # TRY TO CONVERT MKV TO MP4
|
||||||
[[ ! -f "$HOME/.zen/tmp/$WISHKEY/$ZFILE" ]] && echo "No FILE -- CONTINUE --" && continue
|
[[ ! -f "$HOME/.zen/tmp/yt-dlp/$ZFILE" ]] && echo "No FILE -- CONTINUE --" && continue
|
||||||
echo
|
echo
|
||||||
|
|
||||||
####################################################
|
####################################################
|
||||||
echo "FOUND : ~/.zen/tmp/$WISHKEY/$ZFILE"
|
echo "FOUND : ~/.zen/tmp/yt-dlp/$ZFILE"
|
||||||
FILE_BSIZE=$(du -b "$HOME/.zen/tmp/$WISHKEY/$ZFILE" | awk '{print $1}')
|
FILE_BSIZE=$(du -b "$HOME/.zen/tmp/yt-dlp/$ZFILE" | awk '{print $1}')
|
||||||
FILE_SIZE=$(echo "${FILE_BSIZE}" | awk '{ split( "B KB MB GB TB PB" , v ); s=1; while( $1>1024 ){ $1/=1024; s++ } printf "%.2f %s", $1, v[s] }')
|
FILE_SIZE=$(echo "${FILE_BSIZE}" | awk '{ split( "B KB MB GB TB PB" , v ); s=1; while( $1>1024 ){ $1/=1024; s++ } printf "%.2f %s", $1, v[s] }')
|
||||||
echo "FILE SIZE = $FILE_SIZE ($FILE_BSIZE octets)"
|
echo "FILE SIZE = $FILE_SIZE ($FILE_BSIZE octets)"
|
||||||
|
|
||||||
echo "Adding to IPFS"
|
echo "Adding to IPFS"
|
||||||
ILINK=$(ipfs add -q "$HOME/.zen/tmp/$WISHKEY/$ZFILE" | tail -n 1)
|
ILINK=$(ipfs add -q "$HOME/.zen/tmp/yt-dlp/$ZFILE" | tail -n 1)
|
||||||
echo "/ipfs/$ILINK <=> $ZFILE"
|
echo "/ipfs/$ILINK <=> $ZFILE"
|
||||||
|
|
||||||
MIME=$(file --mime-type -b "$HOME/.zen/tmp/$WISHKEY/$ZFILE")
|
MIME=$(file --mime-type -b "$HOME/.zen/tmp/yt-dlp/$ZFILE")
|
||||||
|
|
||||||
## ADD TAGS
|
## ADD TAGS
|
||||||
SEC=$(yt-dlp --print "%(duration)s" "${ZYURL}")
|
SEC=$(yt-dlp --print "%(duration)s" "${ZYURL}")
|
||||||
|
@ -138,7 +169,7 @@ while read YID;
|
||||||
"tags": "'ipfs G1CopierYoutube ${WISHKEY} ${EXTRATAG} ${MIME}'"
|
"tags": "'ipfs G1CopierYoutube ${WISHKEY} ${EXTRATAG} ${MIME}'"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
' > "$HOME/.zen/tmp/$WISHKEY/$YID.TW.json"
|
' > "$HOME/.zen/tmp/$IPFSNODEID/$YID.TW.json"
|
||||||
|
|
||||||
|
|
||||||
#################################################################
|
#################################################################
|
||||||
|
@ -147,22 +178,23 @@ while read YID;
|
||||||
echo "=========================="
|
echo "=========================="
|
||||||
echo "Adding $YID tiddler to TW /ipns/$TWNS "
|
echo "Adding $YID tiddler to TW /ipns/$TWNS "
|
||||||
|
|
||||||
rm -f ~/.zen/tmp/$WISHKEY/newindex.html
|
rm -f ~/.zen/tmp/$IPFSNODEID/newindex.html
|
||||||
|
|
||||||
echo ">>> Importing $HOME/.zen/tmp/$WISHKEY/$YID.TW.json"
|
echo ">>> Importing $HOME/.zen/tmp/$IPFSNODEID/$YID.TW.json"
|
||||||
|
|
||||||
tiddlywiki --load $INDEX \
|
tiddlywiki --load $INDEX \
|
||||||
--import "$HOME/.zen/tmp/$WISHKEY/$YID.TW.json" "application/json" \
|
--import "$HOME/.zen/tmp/$IPFSNODEID/$YID.TW.json" "application/json" \
|
||||||
--deletetiddlers '[tag[CopierYoutube]]' \
|
--output ~/.zen/tmp/$IPFSNODEID --render "$:/core/save/all" "newindex.html" "text/plain"
|
||||||
--output ~/.zen/tmp/$WISHKEY --render "$:/core/save/all" "newindex.html" "text/plain"
|
|
||||||
|
|
||||||
if [[ -s ~/.zen/tmp/$WISHKEY/newindex.html ]]; then
|
# --deletetiddlers '[tag[CopierYoutube]]' ### REFRESH CHANNEL COPY
|
||||||
|
|
||||||
|
if [[ -s ~/.zen/tmp/$IPFSNODEID/newindex.html ]]; then
|
||||||
echo "$$$ Mise à jour $INDEX"
|
echo "$$$ Mise à jour $INDEX"
|
||||||
cp ~/.zen/tmp/$WISHKEY/newindex.html $INDEX
|
cp ~/.zen/tmp/$IPFSNODEID/newindex.html $INDEX
|
||||||
else
|
else
|
||||||
echo "Problem with tiddlywiki command. Missing ~/.zen/tmp/$WISHKEY/newindex.html"
|
echo "Problem with tiddlywiki command. Missing ~/.zen/tmp/$IPFSNODEID/newindex.html"
|
||||||
echo "XXXXXXXXXXXXXXXXXXXXXXX"
|
echo "XXXXXXXXXXXXXXXXXXXXXXX"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
done < ~/.zen/tmp/$WISHKEY/ytids.$MOATS # FINISH YID loop 1
|
done < ~/.zen/tmp/$IPFSNODEID/yt-dlp.cache # FINISH YID loop 1
|
||||||
exit 0
|
exit 0
|
||||||
|
|
Loading…
Reference in New Issue