STI/Astroport.ONE
STI
/
Astroport.ONE
4
1
Fork 2
Code Issues 11 Pull Requests Projects Releases 1 Wiki Activity
Astroport.ONE/ASTROBOT/G1CopierYoutube.sh

286 lines
13 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
########################################################################
# Version: 0.4
# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/)
########################################################################
MY_PATH="`dirname \"$0\"`" # relative
MY_PATH="`( cd \"$MY_PATH\" && pwd )`" # absolutized and normalized
ME="${0##*/}"
. "${MY_PATH}/../tools/my.sh"
echo "-----"
echo "$ME RUNNING"
# ASTROBOT FIRST SPECIFIC PROCESS
# "Copier youtube" + (voeu) => CopierYoutube (G1Voeu G1CopierYoutube) = ASTROBOT copy Ŋ1 "(G1CopierYoutube)"
INDEX="$1"
[[ ! ${INDEX} ]] && echo "ERROR - Please provide path to source TW index.html" && exit 1
[[ ! -s ${INDEX} ]] && echo "ERROR - Fichier TW absent. ${INDEX}" && exit 1
PLAYER="$2"
[[ ! ${PLAYER} ]] && echo "ERROR - Please provide IPFS publish key" && exit 1
ASTRONAUTENS=$(ipfs key list -l | grep -w ${PLAYER} | cut -d ' ' -f1)
[[ ! $ASTRONAUTENS ]] && echo "ERROR - Clef IPNS ${PLAYER} introuvable!" && exit 1
G1PUB=$(cat ~/.zen/game/players/${PLAYER}/.g1pub)
# Extract tag=tube from TW
MOATS="$3"
[[ ! $MOATS ]] && MOATS=$(date -u +"%Y%m%d%H%M%S%4N")
###################################################################
## CREATE APP NODE PLAYER PUBLICATION DIRECTORY
###################################################################
mkdir -p ${HOME}/.zen/tmp/${IPFSNODEID}/G1CopierYoutube/${PLAYER}/
mkdir -p ${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/
###################################################################
## tag[CopierYoutube] EXTRACT ~/.zen/tmp/CopierYoutube.json FROM TW
###################################################################
rm -f ~/.zen/game/players/${PLAYER}/G1CopierYoutube/CopierYoutube.json
tiddlywiki --load ${INDEX} \
--output ~/.zen/game/players/${PLAYER}/G1CopierYoutube \
--render '.' 'CopierYoutube.json' 'text/plain' '$:/core/templates/exporters/JsonFile' 'exportFilter' '[tag[CopierYoutube]]'
echo "DEBUG : cat ~/.zen/game/players/${PLAYER}/G1CopierYoutube/CopierYoutube.json | jq -r"
BZER=$(xdg-settings get default-web-browser | cut -d '.' -f 1 | cut -d '-' -f 1) ## GET cookies-from-browser
[[ $BZER ]] && BROWSER="--cookies-from-browser $BZER " || BROWSER=""
[[ ! $isLAN ]] && BROWSER=""
###################################################################
## URL EXTRACTION & yt-dlp.cache.${PLAYER} upgrade
for YURL in $(cat ~/.zen/game/players/${PLAYER}/G1CopierYoutube/CopierYoutube.json | jq -r '.[].text' | grep 'http'); do
[[ ! $(echo $YURL | grep "http" ) ]] && echo "$YURL error" && continue
echo "G1CopierYoutube : $YURL"
echo "Extracting video playlist into yt-dlp.cache.${PLAYER}"
### yt-dlp.command
CMD=$(cat ~/.zen/game/players/${PLAYER}/G1CopierYoutube/yt-dlp.command 2>/dev/null | grep "$YURL" | tail -n 1)
if [[ ! $CMD ]]; then
echo "${PLAYER}&$YURL:$MOATS" >> ~/.zen/game/players/${PLAYER}/G1CopierYoutube/yt-dlp.command
echo "NOUVEAU CANAL ${PLAYER}&$YURL:$MOATS"
lastrun=$MOATS
duree=604800000
else
lastrun=$(echo "$CMD" | rev | cut -d ':' -f 1 | rev) && echo "$CMD"
duree=$(expr ${MOATS} - $lastrun)
fi
# ONE WEEK NEW SCAN
if [[ $duree -ge 604800000 || ! -s ~/.zen/game/players/${PLAYER}/G1CopierYoutube/yt-dlp.cache.${PLAYER} ]]; then
/usr/local/bin/yt-dlp $BROWSER --print "%(id)s&%(webpage_url)s" "${YURL}" >> ~/.zen/game/players/${PLAYER}/G1CopierYoutube/yt-dlp.cache.${PLAYER}
sed -i "s~$lastrun~$MOATS~g" ~/.zen/game/players/${PLAYER}/G1CopierYoutube/yt-dlp.command # UPDATE LASTRUN
fi
done # FINISH YURL loop
## CREATE SORT UNIQ SHUFFLED ~/.zen/tmp/${IPFSNODEID}/yt-dlp.cache.${PLAYER} (12345 ONLINE)
cat ~/.zen/game/players/${PLAYER}/G1CopierYoutube/yt-dlp.cache.${PLAYER} 2>/dev/null | sort | uniq | shuf > ~/.zen/tmp/${IPFSNODEID}/yt-dlp.cache.${PLAYER}
###################################################################
[[ ! -s ~/.zen/tmp/${IPFSNODEID}/yt-dlp.cache.${PLAYER} ]] && echo "AUCUN YOUTUBEID pour CopierYoutube" && exit 0
###################################################################
boucle=0
tot=0
###################################################################
# PROCESS YOUTUBEID VIDEO DOWNLOAD AND CREATE TIDDLER in TW
###################################################################
while read LINE;
do
boucle=$((boucle+1))
echo "_____ $LINE _____ $boucle"
YID="$(echo "$LINE" | rev | cut -d '=' -f 1 | rev )"
###################################################################
## Search for $YID.TW.json TIDDLER in local & MySwarm cache
echo "--- CACHE SEARCH FOR $YID ---" && TIDDLER=$(ls -t "${HOME}/.zen/game/players/"*"/G1CopierYoutube/$YID.TW.json" 2>/dev/null | head -n 1)
[[ ! $TIDDLER ]] && TIDDLER=$(ls -t "${HOME}/.zen/tmp/${IPFSNODEID}/G1CopierYoutube/"*"/$YID.TW.json" 2>/dev/null | head -n 1)
[[ ! $TIDDLER ]] && TIDDLER=$(ls -t "${HOME}/.zen/tmp/swarm/"*"/G1CopierYoutube/"*"/$YID.TW.json" 2>/dev/null | head -n 1)
[[ $TIDDLER ]] && echo "Tiddler Found in CACHE : $TIDDLER" \
|| echo "EMPTY."
###################################################################
if [[ ! ${TIDDLER} ]]; then
###################################################################
# COPY VIDEO AND MAKE TIDDLER
###################################################################
ZYURL=$(echo "$LINE" | cut -d '&' -f 2-)
echo "COPIE : $ZYURL"
## LIMIT TO 12 MAXIMUM COPY PER DAY PER PLAYER
[[ $tot == 13 ]] && echo "MAXIMUM COPY REACHED FOR TODAY" && continue
TITLE="$(/usr/local/bin/yt-dlp $BROWSER --print "%(title)s" "${ZYURL}" | detox --inline)"
[[ ! $TITLE ]] && echo "NO TITLE" && continue
start=`date +%s`
echo ".... Downloading $TITLE.mp4"
# https://github.com/yt-dlp/yt-dlp#format-selection-examples
# SUBS ? --write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs
# (bv*[height<=720][vcodec~='^((he|a)vc|h26[45])']+ba)
# TODO : DELAY COPY OPERATION... Astro can download quicker at 03:00 AM
echo "/usr/local/bin/yt-dlp -f \"(bv*[ext=mp4][height<=720]+ba/b[height<=720])\" --no-mtime --embed-thumbnail --add-metadata -o \"${HOME}/.zen/tmp/yt-dlp/$TITLE.%(ext)s\" ${ZYURL}"
#############################################################################
## COPY FROM YOUTUBE (TODO DOUBLE COPY & MKV to MP4 OPTIMISATION)
## EXTRA PARAM TO TRY
# --write-subs --write-auto-subs --sub-langs "fr, en, en-orig" --embed-subs
/usr/local/bin/yt-dlp -f "(bv*[ext=mp4][height<=720]+ba/b[height<=720])" \
$BROWSER \
--download-archive ${HOME}/.zen/.yt-dlp.list \
-S res,ext:mp4:m4a --recode mp4 --no-mtime --embed-thumbnail --add-metadata \
-o "${HOME}/.zen/tmp/yt-dlp/$TITLE.%(ext)s" ${ZYURL}
################################################################################
### ADAPT TO TW RYTHM (DELAY COPY?)
echo
ZFILE="$TITLE.mp4"
############################################################################
### CHECK RESULT CONVERT MKV TO MP4
[[ -s "${HOME}/.zen/tmp/yt-dlp/$TITLE.mkv" ]] && ffmpeg -loglevel quiet -i "${HOME}/.zen/tmp/yt-dlp/$TITLE.mkv" -c:v libx264 -c:a aac "${HOME}/.zen/tmp/yt-dlp/$TITLE.mp4" # TRY TO CONVERT MKV TO MP4
if [[ ! -s "${HOME}/.zen/tmp/yt-dlp/${ZFILE}" ]]; then
echo "No FILE -- TRYING TO RESTORE CACHE FROM TW --"
tiddlywiki --load ${INDEX} \
--output ~/.zen/game/players/${PLAYER}/G1CopierYoutube \
--render '.' "$YID.TW.json" 'text/plain' '$:/core/templates/exporters/JsonFile' 'exportFilter' "${ZFILE}" \
&& rm "${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/${ZFILE}.json" && ln -s "${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/$YID.TW.json" "${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/${ZFILE}.json"
continue
fi
echo
####################################################
echo "FOUND : ~/.zen/tmp/yt-dlp/${ZFILE}"
FILE_BSIZE=$(du -b "${HOME}/.zen/tmp/yt-dlp/${ZFILE}" | awk '{print $1}')
[[ ! $FILE_BSIZE ]] && echo "SIZE ERROR" && continue
FILE_SIZE=$(echo "${FILE_BSIZE}" | awk '{ split( "B KB MB GB TB PB" , v ); s=1; while( $1>1024 ){ $1/=1024; s++ } printf "%.2f %s", $1, v[s] }')
echo "$boucle - ${ZFILE} - FILE SIZE = $FILE_SIZE ($FILE_BSIZE octets)"
espeak "GOOD! Video Number $boucle = $FILE_SIZE" > /dev/null 2>&1
### CREATE GIF ANIM : make_video_gifanim_ipfs.sh
$(${MY_PATH}/../tools/make_video_gifanim_ipfs.sh "${HOME}/.zen/tmp/yt-dlp" "${ZFILE}" | tail -n 1) ## export ANIMH
echo "HOP=$HOP
ANIMH=$ANIMH
PROBETIME=$PROBETIME
DURATION=$DURATION
DUREE=$DUREE
RES=$RES
MIME=$MIME
VTRATIO=$VTRATIO
file=$file"
## Create gifanime ## TODO Search for similarities BEFORE ADD
echo "Adding to IPFS"
ILINK=$(ipfs add -q "${HOME}/.zen/tmp/yt-dlp/${ZFILE}" | tail -n 1)
echo "/ipfs/$ILINK === ${ZFILE}"
[[ $ILINK == "" ]] && echo ">>>>> BIG PROBLEM PAPA. NO IPFS " && continue
MIME=$(file --mime-type -b "${HOME}/.zen/tmp/yt-dlp/${ZFILE}")
## ADD TAGS
SEC=$(/usr/local/bin/yt-dlp $BROWSER --print "%(duration)s" "${ZYURL}")
CHANNEL=$(/usr/local/bin/yt-dlp $BROWSER --print "%(channel)s" "${ZYURL}" | sed -r 's/\<./\U&/g' | sed 's/ //g') # CapitalGluedWords
PLAYLIST=$(/usr/local/bin/yt-dlp $BROWSER --print "%(playlist)s" "${ZYURL}" | sed -r 's/\<./\U&/g' | sed 's/ //g')
EXTRATAG="$CHANNEL $PLAYLIST"
## PREPARE VIDEO HTML5 CODE
TEXT="<video controls width=100% poster='/ipfs/"${ANIMH}"'>
<source src='/ipfs/"${ILINK}"' type='"${MIME}"'>
</video>
<br>
{{!!filesize}} - {{!!duration}} sec. - vtratio(dur) = {{!!vtratio}} ({{!!dur}})
<br>
<h1><a href='"${ZYURL}"'>"${TITLE}"</a></h1>"
end=`date +%s`
dur=`expr $end - $start`
echo "Creating Youtube \"${YID}\" tiddler : G1CopierYoutube !"
## WAN ADD <<hide tiddler-controls>> TO text jq 'map(.text += "<<hide tiddler-controls>>")'
[[ ! isLAN ]] && TEXT="$TEXT <<hide tiddler-controls>>"
echo $TEXT
echo '[
{
"created": "'${MOATS}'",
"resolution": "'${RES}'",
"duree": "'${DUREE}'",
"duration": "'${DURATION}'",
"giftime": "'${PROBETIME}'",
"gifanime": "'/ipfs/${ANIMH}'",
"modified": "'${MOATS}'",
"title": "'${ZFILE}'",
"type": "'text/vnd.tiddlywiki'",
"vtratio": "'${VTRATIO}'",
"text": "'$TEXT'",
"g1pub": "'${G1PUB}'",
"mime": "'${MIME}'",
"size": "'${FILE_BSIZE}'",
"filesize": "'${FILE_SIZE}'",
"sec": "'${SEC}'",
"dur": "'${dur}'",
"ipfs": "'/ipfs/${ILINK}'",
"youtubeid": "'${YID}'",
"zurl": "'${ZYURL}'",
"tags": "'ipfs G1CopierYoutube ${PLAYER} ${EXTRATAG} ${MIME} $(echo ${ZFILE} | sed 's~_~ ~g' | sed 's~\.~ ~g')'"
}
]
' > "${HOME}/.zen/tmp/${IPFSNODEID}/G1CopierYoutube/${PLAYER}/$YID.TW.json"
tot=$((tot+1))
TIDDLER="${HOME}/.zen/tmp/${IPFSNODEID}/G1CopierYoutube/${PLAYER}/$YID.TW.json"
else
###################################################################
echo
###################################################################
## TODO : ADD EMAIL TAG ( TIMESTAMP & ADD SIGNATURE over existing ones)
continue
fi
cp -f "${TIDDLER}" "${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/"
#################################################################
### ADDING $YID.TW.json to ASTRONAUTENS INDEX.html
#################################################################
echo "=========================="
echo "Adding $YID tiddler to TW /ipns/$ASTRONAUTENS "
rm -f ~/.zen/tmp/${IPFSNODEID}/newindex.html
echo ">>> Importing ${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/$YID.TW.json"
tiddlywiki --load ${INDEX} \
--import "${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/$YID.TW.json" "application/json" \
--output ~/.zen/tmp/${IPFSNODEID} --render "$:/core/save/all" "newindex.html" "text/plain"
# --deletetiddlers '[tag[CopierYoutube]]' ### REFRESH CHANNEL COPY
if [[ -s ~/.zen/tmp/${IPFSNODEID}/newindex.html ]]; then
## COPY JSON TIDDLER TO PLAYER
ln -s "${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/$YID.TW.json" "${HOME}/.zen/game/players/${PLAYER}/G1CopierYoutube/${ZFILE}.json"
[[ $(diff ~/.zen/tmp/${IPFSNODEID}/newindex.html ${INDEX} ) ]] && cp ~/.zen/tmp/${IPFSNODEID}/newindex.html ${INDEX} && echo "===> Mise à jour ${INDEX}"
else
echo "Problem with tiddlywiki command. Missing ~/.zen/tmp/${IPFSNODEID}/newindex.html"
echo "XXXXXXXXXXXXXXXXXXXXXXX"
fi
done < ~/.zen/tmp/${IPFSNODEID}/yt-dlp.cache.${PLAYER} # FINISH YID loop 1
exit 0
Reference in New Issue View Git Blame Copy Permalink