csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Correct parsing, add compression of data after finishing

Browse source
This commit is contained in:
Carlos Solís 2025-05-31 19:37:00 +00:00
parent dbefa868bf
commit 4b3c32e37b
1 changed files with 12 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

17
youtube-download-channel.sh
View file

@ -9,6 +9,7 @@ sleeptime=${3:-"1.0"}
#Internal variables: #Internal variables:
#Via https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script #Via https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script
folder=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) folder=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
echo "${folder}"
#Required to download your own subscriptions. #Required to download your own subscriptions.
#Obtain this file through the procedure listed at #Obtain this file through the procedure listed at
# https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp # https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp
@ -18,7 +19,7 @@ subfolder="${folder}/${channel}"
archive="${subfolder}/${channel}.txt" archive="${subfolder}/${channel}.txt"
sortcsv="${subfolder}/${channel}-sort.csv" sortcsv="${subfolder}/${channel}-sort.csv"
csv="${subfolder}/${channel}.csv" csv="${subfolder}/${channel}.csv"
json="${subfolder}/${channel}.json" json="${subfolder}/${channel}.db"
python="python" python="python"
if [[ -f "/opt/venv/bin/python" ]]; then if [[ -f "/opt/venv/bin/python" ]]; then
python="/opt/venv/bin/python" python="/opt/venv/bin/python"
@ -27,17 +28,20 @@ ytdl="/usr/bin/yt-dlp"
if [[ -f "/opt/venv/bin/yt-dlp" ]]; then if [[ -f "/opt/venv/bin/yt-dlp" ]]; then
ytdl="/opt/venv/bin/yt-dlp" ytdl="/opt/venv/bin/yt-dlp"
fi fi
if [[ -z "${subfolder}" ]]; then if [[ ! -d "${subfolder}" ]]; then
mkdir "${subfolder}" mkdir -v "${subfolder}"
fi fi
cd "${subfolder}" || exit cd "${subfolder}" || exit
if [[ -f "${channel}.tar.zst" ]]; then
tar -xvp -I zstd -f "${channel}.tar.zst"
fi
#If available, you can use the cookies from your browser directly: #If available, you can use the cookies from your browser directly:
# --cookies-from-browser "firefox" # --cookies-from-browser "firefox"
url="https://www.youtube.com/@${channel}" url="https://www.youtube.com/@${channel}"
if [[ "${channel}" = "subscriptions" ]]; then if [[ "${channel}" = "subscriptions" ]]; then
url="https://www.youtube.com/feed/subscriptions" url="https://www.youtube.com/feed/subscriptions"
fi fi
if [[ -z "${cookies}" ]]; then if [[ -z "${cookies}" && ${channel} = "subscriptions" ]]; then
"${python}" "${ytdl}" "${url}" \ "${python}" "${ytdl}" "${url}" \
--skip-download --download-archive "${archive}" \ --skip-download --download-archive "${archive}" \
--dateafter "${breaktime}" \ --dateafter "${breaktime}" \
@ -76,7 +80,9 @@ echo "{\"playlistName\":\"${channel}\",\"protected\":false,\"description\":\"Vid
while read -r line; do while read -r line; do
file=$(echo "${line}" | cut -d ',' -f3-) file=$(echo "${line}" | cut -d ',' -f3-)
echo "${file}" echo "${file}"
jq -c "{\"videoId\": .id, \"title\": .title, \"author\": .uploader, \"authorId\": .channel_id, \"lengthSeconds\": .duration, \"published\": .epoch, \"timeAdded\": $(date +%s), \"playlistItemId\": \"$(cat /proc/sys/kernel/random/uuid)\", \"type\": \"video\"}" "${subfolder}/${file}" | tee -a "/tmp/${channel}.db" jq -c "{\"videoId\": .id, \"title\": .title, \"author\": .uploader, \"authorId\": .channel_id, \
\"lengthSeconds\": .duration, \"published\": ( .timestamp * 1000 ) , \"timeAdded\": $(date +%s)$(date +%N | cut -c-3), \
\"playlistItemId\": \"$(cat /proc/sys/kernel/random/uuid)\", \"type\": \"video\"}" "${subfolder}/${file}" | tee -a "/tmp/${channel}.db"
echo "," >>"/tmp/${channel}.db" echo "," >>"/tmp/${channel}.db"
done <"/tmp/${channel}-sort-ordered.csv" done <"/tmp/${channel}-sort-ordered.csv"
echo "],\"_id\":\"${channel}\",\"createdAt\":$(date +%s),\"lastUpdatedAt\":$(date +%s)}" >>"/tmp/${channel}.db" echo "],\"_id\":\"${channel}\",\"createdAt\":$(date +%s),\"lastUpdatedAt\":$(date +%s)}" >>"/tmp/${channel}.db"
@ -90,3 +96,4 @@ mv "/tmp/${channel}.csv" "${csv}"
rm "/tmp/${channel}-without-header.csv" rm "/tmp/${channel}-without-header.csv"
sort "${archive}" | uniq >"/tmp/${channel}.txt" sort "${archive}" | uniq >"/tmp/${channel}.txt"
mv "/tmp/${channel}.txt" "${archive}" mv "/tmp/${channel}.txt" "${archive}"
tar -cvp -I zstd -f "${channel}.tar.zst" ./*.info.json && rm ./*.info.json