csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix: Correct syntax errors, optimize stale file deletion

Browse source
This commit is contained in:
Carlos Solís 2025-06-09 05:07:30 +00:00
parent 60147fad1d
commit 74af7e44ea
1 changed files with 39 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

79
youtube-download-channel.sh
View file

@ -41,7 +41,7 @@ if [[ ! -f "${archive}" ]]; then
fi fi
if [[ -f "${channel}.tar.zst" ]]; then if [[ -f "${channel}.tar.zst" ]]; then
if [[ "${channel}" = "subscriptions" ]]; then if [[ "${channel}" = "subscriptions" ]]; then
find . -iname "*.tar.zst" | while read -r c; do tar -xvp -I zstd -f "${c}.tar.zst"; done find . -iname "*.tar.zst" | while read -r c; do tar -xvp -I zstd -f "${c}"; done
else else
tar -xvp -I zstd -f "${channel}.tar.zst" tar -xvp -I zstd -f "${channel}.tar.zst"
fi fi
@ -52,7 +52,7 @@ url="https://www.youtube.com/@${channel}"
if [[ "${channel}" = "subscriptions" ]]; then if [[ "${channel}" = "subscriptions" ]]; then
url="https://www.youtube.com/feed/subscriptions" url="https://www.youtube.com/feed/subscriptions"
fi fi
if [[ -z "${cookies}" && "${channel}" = "subscriptions" ]]; then if [[ -f "${cookies}" && "${channel}" = "subscriptions" ]]; then
"${python}" "${ytdl}" "${url}" \ "${python}" "${ytdl}" "${url}" \
--skip-download --download-archive "${archive}" \ --skip-download --download-archive "${archive}" \
--dateafter "${breaktime}" \ --dateafter "${breaktime}" \
@ -68,37 +68,41 @@ else
--break-on-reject --lazy-playlist --write-info-json \ --break-on-reject --lazy-playlist --write-info-json \
--sleep-requests "${sleeptime}" --sleep-requests "${sleeptime}"
fi fi
rm -rf "${csv}" if [[ -f "${csv}" ]]; then
rm -rf "${csv}"
fi
if [[ ! -f "${sortcsv}" ]]; then if [[ ! -f "${sortcsv}" ]]; then
touch "${sortcsv}" touch "${sortcsv}"
fi fi
db=$(date -d"${breaktime}" +"%s")
find . -type f -iname "*.info.json" -exec ls -t {} + | while read -r xp; do find . -type f -iname "*.info.json" -exec ls -t {} + | while read -r xp; do
x="${xp##./}" x="${xp##./}"
df=$(jq -rc '.timestamp' "${subfolder}/${x}") if [[ "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${subfolder}/${x}") != "@${channel}" ]]; then
touch "${subfolder}/${x}" -d "@${df}" echo "Video ${x} not uploaded from ${channel}, removing..." && rm "${subfolder}/${x}" &
#TODO: Read the date directly
#df=$(jq -rc '.timestamp' "${subfolder}/${x}")
#if [[ "${breaktime}" =~ ^[0-9]+$ ]]; then
# db=$(date -d"$breaktime" +"%s")
# touch "${subfolder}/${x}" -d "${df}"
# if [[ "${db}" -ge "${df}" ]]; then
# echo "Video ${file} uploaded on ${uploaddate}, removing..."
# rm "${x}"
# fi
#fi
echo "youtube $(jq -cr '.id' "${x}")" | tee -a "${archive}" &
if [[ ${enablecsv} = "1" ]]; then
jq -c '[.upload_date, .timestamp, .uploader , .title, .webpage_url]' "${subfolder}/${x}" | while read -r i; do
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" | tee -a "${csv}" &
done
fi fi
if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then if [[ -f "${subfolder}/${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then
jq -c '[.upload_date, .timestamp]' "${subfolder}/${x}" | while read -r i; do df=$(jq -rc '.timestamp' "${subfolder}/${x}")
echo "${i},${x}" | sed -e "s/^\[//g" -e "s/\],/,/g" -e "s/\\\\\"/＂/g" | tee -a "${sortcsv}" & if [[ "${db}" -ge "${df}" ]]; then
done echo "Video ${x} uploaded before ${breaktime}, removing..." && rm "${subfolder}/${x}" &
else
touch "${subfolder}/${x}" -d "@${df}" &
fi
fi fi
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 3 * 2)) ]]; then if [[ -f "${subfolder}/${x}" ]]; then
wait -n echo "youtube $(jq -cr '.id' "${subfolder}/${x}")" | tee -a "${archive}" &
if [[ ${enablecsv} = "1" ]]; then
jq -c '[.upload_date, .timestamp, .uploader , .title, .webpage_url]' "${subfolder}/${x}" | while read -r i; do
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" | tee -a "${csv}" &
done
fi
if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then
jq -c '[.upload_date, .timestamp]' "${subfolder}/${x}" | while read -r i; do
echo "${i},${x}" | sed -e "s/^\[//g" -e "s/\],/,/g" -e "s/\\\\\"/＂/g" | tee -a "${sortcsv}" &
done
fi
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 3 * 2)) ]]; then
wait -n
fi
fi fi
done done
wait wait
@ -107,23 +111,18 @@ if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then
fi fi
if [[ ${enabledb} = "1" ]]; then if [[ ${enabledb} = "1" ]]; then
rm "/tmp/${channel}.db" rm "/tmp/${channel}.db"
echo "{\"playlistName\":\"${channel}\",\"protected\":false,\"description\":\"Videos to watch later\",\"videos\":[" >"/tmp/${channel}.db" echo "{\"playlistName\":\"${channel}\",\"protected\":false,\"description\":\"Videos from ${channel} to watch later\",\"videos\":[" >"/tmp/${channel}.db"
fi fi
if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then
while read -r line; do while read -r line; do
file=$(echo "${line}" | cut -d ',' -f3-) file=$(echo "${line}" | cut -d ',' -f3-)
echo "${file}" #if [[ "${breaktime}" =~ ^[0-9]+$ ]]; then
if [[ "${breaktime}" =~ ^[0-9]+$ ]]; then # uploaddate=$(echo "${line}" | cut -d ',' -f1 | sed -e "s/\"//g")
uploaddate=$(echo "${line}" | cut -d ',' -f1 | sed -e "s/\"//g") # if [[ "${uploaddate}" -lt "${breaktime}" ]]; then
if [[ "${uploaddate}" -lt "${breaktime}" ]]; then # echo "Video ${file} uploaded on ${uploaddate}, removing..."
echo "Video ${file} uploaded on ${uploaddate}, removing..." # rm "${file}"
rm "${file}" # fi
fi #fi
fi
if [[ "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${file}") != "@${channel}" ]]; then
echo "Video ${file} not uploaded from ${channel}, removing..."
rm "${file}"
fi
if [[ ${enabledb} = "1" ]]; then if [[ ${enabledb} = "1" ]]; then
if [[ -f "${file}" ]]; then if [[ -f "${file}" ]]; then
jq -c "{\"videoId\": .id, \"title\": .title, \"author\": .uploader, \"authorId\": .channel_id, \"lengthSeconds\": .duration, \"published\": ( .timestamp * 1000 ), \"timeAdded\": $(date +%s)$(date +%N | cut -c-3), \"playlistItemId\": \"$(cat /proc/sys/kernel/random/uuid)\", \"type\": \"video\"}" "${subfolder}/${file}" | tee -a "/tmp/${channel}.db" jq -c "{\"videoId\": .id, \"title\": .title, \"author\": .uploader, \"authorId\": .channel_id, \"lengthSeconds\": .duration, \"published\": ( .timestamp * 1000 ), \"timeAdded\": $(date +%s)$(date +%N | cut -c-3), \"playlistItemId\": \"$(cat /proc/sys/kernel/random/uuid)\", \"type\": \"video\"}" "${subfolder}/${file}" | tee -a "/tmp/${channel}.db"
@ -149,4 +148,4 @@ if [[ ${enablecsv} = "1" ]]; then
fi fi
sort "${archive}" | uniq >"/tmp/${channel}.txt" sort "${archive}" | uniq >"/tmp/${channel}.txt"
mv "/tmp/${channel}.txt" "${archive}" mv "/tmp/${channel}.txt" "${archive}"
tar -cvp -I zstd -f "${channel}.tar.zst" ./*.info.json && rm ./*.info.json tar -cvp -I zstd -f "${channel}.tar.zst" -- *.info.json && rm -- *.info.json