csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: Host more working files in temporary folder

Browse source
This commit is contained in:
Carlos Solís 2025-06-13 20:16:04 +00:00
parent 24e9e9b427
commit 0cacefbcf8
1 changed files with 22 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

52
youtube-download-channel.sh
View file

@ -19,9 +19,9 @@ folder=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
#and place it next to your script. #and place it next to your script.
cookies="${folder}/yt-cookies.txt" cookies="${folder}/yt-cookies.txt"
subfolder="${folder}/subscriptions" subfolder="${folder}/subscriptions"
temporary="/tmp/subscriptions/${channel}" temporary="/tmp/subscriptions-${channel}"
archive="${subfolder}/${channel}.txt" archive="${subfolder}/${channel}.txt"
sortcsv="${subfolder}/${channel}-sort.csv" sortcsv="${temporary}/${channel}-sort.csv"
csv="${subfolder}/${channel}.csv" csv="${subfolder}/${channel}.csv"
json="${subfolder}/${channel}.db" json="${subfolder}/${channel}.db"
python="python" python="python"
@ -35,12 +35,10 @@ fi
if [[ ! -d "${subfolder}" ]]; then if [[ ! -d "${subfolder}" ]]; then
mkdir -v "${subfolder}" mkdir -v "${subfolder}"
fi fi
if [[ ! -d "/tmp/subscriptions" ]]; then
mkdir -v "/tmp/subscriptions"
fi
if [[ ! -d "${temporary}" ]]; then if [[ ! -d "${temporary}" ]]; then
mkdir -v "${temporary}" mkdir -v "${temporary}"
fi fi
#TODO: mount $temporary on RAM, if possible
cd "${temporary}" || exit cd "${temporary}" || exit
if [[ ! -f "${archive}" ]]; then if [[ ! -f "${archive}" ]]; then
touch "${archive}" touch "${archive}"
@ -58,7 +56,7 @@ if [[ "${channel}" = "subscriptions" ]]; then
fi fi
for full_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do for full_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
echo "${full_url}" echo "${full_url}"
if [[ -f "${cookies}" && "${channel}" = "subscriptions" ]]; then if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
#If available, you can use the cookies from your browser directly. Substitute #If available, you can use the cookies from your browser directly. Substitute
# --cookies "${cookies}" # --cookies "${cookies}"
#for the below, substituting for your browser of choice: #for the below, substituting for your browser of choice:
@ -114,6 +112,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
fi fi
fi fi
if [[ -f "${x}" ]]; then if [[ -f "${x}" ]]; then
echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt"
if [[ ${enablecsv} = "1" ]]; then if [[ ${enablecsv} = "1" ]]; then
jq -c '[.upload_date, .timestamp, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do jq -c '[.upload_date, .timestamp, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${csv}" echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${csv}"
@ -135,44 +134,37 @@ done
wait wait
sleep 1 sleep 1
if [[ ${enabledb} = "1" ]]; then if [[ ${enabledb} = "1" ]]; then
sort "${sortcsv}" | uniq >"/tmp/${channel}-sort-ordered.csv" sort "${sortcsv}" | uniq >"${temporary}/${channel}-sort-ordered.csv"
if [[ -f "/tmp/${channel}.db" ]]; then if [[ -f "${temporary}/${channel}.db" ]]; then
rm "/tmp/${channel}.db" rm "${temporary}/${channel}.db"
fi fi
echo "{\"playlistName\":\"${channel}\",\"protected\":false,\"description\":\"Videos from ${channel} to watch later\",\"videos\":[" >"/tmp/${channel}.db" echo "{\"playlistName\":\"${channel}\",\"protected\":false,\"description\":\"Videos from ${channel} to watch later\",\"videos\":[" >"${temporary}/${channel}.db"
count=0 count=0
total=$(wc -l <"/tmp/${channel}-sort-ordered.csv") total=$(wc -l <"${temporary}/${channel}-sort-ordered.csv")
while read -r line; do while read -r line; do
count=$((count + 1)) count=$((count + 1))
file=$(echo "${line}" | cut -d ',' -f3-) file=$(echo "${line}" | cut -d ',' -f3-)
if [[ -f "${file}" ]]; then if [[ -f "${file}" ]]; then
jq -c "{\"videoId\": .id, \"title\": .title, \"author\": .uploader, \"authorId\": .channel_id, \"lengthSeconds\": .duration, \"published\": ( .timestamp * 1000 ), \"timeAdded\": $(date +%s)$(date +%N | cut -c-3), \"playlistItemId\": \"$(cat /proc/sys/kernel/random/uuid)\", \"type\": \"video\"}" "${temporary}/${file}" >>"/tmp/${channel}.db" jq -c "{\"videoId\": .id, \"title\": .title, \"author\": .uploader, \"authorId\": .channel_id, \"lengthSeconds\": .duration, \"published\": ( .timestamp * 1000 ), \"timeAdded\": $(date +%s)$(date +%N | cut -c-3), \"playlistItemId\": \"$(cat /proc/sys/kernel/random/uuid)\", \"type\": \"video\"}" "${temporary}/${file}" >>"${temporary}/${channel}.db"
echo "," >>"/tmp/${channel}.db" echo "," >>"${temporary}/${channel}.db"
echo "${count}/${total} ${file}" echo "${count}/${total} ${file}"
fi fi
done <"/tmp/${channel}-sort-ordered.csv" done <"${temporary}/${channel}-sort-ordered.csv"
echo "],\"_id\":\"${channel}$(date +%s)\",\"createdAt\":$(date +%s),\"lastUpdatedAt\":$(date +%s)}" >>"/tmp/${channel}.db" echo "],\"_id\":\"${channel}$(date +%s)\",\"createdAt\":$(date +%s),\"lastUpdatedAt\":$(date +%s)}" >>"${temporary}/${channel}.db"
rm "${json}" rm "${json}"
grep -v -e ":[ ]*null" "/tmp/${channel}.db" | tr '\n' '\r' | sed -e "s/,\r[,\r]*/,\r/g" | sed -e "s/,\r\]/\]/g" -e "s/\[\r,/\[/g" | tr '\r' '\n' | jq -c . >"${json}" && rm "/tmp/${channel}.db" grep -v -e ":[ ]*null" "${temporary}/${channel}.db" | tr '\n' '\r' | sed -e "s/,\r[,\r]*/,\r/g" | sed -e "s/,\r\]/\]/g" -e "s/\[\r,/\[/g" | tr '\r' '\n' | jq -c . >"${json}" && rm "${temporary}/${channel}.db"
rm "/tmp/${channel}-sort-ordered.csv" "${sortcsv}" rm "${temporary}/${channel}-sort-ordered.csv" "${sortcsv}"
fi fi
if [[ ${enablecsv} = "1" ]]; then if [[ ${enablecsv} = "1" ]]; then
sort "${csv}" | uniq >"/tmp/${channel}-without-header.csv" sort "${csv}" | uniq >"${temporary}/${channel}-without-header.csv"
echo '"Upload Date", "Timestamp", "Uploader", "Title", "Webpage URL"' >"/tmp/${channel}.csv" echo '"Upload Date", "Timestamp", "Uploader", "Title", "Webpage URL"' >"${temporary}/${channel}.csv"
cat "/tmp/${channel}-without-header.csv" >>"/tmp/${channel}.csv" cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}.csv"
mv "/tmp/${channel}.csv" "${csv}" mv "${temporary}/${channel}.csv" "${csv}"
rm "/tmp/${channel}-without-header.csv" rm "${temporary}/${channel}-without-header.csv"
fi fi
cd "${temporary}" || exit cd "${temporary}" || exit
tar -cvp -I "zstd -T0" -f "${subfolder}/${channel}.tar.zst" -- *.info.json tar -cvp -I "zstd -T0" -f "${subfolder}/${channel}.tar.zst" -- *.info.json
count=0 count=0
total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l) total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l)
find "${temporary}" -type f -iname "*.info.json" | while read -r x; do sort "${temporary}/${channel}.txt" | uniq >"${archive}"
count=$((count + 1))
if [[ -f "${x}" ]]; then
echo "youtube $(jq -cr '.id' "${x}")" >>"${archive}"
echo "${count}/${total} ${x}"
fi
done
sort "${archive}" | uniq >"/tmp/${channel}.txt" && mv "/tmp/${channel}.txt" "${archive}"
rm -rf "${temporary}" rm -rf "${temporary}"