csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix: Optimize CSV generation

Browse source
This commit is contained in:
Carlos Solís 2025-06-11 14:17:03 +00:00
parent 2c265e5f4c
commit c67577c624
1 changed files with 13 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

27
youtube-download-channel.sh
View file

@ -83,14 +83,18 @@ else
--break-on-reject --lazy-playlist --write-info-json \ --break-on-reject --lazy-playlist --write-info-json \
--sleep-requests "${sleeptime}" --sleep-requests "${sleeptime}"
fi fi
if [[ -f "${csv}" ]]; then if [[ ${enablecsv} = 1 ]]; then
rm -rf "${csv}" if [[ -f "${csv}" ]]; then
rm -rf "${csv}"
fi
touch "${csv}"
fi fi
touch "${csv}" if [[ ${enabledb} = 1 ]]; then
if [[ -f "${sortcsv}" ]]; then if [[ -f "${sortcsv}" ]]; then
rm -rf "${sortcsv}" rm -rf "${sortcsv}"
fi
touch "${sortcsv}"
fi fi
touch "${sortcsv}"
breaktime_timestamp=$(date -d"${breaktime}" +"%s") breaktime_timestamp=$(date -d"${breaktime}" +"%s")
count=0 count=0
total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l) total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l)
@ -113,7 +117,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${csv}" echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${csv}"
done done
fi fi
if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then if [[ ${enabledb} = "1" ]]; then
jq -c '[.upload_date, .timestamp]' "${x}" | while read -r i; do jq -c '[.upload_date, .timestamp]' "${x}" | while read -r i; do
echo "${i},${x##*/}" | sed -e "s/^\[//g" -e "s/\],/,/g" -e "s/\\\\\"/＂/g" >>"${sortcsv}" echo "${i},${x##*/}" | sed -e "s/^\[//g" -e "s/\],/,/g" -e "s/\\\\\"/＂/g" >>"${sortcsv}"
done done
@ -127,10 +131,9 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
done done
wait wait
if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then sleep 1
sort "${sortcsv}" | uniq >"/tmp/${channel}-sort-ordered.csv"
fi
if [[ ${enabledb} = "1" ]]; then if [[ ${enabledb} = "1" ]]; then
sort "${sortcsv}" | uniq >"/tmp/${channel}-sort-ordered.csv"
if [[ -f "/tmp/${channel}.db" ]]; then if [[ -f "/tmp/${channel}.db" ]]; then
rm "/tmp/${channel}.db" rm "/tmp/${channel}.db"
fi fi
@ -146,13 +149,9 @@ if [[ ${enabledb} = "1" ]]; then
echo "${count}/${total} ${file}" echo "${count}/${total} ${file}"
fi fi
done <"/tmp/${channel}-sort-ordered.csv" done <"/tmp/${channel}-sort-ordered.csv"
fi
if [[ ${enabledb} = "1" ]]; then
echo "],\"_id\":\"${channel}$(date +%s)\",\"createdAt\":$(date +%s),\"lastUpdatedAt\":$(date +%s)}" >>"/tmp/${channel}.db" echo "],\"_id\":\"${channel}$(date +%s)\",\"createdAt\":$(date +%s),\"lastUpdatedAt\":$(date +%s)}" >>"/tmp/${channel}.db"
rm "${json}" rm "${json}"
grep -v -e ":[ ]*null" "/tmp/${channel}.db" | tr '\n' '\r' | sed -e "s/,\r[,\r]*/,\r/g" | sed -e "s/,\r\]/\]/g" -e "s/\[\r,/\[/g" | tr '\r' '\n' | jq -c . >"${json}" && rm "/tmp/${channel}.db" grep -v -e ":[ ]*null" "/tmp/${channel}.db" | tr '\n' '\r' | sed -e "s/,\r[,\r]*/,\r/g" | sed -e "s/,\r\]/\]/g" -e "s/\[\r,/\[/g" | tr '\r' '\n' | jq -c . >"${json}" && rm "/tmp/${channel}.db"
fi
if [[ ${enablecsv} = "1" || ${enabledb} = "1" ]]; then
rm "/tmp/${channel}-sort-ordered.csv" "${sortcsv}" rm "/tmp/${channel}-sort-ordered.csv" "${sortcsv}"
fi fi
if [[ ${enablecsv} = "1" ]]; then if [[ ${enablecsv} = "1" ]]; then