csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: csolisr:06bbe1cf7d3b47b8053da0fe36fe15d8ce8f49fc
Branches Tags
csolisr:main
..
compare: csolisr:0552252055dc4fe9d9d7684a2331ddb1b9de3edf
Branches Tags
csolisr:main

3 commits
06bbe1cf7d ... 0552252055

Author SHA1 Message Date
Carlos Solís
0552252055 fix: Simplify file locks and index generation 2025-07-09 15:50:49 +00:00
Carlos Solís
ec467c3bc2 fix: Increase threshold for file size 2025-07-09 15:50:03 +00:00
Carlos Solís
293b11dad8 feat: Add support for Watch Later, add playlist info to CSV 2025-07-09 15:49:32 +00:00
3 changed files with 34 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

2
friendica-compress-storage.sh
View file

@ -55,7 +55,7 @@ loop_1() {
echo "Generating photo index..." #&> /dev/null echo "Generating photo index..." #&> /dev/null
sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null
echo "Generating list of files..." #&> /dev/null echo "Generating list of files..." #&> /dev/null
find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | ( find "${storagefolder}" -depth -mindepth 2 -type f -size +100k -mtime -8 -not -iname "index.html" | (
while read -r p; do while read -r p; do
loop_1 "${p}" & loop_1 "${p}" &
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do

58
friendica-delete-old-users.sh
View file

@ -17,8 +17,7 @@ starterid=${3:-"0"}
db="friendica" db="friendica"
period="${period_amount} MONTH" period="${period_amount} MONTH"
tmpfile=/tmp/friendica-delete-old-users.csv tmpfile=/tmp/friendica-delete-old-users.csv
tmpreadlock=/tmp/friendica-delete-old-users.read.tmp tmplock=/tmp/friendica-delete-old-users.tmp
tmpwritelock=/tmp/friendica-delete-old-users.write.tmp
loop() { loop() {
baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g")
@ -33,7 +32,7 @@ loop() {
if [[ -n $(type flock) ]]; then if [[ -n $(type flock) ]]; then
isreadlocked=0 isreadlocked=0
while [[ "${isreadlocked}" -eq 0 ]]; do while [[ "${isreadlocked}" -eq 0 ]]; do
exec 9>"${tmpreadlock}" exec 9>"${tmplock}"
if flock -n -e 9; then if flock -n -e 9; then
isreadlocked=1 isreadlocked=1
if [[ -f "${tmpfile}" ]]; then if [[ -f "${tmpfile}" ]]; then
@ -66,11 +65,11 @@ loop() {
flock -u 9 flock -u 9
iswritelocked=0 iswritelocked=0
while [[ "${iswritelocked}" -eq 0 ]]; do while [[ "${iswritelocked}" -eq 0 ]]; do
exec 8>"${tmpwritelock}" exec 9>"${tmplock}"
if flock -n -e 8; then if flock -n -e 9; then
iswritelocked=1 iswritelocked=1
echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}" echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}"
flock -u 8 flock -u 9
fi fi
done done
fi fi
@ -149,23 +148,18 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
if [[ -f "${tmpfile}" ]]; then if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}" rm -rf "${tmpfile}"
fi fi
if [[ -f "${tmpreadlock}" ]]; then if [[ -f "${tmplock}" ]]; then
rm -rf "${tmpreadlock}" rm -rf "${tmplock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi fi
touch "${tmpfile}" touch "${tmpfile}"
echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}" echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}"
if [[ "${intense_optimizations}" -gt 0 ]]; then if [[ "${intense_optimizations}" -gt 0 ]]; then
"${dbengine}" "${db}" -v -e "\ "${dbengine}" "${db}" -v -e "alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr)"
alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \ "${dbengine}" "${db}" -v -e "alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ "${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ "${dbengine}" "${db}" -v -e "alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ "${dbengine}" "${db}" -v -e "alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ "${dbengine}" "${db}" -v -e "alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`)"
alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \
"
fi fi
counter=0 counter=0
was_empty=0 was_empty=0
@ -189,7 +183,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \ c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \`last-item\` > '0001-01-01' and \
c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \ c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \
'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \ 'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \
c.\`id\` > ${currentid} limit 1000") c.\`id\` > ${currentid} limit 1000")
@ -209,27 +203,21 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
" "
"${dboptimizeengine}" "${db}" "${dboptimizeengine}" "${db}"
fi fi
"${dbengine}" "${db}" -v -e "\ "${dbengine}" "${db}" -v -e "alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`"
alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \ "${dbengine}" "${db}" -v -e "alter table \`post-thread\` drop index \`tmp_post_thread_id\`"
alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \ "${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`"
alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \ "${dbengine}" "${db}" -v -e "alter table \`post-user\` drop index \`tmp_post_user_id\`"
alter table \`post-user\` drop index \`tmp_post_user_id\`; \ "${dbengine}" "${db}" -v -e "alter table \`post\` drop index \`tmp_post_id\`"
alter table \`post\` drop index \`tmp_post_id\`; \ "${dbengine}" "${db}" -v -e "alter table \`photo\` drop index \`tmp_photo_id\`"
alter table \`photo\` drop index \`tmp_photo_id\`; \
"
"${dboptimizeengine}" "${db}" "${dboptimizeengine}" "${db}"
if [[ -n $(type flock) ]]; then if [[ -n $(type flock) ]]; then
flock -u 9 flock -u 9 2>/dev/null
flock -u 8
fi fi
if [[ -f "${tmpfile}" ]]; then if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}" rm -rf "${tmpfile}"
fi fi
if [[ -f "${tmpreadlock}" ]]; then if [[ -f "${tmplock}" ]]; then
rm -rf "${tmpreadlock}" rm -rf "${tmplock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi fi
date date
fi fi

16
youtube-download-channel.sh
View file

@ -60,15 +60,19 @@ if [[ -f "${subfolder}/${channel}.tar.zst" ]]; then
fi fi
fi fi
url="https://www.youtube.com/@${channel}" url="https://www.youtube.com/@${channel}"
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
full_url=$("${ytdl}" -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
if [[ "${channel}" = "subscriptions" ]]; then if [[ "${channel}" = "subscriptions" ]]; then
url="https://www.youtube.com/feed/subscriptions" url="https://www.youtube.com/feed/subscriptions"
full_url="${url}"
elif [[ "${channel}" = "WL" ]]; then
url="https://www.youtube.com/playlist?list=WL"
full_url="${url}"
fi fi
#for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do #for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
#full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl") #full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl")
echo "${url} = ${full_url}" echo "${url} = ${full_url}"
if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then if [[ -f "${cookies}" || "${channel}" = "subscriptions" || "${channel}" = "WL" ]]; then
#If available, you can use the cookies from your browser directly. Substitute #If available, you can use the cookies from your browser directly. Substitute
# --cookies "${cookies}" # --cookies "${cookies}"
#for the below, substituting for your browser of choice: #for the below, substituting for your browser of choice:
@ -130,7 +134,7 @@ total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l)
find "${temporary}" -type f -iname "*.info.json" | while read -r x; do find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
count=$((count + 1)) count=$((count + 1))
( (
if [[ -f "${x}" && "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then if [[ -f "${x}" && "${channel}" != "subscriptions" && "${channel}" != "WL" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then
echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}" echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}"
fi fi
if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then
@ -145,7 +149,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
fi fi
echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt" echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt"
if [[ ${enablecsv} = "1" ]]; then if [[ ${enablecsv} = "1" ]]; then
jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url, .was_live]' "${x}" | while read -r i; do
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${tmpcsv}" echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${tmpcsv}"
done done
fi fi
@ -188,7 +192,7 @@ if [[ ${enabledb} = "1" ]]; then
fi fi
if [[ ${enablecsv} = "1" ]]; then if [[ ${enablecsv} = "1" ]]; then
sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv" sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv"
echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL"' >"${temporary}/${channel}-tmp.csv" echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL", "Livestream"' >"${temporary}/${channel}-tmp.csv"
cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv" cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv"
mv "${temporary}/${channel}-tmp.csv" "${csv}" mv "${temporary}/${channel}-tmp.csv" "${csv}"
rm "${temporary}/${channel}-without-header.csv" rm "${temporary}/${channel}-without-header.csv"