diff --git a/friendica-compress-storage.sh b/friendica-compress-storage.sh index 0d63d3d..6018254 100755 --- a/friendica-compress-storage.sh +++ b/friendica-compress-storage.sh @@ -55,7 +55,7 @@ loop_1() { echo "Generating photo index..." #&> /dev/null sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null echo "Generating list of files..." #&> /dev/null -find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | ( +find "${storagefolder}" -depth -mindepth 2 -type f -size +100k -mtime -8 -not -iname "index.html" | ( while read -r p; do loop_1 "${p}" & until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do diff --git a/friendica-delete-old-users.sh b/friendica-delete-old-users.sh index e5b810d..4f29a26 100755 --- a/friendica-delete-old-users.sh +++ b/friendica-delete-old-users.sh @@ -17,8 +17,7 @@ starterid=${3:-"0"} db="friendica" period="${period_amount} MONTH" tmpfile=/tmp/friendica-delete-old-users.csv -tmpreadlock=/tmp/friendica-delete-old-users.read.tmp -tmpwritelock=/tmp/friendica-delete-old-users.write.tmp +tmplock=/tmp/friendica-delete-old-users.tmp loop() { baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") @@ -33,7 +32,7 @@ loop() { if [[ -n $(type flock) ]]; then isreadlocked=0 while [[ "${isreadlocked}" -eq 0 ]]; do - exec 9>"${tmpreadlock}" + exec 9>"${tmplock}" if flock -n -e 9; then isreadlocked=1 if [[ -f "${tmpfile}" ]]; then @@ -66,11 +65,11 @@ loop() { flock -u 9 iswritelocked=0 while [[ "${iswritelocked}" -eq 0 ]]; do - exec 8>"${tmpwritelock}" - if flock -n -e 8; then + exec 9>"${tmplock}" + if flock -n -e 9; then iswritelocked=1 echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}" - flock -u 8 + flock -u 9 fi done fi @@ -149,23 +148,18 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty if [[ -f "${tmpfile}" ]]; then rm -rf "${tmpfile}" fi - if [[ -f "${tmpreadlock}" ]]; then - rm -rf "${tmpreadlock}" - fi - if [[ -f "${tmpwritelock}" ]]; then - rm -rf "${tmpwritelock}" + if [[ -f "${tmplock}" ]]; then + rm -rf "${tmplock}" fi touch "${tmpfile}" echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}" if [[ "${intense_optimizations}" -gt 0 ]]; then - "${dbengine}" "${db}" -v -e "\ - alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \ - alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ - alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ - alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ - alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ - alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \ - " + "${dbengine}" "${db}" -v -e "alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr)" + "${dbengine}" "${db}" -v -e "alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" + "${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" + "${dbengine}" "${db}" -v -e "alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" + "${dbengine}" "${db}" -v -e "alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" + "${dbengine}" "${db}" -v -e "alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`)" fi counter=0 was_empty=0 @@ -189,7 +183,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \ - c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \ + c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \`last-item\` > '0001-01-01' and \ c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \ 'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \ c.\`id\` > ${currentid} limit 1000") @@ -209,27 +203,21 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty " "${dboptimizeengine}" "${db}" fi - "${dbengine}" "${db}" -v -e "\ - alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \ - alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \ - alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \ - alter table \`post-user\` drop index \`tmp_post_user_id\`; \ - alter table \`post\` drop index \`tmp_post_id\`; \ - alter table \`photo\` drop index \`tmp_photo_id\`; \ - " + "${dbengine}" "${db}" -v -e "alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`" + "${dbengine}" "${db}" -v -e "alter table \`post-thread\` drop index \`tmp_post_thread_id\`" + "${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`" + "${dbengine}" "${db}" -v -e "alter table \`post-user\` drop index \`tmp_post_user_id\`" + "${dbengine}" "${db}" -v -e "alter table \`post\` drop index \`tmp_post_id\`" + "${dbengine}" "${db}" -v -e "alter table \`photo\` drop index \`tmp_photo_id\`" "${dboptimizeengine}" "${db}" if [[ -n $(type flock) ]]; then - flock -u 9 - flock -u 8 + flock -u 9 2>/dev/null fi if [[ -f "${tmpfile}" ]]; then rm -rf "${tmpfile}" fi - if [[ -f "${tmpreadlock}" ]]; then - rm -rf "${tmpreadlock}" - fi - if [[ -f "${tmpwritelock}" ]]; then - rm -rf "${tmpwritelock}" + if [[ -f "${tmplock}" ]]; then + rm -rf "${tmplock}" fi date fi diff --git a/youtube-download-channel.sh b/youtube-download-channel.sh index bf32606..ead40dc 100755 --- a/youtube-download-channel.sh +++ b/youtube-download-channel.sh @@ -60,15 +60,19 @@ if [[ -f "${subfolder}/${channel}.tar.zst" ]]; then fi fi url="https://www.youtube.com/@${channel}" +#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141 +full_url=$("${ytdl}" -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}") if [[ "${channel}" = "subscriptions" ]]; then url="https://www.youtube.com/feed/subscriptions" + full_url="${url}" +elif [[ "${channel}" = "WL" ]]; then + url="https://www.youtube.com/playlist?list=WL" + full_url="${url}" fi #for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do -#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141 -full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}") #full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl") echo "${url} = ${full_url}" -if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then +if [[ -f "${cookies}" || "${channel}" = "subscriptions" || "${channel}" = "WL" ]]; then #If available, you can use the cookies from your browser directly. Substitute # --cookies "${cookies}" #for the below, substituting for your browser of choice: @@ -130,7 +134,7 @@ total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l) find "${temporary}" -type f -iname "*.info.json" | while read -r x; do count=$((count + 1)) ( - if [[ -f "${x}" && "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then + if [[ -f "${x}" && "${channel}" != "subscriptions" && "${channel}" != "WL" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}" fi if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then @@ -145,7 +149,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do fi echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt" if [[ ${enablecsv} = "1" ]]; then - jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do + jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url, .was_live]' "${x}" | while read -r i; do echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/"/g" >>"${tmpcsv}" done fi @@ -188,7 +192,7 @@ if [[ ${enabledb} = "1" ]]; then fi if [[ ${enablecsv} = "1" ]]; then sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv" - echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL"' >"${temporary}/${channel}-tmp.csv" + echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL", "Livestream"' >"${temporary}/${channel}-tmp.csv" cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv" mv "${temporary}/${channel}-tmp.csv" "${csv}" rm "${temporary}/${channel}-without-header.csv"