diff --git a/friendica-compress-storage.sh b/friendica-compress-storage.sh index 6018254..0d63d3d 100755 --- a/friendica-compress-storage.sh +++ b/friendica-compress-storage.sh @@ -55,7 +55,7 @@ loop_1() { echo "Generating photo index..." #&> /dev/null sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null echo "Generating list of files..." #&> /dev/null -find "${storagefolder}" -depth -mindepth 2 -type f -size +100k -mtime -8 -not -iname "index.html" | ( +find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | ( while read -r p; do loop_1 "${p}" & until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do diff --git a/friendica-delete-old-users.sh b/friendica-delete-old-users.sh index 4f29a26..e5b810d 100755 --- a/friendica-delete-old-users.sh +++ b/friendica-delete-old-users.sh @@ -17,7 +17,8 @@ starterid=${3:-"0"} db="friendica" period="${period_amount} MONTH" tmpfile=/tmp/friendica-delete-old-users.csv -tmplock=/tmp/friendica-delete-old-users.tmp +tmpreadlock=/tmp/friendica-delete-old-users.read.tmp +tmpwritelock=/tmp/friendica-delete-old-users.write.tmp loop() { baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") @@ -32,7 +33,7 @@ loop() { if [[ -n $(type flock) ]]; then isreadlocked=0 while [[ "${isreadlocked}" -eq 0 ]]; do - exec 9>"${tmplock}" + exec 9>"${tmpreadlock}" if flock -n -e 9; then isreadlocked=1 if [[ -f "${tmpfile}" ]]; then @@ -65,11 +66,11 @@ loop() { flock -u 9 iswritelocked=0 while [[ "${iswritelocked}" -eq 0 ]]; do - exec 9>"${tmplock}" - if flock -n -e 9; then + exec 8>"${tmpwritelock}" + if flock -n -e 8; then iswritelocked=1 echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}" - flock -u 9 + flock -u 8 fi done fi @@ -148,18 +149,23 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty if [[ -f "${tmpfile}" ]]; then rm -rf "${tmpfile}" fi - if [[ -f "${tmplock}" ]]; then - rm -rf "${tmplock}" + if [[ -f "${tmpreadlock}" ]]; then + rm -rf "${tmpreadlock}" + fi + if [[ -f "${tmpwritelock}" ]]; then + rm -rf "${tmpwritelock}" fi touch "${tmpfile}" echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}" if [[ "${intense_optimizations}" -gt 0 ]]; then - "${dbengine}" "${db}" -v -e "alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr)" - "${dbengine}" "${db}" -v -e "alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" - "${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" - "${dbengine}" "${db}" -v -e "alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" - "${dbengine}" "${db}" -v -e "alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" - "${dbengine}" "${db}" -v -e "alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`)" + "${dbengine}" "${db}" -v -e "\ + alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \ + alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ + alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ + alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ + alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ + alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \ + " fi counter=0 was_empty=0 @@ -183,7 +189,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \ - c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \`last-item\` > '0001-01-01' and \ + c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \ c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \ 'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \ c.\`id\` > ${currentid} limit 1000") @@ -203,21 +209,27 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty " "${dboptimizeengine}" "${db}" fi - "${dbengine}" "${db}" -v -e "alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`" - "${dbengine}" "${db}" -v -e "alter table \`post-thread\` drop index \`tmp_post_thread_id\`" - "${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`" - "${dbengine}" "${db}" -v -e "alter table \`post-user\` drop index \`tmp_post_user_id\`" - "${dbengine}" "${db}" -v -e "alter table \`post\` drop index \`tmp_post_id\`" - "${dbengine}" "${db}" -v -e "alter table \`photo\` drop index \`tmp_photo_id\`" + "${dbengine}" "${db}" -v -e "\ + alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \ + alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \ + alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \ + alter table \`post-user\` drop index \`tmp_post_user_id\`; \ + alter table \`post\` drop index \`tmp_post_id\`; \ + alter table \`photo\` drop index \`tmp_photo_id\`; \ + " "${dboptimizeengine}" "${db}" if [[ -n $(type flock) ]]; then - flock -u 9 2>/dev/null + flock -u 9 + flock -u 8 fi if [[ -f "${tmpfile}" ]]; then rm -rf "${tmpfile}" fi - if [[ -f "${tmplock}" ]]; then - rm -rf "${tmplock}" + if [[ -f "${tmpreadlock}" ]]; then + rm -rf "${tmpreadlock}" + fi + if [[ -f "${tmpwritelock}" ]]; then + rm -rf "${tmpwritelock}" fi date fi diff --git a/youtube-download-channel.sh b/youtube-download-channel.sh index ead40dc..bf32606 100755 --- a/youtube-download-channel.sh +++ b/youtube-download-channel.sh @@ -60,19 +60,15 @@ if [[ -f "${subfolder}/${channel}.tar.zst" ]]; then fi fi url="https://www.youtube.com/@${channel}" -#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141 -full_url=$("${ytdl}" -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}") if [[ "${channel}" = "subscriptions" ]]; then url="https://www.youtube.com/feed/subscriptions" - full_url="${url}" -elif [[ "${channel}" = "WL" ]]; then - url="https://www.youtube.com/playlist?list=WL" - full_url="${url}" fi #for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do +#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141 +full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}") #full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl") echo "${url} = ${full_url}" -if [[ -f "${cookies}" || "${channel}" = "subscriptions" || "${channel}" = "WL" ]]; then +if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then #If available, you can use the cookies from your browser directly. Substitute # --cookies "${cookies}" #for the below, substituting for your browser of choice: @@ -134,7 +130,7 @@ total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l) find "${temporary}" -type f -iname "*.info.json" | while read -r x; do count=$((count + 1)) ( - if [[ -f "${x}" && "${channel}" != "subscriptions" && "${channel}" != "WL" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then + if [[ -f "${x}" && "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}" fi if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then @@ -149,7 +145,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do fi echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt" if [[ ${enablecsv} = "1" ]]; then - jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url, .was_live]' "${x}" | while read -r i; do + jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/"/g" >>"${tmpcsv}" done fi @@ -192,7 +188,7 @@ if [[ ${enabledb} = "1" ]]; then fi if [[ ${enablecsv} = "1" ]]; then sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv" - echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL", "Livestream"' >"${temporary}/${channel}-tmp.csv" + echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL"' >"${temporary}/${channel}-tmp.csv" cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv" mv "${temporary}/${channel}-tmp.csv" "${csv}" rm "${temporary}/${channel}-without-header.csv"