diff --git a/friendica-clean-database.sh b/friendica-clean-database.sh new file mode 100755 index 0000000..c363f46 --- /dev/null +++ b/friendica-clean-database.sh @@ -0,0 +1,78 @@ +#!/bin/bash +interval=14 +sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ +"DELETE FROM \`post-origin\` WHERE (\`parent-uri-id\`, \`uid\`) IN (SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \ + \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL $interval DAY));\ +DELETE FROM \`post-user\` WHERE \`gravity\` = 0 AND \ + \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL $interval DAY);\ +DELETE FROM \`post\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`);\ +DELETE FROM \`post-content\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`);\ +DELETE FROM \`post-thread\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`);\ +DELETE FROM \`post-user\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`);\ +DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL $interval DAY) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` \ + WHERE (\`mention\` OR \`starred\` OR \`wall\`) AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-category\` \ + WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-collection\` \ + WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-media\` \ + WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` INNER JOIN \`contact\` ON \`contact\`.\`id\` = \`contact-id\` AND \`notify_new_posts\` \ + WHERE \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` \ + WHERE (\`origin\` OR \`event-id\` != 0 OR \`post-type\` = 128) AND \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` \ + WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`));\ +DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 AND \`received\` < (CURDATE() - INTERVAL $interval DAY) \ + AND NOT \`uri-id\` IN (\ + SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\`\ + ) AND NOT \`uri-id\` IN (\ + SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL $interval DAY)\ + )\ +);\ +DELETE FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`);\ +DELETE FROM \`item-uri\` WHERE (\ + \`id\` < (\ + SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`thr-parent-id\` FROM \`post-user\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`external-id\` FROM \`post-user\` WHERE \`external-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`replies-id\` FROM \`post-user\` WHERE \`replies-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`context-id\` FROM \`post-thread\` WHERE \`context-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`conversation-id\` FROM \`post-thread\` WHERE \`conversation-id\`= \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`mail\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`event\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`user-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`apcontact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`diaspora-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`inbox-status\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`inbox-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`parent-uri-id\` FROM \`mail\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \ + ) AND NOT EXISTS (\ + SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \ + )\ +); \ +DELETE t1 FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\`;\ +DELETE u1 FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\`;\ +DELETE v1 FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\`;" diff --git a/friendica-find-missing-servers.sh b/friendica-find-missing-servers.sh new file mode 100755 index 0000000..6bf3a64 --- /dev/null +++ b/friendica-find-missing-servers.sh @@ -0,0 +1,80 @@ +#!/bin/bash +db="friendica" +tmpfile="/tmp/sitesdown.txt" +idsdownfile="/tmp/idsdown.txt" +loop_1() { + sitereq=$(curl -s -L --head -m 10 --request GET "$a") + status=$(echo "$sitereq" | grep -e "200" -e "cloudflare") + if [[ -z $status ]] + then + echo "$a" >> "$tmpfile" + echo "Added $a" + fi +} +loop_2() { + echo "Finding users for $b" + mariadb "$db" -N -B -q -e "select \`id\` from contact c where \"$b\" not in (select \`contact-id\` from group_member) and (c.baseurl = \"$b\" or c.url = \"$b\")" | sudo tee -a "$idsdownfile" &> /dev/null +} + +loop_3() { + echo "Deleting user $lineb" + mariadb "$db" -N -B -q -e "delete from \`contact\` where \`id\` = $lineb" + mariadb "$db" -N -B -q -e "delete from \`post-thread\` where \`author-id\` = $lineb or \`causer-id\` = $lineb or \`owner-id\` = $lineb" + mariadb "$db" -N -B -q -e "delete from \`post-thread-user\` where \`author-id\` = $lineb or \`causer-id\` = $lineb or \`owner-id\` = $lineb" + mariadb "$db" -N -B -q -e "delete from \`post-user\` where \`author-id\` = $lineb or \`causer-id\` = $lineb or \`owner-id\` = $lineb" + mariadb "$db" -N -B -q -e "delete from \`post-tag\` where cid = $lineb" + mariadb "$db" -N -B -q -e "delete from \`post\` where \`owner-id\` = $lineb or \`author-id\` = $lineb or \`causer-id\` = $lineb" + mariadb "$db" -N -B -q -e "delete from \`photo\` where \`contact-id\` = $lineb" + mariadb "$db" -N -B -q -e "delete from \`contact\` where \`id\` = $lineb" +} + +#Check if our dependencies are installed +if [[ $(type curl) && $(type mariadb) && $(type date) ]] +then + date + if [[ ! -f "$tmpfile" ]] + then + echo "Listing sites" + sites=($(mariadb "$db" -N -B -q -e "select distinct baseurl from contact" | sort -n | uniq )) + echo "Amount of unique sites: ${#sites[@]}" + for a in "${sites[@]}" + do + loop_1 $a & + if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)*2) ]] + then + wait -n + fi + done + wait + fi + sitesdown=() + while read -r line; do + sitesdown+=($line) + done < "$tmpfile" + echo "Amount of sites down: ${#sitesdown[@]} / ${#sites[@]}" + if [[ ! -f "$idsdownfile" ]] + then + for b in "${sitesdown[@]}" + do + loop_2 $b & + if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)/2) ]] + then + wait -n + fi + done + wait + #cat "$idsdownfile" | sort -n | uniq > "$idsdownfile" + fi + while read -r lineb; do + #The community no longer exists, delete + loop_3 $lineb & + if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)/2) ]] + then + wait -n + fi + wait + done < "$idsdownfile" + rm "$tmpfile" 2> /dev/null + rm "$idsdownfile" 2> /dev/null + date +fi diff --git a/friendica-fix-avatar-permissions.sh b/friendica-fix-avatar-permissions.sh new file mode 100755 index 0000000..d4d3d8e --- /dev/null +++ b/friendica-fix-avatar-permissions.sh @@ -0,0 +1,84 @@ +#!/bin/bash +IFS=" +" +#Set your parameters here +site=hub.example.net +user=friendica +group=friendica +fileperm=644 +folderperm=755 +folder=/var/www/friendica +folderescaped=${folder////\\/} +tmpfile=/tmp/friendica-fix-avatar-permissions.txt +avatarfolder=avatar +cd "$folder" || exit +if [ ! -f "$tmpfile" ] +then + sudo -u "$user" bin/console movetoavatarcache | sudo tee "$tmpfile" #&> /dev/null +fi +grep -e "https://$site/$avatarfolder/" "$tmpfile" | sed -e "s/.*$site/$folderescaped/g" -e "s/-.*/\*/g" | ( + while read n + do + find "$folder/$avatarfolder" -path "$n" -type f | ( + while read p + do + if [[ "${p}" =~ ".jpeg" || "${p}" =~ ".jpg" ]] + then + jpegoptim -m 76 "${p}" & #&> /dev/null + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + fi + if [[ "${p}" =~ ".gif" ]] + then + gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" & #&> /dev/null + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + #Specific compression for large GIF files + while [[ $(stat -c%s "${p}") -ge 512000 ]] + do + gifsicle "${p}" `seq -f "#%g" 0 2 99` -O3 --lossy=80 --colors=255 -o "${p}" & #&> /dev/null + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + done + fi + if [[ "${p}" =~ ".png" ]] + then + oxipng -o max "${p}" & #&> /dev/null + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + fi + if [[ "${p}" =~ ".webp" ]] + then + cwebp -mt -af -quiet "${p}" -o /tmp/temp.webp #&> /dev/null + if [ -f /tmp/temp.webp ] + then + size_new=$(stat -c%s "/tmp/temp.webp") + size_original=$(stat -c%s "${p}") + if [ "$size_original" -gt "$size_new" ] + then + mv /tmp/temp.webp "${p}" + else + rm /tmp/temp.webp + fi + fi + fi + done + ) + done +) +wait +rm "$tmpfile" +/usr/bin/find $folder/avatar -type d -empty -delete +/usr/bin/chmod $folderperm $folder/avatar +/usr/bin/chown -R $user:$group $folder/avatar +/usr/bin/find $folder/avatar -depth -not -user $user -or -not -group $group -print0 | xargs -0 -r sudo chown -v $user:$group #&> /dev/null +/usr/bin/find $folder/avatar -depth -type d -and -not -type f -and -not -perm $folderperm -print0 | xargs -0 -r sudo chmod -v $folderperm #&> /dev/null +/usr/bin/find $folder/avatar -depth -type f -and -not -type d -and -not -perm $fileperm -print0 | xargs -0 -r sudo chmod -v $fileperm #&> /dev/null diff --git a/friendica-non-follower-featured-posts.sh b/friendica-non-follower-featured-posts.sh new file mode 100755 index 0000000..ff7e94a --- /dev/null +++ b/friendica-non-follower-featured-posts.sh @@ -0,0 +1,57 @@ +#!/bin/bash +ca=100 +camax=0 +while [[ $ca -gt 0 ]] +do + ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateContact\" limit $ca; select row_count();") + camax=$(( camax + ca )) + printf "\rUpdateContact\t\t%s\r" "$camax" +done +printf "\rUpdateContact\t\t%s\n\r" "$camax" +#echo "UpdateContact $camax" + +cb=100 +cbmax=0 +while [[ $cb -gt 0 ]] +do + cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"ContactDiscovery\" limit $cb; select row_count();") + cbmax=$(( cbmax + cb )) + printf "\rContactDiscovery\t%s\r" "$cbmax" +done +printf "\rContactDiscovery\t%s\n\r" "$cbmax" +#echo "ContactDiscovery $cbmax" + +cc=100 +ccmax=0 +while [[ $cc -gt 0 ]] +do + cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit $cc; select row_count();") + ccmax=$(( ccmax + cc )) + printf "\rFetchFeaturedPosts\t%s\r" "$ccmax" + +done +printf "\rFetchFeaturedPosts\t%s\n\r" "$ccmax" +#echo "FetchFeaturedPosts $ccmax" + +cd=100 +cdmax=0 +while [[ $cd -gt 0 ]] +do + cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit $cd; select row_count();") + cdmax=$(( cdmax + cd )) + printf "\rUpdateGServer\t\t%s\r" "$cdmax" +done +printf "\rUpdateGServer\t\t%s\n\r" "$cdmax" +#echo "UpdateGServer $cdmax" + +ce=100 +cemax=0 +while [[ $ce -gt 0 ]] +do + ce=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where command=\"ProcessQueue\" and pid=0 and done=0 limit $ce; select row_count();") + cemax=$(( cemax + ce )) + printf "\rProcessQueue\t\t%s\r" "$cemax" +done +printf "\rProcessQueue\t\t%s\n\r" "$cemax" +#echo "ProcessQueue $cemax" + diff --git a/friendica-remove-old-photos-parallel.sh b/friendica-remove-old-photos-parallel.sh new file mode 100755 index 0000000..7d68406 --- /dev/null +++ b/friendica-remove-old-photos-parallel.sh @@ -0,0 +1,56 @@ +#!/bin/bash +IFS=" +" +#Set your parameters here +url=hub.example.net +db=friendica +folder=/var/www/friendica +folderavatar=/var/www/friendica/avatar +loop() { + #Parse each file in folder + ky=$(echo "$y" | sed -e "s/$folderescaped/https:\/\/$url/g" -e "s/-[0-9]*\..*\$//g") + f=$(sudo mariadb $db -N -B -q -e "select photo from contact where photo like '$ky%' limit 1") + if [[ $? -eq 0 && -z $f && -f $y ]] + then + ls -lh "$y" + sudo rm -rf "$y" + d=$(( d + 1 )) + fi + #printf "\rPhotos: %s\tFolder %s\tEntry %s " "$d" "$n" "$m" + printf "\rFolder %s\tEntry %s " "$n" "$m" + return $d +} + +date +#Go to the Friendica installation +cd "$folderavatar" || exit +let "indexlength=37+${#url}" +(( indexlength=49+${#url} )) +sudo mariadb $db -e "alter table contact add index if not exists photo_index (photo($indexlength))" +n=0 +d=0 +sudo find "$folderavatar" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x +do + n=$(( n + 1 )) + #If the directory still exists + if [[ -d "$x" ]] + then + folderescaped=${folder////\\/} + kx=$(echo "$x" | sed -e "s/$folderescaped/https:\/\/$url/g" -e "s/-[0-9]*\..*\$//g") + if [[ -d $x ]] + then + m=0 + while read -r y + do + m=$(( m + 1 )) + loop "$x" "$m" "$n" "$d" & + if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 1 )) ]] + then + wait -n + fi + done < <(sudo find "$x" -type f -mtime -8) + fi + fi +done +sudo mariadb $db -e "alter table contact drop index photo_index" +date diff --git a/friendica-workerqueue.sh b/friendica-workerqueue.sh new file mode 100755 index 0000000..0df20aa --- /dev/null +++ b/friendica-workerqueue.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sudo mariadb friendica --execute="select distinct command, count(*) from workerqueue where done = 0 group by command; select count(*) from workerqueue where done = 0; select count(*) from workerqueue where command = \"ProcessQueue\" and pid = 0 and done = 0" diff --git a/media-optimize-local.sh b/media-optimize-local.sh new file mode 100755 index 0000000..c1cf8c8 --- /dev/null +++ b/media-optimize-local.sh @@ -0,0 +1,56 @@ +#!/bin/bash +IFS=" +" + #media-optimize + find . -type f -iname "*.jp*" -size +50k | ( + while read p + do + nice -n 15 jpegoptim -m 76 "${p}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + done; + wait + ) + find . -type f -iname "*.gif" -size +500k | ( + while read q + do + nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + done; + wait + ) + find . -type f -iname "*.png" -size +300k | ( + while read r + do + nice -n 15 oxipng -o max "${r}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + done; + wait + ) + #compress-webp + find . -type f -iname "*.webp" -size +200k | ( + while read s + do + nice -n 15 cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp + if [ -f /tmp/"${s##.*\/}"_temp.webp ] + then + size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp) + size_original=$(stat -c%s "${s}") + if [ "$size_original" -gt "$size_new" ] + then + mv /tmp/"${s##.*\/}"_temp.webp $s + else + rm /tmp/"${s##.*\/}"_temp.webp + fi + fi + done; + wait + )