fix: Implement more Shellcheck suggestions, make display more clear, timeout more lenient

fix: Correct default file and folder permissions
fix: Separate each part of the database cleaning to prevent one failure from stopping the rest of the script
2025-02-02 21:15:23 +00:00 · 2025-02-02 21:14:32 +00:00 · 2025-02-02 21:13:58 +00:00
3 changed files with 54 additions and 51 deletions
--- a/friendica-clean-database.sh
+++ b/friendica-clean-database.sh
@ -1,32 +1,31 @@
 #!/bin/bash
 interval=14
 sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"\
-CREATE TEMPORARY TABLE \`tmp-post-origin-deleted\` (SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \
+"CREATE TEMPORARY TABLE \`tmp-post-origin-deleted\` (SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \
  \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL "$interval" DAY)); \
-DELETE FROM \`post-origin\` WHERE (\`parent-uri-id\`, \`uid\`) IN (SELECT * FROM \`tmp-post-origin-deleted\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-post-user-deleted\` (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \
+DELETE FROM \`post-origin\` WHERE (\`parent-uri-id\`, \`uid\`) IN (SELECT * FROM \`tmp-post-origin-deleted\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-post-user-deleted\` (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \
  \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL "$interval" DAY)); \
-DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-user-deleted\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-post-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post\` \
+DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-user-deleted\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-post-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post\` \
  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \
-DELETE FROM \`post\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-uri-id-not-in-post-user\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-post-content-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-content\` \
-  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`));\
-DELETE FROM \`post-content\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-content-uri-id-not-in-post-user\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-post-thread-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-thread\` \
+DELETE FROM \`post\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-uri-id-not-in-post-user\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-post-content-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-content\` \
  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \
-DELETE FROM \`post-thread\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-thread-uri-id-not-in-post-user\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-post-user-uri-id-not-in-post\` (SELECT \`uri-id\` FROM \`post-user\` \
+DELETE FROM \`post-content\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-content-uri-id-not-in-post-user\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-post-thread-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-thread\` \
+  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \
+DELETE FROM \`post-thread\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-thread-uri-id-not-in-post-user\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-post-user-uri-id-not-in-post\` (SELECT \`uri-id\` FROM \`post-user\` \
  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`)); \
-DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT \`uri-id\` FROM \`tmp-post-user-uri-id-not-in-post\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-thread\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN \
+DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT \`uri-id\` FROM \`tmp-post-user-uri-id-not-in-post\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-thread\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN \
  (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL $interval DAY) \
    AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` \
      WHERE (\`mention\` OR \`starred\` OR \`wall\`) AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \
@ -43,9 +42,9 @@ CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-thread\` (SELECT \`id\`
    AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` \
      WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`)) \
 ); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-thread\`) ;
-\
-CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-user\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \
+DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-thread\`) ;"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-user\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \
  AND \`received\` < (CURDATE() - INTERVAL $interval DAY) \
  AND NOT \`uri-id\` IN ( \
    SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` \
@ -53,12 +52,12 @@ CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-user\` (SELECT \`id\` FR
    SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL $interval DAY) \
  ) \
 )); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-user\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-attach-not-in-post-media\` (SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`)); \
-DELETE FROM \`attach\` WHERE \`id\` IN (SELECT * FROM \`tmp-attach-not-in-post-media\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-item-uri-not-valid\` (SELECT \`id\` FROM \`item-uri\` WHERE (\
+DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-user\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-attach-not-in-post-media\` (SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`)); \
+DELETE FROM \`attach\` WHERE \`id\` IN (SELECT * FROM \`tmp-attach-not-in-post-media\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-item-uri-not-valid\` (SELECT \`id\` FROM \`item-uri\` WHERE (\
  \`id\` < (\
    SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 \
  ) AND NOT EXISTS (\
@ -99,13 +98,13 @@ CREATE TEMPORARY TABLE \`tmp-item-uri-not-valid\` (SELECT \`id\` FROM \`item-uri
    SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \
  )\
 )); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-valid\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-item-uri-duplicate\` (SELECT \`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\`); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-duplicate\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-post-media-duplicate\` (SELECT \`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\`); \
-DELETE FROM \`post-media\` WHERE \`id\` IN (SELECT * FROM \`tmp-post-media-duplicate\`); \
-\
-CREATE TEMPORARY TABLE \`tmp-post-user-duplicate\` (SELECT \`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\`); \
+DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-valid\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-item-uri-duplicate\` (SELECT \`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\`); \
+DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-duplicate\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-post-media-duplicate\` (SELECT \`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\`); \
+DELETE FROM \`post-media\` WHERE \`id\` IN (SELECT * FROM \`tmp-post-media-duplicate\`);"
+sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
+"CREATE TEMPORARY TABLE \`tmp-post-user-duplicate\` (SELECT \`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\`); \
 DELETE FROM \`post-user\` WHERE \`id\` IN (SELECT * FROM \`tmp-post-user-duplicate\`;"
--- a/friendica-find-missing-servers.sh
+++ b/friendica-find-missing-servers.sh
@ -3,12 +3,11 @@ db="friendica"
 tmpfile="/tmp/sitesdown.txt"
 idsdownfile="/tmp/idsdown.txt"
 url=friendica.example.net
-folder=/var/www/friendica
-folderescaped=${folder////\\/}
 avatarfolder=/var/www/friendica/avatar
 avatarfolderescaped=${avatarfolder////\\/}
 loop_1() {
-	sitereq=$(curl -s -L --head -m 20 --request GET "${a}")
+	sitereq=$(curl -s -L --head -m 30 --request GET "${a}")
+	#Skip check if the message contains a reference to Cloudflare
 	status=$(echo "${sitereq}" | grep -e "200" -e "cloudflare")
 	if [[ -z ${status} ]]
 	then
@ -18,7 +17,7 @@ loop_1() {
 }
 loop_2() {
 	echo "Finding users for ${b}"
-	"${dbengine}" "${db}" -N -B -q -e "select \`id\`, \`name\` from contact c where c.\`id\` not in (select \`contact-id\` from group_member) and (c.baseurl = \"${b}\" or c.url = \"${b}\")" | sudo tee -a "${idsdownfile}" #&> /dev/null
+	"${dbengine}" "${db}" -N -B -q -e "select \`id\`, \`nick\`, \`baseurl\` from contact c where c.\`id\` not in (select \`contact-id\` from group_member) and (c.baseurl = \"${b}\" or c.url = \"${b}\")" | sudo tee -a "${idsdownfile}" #&> /dev/null
 }

 loop_3() {
@ -28,9 +27,12 @@ loop_3() {
 	"${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro
 	do
 		#If stored in avatar folder
-		if [[ -z $(echo "${photo}" | grep "${url}/avatar") ]]
+		if $(echo "${photo}" | grep -q "${url}/avatar")
+		#isavatar=$(grep -q "${url}/avatar" <<< "${photo}")
+		#if [[ -z "${isavatar}" ]]
 		then
 			phototrimmed=$(echo "${photo}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
+			echo "${phototrimmed}"
 			rm -rfv "${phototrimmed}"
 			thumbtrimmed=$(echo "${thumb}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
 			rm -rfv "${thumbtrimmed}"
@ -63,12 +65,14 @@ then
 	if [[ ! -f "${tmpfile}" ]]
 	then
 		echo "Listing sites"
-		sites=($("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl from contact" | sort -n | uniq ))
+		#sites=($("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl from contact where baseurl != \"\"" | sort -n | uniq ))
+		sites=()
+		mapfile -t sites < <("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl from contact where baseurl != \"\"" | sort -b -f -n | uniq -i )
 		echo "Amount of unique sites: ${#sites[@]}"
 		for a in "${sites[@]}"
 		do
 			loop_1 "${a}" &
-			if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)*2) ]]
+			if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) * 2 )) ]]
 			then
 				wait -n
 			fi
@ -77,7 +81,7 @@ then
 	fi
 	sitesdown=()
 	while read -r line; do
-		sitesdown+=(${line})
+		sitesdown+=("${line}")
 	done < "${tmpfile}"
 	echo "Amount of sites down: ${#sitesdown[@]} / ${#sites[@]}"
 	if [[ ! -f "${idsdownfile}" ]]
@ -85,7 +89,7 @@ then
 		for b in "${sitesdown[@]}"
 		do
 			loop_2 "${b}" &
-			if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)/2) ]]
+			if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 2 )) ]]
 			then
 				wait -n
 			fi
@ -99,7 +103,7 @@ then
 		#idsdown+=($lineb)
 		#The community no longer exists, delete
 		loop_3 "${lineb}" "${nick}" "${baseurl}" &
-		if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)/2) ]]
+		if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 2 )) ]]
 		then
 			wait -n
 		fi
--- a/friendica-fix-avatar-permissions.sh
+++ b/friendica-fix-avatar-permissions.sh
@ -5,8 +5,8 @@ IFS="
 site=friendica.example.net
 user=friendica
 group=friendica
-fileperm=644
-folderperm=755
+fileperm=640
+folderperm=750
 folder=/var/www/friendica
 folderescaped=${folder////\\/}
 tmpfile=/tmp/friendica-fix-avatar-permissions.txt
Author	SHA1	Message	Date
Carlos Solís	b1fe5f6698	fix: Implement more Shellcheck suggestions, make display more clear, timeout more lenient	2025-02-02 21:15:23 +00:00
Carlos Solís	90cdec87b4	fix: Correct default file and folder permissions	2025-02-02 21:14:32 +00:00
Carlos Solís	4c8fd869f7	fix: Separate each part of the database cleaning to prevent one failure from stopping the rest of the script	2025-02-02 21:13:58 +00:00