feat: Add new script to fix invalid avatars

friendica-clean-database.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
-interval=14
-limit=100
+interval=7
+limit=100000
 
+echo "tmp_post_origin_deleted"
 tmp_post_origin_deleted_q="${limit}"
 until [[ "${tmp_post_origin_deleted_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_origin_deleted"
 	tmp_post_origin_deleted=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) LIMIT ${limit}");
 	#tmp_post_origin_deleted_q="${#tmp_post_origin_deleted[@]}"
@@ -25,10 +25,10 @@ do
 	fi
 done
 
+echo "tmp_post_user_deleted"
 tmp_post_user_deleted_q="${limit}"
 until [[ "${tmp_post_user_deleted_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_user_deleted"
 	tmp_post_user_deleted=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) LIMIT ${limit}");
 	tmp_post_user_deleted_q=$(echo "${tmp_post_user_deleted}" | wc -l)
@@ -47,10 +47,10 @@ do
 	fi
 done
 
+echo "tmp_post_uri_id_not_in_post_user"
 tmp_post_uri_id_not_in_post_user_q="${limit}"
 until [[ "${tmp_post_uri_id_not_in_post_user_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_uri_id_not_in_post_user"
 	tmp_post_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`uri-id\` FROM \`post\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}");
 	tmp_post_uri_id_not_in_post_user_q=$(echo "${tmp_post_uri_id_not_in_post_user}" | wc -l)
@@ -69,10 +69,10 @@ do
 	fi
 done
 
+echo "tmp_post_content_uri_id_not_in_post_user"
 tmp_post_content_uri_id_not_in_post_user_q="${limit}"
 until [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_content_uri_id_not_in_post_user"
 	tmp_post_content_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`uri-id\` FROM \`post-content\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}");
 	tmp_post_content_uri_id_not_in_post_user_q=$(echo "${tmp_post_content_uri_id_not_in_post_user}" | wc -l)
@@ -91,10 +91,10 @@ do
 	fi
 done
 
+echo "tmp_post_thread_uri_id_not_in_post_user"
 tmp_post_thread_uri_id_not_in_post_user_q="${limit}"
 until [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_thread_uri_id_not_in_post_user"
 	tmp_post_thread_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`uri-id\` FROM \`post-thread\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}");
 	tmp_post_thread_uri_id_not_in_post_user_q=$(echo "${tmp_post_thread_uri_id_not_in_post_user}" | wc -l)
@@ -113,10 +113,10 @@ do
 	fi
 done
 
+echo "tmp_post_user_uri_id_not_in_post"
 tmp_post_user_uri_id_not_in_post_q="${limit}"
 until [[ "${tmp_post_user_uri_id_not_in_post_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_user_uri_id_not_in_post"
 	tmp_post_user_uri_id_not_in_post=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`) LIMIT ${limit}");
 	tmp_post_user_uri_id_not_in_post_q=$(echo "${tmp_post_user_uri_id_not_in_post}" | wc -l)
@@ -135,10 +135,10 @@ do
 	fi
 done
 
+echo "tmp_item_uri_not_in_valid_post_thread"
 tmp_item_uri_not_in_valid_post_thread_q="${limit}"
 until [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -lt "${limit}" ]]
 do
-	echo "tmp_item_uri_not_in_valid_post_thread"
 	tmp_item_uri_not_in_valid_post_thread=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL ${interval} DAY) \
 			AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` WHERE (\`mention\` OR \`starred\` OR \`wall\`) \
@@ -168,10 +168,10 @@ do
 	fi
 done
 
+echo "tmp_item_uri_not_in_valid_post_user"
 tmp_item_uri_not_in_valid_post_user_q="${limit}"
 until [[ "${tmp_item_uri_not_in_valid_post_user_q}" -lt "${limit}" ]]
 do
-	echo "tmp_item_uri_not_in_valid_post_user"
 	tmp_item_uri_not_in_valid_post_user=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \
 		AND \`received\` < (CURDATE() - INTERVAL ${interval} DAY) AND NOT \`uri-id\` IN ( SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 \
@@ -193,10 +193,10 @@ do
 	fi
 done
 
+echo "tmp_attach_not_in_post_media"
 tmp_attach_not_in_post_media_q="${limit}"
 until [[ "${tmp_attach_not_in_post_media_q}" -lt "${limit}" ]]
 do
-	echo "tmp_attach_not_in_post_media"
 	tmp_attach_not_in_post_media=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`) LIMIT ${limit}");
 	tmp_attach_not_in_post_media_q=$(echo "${tmp_attach_not_in_post_media}" | wc -l)
@@ -215,12 +215,14 @@ do
 	fi
 done
 
+echo "tmp_item_uri_not_valid"
 tmp_item_uri_not_valid_q="${limit}"
 until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]
 do
-	echo "tmp_item_uri_not_valid"
+	tmp_item_uri_not_valid_last_id=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1")
 	tmp_item_uri_not_valid=$(sudo mariadb friendica -N -B -q -e \
-		"SELECT \`id\` FROM \`item-uri\` WHERE ( \`id\` < ( SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 ) \
+		"SELECT \`id\` FROM \`item-uri\` WHERE ( \`id\` < ${tmp_item_uri_not_valid_last_id} ) \
 		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
 		AND NOT EXISTS ( SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` ) \
 		AND NOT EXISTS ( SELECT \`thr-parent-id\` FROM \`post-user\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` ) \
@@ -249,17 +251,17 @@ do
 			if [[ -n "${id}" ]]
 			then
 				sudo mariadb friendica -N -B -q -e \
-				"DELETE FROM \`item_uri\` WHERE \`id\` = ${id}"
+				"DELETE FROM \`item-uri\` WHERE \`id\` = ${id}"
 				echo "${id}"
 			fi
 		done
 	fi
 done
 
+echo "tmp_item_uri_duplicate"
 tmp_item_uri_duplicate_q="${limit}"
 until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]
 do
-	echo "tmp_item_uri_duplicate"
 	tmp_item_uri_duplicate=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}")
 	tmp_item_uri_duplicate_q=$(echo "${tmp_item_uri_duplicate}" | wc -l)
@@ -278,10 +280,10 @@ do
 	fi
 done
 
+echo "tmp_post_media_duplicate"
 tmp_post_media_duplicate_q="${limit}"
 until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_media_duplicate"
 	tmp_post_media_duplicate=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}")
 	tmp_post_media_duplicate_q=$(echo "${tmp_post_media_duplicate}" | wc -l)
@@ -300,10 +302,10 @@ do
 	fi
 done
 
+echo "tmp_post_user_duplicate"
 tmp_post_user_duplicate_q="${limit}"
 until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]
 do
-	echo "tmp_post_user_duplicate"
 	tmp_post_user_duplicate=$(sudo mariadb friendica -N -B -q -e \
 		"SELECT \`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}")
 	tmp_post_user_duplicate_q=$(echo "${tmp_post_user_duplicate}" | wc -l)

friendica-remove-invalid-photos.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+#Set your parameters here
+url=hub.azkware.net
+user=friendica
+group=www-data
+fileperm=640
+folderperm=750
+db=friendica
+folder=/var/www/friendica
+#Internal parameters:
+#Amount of times the loop has run
+iteration=0
+#Number of invalid avatars found. Set to 1 initially so we can run the loop at least once
+n=1
+#Number of entries processed
+nx=0
+#Generate an index to make searches faster
+((indexlength=37+${#url}))
+echo "Generating photo index..."
+mariadb $db -e "alter table contact add index if not exists photo_index (photo($indexlength))"
+#Go to the Friendica installation
+cd $folder || exit
+#Loop at least once, until no invalid avatars are found
+until [[ $n -eq 0 ]]
+do
+        #Add to the loop, reset values
+        iteration=$(("$iteration" + 1))
+        n=0
+        nx=0
+        dblist=$(mariadb $db -B -N -q -e "select id, photo, thumb, micro from contact where photo like 'https:\/\/$url/avatar/%' order by id")
+        m=$(echo "$dblist" | wc -l)
+        echo "$dblist" | while read -r id photo thumb micro
+        do
+                nx=$(("$nx" + 1))
+                folderescaped=${folder////\\/}
+                #Substitute the URL path with the folder path so we can search for it in the local file system
+                #Photo is nominally 320px, actually 300px
+                k_photo=$(echo "$photo" | sed -e "s/https:\/\/$url/$folderescaped/g" -e "s/\?ts=.*//g")
+                #Thumb is 80px
+                k_thumb=$(echo "$thumb" | sed -e "s/https:\/\/$url/$folderescaped/g" -e "s/\?ts=.*//g")
+                #Micro is 48px
+                k_micro=$(echo "$micro" | sed -e "s/https:\/\/$url/$folderescaped/g" -e "s/\?ts=.*//g")
+                #If any of the images is not found in the filesystem
+                if [[ ! -e "$k_photo" || ! -e "$k_thumb" || ! -e "$k_micro" ]]
+                then
+                        #If the avatar uses the standard fallback picture or is local, we cannot use it as a base
+                        avatar=$(mariadb $db -B -N -q -e "select avatar from contact where id = \"$id\" and not avatar like \"%$url\" and not avatar like \"%images/person%\"")
+                        #If we have a remote avatar as a fallback, download it
+                        if [[ $! -eq 0 && -n $avatar ]]
+                        then
+				echo "$id $avatar"
+                                sudo -u $user curl "$avatar" -s -o "$k_photo"
+                                #If the file is a valid picture (not empty, not text)
+                                if file "$k_photo" | grep -q -v -e "text" -e "Empty"
+                                then
+                                        #Also fetch for thumb/micro and resize
+                                        #As the photo is the largest version we have, we will use it as the base, and leave it last to convert
+                                        convert "$k_photo" -resize 80x80 -depth 16 "$k_thumb" && chmod "$fileperm" "$k_thumb" && chown "$user:$group" "$k_thumb"
+                                        convert "$k_photo" -resize 48x48 -depth 16 "$k_micro" && chmod "$fileperm" "$k_micro" && chown "$user:$group" "$k_micro"
+                                        convert "$k_photo" -resize 300x300 -depth 16 "$k_photo" && chmod "$fileperm" "$k_photo" && chown "$user:$group" "$k_photo"
+                                else
+                                        #If the avatar is not valid, set it as blank in the database
+                                        mariadb $db -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"$id\""
+                                        rm -rf "$k_photo"
+                                fi
+                        else
+                                #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later
+                                mariadb $db -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"$id\""
+                        fi
+                        n=$(( n + 1 ))
+                fi
+                printf "\rIteration %s\tPhotos: %s\tEntry %s/%s " "$iteration" "$n" "$nx" "$m"
+        done
+        wait
+        printf "\nFixing folders and moving to avatar cache...\n"
+        sudo -u $user bin/console movetoavatarcache #&> /dev/null
+        find ./avatar -depth -not -user "$user" -or -not -group "$group" -exec chown -v "$user:$group" {} \;
+        find ./avatar -depth -type f -and -not -type d -and -not -perm "$fileperm" -exec chmod -v "$fileperm" {} \;
+        find ./avatar -depth -type d -and -not -perm "$folderperm" -exec chmod -v "$folderperm" {} \;
+	#chown -R "$user:$group" ./avatar
+done
+#Drop index in the end to save storage
+mariadb $db -e "alter table contact drop index photo_index"