feat: Rewrite completely to ensure fewer database locks during each cycle

friendica-clean-database.sh

@@ -1,110 +1,309 @@
 #!/bin/bash
 interval=14
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-origin-deleted\` (SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \
-  \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL "$interval" DAY)); \
-DELETE FROM \`post-origin\` WHERE (\`parent-uri-id\`, \`uid\`) IN (SELECT * FROM \`tmp-post-origin-deleted\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-user-deleted\` (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \
-  \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL "$interval" DAY)); \
-DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-user-deleted\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post\` \
-  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \
-DELETE FROM \`post\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-uri-id-not-in-post-user\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-content-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-content\` \
-  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \
-DELETE FROM \`post-content\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-content-uri-id-not-in-post-user\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-thread-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-thread\` \
-  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \
-DELETE FROM \`post-thread\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-thread-uri-id-not-in-post-user\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-user-uri-id-not-in-post\` (SELECT \`uri-id\` FROM \`post-user\` \
-  WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`)); \
-DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT \`uri-id\` FROM \`tmp-post-user-uri-id-not-in-post\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-thread\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN \
-  (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL $interval DAY) \
-    AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` \
-      WHERE (\`mention\` OR \`starred\` OR \`wall\`) AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \
-    AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-category\` \
-      WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
-    AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-collection\` \
-      WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
-    AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-media\` \
-      WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
-    AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` INNER JOIN \`contact\` ON \`contact\`.\`id\` = \`contact-id\` AND \`notify_new_posts\` \
-      WHERE \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \
-    AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` \
-      WHERE (\`origin\` OR \`event-id\` != 0 OR \`post-type\` = 128) AND \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \
-    AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` \
-      WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`)) \
-); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-thread\`) ;"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-user\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \
-  AND \`received\` < (CURDATE() - INTERVAL $interval DAY) \
-  AND NOT \`uri-id\` IN ( \
-    SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` \
-  ) AND NOT \`uri-id\` IN ( \
-    SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL $interval DAY) \
-  ) \
-)); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-user\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-attach-not-in-post-media\` (SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`)); \
-DELETE FROM \`attach\` WHERE \`id\` IN (SELECT * FROM \`tmp-attach-not-in-post-media\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-item-uri-not-valid\` (SELECT \`id\` FROM \`item-uri\` WHERE (\
-  \`id\` < (\
-    SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`thr-parent-id\` FROM \`post-user\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`external-id\` FROM \`post-user\` WHERE \`external-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`replies-id\` FROM \`post-user\` WHERE \`replies-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`context-id\` FROM \`post-thread\` WHERE \`context-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`conversation-id\` FROM \`post-thread\` WHERE \`conversation-id\`= \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`mail\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`event\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`user-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`apcontact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`diaspora-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`inbox-status\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`inbox-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`parent-uri-id\` FROM \`mail\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \
-  ) AND NOT EXISTS (\
-    SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \
-  )\
-)); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-valid\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-item-uri-duplicate\` (SELECT \`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\`); \
-DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-duplicate\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-media-duplicate\` (SELECT \`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\`); \
-DELETE FROM \`post-media\` WHERE \`id\` IN (SELECT * FROM \`tmp-post-media-duplicate\`);"
-sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
-"CREATE TEMPORARY TABLE \`tmp-post-user-duplicate\` (SELECT \`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\`); \
-DELETE FROM \`post-user\` WHERE \`id\` IN (SELECT * FROM \`tmp-post-user-duplicate\`;"
+limit=100
+
+tmp_post_origin_deleted_q="${limit}"
+until [[ "${tmp_post_origin_deleted_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_origin_deleted"
+	tmp_post_origin_deleted=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) LIMIT ${limit}" \
+	);
+	tmp_post_origin_deleted_q="${#tmp_post_origin_deleted[@]}"
+	if [[ "${tmp_post_origin_deleted_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_origin_deleted}" | while read -r uri_id uid
+		do
+			if [[ -n "${uri_id}" && -n "${uid}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post-origin\` WHERE \`parent-uri-id\` = ${uri_id} AND \`uid\` = ${uid}"
+			fi
+		done
+	fi
+done
+
+tmp_post_user_deleted_q="${limit}"
+until [[ "${tmp_post_user_deleted_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_user_deleted"
+	tmp_post_user_deleted=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) LIMIT ${limit}" \
+	);
+	tmp_post_user_deleted_q="${#tmp_post_user_deleted[@]}"
+	if [[ "${tmp_post_user_deleted_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_user_deleted}" | while read -r uid
+		do
+			if [[ -n "${uid}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uid}"
+			fi
+		done
+	fi
+done
+
+tmp_post_uri_id_not_in_post_user_q="${limit}"
+until [[ "${tmp_post_uri_id_not_in_post_user_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_uri_id_not_in_post_user"
+	tmp_post_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`uri-id\` FROM \`post\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}" \
+	);
+	tmp_post_uri_id_not_in_post_user_q="${#tmp_post_uri_id_not_in_post_user[@]}"
+	if [[ "${tmp_post_uri_id_not_in_post_user_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_uri_id_not_in_post_user}" | while read -r uri_id
+		do
+			if [[ -n "${uri_id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post\` WHERE \`uri-id\` = ${uri_id}"
+			fi
+		done
+	fi
+done
+
+tmp_post_content_uri_id_not_in_post_user_q="${limit}"
+until [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_content_uri_id_not_in_post_user"
+	tmp_post_content_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`uri-id\` FROM \`post-content\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}" \
+	);
+	tmp_post_content_uri_id_not_in_post_user_q="${#tmp_post_content_uri_id_not_in_post_user[@]}"
+	if [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_content_uri_id_not_in_post_user}" | while read -r uri_id
+		do
+			if [[ -n "${uri_id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post-content\` WHERE \`uri-id\` = ${uri_id}"
+			fi
+		done
+	fi
+done
+
+tmp_post_thread_uri_id_not_in_post_user_q="${limit}"
+until [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_thread_uri_id_not_in_post_user"
+	tmp_post_thread_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`uri-id\` FROM \`post-thread\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}" \
+	);
+	tmp_post_thread_uri_id_not_in_post_user_q="${#tmp_post_thread_uri_id_not_in_post_user[@]}"
+	if [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_thread_uri_id_not_in_post_user}" | while read -r uri_id
+		do
+			if [[ -n "${uri_id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post-thread\` WHERE \`uri-id\` = ${uri_id}"
+			fi
+		done
+	fi
+done
+
+tmp_post_user_uri_id_not_in_post_q="${limit}"
+until [[ "${tmp_post_user_uri_id_not_in_post_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_user_uri_id_not_in_post"
+	tmp_post_user_uri_id_not_in_post=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`) LIMIT ${limit}" \
+	);
+	tmp_post_user_uri_id_not_in_post_q="${#tmp_post_user_uri_id_not_in_post[@]}"
+	if [[ "${tmp_post_user_uri_id_not_in_post_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_user_uri_id_not_in_post}" | while read -r uri_id
+		do
+			if [[ -n "${uri_id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}"
+			fi
+		done
+	fi
+done
+
+tmp_item_uri_not_in_valid_post_thread_q="${limit}"
+until [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -lt "${limit}" ]]
+do
+	echo "tmp_item_uri_not_in_valid_post_thread"
+	tmp_item_uri_not_in_valid_post_thread=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL ${interval} DAY) \
+			AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` WHERE (\`mention\` OR \`starred\` OR \`wall\`) \
+			AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \
+			AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-category\` WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
+			AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-collection\` WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
+			AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-media\` WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
+			AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` INNER JOIN \`contact\` ON \`contact\`.\`id\` = \`contact-id\` \
+				AND \`notify_new_posts\` WHERE \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \
+			AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` WHERE (\`origin\` OR \`event-id\` != 0 OR \`post-type\` = 128) \
+				AND \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \
+			AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`)) \
+			LIMIT ${limit}" \
+	);
+	tmp_item_uri_not_in_valid_post_thread_q="${#tmp_item_uri_not_in_valid_post_thread[@]}"
+	if [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -gt 0 ]]
+	then
+		echo "${tmp_item_uri_not_in_valid_post_thread}" | while read -r id
+		do
+			if [[ -n "${id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`item-uri\` WHERE \`id\` = ${id}"
+			fi
+		done
+	fi
+done
+
+tmp_item_uri_not_in_valid_post_user_q="${limit}"
+until [[ "${tmp_item_uri_not_in_valid_post_user_q}" -lt "${limit}" ]]
+do
+	echo "tmp_item_uri_not_in_valid_post_user"
+	tmp_item_uri_not_in_valid_post_user=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \
+		AND \`received\` < (CURDATE() - INTERVAL ${interval} DAY) AND NOT \`uri-id\` IN ( SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 \
+		AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` ) AND NOT \`uri-id\` IN ( SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 \
+		AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL ${interval} DAY) ) ) LIMIT ${limit}" \
+	);
+	tmp_item_uri_not_in_valid_post_user_q="${#tmp_item_uri_not_in_valid_post_user[@]}"
+	if [[ "${tmp_item_uri_not_in_valid_post_user_q}" -gt 0 ]]
+	then
+		echo "${tmp_item_uri_not_in_valid_post_user}" | while read -r id
+		do
+			if [[ -n "${id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`item-uri\` WHERE \`id\` = ${id}"
+			fi
+		done
+	fi
+done
+
+tmp_attach_not_in_post_media_q="${limit}"
+until [[ "${tmp_attach_not_in_post_media_q}" -lt "${limit}" ]]
+do
+	echo "tmp_attach_not_in_post_media"
+	tmp_attach_not_in_post_media=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`) LIMIT ${limit}" \
+	);
+	tmp_attach_not_in_post_media_q="${#tmp_attach_not_in_post_media[@]}"
+	if [[ "${tmp_attach_not_in_post_media_q}" -gt 0 ]]
+	then
+		echo "${tmp_attach_not_in_post_media}" | while read -r id
+		do
+			if [[ -n "${id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`attach\` WHERE \`id\` = ${id}"
+			fi
+		done
+	fi
+done
+
+tmp_item_uri_not_valid_q="${limit}"
+until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]
+do
+	echo "tmp_item_uri_not_valid"
+	tmp_item_uri_not_valid=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`id\` FROM \`item-uri\` WHERE ( \`id\` < ( SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`thr-parent-id\` FROM \`post-user\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`external-id\` FROM \`post-user\` WHERE \`external-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`replies-id\` FROM \`post-user\` WHERE \`replies-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`context-id\` FROM \`post-thread\` WHERE \`context-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`conversation-id\` FROM \`post-thread\` WHERE \`conversation-id\`= \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`mail\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`event\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`user-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`apcontact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`diaspora-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`inbox-status\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`inbox-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`parent-uri-id\` FROM \`mail\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` ) \
+		AND NOT EXISTS ( SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` )) \
+		LIMIT ${limit}" \
+	);
+	tmp_item_uri_not_valid_q="${#tmp_item_uri_not_valid[@]}"
+	if [[ "${tmp_item_uri_not_valid_q}" -gt 0 ]]
+	then
+		echo "${tmp_item_uri_not_valid}" | while read -r id
+		do
+			if [[ -n "${id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`item_uri\` WHERE \`id\` = ${id}"
+			fi
+		done
+	fi
+done
+
+tmp_item_uri_duplicate_q="${limit}"
+until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]
+do
+	echo "tmp_item_uri_duplicate"
+	tmp_item_uri_duplicate=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}" \
+	)
+	tmp_item_uri_duplicate_q="${#tmp_item_uri_duplicate[@]}"
+	if [[ "${tmp_item_uri_duplicate_q}" -gt 0 ]]
+	then
+		echo "${tmp_item_uri_duplicate}" | while read -r id
+		do
+			if [[ -n "${id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`item-uri\` WHERE \`id\` = ${id}"
+			fi
+		done
+	fi
+done
+
+tmp_post_media_duplicate_q="${limit}"
+until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_media_duplicate"
+	tmp_post_media_duplicate=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}" \
+	)
+	tmp_post_media_duplicate_q="${#tmp_post_media_duplicate[@]}"
+	if [[ "${tmp_post_media_duplicate_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_media_duplicate}" | while read -r id
+		do
+			if [[ -n "${id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post-media\` WHERE \`id\` = ${id}"
+			fi
+		done
+	fi
+done
+
+tmp_post_user_duplicate_q="${limit}"
+until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]
+do
+	echo "tmp_post_user_duplicate"
+	tmp_post_user_duplicate=$(sudo mariadb friendica -N -B -q -e \
+		"SELECT \`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}" \
+	)
+	tmp_post_user_duplicate_q="${#tmp_post_user_duplicate[@]}"
+	if [[ "${tmp_post_user_duplicate_q}" -gt 0 ]]
+	then
+		echo "${tmp_post_user_duplicate}" | while read -r id
+		do
+			if [[ -n "${id}" ]]
+			then
+				sudo mariadb friendica -v -v -v --show-warnings -e \
+				"DELETE FROM \`post-user\` WHERE \`id\` = ${id}"
+			fi
+		done
+	fi
+done

friendica-fix-avatar-permissions.sh

@@ -22,7 +22,8 @@ loop_1(){
 		#Specific compression for large GIF files
 		while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]
 		do
-			nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 99) -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
+			frameamount=$(exiftool -b -FrameCount "${p}" || 1)
+			nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
 		done
 	elif [[ "${p}" =~ .png ]]
 	then
@@ -30,7 +31,7 @@ loop_1(){
 	elif [[ "${p}" =~ .webp ]]
 	then
 		#If file is not animated
-		if [[ $(grep -v -q -e "ANIM" -e "ANMF" "${p}") ]]
+		if ! grep -v -q -e "ANIM" -e "ANMF" "${p}"
 		then
 			nice -n 10 cwebp -mt -af -quiet "${p}" -o /tmp/temp.webp #&> /dev/null
 			if [[ -f /tmp/temp.webp ]]

friendica-remove-old-photos-parallel.sh

@@ -2,7 +2,7 @@
 IFS="
 "
 #Set your parameters here
-url=hub.example.net
+url=friendica.example.net
 db=friendica
 folder=/var/www/friendica
 folderavatar=/var/www/friendica/avatar
@@ -44,11 +44,12 @@ do
 			do
 				m=$(( m + 1 ))
 				loop "${x}" "${m}" "${n}" "${d}" &
-			        if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 1 )) ]]
+			        if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) * 2 )) ]]
 			        then
                 			wait -n
 			        fi
-			done < <(sudo find "${x}" -type f -mtime -8)
+			#done < <(sudo find "${x}" -type f -mtime -8)
+			done < <(sudo find "${x}" -type f)
                 fi
         fi
 done

media-optimize-local.sh

@@ -1,56 +1,71 @@
 #!/bin/bash
 IFS="
 "
-    #media-optimize
-    find . -type f -iname "*.jp*" -size +50k | (
-        while read p
-	do
-	    nice -n 15 jpegoptim -m 76 "${p}" &
-	    if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
-	    then
-	        wait -n
-            fi
-	done;
-    	wait
-    )
-    find . -type f -iname "*.gif" -size +500k | (
-        while read q
-	do
-            nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" &
-	    if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
-	    then
-	        wait -n
-            fi
-	done;
-    	wait
-    )
-    find . -type f -iname "*.png" -size +300k | (
-        while read r
-	do
-	    nice -n 15 oxipng -o max "${r}" &
-	    if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
-	    then
-	        wait -n
-            fi
-	done;
-    	wait
-    )
-    #compress-webp
-    find . -type f -iname "*.webp" -size +200k | (
-        while read s
-	do
-            nice -n 15 cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp
-            if [[ -f /tmp/"${s##.*\/}"_temp.webp ]]
+#media-optimize
+find . -type f -iname "*.jp*" -size +50k | (
+    while read p
+    do
+        nice -n 15 jpegoptim -m 76 "${p}" &
+        if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
+        then
+            wait -n
+        fi
+    done;
+    wait
+)
+find . -type f -iname "*.gif" -size +500k | (
+    while read q
+    do
+        nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" &
+        if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
+        then
+            wait -n
+        fi
+    done;
+    wait
+)
+#Specific compression for large GIF files
+find . -type f -iname "*.gif" -size +512000 | (
+    while read p
+    do
+        while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]
+        do
+            frameamount=$(exiftool -b -FrameCount "${p}")
+            nice -n 15 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}"
+        done
+    done
+)
+find . -type f -iname "*.png" -size +500k | (
+    while read r
+        do
+        nice -n 15 oxipng -o max "${r}" &
+        if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
+        then
+            wait -n
+        fi
+    done;
+wait
+)
+#compress-webp
+find . -type f -iname "*.webp" -size +50k | (
+    while read s
+    do
+        #If file is not animated
+        if ! grep -v -q -e "ANIM" -e "ANMF" "${s}"
+        then
+            cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp
+            if [ -f /tmp/"${s##.*\/}"_temp.webp ]
             then
-                size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp)
-                size_original=$(stat -c%s "${s}")
-                if [[ "${size_original}" -gt "${size_new}" ]]
+                size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0)
+                size_original=$(stat -c%s "${s}" || 0)
+                if [ "$size_original" -gt "$size_new" ]
                 then
-                    mv /tmp/"${s##.*\/}"_temp.webp "${s}"
+                    mv /tmp/"${s##.*\/}"_temp.webp "$s"
                 else
                     rm /tmp/"${s##.*\/}"_temp.webp
                 fi
             fi
-        done;
-	wait
-    )
+        fi
+    done;
+    wait
+)