diff --git a/friendica-clean-database.sh b/friendica-clean-database.sh index 7575d8e..acef858 100755 --- a/friendica-clean-database.sh +++ b/friendica-clean-database.sh @@ -1,110 +1,309 @@ #!/bin/bash interval=14 -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-origin-deleted\` (SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \ - \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL "$interval" DAY)); \ -DELETE FROM \`post-origin\` WHERE (\`parent-uri-id\`, \`uid\`) IN (SELECT * FROM \`tmp-post-origin-deleted\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-user-deleted\` (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \ - \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL "$interval" DAY)); \ -DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-user-deleted\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post\` \ - WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \ -DELETE FROM \`post\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-uri-id-not-in-post-user\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-content-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-content\` \ - WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \ -DELETE FROM \`post-content\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-content-uri-id-not-in-post-user\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-thread-uri-id-not-in-post-user\` (SELECT \`uri-id\` FROM \`post-thread\` \ - WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`)); \ -DELETE FROM \`post-thread\` WHERE \`uri-id\` IN (SELECT * FROM \`tmp-post-thread-uri-id-not-in-post-user\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-user-uri-id-not-in-post\` (SELECT \`uri-id\` FROM \`post-user\` \ - WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`)); \ -DELETE FROM \`post-user\` WHERE \`uri-id\` IN (SELECT \`uri-id\` FROM \`tmp-post-user-uri-id-not-in-post\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-thread\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN \ - (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL $interval DAY) \ - AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` \ - WHERE (\`mention\` OR \`starred\` OR \`wall\`) AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \ - AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-category\` \ - WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ - AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-collection\` \ - WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ - AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-media\` \ - WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ - AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` INNER JOIN \`contact\` ON \`contact\`.\`id\` = \`contact-id\` AND \`notify_new_posts\` \ - WHERE \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \ - AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` \ - WHERE (\`origin\` OR \`event-id\` != 0 OR \`post-type\` = 128) AND \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \ - AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` \ - WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`)) \ -); \ -DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-thread\`) ;" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-item-uri-not-in-valid-post-user\` (SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \ - AND \`received\` < (CURDATE() - INTERVAL $interval DAY) \ - AND NOT \`uri-id\` IN ( \ - SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` \ - ) AND NOT \`uri-id\` IN ( \ - SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL $interval DAY) \ - ) \ -)); \ -DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-in-valid-post-user\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-attach-not-in-post-media\` (SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`)); \ -DELETE FROM \`attach\` WHERE \`id\` IN (SELECT * FROM \`tmp-attach-not-in-post-media\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-item-uri-not-valid\` (SELECT \`id\` FROM \`item-uri\` WHERE (\ - \`id\` < (\ - SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`thr-parent-id\` FROM \`post-user\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`external-id\` FROM \`post-user\` WHERE \`external-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`replies-id\` FROM \`post-user\` WHERE \`replies-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`context-id\` FROM \`post-thread\` WHERE \`context-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`conversation-id\` FROM \`post-thread\` WHERE \`conversation-id\`= \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`mail\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`event\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`user-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`apcontact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`diaspora-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`inbox-status\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`inbox-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`parent-uri-id\` FROM \`mail\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \ - ) AND NOT EXISTS (\ - SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \ - )\ -)); \ -DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-not-valid\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-item-uri-duplicate\` (SELECT \`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\`); \ -DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT * FROM \`tmp-item-uri-duplicate\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-media-duplicate\` (SELECT \`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\`); \ -DELETE FROM \`post-media\` WHERE \`id\` IN (SELECT * FROM \`tmp-post-media-duplicate\`);" -sudo mariadb friendica --verbose -v -v --show-warnings --execute=\ -"CREATE TEMPORARY TABLE \`tmp-post-user-duplicate\` (SELECT \`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\`); \ -DELETE FROM \`post-user\` WHERE \`id\` IN (SELECT * FROM \`tmp-post-user-duplicate\`;" +limit=100 + +tmp_post_origin_deleted_q="${limit}" +until [[ "${tmp_post_origin_deleted_q}" -lt "${limit}" ]] +do + echo "tmp_post_origin_deleted" + tmp_post_origin_deleted=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) LIMIT ${limit}" \ + ); + tmp_post_origin_deleted_q="${#tmp_post_origin_deleted[@]}" + if [[ "${tmp_post_origin_deleted_q}" -gt 0 ]] + then + echo "${tmp_post_origin_deleted}" | while read -r uri_id uid + do + if [[ -n "${uri_id}" && -n "${uid}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post-origin\` WHERE \`parent-uri-id\` = ${uri_id} AND \`uid\` = ${uid}" + fi + done + fi +done + +tmp_post_user_deleted_q="${limit}" +until [[ "${tmp_post_user_deleted_q}" -lt "${limit}" ]] +do + echo "tmp_post_user_deleted" + tmp_post_user_deleted=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) LIMIT ${limit}" \ + ); + tmp_post_user_deleted_q="${#tmp_post_user_deleted[@]}" + if [[ "${tmp_post_user_deleted_q}" -gt 0 ]] + then + echo "${tmp_post_user_deleted}" | while read -r uid + do + if [[ -n "${uid}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uid}" + fi + done + fi +done + +tmp_post_uri_id_not_in_post_user_q="${limit}" +until [[ "${tmp_post_uri_id_not_in_post_user_q}" -lt "${limit}" ]] +do + echo "tmp_post_uri_id_not_in_post_user" + tmp_post_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`uri-id\` FROM \`post\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}" \ + ); + tmp_post_uri_id_not_in_post_user_q="${#tmp_post_uri_id_not_in_post_user[@]}" + if [[ "${tmp_post_uri_id_not_in_post_user_q}" -gt 0 ]] + then + echo "${tmp_post_uri_id_not_in_post_user}" | while read -r uri_id + do + if [[ -n "${uri_id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post\` WHERE \`uri-id\` = ${uri_id}" + fi + done + fi +done + +tmp_post_content_uri_id_not_in_post_user_q="${limit}" +until [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -lt "${limit}" ]] +do + echo "tmp_post_content_uri_id_not_in_post_user" + tmp_post_content_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`uri-id\` FROM \`post-content\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}" \ + ); + tmp_post_content_uri_id_not_in_post_user_q="${#tmp_post_content_uri_id_not_in_post_user[@]}" + if [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -gt 0 ]] + then + echo "${tmp_post_content_uri_id_not_in_post_user}" | while read -r uri_id + do + if [[ -n "${uri_id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post-content\` WHERE \`uri-id\` = ${uri_id}" + fi + done + fi +done + +tmp_post_thread_uri_id_not_in_post_user_q="${limit}" +until [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -lt "${limit}" ]] +do + echo "tmp_post_thread_uri_id_not_in_post_user" + tmp_post_thread_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`uri-id\` FROM \`post-thread\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) LIMIT ${limit}" \ + ); + tmp_post_thread_uri_id_not_in_post_user_q="${#tmp_post_thread_uri_id_not_in_post_user[@]}" + if [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -gt 0 ]] + then + echo "${tmp_post_thread_uri_id_not_in_post_user}" | while read -r uri_id + do + if [[ -n "${uri_id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post-thread\` WHERE \`uri-id\` = ${uri_id}" + fi + done + fi +done + +tmp_post_user_uri_id_not_in_post_q="${limit}" +until [[ "${tmp_post_user_uri_id_not_in_post_q}" -lt "${limit}" ]] +do + echo "tmp_post_user_uri_id_not_in_post" + tmp_post_user_uri_id_not_in_post=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`) LIMIT ${limit}" \ + ); + tmp_post_user_uri_id_not_in_post_q="${#tmp_post_user_uri_id_not_in_post[@]}" + if [[ "${tmp_post_user_uri_id_not_in_post_q}" -gt 0 ]] + then + echo "${tmp_post_user_uri_id_not_in_post}" | while read -r uri_id + do + if [[ -n "${uri_id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}" + fi + done + fi +done + +tmp_item_uri_not_in_valid_post_thread_q="${limit}" +until [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -lt "${limit}" ]] +do + echo "tmp_item_uri_not_in_valid_post_thread" + tmp_item_uri_not_in_valid_post_thread=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL ${interval} DAY) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` WHERE (\`mention\` OR \`starred\` OR \`wall\`) \ + AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-category\` WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-collection\` WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-media\` WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` INNER JOIN \`contact\` ON \`contact\`.\`id\` = \`contact-id\` \ + AND \`notify_new_posts\` WHERE \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` WHERE (\`origin\` OR \`event-id\` != 0 OR \`post-type\` = 128) \ + AND \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \ + AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`)) \ + LIMIT ${limit}" \ + ); + tmp_item_uri_not_in_valid_post_thread_q="${#tmp_item_uri_not_in_valid_post_thread[@]}" + if [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -gt 0 ]] + then + echo "${tmp_item_uri_not_in_valid_post_thread}" | while read -r id + do + if [[ -n "${id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" + fi + done + fi +done + +tmp_item_uri_not_in_valid_post_user_q="${limit}" +until [[ "${tmp_item_uri_not_in_valid_post_user_q}" -lt "${limit}" ]] +do + echo "tmp_item_uri_not_in_valid_post_user" + tmp_item_uri_not_in_valid_post_user=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`id\` FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \ + AND \`received\` < (CURDATE() - INTERVAL ${interval} DAY) AND NOT \`uri-id\` IN ( SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 \ + AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` ) AND NOT \`uri-id\` IN ( SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 \ + AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL ${interval} DAY) ) ) LIMIT ${limit}" \ + ); + tmp_item_uri_not_in_valid_post_user_q="${#tmp_item_uri_not_in_valid_post_user[@]}" + if [[ "${tmp_item_uri_not_in_valid_post_user_q}" -gt 0 ]] + then + echo "${tmp_item_uri_not_in_valid_post_user}" | while read -r id + do + if [[ -n "${id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" + fi + done + fi +done + +tmp_attach_not_in_post_media_q="${limit}" +until [[ "${tmp_attach_not_in_post_media_q}" -lt "${limit}" ]] +do + echo "tmp_attach_not_in_post_media" + tmp_attach_not_in_post_media=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`) LIMIT ${limit}" \ + ); + tmp_attach_not_in_post_media_q="${#tmp_attach_not_in_post_media[@]}" + if [[ "${tmp_attach_not_in_post_media_q}" -gt 0 ]] + then + echo "${tmp_attach_not_in_post_media}" | while read -r id + do + if [[ -n "${id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`attach\` WHERE \`id\` = ${id}" + fi + done + fi +done + +tmp_item_uri_not_valid_q="${limit}" +until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]] +do + echo "tmp_item_uri_not_valid" + tmp_item_uri_not_valid=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`id\` FROM \`item-uri\` WHERE ( \`id\` < ( SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`thr-parent-id\` FROM \`post-user\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`external-id\` FROM \`post-user\` WHERE \`external-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`replies-id\` FROM \`post-user\` WHERE \`replies-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`context-id\` FROM \`post-thread\` WHERE \`context-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`conversation-id\` FROM \`post-thread\` WHERE \`conversation-id\`= \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`mail\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`event\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`user-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`apcontact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`diaspora-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`inbox-status\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`inbox-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`parent-uri-id\` FROM \`mail\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` ) \ + AND NOT EXISTS ( SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` )) \ + LIMIT ${limit}" \ + ); + tmp_item_uri_not_valid_q="${#tmp_item_uri_not_valid[@]}" + if [[ "${tmp_item_uri_not_valid_q}" -gt 0 ]] + then + echo "${tmp_item_uri_not_valid}" | while read -r id + do + if [[ -n "${id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`item_uri\` WHERE \`id\` = ${id}" + fi + done + fi +done + +tmp_item_uri_duplicate_q="${limit}" +until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]] +do + echo "tmp_item_uri_duplicate" + tmp_item_uri_duplicate=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}" \ + ) + tmp_item_uri_duplicate_q="${#tmp_item_uri_duplicate[@]}" + if [[ "${tmp_item_uri_duplicate_q}" -gt 0 ]] + then + echo "${tmp_item_uri_duplicate}" | while read -r id + do + if [[ -n "${id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" + fi + done + fi +done + +tmp_post_media_duplicate_q="${limit}" +until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]] +do + echo "tmp_post_media_duplicate" + tmp_post_media_duplicate=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}" \ + ) + tmp_post_media_duplicate_q="${#tmp_post_media_duplicate[@]}" + if [[ "${tmp_post_media_duplicate_q}" -gt 0 ]] + then + echo "${tmp_post_media_duplicate}" | while read -r id + do + if [[ -n "${id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post-media\` WHERE \`id\` = ${id}" + fi + done + fi +done + +tmp_post_user_duplicate_q="${limit}" +until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]] +do + echo "tmp_post_user_duplicate" + tmp_post_user_duplicate=$(sudo mariadb friendica -N -B -q -e \ + "SELECT \`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}" \ + ) + tmp_post_user_duplicate_q="${#tmp_post_user_duplicate[@]}" + if [[ "${tmp_post_user_duplicate_q}" -gt 0 ]] + then + echo "${tmp_post_user_duplicate}" | while read -r id + do + if [[ -n "${id}" ]] + then + sudo mariadb friendica -v -v -v --show-warnings -e \ + "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" + fi + done + fi +done diff --git a/friendica-fix-avatar-permissions.sh b/friendica-fix-avatar-permissions.sh index e09e47d..46f22d9 100755 --- a/friendica-fix-avatar-permissions.sh +++ b/friendica-fix-avatar-permissions.sh @@ -22,7 +22,8 @@ loop_1(){ #Specific compression for large GIF files while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]] do - nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 99) -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null + frameamount=$(exiftool -b -FrameCount "${p}" || 1) + nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null done elif [[ "${p}" =~ .png ]] then @@ -30,7 +31,7 @@ loop_1(){ elif [[ "${p}" =~ .webp ]] then #If file is not animated - if [[ $(grep -v -q -e "ANIM" -e "ANMF" "${p}") ]] + if ! grep -v -q -e "ANIM" -e "ANMF" "${p}" then nice -n 10 cwebp -mt -af -quiet "${p}" -o /tmp/temp.webp #&> /dev/null if [[ -f /tmp/temp.webp ]] diff --git a/friendica-remove-old-photos-parallel.sh b/friendica-remove-old-photos-parallel.sh index 5d2504c..1534713 100755 --- a/friendica-remove-old-photos-parallel.sh +++ b/friendica-remove-old-photos-parallel.sh @@ -2,7 +2,7 @@ IFS=" " #Set your parameters here -url=hub.example.net +url=friendica.example.net db=friendica folder=/var/www/friendica folderavatar=/var/www/friendica/avatar @@ -44,11 +44,12 @@ do do m=$(( m + 1 )) loop "${x}" "${m}" "${n}" "${d}" & - if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 1 )) ]] + if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) * 2 )) ]] then wait -n fi - done < <(sudo find "${x}" -type f -mtime -8) + #done < <(sudo find "${x}" -type f -mtime -8) + done < <(sudo find "${x}" -type f) fi fi done diff --git a/media-optimize-local.sh b/media-optimize-local.sh index abbc6f0..b745d98 100755 --- a/media-optimize-local.sh +++ b/media-optimize-local.sh @@ -1,56 +1,71 @@ #!/bin/bash IFS=" " - #media-optimize - find . -type f -iname "*.jp*" -size +50k | ( - while read p - do - nice -n 15 jpegoptim -m 76 "${p}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] - then - wait -n - fi - done; - wait - ) - find . -type f -iname "*.gif" -size +500k | ( - while read q - do - nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] - then - wait -n - fi - done; - wait - ) - find . -type f -iname "*.png" -size +300k | ( - while read r - do - nice -n 15 oxipng -o max "${r}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] - then - wait -n - fi - done; - wait - ) - #compress-webp - find . -type f -iname "*.webp" -size +200k | ( - while read s - do - nice -n 15 cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp - if [[ -f /tmp/"${s##.*\/}"_temp.webp ]] +#media-optimize +find . -type f -iname "*.jp*" -size +50k | ( + while read p + do + nice -n 15 jpegoptim -m 76 "${p}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + done; + wait +) +find . -type f -iname "*.gif" -size +500k | ( + while read q + do + nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + done; + wait +) +#Specific compression for large GIF files +find . -type f -iname "*.gif" -size +512000 | ( + while read p + do + while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]] + do + frameamount=$(exiftool -b -FrameCount "${p}") + nice -n 15 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" + done + done +) +find . -type f -iname "*.png" -size +500k | ( + while read r + do + nice -n 15 oxipng -o max "${r}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] + then + wait -n + fi + done; +wait +) +#compress-webp +find . -type f -iname "*.webp" -size +50k | ( + while read s + do + #If file is not animated + if ! grep -v -q -e "ANIM" -e "ANMF" "${s}" + then + cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp + if [ -f /tmp/"${s##.*\/}"_temp.webp ] then - size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp) - size_original=$(stat -c%s "${s}") - if [[ "${size_original}" -gt "${size_new}" ]] + size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0) + size_original=$(stat -c%s "${s}" || 0) + if [ "$size_original" -gt "$size_new" ] then - mv /tmp/"${s##.*\/}"_temp.webp "${s}" + mv /tmp/"${s##.*\/}"_temp.webp "$s" else rm /tmp/"${s##.*\/}"_temp.webp fi fi - done; - wait - ) + fi + done; + wait +)