diff --git a/friendica-clean-database.sh b/friendica-clean-database.sh index e92e811..474c5f6 100755 --- a/friendica-clean-database.sh +++ b/friendica-clean-database.sh @@ -4,17 +4,11 @@ limit=1000 folder=/var/www/friendica user=friendica phpversion=php8.2 -tmpfile=/tmp/fcb dbengine=mariadb db=friendica dboptimizer=mariadboptimize bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null; -if [[ -f "${tmpfile}" ]]; then - rm "${tmpfile}" && touch "${tmpfile}" -else - touch "${tmpfile}" -fi echo "tmp_post_origin_deleted" tmp_post_origin_deleted_q="${limit}" @@ -23,7 +17,7 @@ until [[ "${tmp_post_origin_deleted_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_post_origin_deleted_q=0 while read -r uri_id uid; do - if [[ -n "${uri_id}" && -n "${uid}" ]]; then + if [[ -s "${uri_id}" && -s "${uid}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`post-origin\` WHERE \`parent-uri-id\` = ${uri_id} AND \`uid\` = ${uid}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -39,6 +33,7 @@ until [[ "${tmp_post_origin_deleted_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_origin_deleted_q} item(s) deleted until ${tmp_post_origin_deleted_current_uri_id} in ${final_i}s" done +wait echo "tmp_post_user_deleted" tmp_post_user_deleted_q="${limit}" @@ -47,7 +42,7 @@ until [[ "${tmp_post_user_deleted_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_post_user_deleted_q=0 while read -r uri_id; do - if [[ -n "${uri_id}" ]]; then + if [[ -s "${uri_id}" ]]; then tmp_post_user_deleted_q=$((tmp_post_user_deleted_q + 1)) "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}" & @@ -63,6 +58,7 @@ until [[ "${tmp_post_user_deleted_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_user_deleted_q} item(s) deleted until ${tmp_post_user_deleted_current_uri_id} in ${final_i}s" done +wait echo "tmp_post_uri_id_not_in_post_user" tmp_post_uri_id_not_in_post_user_q="${limit}" @@ -71,7 +67,7 @@ until [[ "${tmp_post_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_post_uri_id_not_in_post_user_q=0 while read -r uri_id; do - if [[ -n "${uri_id}" ]]; then + if [[ -s "${uri_id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`post\` WHERE \`uri-id\` = ${uri_id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -86,6 +82,7 @@ until [[ "${tmp_post_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_uri_id_not_in_post_user_current_uri_id} in ${final_i}s" done +wait echo "tmp_post_content_uri_id_not_in_post_user" tmp_post_content_uri_id_not_in_post_user_q="${limit}" @@ -94,7 +91,7 @@ until [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_post_content_uri_id_not_in_post_user_q=0 while read -r uri_id; do - if [[ -n "${uri_id}" ]]; then + if [[ -s "${uri_id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`post-content\` WHERE \`uri-id\` = ${uri_id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -109,6 +106,7 @@ until [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_content_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_content_uri_id_not_in_post_user_current_uri_id} in ${final_i}s" done +wait echo "tmp_post_thread_uri_id_not_in_post_user" tmp_post_thread_uri_id_not_in_post_user_q="${limit}" @@ -117,7 +115,7 @@ until [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_post_thread_uri_id_not_in_post_user_q=0 while read -r uri_id; do - if [[ -n "${uri_id}" ]]; then + if [[ -s "${uri_id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`post-thread\` WHERE \`uri-id\` = ${uri_id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -133,6 +131,7 @@ until [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_thread_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_thread_uri_id_not_in_post_user_current_uri_id} in ${final_i}s" done +wait echo "tmp_post_user_uri_id_not_in_post" tmp_post_user_uri_id_not_in_post_q="${limit}" @@ -141,7 +140,7 @@ until [[ "${tmp_post_user_uri_id_not_in_post_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_post_user_uri_id_not_in_post_q=0 while read -r uri_id; do - if [[ -n "${uri_id}" ]]; then + if [[ -s "${uri_id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -157,6 +156,7 @@ until [[ "${tmp_post_user_uri_id_not_in_post_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_user_uri_id_not_in_post_q} item(s) deleted until ${tmp_post_user_uri_id_not_in_post_current_uri_id} in ${final_i}s" done +wait echo "tmp_item_uri_not_in_valid_post_thread" tmp_item_uri_not_in_valid_post_thread_q="${limit}" @@ -165,7 +165,7 @@ until [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_item_uri_not_in_valid_post_thread_q=0 while read -r id; do - if [[ -n "${id}" ]]; then + if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -190,6 +190,7 @@ until [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_item_uri_not_in_valid_post_thread_q} item(s) deleted until ${tmp_item_uri_not_in_valid_post_thread_current_id} in ${final_i}s" done +wait echo "tmp_item_uri_not_in_valid_post_user" tmp_item_uri_not_in_valid_post_user_q="${limit}" @@ -198,7 +199,7 @@ until [[ "${tmp_item_uri_not_in_valid_post_user_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_item_uri_not_in_valid_post_user_q=0 while read -r id; do - if [[ -n "${id}" ]]; then + if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -216,6 +217,7 @@ until [[ "${tmp_item_uri_not_in_valid_post_user_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_item_uri_not_in_valid_post_user_q} item(s) deleted until ${tmp_item_uri_not_in_valid_post_user_current_id} in ${final_i}s" done +wait echo "tmp_attach_not_in_post_media" tmp_attach_not_in_post_media_q="${limit}" @@ -224,7 +226,7 @@ until [[ "${tmp_attach_not_in_post_media_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_attach_not_in_post_media_q=0 while read -r id; do - if [[ -n "${id}" ]]; then + if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`attach\` WHERE \`id\` = ${id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -240,6 +242,7 @@ until [[ "${tmp_attach_not_in_post_media_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_attach_not_in_post_media_q} item(s) deleted until ${tmp_attach_not_in_post_media_current_id} in ${final_i}s" done +wait echo "tmp_item_uri_not_valid" tmp_item_uri_not_valid_q="${limit}" @@ -250,7 +253,7 @@ until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_item_uri_not_valid_q=0 while read -r id; do - if [[ -n "${id}" ]]; then + if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -284,6 +287,7 @@ until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]; do final_i=$(($(date +%s) - initial_i)) echo "${tmp_item_uri_not_valid_q} item(s) deleted until ${tmp_item_uri_not_valid_current_id} in ${final_i}s" done +wait echo "tmp_item_uri_duplicate" tmp_item_uri_duplicate_q="${limit}" @@ -292,7 +296,7 @@ until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) tmp_item_uri_duplicate_q=0 while read -r id; do - if [[ -n "${id}" ]]; then + if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then @@ -302,10 +306,12 @@ until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do tmp_item_uri_duplicate_current_id="${id}" fi done < <("${dbengine}" "${db}" -N -B -q -e \ - "SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} AND t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}") + "SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} \ + AND t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}") final_i=$(($(date +%s) - initial_i)) echo "${tmp_item_uri_duplicate_q} item(s) deleted until ${tmp_item_uri_duplicate_current_id} in ${final_i}s" done +wait echo "tmp_post_media_duplicate" tmp_post_media_duplicate_q="${limit}" @@ -313,44 +319,45 @@ tmp_post_media_duplicate_current_id=0 until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) while read -r id; do - if [[ -n "${id}" ]]; then + if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ "DELETE FROM \`post-media\` WHERE \`id\` = ${id}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then wait -n fi - echo "${id}" tmp_post_media_duplicate_q=$((tmp_post_media_duplicate_q + 1)) tmp_post_media_duplicate_current_id="${id}" fi done < <("${dbengine}" "${db}" -N -B -q -e \ - "SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} AND u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}") + "SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} \ + AND u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}") final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_media_duplicate_q} item(s) deleted until ${tmp_post_media_duplicate_current_id} in ${final_i}s" done +wait echo "tmp_post_user_duplicate" tmp_post_user_duplicate_q="${limit}" tmp_post_user_duplicate_current_id=0 until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) - if [[ "${tmp_post_user_duplicate_q}" -gt 0 ]]; then - while read -r id; do - if [[ -n "${id}" ]]; then - "${dbengine}" "${db}" -N -B -q -e \ - "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then - wait -n - fi - tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1)) - tmp_post_user_duplicate_current_id="${id}" + while read -r id; do + if [[ -s "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" & + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then + wait -n fi - done < <("${dbengine}" "${db}" -N -B -q -e \ - "SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} AND v1.\`id\` < v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}") - fi + tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1)) + tmp_post_user_duplicate_current_id="${id}" + fi + done < <("${dbengine}" "${db}" -N -B -q -e \ + "SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} \ + AND v1.\`id\` < v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}") final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_user_duplicate_q} item(s) deleted until ${tmp_post_user_duplicate_current_id} in ${final_i}s" done +wait "${dboptimizer}" "${db}" #&> /dev/null; bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null;