diff --git a/friendica-clean-database.sh b/friendica-clean-database.sh index ca35b81..ce256b7 100755 --- a/friendica-clean-database.sh +++ b/friendica-clean-database.sh @@ -7,9 +7,8 @@ phpversion=php8.2 dbengine=mariadb db=friendica dboptimizer=mariadb-optimize -intense_optimizations=${1:-"0"} -bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null +bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null; echo "tmp_post_origin_deleted" tmp_post_origin_deleted_q="${limit}" @@ -290,79 +289,77 @@ until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]; do done wait -if [[ "${intense_optimizations}" -gt 0 ]]; then - echo "tmp_item_uri_duplicate" - tmp_item_uri_duplicate_q="${limit}" - tmp_item_uri_duplicate_current_id=0 - until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do - initial_i=$(date +%s) - tmp_item_uri_duplicate_q=0 - while read -r id; do - if [[ -s "${id}" ]]; then - "${dbengine}" "${db}" -N -B -q -e \ - "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then - wait -n - fi - tmp_item_uri_duplicate_q=$((tmp_item_uri_duplicate_q + 1)) - tmp_item_uri_duplicate_current_id="${id}" +echo "tmp_item_uri_duplicate" +tmp_item_uri_duplicate_q="${limit}" +tmp_item_uri_duplicate_current_id=0 +until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_item_uri_duplicate_q=0 + while read -r id; do + if [[ -s "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" & + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then + wait -n fi - done < <("${dbengine}" "${db}" -N -B -q -e \ - "SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} \ + tmp_item_uri_duplicate_q=$((tmp_item_uri_duplicate_q + 1)) + tmp_item_uri_duplicate_current_id="${id}" + fi + done < <("${dbengine}" "${db}" -N -B -q -e \ + "SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} \ AND t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}") - final_i=$(($(date +%s) - initial_i)) - echo "${tmp_item_uri_duplicate_q} item(s) deleted until ${tmp_item_uri_duplicate_current_id} in ${final_i}s" - done - wait + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_item_uri_duplicate_q} item(s) deleted until ${tmp_item_uri_duplicate_current_id} in ${final_i}s" +done +wait - echo "tmp_post_media_duplicate" - tmp_post_media_duplicate_q="${limit}" - tmp_post_media_duplicate_current_id=0 - until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do - initial_i=$(date +%s) - tmp_post_media_duplicate_q=0 - while read -r id; do - if [[ -s "${id}" ]]; then - "${dbengine}" "${db}" -N -B -q -e \ - "DELETE FROM \`post-media\` WHERE \`id\` = ${id}" & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then - wait -n - fi - tmp_post_media_duplicate_q=$((tmp_post_media_duplicate_q + 1)) - tmp_post_media_duplicate_current_id="${id}" +echo "tmp_post_media_duplicate" +tmp_post_media_duplicate_q="${limit}" +tmp_post_media_duplicate_current_id=0 +until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_media_duplicate_q=0 + while read -r id; do + if [[ -s "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-media\` WHERE \`id\` = ${id}" & + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then + wait -n fi - done < <("${dbengine}" "${db}" -N -B -q -e \ - "SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} \ + tmp_post_media_duplicate_q=$((tmp_post_media_duplicate_q + 1)) + tmp_post_media_duplicate_current_id="${id}" + fi + done < <("${dbengine}" "${db}" -N -B -q -e \ + "SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} \ AND u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}") - final_i=$(($(date +%s) - initial_i)) - echo "${tmp_post_media_duplicate_q} item(s) deleted until ${tmp_post_media_duplicate_current_id} in ${final_i}s" - done - wait + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_media_duplicate_q} item(s) deleted until ${tmp_post_media_duplicate_current_id} in ${final_i}s" +done +wait - echo "tmp_post_user_duplicate" - tmp_post_user_duplicate_q="${limit}" - tmp_post_user_duplicate_current_id=0 - until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do - initial_i=$(date +%s) - tmp_post_user_duplicate_q=0 - while read -r id; do - if [[ -s "${id}" ]]; then - "${dbengine}" "${db}" -N -B -q -e \ - "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then - wait -n - fi - tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1)) - tmp_post_user_duplicate_current_id="${id}" +echo "tmp_post_user_duplicate" +tmp_post_user_duplicate_q="${limit}" +tmp_post_user_duplicate_current_id=0 +until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_user_duplicate_q=0 + while read -r id; do + if [[ -s "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" & + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then + wait -n fi - done < <("${dbengine}" "${db}" -N -B -q -e \ - "SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} \ + tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1)) + tmp_post_user_duplicate_current_id="${id}" + fi + done < <("${dbengine}" "${db}" -N -B -q -e \ + "SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} \ AND v1.\`id\` < v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}") - final_i=$(($(date +%s) - initial_i)) - echo "${tmp_post_user_duplicate_q} item(s) deleted until ${tmp_post_user_duplicate_current_id} in ${final_i}s" - done - wait + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_user_duplicate_q} item(s) deleted until ${tmp_post_user_duplicate_current_id} in ${final_i}s" +done +wait - "${dboptimizer}" "${db}" #&> /dev/null -fi -bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null +"${dboptimizer}" "${db}" #&> /dev/null; +bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null; diff --git a/friendica-remove-invalid-photos.sh b/friendica-remove-invalid-photos.sh index 242000b..087d828 100755 --- a/friendica-remove-invalid-photos.sh +++ b/friendica-remove-invalid-photos.sh @@ -6,20 +6,14 @@ group=www-data fileperm=660 db=friendica folder=/var/www/friendica -intense_optimizations=${1:-"0"} -thread_multiplier=1 -nfolder="/tmpʾfriendica-remove-invalid-photos" -nfile="${nfolder}/n$(date +%s).csv" -nlock="${nfolder}/n$(date +%s).lock" -if [[ ! -d "${nfolder}" ]]; then - mkdir "${nfolder}" -fi -if [[ -f "${nfile}" ]]; then +nfile=/tmp/n.csv +nlock=/tmp/n.lock +if [[ -f ${nfile} ]]; then rm -rf "${nfile}" && touch "${nfile}" else touch "${nfile}" fi -if [[ -f "${nlock}" ]]; then +if [[ -f ${nlock} ]]; then rm -rf "${nlock}" fi #Internal parameters: @@ -33,15 +27,12 @@ lastid=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") #Limit per batch limit=$(((maxid / 1000) + 1)) -dbcount=0 -if [[ "${intense_optimizations}" -gt 0 ]]; then - #https:// = 8 characters | /avatar/ = 8 characters - indexlength=$(("${#url}" + 16)) - mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" - dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") -else - dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") -fi +#https:// = 8 characters | /avatar/ = 8 characters +#indexlength=$(("${#url}" + 16)) +#mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" +#Add to the loop, reset values +#dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") +dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") loop() { result_string="" @@ -157,9 +148,6 @@ loop() { fi else result_string=$(printf "%s No remote" "${result_string}") - #If the avatar is not valid, set it as blank in the database - mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - result_string=$(printf "%s (blanked)" "${result_string}") #If no remote avatar is found, we would blank the photo/thumb/micro and let the avatar cache process fix them later, but it's empty already here error_found=1 fi @@ -219,30 +207,15 @@ loop() { cd "${folder}" || exit echo "${n} ${nt}" >"${nfile}" until [[ $((nt + limit)) -gt "${dbcount}" ]]; do - c="" - if [[ "${intense_optimizations}" -gt 0 ]]; then - c=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") - else - c=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") - fi while read -r id; do lastid="${id}" loop & - until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) * thread_multiplier)) ]]; do + until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; do wait -n done - done < <(echo "${c}") + done < <(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") wait done -if [[ -f "${nfile}" ]]; then - rm -rf "${nfile}" -fi -if [[ -f "${nlock}" ]]; then - rm -rf "${nlock}" -fi -if [[ ! -d "${nfolder}" && $(find "${nfolder}" | wc -l) -eq 0 ]]; then - rm -rf "${nfolder}" -fi -if [[ "${intense_optimizations}" -gt 0 ]]; then - mariadb "${db}" -e "alter table contact drop index photo_index" -fi +rm -rf "${nfile}" "${nlock}" +#mariadb "${db}" -e "alter table contact drop index photo_index" +#printf "\nFixing folders and moving to avatar cache...\n"