From 43cb4e881099582ba77281751bebfca784dfbd82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Sol=C3=ADs?= Date: Fri, 7 Feb 2025 18:58:48 +0000 Subject: [PATCH 1/3] fix: Correct name of database optimizer, prevent infinite loops --- friendica-clean-database.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/friendica-clean-database.sh b/friendica-clean-database.sh index 474c5f6..ce256b7 100755 --- a/friendica-clean-database.sh +++ b/friendica-clean-database.sh @@ -6,7 +6,7 @@ user=friendica phpversion=php8.2 dbengine=mariadb db=friendica -dboptimizer=mariadboptimize +dboptimizer=mariadb-optimize bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null; @@ -318,6 +318,7 @@ tmp_post_media_duplicate_q="${limit}" tmp_post_media_duplicate_current_id=0 until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) + tmp_post_media_duplicate_q=0 while read -r id; do if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ @@ -341,6 +342,7 @@ tmp_post_user_duplicate_q="${limit}" tmp_post_user_duplicate_current_id=0 until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) + tmp_post_user_duplicate_q=0 while read -r id; do if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ From a42824cc06693887dc1ca2148e6629ab18c410cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Sol=C3=ADs?= Date: Fri, 7 Feb 2025 18:59:25 +0000 Subject: [PATCH 2/3] fix: Correct some errors --- friendica-remove-old-photos-parallel.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/friendica-remove-old-photos-parallel.sh b/friendica-remove-old-photos-parallel.sh index 7e0272a..c49878a 100755 --- a/friendica-remove-old-photos-parallel.sh +++ b/friendica-remove-old-photos-parallel.sh @@ -10,10 +10,13 @@ loop() { #Parse each file in folder folderescaped=${folder////\\/} ky=$(echo "${y}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g") - f=$(sudo mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1") + f=$(mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1") if [[ $? -eq 0 && -z ${f} && -f ${y} ]]; then - ls -lh "${y}" - sudo rm -rf "${y}" + ( ls -lh "${y}" && rm -rf "${y}" ) & + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then + wait -n + fi + d=$((d + 1)) fi #printf "\rPhotos: %s\tFolder %s\tEntry %s " "$d" "$n" "$m" @@ -24,12 +27,11 @@ loop() { date #Go to the Friendica installation cd "${folderavatar}" || exit -#let "indexlength=37+${#url}" -((indexlength = 49 + ${#url})) -sudo mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" +indexlength=$((49 + ${#url})) +mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" n=0 d=0 -sudo find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do +find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do n=$((n + 1)) #If the directory still exists if [[ -d "${x}" ]]; then @@ -43,10 +45,10 @@ sudo find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then wait -n fi - #done < <(sudo find "${x}" -type f -mtime -8) - done < <(sudo find "${x}" -type f) + #done < <(find "${x}" -type f -mtime -8) + done < <(find "${x}" -type f) fi fi done -sudo mariadb "${db}" -e "alter table contact drop index photo_index" +mariadb "${db}" -e "alter table contact drop index photo_index" date From 4095a8351117e6d510f327739a7426d40baa8c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Sol=C3=ADs?= Date: Fri, 7 Feb 2025 18:59:53 +0000 Subject: [PATCH 3/3] fix: Rewrite and reorder code to print output better --- friendica-remove-invalid-photos.sh | 218 +++++++++++++++++++---------- 1 file changed, 144 insertions(+), 74 deletions(-) diff --git a/friendica-remove-invalid-photos.sh b/friendica-remove-invalid-photos.sh index 76ca3d0..694dea9 100755 --- a/friendica-remove-invalid-photos.sh +++ b/friendica-remove-invalid-photos.sh @@ -30,12 +30,10 @@ fi #Go to the Friendica installation cd "${folder}" || exit #Add to the loop, reset values -n=0 -nt=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") #dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") -dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") -echo "${dbcount}" +dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' or photo like ''") +echo "${n} ${nt} ${lastid}" >/tmp/lastid until [[ $((nt + limit)) -gt ${dbcount} ]]; do nx=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") @@ -55,83 +53,155 @@ until [[ $((nt + limit)) -gt ${dbcount} ]]; do done ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") - dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" order by id limit ${limit}") - while read -r id photo thumb micro; do - if [[ -n "${id}" && -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then - nx=$(("${nx}" + 1)) - nt=$(("${nt}" + 1)) - error_found=0 - folderescaped=${folder////\\/} - #Substitute the URL path with the folder path so we can search for it in the local file system - #Photo is nominally 320px, actually 300px - k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") - #Thumb is 80px - k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") - #Micro is 48px - k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") - #If fetching any of the images causes an error - if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || - curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || - curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then - #Request the user data to be regenerated in the system through the database - mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" - if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" - fi - echo "${id} ${photo}" - error_found=1 - fi - #If any of the images is not found in the filesystem - if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then - #If the avatar uses the standard fallback picture or is local, we cannot use it as a base - avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") - #If we have a remote avatar as a fallback, download it - if [[ $! -eq 0 && -n ${avatar} ]]; then - echo "${id} ${avatar}" - sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" - #If the file is a valid picture (not empty, not text) - if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then - #Also fetch for thumb/micro and resize - #As the photo is the largest version we have, we will use it as the base, and leave it last to convert - convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}" - convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}" - convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}" - else - #If the avatar is not valid, set it as blank in the database + #dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") + dbid=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") + while read -r id; do + result_string="" + nx=$(("${nx}" + 1)) + nt=$(("${nt}" + 1)) + error_found=0 + dboutput=$(mariadb "${db}" -B -N -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") + #echo "$id" + if [[ -n "${id}" ]]; then + while read -r photo thumb micro; do + #echo "$photo $thumb $micro" + if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then + folderescaped=${folder////\\/} + #Substitute the URL path with the folder path so we can search for it in the local file system + #Photo is nominally 320px, actually 300px + k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #Thumb is 80px + k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #Micro is 48px + k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #If fetching any of the images causes an error + if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Request the user data to be regenerated in the system through the database mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" - rm -rf "${k_photo}" + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi + result_string=$(printf "${result_string} Fetch error: %s %s\n" "${id}" "${photo}") + error_found=1 fi - #Request the user data to be regenerated in the system through the database - if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" + #If any of the images is not found in the filesystem + if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then + #If the avatar uses the standard fallback picture or is local, we cannot use it as a base + avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") + #If we have a remote avatar as a fallback, download it + if [[ -n "${avatar}" ]]; then + result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}") + sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" + #If the file is a valid picture (not empty, not text) + if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Also fetch for thumb/micro and resize + #As the photo is the largest version we have, we will use it as the base, and leave it last to convert + (convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}") & + (convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") & + (convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") & + result_string=$(printf "%s (generated)" "${result_string}") + else + #If the avatar is not valid, set it as blank in the database + mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & + rm -rf "${k_photo}" & + result_string=$(printf "%s (blanked)" "${result_string}") + fi + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi + else + result_string=$(printf "${result_string} No remote avatar: %s" "${id}") + #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later + mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi + fi + error_found=1 + k_photo_delta="${photo//.*?ts=//}" + #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") + else + #k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g") + k_photo_original_time="${photo//.*?ts=//}" + k_photo_found_time=$(stat -c%W "${k_photo}") + k_photo_delta=$((k_photo_found_time - k_photo_original_time)) fi else - echo "${id}" - #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later - mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" - #Request the user data to be regenerated in the system through the database - if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" + result_string=$(printf "${result_string} No local photo: %s" "${id}") + #If the avatar uses the standard fallback picture or is local, we cannot use it as a base + avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") + #If we have a remote avatar as a fallback, download it + if [[ $! -eq 0 && -n ${avatar} ]]; then + result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}") + sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" + #If the file is a valid picture (not empty, not text) + if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Also fetch for thumb/micro and resize + #As the photo is the largest version we have, we will use it as the base, and leave it last to convert + (convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}") & + (convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") & + (convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") & + result_string=$(printf "%s (generated)" "${result_string}") + else + #If the avatar is not valid, set it as blank in the database + mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & + rm -rf "${k_photo}" & + result_string=$(printf "%s (blanked)" "${result_string}") + fi + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi + else + result_string=$(printf "${result_string} No remote avatar: %s" "${id}") + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi fi + error_found=1 + k_photo_delta=0 + #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") fi - error_found=1 - k_photo_delta="${photo//.*?ts=//}" - #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") - else - #k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g") - k_photo_original_time="${photo//.*?ts=//}" - k_photo_found_time=$(stat -c%W "${k_photo}") - k_photo_delta=$((k_photo_found_time - k_photo_original_time)) - fi - if [[ "${error_found}" -gt 0 ]]; then - n=$((n + 1)) - fi - lastid="${id}" - touch /tmp/lastid - echo "${n} ${nt} ${lastid}" >/tmp/lastid + if [[ "${error_found}" -gt 0 ]]; then + n=$((n + 1)) + fi + lastid="${id}" + touch /tmp/lastid + echo "${n} ${nt} ${lastid}" >/tmp/lastid + done < <(echo "${dboutput}") fi - printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %6d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}" - done < <(echo "${dboutput}") + printf "\rFound %8d/%8d Total %8d/%8d Delta %6d %s " "${n}" "${nt}" "${lastid}" "${maxid}" "${k_photo_delta}" "${result_string}" + #Line clearance + printf "\r" + #for space in $(seq 1 "${COLUMNS}") + #do + #printf " " + #done + for space in $(seq 1 "${COLUMNS}") + do + printf "\b" + done + done < <(echo "${dbid}") done printf "\nFixing folders and moving to avatar cache...\n" #sudo -u "${user}" bin/console movetoavatarcache #&> /dev/null