diff --git a/friendica-clean-database.sh b/friendica-clean-database.sh index ce256b7..474c5f6 100755 --- a/friendica-clean-database.sh +++ b/friendica-clean-database.sh @@ -6,7 +6,7 @@ user=friendica phpversion=php8.2 dbengine=mariadb db=friendica -dboptimizer=mariadb-optimize +dboptimizer=mariadboptimize bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null; @@ -318,7 +318,6 @@ tmp_post_media_duplicate_q="${limit}" tmp_post_media_duplicate_current_id=0 until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) - tmp_post_media_duplicate_q=0 while read -r id; do if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ @@ -342,7 +341,6 @@ tmp_post_user_duplicate_q="${limit}" tmp_post_user_duplicate_current_id=0 until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do initial_i=$(date +%s) - tmp_post_user_duplicate_q=0 while read -r id; do if [[ -s "${id}" ]]; then "${dbengine}" "${db}" -N -B -q -e \ diff --git a/friendica-remove-invalid-photos.sh b/friendica-remove-invalid-photos.sh index 694dea9..76ca3d0 100755 --- a/friendica-remove-invalid-photos.sh +++ b/friendica-remove-invalid-photos.sh @@ -30,10 +30,12 @@ fi #Go to the Friendica installation cd "${folder}" || exit #Add to the loop, reset values +n=0 +nt=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") #dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") -dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' or photo like ''") -echo "${n} ${nt} ${lastid}" >/tmp/lastid +dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") +echo "${dbcount}" until [[ $((nt + limit)) -gt ${dbcount} ]]; do nx=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") @@ -53,155 +55,83 @@ until [[ $((nt + limit)) -gt ${dbcount} ]]; do done ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") - #dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") - dbid=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") - while read -r id; do - result_string="" - nx=$(("${nx}" + 1)) - nt=$(("${nt}" + 1)) - error_found=0 - dboutput=$(mariadb "${db}" -B -N -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") - #echo "$id" - if [[ -n "${id}" ]]; then - while read -r photo thumb micro; do - #echo "$photo $thumb $micro" - if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then - folderescaped=${folder////\\/} - #Substitute the URL path with the folder path so we can search for it in the local file system - #Photo is nominally 320px, actually 300px - k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") - #Thumb is 80px - k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") - #Micro is 48px - k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") - #If fetching any of the images causes an error - if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || - curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || - curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then - #Request the user data to be regenerated in the system through the database - mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & - result_string=$(printf "%s (added)" "${result_string}") - else - result_string=$(printf "%s (already added)" "${result_string}") - fi - result_string=$(printf "${result_string} Fetch error: %s %s\n" "${id}" "${photo}") - error_found=1 - fi - #If any of the images is not found in the filesystem - if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then - #If the avatar uses the standard fallback picture or is local, we cannot use it as a base - avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") - #If we have a remote avatar as a fallback, download it - if [[ -n "${avatar}" ]]; then - result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}") - sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" - #If the file is a valid picture (not empty, not text) - if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then - #Also fetch for thumb/micro and resize - #As the photo is the largest version we have, we will use it as the base, and leave it last to convert - (convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}") & - (convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") & - (convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") & - result_string=$(printf "%s (generated)" "${result_string}") - else - #If the avatar is not valid, set it as blank in the database - mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - rm -rf "${k_photo}" & - result_string=$(printf "%s (blanked)" "${result_string}") - fi - #Request the user data to be regenerated in the system through the database - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & - result_string=$(printf "%s (added)" "${result_string}") - else - result_string=$(printf "%s (already added)" "${result_string}") - fi - else - result_string=$(printf "${result_string} No remote avatar: %s" "${id}") - #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later - mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - #Request the user data to be regenerated in the system through the database - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & - result_string=$(printf "%s (added)" "${result_string}") - else - result_string=$(printf "%s (already added)" "${result_string}") - fi - fi - error_found=1 - k_photo_delta="${photo//.*?ts=//}" - #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") + dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" order by id limit ${limit}") + while read -r id photo thumb micro; do + if [[ -n "${id}" && -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then + nx=$(("${nx}" + 1)) + nt=$(("${nt}" + 1)) + error_found=0 + folderescaped=${folder////\\/} + #Substitute the URL path with the folder path so we can search for it in the local file system + #Photo is nominally 320px, actually 300px + k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #Thumb is 80px + k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #Micro is 48px + k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #If fetching any of the images causes an error + if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Request the user data to be regenerated in the system through the database + mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" + if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" + fi + echo "${id} ${photo}" + error_found=1 + fi + #If any of the images is not found in the filesystem + if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then + #If the avatar uses the standard fallback picture or is local, we cannot use it as a base + avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") + #If we have a remote avatar as a fallback, download it + if [[ $! -eq 0 && -n ${avatar} ]]; then + echo "${id} ${avatar}" + sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" + #If the file is a valid picture (not empty, not text) + if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Also fetch for thumb/micro and resize + #As the photo is the largest version we have, we will use it as the base, and leave it last to convert + convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}" + convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}" + convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}" else - #k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g") - k_photo_original_time="${photo//.*?ts=//}" - k_photo_found_time=$(stat -c%W "${k_photo}") - k_photo_delta=$((k_photo_found_time - k_photo_original_time)) + #If the avatar is not valid, set it as blank in the database + mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" + rm -rf "${k_photo}" + fi + #Request the user data to be regenerated in the system through the database + if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" fi else - result_string=$(printf "${result_string} No local photo: %s" "${id}") - #If the avatar uses the standard fallback picture or is local, we cannot use it as a base - avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") - #If we have a remote avatar as a fallback, download it - if [[ $! -eq 0 && -n ${avatar} ]]; then - result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}") - sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" - #If the file is a valid picture (not empty, not text) - if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then - #Also fetch for thumb/micro and resize - #As the photo is the largest version we have, we will use it as the base, and leave it last to convert - (convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}") & - (convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") & - (convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") & - result_string=$(printf "%s (generated)" "${result_string}") - else - #If the avatar is not valid, set it as blank in the database - mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - rm -rf "${k_photo}" & - result_string=$(printf "%s (blanked)" "${result_string}") - fi - #Request the user data to be regenerated in the system through the database - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & - result_string=$(printf "%s (added)" "${result_string}") - else - result_string=$(printf "%s (already added)" "${result_string}") - fi - else - result_string=$(printf "${result_string} No remote avatar: %s" "${id}") - #Request the user data to be regenerated in the system through the database - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & - result_string=$(printf "%s (added)" "${result_string}") - else - result_string=$(printf "%s (already added)" "${result_string}") - fi + echo "${id}" + #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later + mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" + #Request the user data to be regenerated in the system through the database + if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" fi - error_found=1 - k_photo_delta=0 - #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") fi - if [[ "${error_found}" -gt 0 ]]; then - n=$((n + 1)) - fi - lastid="${id}" - touch /tmp/lastid - echo "${n} ${nt} ${lastid}" >/tmp/lastid - done < <(echo "${dboutput}") + error_found=1 + k_photo_delta="${photo//.*?ts=//}" + #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") + else + #k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g") + k_photo_original_time="${photo//.*?ts=//}" + k_photo_found_time=$(stat -c%W "${k_photo}") + k_photo_delta=$((k_photo_found_time - k_photo_original_time)) + fi + if [[ "${error_found}" -gt 0 ]]; then + n=$((n + 1)) + fi + lastid="${id}" + touch /tmp/lastid + echo "${n} ${nt} ${lastid}" >/tmp/lastid fi - printf "\rFound %8d/%8d Total %8d/%8d Delta %6d %s " "${n}" "${nt}" "${lastid}" "${maxid}" "${k_photo_delta}" "${result_string}" - #Line clearance - printf "\r" - #for space in $(seq 1 "${COLUMNS}") - #do - #printf " " - #done - for space in $(seq 1 "${COLUMNS}") - do - printf "\b" - done - done < <(echo "${dbid}") + printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %6d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}" + done < <(echo "${dboutput}") done printf "\nFixing folders and moving to avatar cache...\n" #sudo -u "${user}" bin/console movetoavatarcache #&> /dev/null diff --git a/friendica-remove-old-photos-parallel.sh b/friendica-remove-old-photos-parallel.sh index c49878a..7e0272a 100755 --- a/friendica-remove-old-photos-parallel.sh +++ b/friendica-remove-old-photos-parallel.sh @@ -10,13 +10,10 @@ loop() { #Parse each file in folder folderescaped=${folder////\\/} ky=$(echo "${y}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g") - f=$(mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1") + f=$(sudo mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1") if [[ $? -eq 0 && -z ${f} && -f ${y} ]]; then - ( ls -lh "${y}" && rm -rf "${y}" ) & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then - wait -n - fi - + ls -lh "${y}" + sudo rm -rf "${y}" d=$((d + 1)) fi #printf "\rPhotos: %s\tFolder %s\tEntry %s " "$d" "$n" "$m" @@ -27,11 +24,12 @@ loop() { date #Go to the Friendica installation cd "${folderavatar}" || exit -indexlength=$((49 + ${#url})) -mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" +#let "indexlength=37+${#url}" +((indexlength = 49 + ${#url})) +sudo mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" n=0 d=0 -find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do +sudo find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do n=$((n + 1)) #If the directory still exists if [[ -d "${x}" ]]; then @@ -45,10 +43,10 @@ find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then wait -n fi - #done < <(find "${x}" -type f -mtime -8) - done < <(find "${x}" -type f) + #done < <(sudo find "${x}" -type f -mtime -8) + done < <(sudo find "${x}" -type f) fi fi done -mariadb "${db}" -e "alter table contact drop index photo_index" +sudo mariadb "${db}" -e "alter table contact drop index photo_index" date