diff --git a/friendica-remove-invalid-photos.sh b/friendica-remove-invalid-photos.sh index 9bda8c3..26d808e 100755 --- a/friendica-remove-invalid-photos.sh +++ b/friendica-remove-invalid-photos.sh @@ -8,79 +8,123 @@ folderperm=750 db=friendica folder=/var/www/friendica #Internal parameters: -#Amount of times the loop has run -iteration=0 +#Amount of times the internal loop has run +batch=0 #Number of invalid avatars found. Set to 1 initially so we can run the loop at least once n=1 -#Number of entries processed +#Number of entries processed this loop nx=0 +#Total number of entries processed +nt=0 #Last known ID to have been successfully processed lastid=0 -#Generate an index to make searches faster -((indexlength=37+${#url})) -echo "Generating photo index..." -mariadb $db -e "alter table contact add index if not exists photo_index (photo($indexlength))" +#Highest possible ID known +maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") +#Limit per batch +#limit=1000 +limit=$maxid +if [[ -f /tmp/lastid ]]; then + rm /tmp/lastid && touch /tmp/lastid +else + touch /tmp/lastid +fi #Go to the Friendica installation -cd $folder || exit -#Loop at least once, until no invalid avatars are found -until [[ $n -eq 0 ]] -do - #Add to the loop, reset values - iteration=$(("$iteration" + 1)) - n=0 - nx=0 - dblist=$(mariadb $db -B -N -q -e "select id, photo, thumb, micro from contact where id > $lastid and photo like 'https:\/\/$url/avatar/%' order by id") - m=$(echo "$dblist" | wc -l) - echo "$dblist" | while read -r id photo thumb micro - do - nx=$(("$nx" + 1)) - folderescaped=${folder////\\/} - #Substitute the URL path with the folder path so we can search for it in the local file system - #Photo is nominally 320px, actually 300px - k_photo=$(echo "$photo" | sed -e "s/https:\/\/$url/$folderescaped/g" -e "s/\?ts=.*//g") - #Thumb is 80px - k_thumb=$(echo "$thumb" | sed -e "s/https:\/\/$url/$folderescaped/g" -e "s/\?ts=.*//g") - #Micro is 48px - k_micro=$(echo "$micro" | sed -e "s/https:\/\/$url/$folderescaped/g" -e "s/\?ts=.*//g") - #If any of the images is not found in the filesystem - if [[ ! -e "$k_photo" || ! -e "$k_thumb" || ! -e "$k_micro" ]] - then - #If the avatar uses the standard fallback picture or is local, we cannot use it as a base - avatar=$(mariadb $db -B -N -q -e "select avatar from contact where id = \"$id\" and not avatar like \"%$url\" and not avatar like \"%images/person%\"") - #If we have a remote avatar as a fallback, download it - if [[ $! -eq 0 && -n $avatar ]] - then - echo "$id $avatar" - sudo -u $user curl "$avatar" -s -o "$k_photo" - #If the file is a valid picture (not empty, not text) - if file "$k_photo" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory" - then - #Also fetch for thumb/micro and resize - #As the photo is the largest version we have, we will use it as the base, and leave it last to convert - convert "$k_photo" -resize 80x80 -depth 16 "$k_thumb" && chmod "$fileperm" "$k_thumb" && chown "$user:$group" "$k_thumb" - convert "$k_photo" -resize 48x48 -depth 16 "$k_micro" && chmod "$fileperm" "$k_micro" && chown "$user:$group" "$k_micro" - convert "$k_photo" -resize 300x300 -depth 16 "$k_photo" && chmod "$fileperm" "$k_photo" && chown "$user:$group" "$k_photo" - else - #If the avatar is not valid, set it as blank in the database - mariadb $db -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"$id\"" - rm -rf "$k_photo" - fi - else - #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later - mariadb $db -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"$id\"" - fi - n=$(( n + 1 )) - fi - lastid="${id}" - printf "\rIteration %s\tPhotos: %s\tEntry %s/%s " "$iteration" "$n" "$nx" "$m" - done - wait - printf "\nFixing folders and moving to avatar cache...\n" - sudo -u $user bin/console movetoavatarcache #&> /dev/null - find ./avatar -depth -not -user "$user" -or -not -group "$group" -exec chown -v "$user:$group" {} \; - find ./avatar -depth -type f -and -not -type d -and -not -perm "$fileperm" -exec chmod -v "$fileperm" {} \; - find ./avatar -depth -type d -and -not -perm "$folderperm" -exec chmod -v "$folderperm" {} \; - #chown -R "$user:$group" ./avatar +cd "${folder}" || exit +#Add to the loop, reset values +n=0 +nt=0 +maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") +dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") +echo "${dbcount}" +until [[ $((nt + limit)) -ge ${dbcount} ]]; do + nx=0 + maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") + batch=$(("${batch}" + 1)) + #Read lastid outside of the loop with a temporary file + if [[ -f /tmp/lastid && -s /tmp/lastid ]]; then + while read -r n_i nt_i lastid_i; do + if [[ -s "${n_i}" && -s "${nt_i}" && -n "${lastid_i}" ]]; then + n="${n_i}" + nt="${nt_i}" + lastid="${lastid_i}" + fi + done < <(cat /tmp/lastid) + fi + dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") + echo "${dboutput}" | while read -r id photo thumb micro; do + if [[ -n "${id}" && -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then + nx=$(("${nx}" + 1)) + nt=$(("${nt}" + 1)) + error_found=0 + folderescaped=${folder////\\/} + #Substitute the URL path with the folder path so we can search for it in the local file system + #Photo is nominally 320px, actually 300px + k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #Thumb is 80px + k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #Micro is 48px + k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #If fetching any of the images causes an error + if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Request the user data to be regenerated in the system through the database + mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority) values (\"UpdateContact\", \"[${id}]\", 20);" + echo "${id} ${photo}" + error_found=1 + fi + #If any of the images is not found in the filesystem + if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then + #If the avatar uses the standard fallback picture or is local, we cannot use it as a base + avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") + #If we have a remote avatar as a fallback, download it + if [[ $! -eq 0 && -n ${avatar} ]]; then + echo "${id} ${avatar}" + sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" + #If the file is a valid picture (not empty, not text) + if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Also fetch for thumb/micro and resize + #As the photo is the largest version we have, we will use it as the base, and leave it last to convert + convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}" + convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}" + convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}" + else + #If the avatar is not valid, set it as blank in the database + mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" + rm -rf "${k_photo}" + fi + #Request the user data to be regenerated in the system through the database + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" + else + echo "${id}" + #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later + mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" + #Request the user data to be regenerated in the system through the database + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" + fi + error_found=1 + k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") + else + k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g") + k_photo_found_time=$(stat -c%W "${k_photo}") + k_photo_delta=$((k_photo_found_time - k_photo_original_time)) + fi + if [[ "${error_found}" -gt 0 ]]; then + n=$((n + 1)) + fi + fi + lastid="${id}" + touch /tmp/lastid + echo "${n} ${nt} ${lastid}" >/tmp/lastid + printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}" + done done -#Drop index in the end to save storage -mariadb $db -e "alter table contact drop index photo_index" +printf "\nFixing folders and moving to avatar cache...\n" +#sudo -u "${user}" bin/console movetoavatarcache #&> /dev/null +"${folder}"/bin/console movetoavatarcache #&> /dev/null +find ./avatar -depth -not -user "${user}" -or -not -group "${group}" -exec chown -v "${user}:${group}" {} \; +find ./avatar -depth -type f -and -not -type d -and -not -perm "${fileperm}" -exec chmod -v "${fileperm}" {} \; +find ./avatar -depth -type d -and -not -perm "${folderperm}" -exec chmod -v "${folderperm}" {} \; +chown -R "${user}:${group}" ./avatar +rm /tmp/lastid