diff --git a/friendica-fix-avatar-permissions.sh b/friendica-fix-avatar-permissions.sh index 0a70c0f..b7584e6 100755 --- a/friendica-fix-avatar-permissions.sh +++ b/friendica-fix-avatar-permissions.sh @@ -5,7 +5,7 @@ IFS=" site=friendica.example.net user=friendica group=www-data -fileperm=660 +fileperm=640 folderperm=770 folder=/var/www/friendica folderescaped=${folder////\\/} @@ -19,7 +19,7 @@ loop_1() { nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null #Specific compression for large GIF files while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do - frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1)) + frameamount=$(exiftool -b -FrameCount "${p}" || 1) nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null done elif [[ "${p}" =~ .png ]]; then @@ -43,8 +43,7 @@ loop_1() { cd "${folder}" || exit if [[ ! -f "${tmpfile}" ]]; then - #sudo -u "${user}" bin/console movetoavatarcache | sudo tee "${tmpfile}" #&> /dev/null - sudo bin/console movetoavatarcache | sudo tee "${tmpfile}" #&> /dev/null + sudo -u "${user}" bin/console movetoavatarcache | sudo tee "${tmpfile}" #&> /dev/null fi grep -e "https://${site}/${avatarfolder}/" "${tmpfile}" | sed -e "s/.*${site}/${folderescaped}/g" -e "s/?ts=.*//g" | ( while read -r i; do diff --git a/friendica-remove-invalid-photos.sh b/friendica-remove-invalid-photos.sh index 252f948..5f12ac1 100755 --- a/friendica-remove-invalid-photos.sh +++ b/friendica-remove-invalid-photos.sh @@ -4,13 +4,14 @@ url=friendica.example.net user=friendica group=www-data fileperm=660 +folderperm=770 db=friendica folder=/var/www/friendica #Internal parameters: #Amount of times the internal loop has run batch=0 -#Number of invalid avatars found. -n=0 +#Number of invalid avatars found. Set to 1 initially so we can run the loop at least once +n=1 #Number of entries processed this loop nx=0 #Total number of entries processed @@ -21,46 +22,80 @@ lastid=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") #Limit per batch limit=$(((maxid / 1000) + 1)) -#https:// = 8 characters | /avatar/ = 8 characters -#indexlength=$(("${#url}" + 16)) -#mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" +if [[ -f /tmp/lastid ]]; then + rm /tmp/lastid && touch /tmp/lastid +else + touch /tmp/lastid +fi #Go to the Friendica installation cd "${folder}" || exit #Add to the loop, reset values +maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") #dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") -dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") -until [[ $((nt + limit)) -gt "${dbcount}" ]]; do +dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' or photo like ''") +echo "${n} ${nt} ${lastid}" >/tmp/lastid +until [[ $((nt + limit)) -gt ${dbcount} ]]; do nx=0 + maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") batch=$(("${batch}" + 1)) + #Read lastid outside of the loop with a temporary file + if [[ -f /tmp/lastid && -s /tmp/lastid ]]; then + while read -r n_i nt_i lastid_i; do + if [[ -s "${n_i}" ]]; then + n="${n_i}" + fi + if [[ -s "${nt_i}" ]]; then + nt="${nt_i}" + fi + if [[ -s "${lastid_i}" ]]; then + lastid="${lastid_i}" + fi + done ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") + #dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") + dbid=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") while read -r id; do result_string="" - nl=0 nx=$(("${nx}" + 1)) nt=$(("${nt}" + 1)) error_found=0 - t_id=$(($(date +%s%N) / 1000000)) + dboutput=$(mariadb "${db}" -B -N -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") + #echo "$id" if [[ -n "${id}" ]]; then while read -r photo thumb micro; do + #echo "$photo $thumb $micro" if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then - #If there is a photo folderescaped=${folder////\\/} #Substitute the URL path with the folder path so we can search for it in the local file system #Photo is nominally 320px, actually 300px - k_photo=$(sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g" <<<"${photo}") + k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") #Thumb is 80px - k_thumb=$(sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g" <<<"${thumb}") + k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") #Micro is 48px - k_micro=$(sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g" <<<"${micro}") + k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g") + #If fetching any of the images causes an error + if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || + curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then + #Request the user data to be regenerated in the system through the database + mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -eq 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi + result_string=$(printf "${result_string} Fetch error: %s %s\n" "${id}" "${photo}") + error_found=1 + fi #If any of the images is not found in the filesystem if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then #If the avatar uses the standard fallback picture or is local, we cannot use it as a base - t=$(($(date +%s%N) / 1000000)) avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") - result_string=$(printf "%s %d ms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t))) #If we have a remote avatar as a fallback, download it - if [[ "$!" -eq 0 && -n "${avatar}" ]]; then - result_string=$(printf "%s Remote %s" "${result_string}" "${avatar}") - nl=1 + if [[ -n "${avatar}" ]]; then + result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}") sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" #If the file is a valid picture (not empty, not text) if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then @@ -70,46 +105,47 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do (convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") & (convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") & result_string=$(printf "%s (generated)" "${result_string}") - error_found=1 else #If the avatar is not valid, set it as blank in the database mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - rm -rf "${k_photo}" "${k_thumb}" "${k_micro}" & + rm -rf "${k_photo}" & result_string=$(printf "%s (blanked)" "${result_string}") - error_found=1 + fi + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -eq 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") fi else - result_string=$(printf "%s No remote avatar" "${result_string}") + result_string=$(printf "${result_string} No remote avatar: %s" "${id}") #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - error_found=1 + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -eq 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi fi + error_found=1 + k_photo_delta="${photo//.*?ts=//}" + #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") else - #If the images are all found in the filesystem, but fetching any of the images causes an error - if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || - curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || - curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then - result_string=$(printf "${result_string} Fetch error: %s" "${photo}") - mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - result_string=$(printf "%s (blanked)" "${result_string}") - nl=1 - error_found=1 - else - result_string=$(printf "%s (FOUND)" "${result_string}") - error_found=0 - fi + #k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g") + k_photo_original_time="${photo//.*?ts=//}" + k_photo_found_time=$(stat -c%W "${k_photo}") + k_photo_delta=$((k_photo_found_time - k_photo_original_time)) fi else - #If there is no photo - result_string=$(printf "%s No local" "${result_string}") + result_string=$(printf "${result_string} No local photo: %s" "${id}") #If the avatar uses the standard fallback picture or is local, we cannot use it as a base - t=$(($(date +%s%N) / 1000000)) avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"") - result_string=$(printf "%s %d ms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t))) #If we have a remote avatar as a fallback, download it - if [[ "$!" -eq 0 && -n "${avatar}" ]]; then - result_string=$(printf "${result_string} Remote %s" "${avatar}") - nl=1 + if [[ $! -eq 0 && -n ${avatar} ]]; then + result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}") sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" #If the file is a valid picture (not empty, not text) if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then @@ -119,58 +155,54 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do (convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") & (convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") & result_string=$(printf "%s (generated)" "${result_string}") - error_found=1 else #If the avatar is not valid, set it as blank in the database mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & - rm -rf "${k_photo}" "${k_thumb}" "{k_micro}" & + rm -rf "${k_photo}" & result_string=$(printf "%s (blanked)" "${result_string}") - error_found=1 + fi + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -eq 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") fi else - result_string=$(printf "%s No remote" "${result_string}") - #If no remote avatar is found, we would blank the photo/thumb/micro and let the avatar cache process fix them later, but it's empty already here - error_found=1 + result_string=$(printf "${result_string} No remote avatar: %s" "${id}") + #Request the user data to be regenerated in the system through the database + if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -eq 0 ]]; then + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" & + result_string=$(printf "%s (added)" "${result_string}") + else + result_string=$(printf "%s (already added)" "${result_string}") + fi fi + error_found=1 + k_photo_delta=0 + #k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") fi if [[ "${error_found}" -gt 0 ]]; then - #Request the user data to be regenerated in the system through the database - #if [[ $(mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) \ - #select \"UpdateContact\" as command, \"[${id}]\" as parameter, 20 as priority, concat(curdate(), \" \", curtime()) as created \ - #from workerqueue where (select count(*) from workerqueue \ - #where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0) = 0; \ - #select row_count();") -gt 0 ]]; then - #result_string=$(printf "%s (added)" "${result_string}") - #nl=1 - #else - #result_string=$(printf "%s (already)" "${result_string}") - #fi - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) \ - values (\"UpdateContact\", \"[${id}]\", 20, concat(curdate(), \" \", curtime()));" & - result_string=$(printf "%s (added)" "${result_string}") - nl=1 n=$((n + 1)) fi lastid="${id}" - done < <(mariadb "${db}" -B -N -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") + touch /tmp/lastid + echo "${n} ${nt} ${lastid}" >/tmp/lastid + done < <(echo "${dboutput}") fi - result_string=$(printf "%s %d ms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_id))) - final_string=$(printf "E%8d F%8d/%8d T%8d/%8d %s" "${n}" "${nt}" "${dbcount}" "${lastid}" "${maxid}" "${result_string}") - final_string_length="${#final_string}" - #Previous line clearance - #Measure length of string, blank only the excess - blank_string="" - blank_string_length=$((COLUMNS - final_string_length)) - for ((count = 0; count < "${blank_string_length}"; count++)); do - blank_string=$(printf "%s " "${blank_string}") + printf "\rFound %8d/%8d Total %8d/%8d Delta %6d %s " "${n}" "${nt}" "${lastid}" "${maxid}" "${k_photo_delta}" "${result_string}" + #Line clearance + printf "\r" + for ((count = 0; count < "${COLUMNS}"; count++)); do + printf "\b" done - final_string=$(printf "%s%s" "${final_string}" "${blank_string}") - #Add a new line only when necessary - if [[ "${nl}" -eq 1 ]]; then - final_string=$(printf "%s\n\r\n" "${final_string}") - fi - printf "%s\r" "${final_string}" - done < <(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") + done < <(echo "${dbid}") done -#mariadb "${db}" -e "alter table contact drop index photo_index" -#printf "\nFixing folders and moving to avatar cache...\n" +printf "\nFixing folders and moving to avatar cache...\n" +#sudo -u "${user}" bin/console movetoavatarcache #&> /dev/null +"${folder}"/bin/console movetoavatarcache #&> /dev/null +find ./avatar -depth -not -user "${user}" -or -not -group "${group}" -exec chown -v "${user}:${group}" {} \; +find ./avatar -depth -type f -and -not -type d -and -not -perm "${fileperm}" -exec chmod -v "${fileperm}" {} \; +find ./avatar -depth -type d -and -not -perm "${folderperm}" -exec chmod -v "${folderperm}" {} \; +chown -R "${user}:${group}" ./avatar +rm /tmp/lastid diff --git a/friendica-remove-old-photos-parallel.sh b/friendica-remove-old-photos-parallel.sh index dfb7d39..9227d78 100755 --- a/friendica-remove-old-photos-parallel.sh +++ b/friendica-remove-old-photos-parallel.sh @@ -12,7 +12,7 @@ loop() { ky=$(echo "${y}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g") f=$(mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1") if [[ $? -eq 0 && -z ${f} && -f ${y} ]]; then - rm -rvf "${y}" & + (ls -lh "${y}" && rm -rf "${y}") & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then wait -n fi @@ -27,23 +27,28 @@ loop() { date #Go to the Friendica installation cd "${folderavatar}" || exit -#indexlength=$((49 + ${#url})) -#mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" +indexlength=$((49 + ${#url})) +mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" n=0 d=0 -while read -r x; do +find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do n=$((n + 1)) #If the directory still exists if [[ -d "${x}" ]]; then - m=0 - while read -r y; do - m=$((m + 1)) - loop "${x}" "${m}" "${n}" "${d}" "${y}" & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then - wait -n - fi - done < <(find "${x}" -type f) + folderescaped=${folder////\\/} + #kx=$(echo "${x}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g") + if [[ -d ${x} ]]; then + m=0 + while read -r y; do + m=$((m + 1)) + loop "${x}" "${m}" "${n}" "${d}" & + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then + wait -n + fi + #done < <(find "${x}" -type f -mtime -8) + done < <(find "${x}" -type f) + fi fi -done < <(find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d) -#mariadb "${db}" -e "alter table contact drop index photo_index" +done +mariadb "${db}" -e "alter table contact drop index photo_index" date