csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Add thread parallelism, optimize certain fetches even further

Browse source
This commit is contained in:
Carlos Solís 2025-02-09 23:29:49 +00:00
parent 1ee6c0dfad
commit 89d876a1fa
1 changed files with 143 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
friendica-remove-invalid-photos.sh
View file

@ -6,6 +6,16 @@ group=www-data
fileperm=660 fileperm=660
db=friendica db=friendica
folder=/var/www/friendica folder=/var/www/friendica
nfile=/tmp/n.csv
nlock=/tmp/n.lock
if [[ -f ${nfile} ]]; then
rm -rf "${nfile}" && touch "${nfile}"
else
touch "${nfile}"
fi
if [[ -f ${nlock} ]]; then
rm -rf "${nlock}"
fi
#Internal parameters: #Internal parameters:
#Amount of times the internal loop has run #Amount of times the internal loop has run
batch=0 batch=0
@ -24,23 +34,25 @@ limit=$(((maxid / 1000) + 1))
#https:// = 8 characters | /avatar/ = 8 characters #https:// = 8 characters | /avatar/ = 8 characters
#indexlength=$(("${#url}" + 16)) #indexlength=$(("${#url}" + 16))
#mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" #mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
#Go to the Friendica installation
cd "${folder}" || exit
#Add to the loop, reset values #Add to the loop, reset values
#dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") #dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'")
until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
nx=0 loop() {
batch=$(("${batch}" + 1)) #Wait until lock no longer exists
while read -r id; do r=0
result_string="" while [[ "${r}" -eq 0 ]]; do
nl=0 #Read data from file, delete lock
if [[ ! -f "${nlock}" ]]; then
touch "${nlock}" && read -r lastid n nx nt <"${nfile}" && rm -rf "${nlock}" && r=1
fi
done
nx=$(("${nx}" + 1)) nx=$(("${nx}" + 1))
nt=$(("${nt}" + 1)) nt=$(("${nt}" + 1))
error_found=0 #Continue only if lastid is lower than current id
t_id=$(($(date +%s%N) / 1000000)) t_id=$(($(date +%s%N) / 1000000))
if [[ -n "${id}" ]]; then if [[ -n "${id}" ]]; then
while read -r photo thumb micro; do while read -r avatar photo thumb micro; do
if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then
#If there is a photo #If there is a photo
folderescaped=${folder////\\/} folderescaped=${folder////\\/}
@ -54,11 +66,8 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
#If any of the images is not found in the filesystem #If any of the images is not found in the filesystem
if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then
#If the avatar uses the standard fallback picture or is local, we cannot use it as a base #If the avatar uses the standard fallback picture or is local, we cannot use it as a base
t=$(($(date +%s%N) / 1000000))
avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"")
result_string=$(printf "%s %d ms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t)))
#If we have a remote avatar as a fallback, download it #If we have a remote avatar as a fallback, download it
if [[ "$!" -eq 0 && -n "${avatar}" ]]; then if [[ -n "${avatar}" && $(grep -q -v -e "${url}" -e "images/person" <(echo "${avatar}")) -gt 0 ]]; then
result_string=$(printf "%s Remote %s" "${result_string}" "${avatar}") result_string=$(printf "%s Remote %s" "${result_string}" "${avatar}")
nl=1 nl=1
sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}"
@ -79,22 +88,28 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
error_found=1 error_found=1
fi fi
else else
result_string=$(printf "%s No remote avatar" "${result_string}") result_string=$(printf "%s No remote" "${result_string}")
#If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later
mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & mariadb "${db}" -N -B -q -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" &
result_string=$(printf "%s (blanked)" "${result_string}")
error_found=1 error_found=1
fi fi
else else
t=$(($(date +%s%N) / 1000000))
#If the images are all found in the filesystem, but fetching any of the images causes an error #If the images are all found in the filesystem, but fetching any of the images causes an error
if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || if [[ -s $(curl --fail-early \
curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" || -s "${photo}" -X HEAD -I --http2-prior-knowledge -4 -N --next \
curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then -s "${thumb}" -X HEAD -I --http2-prior-knowledge -4 -N --next \
-s "${micro}" -X HEAD -I --http2-prior-knowledge -4 -N |
grep -q "content-type: image") ]]; then
result_string=$(printf "%s F%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t)))
result_string=$(printf "${result_string} Fetch error: %s" "${photo}") result_string=$(printf "${result_string} Fetch error: %s" "${photo}")
mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" & mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" &
result_string=$(printf "%s (blanked)" "${result_string}") result_string=$(printf "%s (blanked)" "${result_string}")
nl=1 nl=1
error_found=1 error_found=1
else else
result_string=$(printf "%s F%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t)))
result_string=$(printf "%s (FOUND)" "${result_string}") result_string=$(printf "%s (FOUND)" "${result_string}")
error_found=0 error_found=0
fi fi
@ -103,11 +118,8 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
#If there is no photo #If there is no photo
result_string=$(printf "%s No local" "${result_string}") result_string=$(printf "%s No local" "${result_string}")
#If the avatar uses the standard fallback picture or is local, we cannot use it as a base #If the avatar uses the standard fallback picture or is local, we cannot use it as a base
t=$(($(date +%s%N) / 1000000))
avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"")
result_string=$(printf "%s %d ms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t)))
#If we have a remote avatar as a fallback, download it #If we have a remote avatar as a fallback, download it
if [[ "$!" -eq 0 && -n "${avatar}" ]]; then if [[ -n "${avatar}" && $(grep -q -v -e "${url}" -e "images/person" <(echo "${avatar}")) -gt 0 ]]; then
result_string=$(printf "${result_string} Remote %s" "${avatar}") result_string=$(printf "${result_string} Remote %s" "${avatar}")
nl=1 nl=1
sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}" sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}"
@ -134,17 +146,6 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
fi fi
fi fi
if [[ "${error_found}" -gt 0 ]]; then if [[ "${error_found}" -gt 0 ]]; then
#Request the user data to be regenerated in the system through the database
#if [[ $(mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) \
#select \"UpdateContact\" as command, \"[${id}]\" as parameter, 20 as priority, concat(curdate(), \" \", curtime()) as created \
#from workerqueue where (select count(*) from workerqueue \
#where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0) = 0; \
#select row_count();") -gt 0 ]]; then
#result_string=$(printf "%s (added)" "${result_string}")
#nl=1
#else
#result_string=$(printf "%s (already)" "${result_string}")
#fi
mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) \ mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) \
values (\"UpdateContact\", \"[${id}]\", 20, concat(curdate(), \" \", curtime()));" & values (\"UpdateContact\", \"[${id}]\", 20, concat(curdate(), \" \", curtime()));" &
result_string=$(printf "%s (added)" "${result_string}") result_string=$(printf "%s (added)" "${result_string}")
@ -152,9 +153,18 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
n=$((n + 1)) n=$((n + 1))
fi fi
lastid="${id}" lastid="${id}"
done < <(mariadb "${db}" -B -N -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") done < <(mariadb "${db}" -B -N -q -e "select \`avatar\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}")
fi fi
result_string=$(printf "%s %d ms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_id))) w=0
while [[ "${w}" -eq 0 ]]; do
if [[ ! -f "${nlock}" ]]; then
#Write data to file, delete lock
#n is increased only if error_found = 1
touch "${nlock}" && read -r lastid n nx nt <"${nfile}" && n=$((n + error_found)) && nx=$((nx + 1)) && nt=$((nt + 1)) && lastid="${id}" &&
echo "${lastid} ${n} ${nx} ${nt}" >"${nfile}" && rm -rf "${nlock}" && w=1
fi
done
result_string=$(printf "%s T%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_id)))
final_string=$(printf "E%8d F%8d/%8d T%8d/%8d %s" "${n}" "${nt}" "${dbcount}" "${lastid}" "${maxid}" "${result_string}") final_string=$(printf "E%8d F%8d/%8d T%8d/%8d %s" "${n}" "${nt}" "${dbcount}" "${lastid}" "${maxid}" "${result_string}")
final_string_length="${#final_string}" final_string_length="${#final_string}"
#Previous line clearance #Previous line clearance
@ -170,7 +180,25 @@ until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
final_string=$(printf "%s\n\r\n" "${final_string}") final_string=$(printf "%s\n\r\n" "${final_string}")
fi fi
printf "%s\r" "${final_string}" printf "%s\r" "${final_string}"
}
#Go to the Friendica installation
cd "${folder}" || exit
until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
nx=0
batch=$(("${batch}" + 1))
result_string=""
nl=0
error_found=0
echo "${lastid} ${n} ${nx} ${nt}" >"${nfile}"
while read -r id; do
loop "${batch}" "${result_string}" "${nl}" "${nx}" "${nt}" "${error_found}" &
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; do
wait -n
done
done < <(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") done < <(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}")
wait
done done
rm -rf "${nfile}" "${nlock}"
#mariadb "${db}" -e "alter table contact drop index photo_index" #mariadb "${db}" -e "alter table contact drop index photo_index"
#printf "\nFixing folders and moving to avatar cache...\n" #printf "\nFixing folders and moving to avatar cache...\n"