csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Add a fifth parameter to disable the lockfile implementation

Browse source
This commit is contained in:
Carlos Solís 2025-02-24 11:03:57 -06:00
parent 44da143777
commit c0d998342d
1 changed files with 129 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
friendica-remove-invalid-photos.sh
View file

@ -35,6 +35,11 @@ lastid=${4:-"0"}
if [[ ! "${lastid}" =~ ^[0-9]+$ || $((10#${lastid})) -le 0 ]]; then if [[ ! "${lastid}" =~ ^[0-9]+$ || $((10#${lastid})) -le 0 ]]; then
lastid=0 lastid=0
fi fi
#Command-line parameter number 5: whether to enable the lockfile and the item counter, or disable it for a performance boost (0=off). Defaults to 1=on.
lockfile_enabled=${5:-"1"}
if [[ "${lockfile_enabled}" != "0" && "${lockfile_enabled}" != "1" ]]; then
lockfile_enabled=1
fi
nfolder="/tmp/friendica-remove-invalid-photos" nfolder="/tmp/friendica-remove-invalid-photos"
nfile="${nfolder}/n$(date +%s).csv" nfile="${nfolder}/n$(date +%s).csv"
nlock="${nfolder}/n$(date +%s).lock" nlock="${nfolder}/n$(date +%s).lock"
@ -68,8 +73,12 @@ if [[ "${intensive_optimizations}" -gt 0 ]]; then
indexlength=$(("${#url}" + 16)) indexlength=$(("${#url}" + 16))
"${dbengine}" "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" "${dbengine}" "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = ''))") dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = ''))")
#dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%')")
#dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = ''))")
else else
dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
#dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%') and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
#dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
fi fi
loop() { loop() {
@ -80,6 +89,7 @@ loop() {
nl=0 nl=0
error_found=0 error_found=0
#Lockfile-protected read #Lockfile-protected read
if [[ ${lockfile_enabled} -eq 1 ]]; then
r=0 r=0
if [[ "${time_counter}" -eq 1 ]]; then if [[ "${time_counter}" -eq 1 ]]; then
t_r=$(($(date +%s%N) / 1000000)) t_r=$(($(date +%s%N) / 1000000))
@ -126,6 +136,7 @@ loop() {
if [[ "${time_counter}" -eq 1 ]]; then if [[ "${time_counter}" -eq 1 ]]; then
result_string=$(printf "%s R%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_r))) result_string=$(printf "%s R%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_r)))
fi fi
fi
if [[ -n "${id}" ]]; then if [[ -n "${id}" ]]; then
while read -r avatar photo thumb micro; do while read -r avatar photo thumb micro; do
if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then
@ -238,6 +249,7 @@ loop() {
lastid="${id}" lastid="${id}"
done < <("${dbengine}" "${db}" -B -N -q -e "select \`avatar\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") done < <("${dbengine}" "${db}" -B -N -q -e "select \`avatar\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}")
fi fi
if [[ ${lockfile_enabled} -eq 1 ]]; then
w=0 w=0
if [[ "${time_counter}" -eq 1 ]]; then if [[ "${time_counter}" -eq 1 ]]; then
t_w=$(($(date +%s%N) / 1000000)) t_w=$(($(date +%s%N) / 1000000))
@ -278,9 +290,16 @@ loop() {
done done
if [[ "${time_counter}" -eq 1 ]]; then if [[ "${time_counter}" -eq 1 ]]; then
result_string=$(printf "%s W%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_w))) result_string=$(printf "%s W%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_w)))
fi
fi
if [[ "${time_counter}" -eq 1 ]]; then
result_string=$(printf "%s T%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_id))) result_string=$(printf "%s T%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_id)))
fi fi
final_string=$(printf "E%8d F%8d/%8d T%8d/%8d %s" "${n}" "${nt}" "${dbcount}" "${lastid}" "${maxid}" "${result_string}") final_string=""
if [[ ${lockfile_enabled} -eq 1 ]]; then
final_string=$(printf "E%8d F%8d/%8d " "${n}" "${nt}" "${dbcount}")
fi
final_string=$(printf "%sT%8d/%8d %s" "${final_string}" "${lastid}" "${maxid}" "${result_string}")
final_string_length="${#final_string}" final_string_length="${#final_string}"
#Previous line clearance #Previous line clearance
#Measure length of string, blank only the excess #Measure length of string, blank only the excess
@ -307,8 +326,12 @@ echo "${n} ${nt}" >"${nfile}"
c="" c=""
if [[ "${intensive_optimizations}" -gt 0 ]]; then if [[ "${intensive_optimizations}" -gt 0 ]]; then
c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) order by id limit ${limit}") c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) order by id limit ${limit}")
#c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%') order by id limit ${limit}")
#c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) order by id limit ${limit}")
else else
c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
#c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%') and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
#c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
fi fi
while read -r id; do while read -r id; do
if [[ -n "${id}" && $((10#${id})) -ge "${lastid}" ]]; then if [[ -n "${id}" && $((10#${id})) -ge "${lastid}" ]]; then
@ -324,6 +347,7 @@ while read -r id; do
done done
done < <(echo "${c}") done < <(echo "${c}")
wait wait
if [[ ${lockfile_enabled} -eq 1 ]]; then
#Read data before next iteration #Read data before next iteration
rl=0 rl=0
( (
@ -359,7 +383,7 @@ while [[ "${rl}" -eq 0 ]]; do
fi fi
fi fi
done done
#done fi
if [[ -f "${nfile}" ]]; then if [[ -f "${nfile}" ]]; then
rm -rf "${nfile}" rm -rf "${nfile}"
fi fi