csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: csolisr:4924cd7933d7b7f46db20d3f43d885a0eb144100
Branches Tags
csolisr:main
...
compare: csolisr:0ac788c88291d336e0534a39a6145789ff6912ef
Branches Tags
csolisr:main

2 commits
4924cd7933 ... 0ac788c882

Author SHA1 Message Date
Carlos Solís
0ac788c882 feat: Add parameter to reserve intensive optimizations 2025-02-10 03:25:56 +00:00
Carlos Solís
d989d762f1 fix: Correct loop behavior, add parameter to reserve intensive optimizations 2025-02-10 03:25:14 +00:00
2 changed files with 111 additions and 81 deletions
Show stats Expand all files Collapse all files

135
friendica-clean-database.sh
View file

@ -7,8 +7,9 @@ phpversion=php8.2
dbengine=mariadb dbengine=mariadb
db=friendica db=friendica
dboptimizer=mariadb-optimize dboptimizer=mariadb-optimize
intense_optimizations=${1:-"0"}
bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null; bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null
echo "tmp_post_origin_deleted" echo "tmp_post_origin_deleted"
tmp_post_origin_deleted_q="${limit}" tmp_post_origin_deleted_q="${limit}"
@ -289,77 +290,79 @@ until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]; do
done done
wait wait
echo "tmp_item_uri_duplicate" if [[ "${intense_optimizations}" -gt 0 ]]; then
tmp_item_uri_duplicate_q="${limit}" echo "tmp_item_uri_duplicate"
tmp_item_uri_duplicate_current_id=0 tmp_item_uri_duplicate_q="${limit}"
until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do tmp_item_uri_duplicate_current_id=0
initial_i=$(date +%s) until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do
tmp_item_uri_duplicate_q=0 initial_i=$(date +%s)
while read -r id; do tmp_item_uri_duplicate_q=0
if [[ -s "${id}" ]]; then while read -r id; do
"${dbengine}" "${db}" -N -B -q -e \ if [[ -s "${id}" ]]; then
"DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" & "${dbengine}" "${db}" -N -B -q -e \
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" &
wait -n if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
wait -n
fi
tmp_item_uri_duplicate_q=$((tmp_item_uri_duplicate_q + 1))
tmp_item_uri_duplicate_current_id="${id}"
fi fi
tmp_item_uri_duplicate_q=$((tmp_item_uri_duplicate_q + 1)) done < <("${dbengine}" "${db}" -N -B -q -e \
tmp_item_uri_duplicate_current_id="${id}" "SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} \
fi
done < <("${dbengine}" "${db}" -N -B -q -e \
"SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} \
AND t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}") AND t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}")
final_i=$(($(date +%s) - initial_i)) final_i=$(($(date +%s) - initial_i))
echo "${tmp_item_uri_duplicate_q} item(s) deleted until ${tmp_item_uri_duplicate_current_id} in ${final_i}s" echo "${tmp_item_uri_duplicate_q} item(s) deleted until ${tmp_item_uri_duplicate_current_id} in ${final_i}s"
done done
wait wait
echo "tmp_post_media_duplicate" echo "tmp_post_media_duplicate"
tmp_post_media_duplicate_q="${limit}" tmp_post_media_duplicate_q="${limit}"
tmp_post_media_duplicate_current_id=0 tmp_post_media_duplicate_current_id=0
until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do
initial_i=$(date +%s) initial_i=$(date +%s)
tmp_post_media_duplicate_q=0 tmp_post_media_duplicate_q=0
while read -r id; do while read -r id; do
if [[ -s "${id}" ]]; then if [[ -s "${id}" ]]; then
"${dbengine}" "${db}" -N -B -q -e \ "${dbengine}" "${db}" -N -B -q -e \
"DELETE FROM \`post-media\` WHERE \`id\` = ${id}" & "DELETE FROM \`post-media\` WHERE \`id\` = ${id}" &
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
wait -n wait -n
fi
tmp_post_media_duplicate_q=$((tmp_post_media_duplicate_q + 1))
tmp_post_media_duplicate_current_id="${id}"
fi fi
tmp_post_media_duplicate_q=$((tmp_post_media_duplicate_q + 1)) done < <("${dbengine}" "${db}" -N -B -q -e \
tmp_post_media_duplicate_current_id="${id}" "SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} \
fi
done < <("${dbengine}" "${db}" -N -B -q -e \
"SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} \
AND u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}") AND u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}")
final_i=$(($(date +%s) - initial_i)) final_i=$(($(date +%s) - initial_i))
echo "${tmp_post_media_duplicate_q} item(s) deleted until ${tmp_post_media_duplicate_current_id} in ${final_i}s" echo "${tmp_post_media_duplicate_q} item(s) deleted until ${tmp_post_media_duplicate_current_id} in ${final_i}s"
done done
wait wait
echo "tmp_post_user_duplicate" echo "tmp_post_user_duplicate"
tmp_post_user_duplicate_q="${limit}" tmp_post_user_duplicate_q="${limit}"
tmp_post_user_duplicate_current_id=0 tmp_post_user_duplicate_current_id=0
until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do
initial_i=$(date +%s) initial_i=$(date +%s)
tmp_post_user_duplicate_q=0 tmp_post_user_duplicate_q=0
while read -r id; do while read -r id; do
if [[ -s "${id}" ]]; then if [[ -s "${id}" ]]; then
"${dbengine}" "${db}" -N -B -q -e \ "${dbengine}" "${db}" -N -B -q -e \
"DELETE FROM \`post-user\` WHERE \`id\` = ${id}" & "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" &
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
wait -n wait -n
fi
tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1))
tmp_post_user_duplicate_current_id="${id}"
fi fi
tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1)) done < <("${dbengine}" "${db}" -N -B -q -e \
tmp_post_user_duplicate_current_id="${id}" "SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} \
fi
done < <("${dbengine}" "${db}" -N -B -q -e \
"SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} \
AND v1.\`id\` < v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}") AND v1.\`id\` < v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}")
final_i=$(($(date +%s) - initial_i)) final_i=$(($(date +%s) - initial_i))
echo "${tmp_post_user_duplicate_q} item(s) deleted until ${tmp_post_user_duplicate_current_id} in ${final_i}s" echo "${tmp_post_user_duplicate_q} item(s) deleted until ${tmp_post_user_duplicate_current_id} in ${final_i}s"
done done
wait wait
"${dboptimizer}" "${db}" #&> /dev/null; "${dboptimizer}" "${db}" #&> /dev/null
bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null; fi
bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null

57
friendica-remove-invalid-photos.sh
View file

@ -6,14 +6,20 @@ group=www-data
fileperm=660 fileperm=660
db=friendica db=friendica
folder=/var/www/friendica folder=/var/www/friendica
nfile=/tmp/n.csv intense_optimizations=${1:-"0"}
nlock=/tmp/n.lock thread_multiplier=1
if [[ -f ${nfile} ]]; then nfolder="/tmpʾfriendica-remove-invalid-photos"
nfile="${nfolder}/n$(date +%s).csv"
nlock="${nfolder}/n$(date +%s).lock"
if [[ ! -d "${nfolder}" ]]; then
mkdir "${nfolder}"
fi
if [[ -f "${nfile}" ]]; then
rm -rf "${nfile}" && touch "${nfile}" rm -rf "${nfile}" && touch "${nfile}"
else else
touch "${nfile}" touch "${nfile}"
fi fi
if [[ -f ${nlock} ]]; then if [[ -f "${nlock}" ]]; then
rm -rf "${nlock}" rm -rf "${nlock}"
fi fi
#Internal parameters: #Internal parameters:
@ -27,12 +33,15 @@ lastid=0
maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
#Limit per batch #Limit per batch
limit=$(((maxid / 1000) + 1)) limit=$(((maxid / 1000) + 1))
#https:// = 8 characters | /avatar/ = 8 characters dbcount=0
#indexlength=$(("${#url}" + 16)) if [[ "${intense_optimizations}" -gt 0 ]]; then
#mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" #https:// = 8 characters | /avatar/ = 8 characters
#Add to the loop, reset values indexlength=$(("${#url}" + 16))
#dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'")
else
dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
fi
loop() { loop() {
result_string="" result_string=""
@ -148,6 +157,9 @@ loop() {
fi fi
else else
result_string=$(printf "%s No remote" "${result_string}") result_string=$(printf "%s No remote" "${result_string}")
#If the avatar is not valid, set it as blank in the database
mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" &
result_string=$(printf "%s (blanked)" "${result_string}")
#If no remote avatar is found, we would blank the photo/thumb/micro and let the avatar cache process fix them later, but it's empty already here #If no remote avatar is found, we would blank the photo/thumb/micro and let the avatar cache process fix them later, but it's empty already here
error_found=1 error_found=1
fi fi
@ -207,15 +219,30 @@ loop() {
cd "${folder}" || exit cd "${folder}" || exit
echo "${n} ${nt}" >"${nfile}" echo "${n} ${nt}" >"${nfile}"
until [[ $((nt + limit)) -gt "${dbcount}" ]]; do until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
c=""
if [[ "${intense_optimizations}" -gt 0 ]]; then
c=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}")
else
c=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
fi
while read -r id; do while read -r id; do
lastid="${id}" lastid="${id}"
loop & loop &
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; do until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) * thread_multiplier)) ]]; do
wait -n wait -n
done done
done < <(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}") done < <(echo "${c}")
wait wait
done done
rm -rf "${nfile}" "${nlock}" if [[ -f "${nfile}" ]]; then
#mariadb "${db}" -e "alter table contact drop index photo_index" rm -rf "${nfile}"
#printf "\nFixing folders and moving to avatar cache...\n" fi
if [[ -f "${nlock}" ]]; then
rm -rf "${nlock}"
fi
if [[ ! -d "${nfolder}" && $(find "${nfolder}" | wc -l) -eq 0 ]]; then
rm -rf "${nfolder}"
fi
if [[ "${intense_optimizations}" -gt 0 ]]; then
mariadb "${db}" -e "alter table contact drop index photo_index"
fi