Compare commits
No commits in common. "0ac788c88291d336e0534a39a6145789ff6912ef" and "4924cd7933d7b7f46db20d3f43d885a0eb144100" have entirely different histories.
0ac788c882
...
4924cd7933
2 changed files with 81 additions and 111 deletions
|
|
@ -7,9 +7,8 @@ phpversion=php8.2
|
||||||
dbengine=mariadb
|
dbengine=mariadb
|
||||||
db=friendica
|
db=friendica
|
||||||
dboptimizer=mariadb-optimize
|
dboptimizer=mariadb-optimize
|
||||||
intense_optimizations=${1:-"0"}
|
|
||||||
|
|
||||||
bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null
|
bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null;
|
||||||
|
|
||||||
echo "tmp_post_origin_deleted"
|
echo "tmp_post_origin_deleted"
|
||||||
tmp_post_origin_deleted_q="${limit}"
|
tmp_post_origin_deleted_q="${limit}"
|
||||||
|
|
@ -290,79 +289,77 @@ until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]; do
|
||||||
done
|
done
|
||||||
wait
|
wait
|
||||||
|
|
||||||
if [[ "${intense_optimizations}" -gt 0 ]]; then
|
echo "tmp_item_uri_duplicate"
|
||||||
echo "tmp_item_uri_duplicate"
|
tmp_item_uri_duplicate_q="${limit}"
|
||||||
tmp_item_uri_duplicate_q="${limit}"
|
tmp_item_uri_duplicate_current_id=0
|
||||||
tmp_item_uri_duplicate_current_id=0
|
until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do
|
||||||
until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do
|
initial_i=$(date +%s)
|
||||||
initial_i=$(date +%s)
|
tmp_item_uri_duplicate_q=0
|
||||||
tmp_item_uri_duplicate_q=0
|
while read -r id; do
|
||||||
while read -r id; do
|
if [[ -s "${id}" ]]; then
|
||||||
if [[ -s "${id}" ]]; then
|
"${dbengine}" "${db}" -N -B -q -e \
|
||||||
"${dbengine}" "${db}" -N -B -q -e \
|
"DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" &
|
||||||
"DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" &
|
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
|
||||||
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
|
wait -n
|
||||||
wait -n
|
|
||||||
fi
|
|
||||||
tmp_item_uri_duplicate_q=$((tmp_item_uri_duplicate_q + 1))
|
|
||||||
tmp_item_uri_duplicate_current_id="${id}"
|
|
||||||
fi
|
fi
|
||||||
done < <("${dbengine}" "${db}" -N -B -q -e \
|
tmp_item_uri_duplicate_q=$((tmp_item_uri_duplicate_q + 1))
|
||||||
"SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} \
|
tmp_item_uri_duplicate_current_id="${id}"
|
||||||
|
fi
|
||||||
|
done < <("${dbengine}" "${db}" -N -B -q -e \
|
||||||
|
"SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` > ${tmp_item_uri_duplicate_current_id} \
|
||||||
AND t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}")
|
AND t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}")
|
||||||
final_i=$(($(date +%s) - initial_i))
|
final_i=$(($(date +%s) - initial_i))
|
||||||
echo "${tmp_item_uri_duplicate_q} item(s) deleted until ${tmp_item_uri_duplicate_current_id} in ${final_i}s"
|
echo "${tmp_item_uri_duplicate_q} item(s) deleted until ${tmp_item_uri_duplicate_current_id} in ${final_i}s"
|
||||||
done
|
done
|
||||||
wait
|
wait
|
||||||
|
|
||||||
echo "tmp_post_media_duplicate"
|
echo "tmp_post_media_duplicate"
|
||||||
tmp_post_media_duplicate_q="${limit}"
|
tmp_post_media_duplicate_q="${limit}"
|
||||||
tmp_post_media_duplicate_current_id=0
|
tmp_post_media_duplicate_current_id=0
|
||||||
until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do
|
until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do
|
||||||
initial_i=$(date +%s)
|
initial_i=$(date +%s)
|
||||||
tmp_post_media_duplicate_q=0
|
tmp_post_media_duplicate_q=0
|
||||||
while read -r id; do
|
while read -r id; do
|
||||||
if [[ -s "${id}" ]]; then
|
if [[ -s "${id}" ]]; then
|
||||||
"${dbengine}" "${db}" -N -B -q -e \
|
"${dbengine}" "${db}" -N -B -q -e \
|
||||||
"DELETE FROM \`post-media\` WHERE \`id\` = ${id}" &
|
"DELETE FROM \`post-media\` WHERE \`id\` = ${id}" &
|
||||||
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
|
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
|
||||||
wait -n
|
wait -n
|
||||||
fi
|
|
||||||
tmp_post_media_duplicate_q=$((tmp_post_media_duplicate_q + 1))
|
|
||||||
tmp_post_media_duplicate_current_id="${id}"
|
|
||||||
fi
|
fi
|
||||||
done < <("${dbengine}" "${db}" -N -B -q -e \
|
tmp_post_media_duplicate_q=$((tmp_post_media_duplicate_q + 1))
|
||||||
"SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} \
|
tmp_post_media_duplicate_current_id="${id}"
|
||||||
|
fi
|
||||||
|
done < <("${dbengine}" "${db}" -N -B -q -e \
|
||||||
|
"SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` > ${tmp_post_media_duplicate_current_id} \
|
||||||
AND u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}")
|
AND u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}")
|
||||||
final_i=$(($(date +%s) - initial_i))
|
final_i=$(($(date +%s) - initial_i))
|
||||||
echo "${tmp_post_media_duplicate_q} item(s) deleted until ${tmp_post_media_duplicate_current_id} in ${final_i}s"
|
echo "${tmp_post_media_duplicate_q} item(s) deleted until ${tmp_post_media_duplicate_current_id} in ${final_i}s"
|
||||||
done
|
done
|
||||||
wait
|
wait
|
||||||
|
|
||||||
echo "tmp_post_user_duplicate"
|
echo "tmp_post_user_duplicate"
|
||||||
tmp_post_user_duplicate_q="${limit}"
|
tmp_post_user_duplicate_q="${limit}"
|
||||||
tmp_post_user_duplicate_current_id=0
|
tmp_post_user_duplicate_current_id=0
|
||||||
until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do
|
until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do
|
||||||
initial_i=$(date +%s)
|
initial_i=$(date +%s)
|
||||||
tmp_post_user_duplicate_q=0
|
tmp_post_user_duplicate_q=0
|
||||||
while read -r id; do
|
while read -r id; do
|
||||||
if [[ -s "${id}" ]]; then
|
if [[ -s "${id}" ]]; then
|
||||||
"${dbengine}" "${db}" -N -B -q -e \
|
"${dbengine}" "${db}" -N -B -q -e \
|
||||||
"DELETE FROM \`post-user\` WHERE \`id\` = ${id}" &
|
"DELETE FROM \`post-user\` WHERE \`id\` = ${id}" &
|
||||||
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
|
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; then
|
||||||
wait -n
|
wait -n
|
||||||
fi
|
|
||||||
tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1))
|
|
||||||
tmp_post_user_duplicate_current_id="${id}"
|
|
||||||
fi
|
fi
|
||||||
done < <("${dbengine}" "${db}" -N -B -q -e \
|
tmp_post_user_duplicate_q=$((tmp_post_user_duplicate_q + 1))
|
||||||
"SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} \
|
tmp_post_user_duplicate_current_id="${id}"
|
||||||
|
fi
|
||||||
|
done < <("${dbengine}" "${db}" -N -B -q -e \
|
||||||
|
"SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` > ${tmp_post_user_duplicate_current_id} \
|
||||||
AND v1.\`id\` < v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}")
|
AND v1.\`id\` < v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}")
|
||||||
final_i=$(($(date +%s) - initial_i))
|
final_i=$(($(date +%s) - initial_i))
|
||||||
echo "${tmp_post_user_duplicate_q} item(s) deleted until ${tmp_post_user_duplicate_current_id} in ${final_i}s"
|
echo "${tmp_post_user_duplicate_q} item(s) deleted until ${tmp_post_user_duplicate_current_id} in ${final_i}s"
|
||||||
done
|
done
|
||||||
wait
|
wait
|
||||||
|
|
||||||
"${dboptimizer}" "${db}" #&> /dev/null
|
"${dboptimizer}" "${db}" #&> /dev/null;
|
||||||
fi
|
bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null;
|
||||||
bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null
|
|
||||||
|
|
|
||||||
|
|
@ -6,20 +6,14 @@ group=www-data
|
||||||
fileperm=660
|
fileperm=660
|
||||||
db=friendica
|
db=friendica
|
||||||
folder=/var/www/friendica
|
folder=/var/www/friendica
|
||||||
intense_optimizations=${1:-"0"}
|
nfile=/tmp/n.csv
|
||||||
thread_multiplier=1
|
nlock=/tmp/n.lock
|
||||||
nfolder="/tmpʾfriendica-remove-invalid-photos"
|
if [[ -f ${nfile} ]]; then
|
||||||
nfile="${nfolder}/n$(date +%s).csv"
|
|
||||||
nlock="${nfolder}/n$(date +%s).lock"
|
|
||||||
if [[ ! -d "${nfolder}" ]]; then
|
|
||||||
mkdir "${nfolder}"
|
|
||||||
fi
|
|
||||||
if [[ -f "${nfile}" ]]; then
|
|
||||||
rm -rf "${nfile}" && touch "${nfile}"
|
rm -rf "${nfile}" && touch "${nfile}"
|
||||||
else
|
else
|
||||||
touch "${nfile}"
|
touch "${nfile}"
|
||||||
fi
|
fi
|
||||||
if [[ -f "${nlock}" ]]; then
|
if [[ -f ${nlock} ]]; then
|
||||||
rm -rf "${nlock}"
|
rm -rf "${nlock}"
|
||||||
fi
|
fi
|
||||||
#Internal parameters:
|
#Internal parameters:
|
||||||
|
|
@ -33,15 +27,12 @@ lastid=0
|
||||||
maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
|
maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
|
||||||
#Limit per batch
|
#Limit per batch
|
||||||
limit=$(((maxid / 1000) + 1))
|
limit=$(((maxid / 1000) + 1))
|
||||||
dbcount=0
|
#https:// = 8 characters | /avatar/ = 8 characters
|
||||||
if [[ "${intense_optimizations}" -gt 0 ]]; then
|
#indexlength=$(("${#url}" + 16))
|
||||||
#https:// = 8 characters | /avatar/ = 8 characters
|
#mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
|
||||||
indexlength=$(("${#url}" + 16))
|
#Add to the loop, reset values
|
||||||
mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
|
#dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
|
||||||
dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'")
|
dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'")
|
||||||
else
|
|
||||||
dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
|
|
||||||
fi
|
|
||||||
|
|
||||||
loop() {
|
loop() {
|
||||||
result_string=""
|
result_string=""
|
||||||
|
|
@ -157,9 +148,6 @@ loop() {
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
result_string=$(printf "%s No remote" "${result_string}")
|
result_string=$(printf "%s No remote" "${result_string}")
|
||||||
#If the avatar is not valid, set it as blank in the database
|
|
||||||
mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" &
|
|
||||||
result_string=$(printf "%s (blanked)" "${result_string}")
|
|
||||||
#If no remote avatar is found, we would blank the photo/thumb/micro and let the avatar cache process fix them later, but it's empty already here
|
#If no remote avatar is found, we would blank the photo/thumb/micro and let the avatar cache process fix them later, but it's empty already here
|
||||||
error_found=1
|
error_found=1
|
||||||
fi
|
fi
|
||||||
|
|
@ -219,30 +207,15 @@ loop() {
|
||||||
cd "${folder}" || exit
|
cd "${folder}" || exit
|
||||||
echo "${n} ${nt}" >"${nfile}"
|
echo "${n} ${nt}" >"${nfile}"
|
||||||
until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
|
until [[ $((nt + limit)) -gt "${dbcount}" ]]; do
|
||||||
c=""
|
|
||||||
if [[ "${intense_optimizations}" -gt 0 ]]; then
|
|
||||||
c=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}")
|
|
||||||
else
|
|
||||||
c=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
|
|
||||||
fi
|
|
||||||
while read -r id; do
|
while read -r id; do
|
||||||
lastid="${id}"
|
lastid="${id}"
|
||||||
loop &
|
loop &
|
||||||
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) * thread_multiplier)) ]]; do
|
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) * 1)) ]]; do
|
||||||
wait -n
|
wait -n
|
||||||
done
|
done
|
||||||
done < <(echo "${c}")
|
done < <(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}")
|
||||||
wait
|
wait
|
||||||
done
|
done
|
||||||
if [[ -f "${nfile}" ]]; then
|
rm -rf "${nfile}" "${nlock}"
|
||||||
rm -rf "${nfile}"
|
#mariadb "${db}" -e "alter table contact drop index photo_index"
|
||||||
fi
|
#printf "\nFixing folders and moving to avatar cache...\n"
|
||||||
if [[ -f "${nlock}" ]]; then
|
|
||||||
rm -rf "${nlock}"
|
|
||||||
fi
|
|
||||||
if [[ ! -d "${nfolder}" && $(find "${nfolder}" | wc -l) -eq 0 ]]; then
|
|
||||||
rm -rf "${nfolder}"
|
|
||||||
fi
|
|
||||||
if [[ "${intense_optimizations}" -gt 0 ]]; then
|
|
||||||
mariadb "${db}" -e "alter table contact drop index photo_index"
|
|
||||||
fi
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue