csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: Remove only necessary data, optimize display

Browse source
This commit is contained in:
Carlos Solís 2025-07-04 16:43:10 +00:00
parent f38bd73bf8
commit 06bbe1cf7d
1 changed files with 119 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

180
friendica-delete-old-users.sh
View file

@ -13,30 +13,15 @@ else
fi fi
intense_optimizations=${1:-"0"} intense_optimizations=${1:-"0"}
period_amount=${2:-"12"} period_amount=${2:-"12"}
starterid=${3:-"0"}
db="friendica" db="friendica"
period="${period_amount} MONTH" period="${period_amount} MONTH"
tmpfile=/tmp/friendica-delete-old-users.csv tmpfile=/tmp/friendica-delete-old-users.csv
url=friendica.example.net tmpreadlock=/tmp/friendica-delete-old-users.read.tmp
avatarfolder=/var/www/friendica/avatar tmpwritelock=/tmp/friendica-delete-old-users.write.tmp
avatarfolderescaped=${avatarfolder////\\/}
loop() { loop() {
baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g")
#Find the pictures in the avatar folders and delete them
picturecount=0
while read -r photo thumb micro; do
#If stored in avatar folder
if grep -v -q "${url}/avatar" <(echo "${photo}"); then
#if [[ -z "${isavatar}" ]]
phototrimmed=$(echo "${photo}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
rm -rfv "${phototrimmed}"
thumbtrimmed=$(echo "${thumb}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
rm -rfv "${thumbtrimmed}"
microtrimmed=$(echo "${micro}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
rm -rfv "${microtrimmed}"
picturecount=1
fi
done < <("${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}")
postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();" || echo 0) postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();" || echo 0)
@ -44,30 +29,86 @@ loop() {
postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${id}; select row_count();" || echo 0) photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${id}; select row_count();" || echo 0)
contactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${id}; select row_count();" || echo 0) lastitemid="${id}"
apcontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${id}; select row_count();" || echo 0) if [[ -n $(type flock) ]]; then
diasporacontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${id}; select row_count();" || echo 0) isreadlocked=0
while read -r tmp_counter tmp_picturecount tmp_postthreadcount tmp_postthreadusercount tmp_postusercount tmp_posttagcount tmp_postcontentcount tmp_postcount tmp_photocount tmp_contactcount tmp_apcontactcount tmp_diasporacontactcount; do while [[ "${isreadlocked}" -eq 0 ]]; do
if [[ -n "${tmp_counter}" && -n "${tmp_picturecount}" && -n "${tmp_postthreadcount}" && -n "${tmp_postthreadusercount}" && -n "${tmp_postusercount}" && -n "${tmp_posttagcount}" && -n "${tmp_postcontentcount}" && -n "${tmp_postcount}" && -n "${tmp_photocount}" && -n "${tmp_contactcount}" && -n "${tmp_apcontactcount}" && -n "${tmp_diasporacontactcount}" ]]; then exec 9>"${tmpreadlock}"
picturecount=$((picturecount + tmp_picturecount)) if flock -n -e 9; then
postthreadcount=$((postthreadcount + tmp_postthreadcount)) isreadlocked=1
postthreadusercount=$((postthreadusercount + tmp_postthreadusercount)) if [[ -f "${tmpfile}" ]]; then
postusercount=$((postusercount + tmp_postusercount)) while read -r tmp_counter tmp_lastitemid tmp_postthreadcount tmp_postthreadusercount tmp_postusercount tmp_posttagcount tmp_postcontentcount tmp_postcount tmp_photocount; do
posttagcount=$((posttagcount + tmp_posttagcount)) if [[ "${id}" -gt "${lastitemid}" ]]; then
postcontentcount=$((postcontentcount + tmp_postcontentcount)) lastitemid="${id}"
postcount=$((postcount + tmp_postcount)) fi
photocount=$((photocount + tmp_photocount)) if [[ -n "${tmp_postthreadcount}" ]]; then
contactcount=$((contactcount + tmp_contactcount)) postthreadcount=$((postthreadcount + tmp_postthreadcount))
apcontactcount=$((apcontactcount + tmp_apcontactcount)) fi
diasporacontactcount=$((diasporacontactcount + tmp_diasporacontactcount)) if [[ -n "${tmp_postthreadusercount}" ]]; then
postthreadusercount=$((postthreadusercount + tmp_postthreadusercount))
fi
if [[ -n "${tmp_postusercount}" ]]; then
postusercount=$((postusercount + tmp_postusercount))
fi
if [[ -n "${tmp_posttagcount}" ]]; then
posttagcount=$((posttagcount + tmp_posttagcount))
fi
if [[ -n "${tmp_postcontentcount}" ]]; then
postcontentcount=$((postcontentcount + tmp_postcontentcount))
fi
if [[ -n "${tmp_postcount}" ]]; then
postcount=$((postcount + tmp_postcount))
fi
if [[ -n "${tmp_photocount}" ]]; then
photocount=$((photocount + tmp_photocount))
fi
done <"${tmpfile}"
flock -u 9
iswritelocked=0
while [[ "${iswritelocked}" -eq 0 ]]; do
exec 8>"${tmpwritelock}"
if flock -n -e 8; then
iswritelocked=1
echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}"
flock -u 8
fi
done
fi
fi
done
else
if [[ -f "${tmpfile}" ]]; then
while read -r tmp_counter tmp_lastitemid tmp_postthreadcount tmp_postthreadusercount tmp_postusercount tmp_posttagcount tmp_postcontentcount tmp_postcount tmp_photocount; do
if [[ "${id}" -gt "${lastitemid}" ]]; then
lastitemid="${id}"
fi
if [[ -n "${tmp_postthreadcount}" ]]; then
postthreadcount=$((postthreadcount + tmp_postthreadcount))
fi
if [[ -n "${tmp_postthreadusercount}" ]]; then
postthreadusercount=$((postthreadusercount + tmp_postthreadusercount))
fi
if [[ -n "${tmp_postusercount}" ]]; then
postusercount=$((postusercount + tmp_postusercount))
fi
if [[ -n "${tmp_posttagcount}" ]]; then
posttagcount=$((posttagcount + tmp_posttagcount))
fi
if [[ -n "${tmp_postcontentcount}" ]]; then
postcontentcount=$((postcontentcount + tmp_postcontentcount))
fi
if [[ -n "${tmp_postcount}" ]]; then
postcount=$((postcount + tmp_postcount))
fi
if [[ -n "${tmp_photocount}" ]]; then
photocount=$((photocount + tmp_photocount))
fi
done <"${tmpfile}"
echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}"
fi fi
done <"${tmpfile}"
if [[ -n "${counter}" && -n "${picturecount}" && -n "${postthreadcount}" && -n "${postthreadusercount}" && -n "${postusercount}" && -n "${posttagcount}" && -n "${postcontentcount}" && -n "${postcount}" && -n "${photocount}" && -n "${contactcount}" && -n "${apcontactcount}" && -n "${diasporacontactcount}" ]]; then
echo "${counter}" "${picturecount}" "${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount} ${contactcount} ${apcontactcount} ${diasporacontactcount}" >"${tmpfile}"
fi fi
if [[ -n "${lastitem}" && "${#lastitem}" -gt 9 ]]; then if [[ -n "${lastitem}" && "${#lastitem}" -gt 9 ]]; then
response_left=$(printf "%s %s %s %s@%s " "${counter}" "${id}" "${lastitem::-9}" "${nick}" "${baseurltrimmed}") response_left=$(printf "%s %s %s %s@%s " "${counter}" "${id}" "${lastitem::-9}" "${nick}" "${baseurltrimmed}")
response=$(printf "%spicture:%s " "${response}" "${picturecount}")
response=$(printf "%spost-thread:%s " "${response}" "${postthreadcount}") response=$(printf "%spost-thread:%s " "${response}" "${postthreadcount}")
response=$(printf "%spost-thread-user:%s " "${response}" "${postthreadusercount}") response=$(printf "%spost-thread-user:%s " "${response}" "${postthreadusercount}")
response=$(printf "%spost-user:%s " "${response}" "${postusercount}") response=$(printf "%spost-user:%s " "${response}" "${postusercount}")
@ -75,9 +116,6 @@ loop() {
response=$(printf "%spost-content:%s " "${response}" "${postcontentcount}") response=$(printf "%spost-content:%s " "${response}" "${postcontentcount}")
response=$(printf "%spost:%s " "${response}" "${postcount}") response=$(printf "%spost:%s " "${response}" "${postcount}")
response=$(printf "%sphoto:%s " "${response}" "${photocount}") response=$(printf "%sphoto:%s " "${response}" "${photocount}")
response=$(printf "%scontact:%s " "${response}" "${contactcount}")
response=$(printf "%sapcontact:%s " "${response}" "${apcontactcount}")
response=$(printf "%sdiaspora-contact:%s " "${response}" "${diasporacontactcount}")
#Previous line clearance #Previous line clearance
#Measure length of string, blank only the excess #Measure length of string, blank only the excess
#Since this string is panned to both sides, we will need to account for two lengths #Since this string is panned to both sides, we will need to account for two lengths
@ -108,8 +146,17 @@ loop() {
#Check if our dependencies are installed #Check if our dependencies are installed
if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then
date date
if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}"
fi
if [[ -f "${tmpreadlock}" ]]; then
rm -rf "${tmpreadlock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi
touch "${tmpfile}" touch "${tmpfile}"
echo "0 0 0 0 0 0 0 0 0 0 0 0" >"${tmpfile}" echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}"
if [[ "${intense_optimizations}" -gt 0 ]]; then if [[ "${intense_optimizations}" -gt 0 ]]; then
"${dbengine}" "${db}" -v -e "\ "${dbengine}" "${db}" -v -e "\
alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \ alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \
@ -118,19 +165,18 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \ alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \ alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \
select count(\`id\`) from contact c where \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \
c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \
'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin');
" "
fi fi
counter=0 counter=0
was_empty=0 was_empty=0
while [[ "${was_empty}" -eq 0 ]]; do while [[ "${was_empty}" -eq 0 ]]; do
current_counter=0 current_counter=0
currentid="${starterid}"
while read -r tmp_counter tmp_lastitemid tmp_postthreadcount tmp_postthreadusercount tmp_postusercount tmp_posttagcount tmp_postcontentcount tmp_postcount tmp_photocount; do
if [[ -n "${tmp_counter}" && -n "${tmp_lastitemid}" && "${currentid}" -lt "${tmp_lastitemid}" ]]; then
currentid="${tmp_lastitemid}"
fi
done <"${tmpfile}"
while read -r id nick baseurl lastitem; do while read -r id nick baseurl lastitem; do
counter=$((counter + 1)) counter=$((counter + 1))
current_counter=$((current_counter + 1)) current_counter=$((current_counter + 1))
@ -145,9 +191,8 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \ c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \
c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \ c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \
'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') \ 'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \
limit 1000") c.\`id\` > ${currentid} limit 1000")
#not regexp_replace(c.\`nick\`, '[[:punct:]].*', '') like regexp_replace( replace(c.\`baseurl\`, 'https://', ''), '[[:punct:]].+', '')
wait wait
if [[ "${current_counter}" -eq 0 ]]; then if [[ "${current_counter}" -eq 0 ]]; then
was_empty=1 was_empty=1
@ -161,17 +206,30 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
alter table \`post-user\` auto_increment = 1; \ alter table \`post-user\` auto_increment = 1; \
alter table \`post-tag\` auto_increment = 1; \ alter table \`post-tag\` auto_increment = 1; \
alter table \`post\` auto_increment = 1; \ alter table \`post\` auto_increment = 1; \
alter table \`photo\` auto_increment = 1; \
alter table \`contact\` auto_increment = 1; \
alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \
alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \
alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \
alter table \`post-user\` drop index \`tmp_post_user_id\`; \
alter table \`post\` drop index \`tmp_post_id\`; \
alter table \`photo\` drop index\`tmp_photo_id\`; \
" "
"${dboptimizeengine}" "${db}" "${dboptimizeengine}" "${db}"
fi fi
rm -rf "${tmpfile}" "${dbengine}" "${db}" -v -e "\
alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \
alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \
alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \
alter table \`post-user\` drop index \`tmp_post_user_id\`; \
alter table \`post\` drop index \`tmp_post_id\`; \
alter table \`photo\` drop index \`tmp_photo_id\`; \
"
"${dboptimizeengine}" "${db}"
if [[ -n $(type flock) ]]; then
flock -u 9
flock -u 8
fi
if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}"
fi
if [[ -f "${tmpreadlock}" ]]; then
rm -rf "${tmpreadlock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi
date date
fi fi