csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: csolisr:63c76c137ae5c899793174427ff8e3e2e7ed0974
Branches Tags
csolisr:main
...
compare: csolisr:e203b41209f8654c3788fb882a942035738c317e
Branches Tags
csolisr:main

3 commits
63c76c137a ... e203b41209

Author SHA1 Message Date
Carlos Solís
e203b41209 fix: Correct the display where the loop is restarted from a given lastid 2025-02-13 18:59:32 +00:00
Carlos Solís
6bd794f4d2 Fix: Correct some queries, make the default limit per batch higher 2025-02-13 18:58:46 +00:00
Carlos Solís
d4ff1deadc feat: Add a display of how many rows are deleted per user, improve queries 2025-02-13 18:57:59 +00:00
3 changed files with 46 additions and 32 deletions
Show stats Expand all files Collapse all files

43
friendica-find-missing-servers.sh
View file

@ -39,7 +39,7 @@ loop_1() {
}
loop_2() {
echo "Finding users for ${b}"
"${dbengine}" "${db}" -N -B -q -e "select \`id\`, \`nick\`, \`baseurl\` from contact c where c.\`id\` not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and (c.baseurl = \"http://${b}\" or c.url = \"http://${b}\" or c.baseurl = \"https://${b}\" or c.url = \"https://${b}\")" | sudo tee -a "${idsdownfile}" #&> /dev/null
"${dbengine}" "${db}" -N -B -q -e "select \`id\`, \`nick\`, \`baseurl\` from contact c where c.\`id\` not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and (c.baseurl = \"http://${b}\" or c.baseurl = \"https://${b}\")" | sudo tee -a "${idsdownfile}" #&> /dev/null
}
loop_3() {
@ -58,26 +58,38 @@ loop_3() {
rm -rfv "${microtrimmed}"
fi
done
"${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread-user\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "delete from \`post-user\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`;"
"${dbengine}" "${db}" -N -B -q -e "delete from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}"
"${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}"
printf "post-thread:"
"${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();"
printf "post-thread-user:"
"${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();"
printf "post-user":
"${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();"
printf "post-tag:"
"${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}; select row_count();"
printf "post-content:"
"${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();"
printf "post:"
"${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();"
printf "photo:"
"${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}; select row_count();"
printf "contact:"
"${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}; select row_count();"
printf "apcontact:"
"${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}; select row_count();"
printf "diaspora-contact:"
"${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}; select row_count();"
}
#Check if our dependencies are installed
if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then
date
"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`contact_baseurl\` (baseurl)"
"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`contact_baseurl\` (\`baseurl\`)"
"${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` add index if not exists \`post_user_id\` (\`author-id\`, \`causer-id\`, \`owner-id\`)"
if [[ ! -f "${tmpfile}" ]]; then
echo "Listing sites"
siteslist=$("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl, protocol from contact where baseurl != ''" | sort -b -f -n | sed -e "s/http:/https:/g" | uniq -i)
echo "Amount of unique sites: ${#siteslist[@]}"
siteslistamount=$(echo "${siteslist}" | wc -l)
echo "Amount of unique sites: ${siteslistamount}"
while read -r sites protocol; do
loop_1 "${sites}" "${protocol}" &
if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then
@ -92,7 +104,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
done <"${tmpfile}"
t=$(sort -n "${tmpfile}" | uniq)
echo "${t}" >"${tmpfile}"
echo "Amount of sites down: ${#sitesdown[@]} / ${#siteslist[@]}"
echo "Amount of sites down: ${#sitesdown[@]} / ${siteslistamount}"
if [[ ! -f "${idsdownfile}" ]]; then
for b in "${sitesdown[@]}"; do
loop_2 "${b}" &
@ -121,6 +133,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
"${dbengine}" "${db}" -N -B -q -e "alter table \`post\` auto_increment = 1"
"${dbengine}" "${db}" -N -B -q -e "alter table \`photo\` auto_increment = 1"
"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` auto_increment = 1"
"${dbengine}" "${db}" -e "alter table contact drop index \`contact_baseurl\`"
"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` drop index \`contact_baseurl\`"
"${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` drop index \`post_user_id\`"
date
fi

31
friendica-non-follower-featured-posts.sh
View file

@ -1,7 +1,8 @@
#!/bin/bash
ca=100
limit=1000
ca=${limit}
camax=0
while [[ ${ca} -gt 0 ]]; do
until [[ ${ca} -lt ${limit} ]]; do
ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"UpdateContact\" limit ${ca}; select row_count();")
camax=$((camax + ca))
printf "\rUpdateContact\t\t%s\r" "${camax}"
@ -9,9 +10,9 @@ done
printf "\rUpdateContact\t\t%s\n\r" "${camax}"
#echo "UpdateContact $camax"
cb=100
cb=${limit}
cbmax=0
while [[ ${cb} -gt 0 ]]; do
until [[ ${cb} -lt ${limit} ]]; do
cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"ContactDiscovery\" limit ${cb}; select row_count();")
cbmax=$((cbmax + cb))
printf "\rContactDiscovery\t%s\r" "${cbmax}"
@ -19,19 +20,19 @@ done
printf "\rContactDiscovery\t%s\n\r" "${cbmax}"
#echo "ContactDiscovery $cbmax"
cc=100
cc=${limit}
ccmax=0
while [[ ${cc} -gt 0 ]]; do
until [[ ${cc} -lt ${limit} ]]; do
cc=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_addcontact (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\`= \"AddContact\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_addcontact) limit ${cc}; select row_count();")
ccmax=$((ccmax + cc))
printf "\rAddContact\t%s\r" "${ccmax}"
done
printf "\rAddContact\t%s\n\r" "${ccmax}"
printf "\rAddContact \t%s\n\r" "${ccmax}"
#echo "AddContact $ccmax"
cd=100
cd=${limit}
cdmax=0
while [[ ${cd} -gt 0 ]]; do
until [[ ${cd} -lt ${limit} ]]; do
cd=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_updategserver (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\` = \"UpdateGServer\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_updategserver) limit ${cd}; select row_count();")
cdmax=$((cdmax + cd))
printf "\rUpdateGServer\t\t%s\r" "${cdmax}"
@ -39,9 +40,9 @@ done
printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}"
#echo "UpdateGServer $cdmax"
ce=100
ce=${limit}
cemax=0
while [[ ${ce} -gt 0 ]]; do
until [[ ${ce} -lt ${limit} ]]; do
ce=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_fetchfeaturedposts (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\`= \"FetchFeaturedPosts\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_fetchfeaturedposts) limit ${ce}; select row_count();")
cemax=$((cemax + ce))
printf "\rFetchFeaturedPosts\t%s\r" "${cemax}"
@ -49,9 +50,9 @@ done
printf "\rFetchFeaturedPosts\t%s\n\r" "${cemax}"
#echo "FetchFeaturedPosts $cemax"
cf=100
cf=${limit}
cfmax=0
while [[ ${cf} -gt 0 ]]; do
until [[ ${cf} -lt ${limit} ]]; do
cf=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where command=\"ProcessQueue\" and pid=0 and done=0 limit ${cf}; select row_count();")
cfmax=$((cfmax + cf))
printf "\rProcessQueue\t\t%s\r" "${cfmax}"
@ -59,9 +60,9 @@ done
printf "\rProcessQueue\t\t%s\n\r" "${cfmax}"
#echo "ProcessQueue $cfmax"
cg=100
cg=${limit}
cgmax=0
while [[ ${cg} -gt 0 ]]; do
until [[ ${cg} -lt ${limit} ]]; do
cg=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where \`id\` in (select distinct w2.\`id\` from workerqueue w1 inner join workerqueue w2 where w1.\`id\` > w2.\`id\` and w1.\`parameter\` = w2.\`parameter\` and w1.command = \"UpdateContact\" and w1.\`pid\` = 0 and w1.\`done\` = 0) limit ${cg}; select row_count();")
cgmax=$((cgmax + cg))
printf "\rWorkerQueue\t\t%s\r" "${cgmax}"

4
friendica-remove-invalid-photos.sh
View file

@ -67,9 +67,9 @@ if [[ "${intensive_optimizations}" -gt 0 ]]; then
#https:// = 8 characters | /avatar/ = 8 characters
indexlength=$(("${#url}" + 16))
"${dbengine}" "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = ''))")
dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = ''))")
else
dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
fi
loop() {