From 477637f7e4e022e40c935bfb0ad91600c7d42e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Sol=C3=ADs?= Date: Wed, 5 Mar 2025 09:20:23 -0600 Subject: [PATCH 1/3] fix: Correct the comparison displayed on diff --- copy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy.sh b/copy.sh index f32a014..eb1f1df 100755 --- a/copy.sh +++ b/copy.sh @@ -7,7 +7,7 @@ for i in ./*.sh; do find /etc/cron.* -iname "${i_tmp%.sh}" | while read -r j; do echo "${j}" diff <(sed -e "s/friendica.example.net/hub.azkware.net/g" -e "s/1\:-\"0\"/1\:-\"1\"/g" -e "s/#&>/\&\>/g" "${i}") "${j}" - sed -e "s/friendica.example.net/hub.azkware.net/g" -e "s/#&>/\&\>/g" "${i}" | sudo tee "${j}" &>/dev/null + sed -e "s/friendica.example.net/hub.azkware.net/g" -e "s/1\:-\"0\"/1\:-\"1\"/g" -e "s/#&>/\&\>/g" "${i}" | sudo tee "${j}" &>/dev/null done find ../../Scripts -iname "${i_tmp}" | while read -r k; do echo "${k}" From 94b6d9f996c8ed6df25c482563abba5e723581d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Sol=C3=ADs?= Date: Wed, 5 Mar 2025 09:21:37 -0600 Subject: [PATCH 2/3] feat: Improve data display, add switch for intensive optimizations --- friendica-delete-old-users.sh | 120 +++++++++++++++++++++--------- friendica-find-missing-servers.sh | 117 ++++++++++++++++++++--------- 2 files changed, 164 insertions(+), 73 deletions(-) diff --git a/friendica-delete-old-users.sh b/friendica-delete-old-users.sh index e41fb65..40e8e0d 100755 --- a/friendica-delete-old-users.sh +++ b/friendica-delete-old-users.sh @@ -8,16 +8,19 @@ elif [[ -n $(type mysql) ]]; then else exit fi +intense_optimizations=${1:-"0"} db="friendica" +period="3 MONTH" +tmpfile=/tmp/friendica-delete-old-users.csv url=friendica.example.net avatarfolder=/var/www/friendica/avatar avatarfolderescaped=${avatarfolder////\\/} loop() { baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") - printf "%s - %s@%s " "${lineb}" "${nick}" "${baseurltrimmed}" #Find the pictures in the avatar folders and delete them - "${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro; do + picturecount=0 + while read -r photo thumb micro; do #If stored in avatar folder if grep -v -q "${url}/avatar" <(echo "${photo}"); then #if [[ -z "${isavatar}" ]] @@ -27,55 +30,98 @@ loop() { rm -rfv "${thumbtrimmed}" microtrimmed=$(echo "${micro}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") rm -rfv "${microtrimmed}" + picturecount=1 fi + done < <("${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") + postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) + postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) + postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();" || echo 0) + posttagcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${id}; select row_count();" || echo 0) + postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) + postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0) + photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${id}; select row_count();" || echo 0) + contactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${id}; select row_count();" || echo 0) + apcontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${id}; select row_count();" || echo 0) + diasporacontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${id}; select row_count();" || echo 0) + while read -r tmp_picturecount tmp_postthreadcount tmp_postthreadusercount tmp_postusercount tmp_posttagcount tmp_postcontentcount tmp_postcount tmp_photocount tmp_contactcount tmp_apcontactcount tmp_diasporacontactcount; do + picturecount=$((picturecount + tmp_picturecount)) + postthreadcount=$((postthreadcount + tmp_postthreadcount)) + postthreadusercount=$((postthreadusercount + tmp_postthreadusercount)) + postusercount=$((postusercount + tmp_postusercount)) + posttagcount=$((posttagcount + tmp_posttagcount)) + postcontentcount=$((postcontentcount + tmp_postcontentcount)) + postcount=$((postcount + tmp_postcount)) + photocount=$((photocount + tmp_photocount)) + contactcount=$((contactcount + tmp_contactcount)) + apcontactcount=$((apcontactcount + tmp_apcontactcount)) + diasporacontactcount=$((diasporacontactcount + tmp_diasporacontactcount)) + done <"${tmpfile}" + response_left=$(printf "%s %s %s@%s " "${id}" "${lastitem::-9}" "${nick}" "${baseurltrimmed}") + response=$(printf "%spicture:%s " "${response}" "${picturecount}") + response=$(printf "%spost-thread:%s " "${response}" "${postthreadcount}") + response=$(printf "%spost-thread-user:%s " "${response}" "${postthreadusercount}") + response=$(printf "%spost-user:%s " "${response}" "${postusercount}") + response=$(printf "%spost-tag:%s " "${response}" "${posttagcount}") + response=$(printf "%spost-content:%s " "${response}" "${postcontentcount}") + response=$(printf "%spost:%s " "${response}" "${postcount}") + response=$(printf "%sphoto:%s " "${response}" "${photocount}") + response=$(printf "%scontact:%s " "${response}" "${contactcount}") + response=$(printf "%sapcontact:%s " "${response}" "${apcontactcount}") + response=$(printf "%sdiaspora-contact:%s " "${response}" "${diasporacontactcount}") + echo "${picturecount}" "${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount} ${contactcount} ${apcontactcount} ${diasporacontactcount}" >"${tmpfile}" + #Previous line clearance + #Measure length of string, blank only the excess + final_string_length_left="${#response_left}" + final_string_length_right="${#response}" + final_string_length=$((final_string_length_left + final_string_length_right)) + blank_string="" + columns_length="${COLUMNS}" + while [[ "${final_string_length}" -gt "${columns_length}" ]]; do + columns_length=$((columns_length + COLUMNS)) done - postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();") - postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();") - postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();") - posttagcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}; select row_count();") - postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") - postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") - photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}; select row_count();") - contactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}; select row_count();") - apcontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}; select row_count();") - diasporacontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}; select row_count();") - printf "post-thread: %s " "${postthreadcount}" - printf "post-thread-user: %s " "${postthreadusercount}" - printf "post-user: %s " "${postusercount}" - printf "post-tag: %s " "${posttagcount}" - printf "post-content: %s " "${postcontentcount}" - printf "post: %s " "${postcount}" - printf "photo: %s " "${photocount}" - printf "contact: %s " "${contactcount}" - printf "apcontact: %s " "${apcontactcount}" - printf "diaspora-contact: %s " "${diasporacontactcount}" - printf "\r\n" + blank_string_length=$((columns_length - final_string_length)) + for ((count = 0; count < "${blank_string_length}"; count++)); do + blank_string=$(printf "%s " "${blank_string}") + done + response=$(printf "%s%s%s" "${response_left}" "${blank_string}" "${response}") + for ((count = 0; count < "${final_string_length}"; count++)); do + response=$(printf "%s\b" "${response}") + done + printf "%s\r" "${response}" } #Check if our dependencies are installed if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then date - "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`contact_baseurl\` (baseurl)" + touch "${tmpfile}" + echo "0 0 0 0 0 0 0 0 0 0 0" >"${tmpfile}" + if [[ "${intense_optimizations}" -gt 0 ]]; then + "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`contact_baseurl\` (baseurl)" + fi "${dbengine}" "${db}" -N -B -q -e \ - "select \`id\`, \`nick\`, \`baseurl\` from contact c where \ + "select \`id\`, \`nick\`, \`baseurl\`, \`last-item\` from contact c where \ c.\`id\` not in (select \`cid\` from \`user-contact\`) and \ c.\`id\` not in (select \`uid\` from \`user\`) and \ c.\`id\` not in ( select \`contact-id\` from \`group_member\`) and \ - c.\`contact-type\` != 4 and not pending and \`last-discovery\` < CURDATE() - INTERVAL 1 YEAR and \`last-item\` < CURDATE() - INTERVAL 1 YEAR" | - while read -r lineb nick baseurl; do - loop "${lineb}" "${nick}" "${baseurl}" & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; then + c.\`contact-type\` != 4 and not pending and \`last-discovery\` < CURDATE() - INTERVAL ${period} and \`last-item\` < CURDATE() - INTERVAL ${period}" | + while read -r id nick baseurl lastitem; do + loop "${id}" "${nick}" "${baseurl}" & + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) / 1)) ]]; then wait -n fi done + printf "\r\n" wait - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-thread\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-thread-user\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-tag\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`photo\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` drop index \`contact_baseurl\`" + "${dbengine}" "${db}" -N -B -q -e "alter table \`post-thread\` auto_increment = 1; \ + alter table \`post-thread-user\` auto_increment = 1; \ + alter table \`post-user\` auto_increment = 1; \ + alter table \`post-tag\` auto_increment = 1; \ + alter table \`post\` auto_increment = 1; \ + alter table \`photo\` auto_increment = 1; \ + alter table \`contact\` auto_increment = 1" + if [[ "${intense_optimizations}" -gt 0 ]]; then + "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` drop index \`contact_baseurl\`" + fi + rm -rf "${tmpfile}" date fi diff --git a/friendica-find-missing-servers.sh b/friendica-find-missing-servers.sh index 3b5128e..7c3deb3 100755 --- a/friendica-find-missing-servers.sh +++ b/friendica-find-missing-servers.sh @@ -8,9 +8,11 @@ elif [[ -n $(type mysql) ]]; then else exit fi +intense_optimizations=${1:-"0"} db="friendica" tmpfile="/tmp/sitesdown.txt" idsdownfile="/tmp/idsdown.txt" +usrfile="/tmp/usersdown.txt" url=friendica.example.net avatarfolder=/var/www/friendica/avatar avatarfolderescaped=${avatarfolder////\\/} @@ -44,9 +46,9 @@ loop_2() { loop_3() { baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") - printf "%s - %s@%s " "${lineb}" "${nick}" "${baseurltrimmed}" #Find the pictures in the avatar folders and delete them - "${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro; do + picturecount=0 + while read -r photo thumb micro; do #If stored in avatar folder if grep -v -q "${url}/avatar" <(echo "${photo}"); then #if [[ -z "${isavatar}" ]] @@ -56,36 +58,73 @@ loop_3() { rm -rfv "${thumbtrimmed}" microtrimmed=$(echo "${micro}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") rm -rfv "${microtrimmed}" + picturecount=1 fi + done < <("${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") + postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0) + postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0) + postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select_row_count();" || echo 0) + posttagcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${id}; select_row_count();" || echo 0) + postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0) + postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0) + photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${id}; select_row_count();" || echo 0) + contactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${id}; select_row_count();" || echo 0) + apcontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${id}; select_row_count();" || echo 0) + diasporacontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${id}; select_row_count();" || echo 0) + while read -r tmp_picturecount tmp_postthreadcount tmp_postthreadusercount tmp_postusercount tmp_posttagcount tmp_postcontentcount tmp_postcount tmp_photocount tmp_contactcount tmp_apcontactcount tmp_diasporacontactcount; do + picturecount=$((picturecount + tmp_picturecount)) + postthreadcount=$((postthreadcount + tmp_postthreadcount)) + postthreadusercount=$((postthreadusercount + tmp_postthreadusercount)) + postusercount=$((postusercount + tmp_postusercount)) + posttagcount=$((posttagcount + tmp_posttagcount)) + postcontentcount=$((postcontentcount + tmp_postcontentcount)) + postcount=$((postcount + tmp_postcount)) + photocount=$((photocount + tmp_photocount)) + contactcount=$((contactcount + tmp_contactcount)) + apcontactcount=$((apcontactcount + tmp_apcontactcount)) + diasporacontactcount=$((diasporacontactcount + tmp_diasporacontactcount)) + done <"${usrfile}" + response_left=$(printf "%s %s@%s " "${id}" "${nick}" "${baseurltrimmed}") + response=$(printf "%spicture:%s " "${response}" "${picturecount}") + response=$(printf "%spost-thread:%s " "${response}" "${postthreadcount}") + response=$(printf "%spost-thread-user:%s " "${response}" "${postthreadusercount}") + response=$(printf "%spost-user:%s " "${response}" "${postusercount}") + response=$(printf "%spost-tag:%s " "${response}" "${posttagcount}") + response=$(printf "%spost-content:%s " "${response}" "${postcontentcount}") + response=$(printf "%spost:%s " "${response}" "${postcount}") + response=$(printf "%sphoto:%s " "${response}" "${photocount}") + response=$(printf "%scontact:%s " "${response}" "${contactcount}") + response=$(printf "%sapcontact:%s " "${response}" "${apcontactcount}") + response=$(printf "%sdiaspora-contact:%s " "${response}" "${diasporacontactcount}") + echo "${picturecount}" "${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount} ${contactcount} ${apcontactcount} ${diasporacontactcount}" >"${usrfile}" + #Previous line clearance + #Measure length of string, blank only the excess + final_string_length_left="${#response_left}" + final_string_length_right="${#response}" + final_string_length=$((final_string_length_left + final_string_length_right)) + blank_string="" + columns_length="${COLUMNS}" + while [[ "${final_string_length}" -gt "${columns_length}" ]]; do + columns_length=$((columns_length + COLUMNS)) done - postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();") - postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();") - postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();") - posttagcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}; select row_count();") - postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") - postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") - photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}; select row_count();") - contactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}; select row_count();") - apcontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}; select row_count();") - diasporacontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}; select row_count();") - printf "post-thread: %s " "${postthreadcount}" - printf "post-thread-user: %s " "${postthreadusercount}" - printf "post-user: %s " "${postusercount}" - printf "post-tag: %s " "${posttagcount}" - printf "post-content: %s " "${postcontentcount}" - printf "post: %s " "${postcount}" - printf "photo: %s " "${photocount}" - printf "contact: %s " "${contactcount}" - printf "apcontact: %s " "${apcontactcount}" - printf "diaspora-contact: %s " "${diasporacontactcount}" - printf "\r\n" + blank_string_length=$((columns_length - final_string_length)) + for ((count = 0; count < "${blank_string_length}"; count++)); do + blank_string=$(printf "%s " "${blank_string}") + done + response=$(printf "%s%s%s" "${response_left}" "${blank_string}" "${response}") + for ((count = 0; count < "${final_string_length}"; count++)); do + response=$(printf "%s\b" "${response}") + done + printf "%s\r" "${response}" } #Check if our dependencies are installed if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then date - "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`contact_baseurl\` (\`baseurl\`)" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` add index if not exists \`post_user_id\` (\`author-id\`, \`causer-id\`, \`owner-id\`)" + if [[ "${intense_optimizations}" -gt 0 ]]; then + "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`contact_baseurl\` (\`baseurl\`)" + "${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` add index if not exists \`post_user_id\` (\`author-id\`, \`causer-id\`, \`owner-id\`)" + fi if [[ ! -f "${tmpfile}" ]]; then echo "Listing sites" siteslist=$("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl, protocol from contact where baseurl != ''" | sort -b -f -n | sed -e "s/http:/https:/g" | uniq -i) @@ -117,24 +156,30 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty u=$(sort -n "${idsdownfile}" | uniq) echo "${u}" >"${idsdownfile}" fi - while read -r lineb nick baseurl; do + touch "${usrfile}" + echo "0 0 0 0 0 0 0 0 0 0 0" >"${usrfile}" + while read -r id nick baseurl; do #The community no longer exists, delete - loop_3 "${lineb}" "${nick}" "${baseurl}" & + loop_3 "${id}" "${nick}" "${baseurl}" & if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; then wait -n fi wait done <"${idsdownfile}" + printf "\r\n" rm "${tmpfile}" 2>/dev/null rm "${idsdownfile}" 2>/dev/null - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-thread\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-thread-user\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-tag\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`photo\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` auto_increment = 1" - "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` drop index \`contact_baseurl\`" - "${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` drop index \`post_user_id\`" + rm "${usrfile}" 2>/dev/null + "${dbengine}" "${db}" -N -B -q -e "alter table \`post-thread\` auto_increment = 1; \ + alter table \`post-thread-user\` auto_increment = 1; \ + alter table \`post-user\` auto_increment = 1; \ + alter table \`post-tag\` auto_increment = 1; \ + alter table \`post\` auto_increment = 1; \ + alter table \`photo\` auto_increment = 1; \ + alter table \`contact\` auto_increment = 1" + if [[ "${intense_optimizations}" -gt 0 ]]; then + "${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` drop index \`contact_baseurl\`" + "${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` drop index \`post_user_id\`" + fi date fi From 78e26fce2d67b4b657deb5531ec78470da84905d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Sol=C3=ADs?= Date: Wed, 5 Mar 2025 09:22:28 -0600 Subject: [PATCH 3/3] fix: Limit filter for worker queue selection to not-done items --- friendica-non-follower-featured-posts.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/friendica-non-follower-featured-posts.sh b/friendica-non-follower-featured-posts.sh index d7f56b6..7bf5d4a 100755 --- a/friendica-non-follower-featured-posts.sh +++ b/friendica-non-follower-featured-posts.sh @@ -3,7 +3,7 @@ limit=1000 ca=${limit} camax=0 until [[ ${ca} -lt ${limit} ]]; do - ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"UpdateContact\" limit ${ca}; select row_count();") + ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"UpdateContact\" and \`done\` = 0 limit ${ca}; select row_count();") camax=$((camax + ca)) printf "\rUpdateContact\t\t%s\r" "${camax}" done @@ -13,7 +13,7 @@ printf "\rUpdateContact\t\t%s\n\r" "${camax}" cb=${limit} cbmax=0 until [[ ${cb} -lt ${limit} ]]; do - cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"ContactDiscovery\" limit ${cb}; select row_count();") + cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"ContactDiscovery\" and \`done\` = 0 limit ${cb}; select row_count();") cbmax=$((cbmax + cb)) printf "\rContactDiscovery\t%s\r" "${cbmax}" done @@ -23,7 +23,7 @@ printf "\rContactDiscovery\t%s\n\r" "${cbmax}" cc=${limit} ccmax=0 until [[ ${cc} -lt ${limit} ]]; do - cc=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_addcontact (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\`= \"AddContact\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_addcontact) limit ${cc}; select row_count();") + cc=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_addcontact (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\`= \"AddContact\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_addcontact) and \`done\` = 0 limit ${cc}; select row_count();") ccmax=$((ccmax + cc)) printf "\rAddContact \t%s\r" "${ccmax}" done @@ -33,7 +33,7 @@ printf "\rAddContact \t%s\n\r" "${ccmax}" cd=${limit} cdmax=0 until [[ ${cd} -lt ${limit} ]]; do - cd=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_updategserver (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\` = \"UpdateGServer\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_updategserver) limit ${cd}; select row_count();") + cd=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_updategserver (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\` = \"UpdateGServer\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_updategserver) and \`done\` = 0 limit ${cd}; select row_count();") cdmax=$((cdmax + cd)) printf "\rUpdateGServer\t\t%s\r" "${cdmax}" done @@ -43,7 +43,7 @@ printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}" ce=${limit} cemax=0 until [[ ${ce} -lt ${limit} ]]; do - ce=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_fetchfeaturedposts (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\`= \"FetchFeaturedPosts\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_fetchfeaturedposts) limit ${ce}; select row_count();") + ce=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_fetchfeaturedposts (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\`= \"FetchFeaturedPosts\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_fetchfeaturedposts) and \`done\` = 0 limit ${ce}; select row_count();") cemax=$((cemax + ce)) printf "\rFetchFeaturedPosts\t%s\r" "${cemax}" done