diff --git a/friendica-delete-old-users.sh b/friendica-delete-old-users.sh index 4668190..932b9c0 100755 --- a/friendica-delete-old-users.sh +++ b/friendica-delete-old-users.sh @@ -19,7 +19,8 @@ loop() { #Find the pictures in the avatar folders and delete them "${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro; do #If stored in avatar folder - if ! grep -v -q "${url}/avatar" <(echo "${photo}"); then + if grep -v -q "${url}/avatar" <(echo "${photo}"); then + #if [[ -z "${isavatar}" ]] phototrimmed=$(echo "${photo}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") rm -rfv "${phototrimmed}" thumbtrimmed=$(echo "${thumb}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") @@ -28,16 +29,17 @@ loop() { rm -rfv "${microtrimmed}" fi done - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread-user\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-user\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`;" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}" + printf "post-thread: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();") + printf "post-thread-user: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();") + printf "post-user: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();") + printf "post-tag: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}; select row_count();") + printf "post-content: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") + printf "post: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") + printf "photo: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}; select row_count();") + printf "contact: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}; select row_count();") + printf "apcontact: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}; select row_count();") + printf "diaspora-contact: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}; select row_count();") + printf "\r\n" } #Check if our dependencies are installed diff --git a/friendica-find-missing-servers.sh b/friendica-find-missing-servers.sh index 18a40c7..b0d149d 100755 --- a/friendica-find-missing-servers.sh +++ b/friendica-find-missing-servers.sh @@ -16,18 +16,18 @@ avatarfolder=/var/www/friendica/avatar avatarfolderescaped=${avatarfolder////\\/} loop_1() { site=$(echo "${sites}" | sed -e "s/http[s]*:\/\///g") - if [[ "${protocol}" == "apub" ]]; then - #For ActivityPub sites, we test the well-known Webfinger - #We also need a valid (known) user for the Webfinger test - user=$("${dbengine}" "${db}" -N -B -q -e "select \`addr\` from contact where baseurl = \"http://${site}\" or url = \"http://${site}\" or baseurl = \"https://${site}\" or url = \"https://${site}\" limit 1") - site_test="https://${site}/.well-known/webfinger?resource=acct:${user}" - #If the return message is in "application/jrd+json" format, the site is still up - #If the message contains a reference to Cloudflare, we don't add it to the list either, just in case - if ! grep -q -e "application/jrd+json" -e "HTTP.*200" -e "cloudflare" <(curl -s -L -I -m 30 -X HEAD "${site_test}"); then - echo "${site}" >>"${tmpfile}" - echo "Added ${site}" - fi - fi + # if [[ "${protocol}" == "apub" ]]; then + # #For ActivityPub sites, we test the well-known Webfinger + # #We also need a valid (known) user for the Webfinger test + # user=$("${dbengine}" "${db}" -N -B -q -e "select \`addr\` from contact where baseurl = \"http://${site}\" or url = \"http://${site}\" or baseurl = \"https://${site}\" or url = \"https://${site}\" limit 1") + # site_test="https://${site}/.well-known/webfinger?resource=acct:${user}" + # #If the return message is in "application/jrd+json" format, the site is still up + # #If the message contains a reference to Cloudflare, we don't add it to the list either, just in case + # if ! grep -q -e "application/jrd+json" -e "HTTP.*200" -e "cloudflare" <(curl -s -L -I -m 30 -X HEAD "${site_test}"); then + # echo "${site}" >>"${tmpfile}" + # echo "Added ${site}" + # fi + # fi #This is mostly for RSS feeds, we only check whether the site itself is up #Skip check if the message contains a reference to Cloudflare if [[ "${protocol}" != "bsky" ]]; then @@ -58,26 +58,14 @@ loop_3() { rm -rfv "${microtrimmed}" fi done - printf "post-thread:" - "${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();" - printf "post-thread-user:" - "${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();" - printf "post-user": - "${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();" - printf "post-tag:" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}; select row_count();" - printf "post-content:" - "${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" - printf "post:" - "${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" - printf "photo:" - "${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}; select row_count();" - printf "contact:" - "${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}; select row_count();" - printf "apcontact:" - "${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}; select row_count();" - printf "diaspora-contact:" - "${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}; select row_count();" + printf "post-thread: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();") printf "post-thread-user: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();") + printf "post-user: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();") printf "post-tag: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}; select row_count();") + printf "post-content: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") printf "post: %s " $("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();") + printf "photo: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}; select row_count();") + printf "contact: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}; select row_count();") + printf "apcontact: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${lineb}; select row_count();") + printf "diaspora-contact: %s " $("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${lineb}; select row_count();") + printf "\r\n" } #Check if our dependencies are installed diff --git a/friendica-fix-avatar-permissions.sh b/friendica-fix-avatar-permissions.sh index 274d2a1..b8072c4 100755 --- a/friendica-fix-avatar-permissions.sh +++ b/friendica-fix-avatar-permissions.sh @@ -31,8 +31,8 @@ loop_1() { tmppic="/tmp/temp_$(date +%s).webp" nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null if [[ -f "${tmppic}" ]]; then - size_new=$(stat -c%s "${tmppic}" || echo 0) - size_original=$(stat -c%s "${p}" || echo 0) + size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0) + size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0) if [[ "${size_original}" -gt "${size_new}" ]]; then mv "${tmppic}" "${p}" #&> /dev/null else diff --git a/friendica-non-follower-featured-posts.sh b/friendica-non-follower-featured-posts.sh index 7c9478e..d7f56b6 100755 --- a/friendica-non-follower-featured-posts.sh +++ b/friendica-non-follower-featured-posts.sh @@ -25,9 +25,9 @@ ccmax=0 until [[ ${cc} -lt ${limit} ]]; do cc=$(sudo mariadb friendica -B -N -q -e "create temporary table tmp_addcontact (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)); delete from workerqueue where \`command\`= \"AddContact\" and regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from tmp_addcontact) limit ${cc}; select row_count();") ccmax=$((ccmax + cc)) - printf "\rAddContact\t%s\r" "${ccmax}" + printf "\rAddContact \t%s\r" "${ccmax}" done -printf "\rAddContact \t%s\n\r" "${ccmax}" +printf "\rAddContact \t%s\n\r" "${ccmax}" #echo "AddContact $ccmax" cd=${limit} diff --git a/friendica-remove-invalid-photos.sh b/friendica-remove-invalid-photos.sh index 80e241e..284d3ad 100755 --- a/friendica-remove-invalid-photos.sh +++ b/friendica-remove-invalid-photos.sh @@ -35,6 +35,11 @@ lastid=${4:-"0"} if [[ ! "${lastid}" =~ ^[0-9]+$ || $((10#${lastid})) -le 0 ]]; then lastid=0 fi +#Command-line parameter number 5: whether to enable the lockfile and the item counter, or disable it for a performance boost (0=off). Defaults to 1=on. +lockfile_enabled=${5:-"1"} +if [[ "${lockfile_enabled}" != "0" && "${lockfile_enabled}" != "1" ]]; then + lockfile_enabled=1 +fi nfolder="/tmp/friendica-remove-invalid-photos" nfile="${nfolder}/n$(date +%s).csv" nlock="${nfolder}/n$(date +%s).lock" @@ -68,8 +73,12 @@ if [[ "${intensive_optimizations}" -gt 0 ]]; then indexlength=$(("${#url}" + 16)) "${dbengine}" "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = ''))") + #dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%')") + #dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = ''))") else dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") + #dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%') and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") + #dbcount=$("${dbengine}" "${db}" -B -N -q -e "select count(\`id\`) from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") fi loop() { @@ -80,51 +89,53 @@ loop() { nl=0 error_found=0 #Lockfile-protected read - r=0 - if [[ "${time_counter}" -eq 1 ]]; then - t_r=$(($(date +%s%N) / 1000000)) - fi - #Fallback in case the process dies - ( - sleep "${timeout}"s - if [[ "${r}" -eq 0 ]]; then - rm -rf "${nlock}" + if [[ ${lockfile_enabled} -eq 1 ]]; then + r=0 + if [[ "${time_counter}" -eq 1 ]]; then + t_r=$(($(date +%s%N) / 1000000)) fi - ) & - while [[ "${r}" -eq 0 ]]; do - if [[ ! -d "${nfolder}" ]]; then - mkdir "${nfolder}" - fi - if [[ ! -f "${nlock}" ]]; then - touch "${nlock}" - fi - if [[ -f "${nlock}" && $(cat "${nlock}" 2>/dev/null || echo 0) == "" ]]; then - rm -rf "${nlock}" && touch "${nlock}" && echo "${id}" | tee "${nlock}" &>/dev/null - if [[ -f "${nlock}" && $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${id}" ]]; then - read -r n_tmp nt_tmp <"${nfile}" || break - if [[ -n "${n_tmp}" && -n "${nt_tmp}" ]]; then - n="${n_tmp}" - nt="${nt_tmp}" - if [[ -f "${nlock}" ]]; then + #Fallback in case the process dies + ( + sleep "${timeout}"s + if [[ "${r}" -eq 0 ]]; then + rm -rf "${nlock}" + fi + ) & + while [[ "${r}" -eq 0 ]]; do + if [[ ! -d "${nfolder}" ]]; then + mkdir "${nfolder}" + fi + if [[ ! -f "${nlock}" ]]; then + touch "${nlock}" + fi + if [[ -f "${nlock}" && $(cat "${nlock}" 2>/dev/null || echo 0) == "" ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "${id}" | tee "${nlock}" &>/dev/null + if [[ -f "${nlock}" && $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${id}" ]]; then + read -r n_tmp nt_tmp <"${nfile}" || break + if [[ -n "${n_tmp}" && -n "${nt_tmp}" ]]; then + n="${n_tmp}" + nt="${nt_tmp}" + if [[ -f "${nlock}" ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" + fi + r=1 + fi + elif [[ -f "${nlock}" ]]; then + nlm=$(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) + if [[ -n "${nlm}" ]]; then + nlmt=$((10#${nlm})) + if [[ "${nlmt}" -lt $((id + limit)) ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" + fi + else rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" fi - r=1 - fi - elif [[ -f "${nlock}" ]]; then - nlm=$(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) - if [[ -n "${nlm}" ]]; then - nlmt=$((10#${nlm})) - if [[ "${nlmt}" -lt $((id + limit)) ]]; then - rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" - fi - else - rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" fi fi + done + if [[ "${time_counter}" -eq 1 ]]; then + result_string=$(printf "%s R%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_r))) fi - done - if [[ "${time_counter}" -eq 1 ]]; then - result_string=$(printf "%s R%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_r))) fi if [[ -n "${id}" ]]; then while read -r avatar photo thumb micro; do @@ -238,49 +249,57 @@ loop() { lastid="${id}" done < <("${dbengine}" "${db}" -B -N -q -e "select \`avatar\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}") fi - w=0 - if [[ "${time_counter}" -eq 1 ]]; then - t_w=$(($(date +%s%N) / 1000000)) - fi - while [[ "${w}" -eq 0 ]]; do - if [[ ! -d "${nfolder}" ]]; then - mkdir "${nfolder}" + if [[ ${lockfile_enabled} -eq 1 ]]; then + w=0 + if [[ "${time_counter}" -eq 1 ]]; then + t_w=$(($(date +%s%N) / 1000000)) fi - if [[ ! -f "${nlock}" ]]; then - #n is increased only if error_found = 1 - touch "${nlock}" - fi - if [[ -f "${nlock}" && $(cat "${nlock}" 2>/dev/null || echo "") == "" ]]; then - rm -rf "${nlock}" && touch "${nlock}" && echo "${id}" | tee "${nlock}" &>/dev/null - if [[ -f "${nlock}" && $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${id}" ]]; then - read -r n_tmp nt_tmp <"${nfile}" || break - if [[ -n "${n_tmp}" && -n "${nt_tmp}" ]]; then - if [[ $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${id}" ]]; then - if [[ "${n_tmp}" -ge "${n}" ]]; then - n=$((n_tmp + error_found)) - else - n=$((n + error_found)) + while [[ "${w}" -eq 0 ]]; do + if [[ ! -d "${nfolder}" ]]; then + mkdir "${nfolder}" + fi + if [[ ! -f "${nlock}" ]]; then + #n is increased only if error_found = 1 + touch "${nlock}" + fi + if [[ -f "${nlock}" && $(cat "${nlock}" 2>/dev/null || echo "") == "" ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "${id}" | tee "${nlock}" &>/dev/null + if [[ -f "${nlock}" && $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${id}" ]]; then + read -r n_tmp nt_tmp <"${nfile}" || break + if [[ -n "${n_tmp}" && -n "${nt_tmp}" ]]; then + if [[ $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${id}" ]]; then + if [[ "${n_tmp}" -ge "${n}" ]]; then + n=$((n_tmp + error_found)) + else + n=$((n + error_found)) + fi + if [[ "${nt_tmp}" -ge "${nt}" ]]; then + nt=$((nt_tmp + 1)) + else + nt=$((nt + 1)) + fi + echo "${n} ${nt}" >"${nfile}" + if [[ -f "${nlock}" ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" + fi + w=1 fi - if [[ "${nt_tmp}" -ge "${nt}" ]]; then - nt=$((nt_tmp + 1)) - else - nt=$((nt + 1)) - fi - echo "${n} ${nt}" >"${nfile}" - if [[ -f "${nlock}" ]]; then - rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" - fi - w=1 fi fi fi + done + if [[ "${time_counter}" -eq 1 ]]; then + result_string=$(printf "%s W%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_w))) fi - done + fi if [[ "${time_counter}" -eq 1 ]]; then - result_string=$(printf "%s W%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_w))) result_string=$(printf "%s T%dms" "${result_string}" $(($(($(date +%s%N) / 1000000)) - t_id))) fi - final_string=$(printf "E%8d F%8d/%8d T%8d/%8d %s" "${n}" "${nt}" "${dbcount}" "${lastid}" "${maxid}" "${result_string}") + final_string="" + if [[ ${lockfile_enabled} -eq 1 ]]; then + final_string=$(printf "E%8d F%8d/%8d " "${n}" "${nt}" "${dbcount}") + fi + final_string=$(printf "%sT%8d/%8d %s" "${final_string}" "${lastid}" "${maxid}" "${result_string}") final_string_length="${#final_string}" #Previous line clearance #Measure length of string, blank only the excess @@ -307,8 +326,12 @@ echo "${n} ${nt}" >"${nfile}" c="" if [[ "${intensive_optimizations}" -gt 0 ]]; then c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) order by id limit ${limit}") + #c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%') order by id limit ${limit}") + #c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) order by id limit ${limit}") else c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%' or (\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") + #c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like 'https:\/\/${url}/avatar/%') and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") + #c=$("${dbengine}" "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and ((\`photo\` = '' and not \`avatar\` = '') or (\`avatar\` = '' and not \`photo\` = '')) and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") fi while read -r id; do if [[ -n "${id}" && $((10#${id})) -ge "${lastid}" ]]; then @@ -324,42 +347,43 @@ while read -r id; do done done < <(echo "${c}") wait -#Read data before next iteration -rl=0 -( - sleep 60s - if [[ "${rl}" -eq 0 ]]; then rm -rf "${nlock}"; fi -) & -while [[ "${rl}" -eq 0 ]]; do - if [[ ! -f "${nlock}" ]]; then - touch "${nlock}" - fi - if [[ -f "${nlock}" && $(cat "${nlock}" 2>/dev/null || echo "") == "" ]]; then - rm -rf "${nlock}" && touch "${nlock}" && echo "${lastid}" | tee "${nlock}" &>/dev/null - if [[ -f "${nlock}" && $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${lastid}" ]]; then - read -r n_tmp_l nt_tmp_l <"${nfile}" || break - if [[ -n "${n_tmp_l}" && -n "${nt_tmp_l}" ]]; then - n="${n_tmp_l}" - nt="${nt_tmp_l}" - if [[ -f "${nlock}" ]]; then +if [[ ${lockfile_enabled} -eq 1 ]]; then + #Read data before next iteration + rl=0 + ( + sleep 60s + if [[ "${rl}" -eq 0 ]]; then rm -rf "${nlock}"; fi + ) & + while [[ "${rl}" -eq 0 ]]; do + if [[ ! -f "${nlock}" ]]; then + touch "${nlock}" + fi + if [[ -f "${nlock}" && $(cat "${nlock}" 2>/dev/null || echo "") == "" ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "${lastid}" | tee "${nlock}" &>/dev/null + if [[ -f "${nlock}" && $(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) == "${lastid}" ]]; then + read -r n_tmp_l nt_tmp_l <"${nfile}" || break + if [[ -n "${n_tmp_l}" && -n "${nt_tmp_l}" ]]; then + n="${n_tmp_l}" + nt="${nt_tmp_l}" + if [[ -f "${nlock}" ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" + fi + rl=1 + fi + elif [[ -f "${nlock}" ]]; then + nlm=$(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) + if [[ -n "${nlm}" ]]; then + nlmt=$((10#${nlm})) + if [[ "${nlmt}" -lt $((id + limit)) ]]; then + rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" + fi + else rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" fi - rl=1 - fi - elif [[ -f "${nlock}" ]]; then - nlm=$(grep -e "[0-9]" "${nlock}" 2>/dev/null || echo 0) - if [[ -n "${nlm}" ]]; then - nlmt=$((10#${nlm})) - if [[ "${nlmt}" -lt $((id + limit)) ]]; then - rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" - fi - else - rm -rf "${nlock}" && touch "${nlock}" && echo "" >"${nlock}" fi fi - fi -done -#done + done +fi if [[ -f "${nfile}" ]]; then rm -rf "${nfile}" fi