diff --git a/friendica-clean-database.sh b/friendica-clean-database.sh index 7b21aa7..08474fc 100755 --- a/friendica-clean-database.sh +++ b/friendica-clean-database.sh @@ -1,225 +1,186 @@ #!/bin/bash interval=7 limit=1000 +tmpfile=/tmp/fcb +dbengine=mariadb +db=friendica -touch /tmp/fcb +touch "${tmpfile}" echo "tmp_post_origin_deleted" tmp_post_origin_deleted_q="${limit}" tmp_post_origin_deleted_current_uri_id=0 -until [[ "${tmp_post_origin_deleted_q}" -lt "${limit}" ]] -do - tmp_post_origin_deleted=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_origin_deleted_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_origin_deleted=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) \ AND ( \`uri-id\` > ${tmp_post_origin_deleted_current_uri_id} ) \ - ORDER BY \`uri-id\`, \`uid\` LIMIT ${limit}"); + ORDER BY \`uri-id\`, \`uid\` LIMIT ${limit}") tmp_post_origin_deleted_q=$(echo "${tmp_post_origin_deleted}" | grep -c '.') #echo "${tmp_post_origin_deleted_q}" - if [[ "${tmp_post_origin_deleted_q}" -gt 0 ]] - then - echo "${tmp_post_origin_deleted}" | while read -r uri_id uid - do - if [[ -n "${uri_id}" && -n "${uid}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post-origin\` WHERE \`parent-uri-id\` = ${uri_id} AND \`uid\` = ${uid}" + if [[ "${tmp_post_origin_deleted_q}" -gt 0 ]]; then + echo "${tmp_post_origin_deleted}" | while read -r uri_id uid; do + if [[ -n "${uri_id}" && -n "${uid}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-origin\` WHERE \`parent-uri-id\` = ${uri_id} AND \`uid\` = ${uid}" #echo "${uri_id} ${uid}" - echo "${uri_id}" > /tmp/fcb + echo "${uri_id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_post_origin_deleted}") && "${tmp_post_origin_deleted_current_uid}" -le $(tac $(echo "${tmp_post_origin_deleted}") | grep -m 1 '.') ]] - then - tmp_post_origin_deleted_current_uid=$(tac $(echo "${tmp_post_origin_deleted}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_post_origin_deleted_current_uri_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_post_origin_deleted_current_uid=$(cat /tmp/fcb) - fi - echo "${tmp_post_origin_deleted_q} item(s) deleted until ${tmp_post_origin_deleted_current_uri_id} ${tmp_post_origin_deleted_current_uid}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_origin_deleted_q} item(s) deleted until ${tmp_post_origin_deleted_current_uri_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_post_user_deleted" tmp_post_user_deleted_q="${limit}" tmp_post_user_deleted_current_uri_id=0 -until [[ "${tmp_post_user_deleted_q}" -lt "${limit}" ]] -do - tmp_post_user_deleted=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_user_deleted_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_user_deleted=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`deleted\` AND \`edited\` < (CURDATE() - INTERVAL ${interval} DAY) \ - AND \`uri-id\` > ${tmp_post_user_deleted_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}"); + AND \`uri-id\` > ${tmp_post_user_deleted_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}") tmp_post_user_deleted_q=$(echo "${tmp_post_user_deleted}" | grep -c '.') #echo "${tmp_post_user_deleted_q}" - if [[ "${tmp_post_user_deleted_q}" -gt 0 ]] - then - echo "${tmp_post_user_deleted}" | while read -r uri_id - do - if [[ -n "${uri_id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}" + if [[ "${tmp_post_user_deleted_q}" -gt 0 ]]; then + echo "${tmp_post_user_deleted}" | while read -r uri_id; do + if [[ -n "${uri_id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}" #echo "${uri_id}" - echo "${uri_id}" > /tmp/fcb + echo "${uri_id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_post_user_deleted}") && "${tmp_post_user_deleted_current_uri_id}" -lt $(tac $(echo "${tmp_post_user_deleted}") | grep -m 1 '.') ]] - then - tmp_post_user_deleted_current_uri_id=$(tac $(echo "${tmp_post_user_deleted}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_post_user_deleted_current_uri_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_post_user_deleted_current_uri_id=$(cat /tmp/fcb) - fi - echo "${tmp_post_user_deleted_q} item(s) deleted until ${tmp_post_user_deleted_current_uri_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_user_deleted_q} item(s) deleted until ${tmp_post_user_deleted_current_uri_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_post_uri_id_not_in_post_user" tmp_post_uri_id_not_in_post_user_q="${limit}" tmp_post_uri_id_not_in_post_user_current_uri_id=0 -until [[ "${tmp_post_uri_id_not_in_post_user_q}" -lt "${limit}" ]] -do - tmp_post_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_uri_id_not_in_post_user=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) \ - AND \`uri-id\` > ${tmp_post_uri_id_not_in_post_user_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}"); + AND \`uri-id\` > ${tmp_post_uri_id_not_in_post_user_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}") tmp_post_uri_id_not_in_post_user_q=$(echo "${tmp_post_uri_id_not_in_post_user}" | grep -c '.') #echo "${tmp_post_uri_id_not_in_post_user_q}" - if [[ "${tmp_post_uri_id_not_in_post_user_q}" -gt 0 ]] - then - echo "${tmp_post_uri_id_not_in_post_user}" | while read -r uri_id - do - if [[ -n "${uri_id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post\` WHERE \`uri-id\` = ${uri_id}" + if [[ "${tmp_post_uri_id_not_in_post_user_q}" -gt 0 ]]; then + echo "${tmp_post_uri_id_not_in_post_user}" | while read -r uri_id; do + if [[ -n "${uri_id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post\` WHERE \`uri-id\` = ${uri_id}" #echo "${uri_id}" - echo "${uri_id}" > /tmp/fcb + echo "${uri_id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_post_uri_not_in_post_user}") && "${tmp_post_uri_id_not_in_post_user_current_uri_id}" -lt $(tac $(echo "${tmp_post_uri_id_not_in_post_user}") | grep -m 1 '.') ]] - then - tmp_post_uri_id_not_in_post_user_current_uri_id=$(tac $(echo "${tmp_post_uri_id_not_in_post_user}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_post_uri_id_not_in_post_user_current_uri_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_post_uri_id_not_in_post_user_current_uri_id=$(cat /tmp/fcb) - fi - echo "${tmp_post_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_uri_id_not_in_post_user_current_uri_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_uri_id_not_in_post_user_current_uri_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_post_content_uri_id_not_in_post_user" tmp_post_content_uri_id_not_in_post_user_q="${limit}" tmp_post_content_uri_id_not_in_post_user_current_uri_id=0 -until [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -lt "${limit}" ]] -do - tmp_post_content_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_content_uri_id_not_in_post_user=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post-content\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) \ - AND \`uri-id\` > ${tmp_post_content_uri_id_not_in_post_user_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}"); + AND \`uri-id\` > ${tmp_post_content_uri_id_not_in_post_user_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}") tmp_post_content_uri_id_not_in_post_user_q=$(echo "${tmp_post_content_uri_id_not_in_post_user}" | grep -c '.') #echo "${tmp_post_content_uri_id_not_in_post_user_q}" - if [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -gt 0 ]] - then - echo "${tmp_post_content_uri_id_not_in_post_user}" | while read -r uri_id - do - if [[ -n "${uri_id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post-content\` WHERE \`uri-id\` = ${uri_id}" + if [[ "${tmp_post_content_uri_id_not_in_post_user_q}" -gt 0 ]]; then + echo "${tmp_post_content_uri_id_not_in_post_user}" | while read -r uri_id; do + if [[ -n "${uri_id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-content\` WHERE \`uri-id\` = ${uri_id}" #echo "${uri_id}" - echo "${uri_id}" > /tmp/fcb + echo "${uri_id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_post_content_uri_not_in_post_user}") && "${tmp_post_content_uri_id_not_in_post_user_current_uri_id}" -lt $(tac $(echo "${tmp_post_content_uri_id_not_in_post_user}") | grep -m 1 '.') ]] - then - tmp_post_content_uri_id_not_in_post_user_current_uri_id=$(tac $(echo "${tmp_post_content_uri_id_not_in_post_user}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_post_content_uri_id_not_in_post_user_current_uri_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_post_content_uri_id_not_in_post_user_current_uri_id=$(cat /tmp/fcb) - fi - echo "${tmp_post_content_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_content_uri_id_not_in_post_user_current_uri_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_content_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_content_uri_id_not_in_post_user_current_uri_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_post_thread_uri_id_not_in_post_user" tmp_post_thread_uri_id_not_in_post_user_q="${limit}" tmp_post_thread_uri_id_not_in_post_user_current_uri_id=0 -until [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -lt "${limit}" ]] -do - tmp_post_thread_uri_id_not_in_post_user=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_thread_uri_id_not_in_post_user=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post-thread\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`) \ - AND \`uri-id\` > ${tmp_post_thread_uri_id_not_in_post_user_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}"); + AND \`uri-id\` > ${tmp_post_thread_uri_id_not_in_post_user_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}") tmp_post_thread_uri_id_not_in_post_user_q=$(echo "${tmp_post_thread_uri_id_not_in_post_user}" | grep -c '.') #echo "${tmp_post_thread_uri_id_not_in_post_user_q}" - if [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -gt 0 ]] - then - echo "${tmp_post_thread_uri_id_not_in_post_user}" | while read -r uri_id - do - if [[ -n "${uri_id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post-thread\` WHERE \`uri-id\` = ${uri_id}" + if [[ "${tmp_post_thread_uri_id_not_in_post_user_q}" -gt 0 ]]; then + echo "${tmp_post_thread_uri_id_not_in_post_user}" | while read -r uri_id; do + if [[ -n "${uri_id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-thread\` WHERE \`uri-id\` = ${uri_id}" #echo "${uri_id}" - echo "${uri_id}" > /tmp/fcb + echo "${uri_id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_post_thread_uri_id_not_in_post_user}") && "${tmp_post_thread_uri_id_not_in_post_user_current_uri_id}" -lt $(tac $(echo "${tmp_post_thread_uri_id_not_in_post_user}") | grep -m 1 '.') ]] - then - tmp_post_thread_uri_id_not_in_post_user_current_uri_id=$(tac $(echo "${tmp_post_thread_uri_id_not_in_post_user}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_post_thread_uri_id_not_in_post_user_current_uri_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_post_thread_uri_id_not_in_post_user_current_uri_id=$(cat /tmp/fcb) - fi - echo "${tmp_post_thread_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_thread_uri_id_not_in_post_user_current_uri_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_thread_uri_id_not_in_post_user_q} item(s) deleted until ${tmp_post_thread_uri_id_not_in_post_user_current_uri_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_post_user_uri_id_not_in_post" tmp_post_user_uri_id_not_in_post_q="${limit}" tmp_post_user_uri_id_not_in_post_current_uri_id=0 -until [[ "${tmp_post_user_uri_id_not_in_post_q}" -lt "${limit}" ]] -do - tmp_post_user_uri_id_not_in_post=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_user_uri_id_not_in_post_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_post_user_uri_id_not_in_post=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`) \ - AND \`uri-id\` > ${tmp_post_user_uri_id_not_in_post_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}"); + AND \`uri-id\` > ${tmp_post_user_uri_id_not_in_post_current_uri_id} ORDER BY \`uri-id\` LIMIT ${limit}") tmp_post_user_uri_id_not_in_post_q=$(echo "${tmp_post_user_uri_id_not_in_post}" | grep -c '.') #echo "${tmp_post_user_uri_id_not_in_post_q}" - if [[ "${tmp_post_user_uri_id_not_in_post_q}" -gt 0 ]] - then - echo "${tmp_post_user_uri_id_not_in_post}" | while read -r uri_id - do - if [[ -n "${uri_id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}" + if [[ "${tmp_post_user_uri_id_not_in_post_q}" -gt 0 ]]; then + echo "${tmp_post_user_uri_id_not_in_post}" | while read -r uri_id; do + if [[ -n "${uri_id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-user\` WHERE \`uri-id\` = ${uri_id}" #echo "${uri_id}" - echo "${uri_id}" > /tmp/fcb + echo "${uri_id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_post_user_uri_id_not_in_post}") && "${tmp_post_user_uri_id_not_in_post_current_uri_id}" -lt $(tac $(echo "${tmp_post_user_uri_id_not_in_post}") | grep -m 1 '.') ]] - then - tmp_post_user_uri_id_not_in_post_current_uri_id=$(tac $(echo "${tmp_post_user_uri_id_not_in_post}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_post_user_uri_id_not_in_post_current_uri_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_post_user_uri_id_not_in_post_current_uri_id=$(cat /tmp/fcb) - fi - echo "${tmp_post_user_uri_id_not_in_post_q} item(s) deleted until ${tmp_post_user_uri_id_not_in_post_current_uri_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_post_user_uri_id_not_in_post_q} item(s) deleted until ${tmp_post_user_uri_id_not_in_post_current_uri_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_item_uri_not_in_valid_post_thread" tmp_item_uri_not_in_valid_post_thread_q="${limit}" tmp_item_uri_not_in_valid_post_thread_current_id=0 -until [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -lt "${limit}" ]] -do - tmp_item_uri_not_in_valid_post_thread=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_item_uri_not_in_valid_post_thread=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL ${interval} DAY) \ AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` WHERE (\`mention\` OR \`starred\` OR \`wall\`) \ AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \ @@ -231,115 +192,94 @@ do AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` WHERE (\`origin\` OR \`event-id\` != 0 OR \`post-type\` = 128) \ AND \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \ AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \ - AND \`uri-id\` > ${tmp_item_uri_not_in_valid_post_thread_current_id} ORDER BY \`uri-id\` LIMIT ${limit}"); + AND \`uri-id\` > ${tmp_item_uri_not_in_valid_post_thread_current_id} ORDER BY \`uri-id\` LIMIT ${limit}") tmp_item_uri_not_in_valid_post_thread_q=$(echo "${tmp_item_uri_not_in_valid_post_thread}" | grep -c '.') #echo "${tmp_item_uri_not_in_valid_post_thread_q}" - if [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -gt 0 ]] - then - echo "${tmp_item_uri_not_in_valid_post_thread}" | while read -r id - do - if [[ -n "${id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" + if [[ "${tmp_item_uri_not_in_valid_post_thread_q}" -gt 0 ]]; then + echo "${tmp_item_uri_not_in_valid_post_thread}" | while read -r id; do + if [[ -n "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" #echo "${id}" - echo "${id}" > /tmp/fcb + echo "${id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_item_uri_not_in_valid_post_thread}") && "${tmp_item_uri_not_in_valid_post_thread_current_id}" -lt $(tac $(echo "${tmp_item_uri_not_in_valid_post_thread}") | grep -m 1 '.') ]] - then - tmp_item_uri_not_in_valid_post_thread_current_id=$(tac $(echo "${tmp_item_uri_not_in_valid_post_thread}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_item_uri_not_in_valid_post_thread_current_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_item_uri_not_in_valid_post_thread_current_id=$(cat /tmp/fcb) - fi - echo "${tmp_item_uri_not_in_valid_post_thread_q} item(s) deleted until ${tmp_item_uri_not_in_valid_post_thread_current_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_item_uri_not_in_valid_post_thread_q} item(s) deleted until ${tmp_item_uri_not_in_valid_post_thread_current_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_item_uri_not_in_valid_post_user" tmp_item_uri_not_in_valid_post_user_q="${limit}" tmp_item_uri_not_in_valid_post_user_current_id=0 -until [[ "${tmp_item_uri_not_in_valid_post_user_q}" -lt "${limit}" ]] -do - tmp_item_uri_not_in_valid_post_user=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_item_uri_not_in_valid_post_user_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_item_uri_not_in_valid_post_user=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 \ AND \`received\` < (CURDATE() - INTERVAL ${interval} DAY) AND NOT \`uri-id\` IN ( SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 \ AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` ) AND NOT \`uri-id\` IN ( SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 \ AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL ${interval} DAY) ) \ - AND \`uri-id\` > ${tmp_item_uri_not_in_valid_post_user_current_id} ORDER BY \`uri-id\` LIMIT ${limit}"); + AND \`uri-id\` > ${tmp_item_uri_not_in_valid_post_user_current_id} ORDER BY \`uri-id\` LIMIT ${limit}") tmp_item_uri_not_in_valid_post_user_q=$(echo "${tmp_item_uri_not_in_valid_post_user}" | grep -c '.') #echo "${tmp_item_uri_not_in_valid_post_user_q}" - if [[ "${tmp_item_uri_not_in_valid_post_user_q}" -gt 0 ]] - then - echo "${tmp_item_uri_not_in_valid_post_user}" | while read -r id - do - if [[ -n "${id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" + if [[ "${tmp_item_uri_not_in_valid_post_user_q}" -gt 0 ]]; then + echo "${tmp_item_uri_not_in_valid_post_user}" | while read -r id; do + if [[ -n "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" #echo "${id}" - echo "${id}" > /tmp/fcb + echo "${id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_item_uri_not_in_valid_post_user}") && "${tmp_item_uri_not_in_valid_post_user_current_id}" -lt $(tac $(echo "${tmp_item_uri_not_in_valid_post_user}") | grep -m 1 '.') ]] - then - tmp_item_uri_not_in_valid_post_user_current_id=$(tac $(echo "${tmp_item_uri_not_in_valid_post_user}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_item_uri_not_in_valid_post_user_current_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_item_uri_id_not_in_valid_post_current_id=$(cat /tmp/fcb) - fi - echo "${tmp_item_uri_not_in_valid_post_user_q} item(s) deleted until ${tmp_item_uri_not_in_valid_post_user_current_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_item_uri_not_in_valid_post_user_q} item(s) deleted until ${tmp_item_uri_not_in_valid_post_user_current_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_attach_not_in_post_media" tmp_attach_not_in_post_media_q="${limit}" tmp_attach_not_in_post_media_current_id=0 -until [[ "${tmp_attach_not_in_post_media_q}" -lt "${limit}" ]] -do - tmp_attach_not_in_post_media=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_attach_not_in_post_media_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_attach_not_in_post_media=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`id\` FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`) \ - AND \`id\` > ${tmp_attach_not_in_post_media_current_id} ORDER BY \`id\` LIMIT ${limit}"); + AND \`id\` > ${tmp_attach_not_in_post_media_current_id} ORDER BY \`id\` LIMIT ${limit}") tmp_attach_not_in_post_media_q=$(echo "${tmp_attach_not_in_post_media}" | grep -c '.') #echo "${tmp_attach_not_in_post_media_q}" - if [[ "${tmp_attach_not_in_post_media_q}" -gt 0 ]] - then - echo "${tmp_attach_not_in_post_media}" | while read -r id - do - if [[ -n "${id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`attach\` WHERE \`id\` = ${id}" + if [[ "${tmp_attach_not_in_post_media_q}" -gt 0 ]]; then + echo "${tmp_attach_not_in_post_media}" | while read -r id; do + if [[ -n "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`attach\` WHERE \`id\` = ${id}" #echo "${id}" - echo "${id}" > /tmp/fcb + echo "${id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_attach_not_in_post_media}") && "${tmp_attach_not_in_post_media_current_id}" -lt $(tac $(echo "${tmp_attach_not_in_post_media}") | grep -m 1 '.') ]] - then - tmp_attach_not_in_post_media_current_id=$(tac $(echo "${tmp_attach_not_in_post_media}") | grep -m 1 '.') - fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_attach_not_in_post_media_current_id=$(cat /tmp/fcb) + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_attach_not_in_post_media_current_id=$(cat "${tmpfile}") fi + final_i=$(($(date +%s) - initial_i)) echo "${tmp_attach_not_in_post_media_q} item(s) deleted until ${tmp_attach_not_in_post_media_current_id}" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_item_uri_not_valid" tmp_item_uri_not_valid_q="${limit}" tmp_item_uri_not_valid_current_id=0 -tmp_item_uri_not_valid_last_id=$(sudo mariadb friendica -N -B -q -e \ +tmp_item_uri_not_valid_last_id=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1") -until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]] -do - tmp_item_uri_not_valid=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_item_uri_not_valid_q}" -lt "${limit}" ]]; do + initial_i=$(date +%s) + tmp_item_uri_not_valid=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT \`id\` FROM \`item-uri\` WHERE ( \`id\` < ${tmp_item_uri_not_valid_last_id} ) \ AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` ) \ AND NOT EXISTS ( SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` ) \ @@ -359,50 +299,39 @@ do AND NOT EXISTS ( SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`inbox-id\` = \`item-uri\`.\`id\` ) \ AND NOT EXISTS ( SELECT \`parent-uri-id\` FROM \`mail\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` ) \ AND NOT EXISTS ( SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` ) \ - AND (\`id\` > ${tmp_item_uri_not_valid_current_id} ) ORDER BY \`id\` LIMIT ${limit}"); + AND (\`id\` > ${tmp_item_uri_not_valid_current_id} ) ORDER BY \`id\` LIMIT ${limit}") tmp_item_uri_not_valid_q=$(echo "${tmp_item_uri_not_valid}" | grep -c '.') #echo "${tmp_item_uri_not_valid_q}" - if [[ "${tmp_item_uri_not_valid_q}" -gt 0 ]] - then - echo "${tmp_item_uri_not_valid}" | while read -r id - do - if [[ -n "${id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" - #echo "${id}" - echo "${id}" > /tmp/fcb + if [[ "${tmp_item_uri_not_valid_q}" -gt 0 ]]; then + echo "${tmp_item_uri_not_valid}" | while read -r id; do + if [[ -n "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" + echo "${id}" >"${tmpfile}" fi done fi - if [[ -s $(echo "${tmp_item_uri_not_valid_current_id}") && "${tmp_item_uri_not_valid_current_id}" -lt $(tac $(echo "${tmp_item_uri_not_valid}") | grep -m 1 '.') ]] - then - tmp_item_uri_not_valid_current_id=$(tac $(echo "${tmp_item_uri_not_valid}") | grep -m 1 '.') + if [[ -f "${tmpfile}" && -s "${tmpfile}" ]]; then + tmp_item_uri_not_valid_current_id=$(cat "${tmpfile}") fi - if [[ -f /tmp/fcb && -s /tmp/fcb ]] - then - tmp_item_uri_id_not_valid_current_id=$(cat /tmp/fcb) - fi - echo "${tmp_item_uri_not_valid_q} item(s) deleted until ${tmp_item_uri_not_valid_current_id}" + final_i=$(($(date +%s) - initial_i)) + echo "${tmp_item_uri_not_valid_q} item(s) deleted until ${tmp_item_uri_not_valid_current_id} in ${final_i}s" done -rm /tmp/fcb && touch /tmp/fcb +rm "${tmpfile}" && touch "${tmpfile}" echo "tmp_item_uri_duplicate" tmp_item_uri_duplicate_q="${limit}" -until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]] -do - tmp_item_uri_duplicate=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_item_uri_duplicate_q}" -lt "${limit}" ]]; do + tmp_item_uri_duplicate=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT t1.\`id\` FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\` LIMIT ${limit}") tmp_item_uri_duplicate_q=$(echo "${tmp_item_uri_duplicate}" | grep -c '.') + final_i=$(($(date +%s) - initial_i)) echo "${tmp_item_uri_duplicate_q}" - if [[ "${tmp_item_uri_duplicate_q}" -gt 0 ]] - then - echo "${tmp_item_uri_duplicate}" | while read -r id - do - if [[ -n "${id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" + if [[ "${tmp_item_uri_duplicate_q}" -gt 0 ]]; then + echo "${tmp_item_uri_duplicate}" | while read -r id; do + if [[ -n "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`item-uri\` WHERE \`id\` = ${id}" echo "${id}" fi done @@ -411,20 +340,17 @@ done echo "tmp_post_media_duplicate" tmp_post_media_duplicate_q="${limit}" -until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]] -do - tmp_post_media_duplicate=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do + tmp_post_media_duplicate=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT u1.\`id\` FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\` LIMIT ${limit}") tmp_post_media_duplicate_q=$(echo "${tmp_post_media_duplicate}" | grep -c '.') + final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_media_duplicate_q}" - if [[ "${tmp_post_media_duplicate_q}" -gt 0 ]] - then - echo "${tmp_post_media_duplicate}" | while read -r id - do - if [[ -n "${id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post-media\` WHERE \`id\` = ${id}" + if [[ "${tmp_post_media_duplicate_q}" -gt 0 ]]; then + echo "${tmp_post_media_duplicate}" | while read -r id; do + if [[ -n "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-media\` WHERE \`id\` = ${id}" echo "${id}" fi done @@ -433,20 +359,17 @@ done echo "tmp_post_user_duplicate" tmp_post_user_duplicate_q="${limit}" -until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]] -do - tmp_post_user_duplicate=$(sudo mariadb friendica -N -B -q -e \ +until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do + tmp_post_user_duplicate=$("${dbengine}" "${db}" -N -B -q -e \ "SELECT v1.\`id\` FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\` LIMIT ${limit}") tmp_post_user_duplicate_q=$(echo "${tmp_post_user_duplicate}" | grep -c '.') + final_i=$(($(date +%s) - initial_i)) echo "${tmp_post_user_duplicate_q}" - if [[ "${tmp_post_user_duplicate_q}" -gt 0 ]] - then - echo "${tmp_post_user_duplicate}" | while read -r id - do - if [[ -n "${id}" ]] - then - sudo mariadb friendica -N -B -q -e \ - "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" + if [[ "${tmp_post_user_duplicate_q}" -gt 0 ]]; then + echo "${tmp_post_user_duplicate}" | while read -r id; do + if [[ -n "${id}" ]]; then + "${dbengine}" "${db}" -N -B -q -e \ + "DELETE FROM \`post-user\` WHERE \`id\` = ${id}" echo "${id}" fi done diff --git a/friendica-find-missing-servers.sh b/friendica-find-missing-servers.sh index 4b624ff..a23c64f 100755 --- a/friendica-find-missing-servers.sh +++ b/friendica-find-missing-servers.sh @@ -9,9 +9,8 @@ loop_1() { sitereq=$(curl -s -L --head -m 30 --request GET "${a}") #Skip check if the message contains a reference to Cloudflare status=$(echo "${sitereq}" | grep -e "200" -e "cloudflare") - if [[ -z ${status} ]] - then - echo "${a}" >> "${tmpfile}" + if [[ -z ${status} ]]; then + echo "${a}" >>"${tmpfile}" echo "Added ${a}" fi } @@ -24,13 +23,10 @@ loop_3() { baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") echo "Deleting user ${lineb} - ${nick}@${baseurltrimmed}" #Find the pictures in the avatar folders and delete them - "${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro - do + "${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro; do #If stored in avatar folder - if $(echo "${photo}" | grep -q "${url}/avatar") - #isavatar=$(grep -q "${url}/avatar" <<< "${photo}") - #if [[ -z "${isavatar}" ]] - then + if $(echo "${photo}" | grep -q "${url}/avatar"); then #isavatar=$(grep -q "${url}/avatar" <<< "${photo}") + #if [[ -z "${isavatar}" ]] phototrimmed=$(echo "${photo}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") echo "${phototrimmed}" rm -rfv "${phototrimmed}" @@ -51,29 +47,23 @@ loop_3() { #Check for mariadb vs. mysql dbengine="" -if [[ -n $(type mariadb) ]] -then - dbengine="mariadb" -elif [[ -n $(type mysql) ]] -then - dbengine="mysql" +if [[ -n $(type mariadb) ]]; then + dbengine="mariadb" +elif [[ -n $(type mysql) ]]; then + dbengine="mysql" fi #Check if our dependencies are installed -if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]] -then +if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then date - if [[ ! -f "${tmpfile}" ]] - then + if [[ ! -f "${tmpfile}" ]]; then echo "Listing sites" #sites=($("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl from contact where baseurl != \"\"" | sort -n | uniq )) sites=() - mapfile -t sites < <("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl from contact where baseurl != \"\"" | sort -b -f -n | uniq -i ) + mapfile -t sites < <("${dbengine}" "${db}" -N -B -q -e "select distinct baseurl from contact where baseurl != \"\"" | sort -b -f -n | uniq -i) echo "Amount of unique sites: ${#sites[@]}" - for a in "${sites[@]}" - do + for a in "${sites[@]}"; do loop_1 "${a}" & - if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) * 2 )) ]] - then + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then wait -n fi done @@ -82,15 +72,12 @@ then sitesdown=() while read -r line; do sitesdown+=("${line}") - done < "${tmpfile}" + done <"${tmpfile}" echo "Amount of sites down: ${#sitesdown[@]} / ${#sites[@]}" - if [[ ! -f "${idsdownfile}" ]] - then - for b in "${sitesdown[@]}" - do + if [[ ! -f "${idsdownfile}" ]]; then + for b in "${sitesdown[@]}"; do loop_2 "${b}" & - if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 2 )) ]] - then + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; then wait -n fi done @@ -103,13 +90,12 @@ then #idsdown+=($lineb) #The community no longer exists, delete loop_3 "${lineb}" "${nick}" "${baseurl}" & - if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 2 )) ]] - then + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; then wait -n fi wait - done < "${idsdownfile}" - rm "${tmpfile}" 2> /dev/null - rm "${idsdownfile}" 2> /dev/null + done <"${idsdownfile}" + rm "${tmpfile}" 2>/dev/null + rm "${idsdownfile}" 2>/dev/null date fi diff --git a/friendica-fix-avatar-permissions.sh b/friendica-fix-avatar-permissions.sh index 46f22d9..a1e55c8 100755 --- a/friendica-fix-avatar-permissions.sh +++ b/friendica-fix-avatar-permissions.sh @@ -12,34 +12,26 @@ folderescaped=${folder////\\/} tmpfile=/tmp/friendica-fix-avatar-permissions.txt avatarfolder=avatar -loop_1(){ - if [[ "${p}" =~ .jpeg || "${p}" =~ .jpg ]] - then +loop_1() { + if [[ "${p}" =~ .jpeg || "${p}" =~ .jpg ]]; then nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null - elif [[ "${p}" =~ .gif ]] - then + elif [[ "${p}" =~ .gif ]]; then nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null #Specific compression for large GIF files - while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]] - do + while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do frameamount=$(exiftool -b -FrameCount "${p}" || 1) nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null done - elif [[ "${p}" =~ .png ]] - then + elif [[ "${p}" =~ .png ]]; then nice -n 10 oxipng -o max "${p}" #&> /dev/null - elif [[ "${p}" =~ .webp ]] - then + elif [[ "${p}" =~ .webp ]]; then #If file is not animated - if ! grep -v -q -e "ANIM" -e "ANMF" "${p}" - then + if ! grep -v -q -e "ANIM" -e "ANMF" "${p}"; then nice -n 10 cwebp -mt -af -quiet "${p}" -o /tmp/temp.webp #&> /dev/null - if [[ -f /tmp/temp.webp ]] - then - size_new=$(stat -c%s "/tmp/temp.webp" || 0 ) + if [[ -f /tmp/temp.webp ]]; then + size_new=$(stat -c%s "/tmp/temp.webp" || 0) size_original=$(stat -c%s "${p}") - if [[ "${size_original}" -gt "${size_new}" ]] - then + if [[ "${size_original}" -gt "${size_new}" ]]; then mv /tmp/temp.webp "${p}" #&> /dev/null else rm /tmp/temp.webp #&> /dev/null @@ -50,18 +42,14 @@ loop_1(){ } cd "${folder}" || exit -if [[ ! -f "${tmpfile}" ]] -then +if [[ ! -f "${tmpfile}" ]]; then sudo -u "${user}" bin/console movetoavatarcache | sudo tee "${tmpfile}" #&> /dev/null fi grep -e "https://${site}/${avatarfolder}/" "${tmpfile}" | sed -e "s/.*${site}/${folderescaped}/g" -e "s/?ts=.*//g" | ( - while read -r i - do - for p in "${i}" "${i//-320/-80}" "${i//-320/-48}" - do + while read -r i; do + for p in "${i}" "${i//-320/-80}" "${i//-320/-48}"; do loop_1 "${p}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] - then + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then wait -n fi done @@ -72,6 +60,6 @@ rm "${tmpfile}" /usr/bin/find "${folder}"/avatar -type d -empty -delete /usr/bin/chmod "${folderperm}" "${folder}"/avatar /usr/bin/chown -R "${user}":"${group}" "${folder}"/avatar -/usr/bin/find "${folder}"/avatar -depth -not -user "${user}" -or -not -group "${group}" -print0 | xargs -0 -r chown -v "${user}":"${group}" #&> /dev/null +/usr/bin/find "${folder}"/avatar -depth -not -user "${user}" -or -not -group "${group}" -print0 | xargs -0 -r chown -v "${user}":"${group}" #&> /dev/null /usr/bin/find "${folder}"/avatar -depth -type d -and -not -type f -and -not -perm "${folderperm}" -print0 | xargs -0 -r chmod -v "${folderperm}" #&> /dev/null -/usr/bin/find "${folder}"/avatar -depth -type f -and -not -type d -and -not -perm "${fileperm}" -print0 | xargs -0 -r chmod -v "${fileperm}" #&> /dev/null +/usr/bin/find "${folder}"/avatar -depth -type f -and -not -type d -and -not -perm "${fileperm}" -print0 | xargs -0 -r chmod -v "${fileperm}" #&> /dev/null diff --git a/friendica-non-follower-featured-posts.sh b/friendica-non-follower-featured-posts.sh index feb3221..bf3f38d 100755 --- a/friendica-non-follower-featured-posts.sh +++ b/friendica-non-follower-featured-posts.sh @@ -1,10 +1,9 @@ #!/bin/bash ca=100 camax=0 -while [[ ${ca} -gt 0 ]] -do +while [[ ${ca} -gt 0 ]]; do ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateContact\" limit ${ca}; select row_count();") - camax=$(( camax + ca )) + camax=$((camax + ca)) printf "\rUpdateContact\t\t%s\r" "${camax}" done printf "\rUpdateContact\t\t%s\n\r" "${camax}" @@ -12,10 +11,9 @@ printf "\rUpdateContact\t\t%s\n\r" "${camax}" cb=100 cbmax=0 -while [[ ${cb} -gt 0 ]] -do +while [[ ${cb} -gt 0 ]]; do cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"ContactDiscovery\" limit ${cb}; select row_count();") - cbmax=$(( cbmax + cb )) + cbmax=$((cbmax + cb)) printf "\rContactDiscovery\t%s\r" "${cbmax}" done printf "\rContactDiscovery\t%s\n\r" "${cbmax}" @@ -23,10 +21,9 @@ printf "\rContactDiscovery\t%s\n\r" "${cbmax}" cc=100 ccmax=0 -while [[ ${cc} -gt 0 ]] -do +while [[ ${cc} -gt 0 ]]; do cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit ${cc}; select row_count();") - ccmax=$(( ccmax + cc )) + ccmax=$((ccmax + cc)) printf "\rFetchFeaturedPosts\t%s\r" "${ccmax}" done @@ -35,10 +32,9 @@ printf "\rFetchFeaturedPosts\t%s\n\r" "${ccmax}" cd=100 cdmax=0 -while [[ ${cd} -gt 0 ]] -do +while [[ ${cd} -gt 0 ]]; do cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit ${cd}; select row_count();") - cdmax=$(( cdmax + cd )) + cdmax=$((cdmax + cd)) printf "\rUpdateGServer\t\t%s\r" "${cdmax}" done printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}" @@ -46,12 +42,10 @@ printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}" ce=100 cemax=0 -while [[ ${ce} -gt 0 ]] -do +while [[ ${ce} -gt 0 ]]; do ce=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where command=\"ProcessQueue\" and pid=0 and done=0 limit ${ce}; select row_count();") - cemax=$(( cemax + ce )) + cemax=$((cemax + ce)) printf "\rProcessQueue\t\t%s\r" "${cemax}" done printf "\rProcessQueue\t\t%s\n\r" "${cemax}" #echo "ProcessQueue $cemax" - diff --git a/friendica-remove-old-photos-parallel.sh b/friendica-remove-old-photos-parallel.sh index 1534713..517be4f 100755 --- a/friendica-remove-old-photos-parallel.sh +++ b/friendica-remove-old-photos-parallel.sh @@ -10,11 +10,10 @@ loop() { #Parse each file in folder ky=$(echo "${y}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g") f=$(sudo mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1") - if [[ $? -eq 0 && -z ${f} && -f ${y} ]] - then + if [[ $? -eq 0 && -z ${f} && -f ${y} ]]; then ls -lh "${y}" sudo rm -rf "${y}" - d=$(( d + 1 )) + d=$((d + 1)) fi #printf "\rPhotos: %s\tFolder %s\tEntry %s " "$d" "$n" "$m" printf "\rFolder %s\tEntry %s " "${n}" "${m}" @@ -25,33 +24,28 @@ date #Go to the Friendica installation cd "${folderavatar}" || exit let "indexlength=37+${#url}" -(( indexlength=49+${#url} )) +((indexlength = 49 + ${#url})) sudo mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))" n=0 d=0 -sudo find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x -do - n=$(( n + 1 )) +sudo find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do + n=$((n + 1)) #If the directory still exists - if [[ -d "${x}" ]] - then - folderescaped=${folder////\\/} - kx=$(echo "${x}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g") - if [[ -d ${x} ]] - then + if [[ -d "${x}" ]]; then + folderescaped=${folder////\\/} + kx=$(echo "${x}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g") + if [[ -d ${x} ]]; then m=0 - while read -r y - do - m=$(( m + 1 )) + while read -r y; do + m=$((m + 1)) loop "${x}" "${m}" "${n}" "${d}" & - if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) * 2 )) ]] - then - wait -n - fi - #done < <(sudo find "${x}" -type f -mtime -8) + if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then + wait -n + fi + #done < <(sudo find "${x}" -type f -mtime -8) done < <(sudo find "${x}" -type f) - fi - fi + fi + fi done sudo mariadb "${db}" -e "alter table contact drop index photo_index" date diff --git a/media-optimize-local.sh b/media-optimize-local.sh index fd429ce..fd2db6e 100755 --- a/media-optimize-local.sh +++ b/media-optimize-local.sh @@ -3,69 +3,57 @@ IFS=" " #media-optimize find . -type f -iname "*.jp*" -size +50k | ( - while read p - do - nice -n 15 jpegoptim -m 76 "${p}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] - then - wait -n - fi - done; - wait + while read p; do + nice -n 15 jpegoptim -m 76 "${p}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then + wait -n + fi + done + wait ) find . -type f -iname "*.gif" -size +500k | ( - while read q - do - nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] - then - wait -n - fi - done; - wait + while read q; do + nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then + wait -n + fi + done + wait ) #Specific compression for large GIF files: halving the frame rate find . -type f -size +500k -iname "*-320.gif" -or -iname "*-80.gif" -or -iname "*-48.gif" | ( - while read p - do - while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]] - do - frameamount=$(( $(exiftool -b -FrameCount "${p}" || 1) - 1 )) - nice -n 15 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" - done - done + while read p; do + while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do + frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1)) + nice -n 15 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" + done + done ) find . -type f -iname "*.png" -size +500k | ( - while read r - do - nice -n 15 oxipng -o max "${r}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]] - then - wait -n - fi - done; -wait + while read r; do + nice -n 15 oxipng -o max "${r}" & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then + wait -n + fi + done + wait ) #compress-webp find . -type f -iname "*.webp" -size +50k | ( - while read s - do - #If file is not animated - if ! grep -v -q -e "ANIM" -e "ANMF" "${s}" - then - cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp - if [ -f /tmp/"${s##.*\/}"_temp.webp ] - then - size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0) - size_original=$(stat -c%s "${s}" || 0) - if [ "$size_original" -gt "$size_new" ] - then - mv /tmp/"${s##.*\/}"_temp.webp "$s" - else - rm /tmp/"${s##.*\/}"_temp.webp - fi - fi - fi - done; - wait + while read s; do + #If file is not animated + if ! grep -v -q -e "ANIM" -e "ANMF" "${s}"; then + cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp + if [[ -f /tmp/"${s##.*\/}"_temp.webp ]]; then + size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0) + size_original=$(stat -c%s "${s}" || 0) + if [[ "${size_original}" -gt "${size_new}" ]]; then + mv /tmp/"${s##.*\/}"_temp.webp "${s}" + else + rm /tmp/"${s##.*\/}"_temp.webp + fi + fi + fi + done + wait )