diff --git a/friendica-non-follower-featured-posts.sh b/friendica-non-follower-featured-posts.sh index feb3221..54d22e4 100755 --- a/friendica-non-follower-featured-posts.sh +++ b/friendica-non-follower-featured-posts.sh @@ -1,10 +1,9 @@ #!/bin/bash ca=100 camax=0 -while [[ ${ca} -gt 0 ]] -do - ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateContact\" limit ${ca}; select row_count();") - camax=$(( camax + ca )) +while [[ ${ca} -gt 0 ]]; do + ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"UpdateContact\" limit ${ca}; select row_count();") + camax=$((camax + ca)) printf "\rUpdateContact\t\t%s\r" "${camax}" done printf "\rUpdateContact\t\t%s\n\r" "${camax}" @@ -12,10 +11,9 @@ printf "\rUpdateContact\t\t%s\n\r" "${camax}" cb=100 cbmax=0 -while [[ ${cb} -gt 0 ]] -do - cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"ContactDiscovery\" limit ${cb}; select row_count();") - cbmax=$(( cbmax + cb )) +while [[ ${cb} -gt 0 ]]; do + cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"ContactDiscovery\" limit ${cb}; select row_count();") + cbmax=$((cbmax + cb)) printf "\rContactDiscovery\t%s\r" "${cbmax}" done printf "\rContactDiscovery\t%s\n\r" "${cbmax}" @@ -23,10 +21,9 @@ printf "\rContactDiscovery\t%s\n\r" "${cbmax}" cc=100 ccmax=0 -while [[ ${cc} -gt 0 ]] -do - cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit ${cc}; select row_count();") - ccmax=$(( ccmax + cc )) +while [[ ${cc} -gt 0 ]]; do + cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"FetchFeaturedPosts\" limit ${cc}; select row_count();") + ccmax=$((ccmax + cc)) printf "\rFetchFeaturedPosts\t%s\r" "${ccmax}" done @@ -35,10 +32,9 @@ printf "\rFetchFeaturedPosts\t%s\n\r" "${ccmax}" cd=100 cdmax=0 -while [[ ${cd} -gt 0 ]] -do - cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit ${cd}; select row_count();") - cdmax=$(( cdmax + cd )) +while [[ ${cd} -gt 0 ]]; do + cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`) or \`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`)) and \`command\` = \"UpdateGServer\" limit ${cd}; select row_count();") + cdmax=$((cdmax + cd)) printf "\rUpdateGServer\t\t%s\r" "${cdmax}" done printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}" @@ -46,12 +42,10 @@ printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}" ce=100 cemax=0 -while [[ ${ce} -gt 0 ]] -do +while [[ ${ce} -gt 0 ]]; do ce=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where command=\"ProcessQueue\" and pid=0 and done=0 limit ${ce}; select row_count();") - cemax=$(( cemax + ce )) + cemax=$((cemax + ce)) printf "\rProcessQueue\t\t%s\r" "${cemax}" done printf "\rProcessQueue\t\t%s\n\r" "${cemax}" #echo "ProcessQueue $cemax" - diff --git a/media-optimize-local.sh b/media-optimize-local.sh index fd2db6e..e1c2399 100755 --- a/media-optimize-local.sh +++ b/media-optimize-local.sh @@ -2,58 +2,132 @@ IFS=" " #media-optimize -find . -type f -iname "*.jp*" -size +50k | ( - while read p; do - nice -n 15 jpegoptim -m 76 "${p}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then - wait -n - fi - done - wait -) -find . -type f -iname "*.gif" -size +500k | ( - while read q; do - nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then - wait -n - fi - done - wait -) -#Specific compression for large GIF files: halving the frame rate -find . -type f -size +500k -iname "*-320.gif" -or -iname "*-80.gif" -or -iname "*-48.gif" | ( - while read p; do - while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do - frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1)) - nice -n 15 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" - done - done -) -find . -type f -iname "*.png" -size +500k | ( - while read r; do - nice -n 15 oxipng -o max "${r}" & - if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then - wait -n - fi - done - wait -) +while read -r p; do + nice -n 15 jpegoptim -m 76 "${p}" #&>/dev/null & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then + wait -n + fi +done < <(find . -type f -iname "*.jp*" \( -size +50k -and -mtime -8 \)) +wait +while read -r q; do + nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" #&>/dev/null & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then + wait -n + fi +done < <(find . -type f -iname "*.gif" \( -size +500k -and -mtime -8 \)) +wait +while read -r p; do + (while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do + frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1)) + nice -n 15 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&>/dev/null + done) & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then + wait -n + fi +done < <(find . -type f -size +500k \( -iname "*-320.gif" -or -iname "*-80.gif" -or -iname "*-48.gif" \)) +wait +while read -r r; do + nice -n 15 oxipng -o max "${r}" #&>/dev/null & + if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]; then + wait -n + fi +done < <(find . -type f -iname "*.png" -size +500k) + +wait #compress-webp -find . -type f -iname "*.webp" -size +50k | ( - while read s; do - #If file is not animated - if ! grep -v -q -e "ANIM" -e "ANMF" "${s}"; then - cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp - if [[ -f /tmp/"${s##.*\/}"_temp.webp ]]; then - size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0) - size_original=$(stat -c%s "${s}" || 0) - if [[ "${size_original}" -gt "${size_new}" ]]; then - mv /tmp/"${s##.*\/}"_temp.webp "${s}" - else - rm /tmp/"${s##.*\/}"_temp.webp +while read -r s; do + #If file is not animated + if ! grep -v -q -e "ANIM" -e "ANMF" "${s}"; then + cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp #&>/dev/null + if [[ -f /tmp/"${s##.*\/}"_temp.webp ]]; then + size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0) + size_original=$(stat -c%s "${s}" || 0) + if [[ -n "${size_original}" && -n "${size_new}" && "${size_original}" -gt "${size_new}" ]]; then + mv /tmp/"${s##.*\/}"_temp.webp "${s}" + else + rm /tmp/"${s##.*\/}"_temp.webp + fi + fi + fi +done < <(find . -type f -iname "*.webp" \( -size +50k -and -mtime -8 \)) +wait +while read -r i; do + if file -b "${i}" | grep -vq "PNG" | grep -vq "empty" | grep -vq "symbolic link" | grep -vq "directory" | grep -vq "text"; then + if file -b "${i}" | grep -q "JPEG"; then + mv "${i}" "${i%.*}".jpg + jpegoptim -m76 "${i%.*}".jpg #&>/dev/null + elif file "${i}" | grep -q "GIF"; then + mv "${i}" "${i%.*}".gif + gifsicle --batch -O3 --lossy=80 --colors=255 "${i%.*}".gif #&>/dev/null + elif file "${i}" | grep -q "Web/P"; then + mv "${i}" "${i%.*}".webp + #cwebp -mt -af -quiet $i + fi + fi +done < <(find . -iname "*.png" -mtime -8) +while read -r j; do + if file -b "${j}" | grep -v -q -e "JPEG" -e "empty" -e "symbolic link" -e "directory" -e "text"; then + if file -b "${j}" | grep -q "PNG"; then + mv "${j}" "${j%.*}".png + oxipng -o max "${j%.*}".png #&>/dev/null + elif file -b "${j}" | grep -q "GIF"; then + mv "${j}" "${j%.*}".gif + gifsicle --batch -O3 --lossy=80 --colors=255 "${j%.*}".gif #&>/dev/null + elif file -b "${j}" | grep -q "Web/P"; then + mv "${j}" "${j%.*}".webp + #If file is not animated + if ! grep -v -q -e "ANIM" -e "ANMF" "${s}"; then + cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp #&>/dev/null + if [[ -f /tmp/"${s##.*\/}"_temp.webp ]]; then + size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0) + size_original=$(stat -c%s "${s}" || 0) + if [[ -n "${size_original}" && -n "${size_new}" && "${size_original}" -gt "${size_new}" ]]; then + mv /tmp/"${s##.*\/}"_temp.webp "${s}" + else + rm /tmp/"${s##.*\/}"_temp.webp + fi fi fi fi - done - wait -) + fi +done < <(find . -mtime -8 \( -iname "*.jpg" -and -iname "*.jpeg" \)) +while read -r k; do + if file -b "${k}" | grep -v -q -e "GIF" -e "empty" -e "symbolic link" -e "directory" -e "text"; then + if file -b "${k}" | grep -q "JPEG"; then + mv "${k}" "${k%.*}".jpg + jpegoptim -m76 "${k%.*}".jpg #&>/dev/null + elif file -b "${k}" | grep -q "PNG"; then + mv "${k}" "${k%.*}".png + oxipng -o max "${k%.*}".png #&>/dev/null + elif file -b "${k}" | grep -q "Web/P"; then + mv "${k}" "${k%.*}".webp + #If file is not animated + if ! grep -v -q -e "ANIM" -e "ANMF" "${s}"; then + cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp #&>/dev/null + if [[ -f /tmp/"${s##.*\/}"_temp.webp ]]; then + size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp || 0) + size_original=$(stat -c%s "${s}" || 0) + if [[ -n "${size_original}" && -n "${size_new}" && "${size_original}" -gt "${size_new}" ]]; then + mv /tmp/"${s##.*\/}"_temp.webp "${s}" + else + rm /tmp/"${s##.*\/}"_temp.webp + fi + fi + fi + fi + fi +done < <(find . -iname "*.gif" -mtime -8) +while read -r l; do + if file -b "${l}" | grep -v -q -e "Web/P" -e "empty" -e "symbolic link" -e "directory" -e "text"; then + if file -b "${l}" | grep -q "JPEG"; then + mv "${l}" "${l%.*}".jpg + jpegoptim -m76 "${l%.*}".jpg #&>/dev/null + elif file -b "${l}" | grep -q "PNG"; then + mv "${l}" "${l%.*}".png + oxipng -o max "${l%.*}".png #&>/dev/null + elif file -b "${l}" | grep -q "GIF"; then + mv "${l}" "${l%.*}".gif + gifsicle --batch -O3 --lossy=80 --colors=255 "${l%.*}".gif #&>/dev/null + fi + fi +done < <(find . -iname "*.webp" -mtime -8)