csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: Perform storage eviction and compression in a single pass

Browse source
This commit is contained in:
Carlos Solís 2025-07-01 15:00:45 +00:00
parent af0e32eb68
commit 11d3594121
1 changed files with 47 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

75
friendica-compress-storage.sh
View file

@ -2,35 +2,48 @@
IFS="
"
#Set your parameters here
folder=/var/www/friendica
storagefolder=storage
#Name of the database
db=friendica
#User of the database
user=root
#Folder with the storage files to check
storagefolder=/var/www/friendica/storage
#The folder storage name, with slashes escaped to work through sed
folderescaped=${storagefolder////\\/}
loop_1() {
t=$(file "${p}")
if [[ "${t}" =~ JPEG ]]; then
nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null
elif [[ "${t}" =~ GIF ]]; then
nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null
#Specific compression for large GIF files
while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do
frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1))
nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
done
elif [[ "${t}" =~ PNG ]]; then
nice -n 10 oxipng -o max "${p}" #&> /dev/null
elif [[ "${t}" =~ Web/P ]]; then
#If file is not animated
if [[ -f "${p}" ]]; then
if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then
tmppic="/tmp/temp_$(date +%s).webp"
nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null
if [[ -f "${tmppic}" ]]; then
size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0)
size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0)
if [[ "${size_original}" -gt "${size_new}" ]]; then
mv "${tmppic}" "${p}" #&> /dev/null
else
rm "${tmppic}" #&> /dev/null
ks=$(echo "${p}" | sed -e "s/${folderescaped}//g" -e "s/\///g")
e=$(sudo -u "${user}" mariadb "${db}" -N -B -q -e "select \`backend-ref\` from photo where \`backend-ref\` = '${ks}'")
#If the file was not found in the database, but still exists in the filesystem, delete it
if [[ -z "${e}" && -f "${p}" ]]; then
sudo rm -rfv "${p}" #&> /dev/null
else
t=$(file "${p}")
if [[ "${t}" =~ JPEG ]]; then
nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null
elif [[ "${t}" =~ GIF ]]; then
nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null
#Specific compression for large GIF files
while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do
frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1))
nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
done
elif [[ "${t}" =~ PNG ]]; then
nice -n 10 oxipng -o max "${p}" #&> /dev/null
elif [[ "${t}" =~ Web/P ]]; then
#If file is not animated
if [[ -f "${p}" ]]; then
if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then
tmppic="/tmp/temp_$(date +%s).webp"
nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null
if [[ -f "${tmppic}" ]]; then
size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0)
size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0)
if [[ "${size_original}" -gt "${size_new}" ]]; then
mv -v "${tmppic}" "${p}" #&> /dev/null
else
rm -v "${tmppic}" #&> /dev/null
fi
fi
fi
fi
@ -38,7 +51,11 @@ loop_1() {
fi
}
find "${folder}/${storagefolder}" -depth -mindepth 2 -type f -size +50k -atime -8 -not -iname "index.html" | (
#Generate an index to make searches faster
echo "Generating photo index..." #&> /dev/null
sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null
echo "Generating list of files..." #&> /dev/null
find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | (
while read -r p; do
loop_1 "${p}" &
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do
@ -47,3 +64,5 @@ find "${folder}/${storagefolder}" -depth -mindepth 2 -type f -size +50k -atime -
done
)
wait
#Drop the index in the end to save storage
sudo mariadb "${db}" -e "alter table photo drop index backend_index" #&> /dev/null