diff --git a/friendica-compress-storage.sh b/friendica-compress-storage.sh index f3f2d62..0d63d3d 100755 --- a/friendica-compress-storage.sh +++ b/friendica-compress-storage.sh @@ -2,35 +2,48 @@ IFS=" " #Set your parameters here -folder=/var/www/friendica -storagefolder=storage +#Name of the database +db=friendica +#User of the database +user=root +#Folder with the storage files to check +storagefolder=/var/www/friendica/storage +#The folder storage name, with slashes escaped to work through sed +folderescaped=${storagefolder////\\/} loop_1() { - t=$(file "${p}") - if [[ "${t}" =~ JPEG ]]; then - nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null - elif [[ "${t}" =~ GIF ]]; then - nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null - #Specific compression for large GIF files - while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do - frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1)) - nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null - done - elif [[ "${t}" =~ PNG ]]; then - nice -n 10 oxipng -o max "${p}" #&> /dev/null - elif [[ "${t}" =~ Web/P ]]; then - #If file is not animated - if [[ -f "${p}" ]]; then - if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then - tmppic="/tmp/temp_$(date +%s).webp" - nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null - if [[ -f "${tmppic}" ]]; then - size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0) - size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0) - if [[ "${size_original}" -gt "${size_new}" ]]; then - mv "${tmppic}" "${p}" #&> /dev/null - else - rm "${tmppic}" #&> /dev/null + ks=$(echo "${p}" | sed -e "s/${folderescaped}//g" -e "s/\///g") + e=$(sudo -u "${user}" mariadb "${db}" -N -B -q -e "select \`backend-ref\` from photo where \`backend-ref\` = '${ks}'") + #If the file was not found in the database, but still exists in the filesystem, delete it + if [[ -z "${e}" && -f "${p}" ]]; then + sudo rm -rfv "${p}" #&> /dev/null + else + t=$(file "${p}") + if [[ "${t}" =~ JPEG ]]; then + nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null + elif [[ "${t}" =~ GIF ]]; then + nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null + #Specific compression for large GIF files + while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do + frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1)) + nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null + done + elif [[ "${t}" =~ PNG ]]; then + nice -n 10 oxipng -o max "${p}" #&> /dev/null + elif [[ "${t}" =~ Web/P ]]; then + #If file is not animated + if [[ -f "${p}" ]]; then + if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then + tmppic="/tmp/temp_$(date +%s).webp" + nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null + if [[ -f "${tmppic}" ]]; then + size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0) + size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0) + if [[ "${size_original}" -gt "${size_new}" ]]; then + mv -v "${tmppic}" "${p}" #&> /dev/null + else + rm -v "${tmppic}" #&> /dev/null + fi fi fi fi @@ -38,7 +51,11 @@ loop_1() { fi } -find "${folder}/${storagefolder}" -depth -mindepth 2 -type f -size +50k -atime -8 -not -iname "index.html" | ( +#Generate an index to make searches faster +echo "Generating photo index..." #&> /dev/null +sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null +echo "Generating list of files..." #&> /dev/null +find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | ( while read -r p; do loop_1 "${p}" & until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do @@ -47,3 +64,5 @@ find "${folder}/${storagefolder}" -depth -mindepth 2 -type f -size +50k -atime - done ) wait +#Drop the index in the end to save storage +sudo mariadb "${db}" -e "alter table photo drop index backend_index" #&> /dev/null diff --git a/youtube-download-channel.sh b/youtube-download-channel.sh index 5fa995c..bf32606 100755 --- a/youtube-download-channel.sh +++ b/youtube-download-channel.sh @@ -63,52 +63,55 @@ url="https://www.youtube.com/@${channel}" if [[ "${channel}" = "subscriptions" ]]; then url="https://www.youtube.com/feed/subscriptions" fi -for full_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do - echo "${full_url}" - if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then - #If available, you can use the cookies from your browser directly. Substitute - # --cookies "${cookies}" - #for the below, substituting for your browser of choice: - # --cookies-from-browser "firefox" - #In case this still fails, you can resort to a PO Token. Follow the instructions at - # https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide - #and add a new variable with the contents of the PO Token in the form - # potoken="INSERTYOURPOTOKENHERE" - #then substitute the "--extractor-args" line below with - # --extractor-args "youtubetab:approximate_date,youtube:player-client=default,mweb;po_token=mweb.gvs+${potoken}" \ - #including the backslash so the multiline command keeps working. - "${python}" "${ytdl}" "${full_url}" \ - --cookies "${cookies}" \ - --extractor-args "youtubetab:approximate_date" \ - --skip-download --download-archive "${archive}" \ - --dateafter "${breaktime}" \ - --break-on-reject --lazy-playlist --write-info-json \ - --sleep-requests "${sleeptime}" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" - else - "${python}" "${ytdl}" "${full_url}" \ - --extractor-args "youtubetab:approximate_date" \ - --skip-download --download-archive "${archive}" \ - --dateafter "${breaktime}" \ - --break-on-reject --lazy-playlist --write-info-json \ - --sleep-requests "${sleeptime}" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" \ - --parse-metadata "video::(?P)" - fi -done +#for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do +#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141 +full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}") +#full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl") +echo "${url} = ${full_url}" +if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then + #If available, you can use the cookies from your browser directly. Substitute + # --cookies "${cookies}" + #for the below, substituting for your browser of choice: + # --cookies-from-browser "firefox" + #In case this still fails, you can resort to a PO Token. Follow the instructions at + # https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide + #and add a new variable with the contents of the PO Token in the form + # potoken="INSERTYOURPOTOKENHERE" + #then substitute the "--extractor-args" line below with + # --extractor-args "youtubetab:approximate_date,youtube:player-client=default,mweb;po_token=mweb.gvs+${potoken}" \ + #including the backslash so the multiline command keeps working. + "${python}" "${ytdl}" "${full_url}" \ + --cookies "${cookies}" \ + --skip-download --download-archive "${archive}" \ + --dateafter "${breaktime}" \ + --extractor-args "youtubetab:approximate_date,youtubetab:skip=webpage" \ + --break-on-reject --lazy-playlist --write-info-json \ + --sleep-requests "${sleeptime}" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" +else + "${python}" "${ytdl}" "${full_url}" \ + --skip-download --download-archive "${archive}" \ + --dateafter "${breaktime}" \ + --extractor-args "youtubetab:approximate_date,youtubetab:skip=webpage" \ + --break-on-reject --lazy-playlist --write-info-json \ + --sleep-requests "${sleeptime}" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" \ + --parse-metadata "video::(?P)" +fi +#done if [[ ${enablecsv} = 1 ]]; then if [[ -f "${tmpcsv}" ]]; then rm -rf "${tmpcsv}"