Compare commits
No commits in common. "f38bd73bf81298d218fc470676d8119a44fdef77" and "af0e32eb68cd83c26356c547f9a0b10a3991a54d" have entirely different histories.
f38bd73bf8
...
af0e32eb68
2 changed files with 74 additions and 96 deletions
|
|
@ -2,48 +2,35 @@
|
||||||
IFS="
|
IFS="
|
||||||
"
|
"
|
||||||
#Set your parameters here
|
#Set your parameters here
|
||||||
#Name of the database
|
folder=/var/www/friendica
|
||||||
db=friendica
|
storagefolder=storage
|
||||||
#User of the database
|
|
||||||
user=root
|
|
||||||
#Folder with the storage files to check
|
|
||||||
storagefolder=/var/www/friendica/storage
|
|
||||||
#The folder storage name, with slashes escaped to work through sed
|
|
||||||
folderescaped=${storagefolder////\\/}
|
|
||||||
|
|
||||||
loop_1() {
|
loop_1() {
|
||||||
ks=$(echo "${p}" | sed -e "s/${folderescaped}//g" -e "s/\///g")
|
t=$(file "${p}")
|
||||||
e=$(sudo -u "${user}" mariadb "${db}" -N -B -q -e "select \`backend-ref\` from photo where \`backend-ref\` = '${ks}'")
|
if [[ "${t}" =~ JPEG ]]; then
|
||||||
#If the file was not found in the database, but still exists in the filesystem, delete it
|
nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null
|
||||||
if [[ -z "${e}" && -f "${p}" ]]; then
|
elif [[ "${t}" =~ GIF ]]; then
|
||||||
sudo rm -rfv "${p}" #&> /dev/null
|
nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null
|
||||||
else
|
#Specific compression for large GIF files
|
||||||
t=$(file "${p}")
|
while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do
|
||||||
if [[ "${t}" =~ JPEG ]]; then
|
frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1))
|
||||||
nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null
|
nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
|
||||||
elif [[ "${t}" =~ GIF ]]; then
|
done
|
||||||
nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null
|
elif [[ "${t}" =~ PNG ]]; then
|
||||||
#Specific compression for large GIF files
|
nice -n 10 oxipng -o max "${p}" #&> /dev/null
|
||||||
while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do
|
elif [[ "${t}" =~ Web/P ]]; then
|
||||||
frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1))
|
#If file is not animated
|
||||||
nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
|
if [[ -f "${p}" ]]; then
|
||||||
done
|
if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then
|
||||||
elif [[ "${t}" =~ PNG ]]; then
|
tmppic="/tmp/temp_$(date +%s).webp"
|
||||||
nice -n 10 oxipng -o max "${p}" #&> /dev/null
|
nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null
|
||||||
elif [[ "${t}" =~ Web/P ]]; then
|
if [[ -f "${tmppic}" ]]; then
|
||||||
#If file is not animated
|
size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0)
|
||||||
if [[ -f "${p}" ]]; then
|
size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0)
|
||||||
if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then
|
if [[ "${size_original}" -gt "${size_new}" ]]; then
|
||||||
tmppic="/tmp/temp_$(date +%s).webp"
|
mv "${tmppic}" "${p}" #&> /dev/null
|
||||||
nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null
|
else
|
||||||
if [[ -f "${tmppic}" ]]; then
|
rm "${tmppic}" #&> /dev/null
|
||||||
size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0)
|
|
||||||
size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0)
|
|
||||||
if [[ "${size_original}" -gt "${size_new}" ]]; then
|
|
||||||
mv -v "${tmppic}" "${p}" #&> /dev/null
|
|
||||||
else
|
|
||||||
rm -v "${tmppic}" #&> /dev/null
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
@ -51,11 +38,7 @@ loop_1() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
#Generate an index to make searches faster
|
find "${folder}/${storagefolder}" -depth -mindepth 2 -type f -size +50k -atime -8 -not -iname "index.html" | (
|
||||||
echo "Generating photo index..." #&> /dev/null
|
|
||||||
sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null
|
|
||||||
echo "Generating list of files..." #&> /dev/null
|
|
||||||
find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | (
|
|
||||||
while read -r p; do
|
while read -r p; do
|
||||||
loop_1 "${p}" &
|
loop_1 "${p}" &
|
||||||
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do
|
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do
|
||||||
|
|
@ -64,5 +47,3 @@ find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -in
|
||||||
done
|
done
|
||||||
)
|
)
|
||||||
wait
|
wait
|
||||||
#Drop the index in the end to save storage
|
|
||||||
sudo mariadb "${db}" -e "alter table photo drop index backend_index" #&> /dev/null
|
|
||||||
|
|
|
||||||
|
|
@ -63,55 +63,52 @@ url="https://www.youtube.com/@${channel}"
|
||||||
if [[ "${channel}" = "subscriptions" ]]; then
|
if [[ "${channel}" = "subscriptions" ]]; then
|
||||||
url="https://www.youtube.com/feed/subscriptions"
|
url="https://www.youtube.com/feed/subscriptions"
|
||||||
fi
|
fi
|
||||||
#for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
|
for full_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
|
||||||
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
|
echo "${full_url}"
|
||||||
full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
|
if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
|
||||||
#full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl")
|
#If available, you can use the cookies from your browser directly. Substitute
|
||||||
echo "${url} = ${full_url}"
|
# --cookies "${cookies}"
|
||||||
if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
|
#for the below, substituting for your browser of choice:
|
||||||
#If available, you can use the cookies from your browser directly. Substitute
|
# --cookies-from-browser "firefox"
|
||||||
# --cookies "${cookies}"
|
#In case this still fails, you can resort to a PO Token. Follow the instructions at
|
||||||
#for the below, substituting for your browser of choice:
|
# https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide
|
||||||
# --cookies-from-browser "firefox"
|
#and add a new variable with the contents of the PO Token in the form
|
||||||
#In case this still fails, you can resort to a PO Token. Follow the instructions at
|
# potoken="INSERTYOURPOTOKENHERE"
|
||||||
# https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide
|
#then substitute the "--extractor-args" line below with
|
||||||
#and add a new variable with the contents of the PO Token in the form
|
# --extractor-args "youtubetab:approximate_date,youtube:player-client=default,mweb;po_token=mweb.gvs+${potoken}" \
|
||||||
# potoken="INSERTYOURPOTOKENHERE"
|
#including the backslash so the multiline command keeps working.
|
||||||
#then substitute the "--extractor-args" line below with
|
"${python}" "${ytdl}" "${full_url}" \
|
||||||
# --extractor-args "youtubetab:approximate_date,youtube:player-client=default,mweb;po_token=mweb.gvs+${potoken}" \
|
--cookies "${cookies}" \
|
||||||
#including the backslash so the multiline command keeps working.
|
--extractor-args "youtubetab:approximate_date" \
|
||||||
"${python}" "${ytdl}" "${full_url}" \
|
--skip-download --download-archive "${archive}" \
|
||||||
--cookies "${cookies}" \
|
--dateafter "${breaktime}" \
|
||||||
--skip-download --download-archive "${archive}" \
|
--break-on-reject --lazy-playlist --write-info-json \
|
||||||
--dateafter "${breaktime}" \
|
--sleep-requests "${sleeptime}" \
|
||||||
--extractor-args "youtubetab:approximate_date,youtubetab:skip=webpage" \
|
--parse-metadata "video::(?P<formats>)" \
|
||||||
--break-on-reject --lazy-playlist --write-info-json \
|
--parse-metadata "video::(?P<thumbnails>)" \
|
||||||
--sleep-requests "${sleeptime}" \
|
--parse-metadata "video::(?P<subtitles>)" \
|
||||||
--parse-metadata "video::(?P<formats>)" \
|
--parse-metadata "video::(?P<automatic_captions>)" \
|
||||||
--parse-metadata "video::(?P<thumbnails>)" \
|
--parse-metadata "video::(?P<chapters>)" \
|
||||||
--parse-metadata "video::(?P<subtitles>)" \
|
--parse-metadata "video::(?P<heatmap>)" \
|
||||||
--parse-metadata "video::(?P<automatic_captions>)" \
|
--parse-metadata "video::(?P<tags>)" \
|
||||||
--parse-metadata "video::(?P<chapters>)" \
|
--parse-metadata "video::(?P<categories>)"
|
||||||
--parse-metadata "video::(?P<heatmap>)" \
|
else
|
||||||
--parse-metadata "video::(?P<tags>)" \
|
"${python}" "${ytdl}" "${full_url}" \
|
||||||
--parse-metadata "video::(?P<categories>)"
|
--extractor-args "youtubetab:approximate_date" \
|
||||||
else
|
--skip-download --download-archive "${archive}" \
|
||||||
"${python}" "${ytdl}" "${full_url}" \
|
--dateafter "${breaktime}" \
|
||||||
--skip-download --download-archive "${archive}" \
|
--break-on-reject --lazy-playlist --write-info-json \
|
||||||
--dateafter "${breaktime}" \
|
--sleep-requests "${sleeptime}" \
|
||||||
--extractor-args "youtubetab:approximate_date,youtubetab:skip=webpage" \
|
--parse-metadata "video::(?P<formats>)" \
|
||||||
--break-on-reject --lazy-playlist --write-info-json \
|
--parse-metadata "video::(?P<thumbnails>)" \
|
||||||
--sleep-requests "${sleeptime}" \
|
--parse-metadata "video::(?P<subtitles>)" \
|
||||||
--parse-metadata "video::(?P<formats>)" \
|
--parse-metadata "video::(?P<automatic_captions>)" \
|
||||||
--parse-metadata "video::(?P<thumbnails>)" \
|
--parse-metadata "video::(?P<chapters>)" \
|
||||||
--parse-metadata "video::(?P<subtitles>)" \
|
--parse-metadata "video::(?P<heatmap>)" \
|
||||||
--parse-metadata "video::(?P<automatic_captions>)" \
|
--parse-metadata "video::(?P<tags>)" \
|
||||||
--parse-metadata "video::(?P<chapters>)" \
|
--parse-metadata "video::(?P<categories>)"
|
||||||
--parse-metadata "video::(?P<heatmap>)" \
|
fi
|
||||||
--parse-metadata "video::(?P<tags>)" \
|
done
|
||||||
--parse-metadata "video::(?P<categories>)"
|
|
||||||
fi
|
|
||||||
#done
|
|
||||||
if [[ ${enablecsv} = 1 ]]; then
|
if [[ ${enablecsv} = 1 ]]; then
|
||||||
if [[ -f "${tmpcsv}" ]]; then
|
if [[ -f "${tmpcsv}" ]]; then
|
||||||
rm -rf "${tmpcsv}"
|
rm -rf "${tmpcsv}"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue