Compare commits
2 commits
af0e32eb68
...
f38bd73bf8
| Author | SHA1 | Date | |
|---|---|---|---|
| f38bd73bf8 | |||
| 11d3594121 |
2 changed files with 96 additions and 74 deletions
|
|
@ -2,35 +2,48 @@
|
|||
IFS="
|
||||
"
|
||||
#Set your parameters here
|
||||
folder=/var/www/friendica
|
||||
storagefolder=storage
|
||||
#Name of the database
|
||||
db=friendica
|
||||
#User of the database
|
||||
user=root
|
||||
#Folder with the storage files to check
|
||||
storagefolder=/var/www/friendica/storage
|
||||
#The folder storage name, with slashes escaped to work through sed
|
||||
folderescaped=${storagefolder////\\/}
|
||||
|
||||
loop_1() {
|
||||
t=$(file "${p}")
|
||||
if [[ "${t}" =~ JPEG ]]; then
|
||||
nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null
|
||||
elif [[ "${t}" =~ GIF ]]; then
|
||||
nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null
|
||||
#Specific compression for large GIF files
|
||||
while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do
|
||||
frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1))
|
||||
nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
|
||||
done
|
||||
elif [[ "${t}" =~ PNG ]]; then
|
||||
nice -n 10 oxipng -o max "${p}" #&> /dev/null
|
||||
elif [[ "${t}" =~ Web/P ]]; then
|
||||
#If file is not animated
|
||||
if [[ -f "${p}" ]]; then
|
||||
if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then
|
||||
tmppic="/tmp/temp_$(date +%s).webp"
|
||||
nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null
|
||||
if [[ -f "${tmppic}" ]]; then
|
||||
size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0)
|
||||
size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0)
|
||||
if [[ "${size_original}" -gt "${size_new}" ]]; then
|
||||
mv "${tmppic}" "${p}" #&> /dev/null
|
||||
else
|
||||
rm "${tmppic}" #&> /dev/null
|
||||
ks=$(echo "${p}" | sed -e "s/${folderescaped}//g" -e "s/\///g")
|
||||
e=$(sudo -u "${user}" mariadb "${db}" -N -B -q -e "select \`backend-ref\` from photo where \`backend-ref\` = '${ks}'")
|
||||
#If the file was not found in the database, but still exists in the filesystem, delete it
|
||||
if [[ -z "${e}" && -f "${p}" ]]; then
|
||||
sudo rm -rfv "${p}" #&> /dev/null
|
||||
else
|
||||
t=$(file "${p}")
|
||||
if [[ "${t}" =~ JPEG ]]; then
|
||||
nice -n 10 jpegoptim -m 76 "${p}" #&> /dev/null
|
||||
elif [[ "${t}" =~ GIF ]]; then
|
||||
nice -n 10 gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" #&> /dev/null
|
||||
#Specific compression for large GIF files
|
||||
while [[ $(stat -c%s "${p}" || 0) -ge 512000 ]]; do
|
||||
frameamount=$(($(exiftool -b -FrameCount "${p}" || 1) - 1))
|
||||
nice -n 10 gifsicle "${p}" $(seq -f "#%g" 0 2 "${frameamount}") -O3 --lossy=80 --colors=255 -o "${p}" #&> /dev/null
|
||||
done
|
||||
elif [[ "${t}" =~ PNG ]]; then
|
||||
nice -n 10 oxipng -o max "${p}" #&> /dev/null
|
||||
elif [[ "${t}" =~ Web/P ]]; then
|
||||
#If file is not animated
|
||||
if [[ -f "${p}" ]]; then
|
||||
if grep -q -a -l -e "ANIM" -e "ANMF" "${p}"; then
|
||||
tmppic="/tmp/temp_$(date +%s).webp"
|
||||
nice -n 10 cwebp -mt -af -quiet "${p}" -o "${tmppic}" #&> /dev/null
|
||||
if [[ -f "${tmppic}" ]]; then
|
||||
size_new=$(stat -c%s "${tmppic}" 2>/dev/null || echo 0)
|
||||
size_original=$(stat -c%s "${p}" 2>/dev/null || echo 0)
|
||||
if [[ "${size_original}" -gt "${size_new}" ]]; then
|
||||
mv -v "${tmppic}" "${p}" #&> /dev/null
|
||||
else
|
||||
rm -v "${tmppic}" #&> /dev/null
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -38,7 +51,11 @@ loop_1() {
|
|||
fi
|
||||
}
|
||||
|
||||
find "${folder}/${storagefolder}" -depth -mindepth 2 -type f -size +50k -atime -8 -not -iname "index.html" | (
|
||||
#Generate an index to make searches faster
|
||||
echo "Generating photo index..." #&> /dev/null
|
||||
sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null
|
||||
echo "Generating list of files..." #&> /dev/null
|
||||
find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | (
|
||||
while read -r p; do
|
||||
loop_1 "${p}" &
|
||||
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do
|
||||
|
|
@ -47,3 +64,5 @@ find "${folder}/${storagefolder}" -depth -mindepth 2 -type f -size +50k -atime -
|
|||
done
|
||||
)
|
||||
wait
|
||||
#Drop the index in the end to save storage
|
||||
sudo mariadb "${db}" -e "alter table photo drop index backend_index" #&> /dev/null
|
||||
|
|
|
|||
|
|
@ -63,52 +63,55 @@ url="https://www.youtube.com/@${channel}"
|
|||
if [[ "${channel}" = "subscriptions" ]]; then
|
||||
url="https://www.youtube.com/feed/subscriptions"
|
||||
fi
|
||||
for full_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
|
||||
echo "${full_url}"
|
||||
if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
|
||||
#If available, you can use the cookies from your browser directly. Substitute
|
||||
# --cookies "${cookies}"
|
||||
#for the below, substituting for your browser of choice:
|
||||
# --cookies-from-browser "firefox"
|
||||
#In case this still fails, you can resort to a PO Token. Follow the instructions at
|
||||
# https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide
|
||||
#and add a new variable with the contents of the PO Token in the form
|
||||
# potoken="INSERTYOURPOTOKENHERE"
|
||||
#then substitute the "--extractor-args" line below with
|
||||
# --extractor-args "youtubetab:approximate_date,youtube:player-client=default,mweb;po_token=mweb.gvs+${potoken}" \
|
||||
#including the backslash so the multiline command keeps working.
|
||||
"${python}" "${ytdl}" "${full_url}" \
|
||||
--cookies "${cookies}" \
|
||||
--extractor-args "youtubetab:approximate_date" \
|
||||
--skip-download --download-archive "${archive}" \
|
||||
--dateafter "${breaktime}" \
|
||||
--break-on-reject --lazy-playlist --write-info-json \
|
||||
--sleep-requests "${sleeptime}" \
|
||||
--parse-metadata "video::(?P<formats>)" \
|
||||
--parse-metadata "video::(?P<thumbnails>)" \
|
||||
--parse-metadata "video::(?P<subtitles>)" \
|
||||
--parse-metadata "video::(?P<automatic_captions>)" \
|
||||
--parse-metadata "video::(?P<chapters>)" \
|
||||
--parse-metadata "video::(?P<heatmap>)" \
|
||||
--parse-metadata "video::(?P<tags>)" \
|
||||
--parse-metadata "video::(?P<categories>)"
|
||||
else
|
||||
"${python}" "${ytdl}" "${full_url}" \
|
||||
--extractor-args "youtubetab:approximate_date" \
|
||||
--skip-download --download-archive "${archive}" \
|
||||
--dateafter "${breaktime}" \
|
||||
--break-on-reject --lazy-playlist --write-info-json \
|
||||
--sleep-requests "${sleeptime}" \
|
||||
--parse-metadata "video::(?P<formats>)" \
|
||||
--parse-metadata "video::(?P<thumbnails>)" \
|
||||
--parse-metadata "video::(?P<subtitles>)" \
|
||||
--parse-metadata "video::(?P<automatic_captions>)" \
|
||||
--parse-metadata "video::(?P<chapters>)" \
|
||||
--parse-metadata "video::(?P<heatmap>)" \
|
||||
--parse-metadata "video::(?P<tags>)" \
|
||||
--parse-metadata "video::(?P<categories>)"
|
||||
fi
|
||||
done
|
||||
#for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
|
||||
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
|
||||
full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
|
||||
#full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl")
|
||||
echo "${url} = ${full_url}"
|
||||
if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
|
||||
#If available, you can use the cookies from your browser directly. Substitute
|
||||
# --cookies "${cookies}"
|
||||
#for the below, substituting for your browser of choice:
|
||||
# --cookies-from-browser "firefox"
|
||||
#In case this still fails, you can resort to a PO Token. Follow the instructions at
|
||||
# https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide
|
||||
#and add a new variable with the contents of the PO Token in the form
|
||||
# potoken="INSERTYOURPOTOKENHERE"
|
||||
#then substitute the "--extractor-args" line below with
|
||||
# --extractor-args "youtubetab:approximate_date,youtube:player-client=default,mweb;po_token=mweb.gvs+${potoken}" \
|
||||
#including the backslash so the multiline command keeps working.
|
||||
"${python}" "${ytdl}" "${full_url}" \
|
||||
--cookies "${cookies}" \
|
||||
--skip-download --download-archive "${archive}" \
|
||||
--dateafter "${breaktime}" \
|
||||
--extractor-args "youtubetab:approximate_date,youtubetab:skip=webpage" \
|
||||
--break-on-reject --lazy-playlist --write-info-json \
|
||||
--sleep-requests "${sleeptime}" \
|
||||
--parse-metadata "video::(?P<formats>)" \
|
||||
--parse-metadata "video::(?P<thumbnails>)" \
|
||||
--parse-metadata "video::(?P<subtitles>)" \
|
||||
--parse-metadata "video::(?P<automatic_captions>)" \
|
||||
--parse-metadata "video::(?P<chapters>)" \
|
||||
--parse-metadata "video::(?P<heatmap>)" \
|
||||
--parse-metadata "video::(?P<tags>)" \
|
||||
--parse-metadata "video::(?P<categories>)"
|
||||
else
|
||||
"${python}" "${ytdl}" "${full_url}" \
|
||||
--skip-download --download-archive "${archive}" \
|
||||
--dateafter "${breaktime}" \
|
||||
--extractor-args "youtubetab:approximate_date,youtubetab:skip=webpage" \
|
||||
--break-on-reject --lazy-playlist --write-info-json \
|
||||
--sleep-requests "${sleeptime}" \
|
||||
--parse-metadata "video::(?P<formats>)" \
|
||||
--parse-metadata "video::(?P<thumbnails>)" \
|
||||
--parse-metadata "video::(?P<subtitles>)" \
|
||||
--parse-metadata "video::(?P<automatic_captions>)" \
|
||||
--parse-metadata "video::(?P<chapters>)" \
|
||||
--parse-metadata "video::(?P<heatmap>)" \
|
||||
--parse-metadata "video::(?P<tags>)" \
|
||||
--parse-metadata "video::(?P<categories>)"
|
||||
fi
|
||||
#done
|
||||
if [[ ${enablecsv} = 1 ]]; then
|
||||
if [[ -f "${tmpcsv}" ]]; then
|
||||
rm -rf "${tmpcsv}"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue