Compare commits
3 commits
06bbe1cf7d
...
0552252055
| Author | SHA1 | Date | |
|---|---|---|---|
| 0552252055 | |||
| ec467c3bc2 | |||
| 293b11dad8 |
3 changed files with 34 additions and 42 deletions
|
|
@ -55,7 +55,7 @@ loop_1() {
|
|||
echo "Generating photo index..." #&> /dev/null
|
||||
sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null
|
||||
echo "Generating list of files..." #&> /dev/null
|
||||
find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | (
|
||||
find "${storagefolder}" -depth -mindepth 2 -type f -size +100k -mtime -8 -not -iname "index.html" | (
|
||||
while read -r p; do
|
||||
loop_1 "${p}" &
|
||||
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do
|
||||
|
|
|
|||
|
|
@ -17,8 +17,7 @@ starterid=${3:-"0"}
|
|||
db="friendica"
|
||||
period="${period_amount} MONTH"
|
||||
tmpfile=/tmp/friendica-delete-old-users.csv
|
||||
tmpreadlock=/tmp/friendica-delete-old-users.read.tmp
|
||||
tmpwritelock=/tmp/friendica-delete-old-users.write.tmp
|
||||
tmplock=/tmp/friendica-delete-old-users.tmp
|
||||
|
||||
loop() {
|
||||
baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g")
|
||||
|
|
@ -33,7 +32,7 @@ loop() {
|
|||
if [[ -n $(type flock) ]]; then
|
||||
isreadlocked=0
|
||||
while [[ "${isreadlocked}" -eq 0 ]]; do
|
||||
exec 9>"${tmpreadlock}"
|
||||
exec 9>"${tmplock}"
|
||||
if flock -n -e 9; then
|
||||
isreadlocked=1
|
||||
if [[ -f "${tmpfile}" ]]; then
|
||||
|
|
@ -66,11 +65,11 @@ loop() {
|
|||
flock -u 9
|
||||
iswritelocked=0
|
||||
while [[ "${iswritelocked}" -eq 0 ]]; do
|
||||
exec 8>"${tmpwritelock}"
|
||||
if flock -n -e 8; then
|
||||
exec 9>"${tmplock}"
|
||||
if flock -n -e 9; then
|
||||
iswritelocked=1
|
||||
echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}"
|
||||
flock -u 8
|
||||
flock -u 9
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
|
@ -149,23 +148,18 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
|
|||
if [[ -f "${tmpfile}" ]]; then
|
||||
rm -rf "${tmpfile}"
|
||||
fi
|
||||
if [[ -f "${tmpreadlock}" ]]; then
|
||||
rm -rf "${tmpreadlock}"
|
||||
fi
|
||||
if [[ -f "${tmpwritelock}" ]]; then
|
||||
rm -rf "${tmpwritelock}"
|
||||
if [[ -f "${tmplock}" ]]; then
|
||||
rm -rf "${tmplock}"
|
||||
fi
|
||||
touch "${tmpfile}"
|
||||
echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}"
|
||||
if [[ "${intense_optimizations}" -gt 0 ]]; then
|
||||
"${dbengine}" "${db}" -v -e "\
|
||||
alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \
|
||||
alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
|
||||
alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
|
||||
alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
|
||||
alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
|
||||
alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \
|
||||
"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr)"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`)"
|
||||
fi
|
||||
counter=0
|
||||
was_empty=0
|
||||
|
|
@ -189,7 +183,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
|
|||
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \
|
||||
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \
|
||||
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \
|
||||
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \
|
||||
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \`last-item\` > '0001-01-01' and \
|
||||
c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \
|
||||
'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \
|
||||
c.\`id\` > ${currentid} limit 1000")
|
||||
|
|
@ -209,27 +203,21 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
|
|||
"
|
||||
"${dboptimizeengine}" "${db}"
|
||||
fi
|
||||
"${dbengine}" "${db}" -v -e "\
|
||||
alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \
|
||||
alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \
|
||||
alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \
|
||||
alter table \`post-user\` drop index \`tmp_post_user_id\`; \
|
||||
alter table \`post\` drop index \`tmp_post_id\`; \
|
||||
alter table \`photo\` drop index \`tmp_photo_id\`; \
|
||||
"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post-thread\` drop index \`tmp_post_thread_id\`"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post-user\` drop index \`tmp_post_user_id\`"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`post\` drop index \`tmp_post_id\`"
|
||||
"${dbengine}" "${db}" -v -e "alter table \`photo\` drop index \`tmp_photo_id\`"
|
||||
"${dboptimizeengine}" "${db}"
|
||||
if [[ -n $(type flock) ]]; then
|
||||
flock -u 9
|
||||
flock -u 8
|
||||
flock -u 9 2>/dev/null
|
||||
fi
|
||||
if [[ -f "${tmpfile}" ]]; then
|
||||
rm -rf "${tmpfile}"
|
||||
fi
|
||||
if [[ -f "${tmpreadlock}" ]]; then
|
||||
rm -rf "${tmpreadlock}"
|
||||
fi
|
||||
if [[ -f "${tmpwritelock}" ]]; then
|
||||
rm -rf "${tmpwritelock}"
|
||||
if [[ -f "${tmplock}" ]]; then
|
||||
rm -rf "${tmplock}"
|
||||
fi
|
||||
date
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -60,15 +60,19 @@ if [[ -f "${subfolder}/${channel}.tar.zst" ]]; then
|
|||
fi
|
||||
fi
|
||||
url="https://www.youtube.com/@${channel}"
|
||||
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
|
||||
full_url=$("${ytdl}" -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
|
||||
if [[ "${channel}" = "subscriptions" ]]; then
|
||||
url="https://www.youtube.com/feed/subscriptions"
|
||||
full_url="${url}"
|
||||
elif [[ "${channel}" = "WL" ]]; then
|
||||
url="https://www.youtube.com/playlist?list=WL"
|
||||
full_url="${url}"
|
||||
fi
|
||||
#for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
|
||||
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
|
||||
full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
|
||||
#full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl")
|
||||
echo "${url} = ${full_url}"
|
||||
if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
|
||||
if [[ -f "${cookies}" || "${channel}" = "subscriptions" || "${channel}" = "WL" ]]; then
|
||||
#If available, you can use the cookies from your browser directly. Substitute
|
||||
# --cookies "${cookies}"
|
||||
#for the below, substituting for your browser of choice:
|
||||
|
|
@ -130,7 +134,7 @@ total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l)
|
|||
find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
|
||||
count=$((count + 1))
|
||||
(
|
||||
if [[ -f "${x}" && "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then
|
||||
if [[ -f "${x}" && "${channel}" != "subscriptions" && "${channel}" != "WL" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then
|
||||
echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}"
|
||||
fi
|
||||
if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then
|
||||
|
|
@ -145,7 +149,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
|
|||
fi
|
||||
echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt"
|
||||
if [[ ${enablecsv} = "1" ]]; then
|
||||
jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do
|
||||
jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url, .was_live]' "${x}" | while read -r i; do
|
||||
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${tmpcsv}"
|
||||
done
|
||||
fi
|
||||
|
|
@ -188,7 +192,7 @@ if [[ ${enabledb} = "1" ]]; then
|
|||
fi
|
||||
if [[ ${enablecsv} = "1" ]]; then
|
||||
sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv"
|
||||
echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL"' >"${temporary}/${channel}-tmp.csv"
|
||||
echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL", "Livestream"' >"${temporary}/${channel}-tmp.csv"
|
||||
cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv"
|
||||
mv "${temporary}/${channel}-tmp.csv" "${csv}"
|
||||
rm "${temporary}/${channel}-without-header.csv"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue