csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: csolisr:0552252055dc4fe9d9d7684a2331ddb1b9de3edf
Branches Tags
csolisr:main
..
compare: csolisr:06bbe1cf7d3b47b8053da0fe36fe15d8ce8f49fc
Branches Tags
csolisr:main

No commits in common. "0552252055dc4fe9d9d7684a2331ddb1b9de3edf" and "06bbe1cf7d3b47b8053da0fe36fe15d8ce8f49fc" have entirely different histories.
0552252055 ... 06bbe1cf7d

3 changed files with 42 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

2
friendica-compress-storage.sh
View file

@ -55,7 +55,7 @@ loop_1() {
echo "Generating photo index..." #&> /dev/null echo "Generating photo index..." #&> /dev/null
sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null
echo "Generating list of files..." #&> /dev/null echo "Generating list of files..." #&> /dev/null
find "${storagefolder}" -depth -mindepth 2 -type f -size +100k -mtime -8 -not -iname "index.html" | ( find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | (
while read -r p; do while read -r p; do
loop_1 "${p}" & loop_1 "${p}" &
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do

58
friendica-delete-old-users.sh
View file

@ -17,7 +17,8 @@ starterid=${3:-"0"}
db="friendica" db="friendica"
period="${period_amount} MONTH" period="${period_amount} MONTH"
tmpfile=/tmp/friendica-delete-old-users.csv tmpfile=/tmp/friendica-delete-old-users.csv
tmplock=/tmp/friendica-delete-old-users.tmp tmpreadlock=/tmp/friendica-delete-old-users.read.tmp
tmpwritelock=/tmp/friendica-delete-old-users.write.tmp
loop() { loop() {
baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g")
@ -32,7 +33,7 @@ loop() {
if [[ -n $(type flock) ]]; then if [[ -n $(type flock) ]]; then
isreadlocked=0 isreadlocked=0
while [[ "${isreadlocked}" -eq 0 ]]; do while [[ "${isreadlocked}" -eq 0 ]]; do
exec 9>"${tmplock}" exec 9>"${tmpreadlock}"
if flock -n -e 9; then if flock -n -e 9; then
isreadlocked=1 isreadlocked=1
if [[ -f "${tmpfile}" ]]; then if [[ -f "${tmpfile}" ]]; then
@ -65,11 +66,11 @@ loop() {
flock -u 9 flock -u 9
iswritelocked=0 iswritelocked=0
while [[ "${iswritelocked}" -eq 0 ]]; do while [[ "${iswritelocked}" -eq 0 ]]; do
exec 9>"${tmplock}" exec 8>"${tmpwritelock}"
if flock -n -e 9; then if flock -n -e 8; then
iswritelocked=1 iswritelocked=1
echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}" echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}"
flock -u 9 flock -u 8
fi fi
done done
fi fi
@ -148,18 +149,23 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
if [[ -f "${tmpfile}" ]]; then if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}" rm -rf "${tmpfile}"
fi fi
if [[ -f "${tmplock}" ]]; then if [[ -f "${tmpreadlock}" ]]; then
rm -rf "${tmplock}" rm -rf "${tmpreadlock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi fi
touch "${tmpfile}" touch "${tmpfile}"
echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}" echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}"
if [[ "${intense_optimizations}" -gt 0 ]]; then if [[ "${intense_optimizations}" -gt 0 ]]; then
"${dbengine}" "${db}" -v -e "alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr)" "${dbengine}" "${db}" -v -e "\
"${dbengine}" "${db}" -v -e "alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \
"${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
"${dbengine}" "${db}" -v -e "alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
"${dbengine}" "${db}" -v -e "alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)" alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
"${dbengine}" "${db}" -v -e "alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`)" alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \
"
fi fi
counter=0 counter=0
was_empty=0 was_empty=0
@ -183,7 +189,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \ c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \`last-item\` > '0001-01-01' and \ c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \
c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \ c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \
'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \ 'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \
c.\`id\` > ${currentid} limit 1000") c.\`id\` > ${currentid} limit 1000")
@ -203,21 +209,27 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
" "
"${dboptimizeengine}" "${db}" "${dboptimizeengine}" "${db}"
fi fi
"${dbengine}" "${db}" -v -e "alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`" "${dbengine}" "${db}" -v -e "\
"${dbengine}" "${db}" -v -e "alter table \`post-thread\` drop index \`tmp_post_thread_id\`" alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \
"${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`" alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \
"${dbengine}" "${db}" -v -e "alter table \`post-user\` drop index \`tmp_post_user_id\`" alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \
"${dbengine}" "${db}" -v -e "alter table \`post\` drop index \`tmp_post_id\`" alter table \`post-user\` drop index \`tmp_post_user_id\`; \
"${dbengine}" "${db}" -v -e "alter table \`photo\` drop index \`tmp_photo_id\`" alter table \`post\` drop index \`tmp_post_id\`; \
alter table \`photo\` drop index \`tmp_photo_id\`; \
"
"${dboptimizeengine}" "${db}" "${dboptimizeengine}" "${db}"
if [[ -n $(type flock) ]]; then if [[ -n $(type flock) ]]; then
flock -u 9 2>/dev/null flock -u 9
flock -u 8
fi fi
if [[ -f "${tmpfile}" ]]; then if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}" rm -rf "${tmpfile}"
fi fi
if [[ -f "${tmplock}" ]]; then if [[ -f "${tmpreadlock}" ]]; then
rm -rf "${tmplock}" rm -rf "${tmpreadlock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi fi
date date
fi fi

16
youtube-download-channel.sh
View file

@ -60,19 +60,15 @@ if [[ -f "${subfolder}/${channel}.tar.zst" ]]; then
fi fi
fi fi
url="https://www.youtube.com/@${channel}" url="https://www.youtube.com/@${channel}"
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
full_url=$("${ytdl}" -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
if [[ "${channel}" = "subscriptions" ]]; then if [[ "${channel}" = "subscriptions" ]]; then
url="https://www.youtube.com/feed/subscriptions" url="https://www.youtube.com/feed/subscriptions"
full_url="${url}"
elif [[ "${channel}" = "WL" ]]; then
url="https://www.youtube.com/playlist?list=WL"
full_url="${url}"
fi fi
#for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do #for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
#full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl") #full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl")
echo "${url} = ${full_url}" echo "${url} = ${full_url}"
if [[ -f "${cookies}" || "${channel}" = "subscriptions" || "${channel}" = "WL" ]]; then if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
#If available, you can use the cookies from your browser directly. Substitute #If available, you can use the cookies from your browser directly. Substitute
# --cookies "${cookies}" # --cookies "${cookies}"
#for the below, substituting for your browser of choice: #for the below, substituting for your browser of choice:
@ -134,7 +130,7 @@ total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l)
find "${temporary}" -type f -iname "*.info.json" | while read -r x; do find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
count=$((count + 1)) count=$((count + 1))
( (
if [[ -f "${x}" && "${channel}" != "subscriptions" && "${channel}" != "WL" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then if [[ -f "${x}" && "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then
echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}" echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}"
fi fi
if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then
@ -149,7 +145,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
fi fi
echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt" echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt"
if [[ ${enablecsv} = "1" ]]; then if [[ ${enablecsv} = "1" ]]; then
jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url, .was_live]' "${x}" | while read -r i; do jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${tmpcsv}" echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${tmpcsv}"
done done
fi fi
@ -192,7 +188,7 @@ if [[ ${enabledb} = "1" ]]; then
fi fi
if [[ ${enablecsv} = "1" ]]; then if [[ ${enablecsv} = "1" ]]; then
sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv" sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv"
echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL", "Livestream"' >"${temporary}/${channel}-tmp.csv" echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL"' >"${temporary}/${channel}-tmp.csv"
cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv" cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv"
mv "${temporary}/${channel}-tmp.csv" "${csv}" mv "${temporary}/${channel}-tmp.csv" "${csv}"
rm "${temporary}/${channel}-without-header.csv" rm "${temporary}/${channel}-without-header.csv"