csolisr/public-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: csolisr:0552252055dc4fe9d9d7684a2331ddb1b9de3edf
Branches Tags
csolisr:main
..
compare: csolisr:06bbe1cf7d3b47b8053da0fe36fe15d8ce8f49fc
Branches Tags
csolisr:main

No commits in common. "0552252055dc4fe9d9d7684a2331ddb1b9de3edf" and "06bbe1cf7d3b47b8053da0fe36fe15d8ce8f49fc" have entirely different histories.
0552252055 ... 06bbe1cf7d

3 changed files with 42 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

2
friendica-compress-storage.sh
View file

@ -55,7 +55,7 @@ loop_1() {
echo "Generating photo index..." #&> /dev/null
sudo mariadb "${db}" -e "alter table photo add index if not exists backend_index (\`backend-ref\`)" #&> /dev/null
echo "Generating list of files..." #&> /dev/null
find "${storagefolder}" -depth -mindepth 2 -type f -size +100k -mtime -8 -not -iname "index.html" | (
find "${storagefolder}" -depth -mindepth 2 -type f -size +50k -mtime -8 -not -iname "index.html" | (
while read -r p; do
loop_1 "${p}" &
until [[ $(jobs -r -p | wc -l) -lt $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; do

58
friendica-delete-old-users.sh
View file

@ -17,7 +17,8 @@ starterid=${3:-"0"}
db="friendica"
period="${period_amount} MONTH"
tmpfile=/tmp/friendica-delete-old-users.csv
tmplock=/tmp/friendica-delete-old-users.tmp
tmpreadlock=/tmp/friendica-delete-old-users.read.tmp
tmpwritelock=/tmp/friendica-delete-old-users.write.tmp
loop() {
baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g")
@ -32,7 +33,7 @@ loop() {
if [[ -n $(type flock) ]]; then
isreadlocked=0
while [[ "${isreadlocked}" -eq 0 ]]; do
exec 9>"${tmplock}"
exec 9>"${tmpreadlock}"
if flock -n -e 9; then
isreadlocked=1
if [[ -f "${tmpfile}" ]]; then
@ -65,11 +66,11 @@ loop() {
flock -u 9
iswritelocked=0
while [[ "${iswritelocked}" -eq 0 ]]; do
exec 9>"${tmplock}"
if flock -n -e 9; then
exec 8>"${tmpwritelock}"
if flock -n -e 8; then
iswritelocked=1
echo "${counter} ${lastitemid} ${postthreadcount} ${postthreadusercount} ${postusercount} ${posttagcount} ${postcontentcount} ${postcount} ${photocount}" >"${tmpfile}"
flock -u 9
flock -u 8
fi
done
fi
@ -148,18 +149,23 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}"
fi
if [[ -f "${tmplock}" ]]; then
rm -rf "${tmplock}"
if [[ -f "${tmpreadlock}" ]]; then
rm -rf "${tmpreadlock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi
touch "${tmpfile}"
echo "0 0 0 0 0 0 0 0 0" >"${tmpfile}"
if [[ "${intense_optimizations}" -gt 0 ]]; then
"${dbengine}" "${db}" -v -e "alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr)"
"${dbengine}" "${db}" -v -e "alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
"${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
"${dbengine}" "${db}" -v -e "alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
"${dbengine}" "${db}" -v -e "alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`)"
"${dbengine}" "${db}" -v -e "alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`)"
"${dbengine}" "${db}" -v -e "\
alter table \`contact\` add index if not exists \`tmp_contact_baseurl_addr\` (baseurl, addr); \
alter table \`post-thread\` add index if not exists \`tmp_post_thread_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
alter table \`post-thread-user\` add index if not exists \`tmp_post_thread_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
alter table \`post\` add index if not exists \`tmp_post_id\` (\`owner-id\`, \`author-id\`, \`causer-id\`); \
alter table \`photo\` add index if not exists \`tmp_photo_id\` (\`contact-id\`); \
"
fi
counter=0
was_empty=0
@ -183,7 +189,7 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`cid\` from \`user-contact\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`uid\` from \`user\`)) and \
c.\`addr\` not in (select \`addr\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \`last-item\` > '0001-01-01' and \
c.\`contact-type\` != 4 and not pending and \`last-item\` < CURDATE() - INTERVAL ${period} and \
c.\`nick\` not in ('threads.sys', 'relay', 'friendica', 'sharkey', 'bot', 'catodon', \
'flipboard', 'lemmy', 'mitra', 'mstdn_bot', 'peertube', 'piefed', 'admin') and \
c.\`id\` > ${currentid} limit 1000")
@ -203,21 +209,27 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
"
"${dboptimizeengine}" "${db}"
fi
"${dbengine}" "${db}" -v -e "alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`"
"${dbengine}" "${db}" -v -e "alter table \`post-thread\` drop index \`tmp_post_thread_id\`"
"${dbengine}" "${db}" -v -e "alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`"
"${dbengine}" "${db}" -v -e "alter table \`post-user\` drop index \`tmp_post_user_id\`"
"${dbengine}" "${db}" -v -e "alter table \`post\` drop index \`tmp_post_id\`"
"${dbengine}" "${db}" -v -e "alter table \`photo\` drop index \`tmp_photo_id\`"
"${dbengine}" "${db}" -v -e "\
alter table \`contact\` drop index \`tmp_contact_baseurl_addr\`; \
alter table \`post-thread\` drop index \`tmp_post_thread_id\`; \
alter table \`post-thread-user\` drop index \`tmp_post_thread_user_id\`; \
alter table \`post-user\` drop index \`tmp_post_user_id\`; \
alter table \`post\` drop index \`tmp_post_id\`; \
alter table \`photo\` drop index \`tmp_photo_id\`; \
"
"${dboptimizeengine}" "${db}"
if [[ -n $(type flock) ]]; then
flock -u 9 2>/dev/null
flock -u 9
flock -u 8
fi
if [[ -f "${tmpfile}" ]]; then
rm -rf "${tmpfile}"
fi
if [[ -f "${tmplock}" ]]; then
rm -rf "${tmplock}"
if [[ -f "${tmpreadlock}" ]]; then
rm -rf "${tmpreadlock}"
fi
if [[ -f "${tmpwritelock}" ]]; then
rm -rf "${tmpwritelock}"
fi
date
fi

16
youtube-download-channel.sh
View file

@ -60,19 +60,15 @@ if [[ -f "${subfolder}/${channel}.tar.zst" ]]; then
fi
fi
url="https://www.youtube.com/@${channel}"
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
full_url=$("${ytdl}" -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
if [[ "${channel}" = "subscriptions" ]]; then
url="https://www.youtube.com/feed/subscriptions"
full_url="${url}"
elif [[ "${channel}" = "WL" ]]; then
url="https://www.youtube.com/playlist?list=WL"
full_url="${url}"
fi
#for section_url in "${url}/videos" "${url}/shorts" "${url}/streams"; do
#Via https://github.com/yt-dlp/yt-dlp/issues/13573#issuecomment-3020152141
full_url=$(yt-dlp -I0 --print "playlist:https://www.youtube.com/playlist?list=UU%(channel_id.2:)s" "${url}")
#full_url=$(curl "${url}" | tr -d "\n\r" | xmlstarlet fo -R -n -H 2>/dev/null | xmlstarlet sel -t -v "/html" -n | grep "/channel/UC" | sed -e "s/var .* = //g" -e "s/\};/\}/g" -e "s/channel\/UC/playlist\?list=UU/g" | jq -r ".metadata .channelMetadataRenderer .channelUrl")
echo "${url} = ${full_url}"
if [[ -f "${cookies}" || "${channel}" = "subscriptions" || "${channel}" = "WL" ]]; then
if [[ -f "${cookies}" || "${channel}" = "subscriptions" ]]; then
#If available, you can use the cookies from your browser directly. Substitute
# --cookies "${cookies}"
#for the below, substituting for your browser of choice:
@ -134,7 +130,7 @@ total=$(find "${temporary}" -type f -iname "*.info.json" | wc -l)
find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
count=$((count + 1))
(
if [[ -f "${x}" && "${channel}" != "subscriptions" && "${channel}" != "WL" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then
if [[ -f "${x}" && "${channel}" != "subscriptions" && $(jq -rc ".uploader_id" "${x}") != "@${channel}" ]]; then
echo "${count}/${total} ${x} not uploaded from ${channel}, removing..." && rm "${x}"
fi
if [[ -f "${x}" && "${breaktime}" =~ ^[0-9]+$ ]]; then
@ -149,7 +145,7 @@ find "${temporary}" -type f -iname "*.info.json" | while read -r x; do
fi
echo "youtube $(jq -cr '.id' "${x}")" >>"${temporary}/${channel}.txt"
if [[ ${enablecsv} = "1" ]]; then
jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url, .was_live]' "${x}" | while read -r i; do
jq -c '[.upload_date, .timestamp, .duration, .uploader , .title, .webpage_url]' "${x}" | while read -r i; do
echo "${i}" | sed -e "s/^\[//g" -e "s/\]$//g" -e "s/\\\\\"/＂/g" >>"${tmpcsv}"
done
fi
@ -192,7 +188,7 @@ if [[ ${enabledb} = "1" ]]; then
fi
if [[ ${enablecsv} = "1" ]]; then
sort "${tmpcsv}" | uniq >"${temporary}/${channel}-without-header.csv"
echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL", "Livestream"' >"${temporary}/${channel}-tmp.csv"
echo '"Upload Date", "Timestamp", "Duration", "Uploader", "Title", "Webpage URL"' >"${temporary}/${channel}-tmp.csv"
cat "${temporary}/${channel}-without-header.csv" >>"${temporary}/${channel}-tmp.csv"
mv "${temporary}/${channel}-tmp.csv" "${csv}"
rm "${temporary}/${channel}-without-header.csv"