chore: Initial release

This commit is contained in:
Carlos Solís 2025-01-24 19:16:21 +00:00
parent d842475857
commit 7f2b750466
7 changed files with 413 additions and 0 deletions

78
friendica-clean-database.sh Executable file
View file

@ -0,0 +1,78 @@
#!/bin/bash
interval=14
sudo mariadb friendica --verbose -v -v --show-warnings --execute=\
"DELETE FROM \`post-origin\` WHERE (\`parent-uri-id\`, \`uid\`) IN (SELECT \`uri-id\`, \`uid\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \
\`deleted\` AND \`edited\` < (CURDATE() - INTERVAL $interval DAY));\
DELETE FROM \`post-user\` WHERE \`gravity\` = 0 AND \
\`deleted\` AND \`edited\` < (CURDATE() - INTERVAL $interval DAY);\
DELETE FROM \`post\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`);\
DELETE FROM \`post-content\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`);\
DELETE FROM \`post-thread\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post-user\`);\
DELETE FROM \`post-user\` WHERE \`uri-id\` NOT IN (SELECT \`uri-id\` FROM \`post\`);\
DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-thread\` WHERE \`received\` < (CURDATE() - INTERVAL $interval DAY) \
AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-thread-user\` \
WHERE (\`mention\` OR \`starred\` OR \`wall\`) AND \`uri-id\` = \`post-thread\`.\`uri-id\`) \
AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-category\` \
WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-collection\` \
WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-media\` \
WHERE \`uri-id\` = \`post-thread\`.\`uri-id\`) \
AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` INNER JOIN \`contact\` ON \`contact\`.\`id\` = \`contact-id\` AND \`notify_new_posts\` \
WHERE \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \
AND NOT \`uri-id\` IN (SELECT \`parent-uri-id\` FROM \`post-user\` \
WHERE (\`origin\` OR \`event-id\` != 0 OR \`post-type\` = 128) AND \`parent-uri-id\` = \`post-thread\`.\`uri-id\`) \
AND NOT \`uri-id\` IN (SELECT \`uri-id\` FROM \`post-content\` \
WHERE \`resource-id\` != 0 AND \`uri-id\` = \`post-thread\`.\`uri-id\`));\
DELETE FROM \`item-uri\` WHERE \`id\` IN (SELECT \`uri-id\` FROM \`post-user\` WHERE \`gravity\` = 0 AND \`uid\` = 0 AND \`received\` < (CURDATE() - INTERVAL $interval DAY) \
AND NOT \`uri-id\` IN (\
SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` != 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\`\
) AND NOT \`uri-id\` IN (\
SELECT \`parent-uri-id\` FROM \`post-user\` AS \`i\` WHERE \`i\`.\`uid\` = 0 AND \`i\`.\`parent-uri-id\` = \`post-user\`.\`uri-id\` AND \`i\`.\`received\` > (CURDATE() - INTERVAL $interval DAY)\
)\
);\
DELETE FROM \`attach\` WHERE \`id\` NOT IN (SELECT \`attach-id\` FROM \`post-media\`);\
DELETE FROM \`item-uri\` WHERE (\
\`id\` < (\
SELECT \`uri-id\` FROM \`post\` WHERE \`received\` < CURDATE() - INTERVAL 1 DAY ORDER BY \`received\` DESC LIMIT 1 \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`post-user\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`parent-uri-id\` FROM \`post-user\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`thr-parent-id\` FROM \`post-user\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`external-id\` FROM \`post-user\` WHERE \`external-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`replies-id\` FROM \`post-user\` WHERE \`replies-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`context-id\` FROM \`post-thread\` WHERE \`context-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`conversation-id\` FROM \`post-thread\` WHERE \`conversation-id\`= \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`mail\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`event\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`user-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`apcontact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`diaspora-contact\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`inbox-status\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`uri-id\` FROM \`post-delivery\` WHERE \`inbox-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`parent-uri-id\` FROM \`mail\` WHERE \`parent-uri-id\` = \`item-uri\`.\`id\` \
) AND NOT EXISTS (\
SELECT \`thr-parent-id\` FROM \`mail\` WHERE \`thr-parent-id\` = \`item-uri\`.\`id\` \
)\
); \
DELETE t1 FROM \`item-uri\` t1 INNER JOIN \`item-uri\` t2 WHERE t1.\`id\` < t2.\`id\` AND t1.\`uri\` = t2.\`uri\`;\
DELETE u1 FROM \`post-media\` u1 INNER JOIN \`post-media\` u2 WHERE u1.\`id\` < u2.\`id\` AND u1.\`uri-id\` = u2.\`uri-id\` AND u1.\`url\`= u2.\`url\`;\
DELETE v1 FROM \`post-user\` v1 INNER JOIN \`post-media\` v2 WHERE v1.\`id\` = v2.\`id\` AND v1.\`uri-id\` = v2.\`uri-id\`;"

View file

@ -0,0 +1,80 @@
#!/bin/bash
db="friendica"
tmpfile="/tmp/sitesdown.txt"
idsdownfile="/tmp/idsdown.txt"
loop_1() {
sitereq=$(curl -s -L --head -m 10 --request GET "$a")
status=$(echo "$sitereq" | grep -e "200" -e "cloudflare")
if [[ -z $status ]]
then
echo "$a" >> "$tmpfile"
echo "Added $a"
fi
}
loop_2() {
echo "Finding users for $b"
mariadb "$db" -N -B -q -e "select \`id\` from contact c where \"$b\" not in (select \`contact-id\` from group_member) and (c.baseurl = \"$b\" or c.url = \"$b\")" | sudo tee -a "$idsdownfile" &> /dev/null
}
loop_3() {
echo "Deleting user $lineb"
mariadb "$db" -N -B -q -e "delete from \`contact\` where \`id\` = $lineb"
mariadb "$db" -N -B -q -e "delete from \`post-thread\` where \`author-id\` = $lineb or \`causer-id\` = $lineb or \`owner-id\` = $lineb"
mariadb "$db" -N -B -q -e "delete from \`post-thread-user\` where \`author-id\` = $lineb or \`causer-id\` = $lineb or \`owner-id\` = $lineb"
mariadb "$db" -N -B -q -e "delete from \`post-user\` where \`author-id\` = $lineb or \`causer-id\` = $lineb or \`owner-id\` = $lineb"
mariadb "$db" -N -B -q -e "delete from \`post-tag\` where cid = $lineb"
mariadb "$db" -N -B -q -e "delete from \`post\` where \`owner-id\` = $lineb or \`author-id\` = $lineb or \`causer-id\` = $lineb"
mariadb "$db" -N -B -q -e "delete from \`photo\` where \`contact-id\` = $lineb"
mariadb "$db" -N -B -q -e "delete from \`contact\` where \`id\` = $lineb"
}
#Check if our dependencies are installed
if [[ $(type curl) && $(type mariadb) && $(type date) ]]
then
date
if [[ ! -f "$tmpfile" ]]
then
echo "Listing sites"
sites=($(mariadb "$db" -N -B -q -e "select distinct baseurl from contact" | sort -n | uniq ))
echo "Amount of unique sites: ${#sites[@]}"
for a in "${sites[@]}"
do
loop_1 $a &
if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)*2) ]]
then
wait -n
fi
done
wait
fi
sitesdown=()
while read -r line; do
sitesdown+=($line)
done < "$tmpfile"
echo "Amount of sites down: ${#sitesdown[@]} / ${#sites[@]}"
if [[ ! -f "$idsdownfile" ]]
then
for b in "${sitesdown[@]}"
do
loop_2 $b &
if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)/2) ]]
then
wait -n
fi
done
wait
#cat "$idsdownfile" | sort -n | uniq > "$idsdownfile"
fi
while read -r lineb; do
#The community no longer exists, delete
loop_3 $lineb &
if [[ $(jobs -r -p | wc -l) -ge $(expr $(getconf _NPROCESSORS_ONLN)/2) ]]
then
wait -n
fi
wait
done < "$idsdownfile"
rm "$tmpfile" 2> /dev/null
rm "$idsdownfile" 2> /dev/null
date
fi

View file

@ -0,0 +1,84 @@
#!/bin/bash
IFS="
"
#Set your parameters here
site=hub.example.net
user=friendica
group=friendica
fileperm=644
folderperm=755
folder=/var/www/friendica
folderescaped=${folder////\\/}
tmpfile=/tmp/friendica-fix-avatar-permissions.txt
avatarfolder=avatar
cd "$folder" || exit
if [ ! -f "$tmpfile" ]
then
sudo -u "$user" bin/console movetoavatarcache | sudo tee "$tmpfile" #&> /dev/null
fi
grep -e "https://$site/$avatarfolder/" "$tmpfile" | sed -e "s/.*$site/$folderescaped/g" -e "s/-.*/\*/g" | (
while read n
do
find "$folder/$avatarfolder" -path "$n" -type f | (
while read p
do
if [[ "${p}" =~ ".jpeg" || "${p}" =~ ".jpg" ]]
then
jpegoptim -m 76 "${p}" & #&> /dev/null
if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
then
wait -n
fi
fi
if [[ "${p}" =~ ".gif" ]]
then
gifsicle --batch -O3 --lossy=80 --colors=255 "${p}" & #&> /dev/null
if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
then
wait -n
fi
#Specific compression for large GIF files
while [[ $(stat -c%s "${p}") -ge 512000 ]]
do
gifsicle "${p}" `seq -f "#%g" 0 2 99` -O3 --lossy=80 --colors=255 -o "${p}" & #&> /dev/null
if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
then
wait -n
fi
done
fi
if [[ "${p}" =~ ".png" ]]
then
oxipng -o max "${p}" & #&> /dev/null
if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
then
wait -n
fi
fi
if [[ "${p}" =~ ".webp" ]]
then
cwebp -mt -af -quiet "${p}" -o /tmp/temp.webp #&> /dev/null
if [ -f /tmp/temp.webp ]
then
size_new=$(stat -c%s "/tmp/temp.webp")
size_original=$(stat -c%s "${p}")
if [ "$size_original" -gt "$size_new" ]
then
mv /tmp/temp.webp "${p}"
else
rm /tmp/temp.webp
fi
fi
fi
done
)
done
)
wait
rm "$tmpfile"
/usr/bin/find $folder/avatar -type d -empty -delete
/usr/bin/chmod $folderperm $folder/avatar
/usr/bin/chown -R $user:$group $folder/avatar
/usr/bin/find $folder/avatar -depth -not -user $user -or -not -group $group -print0 | xargs -0 -r sudo chown -v $user:$group #&> /dev/null
/usr/bin/find $folder/avatar -depth -type d -and -not -type f -and -not -perm $folderperm -print0 | xargs -0 -r sudo chmod -v $folderperm #&> /dev/null
/usr/bin/find $folder/avatar -depth -type f -and -not -type d -and -not -perm $fileperm -print0 | xargs -0 -r sudo chmod -v $fileperm #&> /dev/null

View file

@ -0,0 +1,57 @@
#!/bin/bash
ca=100
camax=0
while [[ $ca -gt 0 ]]
do
ca=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateContact\" limit $ca; select row_count();")
camax=$(( camax + ca ))
printf "\rUpdateContact\t\t%s\r" "$camax"
done
printf "\rUpdateContact\t\t%s\n\r" "$camax"
#echo "UpdateContact $camax"
cb=100
cbmax=0
while [[ $cb -gt 0 ]]
do
cb=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(regexp_replace(\`parameter\`, '\\\[', ''), '\\\]', '') not in (select \`id\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"ContactDiscovery\" limit $cb; select row_count();")
cbmax=$(( cbmax + cb ))
printf "\rContactDiscovery\t%s\r" "$cbmax"
done
printf "\rContactDiscovery\t%s\n\r" "$cbmax"
#echo "ContactDiscovery $cbmax"
cc=100
ccmax=0
while [[ $cc -gt 0 ]]
do
cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit $cc; select row_count();")
ccmax=$(( ccmax + cc ))
printf "\rFetchFeaturedPosts\t%s\r" "$ccmax"
done
printf "\rFetchFeaturedPosts\t%s\n\r" "$ccmax"
#echo "FetchFeaturedPosts $ccmax"
cd=100
cdmax=0
while [[ $cd -gt 0 ]]
do
cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit $cd; select row_count();")
cdmax=$(( cdmax + cd ))
printf "\rUpdateGServer\t\t%s\r" "$cdmax"
done
printf "\rUpdateGServer\t\t%s\n\r" "$cdmax"
#echo "UpdateGServer $cdmax"
ce=100
cemax=0
while [[ $ce -gt 0 ]]
do
ce=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where command=\"ProcessQueue\" and pid=0 and done=0 limit $ce; select row_count();")
cemax=$(( cemax + ce ))
printf "\rProcessQueue\t\t%s\r" "$cemax"
done
printf "\rProcessQueue\t\t%s\n\r" "$cemax"
#echo "ProcessQueue $cemax"

View file

@ -0,0 +1,56 @@
#!/bin/bash
IFS="
"
#Set your parameters here
url=hub.example.net
db=friendica
folder=/var/www/friendica
folderavatar=/var/www/friendica/avatar
loop() {
#Parse each file in folder
ky=$(echo "$y" | sed -e "s/$folderescaped/https:\/\/$url/g" -e "s/-[0-9]*\..*\$//g")
f=$(sudo mariadb $db -N -B -q -e "select photo from contact where photo like '$ky%' limit 1")
if [[ $? -eq 0 && -z $f && -f $y ]]
then
ls -lh "$y"
sudo rm -rf "$y"
d=$(( d + 1 ))
fi
#printf "\rPhotos: %s\tFolder %s\tEntry %s " "$d" "$n" "$m"
printf "\rFolder %s\tEntry %s " "$n" "$m"
return $d
}
date
#Go to the Friendica installation
cd "$folderavatar" || exit
let "indexlength=37+${#url}"
(( indexlength=49+${#url} ))
sudo mariadb $db -e "alter table contact add index if not exists photo_index (photo($indexlength))"
n=0
d=0
sudo find "$folderavatar" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x
do
n=$(( n + 1 ))
#If the directory still exists
if [[ -d "$x" ]]
then
folderescaped=${folder////\\/}
kx=$(echo "$x" | sed -e "s/$folderescaped/https:\/\/$url/g" -e "s/-[0-9]*\..*\$//g")
if [[ -d $x ]]
then
m=0
while read -r y
do
m=$(( m + 1 ))
loop "$x" "$m" "$n" "$d" &
if [[ $(jobs -r -p | wc -l) -ge $(( $(getconf _NPROCESSORS_ONLN) / 1 )) ]]
then
wait -n
fi
done < <(sudo find "$x" -type f -mtime -8)
fi
fi
done
sudo mariadb $db -e "alter table contact drop index photo_index"
date

2
friendica-workerqueue.sh Executable file
View file

@ -0,0 +1,2 @@
#!/bin/bash
sudo mariadb friendica --execute="select distinct command, count(*) from workerqueue where done = 0 group by command; select count(*) from workerqueue where done = 0; select count(*) from workerqueue where command = \"ProcessQueue\" and pid = 0 and done = 0"

56
media-optimize-local.sh Executable file
View file

@ -0,0 +1,56 @@
#!/bin/bash
IFS="
"
#media-optimize
find . -type f -iname "*.jp*" -size +50k | (
while read p
do
nice -n 15 jpegoptim -m 76 "${p}" &
if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
then
wait -n
fi
done;
wait
)
find . -type f -iname "*.gif" -size +500k | (
while read q
do
nice -n 15 gifsicle --batch -O3 --lossy=80 --colors=255 "${q}" &
if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
then
wait -n
fi
done;
wait
)
find . -type f -iname "*.png" -size +300k | (
while read r
do
nice -n 15 oxipng -o max "${r}" &
if [[ $(jobs -r -p | wc -l) -ge $(getconf _NPROCESSORS_ONLN) ]]
then
wait -n
fi
done;
wait
)
#compress-webp
find . -type f -iname "*.webp" -size +200k | (
while read s
do
nice -n 15 cwebp -mt -af -quiet "${s}" -o /tmp/"${s##.*\/}"_temp.webp
if [ -f /tmp/"${s##.*\/}"_temp.webp ]
then
size_new=$(stat -c%s /tmp/"${s##.*\/}"_temp.webp)
size_original=$(stat -c%s "${s}")
if [ "$size_original" -gt "$size_new" ]
then
mv /tmp/"${s##.*\/}"_temp.webp $s
else
rm /tmp/"${s##.*\/}"_temp.webp
fi
fi
done;
wait
)