fix: Remove piping to solve nested shell scope issues

friendica-delete-old-users.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+db="friendica"
+idsdownfile="/tmp/idsdown.txt"
+url=friendica.example.net
+avatarfolder=/var/www/friendica/avatar
+avatarfolderescaped=${avatarfolder////\\/}
+
+loop() {
+	baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g")
+	echo "Deleting user ${lineb} - ${nick}@${baseurltrimmed}"
+	#Find the pictures in the avatar folders and delete them
+	"${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro; do
+		#If stored in avatar folder
+		if [[ -z $(echo "${photo}" | grep "${url}/avatar") ]]; then
+			phototrimmed=$(echo "${photo}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
+			rm -rfv "${phototrimmed}"
+			thumbtrimmed=$(echo "${thumb}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
+			rm -rfv "${thumbtrimmed}"
+			microtrimmed=$(echo "${micro}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g")
+			rm -rfv "${microtrimmed}"
+		fi
+	done
+	"${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}"
+	"${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread-user\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}  or \`owner-id\` = ${lineb}"
+	"${dbengine}" "${db}" -N -B -q -e "delete from \`post-user\` where \`author-id\` = ${lineb}  or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}"
+	"${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}"
+	"${dbengine}" "${db}" -N -B -q -e "delete from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}"
+	"${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}"
+	"${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}"
+}
+
+#Check for mariadb vs. mysql
+dbengine=""
+if [[ -n $(type mariadb) ]]; then
+	dbengine="mariadb"
+elif [[ -n $(type mysql) ]]; then
+	dbengine="mysql"
+fi
+#Check if our dependencies are installed
+if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then
+	date
+	"${dbengine}" "${db}" -N -B -q -e \
+		"select \`id\`, \`nick\`, \`baseurl\` from contact c where \
+		c.\`id\` not in (select \`cid\` from \`user-contact\`) and \
+		c.\`id\` not in (select \`uid\` from \`user\`) and \
+		c.\`id\` not in ( select \`contact-id\` from \`group_member\`) and \
+		not pending and \`last-item\` < CURDATE() - INTERVAL 1 YEAR" |
+		while read -r lineb nick baseurl; do
+			loop "${lineb}" "${nick}" "${baseurl}" &
+			if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; then
+				wait -n
+			fi
+		done
+	date
+fi

friendica-fix-avatar-permissions.sh

@@ -4,7 +4,7 @@ IFS="
 #Set your parameters here
 site=friendica.example.net
 user=friendica
-group=friendica
+group=www-data
 fileperm=640
 folderperm=750
 folder=/var/www/friendica

friendica-non-follower-featured-posts.sh

@@ -22,23 +22,23 @@ printf "\rContactDiscovery\t%s\n\r" "${cbmax}"
 cc=100
 ccmax=0
 while [[ ${cc} -gt 0 ]]; do
-	cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit ${cc}; select row_count();")
+	cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') in (select \`url\` from \`contact\` where \`id\` not in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit ${cd}; select row_count();")
 	ccmax=$((ccmax + cc))
-	printf "\rFetchFeaturedPosts\t%s\r" "${ccmax}"
-
+	printf "\rUpdateGServer\t\t%s\r" "${ccmax}"
 done
-printf "\rFetchFeaturedPosts\t%s\n\r" "${ccmax}"
-#echo "FetchFeaturedPosts $ccmax"
+printf "\rUpdateGServer\t\t%s\n\r" "${ccmax}"
+#echo "UpdateGServer      $ccmax"
 
 cd=100
 cdmax=0
 while [[ ${cd} -gt 0 ]]; do
-	cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit ${cd}; select row_count();")
-	cdmax=$((cdmax + cd))
-	printf "\rUpdateGServer\t\t%s\r" "${cdmax}"
+	cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') in (select \`url\` from \`contact\` where \`id\` not in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit ${cc}; select row_count();")
+	cdmax=$((ccmax + cc))
+	printf "\rFetchFeaturedPosts\t%s\r" "${cdmax}"
+
 done
-printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}"
-#echo "UpdateGServer      $cdmax"
+printf "\rFetchFeaturedPosts\t%s\n\r" "${cdmax}"
+#echo "FetchFeaturedPosts $cdmax"
 
 ce=100
 cemax=0

friendica-remove-invalid-photos.sh

@@ -21,8 +21,7 @@ lastid=0
 #Highest possible ID known
 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
 #Limit per batch
-#limit=1000
-limit=$maxid
+limit=$(( ( maxid / 1000 ) + 1 ))
 if [[ -f /tmp/lastid ]]; then
 	rm /tmp/lastid && touch /tmp/lastid
 else
@@ -34,24 +33,31 @@ cd "${folder}" || exit
 n=0
 nt=0
 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
-dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
+#dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
+dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'")
 echo "${dbcount}"
-until [[ $((nt + limit)) -ge ${dbcount} ]]; do
+until [[ $((nt + limit)) -gt ${dbcount} ]]; do
 	nx=0
 	maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
 	batch=$(("${batch}" + 1))
 	#Read lastid outside of the loop with a temporary file
 	if [[ -f /tmp/lastid && -s /tmp/lastid ]]; then
+		count=0
 		while read -r n_i nt_i lastid_i; do
-			if [[ -s "${n_i}" && -s "${nt_i}" && -n "${lastid_i}" ]]; then
+			if [[ -s "${n_i}" ]]; then
 				n="${n_i}"
+			fi
+			if [[ -s "${nt_i}" ]]; then
 				nt="${nt_i}"
+			fi
+			if [[ -s "${lastid_i}" ]]; then
 				lastid="${lastid_i}"
 			fi
-		done < <(cat /tmp/lastid)
+		done < /tmp/lastid
 	fi
-	dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
-	echo "${dboutput}" | while read -r id photo thumb micro; do
+	#dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
+	dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" order by id limit ${limit}")
+	while read -r id photo thumb micro; do
 		if [[ -n "${id}" && -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then
 			nx=$(("${nx}" + 1))
 			nt=$(("${nt}" + 1))
@@ -70,7 +76,9 @@ until [[ $((nt + limit)) -ge ${dbcount} ]]; do
 				curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then
 				#Request the user data to be regenerated in the system through the database
 				mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\""
-				mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority) values (\"UpdateContact\", \"[${id}]\", 20);"
+				if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then
+					mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+				fi
 				echo "${id} ${photo}"
 				error_found=1
 			fi
@@ -95,13 +103,17 @@ until [[ $((nt + limit)) -ge ${dbcount} ]]; do
 						rm -rf "${k_photo}"
 					fi
 					#Request the user data to be regenerated in the system through the database
-					mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+					if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then
+						mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+					fi
 				else
 					echo "${id}"
 					#If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later
 					mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\""
 					#Request the user data to be regenerated in the system through the database
-					mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+					if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then
+						mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+					fi
 				fi
 				error_found=1
 				k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g")
@@ -113,12 +125,12 @@ until [[ $((nt + limit)) -ge ${dbcount} ]]; do
 			if [[ "${error_found}" -gt 0 ]]; then
 				n=$((n + 1))
 			fi
+			lastid="${id}"
+			touch /tmp/lastid
+			echo "${n} ${nt} ${lastid}" > /tmp/lastid
 		fi
-		lastid="${id}"
-		touch /tmp/lastid
-		echo "${n} ${nt} ${lastid}" >/tmp/lastid
-		printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}"
-	done
+		printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %6d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}"
+	done < <(echo "${dboutput}")
 done
 printf "\nFixing folders and moving to avatar cache...\n"
 #sudo -u "${user}" bin/console movetoavatarcache #&> /dev/null