fix: Rewrite and reorder code to print output better

friendica-clean-database.sh

@@ -6,7 +6,7 @@ user=friendica
 phpversion=php8.2
 dbengine=mariadb
 db=friendica
-dboptimizer=mariadboptimize
+dboptimizer=mariadb-optimize
 
 bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null;
 
@@ -318,6 +318,7 @@ tmp_post_media_duplicate_q="${limit}"
 tmp_post_media_duplicate_current_id=0
 until [[ "${tmp_post_media_duplicate_q}" -lt "${limit}" ]]; do
 	initial_i=$(date +%s)
+	tmp_post_media_duplicate_q=0
 	while read -r id; do
 		if [[ -s "${id}" ]]; then
 			"${dbengine}" "${db}" -N -B -q -e \
@@ -341,6 +342,7 @@ tmp_post_user_duplicate_q="${limit}"
 tmp_post_user_duplicate_current_id=0
 until [[ "${tmp_post_user_duplicate_q}" -lt "${limit}" ]]; do
 	initial_i=$(date +%s)
+	tmp_post_user_duplicate_q=0
 	while read -r id; do
 		if [[ -s "${id}" ]]; then
 			"${dbengine}" "${db}" -N -B -q -e \

friendica-remove-invalid-photos.sh

@@ -30,12 +30,10 @@ fi
 #Go to the Friendica installation
 cd "${folder}" || exit
 #Add to the loop, reset values
-n=0
-nt=0
 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
 #dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))")
-dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'")
+dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' or photo like ''")
-echo "${dbcount}"
+echo "${n} ${nt} ${lastid}" >/tmp/lastid
 until [[ $((nt + limit)) -gt ${dbcount} ]]; do
 	nx=0
 	maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact")
@@ -55,83 +53,155 @@ until [[ $((nt + limit)) -gt ${dbcount} ]]; do
 		done </tmp/lastid
 	fi
 	#dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}")
-	dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" order by id limit ${limit}")
+	#dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}")
-	while read -r id photo thumb micro; do
+	dbid=$(mariadb "${db}" -B -N -q -e "select \`id\` from \`contact\` where \`id\` > ${lastid} and (\`photo\` like \"https:\/\/${url}/avatar/%\" or \`photo\` like \"\") order by id limit ${limit}")
-		if [[ -n "${id}" && -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then
+	while read -r id; do
-			nx=$(("${nx}" + 1))
+		result_string=""
-			nt=$(("${nt}" + 1))
+		nx=$(("${nx}" + 1))
-			error_found=0
+		nt=$(("${nt}" + 1))
-			folderescaped=${folder////\\/}
+		error_found=0
-			#Substitute the URL path with the folder path so we can search for it in the local file system
+		dboutput=$(mariadb "${db}" -B -N -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}")
-			#Photo is nominally 320px, actually 300px
+		#echo "$id"
-			k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g")
+		if [[ -n "${id}" ]]; then
-			#Thumb is 80px
+			while read -r photo thumb micro; do
-			k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g")
+				#echo "$photo $thumb $micro"
-			#Micro is 48px
+				if [[ -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then
-			k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g")
+					folderescaped=${folder////\\/}
-			#If fetching any of the images causes an error
+					#Substitute the URL path with the folder path so we can search for it in the local file system
-			if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" ||
+					#Photo is nominally 320px, actually 300px
-				curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" ||
+					k_photo=$(echo "${photo}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g")
-				curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then
+					#Thumb is 80px
-				#Request the user data to be regenerated in the system through the database
+					k_thumb=$(echo "${thumb}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g")
-				mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\""
+					#Micro is 48px
-				if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then
+					k_micro=$(echo "${micro}" | sed -e "s/https:\/\/${url}/${folderescaped}/g" -e "s/\?ts=.*//g")
-					mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+					#If fetching any of the images causes an error
-				fi
+					if curl -s "${photo}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" ||
-				echo "${id} ${photo}"
+						curl -s "${thumb}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory" ||
-				error_found=1
+						curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then
-			fi
+						#Request the user data to be regenerated in the system through the database
-			#If any of the images is not found in the filesystem
-			if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then
-				#If the avatar uses the standard fallback picture or is local, we cannot use it as a base
-				avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"")
-				#If we have a remote avatar as a fallback, download it
-				if [[ $! -eq 0 && -n ${avatar} ]]; then
-					echo "${id} ${avatar}"
-					sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}"
-					#If the file is a valid picture (not empty, not text)
-					if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then
-						#Also fetch for thumb/micro and resize
-						#As the photo is the largest version we have, we will use it as the base, and leave it last to convert
-						convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}"
-						convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}"
-						convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}"
-					else
-						#If the avatar is not valid, set it as blank in the database
 						mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\""
-						rm -rf "${k_photo}"
+						if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then
+							mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" &
+							result_string=$(printf "%s (added)" "${result_string}")
+						else
+							result_string=$(printf "%s (already added)" "${result_string}")
+						fi
+						result_string=$(printf "${result_string} Fetch error: %s %s\n" "${id}" "${photo}")
+						error_found=1
 					fi
-					#Request the user data to be regenerated in the system through the database
+					#If any of the images is not found in the filesystem
-					if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then
+					if [[ ! -e "${k_photo}" || ! -e "${k_thumb}" || ! -e "${k_micro}" ]]; then
-						mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+						#If the avatar uses the standard fallback picture or is local, we cannot use it as a base
+						avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"")
+						#If we have a remote avatar as a fallback, download it
+						if [[ -n "${avatar}" ]]; then
+							result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}")
+							sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}"
+							#If the file is a valid picture (not empty, not text)
+							if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then
+								#Also fetch for thumb/micro and resize
+								#As the photo is the largest version we have, we will use it as the base, and leave it last to convert
+								(convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}") &
+								(convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") &
+								(convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") &
+								result_string=$(printf "%s (generated)" "${result_string}")
+							else
+								#If the avatar is not valid, set it as blank in the database
+								mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" &
+								rm -rf "${k_photo}" &
+								result_string=$(printf "%s (blanked)" "${result_string}")
+							fi
+							#Request the user data to be regenerated in the system through the database
+							if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then
+								mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" &
+								result_string=$(printf "%s (added)" "${result_string}")
+							else
+								result_string=$(printf "%s (already added)" "${result_string}")
+							fi
+						else
+							result_string=$(printf "${result_string} No remote avatar: %s" "${id}")
+							#If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later
+							mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" &
+							#Request the user data to be regenerated in the system through the database
+							if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then
+								mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" &
+								result_string=$(printf "%s (added)" "${result_string}")
+							else
+								result_string=$(printf "%s (already added)" "${result_string}")
+							fi
+						fi
+						error_found=1
+						k_photo_delta="${photo//.*?ts=//}"
+						#k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g")
+					else
+						#k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g")
+						k_photo_original_time="${photo//.*?ts=//}"
+						k_photo_found_time=$(stat -c%W "${k_photo}")
+						k_photo_delta=$((k_photo_found_time - k_photo_original_time))
 					fi
 				else
-					echo "${id}"
+					result_string=$(printf "${result_string} No local photo: %s" "${id}")
-					#If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later
+					#If the avatar uses the standard fallback picture or is local, we cannot use it as a base
-					mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\""
+					avatar=$(mariadb "${db}" -B -N -q -e "select avatar from contact where id = \"${id}\" and not avatar like \"%${url}\" and not avatar like \"%images/person%\"")
-					#Request the user data to be regenerated in the system through the database
+					#If we have a remote avatar as a fallback, download it
-					if [[ -n $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then
+					if [[ $! -eq 0 && -n ${avatar} ]]; then
-						mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());"
+						result_string=$(printf "${result_string} Remote avatar: %s %s\n" "${id}" "${avatar}")
+						sudo -u "${user}" curl "${avatar}" -s -o "${k_photo}"
+						#If the file is a valid picture (not empty, not text)
+						if file "${k_photo}" | grep -q -v -e "text" -e "empty" -e "symbolic link" -e "directory"; then
+							#Also fetch for thumb/micro and resize
+							#As the photo is the largest version we have, we will use it as the base, and leave it last to convert
+							(convert "${k_photo}" -resize 80x80 -depth 16 "${k_thumb}" && chmod "${fileperm}" "${k_thumb}" && chown "${user}:${group}" "${k_thumb}") &
+							(convert "${k_photo}" -resize 48x48 -depth 16 "${k_micro}" && chmod "${fileperm}" "${k_micro}" && chown "${user}:${group}" "${k_micro}") &
+							(convert "${k_photo}" -resize 300x300 -depth 16 "${k_photo}" && chmod "${fileperm}" "${k_photo}" && chown "${user}:${group}" "${k_photo}") &
+							result_string=$(printf "%s (generated)" "${result_string}")
+						else
+							#If the avatar is not valid, set it as blank in the database
+							mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" &
+							rm -rf "${k_photo}" &
+							result_string=$(printf "%s (blanked)" "${result_string}")
+						fi
+						#Request the user data to be regenerated in the system through the database
+						if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then
+							mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" &
+							result_string=$(printf "%s (added)" "${result_string}")
+						else
+							result_string=$(printf "%s (already added)" "${result_string}")
+						fi
+					else
+						result_string=$(printf "${result_string} No remote avatar: %s" "${id}")
+						#Request the user data to be regenerated in the system through the database
+						if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\" and done = 0") -gt 0 ]]; then
+							mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" &
+							result_string=$(printf "%s (added)" "${result_string}")
+						else
+							result_string=$(printf "%s (already added)" "${result_string}")
+						fi
 					fi
+					error_found=1
+					k_photo_delta=0
+					#k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g")
 				fi
-				error_found=1
+				if [[ "${error_found}" -gt 0 ]]; then
-				k_photo_delta="${photo//.*?ts=//}"
+					n=$((n + 1))
-				#k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g")
+				fi
-			else
+				lastid="${id}"
-				#k_photo_original_time=$(echo "${photo}" | sed -e "s/.*?ts=//g")
+				touch /tmp/lastid
-				k_photo_original_time="${photo//.*?ts=//}"
+				echo "${n} ${nt} ${lastid}" >/tmp/lastid
-				k_photo_found_time=$(stat -c%W "${k_photo}")
+			done < <(echo "${dboutput}")
-				k_photo_delta=$((k_photo_found_time - k_photo_original_time))
-			fi
-			if [[ "${error_found}" -gt 0 ]]; then
-				n=$((n + 1))
-			fi
-			lastid="${id}"
-			touch /tmp/lastid
-			echo "${n} ${nt} ${lastid}" >/tmp/lastid
 		fi
-		printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %6d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}"
+		printf "\rFound %8d/%8d Total %8d/%8d Delta %6d %s " "${n}" "${nt}" "${lastid}" "${maxid}" "${k_photo_delta}" "${result_string}"
-	done < <(echo "${dboutput}")
+		#Line clearance
+		printf "\r"
+		#for space in $(seq 1 "${COLUMNS}")
+		#do
+			#printf " "
+		#done
+		for space in $(seq 1 "${COLUMNS}")
+		do
+			printf "\b"
+		done
+	done < <(echo "${dbid}")
 done
 printf "\nFixing folders and moving to avatar cache...\n"
 #sudo -u "${user}" bin/console movetoavatarcache #&> /dev/null

friendica-remove-old-photos-parallel.sh

@@ -10,10 +10,13 @@ loop() {
 	#Parse each file in folder
 	folderescaped=${folder////\\/}
 	ky=$(echo "${y}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g")
-	f=$(sudo mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1")
+	f=$(mariadb "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1")
 	if [[ $? -eq 0 && -z ${f} && -f ${y} ]]; then
-		ls -lh "${y}"
+		( ls -lh "${y}" && rm -rf "${y}" ) &
-		sudo rm -rf "${y}"
+				if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then
+					wait -n
+				fi
+
 		d=$((d + 1))
 	fi
 	#printf "\rPhotos: %s\tFolder %s\tEntry %s   " "$d" "$n" "$m"
@@ -24,12 +27,11 @@ loop() {
 date
 #Go to the Friendica installation
 cd "${folderavatar}" || exit
-#let "indexlength=37+${#url}"
+indexlength=$((49 + ${#url}))
-((indexlength = 49 + ${#url}))
+mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
-sudo mariadb "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
 n=0
 d=0
-sudo find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do
+find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read -r x; do
 	n=$((n + 1))
 	#If the directory still exists
 	if [[ -d "${x}" ]]; then
@@ -43,10 +45,10 @@ sudo find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d | while read
 				if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then
 					wait -n
 				fi
-				#done < <(sudo find "${x}" -type f -mtime -8)
+				#done < <(find "${x}" -type f -mtime -8)
-			done < <(sudo find "${x}" -type f)
+			done < <(find "${x}" -type f)
 		fi
 	fi
 done
-sudo mariadb "${db}" -e "alter table contact drop index photo_index"
+mariadb "${db}" -e "alter table contact drop index photo_index"
 date