feat: Add intensive optimizations setting; add option to resume at specific index

friendica-clean-database.sh

@@ -21,7 +21,9 @@ elif [[ -n $(type mysqloptimize) ]]; then
 fi
 intense_optimizations=${1:-"0"}
 
-bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null
+if [[ "${intense_optimizations}" -gt 0 ]]; then
+	bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 1 \"Database maintenance\"" #&> /dev/null
+fi
 
 echo "tmp_post_origin_deleted" #&> /dev/null
 tmp_post_origin_deleted_q="${limit}"
@@ -378,4 +380,6 @@ if [[ "${intense_optimizations}" -gt 0 ]]; then
 
 	"${dboptimizer}" "${db}" #&> /dev/null
 fi
-bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null
+if [[ "${intense_optimizations}" -gt 0 ]]; then
+	bash -c "cd ${folder} && sudo -u ${user} ${phpversion} bin/console.php maintenance 0" #&> /dev/null
+fi

friendica-find-missing-servers.sh

@@ -61,16 +61,16 @@ loop_3() {
 			picturecount=1
 		fi
 	done < <("${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${id}")
-	postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0)
-	postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0)
-	postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select_row_count();" || echo 0)
-	posttagcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${id}; select_row_count();" || echo 0)
-	postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0)
-	postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select_row_count();" || echo 0)
-	photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${id}; select_row_count();" || echo 0)
-	contactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${id}; select_row_count();" || echo 0)
-	apcontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${id}; select_row_count();" || echo 0)
-	diasporacontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${id}; select_row_count();" || echo 0)
+	postthreadcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread (select \`uri-id\` from \`post-thread\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete h.* from \`post-thread\` h inner join \`tmp_post_thread\` t where h.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
+	postthreadusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_thread_user (select \`uri-id\` from \`post-thread-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete r.* from \`post-thread-user\` r inner join \`tmp_post_thread_user\` t where r.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
+	postusercount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post_user (select \`id\` from \`post-user\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete u.* from \`post-user\` u inner join \`tmp_post_user\` t where u.\`id\` = t.\`id\`; select row_count();" || echo 0)
+	posttagcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${id}; select row_count();" || echo 0)
+	postcontentcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post-content\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
+	postcount=$("${dbengine}" "${db}" -N -B -q -e "create temporary table tmp_post (select \`uri-id\` from \`post\` where \`owner-id\` = ${id} or \`author-id\` = ${id} or \`causer-id\` = ${id}); delete p.* from \`post\` p inner join \`tmp_post\` t where p.\`uri-id\` = t.\`uri-id\`; select row_count();" || echo 0)
+	photocount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${id}; select row_count();" || echo 0)
+	contactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${id}; select row_count();" || echo 0)
+	apcontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`apcontact\` where \`uri-id\` = ${id}; select row_count();" || echo 0)
+	diasporacontactcount=$("${dbengine}" "${db}" -N -B -q -e "delete from \`diaspora-contact\` where \`uri-id\` = ${id}; select row_count();" || echo 0)
 	while read -r tmp_picturecount tmp_postthreadcount tmp_postthreadusercount tmp_postusercount tmp_posttagcount tmp_postcontentcount tmp_postcount tmp_photocount tmp_contactcount tmp_apcontactcount tmp_diasporacontactcount; do
 		picturecount=$((picturecount + tmp_picturecount))
 		postthreadcount=$((postthreadcount + tmp_postthreadcount))
@@ -127,8 +127,8 @@ loop_3() {
 if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then
 	date
 	if [[ "${intense_optimizations}" -gt 0 ]]; then
-		"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`contact_baseurl\` (\`baseurl\`)"
-		"${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` add index if not exists \`post_user_id\` (\`author-id\`, \`causer-id\`, \`owner-id\`)"
+		"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` add index if not exists \`tmp_contact_baseurl\` (\`baseurl\`)"
+		"${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` add index if not exists \`tmp_post_user_id\` (\`author-id\`, \`causer-id\`, \`owner-id\`)"
 	fi
 	if [[ ! -f "${tmpfile}" ]]; then
 		echo "Listing sites"
@@ -183,8 +183,8 @@ if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(ty
 		alter table \`photo\` auto_increment = 1; \
 		alter table \`contact\` auto_increment = 1"
 	if [[ "${intense_optimizations}" -gt 0 ]]; then
-		"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` drop index \`contact_baseurl\`"
-		"${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` drop index \`post_user_id\`"
+		"${dbengine}" "${db}" -N -B -q -e "alter table \`contact\` drop index \`tmp_contact_baseurl\`"
+		"${dbengine}" "${db}" -N -B -q -e "alter table \`post-user\` drop index \`tmp_post_user_id\`"
 	fi
 	date
 fi

friendica-remove-old-photos-parallel.sh

@@ -9,40 +9,49 @@ elif [[ -n $(type mysql) ]]; then
 else
 	exit
 fi
+intense_optimizations=${1:-"0"}
+input_id=${2:-"1"}
 #Set your parameters here
 url=friendica.example.net
 db=friendica
 folder=/var/www/friendica
 folderavatar=/var/www/friendica/avatar
+folderescaped=${folder////\\/}
 loop() {
 	#Parse each file in folder
-	folderescaped=${folder////\\/}
 	ky=$(echo "${y}" | sed -e "s/${folderescaped}/https:\/\/${url}/g" -e "s/-[0-9]*\..*\$//g")
 	f=$("${dbengine}" "${db}" -N -B -q -e "select photo from contact where photo like '${ky}%' limit 1")
 	if [[ $? -eq 0 && -z ${f} && -f ${y} ]]; then
-		rm -rvf "${y}" &
+		if [[ "${intense_optimizations}" -eq 1 ]]; then
+			rm -f "${y}" &
+		else
+			rm -rvf "${y}" &
+		fi
 		if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) * 2)) ]]; then
 			wait -n
 		fi
-
-		d=$((d + 1))
+		#d=$((d + 1))
 	fi
 	#printf "\rPhotos: %s\tFolder %s\tEntry %s   " "$d" "$n" "$m"
-	printf "\rFolder %s\tEntry %s  " "${n}" "${m}"
+	if [[ "${intense_optimizations}" -eq 0 ]]; then
+		printf "\rFolder %s\tEntry %s  " "${n}" "${m}"
+	fi
 	return "${d}"
 }
 
 date
 #Go to the Friendica installation
 cd "${folderavatar}" || exit
-#indexlength=$((49 + ${#url}))
-#"${dbengine}" "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
+if [[ "${intense_optimizations}" -eq 1 ]]; then
+	indexlength=$((49 + ${#url}))
+	"${dbengine}" "${db}" -e "alter table contact add index if not exists photo_index (photo(${indexlength}))"
+fi
 n=0
 d=0
 while read -r x; do
 	n=$((n + 1))
 	#If the directory still exists
-	if [[ -d "${x}" ]]; then
+	if [[ -d "${x}" && "${n}" -ge "${input_id}" ]]; then
 		m=0
 		while read -r y; do
 			m=$((m + 1))
@@ -52,6 +61,13 @@ while read -r x; do
 			fi
 		done < <(find "${x}" -type f)
 	fi
+	if [[ "${intense_optimizations}" -eq 1 ]]; then
+		printf "\rFolder %d\tDone     " "${n}"
+	else
+		printf "\r\nFolder %d done - %s\n" "${n}" "${x}"
+	fi
 done < <(find "${folderavatar}" -depth -mindepth 1 -maxdepth 1 -type d)
-#"${dbengine}" "${db}" -e "alter table contact drop index photo_index"
+if [[ "${intense_optimizations}" -eq 1 ]]; then
+	"${dbengine}" "${db}" -e "alter table contact drop index photo_index"
+fi
 date