diff --git a/friendica-delete-old-users.sh b/friendica-delete-old-users.sh deleted file mode 100755 index fb10489..0000000 --- a/friendica-delete-old-users.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -db="friendica" -idsdownfile="/tmp/idsdown.txt" -url=friendica.example.net -avatarfolder=/var/www/friendica/avatar -avatarfolderescaped=${avatarfolder////\\/} - -loop() { - baseurltrimmed=$(echo "${baseurl}" | sed -e "s/http[s]*:\/\///g") - echo "Deleting user ${lineb} - ${nick}@${baseurltrimmed}" - #Find the pictures in the avatar folders and delete them - "${dbengine}" "${db}" -N -B -q -e "select \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` = ${lineb}" | while read -r photo thumb micro; do - #If stored in avatar folder - if [[ -z $(echo "${photo}" | grep "${url}/avatar") ]]; then - phototrimmed=$(echo "${photo}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") - rm -rfv "${phototrimmed}" - thumbtrimmed=$(echo "${thumb}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") - rm -rfv "${thumbtrimmed}" - microtrimmed=$(echo "${micro}" | sed -e "s/https:\/\/${url}\/avatar/${avatarfolderescaped}/g" -e "s/\?ts.*//g") - rm -rfv "${microtrimmed}" - fi - done - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-thread-user\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-user\` where \`author-id\` = ${lineb} or \`causer-id\` = ${lineb} or \`owner-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post-tag\` where cid = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`post\` where \`owner-id\` = ${lineb} or \`author-id\` = ${lineb} or \`causer-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`photo\` where \`contact-id\` = ${lineb}" - "${dbengine}" "${db}" -N -B -q -e "delete from \`contact\` where \`id\` = ${lineb}" -} - -#Check for mariadb vs. mysql -dbengine="" -if [[ -n $(type mariadb) ]]; then - dbengine="mariadb" -elif [[ -n $(type mysql) ]]; then - dbengine="mysql" -fi -#Check if our dependencies are installed -if [[ -n $(type curl) && -n "${dbengine}" && -n $(type "${dbengine}") && -n $(type date) ]]; then - date - "${dbengine}" "${db}" -N -B -q -e \ - "select \`id\`, \`nick\`, \`baseurl\` from contact c where \ - c.\`id\` not in (select \`cid\` from \`user-contact\`) and \ - c.\`id\` not in (select \`uid\` from \`user\`) and \ - c.\`id\` not in ( select \`contact-id\` from \`group_member\`) and \ - not pending and \`last-item\` < CURDATE() - INTERVAL 1 YEAR" | - while read -r lineb nick baseurl; do - loop "${lineb}" "${nick}" "${baseurl}" & - if [[ $(jobs -r -p | wc -l) -ge $(($(getconf _NPROCESSORS_ONLN) / 2)) ]]; then - wait -n - fi - done - date -fi diff --git a/friendica-fix-avatar-permissions.sh b/friendica-fix-avatar-permissions.sh index 4ad39db..a1e55c8 100755 --- a/friendica-fix-avatar-permissions.sh +++ b/friendica-fix-avatar-permissions.sh @@ -4,7 +4,7 @@ IFS=" #Set your parameters here site=friendica.example.net user=friendica -group=www-data +group=friendica fileperm=640 folderperm=750 folder=/var/www/friendica diff --git a/friendica-non-follower-featured-posts.sh b/friendica-non-follower-featured-posts.sh index 4c3fd56..bf3f38d 100755 --- a/friendica-non-follower-featured-posts.sh +++ b/friendica-non-follower-featured-posts.sh @@ -22,23 +22,23 @@ printf "\rContactDiscovery\t%s\n\r" "${cbmax}" cc=100 ccmax=0 while [[ ${cc} -gt 0 ]]; do - cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') in (select \`url\` from \`contact\` where \`id\` not in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit ${cd}; select row_count();") + cc=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit ${cc}; select row_count();") ccmax=$((ccmax + cc)) - printf "\rUpdateGServer\t\t%s\r" "${ccmax}" + printf "\rFetchFeaturedPosts\t%s\r" "${ccmax}" + done -printf "\rUpdateGServer\t\t%s\n\r" "${ccmax}" -#echo "UpdateGServer $ccmax" +printf "\rFetchFeaturedPosts\t%s\n\r" "${ccmax}" +#echo "FetchFeaturedPosts $ccmax" cd=100 cdmax=0 while [[ ${cd} -gt 0 ]]; do - cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') in (select \`url\` from \`contact\` where \`id\` not in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"FetchFeaturedPosts\" limit ${cc}; select row_count();") - cdmax=$((ccmax + cc)) - printf "\rFetchFeaturedPosts\t%s\r" "${cdmax}" - + cd=$(sudo mariadb friendica -B -N -q -e "delete from workerqueue where regexp_replace(substring_index(substring_index(\`parameter\`, '\\\"', -2), '\\\"', 1), '\\\\\\\\', '') not in (select \`url\` from \`contact\` where \`id\` in (select \`contact-id\` from \`group_member\`)) and \`command\` = \"UpdateGServer\" limit ${cd}; select row_count();") + cdmax=$((cdmax + cd)) + printf "\rUpdateGServer\t\t%s\r" "${cdmax}" done -printf "\rFetchFeaturedPosts\t%s\n\r" "${cdmax}" -#echo "FetchFeaturedPosts $cdmax" +printf "\rUpdateGServer\t\t%s\n\r" "${cdmax}" +#echo "UpdateGServer $cdmax" ce=100 cemax=0 diff --git a/friendica-remove-invalid-photos.sh b/friendica-remove-invalid-photos.sh index 7782fad..26d808e 100755 --- a/friendica-remove-invalid-photos.sh +++ b/friendica-remove-invalid-photos.sh @@ -21,7 +21,8 @@ lastid=0 #Highest possible ID known maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") #Limit per batch -limit=$(( ( maxid / 1000 ) + 1 )) +#limit=1000 +limit=$maxid if [[ -f /tmp/lastid ]]; then rm /tmp/lastid && touch /tmp/lastid else @@ -33,31 +34,24 @@ cd "${folder}" || exit n=0 nt=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") -#dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") -dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%'") +dbcount=$(mariadb "${db}" -B -N -q -e "select count(\`id\`) from contact where photo like 'https:\/\/${url}/avatar/%' and (id in (select cid from \`user-contact\`) or id in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`))") echo "${dbcount}" -until [[ $((nt + limit)) -gt ${dbcount} ]]; do +until [[ $((nt + limit)) -ge ${dbcount} ]]; do nx=0 maxid=$(mariadb "${db}" -B -N -q -e "select max(\`id\`) from contact") batch=$(("${batch}" + 1)) #Read lastid outside of the loop with a temporary file if [[ -f /tmp/lastid && -s /tmp/lastid ]]; then - count=0 while read -r n_i nt_i lastid_i; do - if [[ -s "${n_i}" ]]; then + if [[ -s "${n_i}" && -s "${nt_i}" && -n "${lastid_i}" ]]; then n="${n_i}" - fi - if [[ -s "${nt_i}" ]]; then nt="${nt_i}" - fi - if [[ -s "${lastid_i}" ]]; then lastid="${lastid_i}" fi - done < /tmp/lastid + done < <(cat /tmp/lastid) fi - #dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") - dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" order by id limit ${limit}") - while read -r id photo thumb micro; do + dboutput=$(mariadb "${db}" -B -N -q -e "select \`id\`, \`photo\`, \`thumb\`, \`micro\` from \`contact\` where \`id\` > ${lastid} and \`photo\` like \"https:\/\/${url}/avatar/%\" and (\`id\` in (select \`cid\` from \`user-contact\`) or \`id\` in (select \`uid\` from \`user\`) or \`id\` in (select \`contact-id\` from \`group_member\`)) order by id limit ${limit}") + echo "${dboutput}" | while read -r id photo thumb micro; do if [[ -n "${id}" && -n "${photo}" && -n "${thumb}" && -n "${micro}" ]]; then nx=$(("${nx}" + 1)) nt=$(("${nt}" + 1)) @@ -76,9 +70,7 @@ until [[ $((nt + limit)) -gt ${dbcount} ]]; do curl -s "${micro}" | file - | grep -q -e "text" -e "empty" -e "symbolic link" -e "directory"; then #Request the user data to be regenerated in the system through the database mariadb "${db}" -N -B -q -e "update contact set avatar= \"\", photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" - fi + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority) values (\"UpdateContact\", \"[${id}]\", 20);" echo "${id} ${photo}" error_found=1 fi @@ -103,17 +95,13 @@ until [[ $((nt + limit)) -gt ${dbcount} ]]; do rm -rf "${k_photo}" fi #Request the user data to be regenerated in the system through the database - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" - fi + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" else echo "${id}" #If no remote avatar is found, then we blank the photo/thumb/micro and let the avatar cache process fix them later mariadb "${db}" -e "update contact set photo = \"\", thumb = \"\", micro = \"\" where id = \"${id}\"" #Request the user data to be regenerated in the system through the database - if [[ $(mariadb "${db}" -N -B -q -e "select count(*) from workerqueue where command = \"UpdateContact\" and parameter = \"[${id}]\"" -gt 0) ]]; then - mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" - fi + mariadb "${db}" -N -B -q -e "insert ignore into workerqueue (command, parameter, priority, created) values (\"UpdateContact\", \"[${id}]\", 20, CURTIME());" fi error_found=1 k_photo_delta=$(echo "${photo}" | sed -e "s/.*?ts=//g") @@ -125,12 +113,12 @@ until [[ $((nt + limit)) -gt ${dbcount} ]]; do if [[ "${error_found}" -gt 0 ]]; then n=$((n + 1)) fi - lastid="${id}" - touch /tmp/lastid - echo "${n} ${nt} ${lastid}" > /tmp/lastid fi - printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %6d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}" - done < <(echo "${dboutput}") + lastid="${id}" + touch /tmp/lastid + echo "${n} ${nt} ${lastid}" >/tmp/lastid + printf "\rB. %5d Fd. %8d E. %8d Ct. %4d/%4d To. %8d/%8d Dt. %d " "${batch}" "${n}" "${nt}" "${nx}" "${limit}" "${lastid}" "${maxid}" "${k_photo_delta}" + done done printf "\nFixing folders and moving to avatar cache...\n" #sudo -u "${user}" bin/console movetoavatarcache #&> /dev/null