From 32c59f044030b875607fcca8c45723968b40bfb9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 3 Feb 2024 13:30:02 -0700 Subject: [PATCH 01/21] Update TransformImports command, fix import service condition --- app/Console/Commands/TransformImports.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/Console/Commands/TransformImports.php b/app/Console/Commands/TransformImports.php index b884011781..a5a4dbb7ab 100644 --- a/app/Console/Commands/TransformImports.php +++ b/app/Console/Commands/TransformImports.php @@ -70,6 +70,11 @@ public function handle() } $idk = ImportService::getId($ip->user_id, $ip->creation_year, $ip->creation_month, $ip->creation_day); + if(!$idk) { + $ip->skip_missing_media = true; + $ip->save(); + continue; + } if(Storage::exists('imports/' . $id . '/' . $ip->filename) === false) { ImportService::clearAttempts($profile->id); From d3ff89e538076debf916386ba92b19eae81aa0c0 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 3 Feb 2024 13:31:11 -0700 Subject: [PATCH 02/21] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6e4513d5d..cc42cb51f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ - Update LoginController, fix captcha validation error message ([0325e171](https://github.com/pixelfed/pixelfed/commit/0325e171)) - Update ApiV1Controller, properly cast boolean sensitive parameter. Fixes #4888 ([0aff126a](https://github.com/pixelfed/pixelfed/commit/0aff126a)) - Update AccountImport.vue, fix new IG export format ([59aa6a4b](https://github.com/pixelfed/pixelfed/commit/59aa6a4b)) +- Update TransformImports command, fix import service condition ([32c59f04](https://github.com/pixelfed/pixelfed/commit/32c59f04)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 7caed381fb784c0b228423888b69d1fb38dbfbe7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 02:40:04 -0700 Subject: [PATCH 03/21] Update AP helpers, more efficently update post counts --- .../Commands/AccountPostCountStatUpdate.php | 57 +++++++++++++++++++ app/Console/Kernel.php | 3 +- app/Services/Account/AccountStatService.php | 26 +++++++++ app/Util/ActivityPub/Helpers.php | 3 +- 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 app/Console/Commands/AccountPostCountStatUpdate.php create mode 100644 app/Services/Account/AccountStatService.php diff --git a/app/Console/Commands/AccountPostCountStatUpdate.php b/app/Console/Commands/AccountPostCountStatUpdate.php new file mode 100644 index 0000000000..6d5ba00a69 --- /dev/null +++ b/app/Console/Commands/AccountPostCountStatUpdate.php @@ -0,0 +1,57 @@ +count(); + if($statusCount != $acct['statuses_count']) { + $profile = Profile::find($id); + if(!$profile) { + AccountStatService::removeFromPostCount($id); + continue; + } + $profile->status_count = $statusCount; + $profile->save(); + AccountService::del($id); + } + AccountStatService::removeFromPostCount($id); + } + return; + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index aadaa1f7ae..7953ea783b 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -38,7 +38,7 @@ protected function schedule(Schedule $schedule) } if(config('import.instagram.enabled')) { - $schedule->command('app:transform-imports')->everyFourMinutes(); + $schedule->command('app:transform-imports')->everyTenMinutes(); $schedule->command('app:import-upload-garbage-collection')->hourlyAt(51); $schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37); $schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32); @@ -49,6 +49,7 @@ protected function schedule(Schedule $schedule) } $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21'); $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25); + $schedule->command('app:account-post-count-stat-update')->everySixHours(25); } /** diff --git a/app/Services/Account/AccountStatService.php b/app/Services/Account/AccountStatService.php new file mode 100644 index 0000000000..0b5d45a3e1 --- /dev/null +++ b/app/Services/Account/AccountStatService.php @@ -0,0 +1,26 @@ +onQueue('low'); + AccountStatService::incrementPostCount($pid); if( $status->in_reply_to_id === null && in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) From ddf7f09ad41bd2b935ebfb360286e48a4c6370f0 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 02:40:59 -0700 Subject: [PATCH 04/21] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc42cb51f7..c37659fba4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ - Update ApiV1Controller, properly cast boolean sensitive parameter. Fixes #4888 ([0aff126a](https://github.com/pixelfed/pixelfed/commit/0aff126a)) - Update AccountImport.vue, fix new IG export format ([59aa6a4b](https://github.com/pixelfed/pixelfed/commit/59aa6a4b)) - Update TransformImports command, fix import service condition ([32c59f04](https://github.com/pixelfed/pixelfed/commit/32c59f04)) +- Update AP helpers, more efficently update post count ([7caed381](https://github.com/pixelfed/pixelfed/commit/7caed381)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From b81ae5773f82e823ce53892e4abdc8cdae509c78 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 02:50:48 -0700 Subject: [PATCH 05/21] Update AP helpers, refactor post count decrement logic --- app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php | 7 ++----- app/Jobs/StatusPipeline/RemoteStatusDelete.php | 6 ++---- app/Services/Account/AccountStatService.php | 5 +++++ app/Util/ActivityPub/Helpers.php | 2 -- app/Util/ActivityPub/Inbox.php | 2 -- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php index 4969fca2fe..77cd5286fa 100644 --- a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php +++ b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php @@ -22,9 +22,9 @@ use App\Services\AccountService; use App\Services\NetworkTimelineService; use App\Services\StatusService; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Jobs\MediaPipeline\MediaDeletePipeline; use Cache; +use App\Services\Account\AccountStatService; class DeleteRemoteStatusPipeline implements ShouldQueue { @@ -56,10 +56,7 @@ public function handle() { $status = $this->status; - if(AccountService::get($status->profile_id, true)) { - DecrementPostCount::dispatch($status->profile_id)->onQueue('low'); - } - + AccountStatService::decrementPostCount($status->profile_id); NetworkTimelineService::del($status->id); StatusService::del($status->id, true); Bookmark::whereStatusId($status->id)->delete(); diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index 07a2f6236d..a816077555 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -39,8 +39,8 @@ use App\Services\CollectionService; use App\Services\StatusService; use App\Jobs\MediaPipeline\MediaDeletePipeline; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Services\NotificationService; +use App\Services\Account\AccountStatService; class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing { @@ -109,9 +109,7 @@ public function handle() } StatusService::del($status->id, true); - - DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox'); - + AccountStatService::decrementPostCount($status->profile_id); return $this->unlinkRemoveMedia($status); } diff --git a/app/Services/Account/AccountStatService.php b/app/Services/Account/AccountStatService.php index 0b5d45a3e1..12fd3f94f6 100644 --- a/app/Services/Account/AccountStatService.php +++ b/app/Services/Account/AccountStatService.php @@ -14,6 +14,11 @@ public static function incrementPostCount($pid) return Redis::zadd(self::REFRESH_CACHE_KEY, $pid, $pid); } + public static function decrementPostCount($pid) + { + return Redis::zadd(self::REFRESH_CACHE_KEY, $pid, $pid); + } + public static function removeFromPostCount($pid) { return Redis::zrem(self::REFRESH_CACHE_KEY, $pid); diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 4e71a2fae2..511ef25021 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -39,8 +39,6 @@ use App\Util\Media\License; use App\Models\Poll; use Illuminate\Contracts\Cache\LockTimeoutException; -use App\Jobs\ProfilePipeline\IncrementPostCount; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Services\DomainService; use App\Services\UserFilterService; use App\Services\Account\AccountStatService; diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index e26f0a48c1..5c9959e17f 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -48,8 +48,6 @@ use App\Services\NetworkTimelineService; use App\Models\Conversation; use App\Models\RemoteReport; -use App\Jobs\ProfilePipeline\IncrementPostCount; -use App\Jobs\ProfilePipeline\DecrementPostCount; use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline; class Inbox From 8b843d620cea62b3a272f8f305574c7ad0ab25a0 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 02:53:40 -0700 Subject: [PATCH 06/21] Update ProfilePipeline jobs --- .../ProfilePipeline/DecrementPostCount.php | 15 +----- .../ProfilePipeline/IncrementPostCount.php | 49 ++----------------- 2 files changed, 5 insertions(+), 59 deletions(-) diff --git a/app/Jobs/ProfilePipeline/DecrementPostCount.php b/app/Jobs/ProfilePipeline/DecrementPostCount.php index b463f1dda7..74d0523b5f 100644 --- a/app/Jobs/ProfilePipeline/DecrementPostCount.php +++ b/app/Jobs/ProfilePipeline/DecrementPostCount.php @@ -35,18 +35,7 @@ public function __construct($id) */ public function handle() { - $id = $this->id; - - $profile = Profile::find($id); - - if(!$profile) { - return 1; - } - - $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0; - $profile->save(); - AccountService::del($id); - - return 1; + // deprecated + return; } } diff --git a/app/Jobs/ProfilePipeline/IncrementPostCount.php b/app/Jobs/ProfilePipeline/IncrementPostCount.php index 1a94f1e6c3..a1f9ceca78 100644 --- a/app/Jobs/ProfilePipeline/IncrementPostCount.php +++ b/app/Jobs/ProfilePipeline/IncrementPostCount.php @@ -14,42 +14,12 @@ use App\Status; use App\Services\AccountService; -class IncrementPostCount implements ShouldQueue, ShouldBeUniqueUntilProcessing +class IncrementPostCount implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public $id; - public $timeout = 900; - public $tries = 3; - public $maxExceptions = 1; - public $failOnTimeout = true; - - /** - * The number of seconds after which the job's unique lock will be released. - * - * @var int - */ - public $uniqueFor = 3600; - - /** - * Get the unique ID for the job. - */ - public function uniqueId(): string - { - return 'propipe:ipc:' . $this->id; - } - - /** - * Get the middleware the job should pass through. - * - * @return array - */ - public function middleware(): array - { - return [(new WithoutOverlapping("propipe:ipc:{$this->id}"))->shared()->dontRelease()]; - } - /** * Create a new job instance. * @@ -67,20 +37,7 @@ public function __construct($id) */ public function handle() { - $id = $this->id; - - $profile = Profile::find($id); - - if(!$profile) { - return 1; - } - - $profile->status_count = $profile->status_count + 1; - $profile->last_status_at = now(); - $profile->save(); - AccountService::del($id); - AccountService::get($id); - - return 1; + // deprecated + return; } } From 09ca96cc2be7910287e5064b1d590e092865cb13 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 02:54:37 -0700 Subject: [PATCH 07/21] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c37659fba4..6e0d1f1040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ - Update AccountImport.vue, fix new IG export format ([59aa6a4b](https://github.com/pixelfed/pixelfed/commit/59aa6a4b)) - Update TransformImports command, fix import service condition ([32c59f04](https://github.com/pixelfed/pixelfed/commit/32c59f04)) - Update AP helpers, more efficently update post count ([7caed381](https://github.com/pixelfed/pixelfed/commit/7caed381)) +- Update AP helpers, refactor post count decrement logic ([b81ae577](https://github.com/pixelfed/pixelfed/commit/b81ae577)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 00ed330cf3aa2442c73591c34c2ae8dae8cb5c0f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 03:16:57 -0700 Subject: [PATCH 08/21] Update AP helpers, fix sensitive bug --- app/Util/ActivityPub/Helpers.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 511ef25021..5819dc0bc8 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -548,10 +548,11 @@ public static function storeStatus($url, $profile, $activity) public static function getSensitive($activity, $url) { - $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($url); - $url = isset($activity['url']) ? self::pluckval($activity['url']) : $id; - $urlDomain = parse_url($url, PHP_URL_HOST); + if(!$url || !strlen($url)) { + return true; + } + $urlDomain = parse_url($url, PHP_URL_HOST); $cw = isset($activity['sensitive']) ? (bool) $activity['sensitive'] : false; if(in_array($urlDomain, InstanceService::getNsfwDomains())) { From 152b6eab9acd9b008ac961ffaa6ad73794c3587c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 03:18:58 -0700 Subject: [PATCH 09/21] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e0d1f1040..becf2298d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ - Update TransformImports command, fix import service condition ([32c59f04](https://github.com/pixelfed/pixelfed/commit/32c59f04)) - Update AP helpers, more efficently update post count ([7caed381](https://github.com/pixelfed/pixelfed/commit/7caed381)) - Update AP helpers, refactor post count decrement logic ([b81ae577](https://github.com/pixelfed/pixelfed/commit/b81ae577)) +- Update AP helpers, fix sensitive bug ([00ed330c](https://github.com/pixelfed/pixelfed/commit/00ed330c)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 4d4013896c936a7541277ccda4a6832b31ffbcd8 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 07:11:05 -0700 Subject: [PATCH 10/21] Update NotificationEpochUpdatePipeline, use more efficient query --- .../InternalPipeline/NotificationEpochUpdatePipeline.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php b/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php index 477b1f9b38..79df5aa9ad 100644 --- a/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php +++ b/app/Jobs/InternalPipeline/NotificationEpochUpdatePipeline.php @@ -61,7 +61,12 @@ public function __construct() */ public function handle(): void { - $rec = Notification::where('created_at', '>', now()->subMonths(6))->first(); + $pid = Cache::get(NotificationService::EPOCH_CACHE_KEY . '6'); + if($pid && $pid > 1) { + $rec = Notification::where('id', '>', $pid)->whereDate('created_at', now()->subMonths(6)->format('Y-m-d'))->first(); + } else { + $rec = Notification::whereDate('created_at', now()->subMonths(6)->format('Y-m-d'))->first(); + } $id = 1; if($rec) { $id = $rec->id; From fa97a1f38e1a0f37c4f1a59e4995e8e78ffb8802 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 07:18:05 -0700 Subject: [PATCH 11/21] Update notification pipelines, fix non-local saving --- app/Jobs/CommentPipeline/CommentPipeline.php | 28 ++++++++++--------- app/Jobs/FollowPipeline/FollowPipeline.php | 22 ++++++++------- app/Jobs/LikePipeline/LikePipeline.php | 22 ++++++++------- .../StatusPipeline/StatusReplyPipeline.php | 26 +++++++++-------- 4 files changed, 53 insertions(+), 45 deletions(-) diff --git a/app/Jobs/CommentPipeline/CommentPipeline.php b/app/Jobs/CommentPipeline/CommentPipeline.php index 3b2d896af5..1917ecea5f 100644 --- a/app/Jobs/CommentPipeline/CommentPipeline.php +++ b/app/Jobs/CommentPipeline/CommentPipeline.php @@ -91,19 +91,21 @@ public function handle() return; } - DB::transaction(function() use($target, $actor, $comment) { - $notification = new Notification(); - $notification->profile_id = $target->id; - $notification->actor_id = $actor->id; - $notification->action = 'comment'; - $notification->item_id = $comment->id; - $notification->item_type = "App\Status"; - $notification->save(); - - NotificationService::setNotification($notification); - NotificationService::set($notification->profile_id, $notification->id); - StatusService::del($comment->id); - }); + if($target->user_id && $target->domain === null) { + DB::transaction(function() use($target, $actor, $comment) { + $notification = new Notification(); + $notification->profile_id = $target->id; + $notification->actor_id = $actor->id; + $notification->action = 'comment'; + $notification->item_id = $comment->id; + $notification->item_type = "App\Status"; + $notification->save(); + + NotificationService::setNotification($notification); + NotificationService::set($notification->profile_id, $notification->id); + StatusService::del($comment->id); + }); + } if($exists = Cache::get('status:replies:all:' . $status->id)) { if($exists && $exists->count() == 3) { diff --git a/app/Jobs/FollowPipeline/FollowPipeline.php b/app/Jobs/FollowPipeline/FollowPipeline.php index 2253343045..67733919fb 100644 --- a/app/Jobs/FollowPipeline/FollowPipeline.php +++ b/app/Jobs/FollowPipeline/FollowPipeline.php @@ -72,16 +72,18 @@ public function handle() $target->save(); AccountService::del($target->id); - try { - $notification = new Notification(); - $notification->profile_id = $target->id; - $notification->actor_id = $actor->id; - $notification->action = 'follow'; - $notification->item_id = $target->id; - $notification->item_type = "App\Profile"; - $notification->save(); - } catch (Exception $e) { - Log::error($e); + if($target->user_id && $target->domain === null) { + try { + $notification = new Notification(); + $notification->profile_id = $target->id; + $notification->actor_id = $actor->id; + $notification->action = 'follow'; + $notification->item_id = $target->id; + $notification->item_type = "App\Profile"; + $notification->save(); + } catch (Exception $e) { + Log::error($e); + } } } } diff --git a/app/Jobs/LikePipeline/LikePipeline.php b/app/Jobs/LikePipeline/LikePipeline.php index b44c90c8b9..e55c64f805 100644 --- a/app/Jobs/LikePipeline/LikePipeline.php +++ b/app/Jobs/LikePipeline/LikePipeline.php @@ -79,16 +79,18 @@ public function handle() return true; } - try { - $notification = new Notification(); - $notification->profile_id = $status->profile_id; - $notification->actor_id = $actor->id; - $notification->action = 'like'; - $notification->item_id = $status->id; - $notification->item_type = "App\Status"; - $notification->save(); - - } catch (Exception $e) { + if($status->uri === null && $status->object_url === null && $status->url === null) { + try { + $notification = new Notification(); + $notification->profile_id = $status->profile_id; + $notification->actor_id = $actor->id; + $notification->action = 'like'; + $notification->item_id = $status->id; + $notification->item_type = "App\Status"; + $notification->save(); + + } catch (Exception $e) { + } } } diff --git a/app/Jobs/StatusPipeline/StatusReplyPipeline.php b/app/Jobs/StatusPipeline/StatusReplyPipeline.php index 35238d293e..d8af7b96b6 100644 --- a/app/Jobs/StatusPipeline/StatusReplyPipeline.php +++ b/app/Jobs/StatusPipeline/StatusReplyPipeline.php @@ -87,18 +87,20 @@ public function handle() Cache::forget('status:replies:all:' . $reply->id); Cache::forget('status:replies:all:' . $status->id); - DB::transaction(function() use($target, $actor, $status) { - $notification = new Notification(); - $notification->profile_id = $target->id; - $notification->actor_id = $actor->id; - $notification->action = 'comment'; - $notification->item_id = $status->id; - $notification->item_type = "App\Status"; - $notification->save(); - - NotificationService::setNotification($notification); - NotificationService::set($notification->profile_id, $notification->id); - }); + if($target->user_id && $target->domain === null) { + DB::transaction(function() use($target, $actor, $status) { + $notification = new Notification(); + $notification->profile_id = $target->id; + $notification->actor_id = $actor->id; + $notification->action = 'comment'; + $notification->item_id = $status->id; + $notification->item_type = "App\Status"; + $notification->save(); + + NotificationService::setNotification($notification); + NotificationService::set($notification->profile_id, $notification->id); + }); + } if($exists = Cache::get('status:replies:all:' . $reply->id)) { if($exists && $exists->count() == 3) { From 80e0ada946c489497835b41fa670286af512da60 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 4 Feb 2024 07:18:54 -0700 Subject: [PATCH 12/21] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index becf2298d4..591025bfd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,8 @@ - Update AP helpers, more efficently update post count ([7caed381](https://github.com/pixelfed/pixelfed/commit/7caed381)) - Update AP helpers, refactor post count decrement logic ([b81ae577](https://github.com/pixelfed/pixelfed/commit/b81ae577)) - Update AP helpers, fix sensitive bug ([00ed330c](https://github.com/pixelfed/pixelfed/commit/00ed330c)) +- Update NotificationEpochUpdatePipeline, use more efficient query ([4d401389](https://github.com/pixelfed/pixelfed/commit/4d401389)) +- Update notification pipelines, fix non-local saving ([fa97a1f3](https://github.com/pixelfed/pixelfed/commit/fa97a1f3)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 240e6bbe4f57b320bb00a3b30f0a1a907c8975d7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 02:47:34 -0700 Subject: [PATCH 13/21] Update NodeinfoService, disable redirects --- app/Services/NodeinfoService.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/Services/NodeinfoService.php b/app/Services/NodeinfoService.php index 10575ff9f2..6284538f08 100644 --- a/app/Services/NodeinfoService.php +++ b/app/Services/NodeinfoService.php @@ -22,7 +22,10 @@ public static function get($domain) $wk = $url . '/.well-known/nodeinfo'; try { - $res = Http::withHeaders($headers) + $res = Http::withOptions([ + 'allow_redirects' => false, + ]) + ->withHeaders($headers) ->timeout(5) ->get($wk); } catch (RequestException $e) { @@ -61,7 +64,10 @@ public static function get($domain) } try { - $res = Http::withHeaders($headers) + $res = Http::withOptions([ + 'allow_redirects' => false, + ]) + ->withHeaders($headers) ->timeout(5) ->get($href); } catch (RequestException $e) { From 289cad470b0a04c2836e19aa57d6566df95ce668 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 02:49:29 -0700 Subject: [PATCH 14/21] Update Instance model, add entity casts --- app/Instance.php | 120 ++++++++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 53 deletions(-) diff --git a/app/Instance.php b/app/Instance.php index 6a7b8e6f2d..77752d4985 100644 --- a/app/Instance.php +++ b/app/Instance.php @@ -6,63 +6,77 @@ class Instance extends Model { - protected $fillable = ['domain', 'banned', 'auto_cw', 'unlisted', 'notes']; + protected $casts = [ + 'last_crawled_at' => 'datetime', + 'actors_last_synced_at' => 'datetime', + 'notes' => 'array', + 'nodeinfo_last_fetched' => 'datetime', + 'delivery_next_after' => 'datetime', + ]; - public function profiles() - { - return $this->hasMany(Profile::class, 'domain', 'domain'); - } + protected $fillable = [ + 'domain', + 'banned', + 'auto_cw', + 'unlisted', + 'notes' + ]; - public function statuses() - { - return $this->hasManyThrough( - Status::class, - Profile::class, - 'domain', - 'profile_id', - 'domain', - 'id' - ); - } + public function profiles() + { + return $this->hasMany(Profile::class, 'domain', 'domain'); + } - public function reported() - { - return $this->hasManyThrough( - Report::class, - Profile::class, - 'domain', - 'reported_profile_id', - 'domain', - 'id' - ); - } + public function statuses() + { + return $this->hasManyThrough( + Status::class, + Profile::class, + 'domain', + 'profile_id', + 'domain', + 'id' + ); + } - public function reports() - { - return $this->hasManyThrough( - Report::class, - Profile::class, - 'domain', - 'profile_id', - 'domain', - 'id' - ); - } + public function reported() + { + return $this->hasManyThrough( + Report::class, + Profile::class, + 'domain', + 'reported_profile_id', + 'domain', + 'id' + ); + } - public function media() - { - return $this->hasManyThrough( - Media::class, - Profile::class, - 'domain', - 'profile_id', - 'domain', - 'id' - ); - } + public function reports() + { + return $this->hasManyThrough( + Report::class, + Profile::class, + 'domain', + 'profile_id', + 'domain', + 'id' + ); + } - public function getUrl() - { - return url("/i/admin/instances/show/{$this->id}"); - } + public function media() + { + return $this->hasManyThrough( + Media::class, + Profile::class, + 'domain', + 'profile_id', + 'domain', + 'id' + ); + } + + public function getUrl() + { + return url("/i/admin/instances/show/{$this->id}"); + } } From ac01f51ab66df0b4f4657e70bbe0befe2c022b9f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 02:51:50 -0700 Subject: [PATCH 15/21] Update FetchNodeinfoPipeline, use more efficient dispatch --- .../FetchNodeinfoPipeline.php | 105 +++++++++++------- 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php b/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php index b8c79d67f6..943281bb4d 100644 --- a/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php +++ b/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php @@ -4,6 +4,7 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -12,45 +13,71 @@ use App\Instance; use App\Profile; use App\Services\NodeinfoService; +use Illuminate\Contracts\Cache\Repository; +use Illuminate\Support\Facades\Cache; -class FetchNodeinfoPipeline implements ShouldQueue +class FetchNodeinfoPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - - protected $instance; - - /** - * Create a new job instance. - * - * @return void - */ - public function __construct(Instance $instance) - { - $this->instance = $instance; - } - - /** - * Execute the job. - * - * @return void - */ - public function handle() - { - $instance = $this->instance; - - $ni = NodeinfoService::get($instance->domain); - if($ni) { - if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) { - $software = $ni['software']['name']; - $instance->software = strtolower(strip_tags($software)); - $instance->last_crawled_at = now(); - $instance->user_count = Profile::whereDomain($instance->domain)->count(); - $instance->save(); - } - } else { - $instance->user_count = Profile::whereDomain($instance->domain)->count(); - $instance->last_crawled_at = now(); - $instance->save(); - } - } + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + + protected $instance; + + /** + * Create a new job instance. + * + * @return void + */ + public function __construct(Instance $instance) + { + $this->instance = $instance; + } + + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 14400; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return $this->instance->id; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $instance = $this->instance; + + if( $instance->nodeinfo_last_fetched && + $instance->nodeinfo_last_fetched->gt(now()->subHours(12)) || + $instance->delivery_timeout && + $instance->delivery_next_after->gt(now()) + ) { + return; + } + + $ni = NodeinfoService::get($instance->domain); + $instance->last_crawled_at = now(); + if($ni) { + if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) { + $software = $ni['software']['name']; + $instance->software = strtolower(strip_tags($software)); + $instance->user_count = Profile::whereDomain($instance->domain)->count(); + $instance->nodeinfo_last_fetched = now(); + $instance->save(); + } + } else { + $instance->delivery_timeout = 1; + $instance->delivery_next_after = now()->addHours(14); + $instance->save(); + } + } } From 1e3acadefb200fbb4d594909083c5a275e413b77 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 02:52:37 -0700 Subject: [PATCH 16/21] Update horizon.php config --- config/horizon.php | 395 +++++++++++++++++++++++---------------------- 1 file changed, 198 insertions(+), 197 deletions(-) diff --git a/config/horizon.php b/config/horizon.php index f9cfd960e8..5aa37f2fe7 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -2,201 +2,202 @@ return [ - /* - |-------------------------------------------------------------------------- - | Horizon Domain - |-------------------------------------------------------------------------- - | - | This is the subdomain where Horizon will be accessible from. If this - | setting is null, Horizon will reside under the same domain as the - | application. Otherwise, this value will serve as the subdomain. - | - */ - - 'domain' => null, - - /* - |-------------------------------------------------------------------------- - | Horizon Path - |-------------------------------------------------------------------------- - | - | This is the URI path where Horizon will be accessible from. Feel free - | to change this path to anything you like. Note that the URI will not - | affect the paths of its internal API that aren't exposed to users. - | - */ - - 'path' => 'horizon', - - /* - |-------------------------------------------------------------------------- - | Horizon Redis Connection - |-------------------------------------------------------------------------- - | - | This is the name of the Redis connection where Horizon will store the - | meta information required for it to function. It includes the list - | of supervisors, failed jobs, job metrics, and other information. - | - */ - - 'use' => 'default', - - /* - |-------------------------------------------------------------------------- - | Horizon Redis Prefix - |-------------------------------------------------------------------------- - | - | This prefix will be used when storing all Horizon data in Redis. You - | may modify the prefix when you are running multiple installations - | of Horizon on the same server so that they don't have problems. - | - */ - - 'prefix' => env('HORIZON_PREFIX', 'horizon-'), - - /* - |-------------------------------------------------------------------------- - | Horizon Route Middleware - |-------------------------------------------------------------------------- - | - | These middleware will get attached onto each Horizon route, giving you - | the chance to add your own middleware to this list or change any of - | the existing middleware. Or, you can simply stick with this list. - | - */ - - 'middleware' => ['web'], - - /* - |-------------------------------------------------------------------------- - | Queue Wait Time Thresholds - |-------------------------------------------------------------------------- - | - | This option allows you to configure when the LongWaitDetected event - | will be fired. Every connection / queue combination may have its - | own, unique threshold (in seconds) before this event is fired. - | - */ - - 'waits' => [ - 'redis:feed' => 30, - 'redis:follow' => 30, - 'redis:shared' => 30, - 'redis:default' => 30, - 'redis:inbox' => 30, - 'redis:low' => 30, - 'redis:high' => 30, - 'redis:delete' => 30, - 'redis:story' => 30, - 'redis:mmo' => 30, - ], - - /* - |-------------------------------------------------------------------------- - | Job Trimming Times - |-------------------------------------------------------------------------- - | - | Here you can configure for how long (in minutes) you desire Horizon to - | persist the recent and failed jobs. Typically, recent jobs are kept - | for one hour while all failed jobs are stored for an entire week. - | - */ - - 'trim' => [ - 'recent' => 60, - 'pending' => 60, - 'completed' => 60, - 'recent_failed' => 10080, - 'failed' => 10080, - 'monitored' => 10080, - ], - - /* - |-------------------------------------------------------------------------- - | Metrics - |-------------------------------------------------------------------------- - | - | Here you can configure how many snapshots should be kept to display in - | the metrics graph. This will get used in combination with Horizon's - | `horizon:snapshot` schedule to define how long to retain metrics. - | - */ - - 'metrics' => [ - 'trim_snapshots' => [ - 'job' => 24, - 'queue' => 24, - ], - ], - - /* - |-------------------------------------------------------------------------- - | Fast Termination - |-------------------------------------------------------------------------- - | - | When this option is enabled, Horizon's "terminate" command will not - | wait on all of the workers to terminate unless the --wait option - | is provided. Fast termination can shorten deployment delay by - | allowing a new instance of Horizon to start while the last - | instance will continue to terminate each of its workers. - | - */ - - 'fast_termination' => false, - - /* - |-------------------------------------------------------------------------- - | Memory Limit (MB) - |-------------------------------------------------------------------------- - | - | This value describes the maximum amount of memory the Horizon worker - | may consume before it is terminated and restarted. You should set - | this value according to the resources available to your server. - | - */ - - 'memory_limit' => env('HORIZON_MEMORY_LIMIT', 64), - - /* - |-------------------------------------------------------------------------- - | Queue Worker Configuration - |-------------------------------------------------------------------------- - | - | Here you may define the queue worker settings used by your application - | in all environments. These supervisors and settings handle all your - | queued jobs and will be provisioned by Horizon during deployment. - | - */ - - 'environments' => [ - 'production' => [ - 'supervisor-1' => [ - 'connection' => 'redis', - 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo'], - 'balance' => env('HORIZON_BALANCE_STRATEGY', 'auto'), - 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), - 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 20), - 'memory' => env('HORIZON_SUPERVISOR_MEMORY', 64), - 'tries' => env('HORIZON_SUPERVISOR_TRIES', 3), - 'nice' => env('HORIZON_SUPERVISOR_NICE', 0), - 'timeout' => env('HORIZON_SUPERVISOR_TIMEOUT', 300), - ], - ], - - 'local' => [ - 'supervisor-1' => [ - 'connection' => 'redis', - 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo'], - 'balance' => 'auto', - 'minProcesses' => 1, - 'maxProcesses' => 20, - 'memory' => 128, - 'tries' => 3, - 'nice' => 0, - 'timeout' => 300 - ], - ], - ], - - 'darkmode' => env('HORIZON_DARKMODE', false), + /* + |-------------------------------------------------------------------------- + | Horizon Domain + |-------------------------------------------------------------------------- + | + | This is the subdomain where Horizon will be accessible from. If this + | setting is null, Horizon will reside under the same domain as the + | application. Otherwise, this value will serve as the subdomain. + | + */ + + 'domain' => null, + + /* + |-------------------------------------------------------------------------- + | Horizon Path + |-------------------------------------------------------------------------- + | + | This is the URI path where Horizon will be accessible from. Feel free + | to change this path to anything you like. Note that the URI will not + | affect the paths of its internal API that aren't exposed to users. + | + */ + + 'path' => 'horizon', + + /* + |-------------------------------------------------------------------------- + | Horizon Redis Connection + |-------------------------------------------------------------------------- + | + | This is the name of the Redis connection where Horizon will store the + | meta information required for it to function. It includes the list + | of supervisors, failed jobs, job metrics, and other information. + | + */ + + 'use' => 'default', + + /* + |-------------------------------------------------------------------------- + | Horizon Redis Prefix + |-------------------------------------------------------------------------- + | + | This prefix will be used when storing all Horizon data in Redis. You + | may modify the prefix when you are running multiple installations + | of Horizon on the same server so that they don't have problems. + | + */ + + 'prefix' => env('HORIZON_PREFIX', 'horizon-'), + + /* + |-------------------------------------------------------------------------- + | Horizon Route Middleware + |-------------------------------------------------------------------------- + | + | These middleware will get attached onto each Horizon route, giving you + | the chance to add your own middleware to this list or change any of + | the existing middleware. Or, you can simply stick with this list. + | + */ + + 'middleware' => ['web'], + + /* + |-------------------------------------------------------------------------- + | Queue Wait Time Thresholds + |-------------------------------------------------------------------------- + | + | This option allows you to configure when the LongWaitDetected event + | will be fired. Every connection / queue combination may have its + | own, unique threshold (in seconds) before this event is fired. + | + */ + + 'waits' => [ + 'redis:feed' => 30, + 'redis:follow' => 30, + 'redis:shared' => 30, + 'redis:default' => 30, + 'redis:inbox' => 30, + 'redis:low' => 30, + 'redis:high' => 30, + 'redis:delete' => 30, + 'redis:story' => 30, + 'redis:mmo' => 30, + 'redis:intbg' => 30, + ], + + /* + |-------------------------------------------------------------------------- + | Job Trimming Times + |-------------------------------------------------------------------------- + | + | Here you can configure for how long (in minutes) you desire Horizon to + | persist the recent and failed jobs. Typically, recent jobs are kept + | for one hour while all failed jobs are stored for an entire week. + | + */ + + 'trim' => [ + 'recent' => 60, + 'pending' => 60, + 'completed' => 60, + 'recent_failed' => 10080, + 'failed' => 10080, + 'monitored' => 10080, + ], + + /* + |-------------------------------------------------------------------------- + | Metrics + |-------------------------------------------------------------------------- + | + | Here you can configure how many snapshots should be kept to display in + | the metrics graph. This will get used in combination with Horizon's + | `horizon:snapshot` schedule to define how long to retain metrics. + | + */ + + 'metrics' => [ + 'trim_snapshots' => [ + 'job' => 24, + 'queue' => 24, + ], + ], + + /* + |-------------------------------------------------------------------------- + | Fast Termination + |-------------------------------------------------------------------------- + | + | When this option is enabled, Horizon's "terminate" command will not + | wait on all of the workers to terminate unless the --wait option + | is provided. Fast termination can shorten deployment delay by + | allowing a new instance of Horizon to start while the last + | instance will continue to terminate each of its workers. + | + */ + + 'fast_termination' => false, + + /* + |-------------------------------------------------------------------------- + | Memory Limit (MB) + |-------------------------------------------------------------------------- + | + | This value describes the maximum amount of memory the Horizon worker + | may consume before it is terminated and restarted. You should set + | this value according to the resources available to your server. + | + */ + + 'memory_limit' => env('HORIZON_MEMORY_LIMIT', 64), + + /* + |-------------------------------------------------------------------------- + | Queue Worker Configuration + |-------------------------------------------------------------------------- + | + | Here you may define the queue worker settings used by your application + | in all environments. These supervisors and settings handle all your + | queued jobs and will be provisioned by Horizon during deployment. + | + */ + + 'environments' => [ + 'production' => [ + 'supervisor-1' => [ + 'connection' => 'redis', + 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo', 'intbg'], + 'balance' => env('HORIZON_BALANCE_STRATEGY', 'auto'), + 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), + 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 20), + 'memory' => env('HORIZON_SUPERVISOR_MEMORY', 64), + 'tries' => env('HORIZON_SUPERVISOR_TRIES', 3), + 'nice' => env('HORIZON_SUPERVISOR_NICE', 0), + 'timeout' => env('HORIZON_SUPERVISOR_TIMEOUT', 300), + ], + ], + + 'local' => [ + 'supervisor-1' => [ + 'connection' => 'redis', + 'queue' => ['high', 'default', 'follow', 'shared', 'inbox', 'feed', 'low', 'story', 'delete', 'mmo', 'intbg'], + 'balance' => 'auto', + 'minProcesses' => 1, + 'maxProcesses' => 20, + 'memory' => 128, + 'tries' => 3, + 'nice' => 0, + 'timeout' => 300 + ], + ], + ], + + 'darkmode' => env('HORIZON_DARKMODE', false), ]; From 01b33fb37efdb595709b5379ba5482b72cf4093f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 03:43:20 -0700 Subject: [PATCH 17/21] Update PublicApiController, consume InstanceService blocked domains for account and statuses endpoints --- app/Http/Controllers/PublicApiController.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/Http/Controllers/PublicApiController.php b/app/Http/Controllers/PublicApiController.php index f888eb512e..78008eda48 100644 --- a/app/Http/Controllers/PublicApiController.php +++ b/app/Http/Controllers/PublicApiController.php @@ -42,6 +42,7 @@ use App\Jobs\StatusPipeline\NewStatusPipeline; use League\Fractal\Serializer\ArraySerializer; use League\Fractal\Pagination\IlluminatePaginatorAdapter; +use App\Services\InstanceService; class PublicApiController extends Controller { @@ -661,6 +662,10 @@ public function relationships(Request $request) public function account(Request $request, $id) { $res = AccountService::get($id); + if($res && isset($res['local'], $res['url']) && !$res['local']) { + $domain = parse_url($res['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } return response()->json($res); } @@ -680,6 +685,11 @@ public function accountStatuses(Request $request, $id) $profile = AccountService::get($id); abort_if(!$profile, 404); + if($profile && isset($profile['local'], $profile['url']) && !$profile['local']) { + $domain = parse_url($profile['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $limit = $request->limit ?? 9; $max_id = $request->max_id; $min_id = $request->min_id; From 5b284cacea474367701041093c445cff545ad742 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 04:41:12 -0700 Subject: [PATCH 18/21] Update ApiV1Controller, enforce blocked instance domain logic --- app/Http/Controllers/Api/ApiV1Controller.php | 69 ++++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index dd0cbd062d..b94a11c925 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -219,6 +219,10 @@ public function accountById(Request $request, $id) if(!$res) { return response()->json(['error' => 'Record not found'], 404); } + if($res && strpos($res['acct'], '@') != -1) { + $domain = parse_url($res['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } return $this->json($res); } @@ -483,6 +487,11 @@ public function accountFollowersById(Request $request, $id) $limit = $request->input('limit', 10); $napi = $request->has(self::PF_API_ENTITY_KEY); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + if(intval($pid) !== intval($account['id'])) { if($account['locked']) { if(!FollowerService::follows($pid, $account['id'])) { @@ -575,6 +584,11 @@ public function accountFollowingById(Request $request, $id) $limit = $request->input('limit', 10); $napi = $request->has(self::PF_API_ENTITY_KEY); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + if(intval($pid) !== intval($account['id'])) { if($account['locked']) { if(!FollowerService::follows($pid, $account['id'])) { @@ -676,6 +690,11 @@ public function accountStatusesById(Request $request, $id) return $this->json(['error' => 'Account not found'], 404); } + if($profile && strpos($profile['acct'], '@') != -1) { + $domain = parse_url($profile['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $limit = $request->limit ?? 20; $max_id = $request->max_id; $min_id = $request->min_id; @@ -766,6 +785,11 @@ public function accountFollowById(Request $request, $id) ->whereNull('status') ->findOrFail($id); + if($target && $target->domain) { + $domain = $target->domain; + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $private = (bool) $target->is_private; $remote = (bool) $target->domain; $blocked = UserFilter::whereUserId($target->id) @@ -1252,14 +1276,19 @@ public function statusFavouriteById(Request $request, $id) $user = $request->user(); abort_if($user->has_roles && !UserRoleService::can('can-like', $user->id), 403, 'Invalid permissions for this action'); - AccountService::setLastActive($user->id); - $status = StatusService::getMastodon($id, false); - abort_unless($status, 400); + abort_unless($status, 404); + + if($status && isset($status['account'], $status['account']['acct']) && strpos($status['account']['acct'], '@') != -1) { + $domain = parse_url($status['account']['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } $spid = $status['account']['id']; + AccountService::setLastActive($user->id); + if(intval($spid) !== intval($user->profile_id)) { if($status['visibility'] == 'private') { abort_if(!FollowerService::follows($user->profile_id, $spid), 403); @@ -1404,6 +1433,11 @@ public function accountFollowRequestAccept(Request $request, $id) return response()->json(['error' => 'Record not found'], 404); } + if($target && strpos($target['acct'], '@') != -1) { + $domain = parse_url($target['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $followRequest = FollowRequest::whereFollowingId($pid)->whereFollowerId($id)->first(); if(!$followRequest) { @@ -2011,6 +2045,11 @@ public function accountMuteById(Request $request, $id) $account = Profile::findOrFail($id); + if($account && $account->domain) { + $domain = $account->domain; + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $count = UserFilterService::muteCount($pid); $maxLimit = intval(config('instance.user_filters.max_user_mutes')); if($count == 0) { @@ -2653,6 +2692,11 @@ public function statusById(Request $request, $id) abort(404); } + if($res && isset($res['account'], $res['account']['acct'], $res['account']['url']) && strpos($res['account']['acct'], '@') != -1) { + $domain = parse_url($res['account']['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + $scope = $res['visibility']; if(!in_array($scope, ['public', 'unlisted'])) { if($scope === 'private') { @@ -2697,6 +2741,11 @@ public function statusContext(Request $request, $id) return response('', 404); } + if($status && isset($status['account'], $status['account']['acct']) && strpos($status['account']['acct'], '@') != -1) { + $domain = parse_url($status['account']['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } + if(intval($status['account']['id']) !== intval($user->profile_id)) { if($status['visibility'] == 'private') { if(!FollowerService::follows($user->profile_id, $status['account']['id'])) { @@ -2780,6 +2829,10 @@ public function statusRebloggedBy(Request $request, $id) $status = Status::findOrFail($id); $account = AccountService::get($status->profile_id, true); abort_if(!$account, 404); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } $author = intval($status->profile_id) === intval($pid) || $user->is_admin; $napi = $request->has(self::PF_API_ENTITY_KEY); @@ -2871,6 +2924,10 @@ public function statusFavouritedBy(Request $request, $id) $pid = $user->profile_id; $status = Status::findOrFail($id); $account = AccountService::get($status->profile_id, true); + if($account && strpos($account['acct'], '@') != -1) { + $domain = parse_url($account['url'], PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } abort_if(!$account, 404); $author = intval($status->profile_id) === intval($pid) || $user->is_admin; $napi = $request->has(self::PF_API_ENTITY_KEY); @@ -3200,7 +3257,11 @@ public function statusShare(Request $request, $id) abort_if($user->has_roles && !UserRoleService::can('can-share', $user->id), 403, 'Invalid permissions for this action'); AccountService::setLastActive($user->id); $status = Status::whereScope('public')->findOrFail($id); - + if($status && ($status->uri || $status->url || $status->object_url)) { + $url = $status->uri ?? $status->url ?? $status->object_url; + $domain = parse_url($url, PHP_URL_HOST); + abort_if(in_array($domain, InstanceService::getBannedDomains()), 404); + } if(intval($status->profile_id) !== intval($user->profile_id)) { if($status->scope == 'private') { abort_if(!FollowerService::follows($user->profile_id, $status->profile_id), 403); From 6921d3568e7662b55268a0b2ab5799758735b74a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 04:42:27 -0700 Subject: [PATCH 19/21] Add InstanceMananger command --- app/Console/Commands/InstanceManager.php | 298 +++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 app/Console/Commands/InstanceManager.php diff --git a/app/Console/Commands/InstanceManager.php b/app/Console/Commands/InstanceManager.php new file mode 100644 index 0000000000..a495d9617d --- /dev/null +++ b/app/Console/Commands/InstanceManager.php @@ -0,0 +1,298 @@ +recalculateStats(); + break; + + case 'Unlisted Instances': + return $this->viewUnlistedInstances(); + break; + + case 'Banned Instances': + return $this->viewBannedInstances(); + break; + + case 'Unlist Instance': + return $this->unlistInstance(); + break; + + case 'Ban Instance': + return $this->banInstance(); + break; + + case 'Unban Instance': + return $this->unbanInstance(); + break; + + case 'Relist Instance': + return $this->relistInstance(); + break; + } + } + + protected function recalculateStats() + { + $instanceCount = Instance::count(); + $confirmed = confirm('Do you want to recalculate stats for all ' . $instanceCount . ' instances?'); + if(!$confirmed) { + $this->error('Aborting...'); + exit; + } + + $users = progress( + label: 'Updating instance stats...', + steps: Instance::all(), + callback: fn ($instance) => $this->updateInstanceStats($instance), + ); + } + + protected function updateInstanceStats($instance) + { + FetchNodeinfoPipeline::dispatch($instance)->onQueue('intbg'); + } + + protected function unlistInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereUnlisted(false)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to unlist this instance?'); + if(!$confirmed) { + $this->error('Aborting instance unlisting'); + exit; + } + + $instance->unlisted = true; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully unlisted ' . $instance->domain . '!'); + exit; + } + + protected function relistInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereUnlisted(true)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to re-list this instance?'); + if(!$confirmed) { + $this->error('Aborting instance re-listing'); + exit; + } + + $instance->unlisted = false; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully re-listed ' . $instance->domain . '!'); + exit; + } + + protected function banInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereBanned(false)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to ban this instance?'); + if(!$confirmed) { + $this->error('Aborting instance ban'); + exit; + } + + $instance->banned = true; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully banned ' . $instance->domain . '!'); + exit; + } + + protected function unbanInstance() + { + $id = search( + 'Search by domain', + fn (string $value) => strlen($value) > 0 + ? Instance::whereBanned(true)->where('domain', 'like', "%{$value}%")->pluck('domain', 'id')->all() + : [] + ); + + $instance = Instance::find($id); + if(!$instance) { + $this->error('Oops, an error occured'); + exit; + } + + $tbl = [ + [ + $instance->domain, + number_format($instance->status_count), + number_format($instance->user_count), + ] + ]; + table( + ['Domain', 'Status Count', 'User Count'], + $tbl + ); + + $confirmed = confirm('Are you sure you want to unban this instance?'); + if(!$confirmed) { + $this->error('Aborting instance unban'); + exit; + } + + $instance->banned = false; + $instance->save(); + InstanceService::refresh(); + $this->info('Successfully un-banned ' . $instance->domain . '!'); + exit; + } + + protected function viewBannedInstances() + { + $data = Instance::whereBanned(true) + ->get(['domain', 'user_count', 'status_count']) + ->map(function($d) { + return [ + 'domain' => $d->domain, + 'user_count' => number_format($d->user_count), + 'status_count' => number_format($d->status_count), + ]; + }) + ->toArray(); + table( + ['Domain', 'User Count', 'Status Count'], + $data + ); + } + + protected function viewUnlistedInstances() + { + $data = Instance::whereUnlisted(true) + ->get(['domain', 'user_count', 'status_count', 'banned']) + ->map(function($d) { + return [ + 'domain' => $d->domain, + 'user_count' => number_format($d->user_count), + 'status_count' => number_format($d->status_count), + 'banned' => $d->banned ? '✅' : null + ]; + }) + ->toArray(); + table( + ['Domain', 'User Count', 'Status Count', 'Banned'], + $data + ); + } +} From 1f3f0cae65f6ef100e0710ddb0919e2a2ed986e8 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 04:43:32 -0700 Subject: [PATCH 20/21] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 591025bfd8..640f6531a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,12 @@ - Update AP helpers, fix sensitive bug ([00ed330c](https://github.com/pixelfed/pixelfed/commit/00ed330c)) - Update NotificationEpochUpdatePipeline, use more efficient query ([4d401389](https://github.com/pixelfed/pixelfed/commit/4d401389)) - Update notification pipelines, fix non-local saving ([fa97a1f3](https://github.com/pixelfed/pixelfed/commit/fa97a1f3)) +- Update NodeinfoService, disable redirects ([240e6bbe](https://github.com/pixelfed/pixelfed/commit/240e6bbe)) +- Update Instance model, add entity casts ([289cad47](https://github.com/pixelfed/pixelfed/commit/289cad47)) +- Update FetchNodeinfoPipeline, use more efficient dispatch ([ac01f51a](https://github.com/pixelfed/pixelfed/commit/ac01f51a)) +- Update horizon.php config ([1e3acade](https://github.com/pixelfed/pixelfed/commit/1e3acade)) +- Update PublicApiController, consume InstanceService blocked domains for account and statuses endpoints ([01b33fb3](https://github.com/pixelfed/pixelfed/commit/01b33fb3)) +- Update ApiV1Controller, enforce blocked instance domain logic ([5b284cac](https://github.com/pixelfed/pixelfed/commit/5b284cac)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 97b7cb2719924ca70a330842fa4b1e3772eb37d5 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 7 Feb 2024 04:49:44 -0700 Subject: [PATCH 21/21] Add migration --- ..._add_active_deliver_to_instances_table.php | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 database/migrations/2023_12_05_092152_add_active_deliver_to_instances_table.php diff --git a/database/migrations/2023_12_05_092152_add_active_deliver_to_instances_table.php b/database/migrations/2023_12_05_092152_add_active_deliver_to_instances_table.php new file mode 100644 index 0000000000..d6e413768f --- /dev/null +++ b/database/migrations/2023_12_05_092152_add_active_deliver_to_instances_table.php @@ -0,0 +1,36 @@ +boolean('active_deliver')->nullable()->index()->after('domain'); + $table->boolean('valid_nodeinfo')->nullable(); + $table->timestamp('nodeinfo_last_fetched')->nullable(); + $table->boolean('delivery_timeout')->default(false); + $table->timestamp('delivery_next_after')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('instances', function (Blueprint $table) { + $table->dropColumn('active_deliver'); + $table->dropColumn('valid_nodeinfo'); + $table->dropColumn('nodeinfo_last_fetched'); + $table->dropColumn('delivery_timeout'); + $table->dropColumn('delivery_next_after'); + }); + } +};