From 01535a6cfead53155621d066f26f3283c286dd50 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 5 Mar 2024 01:56:16 -0700 Subject: [PATCH] Update ApiV1Controller, improve notification filtering --- app/Http/Controllers/Api/ApiV1Controller.php | 35 +- app/MediaTag.php | 8 +- app/Services/NotificationService.php | 512 +++++++++--------- .../Api/NotificationTransformer.php | 121 +++-- 4 files changed, 370 insertions(+), 306 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index 64da64ecf..efd4c634c 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -2247,7 +2247,8 @@ class ApiV1Controller extends Controller 'max_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX, 'since_id' => 'nullable|integer|min:1|max:'.PHP_INT_MAX, 'types[]' => 'sometimes|array', - 'type' => 'sometimes|string|in:mention,reblog,follow,favourite' + 'type' => 'sometimes|string|in:mention,reblog,follow,favourite', + '_pe' => 'sometimes', ]); $pid = $request->user()->profile_id; @@ -2259,6 +2260,7 @@ class ApiV1Controller extends Controller $since = $request->input('since_id'); $min = $request->input('min_id'); $max = $request->input('max_id'); + $pe = $request->filled('_pe'); if(!$since && !$min && !$max) { $min = 1; @@ -2298,12 +2300,39 @@ class ApiV1Controller extends Controller $minId = null; } - $res = collect($res)->filter(function($n) { + $res = collect($res) + ->map(function($n) use($pe) { + if(!$pe) { + if($n['type'] == 'comment') { + $n['type'] = 'mention'; + return $n; + } + + return $n; + } + return $n; + }) + ->filter(function($n) use($pe) { if(in_array($n['type'], ['mention', 'reblog', 'favourite'])) { return isset($n['status'], $n['status']['id']); } - return isset($n['account'], $n['account']['id']); + if(!$pe) { + if(in_array($n['type'], [ + 'tagged', + 'modlog', + 'story:react', + 'story:comment', + 'group:comment', + 'group:join:approved', + 'group:join:rejected', + ])) { + return false; + } + return isset($n['account'], $n['account']['id']); + } + + return true; })->values(); if($maxId) { diff --git a/app/MediaTag.php b/app/MediaTag.php index 49dd52b16..df253fc6e 100644 --- a/app/MediaTag.php +++ b/app/MediaTag.php @@ -8,8 +8,14 @@ class MediaTag extends Model { protected $guarded = []; + protected $visible = [ + 'status_id', + 'profile_id', + 'tagged_username', + ]; + public function status() { - return $this->belongsTo(Status::class); + return $this->belongsTo(Status::class); } } diff --git a/app/Services/NotificationService.php b/app/Services/NotificationService.php index 634038baf..9ee0d77fc 100644 --- a/app/Services/NotificationService.php +++ b/app/Services/NotificationService.php @@ -2,298 +2,324 @@ namespace App\Services; +use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline; +use App\Notification; +use App\Transformer\Api\NotificationTransformer; use Cache; use Illuminate\Support\Facades\Redis; -use App\{ - Notification, - Profile -}; -use App\Transformer\Api\NotificationTransformer; use League\Fractal; use League\Fractal\Serializer\ArraySerializer; -use League\Fractal\Pagination\IlluminatePaginatorAdapter; -use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline; -class NotificationService { +class NotificationService +{ + const CACHE_KEY = 'pf:services:notifications:ids:'; - const CACHE_KEY = 'pf:services:notifications:ids:'; - const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:'; - const ITEM_CACHE_TTL = 86400; - const MASTODON_TYPES = [ - 'follow', - 'follow_request', - 'mention', - 'reblog', - 'favourite', - 'poll', - 'status' - ]; + const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:'; - public static function get($id, $start = 0, $stop = 400) - { - $res = collect([]); - $key = self::CACHE_KEY . $id; - $stop = $stop > 400 ? 400 : $stop; - $ids = Redis::zrangebyscore($key, $start, $stop); - if(empty($ids)) { - $ids = self::coldGet($id, $start, $stop); - } - foreach($ids as $id) { - $n = self::getNotification($id); - if($n != null) { - $res->push($n); - } - } - return $res; - } + const ITEM_CACHE_TTL = 86400; - public static function getEpochId($months = 6) - { - $epoch = Cache::get(self::EPOCH_CACHE_KEY . $months); - if(!$epoch) { - NotificationEpochUpdatePipeline::dispatch(); - return 1; - } - return $epoch; - } + const MASTODON_TYPES = [ + 'follow', + 'follow_request', + 'mention', + 'reblog', + 'favourite', + 'poll', + 'status', + ]; - public static function coldGet($id, $start = 0, $stop = 400) - { - $stop = $stop > 400 ? 400 : $stop; - $ids = Notification::where('id', '>', self::getEpochId()) - ->where('profile_id', $id) - ->orderByDesc('id') - ->skip($start) - ->take($stop) - ->pluck('id'); - foreach($ids as $key) { - self::set($id, $key); - } - return $ids; - } + public static function get($id, $start = 0, $stop = 400) + { + $res = collect([]); + $key = self::CACHE_KEY.$id; + $stop = $stop > 400 ? 400 : $stop; + $ids = Redis::zrangebyscore($key, $start, $stop); + if (empty($ids)) { + $ids = self::coldGet($id, $start, $stop); + } + foreach ($ids as $id) { + $n = self::getNotification($id); + if ($n != null) { + $res->push($n); + } + } - public static function getMax($id = false, $start = 0, $limit = 10) - { - $ids = self::getRankedMaxId($id, $start, $limit); + return $res; + } - if(empty($ids)) { - return []; - } + public static function getEpochId($months = 6) + { + $epoch = Cache::get(self::EPOCH_CACHE_KEY.$months); + if (! $epoch) { + NotificationEpochUpdatePipeline::dispatch(); - $res = collect([]); - foreach($ids as $id) { - $n = self::getNotification($id); - if($n != null) { - $res->push($n); - } - } - return $res->toArray(); - } + return 1; + } - public static function getMin($id = false, $start = 0, $limit = 10) - { - $ids = self::getRankedMinId($id, $start, $limit); + return $epoch; + } - if(empty($ids)) { - return []; - } + public static function coldGet($id, $start = 0, $stop = 400) + { + $stop = $stop > 400 ? 400 : $stop; + $ids = Notification::where('id', '>', self::getEpochId()) + ->where('profile_id', $id) + ->orderByDesc('id') + ->skip($start) + ->take($stop) + ->pluck('id'); + foreach ($ids as $key) { + self::set($id, $key); + } - $res = collect([]); - foreach($ids as $id) { - $n = self::getNotification($id); - if($n != null) { - $res->push($n); - } - } - return $res->toArray(); - } + return $ids; + } + public static function getMax($id = false, $start = 0, $limit = 10) + { + $ids = self::getRankedMaxId($id, $start, $limit); - public static function getMaxMastodon($id = false, $start = 0, $limit = 10) - { - $ids = self::getRankedMaxId($id, $start, $limit); + if (empty($ids)) { + return []; + } - if(empty($ids)) { - return []; - } + $res = collect([]); + foreach ($ids as $id) { + $n = self::getNotification($id); + if ($n != null) { + $res->push($n); + } + } - $res = collect([]); - foreach($ids as $id) { - $n = self::rewriteMastodonTypes(self::getNotification($id)); - if($n != null && in_array($n['type'], self::MASTODON_TYPES)) { - if(isset($n['account'])) { - $n['account'] = AccountService::getMastodon($n['account']['id']); - } + return $res->toArray(); + } - if(isset($n['relationship'])) { - unset($n['relationship']); - } + public static function getMin($id = false, $start = 0, $limit = 10) + { + $ids = self::getRankedMinId($id, $start, $limit); - if(isset($n['status'])) { - $n['status'] = StatusService::getMastodon($n['status']['id'], false); - } + if (empty($ids)) { + return []; + } - $res->push($n); - } - } - return $res->toArray(); - } + $res = collect([]); + foreach ($ids as $id) { + $n = self::getNotification($id); + if ($n != null) { + $res->push($n); + } + } - public static function getMinMastodon($id = false, $start = 0, $limit = 10) - { - $ids = self::getRankedMinId($id, $start, $limit); + return $res->toArray(); + } - if(empty($ids)) { - return []; - } + public static function getMaxMastodon($id = false, $start = 0, $limit = 10) + { + $ids = self::getRankedMaxId($id, $start, $limit); - $res = collect([]); - foreach($ids as $id) { - $n = self::rewriteMastodonTypes(self::getNotification($id)); - if($n != null && in_array($n['type'], self::MASTODON_TYPES)) { - if(isset($n['account'])) { - $n['account'] = AccountService::getMastodon($n['account']['id']); - } + if (empty($ids)) { + return []; + } - if(isset($n['relationship'])) { - unset($n['relationship']); - } + $res = collect([]); + foreach ($ids as $id) { + $n = self::rewriteMastodonTypes(self::getNotification($id)); + if ($n != null && in_array($n['type'], self::MASTODON_TYPES)) { + if (isset($n['account'])) { + $n['account'] = AccountService::getMastodon($n['account']['id']); + } - if(isset($n['status'])) { - $n['status'] = StatusService::getMastodon($n['status']['id'], false); - } + if (isset($n['relationship'])) { + unset($n['relationship']); + } - $res->push($n); - } - } - return $res->toArray(); - } + if ($n['type'] === 'mention' && isset($n['tagged'], $n['tagged']['status_id'])) { + $n['status'] = StatusService::getMastodon($n['tagged']['status_id'], false); + unset($n['tagged']); + } - public static function getRankedMaxId($id = false, $start = null, $limit = 10) - { - if(!$start || !$id) { - return []; - } + if (isset($n['status'])) { + $n['status'] = StatusService::getMastodon($n['status']['id'], false); + } - return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, $start, '-inf', [ - 'withscores' => true, - 'limit' => [1, $limit] - ])); - } + $res->push($n); + } + } - public static function getRankedMinId($id = false, $end = null, $limit = 10) - { - if(!$end || !$id) { - return []; - } + return $res->toArray(); + } - return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, '+inf', $end, [ - 'withscores' => true, - 'limit' => [0, $limit] - ])); - } + public static function getMinMastodon($id = false, $start = 0, $limit = 10) + { + $ids = self::getRankedMinId($id, $start, $limit); - public static function rewriteMastodonTypes($notification) - { - if(!$notification || !isset($notification['type'])) { - return $notification; - } + if (empty($ids)) { + return []; + } - if($notification['type'] === 'comment') { - $notification['type'] = 'mention'; - } + $res = collect([]); + foreach ($ids as $id) { + $n = self::rewriteMastodonTypes(self::getNotification($id)); + if ($n != null && in_array($n['type'], self::MASTODON_TYPES)) { + if (isset($n['account'])) { + $n['account'] = AccountService::getMastodon($n['account']['id']); + } - if($notification['type'] === 'share') { - $notification['type'] = 'reblog'; - } + if (isset($n['relationship'])) { + unset($n['relationship']); + } - return $notification; - } + if ($n['type'] === 'mention' && isset($n['tagged'], $n['tagged']['status_id'])) { + $n['status'] = StatusService::getMastodon($n['tagged']['status_id'], false); + unset($n['tagged']); + } - public static function set($id, $val) - { - if(self::count($id) > 400) { - Redis::zpopmin(self::CACHE_KEY . $id); - } - return Redis::zadd(self::CACHE_KEY . $id, $val, $val); - } + if (isset($n['status'])) { + $n['status'] = StatusService::getMastodon($n['status']['id'], false); + } - public static function del($id, $val) - { - Cache::forget('service:notification:' . $val); - return Redis::zrem(self::CACHE_KEY . $id, $val); - } + $res->push($n); + } + } - public static function add($id, $val) - { - return self::set($id, $val); - } + return $res->toArray(); + } - public static function rem($id, $val) - { - return self::del($id, $val); - } + public static function getRankedMaxId($id = false, $start = null, $limit = 10) + { + if (! $start || ! $id) { + return []; + } - public static function count($id) - { - return Redis::zcount(self::CACHE_KEY . $id, '-inf', '+inf'); - } + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, $start, '-inf', [ + 'withscores' => true, + 'limit' => [1, $limit], + ])); + } - public static function getNotification($id) - { - $notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function() use($id) { - $n = Notification::with('item')->find($id); + public static function getRankedMinId($id = false, $end = null, $limit = 10) + { + if (! $end || ! $id) { + return []; + } - if(!$n) { - return null; - } + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, '+inf', $end, [ + 'withscores' => true, + 'limit' => [0, $limit], + ])); + } - $account = AccountService::get($n->actor_id, true); + public static function rewriteMastodonTypes($notification) + { + if (! $notification || ! isset($notification['type'])) { + return $notification; + } - if(!$account) { - return null; - } + if ($notification['type'] === 'comment') { + $notification['type'] = 'mention'; + } - $fractal = new Fractal\Manager(); - $fractal->setSerializer(new ArraySerializer()); - $resource = new Fractal\Resource\Item($n, new NotificationTransformer()); - return $fractal->createData($resource)->toArray(); - }); + if ($notification['type'] === 'share') { + $notification['type'] = 'reblog'; + } - if(!$notification) { - return; - } + if ($notification['type'] === 'tagged') { + $notification['type'] = 'mention'; + } - if(isset($notification['account'])) { - $notification['account'] = AccountService::get($notification['account']['id'], true); - } + return $notification; + } - return $notification; - } + public static function set($id, $val) + { + if (self::count($id) > 400) { + Redis::zpopmin(self::CACHE_KEY.$id); + } - public static function setNotification(Notification $notification) - { - return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function() use($notification) { - $fractal = new Fractal\Manager(); - $fractal->setSerializer(new ArraySerializer()); - $resource = new Fractal\Resource\Item($notification, new NotificationTransformer()); - return $fractal->createData($resource)->toArray(); - }); - } + return Redis::zadd(self::CACHE_KEY.$id, $val, $val); + } - public static function warmCache($id, $stop = 400, $force = false) - { - if(self::count($id) == 0 || $force == true) { - $ids = Notification::where('profile_id', $id) - ->where('id', '>', self::getEpochId()) - ->orderByDesc('id') - ->limit($stop) - ->pluck('id'); - foreach($ids as $key) { - self::set($id, $key); - } - return 1; - } - return 0; - } + public static function del($id, $val) + { + Cache::forget('service:notification:'.$val); + + return Redis::zrem(self::CACHE_KEY.$id, $val); + } + + public static function add($id, $val) + { + return self::set($id, $val); + } + + public static function rem($id, $val) + { + return self::del($id, $val); + } + + public static function count($id) + { + return Redis::zcount(self::CACHE_KEY.$id, '-inf', '+inf'); + } + + public static function getNotification($id) + { + $notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function () use ($id) { + $n = Notification::with('item')->find($id); + + if (! $n) { + return null; + } + + $account = AccountService::get($n->actor_id, true); + + if (! $account) { + return null; + } + + $fractal = new Fractal\Manager(); + $fractal->setSerializer(new ArraySerializer()); + $resource = new Fractal\Resource\Item($n, new NotificationTransformer()); + + return $fractal->createData($resource)->toArray(); + }); + + if (! $notification) { + return; + } + + if (isset($notification['account'])) { + $notification['account'] = AccountService::get($notification['account']['id'], true); + } + + return $notification; + } + + public static function setNotification(Notification $notification) + { + return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function () use ($notification) { + $fractal = new Fractal\Manager(); + $fractal->setSerializer(new ArraySerializer()); + $resource = new Fractal\Resource\Item($notification, new NotificationTransformer()); + + return $fractal->createData($resource)->toArray(); + }); + } + + public static function warmCache($id, $stop = 400, $force = false) + { + if (self::count($id) == 0 || $force == true) { + $ids = Notification::where('profile_id', $id) + ->where('id', '>', self::getEpochId()) + ->orderByDesc('id') + ->limit($stop) + ->pluck('id'); + foreach ($ids as $key) { + self::set($id, $key); + } + + return 1; + } + + return 0; + } } diff --git a/app/Transformer/Api/NotificationTransformer.php b/app/Transformer/Api/NotificationTransformer.php index 837c027af..cad5732b5 100644 --- a/app/Transformer/Api/NotificationTransformer.php +++ b/app/Transformer/Api/NotificationTransformer.php @@ -4,78 +4,81 @@ namespace App\Transformer\Api; use App\Notification; use App\Services\AccountService; -use App\Services\HashidService; use App\Services\RelationshipService; use App\Services\StatusService; use League\Fractal; class NotificationTransformer extends Fractal\TransformerAbstract { - public function transform(Notification $notification) - { - $res = [ - 'id' => (string) $notification->id, - 'type' => $this->replaceTypeVerb($notification->action), - 'created_at' => (string) str_replace('+00:00', 'Z', $notification->created_at->format(DATE_RFC3339_EXTENDED)), - ]; + public function transform(Notification $notification) + { + $res = [ + 'id' => (string) $notification->id, + 'type' => $this->replaceTypeVerb($notification->action), + 'created_at' => (string) str_replace('+00:00', 'Z', $notification->created_at->format(DATE_RFC3339_EXTENDED)), + ]; - $n = $notification; + $n = $notification; - if($n->actor_id) { - $res['account'] = AccountService::get($n->actor_id); - if($n->profile_id != $n->actor_id) { - $res['relationship'] = RelationshipService::get($n->actor_id, $n->profile_id); - } - } + if ($n->actor_id) { + $res['account'] = AccountService::get($n->actor_id); + if ($n->profile_id != $n->actor_id) { + $res['relationship'] = RelationshipService::get($n->actor_id, $n->profile_id); + } + } - if($n->item_id && $n->item_type == 'App\Status') { - $res['status'] = StatusService::get($n->item_id, false); - } + if ($n->item_id && $n->item_type == 'App\Status') { + $res['status'] = StatusService::get($n->item_id, false); + } - if($n->item_id && $n->item_type == 'App\ModLog') { - $ml = $n->item; - if($ml && $ml->object_uid) { - $res['modlog'] = [ - 'id' => $ml->object_uid, - 'url' => url('/i/admin/users/modlogs/' . $ml->object_uid) - ]; - } - } + if ($n->item_id && $n->item_type == 'App\ModLog') { + $ml = $n->item; + if ($ml && $ml->object_uid) { + $res['modlog'] = [ + 'id' => $ml->object_uid, + 'url' => url('/i/admin/users/modlogs/'.$ml->object_uid), + ]; + } + } - if($n->item_id && $n->item_type == 'App\MediaTag') { - $ml = $n->item; - if($ml && $ml->tagged_username) { - $res['tagged'] = [ - 'username' => $ml->tagged_username, - 'post_url' => '/p/'.HashidService::encode($ml->status_id) - ]; - } - } + if ($n->item_id && $n->item_type == 'App\MediaTag') { + $ml = $n->item; + if ($ml && $ml->tagged_username) { + $np = StatusService::get($ml->status_id, false); + if ($np && isset($np['id'])) { + $res['tagged'] = [ + 'username' => $ml->tagged_username, + 'post_url' => $np['url'], + 'status_id' => $ml->status_id, + 'profile_id' => $ml->profile_id, + ]; + } + } + } - return $res; - } + return $res; + } - public function replaceTypeVerb($verb) - { - $verbs = [ - 'dm' => 'direct', - 'follow' => 'follow', - 'mention' => 'mention', - 'reblog' => 'share', - 'share' => 'share', - 'like' => 'favourite', - 'group:like' => 'favourite', - 'comment' => 'comment', - 'admin.user.modlog.comment' => 'modlog', - 'tagged' => 'tagged', - 'story:react' => 'story:react', - 'story:comment' => 'story:comment', - ]; + public function replaceTypeVerb($verb) + { + $verbs = [ + 'dm' => 'direct', + 'follow' => 'follow', + 'mention' => 'mention', + 'reblog' => 'share', + 'share' => 'share', + 'like' => 'favourite', + 'comment' => 'comment', + 'admin.user.modlog.comment' => 'modlog', + 'tagged' => 'tagged', + 'story:react' => 'story:react', + 'story:comment' => 'story:comment', + ]; - if(!isset($verbs[$verb])) { - return $verb; - } + if (! isset($verbs[$verb])) { + return $verb; + } - return $verbs[$verb]; - } + return $verbs[$verb]; + } }