From fb1deb6e28e61738acdc35fcc928ee7222ae741d Mon Sep 17 00:00:00 2001
From: Daniel Supernault <danielsupernault@gmail.com>
Date: Mon, 25 Sep 2023 00:59:24 -0600
Subject: [PATCH] Add Resilient Media Storage

---
 app/Services/MediaStorageService.php          | 11 ++--
 app/Services/ResilientMediaStorageService.php | 66 +++++++++++++++++++
 config/filesystems.php                        | 28 ++++++++
 config/media.php                              | 38 ++++++-----
 4 files changed, 119 insertions(+), 24 deletions(-)
 create mode 100644 app/Services/ResilientMediaStorageService.php

diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php
index b52662d1f..b547ee39c 100644
--- a/app/Services/MediaStorageService.php
+++ b/app/Services/MediaStorageService.php
@@ -86,12 +86,11 @@ class MediaStorageService {
 		$thumbname = array_pop($pt);
 		$storagePath = implode('/', $p);
 
-		$disk = Storage::disk(config('filesystems.cloud'));
-		$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
-		$url = $disk->url($file);
-		$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
-		$thumbUrl = $disk->url($thumbFile);
-		$media->thumbnail_url = $thumbUrl;
+		$url = ResilientMediaStorageService::store($storagePath, $path, $name);
+		if($thumb) {
+			$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
+			$media->thumbnail_url = $thumbUrl;
+		}
 		$media->cdn_url = $url;
 		$media->optimized_url = $url;
 		$media->replicated_at = now();
diff --git a/app/Services/ResilientMediaStorageService.php b/app/Services/ResilientMediaStorageService.php
new file mode 100644
index 000000000..ac1b089af
--- /dev/null
+++ b/app/Services/ResilientMediaStorageService.php
@@ -0,0 +1,66 @@
+<?php
+
+namespace App\Services;
+
+use Storage;
+use Illuminate\Http\File;
+use Exception;
+use GuzzleHttp\Exception\ClientException;
+use Aws\S3\Exception\S3Exception;
+use GuzzleHttp\Exception\ConnectException;
+use League\Flysystem\UnableToWriteFile;
+
+class ResilientMediaStorageService
+{
+    static $attempts = 0;
+
+    public static function store($storagePath, $path, $name)
+    {
+        return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
+            self::handleResilientStore($storagePath, $path, $name) :
+            self::handleStore($storagePath, $path, $name);
+    }
+
+    public static function handleStore($storagePath, $path, $name)
+    {
+        return retry(3, function() use($storagePath, $path, $name) {
+            $baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
+            $disk = Storage::disk($baseDisk);
+            $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            return $disk->url($file);
+        }, random_int(100, 500));
+    }
+
+    public static function handleResilientStore($storagePath, $path, $name)
+    {
+        $attempts = 0;
+        return retry(4, function() use($storagePath, $path, $name, $attempts) {
+            self::$attempts++;
+            usleep(100000);
+            $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
+            try {
+                $disk = Storage::disk($baseDisk);
+                $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
+            return $disk->url($file);
+        }, function (int $attempt, Exception $exception) {
+            return $attempt * 200;
+        });
+    }
+
+    public static function getAltDriver()
+    {
+        $drivers = [];
+        if(config('filesystems.disks.alt-primary.enabled')) {
+            $drivers[] = 'alt-primary';
+        }
+        if(config('filesystems.disks.alt-secondary.enabled')) {
+            $drivers[] = 'alt-secondary';
+        }
+        if(empty($drivers)) {
+            return false;
+        }
+        $key = array_rand($drivers, 1);
+        return $drivers[$key];
+    }
+}
diff --git a/config/filesystems.php b/config/filesystems.php
index 6817d5e34..80e63ed99 100644
--- a/config/filesystems.php
+++ b/config/filesystems.php
@@ -79,6 +79,34 @@ return [
             'throw' => true,
         ],
 
+        'alt-primary' => [
+            'enabled'  => env('ALT_PRI_ENABLED', false),
+            'driver'   => 's3',
+            'key'      => env('ALT_PRI_AWS_ACCESS_KEY_ID'),
+            'secret'   => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'),
+            'region'   => env('ALT_PRI_AWS_DEFAULT_REGION'),
+            'bucket'   => env('ALT_PRI_AWS_BUCKET'),
+            'visibility' => 'public',
+            'url'      => env('ALT_PRI_AWS_URL'),
+            'endpoint' => env('ALT_PRI_AWS_ENDPOINT'),
+            'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false),
+            'throw' => true,
+        ],
+
+        'alt-secondary' => [
+            'enabled'  => env('ALT_SEC_ENABLED', false),
+            'driver'   => 's3',
+            'key'      => env('ALT_SEC_AWS_ACCESS_KEY_ID'),
+            'secret'   => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'),
+            'region'   => env('ALT_SEC_AWS_DEFAULT_REGION'),
+            'bucket'   => env('ALT_SEC_AWS_BUCKET'),
+            'visibility' => 'public',
+            'url'      => env('ALT_SEC_AWS_URL'),
+            'endpoint' => env('ALT_SEC_AWS_ENDPOINT'),
+            'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false),
+            'throw' => true,
+        ],
+
         'spaces' => [
             'driver' => 's3',
             'key' => env('DO_SPACES_KEY'),
diff --git a/config/media.php b/config/media.php
index b7d6e95cc..f550ff291 100644
--- a/config/media.php
+++ b/config/media.php
@@ -1,24 +1,26 @@
 <?php
 
 return [
-	'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
+    'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
 
-	'exif' => [
-		'database' => env('MEDIA_EXIF_DATABASE', false),
-	],
+    'exif' => [
+        'database' => env('MEDIA_EXIF_DATABASE', false),
+    ],
 
-	'storage' => [
-		'remote' => [
-			/*
-		    |--------------------------------------------------------------------------
-		    | Store remote media on cloud/S3
-		    |--------------------------------------------------------------------------
-		    |
-		    | Set this to cache remote media on cloud/S3 filesystem drivers.
-		    | Disabled by default.
-		    |
-		    */
-			'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false)
-		],
-	]
+    'storage' => [
+        'remote' => [
+            /*
+            |--------------------------------------------------------------------------
+            | Store remote media on cloud/S3
+            |--------------------------------------------------------------------------
+            |
+            | Set this to cache remote media on cloud/S3 filesystem drivers.
+            | Disabled by default.
+            |
+            */
+            'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false),
+
+            'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false),
+        ],
+    ]
 ];