forked from pixelfed/pixelfed
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'upstream/staging' into develop
# Conflicts: # app/Console/Kernel.php
- Loading branch information
Showing
43 changed files
with
695 additions
and
303 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| <?php | ||
|
|
||
| namespace App\Console\Commands; | ||
|
|
||
| use Illuminate\Console\Command; | ||
| use App\Models\ImportPost; | ||
| use App\Jobs\ImportPipeline\ImportMediaToCloudPipeline; | ||
| use function Laravel\Prompts\progress; | ||
|
|
||
| class ImportUploadMediaToCloudStorage extends Command | ||
| { | ||
| /** | ||
| * The name and signature of the console command. | ||
| * | ||
| * @var string | ||
| */ | ||
| protected $signature = 'app:import-upload-media-to-cloud-storage {--limit=500}'; | ||
|
|
||
| /** | ||
| * The console command description. | ||
| * | ||
| * @var string | ||
| */ | ||
| protected $description = 'Migrate media imported from Instagram to S3 cloud storage.'; | ||
|
|
||
| /** | ||
| * Execute the console command. | ||
| */ | ||
| public function handle() | ||
| { | ||
| if( | ||
| (bool) config('import.instagram.storage.cloud.enabled') === false || | ||
| (bool) config_cache('pixelfed.cloud_storage') === false | ||
| ) { | ||
| $this->error('Aborted. Cloud storage is not enabled for IG imports.'); | ||
| return; | ||
| } | ||
|
|
||
| $limit = $this->option('limit'); | ||
|
|
||
| $progress = progress(label: 'Migrating import media', steps: $limit); | ||
|
|
||
| $progress->start(); | ||
|
|
||
| $posts = ImportPost::whereUploadedToS3(false)->take($limit)->get(); | ||
|
|
||
| foreach($posts as $post) { | ||
| ImportMediaToCloudPipeline::dispatch($post)->onQueue('low'); | ||
| $progress->advance(); | ||
| } | ||
|
|
||
| $progress->finish(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| <?php | ||
|
|
||
| namespace App\Jobs\ImportPipeline; | ||
|
|
||
| use Illuminate\Bus\Queueable; | ||
| use Illuminate\Contracts\Queue\ShouldBeUnique; | ||
| use Illuminate\Contracts\Queue\ShouldQueue; | ||
| use Illuminate\Foundation\Bus\Dispatchable; | ||
| use Illuminate\Queue\InteractsWithQueue; | ||
| use Illuminate\Queue\SerializesModels; | ||
| use Illuminate\Queue\Middleware\WithoutOverlapping; | ||
| use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; | ||
| use App\Models\ImportPost; | ||
| use App\Media; | ||
| use App\Services\MediaStorageService; | ||
| use Illuminate\Support\Facades\Storage; | ||
| use App\Jobs\VideoPipeline\VideoThumbnailToCloudPipeline; | ||
|
|
||
| class ImportMediaToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing | ||
| { | ||
| use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | ||
|
|
||
| protected $importPost; | ||
|
|
||
| public $timeout = 900; | ||
| public $tries = 3; | ||
| public $maxExceptions = 1; | ||
| public $failOnTimeout = true; | ||
|
|
||
| /** | ||
| * The number of seconds after which the job's unique lock will be released. | ||
| * | ||
| * @var int | ||
| */ | ||
| public $uniqueFor = 3600; | ||
|
|
||
| /** | ||
| * Get the unique ID for the job. | ||
| */ | ||
| public function uniqueId(): string | ||
| { | ||
| return 'import-media-to-cloud-pipeline:ip-id:' . $this->importPost->id; | ||
| } | ||
|
|
||
| /** | ||
| * Get the middleware the job should pass through. | ||
| * | ||
| * @return array<int, object> | ||
| */ | ||
| public function middleware(): array | ||
| { | ||
| return [(new WithoutOverlapping("import-media-to-cloud-pipeline:ip-id:{$this->importPost->id}"))->shared()->dontRelease()]; | ||
| } | ||
|
|
||
| /** | ||
| * Delete the job if its models no longer exist. | ||
| * | ||
| * @var bool | ||
| */ | ||
| public $deleteWhenMissingModels = true; | ||
|
|
||
| /** | ||
| * Create a new job instance. | ||
| */ | ||
| public function __construct(ImportPost $importPost) | ||
| { | ||
| $this->importPost = $importPost; | ||
| } | ||
|
|
||
| /** | ||
| * Execute the job. | ||
| */ | ||
| public function handle(): void | ||
| { | ||
| $ip = $this->importPost; | ||
|
|
||
| if( | ||
| $ip->status_id === null || | ||
| $ip->uploaded_to_s3 === true || | ||
| (bool) config_cache('pixelfed.cloud_storage') === false) { | ||
| return; | ||
| } | ||
|
|
||
| $media = Media::whereStatusId($ip->status_id)->get(); | ||
|
|
||
| if(!$media || !$media->count()) { | ||
| $importPost = ImportPost::find($ip->id); | ||
| $importPost->uploaded_to_s3 = true; | ||
| $importPost->save(); | ||
| return; | ||
| } | ||
|
|
||
| foreach($media as $mediaPart) { | ||
| $this->handleMedia($mediaPart); | ||
| } | ||
| } | ||
|
|
||
| protected function handleMedia($media) | ||
| { | ||
| $ip = $this->importPost; | ||
|
|
||
| $importPost = ImportPost::find($ip->id); | ||
|
|
||
| if(!$importPost) { | ||
| return; | ||
| } | ||
|
|
||
| $res = MediaStorageService::move($media); | ||
|
|
||
| $importPost->uploaded_to_s3 = true; | ||
| $importPost->save(); | ||
|
|
||
| if(!$res) { | ||
| return; | ||
| } | ||
|
|
||
| if($res === 'invalid file') { | ||
| return; | ||
| } | ||
|
|
||
| if($res === 'success') { | ||
| if($media->mime === 'video/mp4') { | ||
| VideoThumbnailToCloudPipeline::dispatch($media)->onQueue('low'); | ||
| } else { | ||
| Storage::disk('local')->delete($media->media_path); | ||
| } | ||
| } | ||
| } | ||
| } |
147 changes: 147 additions & 0 deletions
147
app/Jobs/VideoPipeline/VideoThumbnailToCloudPipeline.php
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| <?php | ||
|
|
||
| namespace App\Jobs\VideoPipeline; | ||
|
|
||
| use Illuminate\Bus\Queueable; | ||
| use Illuminate\Contracts\Queue\ShouldQueue; | ||
| use Illuminate\Foundation\Bus\Dispatchable; | ||
| use Illuminate\Queue\InteractsWithQueue; | ||
| use Illuminate\Queue\SerializesModels; | ||
| use Illuminate\Queue\Middleware\WithoutOverlapping; | ||
| use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; | ||
| use Illuminate\Http\File; | ||
| use Cache; | ||
| use FFMpeg; | ||
| use Storage; | ||
| use App\Media; | ||
| use App\Jobs\MediaPipeline\MediaStoragePipeline; | ||
| use App\Util\Media\Blurhash; | ||
| use App\Services\MediaService; | ||
| use App\Services\StatusService; | ||
| use App\Services\ResilientMediaStorageService; | ||
|
|
||
| class VideoThumbnailToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing | ||
| { | ||
| use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | ||
|
|
||
| protected $media; | ||
|
|
||
| public $timeout = 900; | ||
| public $tries = 3; | ||
| public $maxExceptions = 1; | ||
| public $failOnTimeout = true; | ||
| public $deleteWhenMissingModels = true; | ||
|
|
||
| /** | ||
| * The number of seconds after which the job's unique lock will be released. | ||
| * | ||
| * @var int | ||
| */ | ||
| public $uniqueFor = 3600; | ||
|
|
||
| /** | ||
| * Get the unique ID for the job. | ||
| */ | ||
| public function uniqueId(): string | ||
| { | ||
| return 'media:video-thumb-to-cloud:id-' . $this->media->id; | ||
| } | ||
|
|
||
| /** | ||
| * Get the middleware the job should pass through. | ||
| * | ||
| * @return array<int, object> | ||
| */ | ||
| public function middleware(): array | ||
| { | ||
| return [(new WithoutOverlapping("media:video-thumb-to-cloud:id-{$this->media->id}"))->shared()->dontRelease()]; | ||
| } | ||
|
|
||
| /** | ||
| * Create a new job instance. | ||
| */ | ||
| public function __construct(Media $media) | ||
| { | ||
| $this->media = $media; | ||
| } | ||
|
|
||
| /** | ||
| * Execute the job. | ||
| */ | ||
| public function handle(): void | ||
| { | ||
| if((bool) config_cache('pixelfed.cloud_storage') === false) { | ||
| return; | ||
| } | ||
|
|
||
| $media = $this->media; | ||
|
|
||
| if($media->mime != 'video/mp4') { | ||
| return; | ||
| } | ||
|
|
||
| if($media->profile_id === null || $media->status_id === null) { | ||
| return; | ||
| } | ||
|
|
||
| if($media->thumbnail_url) { | ||
| return; | ||
| } | ||
|
|
||
| $base = $media->media_path; | ||
| $path = explode('/', $base); | ||
| $name = last($path); | ||
|
|
||
| try { | ||
| $t = explode('.', $name); | ||
| $t = $t[0].'_thumb.jpeg'; | ||
| $i = count($path) - 1; | ||
| $path[$i] = $t; | ||
| $save = implode('/', $path); | ||
| $video = FFMpeg::open($base) | ||
| ->getFrameFromSeconds(1) | ||
| ->export() | ||
| ->toDisk('local') | ||
| ->save($save); | ||
|
|
||
| if(!$save) { | ||
| return; | ||
| } | ||
|
|
||
| $media->thumbnail_path = $save; | ||
| $p = explode('/', $media->media_path); | ||
| array_pop($p); | ||
| $pt = explode('/', $save); | ||
| $thumbname = array_pop($pt); | ||
| $storagePath = implode('/', $p); | ||
| $thumb = storage_path('app/' . $save); | ||
| $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); | ||
| $media->thumbnail_url = $thumbUrl; | ||
| $media->save(); | ||
|
|
||
| $blurhash = Blurhash::generate($media); | ||
| if($blurhash) { | ||
| $media->blurhash = $blurhash; | ||
| $media->save(); | ||
| } | ||
|
|
||
| if(str_starts_with($save, 'public/m/_v2/') && str_ends_with($save, '.jpeg')) { | ||
| Storage::delete($save); | ||
| } | ||
|
|
||
| if(str_starts_with($media->media_path, 'public/m/_v2/') && str_ends_with($media->media_path, '.mp4')) { | ||
| Storage::disk('local')->delete($media->media_path); | ||
| } | ||
| } catch (Exception $e) { | ||
| } | ||
|
|
||
| if($media->status_id) { | ||
| Cache::forget('status:transformer:media:attachments:' . $media->status_id); | ||
| MediaService::del($media->status_id); | ||
| Cache::forget('status:thumb:nsfw0' . $media->status_id); | ||
| Cache::forget('status:thumb:nsfw1' . $media->status_id); | ||
| Cache::forget('pf:services:sh:id:' . $media->status_id); | ||
| StatusService::del($media->status_id); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.