From 553d21063e8a940e07ba54d0e2f4846b614219d5 Mon Sep 17 00:00:00 2001 From: Nuno Maduro Date: Mon, 13 Dec 2021 14:55:26 +0000 Subject: [PATCH] [2.x] Fixes `escapeshellarg: argument exceeds the allowed length` (#113) * Fixes `escapeshellarg: argument exceeds the allowed length` causing thousands of queue invocations * Fixes tests * Adds queue handler test * Apply fixes from StyleCI Co-authored-by: Taylor Otwell --- src/Console/Commands/VaporWorkCommand.php | 19 +++-- src/Events/LambdaEvent.php | 87 +++++++++++++++++++++++ src/Runtime/Handlers/QueueHandler.php | 16 +++-- tests/Fixtures/lambdaEvent.json | 22 ++++++ tests/Unit/LambdaEventTest.php | 41 +++++++++++ tests/Unit/QueueHandlerTest.php | 84 ++++++++++++++++++++++ tests/Unit/VaporWorkCommandTest.php | 65 +++++++++-------- 7 files changed, 286 insertions(+), 48 deletions(-) create mode 100644 src/Events/LambdaEvent.php create mode 100644 tests/Fixtures/lambdaEvent.json create mode 100644 tests/Unit/LambdaEventTest.php create mode 100644 tests/Unit/QueueHandlerTest.php diff --git a/src/Console/Commands/VaporWorkCommand.php b/src/Console/Commands/VaporWorkCommand.php index 48c1283..23624ba 100644 --- a/src/Console/Commands/VaporWorkCommand.php +++ b/src/Console/Commands/VaporWorkCommand.php @@ -5,7 +5,7 @@ use Illuminate\Console\Command; use Illuminate\Queue\Events\JobFailed; use Illuminate\Queue\WorkerOptions; -use InvalidArgumentException; +use Laravel\Vapor\Events\LambdaEvent; use Laravel\Vapor\Queue\VaporJob; use Laravel\Vapor\Queue\VaporWorker; @@ -19,7 +19,6 @@ class VaporWorkCommand extends Command * @var string */ protected $signature = 'vapor:work - {message : The Base64 encoded message payload} {--delay=0 : The number of seconds to delay failed jobs} {--timeout=0 : The number of seconds a child process can run} {--tries=0 : Number of times to attempt a job before logging it failed} @@ -69,9 +68,10 @@ public function __construct(VaporWorker $worker) /** * Execute the console command. * + * @param \Laravel\Vapor\Events\LambdaEvent $event * @return void */ - public function handle() + public function handle(LambdaEvent $event) { if ($this->downForMaintenance()) { return; @@ -86,7 +86,7 @@ public function handle() $this->worker->setCache($this->laravel['cache']->driver()); return $this->worker->runVaporJob( - $this->marshalJob($this->message()), + $this->marshalJob($this->message($event)), 'sqs', $this->gatherWorkerOptions() ); @@ -128,17 +128,14 @@ protected function normalizeMessage(array $message) } /** - * Get the decoded message payload. + * Get the message payload. * + * @param \Laravel\Vapor\Events\LambdaEvent $event * @return array */ - protected function message() + protected function message($event) { - return tap(json_decode(base64_decode($this->argument('message')), true), function ($message) { - if ($message === false) { - throw new InvalidArgumentException('Unable to unserialize message.'); - } - }); + return $event['Records'][0]; } /** diff --git a/src/Events/LambdaEvent.php b/src/Events/LambdaEvent.php new file mode 100644 index 0000000..1f297f1 --- /dev/null +++ b/src/Events/LambdaEvent.php @@ -0,0 +1,87 @@ +event = $event; + } + + /** + * Determine if an item exists at an offset. + * + * @param string $key + * @return bool + */ + #[\ReturnTypeWillChange] + public function offsetExists($key) + { + return Arr::exists($this->event, $key); + } + + /** + * Get an item at a given offset. + * + * @param string $key + * @return array|string|int + */ + #[\ReturnTypeWillChange] + public function offsetGet($key) + { + return Arr::get($this->event, $key); + } + + /** + * Set the item at a given offset. + * + * @param string $key + * @param array|string|int $value + * @return void + */ + #[\ReturnTypeWillChange] + public function offsetSet($key, $value) + { + Arr::set($this->event, $key, $value); + } + + /** + * Unset the item at a given offset. + * + * @param string $key + * @return void + */ + #[\ReturnTypeWillChange] + public function offsetUnset($key) + { + Arr::forget($this->event, $key); + } + + /** + * Get the instance as an array. + * + * @return array + */ + public function toArray() + { + return $this->event; + } +} diff --git a/src/Runtime/Handlers/QueueHandler.php b/src/Runtime/Handlers/QueueHandler.php index 4754b57..d11c3c1 100644 --- a/src/Runtime/Handlers/QueueHandler.php +++ b/src/Runtime/Handlers/QueueHandler.php @@ -4,6 +4,7 @@ use Illuminate\Contracts\Console\Kernel; use Laravel\Vapor\Contracts\LambdaEventHandler; +use Laravel\Vapor\Events\LambdaEvent; use Laravel\Vapor\Runtime\ArrayLambdaResponse; use Laravel\Vapor\Runtime\StorageDirectories; use Symfony\Component\Console\Input\StringInput; @@ -34,7 +35,6 @@ public function __construct() * Handle an incoming Lambda event. * * @param array $event - * @param \Laravel\Vapor\Contracts\LambdaResponse * @return ArrayLambdaResponse */ public function handle(array $event) @@ -52,13 +52,19 @@ public function handle(array $event) $consoleKernel = static::$app->make(Kernel::class); + static::$app->bind(LambdaEvent::class, function () use ($event) { + return new LambdaEvent($event); + }); + $consoleInput = new StringInput( - 'vapor:work '.rtrim(base64_encode(json_encode($event['Records'][0])), '=').' '.$commandOptions.' --no-interaction' + 'vapor:work '.$commandOptions.' --no-interaction' ); - $consoleKernel->terminate($consoleInput, $status = $consoleKernel->handle( + $status = $consoleKernel->handle( $consoleInput, $output = new BufferedOutput - )); + ); + + $consoleKernel->terminate($consoleInput, $status); return new ArrayLambdaResponse([ 'requestId' => $_ENV['AWS_REQUEST_ID'] ?? null, @@ -68,6 +74,8 @@ public function handle(array $event) 'output' => base64_encode($output->fetch()), ]); } finally { + unset(static::$app[LambdaEvent::class]); + $this->terminate(); } } diff --git a/tests/Fixtures/lambdaEvent.json b/tests/Fixtures/lambdaEvent.json new file mode 100644 index 0000000..d2ff8d4 --- /dev/null +++ b/tests/Fixtures/lambdaEvent.json @@ -0,0 +1,22 @@ +{ + "Records":[ + { + "messageId":"58600123-d011-4d76-af5d-960159ca44aa", + "receiptHandle":"AQEBberAbZm/iuRDZevRaZ1cd1arwj3mHxvAZo/972KO8UH+HiNMTOMl66TPi/pZUbNYu+owiBzhyVGafAJuGDz9+LoyzEt6JqxMrzOKV7C3IO6wRZsUKRBKrlfr42KKP/+KS8zQUJE3QIgWiAwEfEwTnbSLhsxfqGxTFWzLh5+Or7u8U10p3K8tdDozssv2Hr39RhkiOKbuE2CS1U6f1oUvHowIr6o5vqNy9xxEiYr/XDXqbsReBE5zw531guvXxJagJjjKhxaNJoIozuYotF/+TeAz8/0Y0kuQTHZY0/tgS79MWGIPEL6izkF5uDm2lKo5PP4SKqfNMvNHS/i5u35mqzOQfHhJytLMWoRmCwUShI4KSaVNkkX+4ZyBpflOpLQl6u/DJ5TbfgkzWOJqhV+DQQ==", + "body":"{\"uuid\":\"0a0bcc75-f78b-4f15-b834-78e71c56afa3\",\"displayName\":\"Closure (web.php:19)\",\"job\":\"Illuminate\\\\Queue\\\\CallQueuedHandler@call\",\"maxTries\":null,\"maxExceptions\":null,\"failOnTimeout\":false,\"backoff\":null,\"timeout\":null,\"retryUntil\":null,\"data\":{\"commandName\":\"Illuminate\\\\Queue\\\\CallQueuedClosure\",\"command\":\"O:34:\\\"Illuminate\\\\Queue\\\\CallQueuedClosure\\\":14:{s:7:\\\"closure\\\";O:47:\\\"Laravel\\\\SerializableClosure\\\\SerializableClosure\\\":1:{s:12:\\\"serializable\\\";O:46:\\\"Laravel\\\\SerializableClosure\\\\Serializers\\\\Signed\\\":2:{s:12:\\\"serializable\\\";s:432:\\\"O:46:\\\"Laravel\\\\SerializableClosure\\\\Serializers\\\\Native\\\":5:{s:3:\\\"use\\\";a:1:{s:10:\\\"collection\\\";O:29:\\\"Illuminate\\\\Support\\\\Collection\\\":2:{s:8:\\\"\\u0000*\\u0000items\\\";a:1:{i:0;s:4:\\\"nuno\\\";}s:28:\\\"\\u0000*\\u0000escapeWhenCastingToString\\\";b:0;}}s:8:\\\"function\\\";s:79:\\\"function () use ($collection) {\\n \\\\info($collection->implode(','));\\n }\\\";s:5:\\\"scope\\\";s:37:\\\"Illuminate\\\\Routing\\\\RouteFileRegistrar\\\";s:4:\\\"this\\\";N;s:4:\\\"self\\\";s:32:\\\"00000000000001a10000000000000000\\\";}\\\";s:4:\\\"hash\\\";s:44:\\\"bl2j1wIRyXIqlgbMDpY7+kCIUvwcJwhHde9gTY7Ma4E=\\\";}}s:16:\\\"failureCallbacks\\\";a:0:{}s:23:\\\"deleteWhenMissingModels\\\";b:1;s:7:\\\"batchId\\\";N;s:3:\\\"job\\\";N;s:10:\\\"connection\\\";N;s:5:\\\"queue\\\";N;s:15:\\\"chainConnection\\\";N;s:10:\\\"chainQueue\\\";N;s:19:\\\"chainCatchCallbacks\\\";N;s:5:\\\"delay\\\";N;s:11:\\\"afterCommit\\\";N;s:10:\\\"middleware\\\";a:0:{}s:7:\\\"chained\\\";a:0:{}}\"},\"attempts\":0}", + "attributes":{ + "ApproximateReceiveCount":"1", + "SentTimestamp":"1639158884706", + "SenderId":"AROATHDQZYADPZZTHGFLF:vapor-laravel-staging", + "ApproximateFirstReceiveTimestamp":"1639158884710" + }, + "messageAttributes":[ + + ], + "md5OfBody":"28381da423cdb6fb5ba21f2eccb4fea1", + "eventSource":"aws:sqs", + "eventSourceARN":"arn:aws:sqs:eu-west-3:221427384326:laravel-staging", + "awsRegion":"eu-west-3" + } + ] +} \ No newline at end of file diff --git a/tests/Unit/LambdaEventTest.php b/tests/Unit/LambdaEventTest.php new file mode 100644 index 0000000..5b87940 --- /dev/null +++ b/tests/Unit/LambdaEventTest.php @@ -0,0 +1,41 @@ +getEvent(); + + $this->assertIsArray($event->toArray()); + } + + public function test_array_access() + { + $event = $this->getEvent(); + + $this->assertIsArray($event['Records']); + + $this->assertSame('58600123-d011-4d76-af5d-960159ca44aa', $event['Records.0.messageId']); + $this->assertSame('1', $event['Records.0.attributes.ApproximateReceiveCount']); + + unset($event['Records']); + $this->assertFalse(isset($event['Records'])); + + $event['Records'] = [['messageId' => 'foo']]; + $this->assertTrue(isset($event['Records'])); + $this->assertSame('foo', $event['Records.0.messageId']); + } + + public function getEvent() + { + return new LambdaEvent(json_decode( + file_get_contents(__DIR__.'/../Fixtures/lambdaEvent.json'), + true + )); + } +} diff --git a/tests/Unit/QueueHandlerTest.php b/tests/Unit/QueueHandlerTest.php new file mode 100644 index 0000000..25236b8 --- /dev/null +++ b/tests/Unit/QueueHandlerTest.php @@ -0,0 +1,84 @@ +assertFalse(FakeJob::$handled); + + $job = new FakeJob; + + $event = $this->getEvent(); + + $event['Records'][0]['body'] = json_encode([ + 'displayName' => FakeJob::class, + 'job' => 'Illuminate\Queue\CallQueuedHandler@call', + 'maxTries' => null, + 'timeout' => null, + 'timeoutAt' => null, + 'data' => [ + 'commandName' => FakeJob::class, + 'command' => serialize($job), + ], + 'attempts' => 0, + ]); + + QueueHandler::$app = $this->app; + + $queueHandler = new QueueHandler(); + + $this->assertFalse(QueueHandler::$app->bound(LambdaEvent::class)); + $queueHandler->handle($event); + $this->assertFalse(QueueHandler::$app->bound(LambdaEvent::class)); + $this->assertTrue(FakeJob::$handled); + } + + protected function getPackageProviders($app) + { + return [ + \Laravel\Vapor\VaporServiceProvider::class, + ]; + } + + protected function getEnvironmentSetUp($app) + { + $app['config']->set('queue.connections.vapor', [ + 'driver' => 'sqs', + 'key' => env('SQS_KEY', 'your-public-key'), + 'secret' => env('SQS_SECRET', 'your-secret-key'), + 'prefix' => env('SQS_PREFIX', 'https://sqs.us-east-1.amazonaws.com/your-account-id'), + 'queue' => env('SQS_QUEUE', 'your-queue-name'), + 'region' => env('SQS_REGION', 'us-east-1'), + 'delay' => env('SQS_DELAY', 0), + 'tries' => env('SQS_TRIES', 0), + 'force' => env('SQS_FORCE', false), + ]); + } + + protected function getEvent() + { + return json_decode( + file_get_contents(__DIR__.'/../Fixtures/lambdaEvent.json'), + true + ); + } +} diff --git a/tests/Unit/VaporWorkCommandTest.php b/tests/Unit/VaporWorkCommandTest.php index d88dcaf..e257e48 100644 --- a/tests/Unit/VaporWorkCommandTest.php +++ b/tests/Unit/VaporWorkCommandTest.php @@ -2,11 +2,19 @@ namespace Laravel\Vapor\Tests\Unit; +use Laravel\Vapor\Events\LambdaEvent; use Mockery; use Orchestra\Testbench\TestCase; class VaporWorkCommandTest extends TestCase { + protected function setUp(): void + { + parent::setUp(); + + FakeJob::$handled = false; + } + protected function tearDown(): void { Mockery::close(); @@ -18,40 +26,28 @@ public function test_command_can_be_called() $job = new FakeJob; - $message = base64_encode(json_encode([ - 'messageId' => 'test-message-id', - 'receiptHandle' => 'test-receipt-handle', - 'body' => json_encode([ - 'displayName' => FakeJob::class, - 'job' => 'Illuminate\Queue\CallQueuedHandler@call', - 'maxTries' => null, - 'timeout' => null, - 'timeoutAt' => null, - 'data' => [ - 'commandName' => FakeJob::class, - 'command' => serialize($job), - ], - 'attempts' => 0, - ]), - 'attributes' => [ - 'ApproximateReceiveCount' => 1, + $event = $this->getEvent(); + + $event['Records.0.body'] = json_encode([ + 'displayName' => FakeJob::class, + 'job' => 'Illuminate\Queue\CallQueuedHandler@call', + 'maxTries' => null, + 'timeout' => null, + 'timeoutAt' => null, + 'data' => [ + 'commandName' => FakeJob::class, + 'command' => serialize($job), ], - 'messageAttributes' => [], - 'eventSourceARN' => 'arn:aws:sqs:us-east-1:959512994844:vapor-test-queue-2', - 'awsRegion' => 'us-east-1', - ])); + 'attempts' => 0, + ]); + + $this->instance(LambdaEvent::class, $event); - $this->artisan('vapor:work', ['message' => $message]); + $this->artisan('vapor:work'); $this->assertTrue(FakeJob::$handled); } - /** - * Get the package's service providers. - * - * @param \Illuminate\Foundation\Application $app - * @return array - */ protected function getPackageProviders($app) { return [ @@ -59,11 +55,6 @@ protected function getPackageProviders($app) ]; } - /** - * Define the environment. - * - * @param \Illuminate\Foundation\Application $app - */ protected function getEnvironmentSetUp($app) { $app['config']->set('queue.connections.vapor', [ @@ -78,4 +69,12 @@ protected function getEnvironmentSetUp($app) 'force' => env('SQS_FORCE', false), ]); } + + protected function getEvent() + { + return new LambdaEvent(json_decode( + file_get_contents(__DIR__.'/../Fixtures/lambdaEvent.json'), + true + )); + } }