Implemented retry-exectution for specific handler (#41)

* added additional field to execution

* added retry to separate-process executor

* added upgrade note

* updated dependencies

Author: wachterjohannes

UPGRADE.md

@@ -1,9 +1,19 @@
 UPGRADE
 =======
 
+- [1.2.0](#1.2.0)
 - [1.1.0](#1.1.0)
 - [0.4.0](#0.4.0)
 
+### 1.2.0
+
+In the database table `ta_task_executions` a new field was introduced. Run following
+command to update the table.
+
+```bash
+bin/console doctrine:schema:update
+```
+
 ### 1.1.0
 
 In the database table `ta_tasks` a new field was introduced. Run following

src/Command/ExecuteCommand.php

@@ -16,6 +16,7 @@
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\ConsoleOutputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
+use Task\Executor\FailedException;
 use Task\Handler\TaskHandlerFactoryInterface;
 use Task\Storage\TaskExecutionRepositoryInterface;
 
@@ -63,15 +64,20 @@ protected function configure()
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $errOutput = $output instanceof ConsoleOutputInterface ? $output->getErrorOutput() : $output;
+        $errorOutput = $output instanceof ConsoleOutputInterface ? $output->getErrorOutput() : $output;
 
         $execution = $this->executionRepository->findByUuid($input->getArgument('uuid'));
         $handler = $this->handlerFactory->create($execution->getHandlerClass());
 
         try {
             $result = $handler->handle($execution->getWorkload());
-        } catch (\Exception $e) {
-            $errOutput->writeln($e->__toString());
+        } catch (\Exception $exception) {
+            if ($exception instanceof FailedException) {
+                $errorOutput->writeln(FailedException::class);
+                $exception = $exception->getPrevious();
+            }
+
+            $errorOutput->writeln($exception->__toString());
 
             // Process exit-code: 0 = OK, >1 = FAIL
             return 1;

src/Executor/ExecutionProcessFactory.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace Task\TaskBundle\Executor;
+
+use Symfony\Component\Process\Process;
+use Symfony\Component\Process\ProcessBuilder;
+
+/**
+ * Factory for execution-process.
+ */
+class ExecutionProcessFactory
+{
+    /**
+     * @var string
+     */
+    private $consolePath;
+
+    /**
+     * @var string
+     */
+    private $environment;
+
+    /**
+     * @param string $consolePath
+     * @param string $environment
+     */
+    public function __construct($consolePath, $environment)
+    {
+        $this->consolePath = $consolePath;
+        $this->environment = $environment;
+    }
+
+    /**
+     * Create process for given execution-uuid.
+     *
+     * @param string $uuid
+     *
+     * @return Process
+     */
+    public function create($uuid)
+    {
+        return $process = ProcessBuilder::create(
+            [$this->consolePath, 'task:execute', $uuid, '-e ' . $this->environment]
+        )->getProcess();
+    }
+}

src/Executor/SeparateProcessExecutor.php

@@ -11,50 +11,132 @@
 
 namespace Task\TaskBundle\Executor;
 
-use Symfony\Component\Process\ProcessBuilder;
 use Task\Execution\TaskExecutionInterface;
 use Task\Executor\ExecutorInterface;
+use Task\Executor\FailedException;
+use Task\Executor\RetryTaskHandlerInterface;
+use Task\Handler\TaskHandlerFactoryInterface;
+use Task\Storage\TaskExecutionRepositoryInterface;
 
 /**
  * Uses a separate process to start the executions via console-command.
  */
 class SeparateProcessExecutor implements ExecutorInterface
 {
     /**
-     * @var string
+     * @var TaskHandlerFactoryInterface
      */
-    private $consolePath;
+    private $handlerFactory;
 
     /**
-     * @var string
+     * @var TaskExecutionRepositoryInterface
      */
-    private $environment;
+    private $executionRepository;
 
     /**
-     * @param string $consolePath
-     * @param string $environment
+     * @var ExecutionProcessFactory
      */
-    public function __construct($consolePath, $environment)
-    {
-        $this->consolePath = $consolePath;
-        $this->environment = $environment;
+    private $processFactory;
+
+    /**
+     * @param TaskHandlerFactoryInterface $handlerFactory
+     * @param TaskExecutionRepositoryInterface $executionRepository
+     * @param ExecutionProcessFactory $processFactory
+     */
+    public function __construct(
+        TaskHandlerFactoryInterface $handlerFactory,
+        TaskExecutionRepositoryInterface $executionRepository,
+        ExecutionProcessFactory $processFactory
+    ) {
+        $this->handlerFactory = $handlerFactory;
+        $this->executionRepository = $executionRepository;
+        $this->processFactory = $processFactory;
     }
 
     /**
      * {@inheritdoc}
      */
     public function execute(TaskExecutionInterface $execution)
     {
-        $process = ProcessBuilder::create(
-            [$this->consolePath, 'task:execute', $execution->getUuid(), '-e ' . $this->environment]
-        )->getProcess();
+        $attempts = $this->getMaximumAttempts($execution->getHandlerClass());
+        $lastException = null;
+
+        for ($attempt = 0; $attempt < $attempts; ++$attempt) {
+            try {
+                return $this->handle($execution);
+            } catch (FailedException $exception) {
+                throw $exception;
+            } catch (SeparateProcessException $exception) {
+                if ($execution->getAttempts() < $attempts) {
+                    $execution->incrementAttempts();
+                    $this->executionRepository->save($execution);
+                }
+
+                $lastException = $exception;
+            }
+        }
+
+        // maximum attempts to pass executions are reached
+        throw new FailedException($lastException);
+    }
+
+    /**
+     * Returns maximum attempts for specified handler.
+     *
+     * @param string $handlerClass
+     *
+     * @return int
+     */
+    private function getMaximumAttempts($handlerClass)
+    {
+        $handler = $this->handlerFactory->create($handlerClass);
+        if (!$handler instanceof RetryTaskHandlerInterface) {
+            return 1;
+        }
 
+        return $handler->getMaximumAttempts();
+    }
+
+    /**
+     * Handle execution by using console-command.
+     *
+     * @param TaskExecutionInterface $execution
+     *
+     * @return string
+     *
+     * @throws FailedException
+     * @throws SeparateProcessException
+     */
+    private function handle(TaskExecutionInterface $execution)
+    {
+        $process = $this->processFactory->create($execution->getUuid());
         $process->run();
 
         if (!$process->isSuccessful()) {
-            throw new SeparateProcessException($process->getErrorOutput());
+            throw $this->createException($process->getErrorOutput());
         }
 
         return $process->getOutput();
     }
+
+    /**
+     * Create the correct exception.
+     *
+     * FailedException for failed executions.
+     * SeparateProcessExceptions for any exception during execution.
+     *
+     * @param string $errorOutput
+     *
+     * @return FailedException|SeparateProcessException
+     */
+    private function createException($errorOutput)
+    {
+        if (strpos($errorOutput, FailedException::class) !== 0) {
+            return new SeparateProcessException($errorOutput);
+        }
+
+        $errorOutput = trim(str_replace(FailedException::class, '', $errorOutput));
+
+        return new FailedException(new SeparateProcessException($errorOutput));
+    }
 }

src/Resources/config/doctrine/TaskExecution.orm.xml

@@ -22,7 +22,8 @@
         <field name="exception" type="text" nullable="true"/>
         <field name="result" type="object" nullable="true"/>
         <field name="status" type="string" length="20"/>
-        
+        <field name="attempts" type="integer"/>
+
         <many-to-one target-entity="Task\TaskBundle\Entity\Task" field="task">
             <join-column name="task_id" referenced-column-name="uuid" on-delete="CASCADE"/>
         </many-to-one>

src/Resources/config/executor/separate.xml

@@ -4,6 +4,12 @@
            xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd">
     <services>
         <service id="task.executor.separate" class="Task\TaskBundle\Executor\SeparateProcessExecutor">
+            <argument type="service" id="task.handler.factory"/>
+            <argument type="service" id="task.storage.task_execution"/>
+            <argument type="service" id="task.executor.separate.process_factory"/>
+        </service>
+
+        <service id="task.executor.separate.process_factory" class="Task\TaskBundle\Executor\ExecutionProcessFactory">
             <argument type="string">%task.executor.console_path%</argument>
             <argument type="string">%kernel.environment%%</argument>
         </service>

src/Resources/config/task_event_listener.xml

@@ -11,6 +11,7 @@
         <parameter key="task.events.finished" type="constant">Task\Event\Events::TASK_FINISHED</parameter>
         <parameter key="task.events.passed" type="constant">Task\Event\Events::TASK_PASSED</parameter>
         <parameter key="task.events.failed" type="constant">Task\Event\Events::TASK_FAILED</parameter>
+        <parameter key="task.events.retried" type="constant">Task\Event\Events::TASK_RETRIED</parameter>
     </parameters>
 
 </container>