✨ feat(icu): wire ICU validation into TM Analysis + fix FastAnalysis transaction lifecycle (#4517)

* ✨ feat(icu): integrate optional ICU MessageFormat validation

- add support for detecting and validating ICU MessageFormat patterns in segments
- wire ICU detection in TMAnalysisWorker and PostProcess
- update FastAnalysis to handle ICU-enabled metadata

Signed-off-by: domenico <[email protected]>

* ✅ test(icu): add ICU validation tests for PostProcess and ICUSourceSegmentDetector

Cover the PostProcess → QA ICU wiring (PR #4517) and the shared
detection gate used by both QAProcessor and TMAnalysisWorker:

- 3 PostProcess tests: broken target detected, skipped without comparator, valid pass
- 4 ICUSourceSegmentDetector tests: enabled+valid, disabled, plain text, simple argument

* 🔧 chore(phpstan): update baseline for FastAnalysis refactoring and baseline APIDoc.php

* 🔧 chore(phpstan): exclude APIDoc.php from analysis paths

- add `excludePaths` to phpstan.neon configuration to skip lib/View/APIDoc.php

Signed-off-by: domenico <[email protected]>

* 🔧 fix(phpstan): remove baseline entries for untracked APIDoc.php

* 🔧 refactor(icu): add null guard in FastAnalysis, rename icuPluralsValidator to icuComparator

* ♻️ refactor(analysis): add Database::transaction() and fix FastAnalysis TX lifecycle

Add Database::transaction(callable) for safe atomic execution with
automatic rollback on failure.

Refactor all 4 transaction sites in FastAnalysis.php:
- Fix rollBack() called with no active transaction (crash on error)
- Fix null dereference + dangling TX in _updateProject()
- Fix mixed API usage in _getLockProjectForVolumeAnalysis()
- Remove dead unset() calls for undefined variables
- Remove 5 stale PHPStan baseline entries

---------

Signed-off-by: domenico <[email protected]>

Author: Ostico

internal_scripts

@@ -5,6 +5,7 @@
 use Exception;
 use PDO;
 use PDOException;
+use Throwable;
 
 /**
  * Class which implements a database using PDO
@@ -181,6 +182,26 @@ public function rollback(): void
         $this->getConnection()->rollBack();
     }
 
+    /**
+     * @Override
+     * {@inheritdoc}
+     */
+    public function transaction(callable $callback): mixed
+    {
+        $this->begin();
+        try {
+            $result = $callback();
+            $this->commit();
+
+            return $result;
+        } catch (Throwable $e) {
+            if ($this->getConnection()->inTransaction()) {
+                $this->rollback();
+            }
+            throw $e;
+        }
+    }
+
     /**
      * @Warning This method does not support all the SQL syntax features. Only AND key/value pair is supported, OR in WHERE condition is not supported, nesting "AND ( .. OR .. ) AND ( .. )" is not supported
      * @Override

lib/Model/DataAccess/Database.php

@@ -3,6 +3,7 @@
 namespace Model\DataAccess;
 
 use PDO;
+use Throwable;
 
 interface IDatabase
 {
@@ -56,6 +57,22 @@ public function commit(): void;
      */
     public function rollback(): void;
 
+    /**
+     * Execute a callback within a database transaction.
+     *
+     * Begins a transaction, executes the callback, and commits.
+     * On any exception, rolls back (if still in transaction) and re-throws.
+     *
+     * @template T
+     *
+     * @param callable(): T $callback
+     *
+     * @return T The value returned by the callback
+     *
+     * @throws Throwable Re-throws the original exception after rollback
+     */
+    public function transaction( callable $callback ): mixed;
+
     /**
      * Execute a update query with an array as argument
      *

lib/Model/DataAccess/IDatabase.php

@@ -13,7 +13,6 @@
 use Model\Jobs\JobsMetadataMarshaller;
 use Model\Jobs\MetadataDao;
 use Model\MTQE\Templates\DTO\MTQEWorkflowParams;
-use Model\Projects\MetadataDao as ProjectsMetadataDao;
 use Model\Projects\ProjectDao;
 use Model\Projects\ProjectsMetadataMarshaller;
 use Model\Projects\ProjectStruct;
@@ -24,6 +23,7 @@
 use PDOException;
 use ReflectionException;
 use Stomp\Transport\Message;
+use Throwable;
 use UnexpectedValueException;
 use Utils\ActiveMQ\AMQHandler;
 use Utils\AsyncTasks\Workers\Traits\ProjectWordCount;
@@ -240,22 +240,38 @@ public function main(array $args = null): void
                 // INSERT DATA
                 $this->logger->debug("Inserting segments...");
 
-                // define variable for the sake of the code, even if empty
                 $projectStruct = new ProjectStruct();
 
                 try {
                     /**
                      * Ensure we have fresh data from the master node
                      */
-                    Database::obtain()->getConnection()->beginTransaction();
-                    $projectStruct = ProjectDao::findById($pid);
-                    $projectFeaturesString = $projectStruct->getMetadataValue(ProjectsMetadataMarshaller::FEATURES_KEY->value);
-                    $mt_evaluation = $projectStruct->getMetadataValue(ProjectsMetadataMarshaller::MT_EVALUATION->value);
-                    $mt_qe_workflow_enabled = $projectStruct->getMetadataValue(ProjectsMetadataMarshaller::MT_QE_WORKFLOW_ENABLED->value);
-                    $mt_qe_workflow_parameters = $projectStruct->getMetadataValue(ProjectsMetadataMarshaller::MT_QE_WORKFLOW_PARAMETERS->value);
-                    $mt_quality_value_in_editor = $projectStruct->getMetadataValue(ProjectsMetadataMarshaller::MT_QUALITY_VALUE_IN_EDITOR->value);
-                    $subfiltering_handlers = (new ProjectsMetadataDao)->getProjectStaticSubfilteringCustomHandlers($projectStruct->id);
-                    Database::obtain()->getConnection()->commit();
+                    $metadataResult = Database::obtain()->transaction(function () use ($pid) {
+                        $projectStruct = ProjectDao::findById($pid);
+                        if ($projectStruct === null) {
+                            return null;
+                        }
+
+                        return [
+                            'project' => $projectStruct,
+                            'metadata' => $projectStruct->getAllMetadataAsKeyValue(),
+                        ];
+                    });
+
+                    if ($metadataResult === null) {
+                        $this->logger->error("Unable to insert fast analysis: project not found for pid $pid");
+                        continue;
+                    }
+
+                    $projectStruct = $metadataResult['project'];
+                    $allMetadata = $metadataResult['metadata'];
+                    $projectFeaturesString = $allMetadata[ProjectsMetadataMarshaller::FEATURES_KEY->value] ?? '';
+                    $mt_evaluation = $allMetadata[ProjectsMetadataMarshaller::MT_EVALUATION->value] ?? false;
+                    $mt_qe_workflow_enabled = $allMetadata[ProjectsMetadataMarshaller::MT_QE_WORKFLOW_ENABLED->value] ?? false;
+                    $mt_qe_workflow_parameters = $allMetadata[ProjectsMetadataMarshaller::MT_QE_WORKFLOW_PARAMETERS->value] ?? null;
+                    $mt_quality_value_in_editor = $allMetadata[ProjectsMetadataMarshaller::MT_QUALITY_VALUE_IN_EDITOR->value] ?? 85;
+                    $subfiltering_handlers = $allMetadata[ProjectsMetadataMarshaller::SUBFILTERING_HANDLERS->value] ?? [];
+                    $icu_enabled = $allMetadata[ProjectsMetadataMarshaller::ICU_ENABLED->value] ?? false;
 
                     $insertReportRes = $this->_insertFastAnalysis(
                         $projectStruct,
@@ -267,11 +283,10 @@ public function main(array $args = null): void
                         $mt_qe_workflow_enabled,
                         $mt_qe_workflow_parameters,
                         $mt_quality_value_in_editor,
-                        $subfiltering_handlers
+                        $subfiltering_handlers,
+                        $icu_enabled
                     );
-                } catch (Exception $e) {
-                    //Logging done and email sent
-                    //set to error
+                } catch (Throwable $e) {
                     $insertReportRes = -1;
                     $this->logger->debug($e->getMessage() . " " . $e->getTraceAsString());
                 }
@@ -399,16 +414,23 @@ public function cleanShutDown(): void
      */
     protected function _updateProject($pid, $status): void
     {
-        Database::obtain()->begin();
-        $project = ProjectDao::findById($pid);
-        if ($project->status_analysis != ProjectStatus::STATUS_DONE) { // avoid concurrency between fast and tm daemons ( they set DONE when complete )
-            $this->logger->debug("*** Project $pid: Changing status...");
-            ProjectDao::changeProjectStatus($pid, $status);
-            $this->logger->debug("*** Project $pid: $status");
-        } else {
-            $this->logger->debug("*** Project $pid: TM Analysis already completed. Skip update...");
-        }
-        Database::obtain()->commit();
+        Database::obtain()->transaction(function () use ($pid, $status) {
+            $project = ProjectDao::findById($pid);
+            if ($project === null) {
+                $this->logger->debug("*** Project $pid: not found. Skip update.");
+
+                return;
+            }
+
+            // avoid concurrency between fast and tm daemons ( they set DONE when complete )
+            if ($project->status_analysis != ProjectStatus::STATUS_DONE) {
+                $this->logger->debug("*** Project $pid: Changing status...");
+                ProjectDao::changeProjectStatus($pid, $status);
+                $this->logger->debug("*** Project $pid: $status");
+            } else {
+                $this->logger->debug("*** Project $pid: TM Analysis already completed. Skip update...");
+            }
+        });
     }
 
     /**
@@ -452,6 +474,7 @@ protected function _executeInsert($tuple_list, $bind_values): void
      * @param MTQEWorkflowParams|null $mt_qe_workflow_parameters
      * @param int|null $mt_quality_value_in_editor
      * @param array|null $subfiltering_handlers
+     * @param bool $icu_enabled
      * @return int
      * @throws Exception
      */
@@ -465,7 +488,8 @@ protected function _insertFastAnalysis(
         ?bool $mt_qe_workflow_enabled = false,
         ?MTQEWorkflowParams $mt_qe_workflow_parameters = null,
         ?int $mt_quality_value_in_editor = 85,
-        ?array $subfiltering_handlers = []
+        ?array $subfiltering_handlers = [],
+        bool $icu_enabled = false
     ): int {
         $pid = $projectStruct->id;
         $total_eq_wc = 0;
@@ -560,45 +584,35 @@ protected function _insertFastAnalysis(
             }
         }
 
-        unset($data);
         unset($tuple_list);
-        unset($chunks_bind_values);
-        unset($chunks_st);
-
-        //_TimeStampMsg( "Done." );
 
         $data2 = ['fast_analysis_wc' => $total_eq_wc];
         $where = ["id" => $pid];
 
-
-        $db = Database::obtain();
-        $db->begin();
-
         try {
-            /*
-             * IF NO TM ANALYSIS, update the jobs global word count
-            */
-            if (!$perform_Tms_Analysis) {
-                $_details = $this->getProjectSegmentsTranslationSummary($pid);
+            $project_creation_success = Database::obtain()->transaction(function () use ($perform_Tms_Analysis, $pid, $data2, $where) {
+                /*
+                 * IF NO TM ANALYSIS, update the jobs global word count
+                 */
+                if (!$perform_Tms_Analysis) {
+                    $_details = $this->getProjectSegmentsTranslationSummary($pid);
 
-                $this->logger->debug("--- trying to initialize job total word count.");
+                    $this->logger->debug("--- trying to initialize job total word count.");
 
-                /** @noinspection PhpUnusedLocalVariableInspection */
-                $query_rollup = array_pop($_details); //Don't remove, needed to remove rollup row
+                    /** @noinspection PhpUnusedLocalVariableInspection */
+                    $query_rollup = array_pop($_details); //Don't remove, needed to remove rollup row
 
-                foreach ($_details as $job_info) {
-                    $counter = new CounterModel();
-                    $counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
+                    foreach ($_details as $job_info) {
+                        $counter = new CounterModel();
+                        $counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
+                    }
                 }
-            }
-            /* IF NO TM ANALYSIS, upload the jobs global word count */
+                /* IF NO TM ANALYSIS, upload the jobs global word count */
 
-            $project_creation_success = $db->update('projects', $data2, $where);
-
-            $db->commit();
+                return Database::obtain()->update('projects', $data2, $where);
+            });
         } catch (PDOException $e) {
             $this->logger->debug($e->getMessage());
-            $db->rollback();
 
             return $e->getCode() * -1;
         }
@@ -700,6 +714,7 @@ protected function _insertFastAnalysis(
                         $queue_element['mt_quality_value_in_editor'] = $mt_quality_value_in_editor ?? false;
 
                         $queue_element[JobsMetadataMarshaller::SUBFILTERING_HANDLERS->value] = $subfiltering_handlers;
+                        $queue_element[ProjectsMetadataMarshaller::ICU_ENABLED->value] = $icu_enabled;
 
                         $element = new QueueElement();
                         $element->params = new Params($queue_element);
@@ -894,14 +909,13 @@ protected function _getLockProjectForVolumeAnalysis(int $limit = 1): array
         ORDER BY id LIMIT " . $limit;
 
         $db = Database::obtain();
-        //Needed to address the query to the master database if exists
-        Database::obtain()->begin();
-
-        $stmt = $db->getConnection()->prepare($query);
-        $stmt->execute($bindParams);
-        $results = $stmt->fetchAll(PDO::FETCH_ASSOC);
+        // Needed to address the query to the master database if exists
+        $results = $db->transaction(function () use ($db, $query, $bindParams) {
+            $stmt = $db->getConnection()->prepare($query);
+            $stmt->execute($bindParams);
 
-        $db->getConnection()->commit();
+            return $stmt->fetchAll(PDO::FETCH_ASSOC);
+        });
 
         foreach ($results as $position => $project) {
             //acquire a lock
@@ -913,7 +927,7 @@ protected function _getLockProjectForVolumeAnalysis(int $limit = 1): array
 
                 try {
                     $this->_updateProject($project['id'], ProjectStatus::STATUS_BUSY);
-                } catch (PDOException) {
+                } catch (Exception) {
                     $this->queueHandler->getRedisClient()->del('_fPid:' . $project['id']);
                 }
             }