[cli] Repository timeout option for never-ending repositories

This commit adds repository timeout option for never-ending repositories.
The default value is 1h. This will make that some new big repositories
won't have metrics until it is analyzed.

Signed-off-by: Jose Javier Merchante <[email protected]>

Author: jjmerchante

README.md

@@ -54,7 +54,8 @@ trustable spdx.xml \
   --grimoirelab-user user --grimoirelab-password password \
   --opensearch-url https://admin:[email protected]:9200 \
   --opensearch-index events \
-  --output metrics.json
+  --output metrics.json \
+  --repository-timeout 3600
 ```
 
 The parameters needed to run the tool are:

tests/unit/test_cli.py

@@ -96,6 +96,34 @@ def request_callback(request, uri, headers):
     return http_requests
 
 
+def setup_get_never_ending_repositories_mock_server():
+    """Setup a mock HTTP server for repository API calls"""
+
+    http_requests = []
+
+    def request_callback(request, uri, headers):
+        last_request = httpretty.last_request()
+        http_requests.append(last_request)
+        last_run = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=365)
+        data = {
+            "results": [
+                {
+                    "task": {
+                        "last_run": last_run.isoformat(),
+                        "status": "running",
+                    }
+                }
+            ]
+        }
+        body = json.dumps(data)
+
+        return 200, headers, body
+
+    httpretty.register_uri(httpretty.GET, REPOSITORIES_URL, responses=[httpretty.Response(body=request_callback)])
+
+    return http_requests
+
+
 class TestCli(unittest.TestCase):
     def setUp(self):
         logging.getLogger().handlers = []
@@ -348,6 +376,42 @@ def test_server_error(self):
         self.assertIn("Error scheduling task", result.output)
         self.assertEqual(len(http_requests), 5)
 
+    @httpretty.activate
+    @patch("trustable_cli.cli.get_repository_metrics")
+    def test_never_ending_repository(self, mock_get_repository_metrics):
+        """Check if it returns a warning when a repository task never ends"""
+
+        http_requests = setup_add_repository_mock_server()
+        http_requests_repos = setup_get_never_ending_repositories_mock_server()
+        mock_get_repository_metrics.return_value = {"metrics": {"num_commits": 10}}
+
+        runner = CliRunner()
+
+        result = runner.invoke(
+            trustable_grimoirelab_score,
+            [
+                "./data/valid.spdx.xml",
+                "--grimoirelab-url",
+                GRIMOIRELAB_URL,
+                "--opensearch-url",
+                OPENSEARCH_URL,
+                "--opensearch-index",
+                OPENSEARCH_INDEX,
+                "--output",
+                self.temp_file.name,
+                "--repository-timeout",
+                15,
+            ],
+        )
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertIn(
+            "Timeout waiting for repository https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux to be ready",
+            result.output,
+        )
+        self.assertEqual(len(http_requests), 5)
+        self.assertEqual(len(http_requests_repos), 10)
+
 
 class TestGetRepository(unittest.TestCase):
     def test_valid_git_repository(self):

trustable_cli/cli.py

@@ -58,6 +58,12 @@
 )
 @click.option("--opensearch-index", help="OpenSearch index", default="events")
 @click.option("--output", help="File where the scores will be written", type=click.File("w"), default=sys.stdout)
+@click.option(
+    "--repository-timeout",
+    type=int,
+    help="Timeout in seconds to wait for a repository to be analyzed",
+    default=3600,
+)
 @click.option(
     "--from-date",
     type=click.DateTime(formats=["%Y-%m-%d"]),
@@ -79,6 +85,7 @@ def trustable_grimoirelab_score(
     opensearch_url: str,
     opensearch_index: str,
     output: str,
+    repository_timeout: int,
     from_date: datetime.datetime | None = None,
     to_date: datetime.datetime | None = None,
     verify_certs: bool = False,
@@ -116,7 +123,14 @@ def trustable_grimoirelab_score(
         schedule_repositories(git_urls, grimoirelab_client)
 
         metrics = generate_metrics_when_ready(
-            grimoirelab_client, git_urls, opensearch_url, opensearch_index, from_date, to_date, verify_certs
+            grimoirelab_client=grimoirelab_client,
+            repositories=git_urls,
+            opensearch_url=opensearch_url,
+            opensearch_index=opensearch_index,
+            from_date=from_date,
+            to_date=to_date,
+            verify_certs=verify_certs,
+            timeout=repository_timeout,
         )
 
         package_metrics = {"packages": {}}
@@ -188,6 +202,7 @@ def generate_metrics_when_ready(
     from_date: datetime.datetime | None = None,
     to_date: datetime.datetime | None = None,
     verify_certs: bool = False,
+    timeout: int = 3600,
 ) -> dict[str:Any]:
     """Generate metrics once the repositories have finished the collection.
 
@@ -198,9 +213,12 @@ def generate_metrics_when_ready(
     :param from_date: Start date for metrics.
     :param to_date: End date for metrics.
     :param verify_certs: Verify SSL/TLS certificates.
+    :param timeout: Seconds to wait before failing getting metrics
     """
     logging.info("Generating metrics")
 
+    limit_time = time.time() + timeout
+
     after_date = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=7)
     pending_repositories = set(repositories)
     metrics = {"repositories": {}}
@@ -221,10 +239,15 @@ def generate_metrics_when_ready(
 
         pending_repositories -= processed
 
-        if pending_repositories:
+        if pending_repositories and time.time() < limit_time:
             logging.info(f"Waiting for {len(pending_repositories)} repositories to be ready")
             logging.debug(f"Repositories not ready: {pending_repositories}")
             time.sleep(25)
+        else:
+            break
+
+    for repository in pending_repositories:
+        logging.warning(f"Timeout waiting for repository {repository} to be ready")
 
     return metrics