[metrics] Add support for custom file type patterns

This commit allows to define custom file type patterns for code and binary files.

Signed-off-by: Jose Javier Merchante <[email protected]>

Author: jjmerchante

README.md

@@ -54,8 +54,10 @@ trustable spdx.xml \
   --grimoirelab-user user --grimoirelab-password password \
   --opensearch-url https://admin:[email protected]:9200 \
   --opensearch-index events \
-  --output metrics.json \
   --repository-timeout 3600
+  --code-file-pattern "\.py$|\.js$" \
+  --binary-file-pattern "\.exe$|\.tar$" \
+  --output metrics.json
 ```
 
 The parameters needed to run the tool are:

tests/end_to_end/test_cli.py

@@ -72,7 +72,7 @@ def test_metrics(self):
                 self.assertEqual(quickstart_metrics["pony_factor"], 2)
                 self.assertEqual(quickstart_metrics["elephant_factor"], 2)
                 self.assertEqual(quickstart_metrics["file_types_other"], 683)
-                self.assertNotIn("file_types_binary", quickstart_metrics)
+                self.assertEqual(quickstart_metrics["file_types_binary"], 0)
                 self.assertEqual(quickstart_metrics["file_types_code"], 479)
                 self.assertEqual(quickstart_metrics["commit_size_added_lines"], 53121)
                 self.assertEqual(quickstart_metrics["commit_size_removed_lines"], 51852)
@@ -149,7 +149,7 @@ def test_from_date(self):
                 self.assertEqual(quickstart_metrics["pony_factor"], 2)
                 self.assertEqual(quickstart_metrics["elephant_factor"], 1)
                 self.assertEqual(quickstart_metrics["file_types_other"], 37)
-                self.assertNotIn("file_types_binary", quickstart_metrics)
+                self.assertEqual(quickstart_metrics["file_types_binary"], 0)
                 self.assertEqual(quickstart_metrics["file_types_code"], 17)
                 self.assertEqual(quickstart_metrics["commit_size_added_lines"], 269)
                 self.assertEqual(quickstart_metrics["commit_size_removed_lines"], 103)
@@ -171,7 +171,7 @@ def test_from_date(self):
                 self.assertEqual(angular_metrics["pony_factor"], 1)
                 self.assertEqual(angular_metrics["elephant_factor"], 1)
                 self.assertEqual(angular_metrics["file_types_other"], 24)
-                self.assertNotIn("file_types_binary", angular_metrics)
+                self.assertEqual(angular_metrics["file_types_binary"], 0)
                 self.assertEqual(angular_metrics["file_types_code"], 13)
                 self.assertEqual(angular_metrics["commit_size_added_lines"], 4849)
                 self.assertEqual(angular_metrics["commit_size_removed_lines"], 149)
@@ -227,7 +227,7 @@ def test_to_date(self):
                 self.assertEqual(quickstart_metrics["pony_factor"], 2)
                 self.assertEqual(quickstart_metrics["elephant_factor"], 2)
                 self.assertEqual(quickstart_metrics["file_types_other"], 646)
-                self.assertNotIn("file_types_binary", quickstart_metrics)
+                self.assertEqual(quickstart_metrics["file_types_binary"], 0)
                 self.assertEqual(quickstart_metrics["file_types_code"], 462)
                 self.assertEqual(quickstart_metrics["commit_size_added_lines"], 52852)
                 self.assertEqual(quickstart_metrics["commit_size_removed_lines"], 51749)
@@ -305,7 +305,7 @@ def test_duplicate_repo(self):
                 self.assertEqual(quickstart_metrics["pony_factor"], 2)
                 self.assertEqual(quickstart_metrics["elephant_factor"], 2)
                 self.assertEqual(quickstart_metrics["file_types_other"], 683)
-                self.assertNotIn("file_types_binary", quickstart_metrics)
+                self.assertEqual(quickstart_metrics["file_types_binary"], 0)
                 self.assertEqual(quickstart_metrics["file_types_code"], 479)
                 self.assertEqual(quickstart_metrics["commit_size_added_lines"], 53121)
                 self.assertEqual(quickstart_metrics["commit_size_removed_lines"], 51852)
@@ -361,7 +361,7 @@ def test_non_git_repo(self):
                 self.assertEqual(quickstart_metrics["pony_factor"], 2)
                 self.assertEqual(quickstart_metrics["elephant_factor"], 2)
                 self.assertEqual(quickstart_metrics["file_types_other"], 683)
-                self.assertNotIn("file_types_binary", quickstart_metrics)
+                self.assertEqual(quickstart_metrics["file_types_binary"], 0)
                 self.assertEqual(quickstart_metrics["file_types_code"], 479)
                 self.assertEqual(quickstart_metrics["commit_size_added_lines"], 53121)
                 self.assertEqual(quickstart_metrics["commit_size_removed_lines"], 51852)

tests/unit/test_metrics.py

@@ -116,14 +116,32 @@ def test_get_elephant_factor(self):
     def test_file_type_metrics(self):
         """Test that file type metrics are calculated correctly"""
 
-        self.assertEqual(self.analyzer.get_file_type_metrics(), {})
-
         self.analyzer.process_events(self.events)
 
         file_metrics = self.analyzer.get_file_type_metrics()
-        self.assertEqual(file_metrics.get("code", 0), 54)
-        self.assertEqual(file_metrics.get("binary", 0), 1)
-        self.assertEqual(file_metrics.get("other", 0), 24)
+        self.assertEqual(file_metrics["code"], 54)
+        self.assertEqual(file_metrics["binary"], 1)
+        self.assertEqual(file_metrics["other"], 24)
+
+    def test_file_type_metrics_empty(self):
+        """Test that file type metrics are calculated correctly without events"""
+
+        file_metrics = self.analyzer.get_file_type_metrics()
+        self.assertEqual(file_metrics["code"], 0)
+        self.assertEqual(file_metrics["binary"], 0)
+        self.assertEqual(file_metrics["other"], 0)
+
+    def test_file_type_metrics_new_regex(self):
+        """Test that file type metrics are calculated correctly with new regex"""
+
+        analyzer = GitEventsAnalyzer(code_file_pattern=r"\.py$", binary_file_pattern=r"\.md$")
+
+        analyzer.process_events(self.events)
+
+        file_metrics = analyzer.get_file_type_metrics()
+        self.assertEqual(file_metrics["code"], 53)
+        self.assertEqual(file_metrics["binary"], 4)
+        self.assertEqual(file_metrics["other"], 22)
 
     def test_commit_size_metrics(self):
         """Test that commit size metrics are calculated correctly"""

trustable_cli/cli.py

@@ -77,6 +77,8 @@
 )
 @click.option("--verify-certs", is_flag=True, default=False, help="Verify SSL/TLS certificates")
 @click.option("--verbose", is_flag=True, default=False, help="Increase output verbosity")
+@click.option("--code-file-pattern", help="Regular expression to match code file types")
+@click.option("--binary-file-pattern", help="Regular expression to match binary file types")
 def trustable_grimoirelab_score(
     filename: str,
     grimoirelab_url: str,
@@ -90,6 +92,8 @@ def trustable_grimoirelab_score(
     to_date: datetime.datetime | None = None,
     verify_certs: bool = False,
     verbose: bool = False,
+    code_file_pattern: str | None = None,
+    binary_file_pattern: str | None = None,
 ) -> None:
     """Calculate metrics for Trustable using GrimoireLab.
 
@@ -131,6 +135,8 @@ def trustable_grimoirelab_score(
             to_date=to_date,
             verify_certs=verify_certs,
             timeout=repository_timeout,
+            code_file_pattern=code_file_pattern,
+            binary_file_pattern=binary_file_pattern,
         )
 
         package_metrics = {"packages": {}}
@@ -203,6 +209,8 @@ def generate_metrics_when_ready(
     to_date: datetime.datetime | None = None,
     verify_certs: bool = False,
     timeout: int = 3600,
+    code_file_pattern: str | None = None,
+    binary_file_pattern: str | None = None,
 ) -> dict[str:Any]:
     """Generate metrics once the repositories have finished the collection.
 
@@ -214,6 +222,8 @@ def generate_metrics_when_ready(
     :param to_date: End date for metrics.
     :param verify_certs: Verify SSL/TLS certificates.
     :param timeout: Seconds to wait before failing getting metrics
+    :param code_file_pattern: Regular expression to match code file types.
+    :param binary_file_pattern: Regular expression to match binary file types.
     """
     logging.info("Generating metrics")
 
@@ -228,12 +238,14 @@ def generate_metrics_when_ready(
         for repository in pending_repositories:
             if repository_ready(grimoirelab_client, repository, after_date):
                 metrics["repositories"][repository] = get_repository_metrics(
-                    repository,
-                    opensearch_url,
-                    opensearch_index,
-                    from_date,
-                    to_date,
-                    verify_certs,
+                    repository=repository,
+                    opensearch_url=opensearch_url,
+                    opensearch_index=opensearch_index,
+                    from_date=from_date,
+                    to_date=to_date,
+                    verify_certs=verify_certs,
+                    code_file_pattern=code_file_pattern,
+                    binary_file_pattern=binary_file_pattern,
                 )
                 processed.add(repository)
 

trustable_cli/metrics.py

@@ -51,14 +51,16 @@
 
 
 class GitEventsAnalyzer:
-    def __init__(self):
+    def __init__(self, code_file_pattern: str | None = None, binary_file_pattern: str | None = None):
         self.total_commits: int = 0
         self.contributors: Counter = Counter()
         self.companies: Counter = Counter()
-        self.file_types: dict = Counter()
+        self.file_types: dict = {"code": 0, "binary": 0, "other": 0}
         self.added_lines: int = 0
         self.removed_lines: int = 0
         self.messages_sizes: list = []
+        self.re_code_pattern = re.compile(code_file_pattern or FILE_TYPE_CODE)
+        self.re_binary_pattern = re.compile(binary_file_pattern or FILE_TYPE_BINARY)
 
     def process_events(self, events: iter(dict[str, Any])):
         for event in events:
@@ -116,7 +118,7 @@ def get_elephant_factor(self):
     def get_file_type_metrics(self):
         """Get the file type metrics"""
 
-        return dict(self.file_types)
+        return self.file_types
 
     def get_commit_size_metrics(self):
         """Get the commit size metrics"""
@@ -195,9 +197,9 @@ def _update_file_metrics(self, event):
             if not file["file"]:
                 continue
             # File type metrics
-            if re.search(FILE_TYPE_CODE, file["file"]):
+            if self.re_code_pattern.search(file["file"]):
                 self.file_types["code"] += 1
-            elif re.search(FILE_TYPE_BINARY, file["file"]):
+            elif self.re_binary_pattern.search(file["file"]):
                 self.file_types["binary"] += 1
             else:
                 self.file_types["other"] += 1
@@ -226,6 +228,8 @@ def get_repository_metrics(
     from_date: datetime.datetime = None,
     to_date: datetime.datetime = None,
     verify_certs: bool = True,
+    code_file_pattern: str | None = None,
+    binary_file_pattern: str | None = None,
 ):
     """
     Get the metrics from a repository.
@@ -236,14 +240,16 @@ def get_repository_metrics(
     :param verify_certs: Boolean, verify SSL/TLS certificates, default True
     :param from_date: Start date, by default None
     :param to_date: End date, by default None
+    :param code_file_pattern: Regular expression to match code file types.
+    :param binary_file_pattern: Regular expression to match binary file types.
     """
     os_conn = connect_to_opensearch(opensearch_url, verify_certs=verify_certs)
 
     metrics = {"metrics": {}}
 
     events = get_repository_events(os_conn, opensearch_index, repository, from_date, to_date)
 
-    analyzer = GitEventsAnalyzer()
+    analyzer = GitEventsAnalyzer(code_file_pattern=code_file_pattern, binary_file_pattern=binary_file_pattern)
     analyzer.process_events(events)
 
     metrics["metrics"]["total_commits"] = analyzer.get_commit_count()