[metrics] Include mean of commits for month and year

This commit adds the mean number of commits per repository for each week,
month, and year. If the number of days requested for metrics is shorter than
the interval for the metric, it is not calculated because it would not be
meaningful.

Signed-off-by: Jose Javier Merchante <[email protected]>

Author: jjmerchante

README.md

@@ -54,6 +54,7 @@ trustable spdx.xml \
   --grimoirelab-user user --grimoirelab-password password \
   --opensearch-url https://admin:[email protected]:9200 \
   --opensearch-index events \
+  --from-date 2024-01-01 --to-date 2025-01-01 \
   --repository-timeout 3600
   --code-file-pattern "\.py$|\.js$" \
   --binary-file-pattern "\.exe$|\.tar$" \
@@ -106,3 +107,11 @@ This is an example of a valid SPDX file:
 This is the list of the metrics generated by this tool:
 
 - Number of commits per repository
+- Number of developers per repository
+- Number of developers producing up to 50% of the total contributions
+- Number of companies producing up to 50% of the total number of code contributions
+- File type metrics (code, binaries or other)
+- Commit side metrics (added lines and removed lines)
+- Message size metrics (total, mean and median)
+- Frequency metrics for commits (week, month and year)
+- Developer categories (core, regular and casual)

tests/end_to_end/test_cli.py

@@ -50,6 +50,7 @@ def test_metrics(self):
                     "--output",
                     self.temp_file.name,
                     "--from-date=2000-01-01",
+                    "--to-date=2025-01-01",
                 ],
             )
             self.assertEqual(result.exit_code, 0)
@@ -82,7 +83,10 @@ def test_metrics(self):
                 self.assertEqual(quickstart_metrics["developer_categories_core"], 3)
                 self.assertEqual(quickstart_metrics["developer_categories_regular"], 13)
                 self.assertEqual(quickstart_metrics["developer_categories_casual"], 9)
-                self.assertAlmostEqual(quickstart_metrics["commits_week_mean"], 0.06418, delta=0.1)
+                # From 2000 to 2025 there are 9132 days
+                self.assertAlmostEqual(quickstart_metrics["commits_per_week"], 164 / (9132 / 7), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_month"], 164 / (9132 / 30), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_year"], 164 / (9132 / 365), delta=0.1)
 
                 self.assertIn("SPDXRef-angular-seed", metrics["packages"])
                 self.assertEqual(
@@ -104,7 +108,10 @@ def test_metrics(self):
                 self.assertEqual(angular_metrics["developer_categories_core"], 16)
                 self.assertEqual(angular_metrics["developer_categories_regular"], 31)
                 self.assertEqual(angular_metrics["developer_categories_casual"], 11)
-                self.assertAlmostEqual(angular_metrics["commits_week_mean"], 0.08101, delta=0.1)
+                # From 2000 to 2025 there are 9132 days
+                self.assertAlmostEqual(angular_metrics["commits_per_week"], 207 / (9132 / 7), delta=0.1)
+                self.assertAlmostEqual(angular_metrics["commits_per_month"], 207 / (9132 / 30), delta=0.1)
+                self.assertAlmostEqual(angular_metrics["commits_per_year"], 207 / (9132 / 365), delta=0.1)
 
     def test_from_date(self):
         """Check if it returns the number of commits of one repository from a particular date"""
@@ -127,6 +134,7 @@ def test_from_date(self):
                     "--output",
                     self.temp_file.name,
                     "--from-date=2017-01-01",
+                    "--to-date=2025-01-01",
                 ],
             )
             self.assertEqual(result.exit_code, 0)
@@ -159,7 +167,10 @@ def test_from_date(self):
                 self.assertEqual(quickstart_metrics["developer_categories_core"], 3)
                 self.assertEqual(quickstart_metrics["developer_categories_regular"], 3)
                 self.assertEqual(quickstart_metrics["developer_categories_casual"], 2)
-                self.assertAlmostEqual(quickstart_metrics["commits_week_mean"], 0.00861, delta=0.1)
+                # From 2017 to 2025 there are 2922 days
+                self.assertAlmostEqual(quickstart_metrics["commits_per_week"], 22 / (2922 / 7), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_month"], 22 / (2922 / 30), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_year"], 22 / (2922 / 365), delta=0.1)
 
                 self.assertIn("SPDXRef-angular-seed", metrics["packages"])
                 self.assertEqual(
@@ -181,7 +192,10 @@ def test_from_date(self):
                 self.assertEqual(angular_metrics["developer_categories_core"], 1)
                 self.assertEqual(angular_metrics["developer_categories_regular"], 2)
                 self.assertEqual(angular_metrics["developer_categories_casual"], 1)
-                self.assertAlmostEqual(angular_metrics["commits_week_mean"], 0.0043, delta=0.1)
+                # From 2017 to 2025 there are 2922 days
+                self.assertAlmostEqual(angular_metrics["commits_per_week"], 11 / (2922 / 7), delta=0.1)
+                self.assertAlmostEqual(angular_metrics["commits_per_month"], 11 / (2922 / 30), delta=0.1)
+                self.assertAlmostEqual(angular_metrics["commits_per_year"], 11 / (2922 / 365), delta=0.1)
 
     def test_to_date(self):
         """Check if it returns the number of commits of one repository up to a particular date"""
@@ -237,7 +251,10 @@ def test_to_date(self):
                 self.assertEqual(quickstart_metrics["developer_categories_core"], 3)
                 self.assertEqual(quickstart_metrics["developer_categories_regular"], 9)
                 self.assertEqual(quickstart_metrics["developer_categories_casual"], 8)
-                self.assertAlmostEqual(quickstart_metrics["commits_week_mean"], 0.003266620657925006, delta=0.1)
+                # From 2000 to 2017 there are 6210 days
+                self.assertAlmostEqual(quickstart_metrics["commits_per_week"], 142 / (6210 / 7), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_month"], 142 / (6210 / 30), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_year"], 142 / (6210 / 365), delta=0.1)
 
                 self.assertIn("SPDXRef-angular-seed", metrics["packages"])
                 self.assertEqual(
@@ -259,7 +276,10 @@ def test_to_date(self):
                 self.assertEqual(angular_metrics["developer_categories_core"], 16)
                 self.assertEqual(angular_metrics["developer_categories_regular"], 30)
                 self.assertEqual(angular_metrics["developer_categories_casual"], 10)
-                self.assertAlmostEqual(angular_metrics["commits_week_mean"], 0.0045088566827697265, delta=0.1)
+                # From 2000 to 2017 there are 6210 days
+                self.assertAlmostEqual(angular_metrics["commits_per_week"], 196 / (6210 / 7), delta=0.1)
+                self.assertAlmostEqual(angular_metrics["commits_per_month"], 196 / (6210 / 30), delta=0.1)
+                self.assertAlmostEqual(angular_metrics["commits_per_year"], 196 / (6210 / 365), delta=0.1)
 
     def test_duplicate_repo(self):
         """Check if it ignores duplicated URLs"""
@@ -282,6 +302,7 @@ def test_duplicate_repo(self):
                     "--output",
                     self.temp_file.name,
                     "--from-date=2000-01-01",
+                    "--to-date=2025-01-01",
                 ],
             )
             self.assertEqual(result.exit_code, 0)
@@ -315,7 +336,10 @@ def test_duplicate_repo(self):
                 self.assertEqual(quickstart_metrics["developer_categories_core"], 3)
                 self.assertEqual(quickstart_metrics["developer_categories_regular"], 13)
                 self.assertEqual(quickstart_metrics["developer_categories_casual"], 9)
-                self.assertAlmostEqual(quickstart_metrics["commits_week_mean"], 0.06418, delta=0.1)
+                # From 2000 to 2025 there are 9132 days
+                self.assertAlmostEqual(quickstart_metrics["commits_per_week"], 164 / (9132 / 7), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_month"], 164 / (9132 / 30), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_year"], 164 / (9132 / 365), delta=0.1)
 
     def test_non_git_repo(self):
         """Check if it flags non-git dependencies"""
@@ -338,6 +362,7 @@ def test_non_git_repo(self):
                     "--output",
                     self.temp_file.name,
                     "--from-date=2000-01-01",
+                    "--to-date=2025-01-01",
                 ],
             )
             self.assertEqual(result.exit_code, 0)
@@ -371,7 +396,10 @@ def test_non_git_repo(self):
                 self.assertEqual(quickstart_metrics["developer_categories_core"], 3)
                 self.assertEqual(quickstart_metrics["developer_categories_regular"], 13)
                 self.assertEqual(quickstart_metrics["developer_categories_casual"], 9)
-                self.assertAlmostEqual(quickstart_metrics["commits_week_mean"], 0.06418, delta=0.1)
+                # From 2000 to 2025 there are 9132 days
+                self.assertAlmostEqual(quickstart_metrics["commits_per_week"], 164 / (9132 / 7), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_month"], 164 / (9132 / 30), delta=0.1)
+                self.assertAlmostEqual(quickstart_metrics["commits_per_year"], 164 / (9132 / 365), delta=0.1)
 
                 self.assertIn("SPDXRef-sql-dk", metrics["packages"])
                 self.assertEqual(metrics["packages"]["SPDXRef-sql-dk"]["metrics"], None)

tests/unit/test_metrics.py

@@ -164,12 +164,23 @@ def test_message_size_metrics(self):
         self.assertAlmostEqual(metrics["mean"], 210.11, delta=0.1)
         self.assertEqual(metrics["median"], 229)
 
-    def test_get_commits_week_mean(self):
+    def test_get_commits_frequency(self):
         """Test whether the average (mean) commits per week is calculated correctly"""
 
         self.analyzer.process_events(self.events)
-        avg = self.analyzer.get_commits_week_mean(days_interval=30)
-        self.assertAlmostEqual(avg, 9 / 30 / 7, delta=0.1)
+        metrics = self.analyzer.get_commit_frequency_metrics(days_interval=30)
+        self.assertAlmostEqual(metrics["week"], 9 / (30 / 7), delta=0.1)
+        self.assertAlmostEqual(metrics["month"], 9, delta=0.1)
+        self.assertIsNone(metrics["year"])
+
+    def test_get_all_commits_frequency(self):
+        """Test whether the average (mean) commits per week and year is calculated correctly"""
+
+        self.analyzer.process_events(self.events)
+        metrics = self.analyzer.get_commit_frequency_metrics(days_interval=365)
+        self.assertAlmostEqual(metrics["week"], 9 / (365 / 7), delta=0.1)
+        self.assertAlmostEqual(metrics["month"], 9 / (365 / 30), delta=0.1)
+        self.assertAlmostEqual(metrics["year"], 9, delta=0.1)
 
     def test_get_developer_categories(self):
         """Test if the developer categories are calculated correctly"""

trustable_cli/cli.py

@@ -73,7 +73,8 @@
 @click.option(
     "--to-date",
     type=click.DateTime(formats=["%Y-%m-%d"]),
-    help="End date",
+    help="End date, by default today",
+    default=datetime.datetime.today().strftime("%Y-%m-%d"),
 )
 @click.option("--verify-certs", is_flag=True, default=False, help="Verify SSL/TLS certificates")
 @click.option("--verbose", is_flag=True, default=False, help="Increase output verbosity")

trustable_cli/metrics.py

@@ -147,13 +147,25 @@ def get_message_size_metrics(self):
         }
         return metrics
 
-    def get_commits_week_mean(self, days_interval: int):
+    def get_commit_frequency_metrics(self, days_interval: int):
         """
-        Get the average (mean) number of commits per week
+        Get the average (mean) number of commits per week, month and
+        year if the days of interval is greater than the metrics interval.
 
         :param days_interval: Interval of days to calculate the mean
         """
-        return self.total_commits / days_interval / 7
+        metrics = {"week": None, "month": None, "year": None}
+
+        if days_interval >= 7:
+            metrics["week"] = self.total_commits / (days_interval / 7)
+
+        if days_interval >= 30:
+            metrics["month"] = self.total_commits / (days_interval / 30)
+
+        if days_interval >= 365:
+            metrics["year"] = self.total_commits / (days_interval / 365)
+
+        return metrics
 
     def get_developer_categories(self):
         """Return the number of core, regular and casual developers"""
@@ -261,14 +273,14 @@ def get_repository_metrics(
         days = (to_date - from_date).days
     else:
         days = 365
-    metrics["metrics"]["commits_week_mean"] = analyzer.get_commits_week_mean(days)
 
     # Flatten two-level metrics
     metrics_to_flatten = {
         "file_types": analyzer.get_file_type_metrics(),
         "commit_size": analyzer.get_commit_size_metrics(),
         "message_size": analyzer.get_message_size_metrics(),
         "developer_categories": analyzer.get_developer_categories(),
+        "commits_per": analyzer.get_commit_frequency_metrics(days),
     }
 
     for prefix, metrics_set in metrics_to_flatten.items():