Remove old overall data & ingest new one

backend/code_coverage_backend/gcp.py

@@ -5,7 +5,9 @@
 import re
 import tempfile
 from datetime import datetime
+from datetime import timedelta
 
+import pytz
 import redis
 import structlog
 import zstandard as zstd
@@ -368,7 +370,7 @@ def get_suites(self, repository):
         suites = self.redis.smembers(KEY_SUITES.format(repository=repository))
         return sorted(map(lambda x: x.decode("utf-8"), suites))
 
-    def ingest_available_reports(self, repository):
+    def ingest_available_reports(self, repository, until=None):
         """
         Ingest all the available reports for a repository
         """
@@ -377,8 +379,13 @@ def ingest_available_reports(self, repository):
         REGEX_BLOB = re.compile(
             r"^{}/(\w+)/([\w\-]+):([\w\-]+).json.zstd$".format(repository)
         )
+        now = datetime.utcnow().replace(tzinfo=pytz.UTC)
         for blob in self.bucket.list_blobs(prefix=repository):
 
+            if isinstance(until, timedelta) and (now - blob.time_created) >= until:
+                logger.debug(f"Skipping old blob {blob}")
+                continue
+
             # Get changeset from blob name
             match = REGEX_BLOB.match(blob.name)
             if match is None:

backend/tools/cleanup.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import redis
+
+
+def cleanup(client, prefix):
+    nb, memory = 0, 0
+    for key in client.keys(f"{prefix}:*"):
+        if key.endswith(b"all:all"):
+            continue
+
+        key_memory = client.memory_usage(key)
+        nb += 1
+        memory += key_memory
+        print(f"Removing {key_memory}b for {key}")
+
+        client.delete(key)
+
+    print(f"Removed {nb} keys for {memory} bytes")
+
+
+if __name__ == "__main__":
+    client = redis.from_url(os.environ["REDIS_URL"])
+    cleanup(client, "overall:mozilla-central")