Keep delivery failures longer than successfu deliveries

Author: dstufft

tests/unit/email/ses/test_tasks.py

@@ -13,7 +13,9 @@
 import datetime
 
 from warehouse.email.ses.models import EmailMessage
-from warehouse.email.ses.tasks import CLEANUP_AFTER, cleanup
+from warehouse.email.ses.tasks import (
+    CLEANUP_DELIVERED_AFTER, CLEANUP_AFTER, cleanup,
+)
 
 from ....common.db.ses import EmailMessageFactory
 
@@ -22,12 +24,26 @@ def test_cleanup_cleans_correctly(db_request):
     now = datetime.datetime.utcnow()
 
     EmailMessageFactory.create(
-        created=(now - CLEANUP_AFTER - datetime.timedelta(hours=1)),
+        status="Delivered",
+        created=(now - CLEANUP_DELIVERED_AFTER - datetime.timedelta(hours=1)),
     )
-    to_be_kept = EmailMessageFactory.create(
-        created=(now - CLEANUP_AFTER + datetime.timedelta(hours=1)),
+    EmailMessageFactory.create(
+        status="Bounced",
+        created=(now - CLEANUP_AFTER - datetime.timedelta(hours=1)),
     )
 
+    to_be_kept = [
+        EmailMessageFactory.create(
+            status="Delivered",
+            created=(now - CLEANUP_DELIVERED_AFTER +
+                     datetime.timedelta(hours=1)),
+        ),
+        EmailMessageFactory.create(
+            status="Bounced",
+            created=(now - CLEANUP_AFTER + datetime.timedelta(hours=1)),
+        ),
+    ]
+
     cleanup(db_request)
 
-    assert db_request.db.query(EmailMessage).all() == [to_be_kept]
+    assert set(db_request.db.query(EmailMessage).all()) == set(to_be_kept)

warehouse/email/ses/tasks.py

@@ -16,11 +16,25 @@
 from warehouse.email.ses.models import EmailMessage
 
 
-CLEANUP_AFTER = datetime.timedelta(days=14)
+CLEANUP_DELIVERED_AFTER = datetime.timedelta(days=14)
+
+CLEANUP_AFTER = datetime.timedelta(days=90)
 
 
 @tasks.task(ignore_result=True, acks_late=True)
 def cleanup(request):
+    # Cleanup any email message whose status is Accepted or Delivered.
+    # We clean these up quicker than failures because we don't really
+    # need them, and this limits the amount of data stored in the
+    # database.
+    (request.db.query(EmailMessage)
+               .filter(EmailMessage.status.in_(["Accepted", "Delivered"]))
+               .filter(EmailMessage.created <
+                       (datetime.datetime.utcnow() - CLEANUP_DELIVERED_AFTER))
+               .delete(synchronize_session="fetch"))
+
+    # Cleanup *all* messages, this is our hard limit after which we
+    # will not continue to save the email message data.
     (request.db.query(EmailMessage)
                .filter(EmailMessage.created <
                        (datetime.datetime.utcnow() - CLEANUP_AFTER))