Production data on dev load automation (#1651) (#1691)

Author: daveoconnor

docker-compose.yml

@@ -1,7 +1,7 @@
 services:
 
   db:
-    image: postgres:12.0
+    image: postgres:16.0
     env_file: .env
     environment:
       - "POSTGRES_HOST_AUTH_METHOD=trust"

env.template

@@ -26,6 +26,12 @@ STATIC_CONTENT_BUCKET_NAME="stage.boost.org.v2"
 STATIC_CONTENT_REGION="us-east-2"
 STATIC_CONTENT_AWS_S3_ENDPOINT_URL="https://s3.us-east-2.amazonaws.com"
 
+PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID=$STATIC_CONTENT_AWS_ACCESS_KEY_ID
+PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY=$STATIC_CONTENT_AWS_SECRET_ACCESS_KEY
+PROD_MEDIA_CONTENT_BUCKET_NAME="boost.org.media"
+PROD_MEDIA_CONTENT_REGION=$STATIC_CONTENT_REGION
+PROD_MEDIA_CONTENT_AWS_S3_ENDPOINT_URL=$STATIC_CONTENT_AWS_S3_ENDPOINT_URL
+
 # Mailman database settings
 HYPERKITTY_DATABASE_NAME="hyperkitty_db"
 DATABASE_URL="postgresql://postgres@db:5432/postgres"
@@ -61,3 +67,7 @@ DEBUG_TOOLBAR=True
 # currently only pycharm is supported, vscode kept for example of alternative options
 # PYTHONBREAKPOINT=pycharm_debugger.set_trace
 # PYTHONBREAKPOINT=vscode_debugger.set_trace
+
+# for production database dumps on Google Cloud Storage
+PROD_DB_DUMP_URL=gs://boostbackups/db1/daily/
+PROD_DB_DUMP_FILE_WILDCARD=boost_production.db1*

justfile

@@ -127,5 +127,16 @@ alias shell := console
 @pip-compile-upgrade:  ## Upgrade existing Python dependencies to their latest versions
     just pip-compile --upgrade
 
+@load_production_data:  ## downloads and loads the latest production database dump
+    bash scripts/load_production_data.sh
+
+@dump_database:  ## dumps the current database to a .dump file in the project root
+    #!/usr/bin/env bash
+    DUMP_FILENAME="database_dump_$(date +"%Y-%m-%d-%H-%M-%S").dump"
+    echo "Dumping database to ${DUMP_FILENAME}..."
+    docker compose exec -T db pg_dump -U "$(grep PGDATABASE .env | cut -d= -f2)" -d "$(grep PGUSER .env | cut -d= -f2)" -F c -f "/tmp/${DUMP_FILENAME}"
+    docker compose cp "db:/tmp/${DUMP_FILENAME}" "./${DUMP_FILENAME}"
+    echo "Database dumped successfully to ${DUMP_FILENAME}"
+
 @manage args:
     docker compose run --rm web python manage.py {{ args }}

libraries/forms.py

@@ -195,7 +195,14 @@ def cache_html(self):
         # ensure we have "cleaned_data"
         if not self.is_valid():
             return ""
-        html = render_to_string(self.html_template_name, self.get_stats())
+        try:
+            html = render_to_string(self.html_template_name, self.get_stats())
+        except FileNotFoundError as e:
+            html = (
+                f"An error occurred generating the report: {e}. To see the image "
+                f"which is broken copy the error and run the report again. This "
+                f"error isn't shown on subsequent runs."
+            )
         self.cache_set(html)
         return html