update generate_pep_pages to consume results of python/peps#898

Author: ewdurbin

peps/converters.py

@@ -1,4 +1,5 @@
 import functools
+import datetime
 import re
 import os
 
@@ -7,47 +8,41 @@
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.core.files import File
+from django.db.models import Max
 
 from pages.models import Page, Image
 
 PEP_TEMPLATE = 'pages/pep-page.html'
 pep_url = lambda num: 'dev/peps/pep-{}/'.format(num)
 
 
-def check_paths(func):
-    """Ensure that our PEP_REPO_PATH is setup correctly."""
-    @functools.wraps(func)
-    def wrapped(*args, **kwargs):
-        if not hasattr(settings, 'PEP_REPO_PATH'):
-            raise ImproperlyConfigured('No PEP_REPO_PATH in settings')
-        if not os.path.exists(settings.PEP_REPO_PATH):
-            raise ImproperlyConfigured('Path set as PEP_REPO_PATH does not exist')
-        return func(*args, **kwargs)
-    return wrapped
+def get_peps_last_updated():
+    return Page.objects.filter(
+        path__startswith='dev/peps',
+    ).aggregate(Max('updated'))['updated__max']
 
 
-@check_paths
-def convert_pep0():
+def convert_pep0(artifact_path):
     """
     Take existing generated pep-0000.html and convert to something suitable
     for a Python.org Page returns the core body HTML necessary only
     """
-    pep0_path = os.path.join(settings.PEP_REPO_PATH, 'pep-0000.html')
+    pep0_path = os.path.join(artifact_path, 'pep-0000.html')
     pep0_content = open(pep0_path).read()
     data = convert_pep_page(0, pep0_content)
     if data is None:
         return
     return data['content']
 
 
-def get_pep0_page(commit=True):
+def get_pep0_page(artifact_path, commit=True):
     """
     Using convert_pep0 above, create a CMS ready pep0 page and return it
 
     pep0 is used as the directory index, but it's also an actual pep, so we
     return both Page objects.
     """
-    pep0_content = convert_pep0()
+    pep0_content = convert_pep0(artifact_path)
     if pep0_content is None:
         return None, None
     pep0_page, _ = Page.objects.get_or_create(path='dev/peps/')
@@ -88,7 +83,6 @@ def fix_headers(soup, data):
     return soup, data
 
 
-@check_paths
 def convert_pep_page(pep_number, content):
     """
     Handle different formats that pep2html.py outputs
@@ -163,12 +157,12 @@ def convert_pep_page(pep_number, content):
     return data
 
 
-def get_pep_page(pep_number, commit=True):
+def get_pep_page(artifact_path, pep_number, commit=True):
     """
     Given a pep_number retrieve original PEP source text, rst, or html.
     Get or create the associated Page and return it
     """
-    pep_path = os.path.join(settings.PEP_REPO_PATH, 'pep-{}.html'.format(pep_number))
+    pep_path = os.path.join(artifact_path, 'pep-{}.html'.format(pep_number))
     if not os.path.exists(pep_path):
         print("PEP Path '{}' does not exist, skipping".format(pep_path))
         return
@@ -177,7 +171,7 @@ def get_pep_page(pep_number, commit=True):
     if pep_content is None:
         return None
     pep_rst_source = os.path.join(
-        settings.PEP_REPO_PATH, 'pep-{}.rst'.format(pep_number),
+        artifact_path, 'pep-{}.rst'.format(pep_number),
     )
     pep_ext = '.rst' if os.path.exists(pep_rst_source) else '.txt'
     source_link = 'https://github.com/python/peps/blob/master/pep-{}{}'.format(
@@ -198,8 +192,8 @@ def get_pep_page(pep_number, commit=True):
     return pep_page
 
 
-def add_pep_image(pep_number, path):
-    image_path = os.path.join(settings.PEP_REPO_PATH, path)
+def add_pep_image(artifact_path, pep_number, path):
+    image_path = os.path.join(artifact_path, path)
     if not os.path.exists(image_path):
         print("Image Path '{}' does not exist, skipping".format(image_path))
         return
@@ -251,9 +245,8 @@ def add_pep_image(pep_number, path):
     return image
 
 
-@check_paths
-def get_peps_rss():
-    rss_feed = os.path.join(settings.PEP_REPO_PATH, 'peps.rss')
+def get_peps_rss(artifact_path):
+    rss_feed = os.path.join(artifact_path, 'peps.rss')
     if not os.path.exists(rss_feed):
         return
 

peps/management/commands/generate_pep_pages.py

@@ -1,11 +1,19 @@
 import re
 import os
 
+from contextlib import ExitStack
+from tarfile import TarFile
+from tempfile import TemporaryDirectory, TemporaryFile
+
+import requests
+
 from django.core.management import BaseCommand
 from django.conf import settings
 
+from dateutil.parser import parse as parsedate
+
 from peps.converters import (
-    get_pep0_page, get_pep_page, add_pep_image, get_peps_rss
+    get_pep0_page, get_pep_page, add_pep_image, get_peps_rss, get_peps_last_updated
 )
 
 pep_number_re = re.compile(r'pep-(\d+)')
@@ -42,60 +50,82 @@ def verbose(msg):
 
         verbose("== Starting PEP page generation")
 
-        verbose("Generating RSS Feed")
-        peps_rss = get_peps_rss()
-        if not peps_rss:
-            verbose("Could not find generated RSS feed. Skipping.")
-
-        verbose("Generating PEP0 index page")
-        pep0_page, _ = get_pep0_page()
-        if pep0_page is None:
-            verbose("HTML version of PEP 0 cannot be generated.")
-            return
-
-        image_paths = set()
-
-        # Find pep pages
-        for f in os.listdir(settings.PEP_REPO_PATH):
-
-            if self.is_image(f):
-                verbose("- Deferring import of image '{}'".format(f))
-                image_paths.add(f)
-                continue
-
-            # Skip files we aren't looking for
-            if not self.is_pep_page(f):
-                verbose("- Skipping non-PEP file '{}'".format(f))
-                continue
-
-            if 'pep-0000.html' in f:
-                verbose("- Skipping duplicate PEP0 index")
-                continue
-
-            verbose("Generating PEP Page from '{}'".format(f))
-            pep_match = pep_number_re.match(f)
-            if pep_match:
-                pep_number = pep_match.groups(1)[0]
-                p = get_pep_page(pep_number)
-                if p is None:
-                    verbose(
-                        "- HTML version PEP {!r} cannot be generated.".format(
-                            pep_number
+        with ExitStack() as stack:
+            verbose(f"== Fetching PEP artifact from {settings.PEP_ARTIFACT_URL}")
+            peps_last_updated = get_peps_last_updated()
+            with requests.get(settings.PEP_ARTIFACT_URL, stream=True) as r:
+                artifact_last_modified = parsedate(r.headers['last-modified'])
+                if peps_last_updated > artifact_last_modified:
+                    verbose(f"== No update to artifacts, we're done here!")
+                    return
+
+                temp_file = stack.enter_context(TemporaryFile())
+                for chunk in r.iter_content(chunk_size=8192):
+                    if chunk:
+                        temp_file.write(chunk)
+
+            temp_file.seek(0)
+
+            temp_dir = stack.enter_context(TemporaryDirectory())
+            tar_ball = stack.enter_context(TarFile.open(fileobj=temp_file, mode='r:gz'))
+            tar_ball.extractall(path=temp_dir, numeric_owner=False)
+
+            artifacts_path = os.path.join(temp_dir, 'peps')
+
+            verbose("Generating RSS Feed")
+            peps_rss = get_peps_rss(artifacts_path)
+            if not peps_rss:
+                verbose("Could not find generated RSS feed. Skipping.")
+
+            verbose("Generating PEP0 index page")
+            pep0_page, _ = get_pep0_page(artifacts_path)
+            if pep0_page is None:
+                verbose("HTML version of PEP 0 cannot be generated.")
+                return
+
+            image_paths = set()
+
+            # Find pep pages
+            for f in os.listdir(artifacts_path):
+
+                if self.is_image(f):
+                    verbose("- Deferring import of image '{}'".format(f))
+                    image_paths.add(f)
+                    continue
+
+                # Skip files we aren't looking for
+                if not self.is_pep_page(f):
+                    verbose("- Skipping non-PEP file '{}'".format(f))
+                    continue
+
+                if 'pep-0000.html' in f:
+                    verbose("- Skipping duplicate PEP0 index")
+                    continue
+
+                verbose("Generating PEP Page from '{}'".format(f))
+                pep_match = pep_number_re.match(f)
+                if pep_match:
+                    pep_number = pep_match.groups(1)[0]
+                    p = get_pep_page(artifacts_path, pep_number)
+                    if p is None:
+                        verbose(
+                            "- HTML version PEP {!r} cannot be generated.".format(
+                                pep_number
+                            )
                         )
-                    )
-                verbose("====== Title: '{}'".format(p.title))
-            else:
-                verbose("- Skipping invalid '{}'".format(f))
-
-        # Find pep images. This needs to happen afterwards, because we need
-        for img in image_paths:
-            pep_match = pep_number_re.match(img)
-            if pep_match:
-                pep_number = pep_match.groups(1)[0]
-                verbose("Generating image for PEP {} at '{}'".format(
-                    pep_number, img))
-                add_pep_image(pep_number, img)
-            else:
-                verbose("- Skipping non-PEP related image '{}'".format(img))
+                    verbose("====== Title: '{}'".format(p.title))
+                else:
+                    verbose("- Skipping invalid '{}'".format(f))
+
+            # Find pep images. This needs to happen afterwards, because we need
+            for img in image_paths:
+                pep_match = pep_number_re.match(img)
+                if pep_match:
+                    pep_number = pep_match.groups(1)[0]
+                    verbose("Generating image for PEP {} at '{}'".format(
+                        pep_number, img))
+                    add_pep_image(artifacts_path, pep_number, img)
+                else:
+                    verbose("- Skipping non-PEP related image '{}'".format(img))
 
         verbose("== Finished")

pydotorg/settings/base.py

@@ -227,7 +227,7 @@
 MAILING_LIST_PSF_MEMBERS = "[email protected]"
 
 ### PEP Repo Location
-PEP_REPO_PATH = ''
+PEP_ARTIFACT_URL = 'https://pythondotorg-assets-staging.s3.amazonaws.com/peps.tar.gz'
 
 ### Fastly ###
 FASTLY_API_KEY = False  # Set to Fastly API key in production to allow pages to

pydotorg/settings/local.py

@@ -24,9 +24,8 @@
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
-# Set the path to where the PEP repo's HTML source files are located
-# For example, PEP_REPO_PATH = '/Users/frank/work/src/pythondotorg/tmp/peps'
-PEP_REPO_PATH = ''
+# Set the URL to where to fetch PEP artifacts from
+PEP_ARTIFACT_URL = 'https://pythondotorg-assets-staging.s3.amazonaws.com/peps.tar.gz'
 
 # Use Dummy SASS compiler to avoid performance issues and remove the need to
 # have a sass compiler installed at all during local development if you aren't