Bug/17 scan keys unavailable (#19)

* move repository uri default location to central decouple config

* add exception for key metadata missing

* remove future deprecated scan keys call and replace with metadata from s3

* add decoupled and pin version of gnupg to best likely supported version

* pin version of gnupg in setup.py

* add decouple settings file

* update docs to include instructions to add key json

Author: andrewkrug

docs/architecture.rst

@@ -28,13 +28,21 @@ Metadata files have been introduced to remove s3 hosting as a requirement for re
 The new repository structure introduces several new requirements.
 
 1. **Optional** The public portion of the GPG used for signing modules and metadata should present at the root of the repository with the filename ``REPO_SIGNING_KEY.asc``.  If the signing key is not present the ``--gpg-no-verify`` flag must be used with Margarita Shotgun.
-2. **Required** A folder must exist in at the path ``/repodata`` which contains the following files.
+2. **Optional** A JSON file with key metadata including the key fingerprint. REPO_SIGNING_KEY.json.
+Example:
+```
+{
+"uids": ["Lime Signing Key (Threat Response Official Lime Signing Key) <security@threatresponse.cloud>"],
+"fingerprint": "80DA92CB09161F241C8F9BC918BA980367172B17"
+}
+```
+3. **Required** A folder must exist in at the path ``/repodata`` which contains the following files.
 
    1. **Required** ``repomd.xml`` contains repository metadata including one or more manifests of kernel modules.
    2. **Optional** ``repomd.xml.sig`` detached signature for ``repomd.xml``.  If not present in the repository the ``--gpg-no-verify`` flag must be used with Margarita Shotgun.
    3. **Optional** Manifest files.  Techinally manifests can be stored at any relative path but it is recommended that they be stored in the ``repodata`` directory.
 
-3. **Optional** A ``modules`` directory is recommended which will contain the following files.  Note the following files can have any location relative to the repository root, the ``modules`` directory is simply best practice.
+4. **Optional** A ``modules`` directory is recommended which will contain the following files.  Note the following files can have any location relative to the repository root, the ``modules`` directory is simply best practice.
 
    1. **Required** Compiled lime kernel modules.  Module filenames are arbitrary as the files are explicitly listed in a manifest
    2. **Optional** Detached kernel module signatures.  If signatures are not present the ``--gpg-no-verify`` flag must be used with Margarita Shotgun.  The signature filename is arbitary as it is explicitly listed in a manifest.
@@ -44,6 +52,7 @@ Below is an example directory listing of the repository structure.
 .. code-block:: text
 
     ├── REPO_SIGNING_KEY.asc
+    ├── REPO_SIGNING_KEY.json
     ├── modules
     │   ├── lime-2.6.32-131.0.15.el6.centos.plus.x86_64.ko
     │   ├── lime-2.6.32-131.0.15.el6.centos.plus.x86_64.ko.sig

margaritashotgun/cli.py

@@ -6,7 +6,7 @@
 from yaml import YAMLError
 from margaritashotgun import __version__
 from margaritashotgun.exceptions import *
-
+from margaritashotgun.settings import *
 logger = logging.getLogger(__name__)
 
 default_allowed_keys = ["aws", "hosts", "workers", "logging", "repository"]
@@ -28,7 +28,7 @@
                       "prefix": None},
                   "repository": {
                       "enabled": False,
-                      "url": "https://threatresponse-lime-modules.s3.amazonaws.com/",
+                      "url": REPOSITORY_BUCKET_URI,
                       "gpg_verify": True,
                       "manifest": "primary"
                   }}

margaritashotgun/exceptions.py

@@ -88,6 +88,18 @@ def __init__(self, url):
         )
         MargaritaShotgunError.__init__(self, msg)
 
+class RepositoryMissingKeyMetadataError(MargaritaShotgunError):
+    """
+    Raised when signing public key is missing from repository
+    """
+    def __init__(self, url):
+        msg = (
+            "Repository missing a key metadata file with signature "
+            "at {0}, contact the repository maintainer or disable "
+            "gpg verification with --gpg-no-verify".format(url)
+        )
+        MargaritaShotgunError.__init__(self, msg)
+
 class RepositoryMissingSignatureError(MargaritaShotgunError):
     """
     Raised when a detached signature is missing in remote repository"

margaritashotgun/repository.py

@@ -1,15 +1,18 @@
 import gnupg
 import gzip
 import hashlib
+import json
 import logging
 import os
 import requests
 import time
 import xmltodict
+
 from datetime import datetime
 from io import BytesIO
 from margaritashotgun.exceptions import *
 from prompt_toolkit import prompt
+from margaritashotgun.settings import *
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +34,8 @@ def __init__(self, url, gpg_verify):
         self.gpg_verify = gpg_verify
         self.metadata_dir = 'repodata'
         self.metadata_file = 'repomd.xml'
-        self.repo_signing_key = 'REPO_SIGNING_KEY.asc'
+        self.repo_signing_key = PUBLIC_SIGNING_KEY_FILE
+        self.key_metadata = PUBLIC_SIGNING_KEY_METADATA
 
         self.gpg = None
         self.key_path = None
@@ -52,23 +56,43 @@ def init_gpg(self):
 
     def get_signing_key(self):
         """
-        Download a local copy of repo signing key and generate key metadata
+        Download a local copy of repo signing key for installation
+        """
+
+        """
+        Download a local copy of repo signing key key metadata.
+        Fixes #17 Scan Keys no available in all GPG versions.
         """
-        tmp_path = "/tmp/{0}".format(self.repo_signing_key)
+        tmp_key_path = "/tmp/{0}".format(self.repo_signing_key)
+        tmp_metadata_path = "/tmp/{0}".format(self.key_metadata)
+
         repo_key_path = "{0}/{1}".format(self.url, self.repo_signing_key)
-        req = requests.get(repo_key_path)
-        if req.status_code is 200:
+        repo_metadata_path = "{0}/{1}".format(self.url, self.key_metadata)
+
+
+        req_key = requests.get(repo_key_path)
+        req_metadata = requests.get(repo_metadata_path)
+
+        # Fetch the key to disk
+        if req_key.status_code is 200:
             logger.debug(("found repository signing key at "
-                         "{0}".format(repo_key_path)))
-            self.raw_key = req.content
-            with open(tmp_path, 'wb') as f:
+                          "{0}".format(repo_key_path)))
+            self.raw_key = req_key.content
+            with open(tmp_key_path, 'wb') as f:
                 f.write(self.raw_key)
-            key_info = self.gpg.scan_keys(tmp_path)
-            # we only scan one key, return a single key_info
-            key_info = key_info[0]
         else:
             raise RepositoryMissingSigningKeyError(repo_key_path)
-        return (tmp_path, key_info)
+
+        # Fetch the fingerprint from the metadata
+        if req_metadata.status_code is 200:
+            logger.debug(("found key metadata at "
+                          "{0}".format(repo_metadata_path)))
+            print(req_metadata.content)
+            key_info = json.loads(req_metadata.content.decode('utf-8'))
+        else:
+            RepositoryMissingKeyMetadataError
+
+        return (tmp_key_path, key_info)
 
     def check_signing_key(self):
         """

margaritashotgun/settings.py

@@ -0,0 +1,5 @@
+from decouple import config
+
+REPOSITORY_BUCKET_URI = 'https://threatresponse-lime-modules.s3.amazonaws.com/'
+PUBLIC_SIGNING_KEY_FILE = 'REPO_SIGNING_KEY.asc'
+PUBLIC_SIGNING_KEY_METADATA = 'REPO_SIGNING_KEY.json'

requirements.txt

@@ -10,5 +10,6 @@ enum34
 requests
 xmltodict
 logutils
-python-gnupg
+python-gnupg==0.3.9
 prompt_toolkit
+python-decouple

setup.py

@@ -20,6 +20,7 @@ def run(self):
       download_url="https://github.com/ThreatResponse/margaritashotgun/archive/v0.4.0.tar.gz",
       use_2to3=True,
       install_requires=['boto3>=1.3.0',
+                        'python-decouple',
                         'paramiko>=1.16.0',
                         'pyyaml>=3.11',
                         's3fs>=0.0.2',
@@ -28,7 +29,7 @@ def run(self):
                         'requests',
                         'xmltodict',
                         'logutils==0.3.3',
-                        'python-gnupg',
+                        'python-gnupg==0.3.9',
                         'prompt_toolkit'],
       tests_require=['pytest',
                      'pytest-cov',