diff --git a/tasks/generate.py b/tasks/generate.py
index 89d5d3e4..a4cdf16d 100644
--- a/tasks/generate.py
+++ b/tasks/generate.py
@@ -1,8 +1,11 @@
 import base64
 import hashlib
+import io
 import json
 import os
 import os.path
+import re
+import zipfile
 
 import urllib.request
 
@@ -19,11 +22,14 @@ def _path(pyversion=None):
     return os.path.join(*filter(None, parts))
 
 
+def _template(name="default.py"):
+    return os.path.join(PROJECT_ROOT, "templates", name)
+
+
 @invoke.task
 def installer(ctx,
               pip_version=None, wheel_version=None, setuptools_version=None,
-              installer_path=_path(),
-              template_path=os.path.join(PROJECT_ROOT, "template.py")):
+              installer_path=_path(), template_path=_template()):
 
     print("[generate.installer] Generating installer {} (using {})".format(
         os.path.relpath(installer_path, PROJECT_ROOT),
@@ -60,6 +66,18 @@ def installer(ctx,
     data = urllib.request.urlopen(url).read()
     assert hashlib.md5(data).hexdigest() == expected_hash
 
+    # We need to repack the downloaded wheel file to remove the .dist-info,
+    # after this it will no longer be a valid wheel, but it will still work
+    # perfectly fine for our use cases.
+    new_data = io.BytesIO()
+    with zipfile.ZipFile(io.BytesIO(data)) as existing_zip:
+        with zipfile.ZipFile(new_data, mode="w") as new_zip:
+            for zinfo in existing_zip.infolist():
+                if re.search(r"pip-.+\.dist-info/", zinfo.filename):
+                    continue
+                new_zip.writestr(zinfo, existing_zip.read(zinfo))
+    data = new_data.getvalue()
+
     # Write out the wrapper script that will take the place of the zip script
     # The reason we need to do this instead of just directly executing the
     # zip script is that while Python will happily execute a zip script if
@@ -103,10 +121,12 @@ def installer(ctx,
     pre=[
         invoke.call(installer),
         invoke.call(installer, installer_path=_path("2.6"),
+                    template_path=_template("pre-10.py"),
                     pip_version="<10",
                     wheel_version="<0.30",
                     setuptools_version="<37"),
         invoke.call(installer, installer_path=_path("3.2"),
+                    template_path=_template("pre-10.py"),
                     pip_version="<8",
                     wheel_version="<0.30",
                     setuptools_version="<30"),
diff --git a/templates/default.py b/templates/default.py
new file mode 100644
index 00000000..9991ffcb
--- /dev/null
+++ b/templates/default.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+#
+# Hi There!
+# You may be wondering what this giant blob of binary data here is, you might
+# even be worried that we're up to something nefarious (good for you for being
+# paranoid!). This is a base85 encoding of a zip file, this zip file contains
+# an entire copy of pip (version {installed_version}).
+#
+# Pip is a thing that installs packages, pip itself is a package that someone
+# might want to install, especially if they're looking to run this get-pip.py
+# script. Pip has a lot of code to deal with the security of installing
+# packages, various edge cases on various platforms, and other such sort of
+# "tribal knowledge" that has been encoded in its code base. Because of this
+# we basically include an entire copy of pip inside this blob. We do this
+# because the alternatives are attempt to implement a "minipip" that probably
+# doesn't do things correctly and has weird edge cases, or compress pip itself
+# down into a single file.
+#
+# If you're wondering how this is created, it is using an invoke task located
+# in tasks/generate.py called "installer". It can be invoked by using
+# ``invoke generate.installer``.
+
+import os.path
+import pkgutil
+import shutil
+import sys
+import struct
+import tempfile
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    iterbytes = iter
+else:
+    def iterbytes(buf):
+        return (ord(byte) for byte in buf)
+
+try:
+    from base64 import b85decode
+except ImportError:
+    _b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                    b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{{|}}~")
+
+    def b85decode(b):
+        _b85dec = [None] * 256
+        for i, c in enumerate(iterbytes(_b85alphabet)):
+            _b85dec[c] = i
+
+        padding = (-len(b)) % 5
+        b = b + b'~' * padding
+        out = []
+        packI = struct.Struct('!I').pack
+        for i in range(0, len(b), 5):
+            chunk = b[i:i + 5]
+            acc = 0
+            try:
+                for c in iterbytes(chunk):
+                    acc = acc * 85 + _b85dec[c]
+            except TypeError:
+                for j, c in enumerate(iterbytes(chunk)):
+                    if _b85dec[c] is None:
+                        raise ValueError(
+                            'bad base85 character at position %d' % (i + j)
+                        )
+                raise
+            try:
+                out.append(packI(acc))
+            except struct.error:
+                raise ValueError('base85 overflow in hunk starting at byte %d'
+                                 % i)
+
+        result = b''.join(out)
+        if padding:
+            result = result[:-padding]
+        return result
+
+
+def bootstrap(tmpdir=None):
+    # Import pip so we can use it to install pip and maybe setuptools too
+    import pip._internal
+    from pip._internal.commands.install import InstallCommand
+    from pip._internal.req import InstallRequirement
+
+    # Wrapper to provide default certificate with the lowest priority
+    class CertInstallCommand(InstallCommand):
+        def parse_args(self, args):
+            # If cert isn't specified in config or environment, we provide our
+            # own certificate through defaults.
+            # This allows user to specify custom cert anywhere one likes:
+            # config, environment variable or argv.
+            if not self.parser.get_default_values().cert:
+                self.parser.defaults["cert"] = cert_path  # calculated below
+            return super(CertInstallCommand, self).parse_args(args)
+
+    pip._internal.commands_dict["install"] = CertInstallCommand
+
+    implicit_pip = True
+    implicit_setuptools = True
+    implicit_wheel = True
+
+    # Check if the user has requested us not to install setuptools
+    if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"):
+        args = [x for x in sys.argv[1:] if x != "--no-setuptools"]
+        implicit_setuptools = False
+    else:
+        args = sys.argv[1:]
+
+    # Check if the user has requested us not to install wheel
+    if "--no-wheel" in args or os.environ.get("PIP_NO_WHEEL"):
+        args = [x for x in args if x != "--no-wheel"]
+        implicit_wheel = False
+
+    # We only want to implicitly install setuptools and wheel if they don't
+    # already exist on the target platform.
+    if implicit_setuptools:
+        try:
+            import setuptools  # noqa
+            implicit_setuptools = False
+        except ImportError:
+            pass
+    if implicit_wheel:
+        try:
+            import wheel  # noqa
+            implicit_wheel = False
+        except ImportError:
+            pass
+
+    # We want to support people passing things like 'pip<8' to get-pip.py which
+    # will let them install a specific version. However because of the dreaded
+    # DoubleRequirement error if any of the args look like they might be a
+    # specific for one of our packages, then we'll turn off the implicit
+    # install of them.
+    for arg in args:
+        try:
+            req = InstallRequirement.from_line(arg)
+        except:
+            continue
+
+        if implicit_pip and req.name == "pip":
+            implicit_pip = False
+        elif implicit_setuptools and req.name == "setuptools":
+            implicit_setuptools = False
+        elif implicit_wheel and req.name == "wheel":
+            implicit_wheel = False
+
+    # Add any implicit installations to the end of our args
+    if implicit_pip:
+        args += ["pip{pip_version}"]
+    if implicit_setuptools:
+        args += ["setuptools{setuptools_version}"]
+    if implicit_wheel:
+        args += ["wheel{wheel_version}"]
+
+    # Add our default arguments
+    args = ["install", "--upgrade", "--force-reinstall"] + args
+
+    delete_tmpdir = False
+    try:
+        # Create a temporary directory to act as a working directory if we were
+        # not given one.
+        if tmpdir is None:
+            tmpdir = tempfile.mkdtemp()
+            delete_tmpdir = True
+
+        # We need to extract the SSL certificates from requests so that they
+        # can be passed to --cert
+        cert_path = os.path.join(tmpdir, "cacert.pem")
+        with open(cert_path, "wb") as cert:
+            cert.write(pkgutil.get_data("pip._vendor.certifi", "cacert.pem"))
+
+        # Execute the included pip and use it to install the latest pip and
+        # setuptools from PyPI
+        sys.exit(pip._internal.main(args))
+    finally:
+        # Remove our temporary directory
+        if delete_tmpdir and tmpdir:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def main():
+    tmpdir = None
+    try:
+        # Create a temporary working directory
+        tmpdir = tempfile.mkdtemp()
+
+        # Unpack the zipfile into the temporary directory
+        pip_zip = os.path.join(tmpdir, "pip.zip")
+        with open(pip_zip, "wb") as fp:
+            fp.write(b85decode(DATA.replace(b"\n", b"")))
+
+        # Add the zipfile to sys.path so that we can import it
+        sys.path.insert(0, pip_zip)
+
+        # Run the bootstrap
+        bootstrap(tmpdir=tmpdir)
+    finally:
+        # Clean up our temporary working directory
+        if tmpdir:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+DATA = b"""
+{zipfile}
+"""
+
+
+if __name__ == "__main__":
+    main()
diff --git a/template.py b/templates/pre-10.py
similarity index 100%
rename from template.py
rename to templates/pre-10.py