Cleaning up

* Moving appengine config files into tests/resources
* Fixing our GAE downloader to match oauth2client & urllib3's.

Author: Jon Wayne Parrott

.travis.yml

@@ -4,41 +4,24 @@ language: python
 
 cache:
   directories:
-        - $HOME/gcloud/
+  - $HOME/.cache
+
 env:
-        - PATH=$PATH:$HOME/gcloud/google-cloud-sdk/bin GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/python-docs-samples.json GAE_PYTHONPATH=${HOME}/gcloud/google-cloud-sdk/platform/google_appengine TEST_BUCKET_NAME=bigquery-devrel-samples-bucket TEST_PROJECT_ID=bigquery-devrel-samples #Other environment variables on same line
+  global:
+  - PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin
+  - GOOGLE_APPLICATION_CREDENTIALS=${TRAVIS_BUILD_DIR}/python-docs-samples.json
+  - GAE_PYTHONPATH=${HOME}/.cache/google_appengine
+  - TEST_BUCKET_NAME=bigquery-devrel-samples-bucket
+  - TEST_PROJECT_ID=bigquery-devrel-samples
 
 before_install:
-        #ENCRYPT YOUR PRIVATE KEY (If you need authentication)
-        # 1. Install and login to the Travis CLI:
-        #       $ gem install travis
-        #       $ travis login
-        # 2. Move your json private key to client_secrets.json
-        # 3. Run:
-        #       $ travis encrypt-file client_secrets.json --add
-        # 4. Commit changes:
-        #       $ git add client_secrets.json.enc
-        #       $ git commit client_secrets.json.enc .travis.yml
-
-        - if [ ! -d $HOME/gcloud/google-cloud-sdk ]; then
-                mkdir -p $HOME/gcloud &&
-                wget https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz --directory-prefix=$HOME/gcloud &&
-                cd $HOME/gcloud &&
-                tar xzf google-cloud-sdk.tar.gz &&
-                printf '\ny\n\ny\ny\n' | ./google-cloud-sdk/install.sh &&
-                cd $TRAVIS_BUILD_DIR;
-          fi
-        - gcloud -q components update app-engine-python
-        - openssl aes-256-cbc -K $encrypted_4fda24e244ca_key -iv $encrypted_4fda24e244ca_iv -in python-docs-samples.json.enc -out python-docs-samples.json -d
-        - if [ -a python-docs-samples.json ]; then
-                gcloud auth activate-service-account --key-file python-docs-samples.json;
-          fi
+- tests/scripts/travis-before-install.sh
 
 install:
-        - pip install tox coveralls
+- pip install tox coveralls
 
 script:
-        - tox
+- tox
 
 after_success:
-    coveralls
+- coveralls

appengine/localtesting/test_task_queue.py

@@ -28,7 +28,7 @@ def setUp(self):
 
         # root_path must be set the the location of queue.yaml.
         # Otherwise, only the 'default' queue will be available.
-        self.testbed.init_taskqueue_stub(root_path='.')
+        self.testbed.init_taskqueue_stub(root_path='tests/resources')
         self.taskqueue_stub = self.testbed.get_stub(
             testbed.TASKQUEUE_SERVICE_NAME)
 

app.yaml renamed to tests/resources/app.yaml

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Retrieved from https://github.com/Google/oauth2client
+"""Fetch the most recent GAE SDK and decompress it in the current directory.
+
+Usage:
+    fetch_gae_sdk.py [<dest_dir>]
+
+Current releases are listed here:
+    https://www.googleapis.com/storage/v1/b/appengine-sdks/o?prefix=featured
+"""
+
+import json
+import os
+import StringIO
+import sys
+import urllib2
+import zipfile
+
+_SDK_URL = (
+    'https://www.googleapis.com/storage/v1/b/appengine-sdks/o?prefix=featured')
+
+
+def get_gae_versions():
+    try:
+        version_info_json = urllib2.urlopen(_SDK_URL).read()
+    except:
+        return {}
+    try:
+        version_info = json.loads(version_info_json)
+    except:
+        return {}
+    return version_info.get('items', {})
+
+
+def _version_tuple(v):
+    version_string = os.path.splitext(v['name'])[0].rpartition('_')[2]
+    return tuple(int(x) for x in version_string.split('.'))
+
+
+def get_sdk_urls(sdk_versions):
+    python_releases = [
+        v for v in sdk_versions
+        if v['name'].startswith('featured/google_appengine')]
+    current_releases = sorted(
+        python_releases, key=_version_tuple, reverse=True)
+    return [release['mediaLink'] for release in current_releases]
+
+
+def main(argv):
+    if len(argv) > 2:
+        print('Usage: {} [<destination_dir>]'.format(argv[0]))
+        return 1
+    dest_dir = argv[1] if len(argv) > 1 else '.'
+    if not os.path.exists(dest_dir):
+        os.makedirs(dest_dir)
+
+    if os.path.exists(os.path.join(dest_dir, 'google_appengine')):
+        print('GAE SDK already installed at {}, exiting.'.format(dest_dir))
+        return 0
+
+    sdk_versions = get_gae_versions()
+    if not sdk_versions:
+        print('Error fetching GAE SDK version info')
+        return 1
+    sdk_urls = get_sdk_urls(sdk_versions)
+    for sdk_url in sdk_urls:
+        try:
+            sdk_contents = StringIO.StringIO(urllib2.urlopen(sdk_url).read())
+            break
+        except:
+            pass
+    else:
+        print('Could not read SDK from any of {}'.format(sdk_urls))
+        return 1
+    sdk_contents.seek(0)
+    try:
+        zip_contents = zipfile.ZipFile(sdk_contents)
+        zip_contents.extractall(dest_dir)
+        print('GAE SDK Installed to {}.'.format(dest_dir))
+    except:
+        print('Error extracting SDK contents')
+        return 1
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[:]))

queue.yaml renamed to tests/resources/queue.yaml

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -ev
+
+# Install Google App Engine Python SDK
+if [[ ! -d "${GAE_PYTHONPATH}" ]]; then
+  python tests/scripts/fetch_gae_sdk.py `dirname "${GAE_PYTHONPATH}"`
+fi
+
+# Google Cloud Service account key.
+# ENCRYPT YOUR PRIVATE KEY (If you need authentication)
+# 1. Install and login to the Travis CLI:
+#       $ gem install travis
+#       $ travis login
+# 2. Move your json private key to client_secrets.json
+# 3. Run:
+#       $ travis encrypt-file client_secrets.json --add
+# 4. Commit changes:
+#       $ git add client_secrets.json.enc
+#       $ git commit client_secrets.json.enc .travis.yml
+openssl aes-256-cbc \
+    -K $encrypted_4fda24e244ca_key \
+    -iv $encrypted_4fda24e244ca_iv \
+    -in ${TRAVIS_BUILD_DIR}/python-docs-samples.json.enc \
+    -out ${TRAVIS_BUILD_DIR}/python-docs-samples.json -d

tests/scripts/fetch_gae_sdk.py

@@ -25,6 +25,7 @@ deps =
     nosegae
 commands = 
     nosetests --with-gae \
+        --gae-app=tests/resources/app.yaml \
         --logging-level=INFO \
         {[testenv]coverargs} \
         {posargs:appengine}