diff --git a/.gitignore b/.gitignore index 5dc43a8..d07d90b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ - .DS_Store extras/ +*_garmin_connect_export/ +*.pyc diff --git a/README.md b/README.md index 689cd72..7b4a3c7 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Usage You will need a little experience running things from the command line to use this script. That said, here are the usage details from the `--help` flag: ``` -usage: gcexport.py [-h] [--version] [--username [USERNAME]] +usage: gcexport3.py [-h] [--version] [--username [USERNAME]] [--password [PASSWORD]] [-c [COUNT]] [-f [{gpx,tcx,original}]] [-d [DIRECTORY]] [-u] @@ -43,11 +43,11 @@ optional arguments: ``` Examples: -`python gcexport.py --count all` will download all of your data to a dated directory. +`python gcexport3.py --count all` will download all of your data to a dated directory. -`python gcexport.py -d ~/MyActivities -c 3 -f original -u --username bobbyjoe --password bestpasswordever1` will download your three most recent activities in the FIT file format (or whatever they were uploaded as) into the `~/MyActivities` directory (unless they already exist). Using the `--username` and `--password` flags are not recommended because your password will be stored in your command line history. Instead, omit them to be prompted (and note that nothing will be displayed when you type your password). +`python gcexport3.py -d ~/MyActivities -c 3 -f original -u --username bobbyjoe --password bestpasswordever1` will download your three most recent activities in the FIT file format (or whatever they were uploaded as) into the `~/MyActivities` directory (unless they already exist). Using the `--username` and `--password` flags are not recommended because your password will be stored in your command line history. Instead, omit them to be prompted (and note that nothing will be displayed when you type your password). -Alternatively, you may run it with `./gcexport.py` if you set the file as executable (i.e., `chmod u+x gcexport.py`). +Alternatively, you may run it with `./gcexport3.py` if you set the file as executable (i.e., `chmod u+x gcexport3.py`). Of course, you must have Python installed to run this. Most Mac and Linux users should already have it. Also, as stated above, you should have some basic command line experience. @@ -63,9 +63,25 @@ Garmin Connect API ------------------ This script is for personal use only. It simulates a standard user session (i.e., in the browser), logging in using cookies and an authorization ticket. This makes the script pretty brittle. If you're looking for a more reliable option, particularly if you wish to use this for some production service, Garmin does offer a paid API service. +### REST endpoints + +As this script doesn't use the paid API, the endpoints to use are known by reverse engineering browser sessions. And as the Garmin Connect website changes over time, chances are that this script gets broken. + +Small history of the endpoint used by `gcexport3.py` to get a list of activities: + +- [activity-search-service-1.0](https://connect.garmin.com/proxy/activity-search-service-1.0/json/activities): initial endpoint used since 2015, worked at least until January 2018 +- [activity-search-service-1.2](https://connect.garmin.com/proxy/activity-search-service-1.2/json/activities): endpoint introduced in `gcexport.py` in August 2016. In March 2018 this still works, but doesn't allow you to fetch more than 20 activities, even split over multiple calls (when doing three consecutive calls with 1,19,19 as `limit` parameter, the third one fails with HTTP error 500). The JSON returned by this endpoint however is quite rich (see example in the `json` folder). +- [activitylist-service](https://connect.garmin.com/modern/proxy/activitylist-service/activities/search/activities): endpoint introduced in `gcexport.py` in March 2018. The JSON returned by this endpoint is very different from the activity-search-service-1.2 one (also here see the example in the `json` folder), e.g. + - it is concise and offers no redundant information (e.g. only speed, not speed and pace) + - the units are not explicitly given and must be deducted (e.g. the speed unit is m/s) + - there is less information, e.g. there is only one set of elevation values (not both corrected and uncorrected), and other values like minimum heart rate are missing. + - some other information is available only as an ID (e.g. `timeZoneId` or `deviceId`), and complete information might be available by another REST call (I didn't reverse further for the time being) + History ------- -The original project was written in PHP (now in the `old` directory), based on "Garmin Connect export to Dailymile" code at http://www.ciscomonkey.net/gc-to-dm-export/ (link has been down for a while). It no longer works due to the way Garmin handles logins. It could be updated, but I decided to rewrite everything in Python for the latest version. +The original project was written in PHP (formerly in the `old` directory, now deleted), based on "Garmin Connect export to Dailymile" code at http://www.ciscomonkey.net/gc-to-dm-export/ (link has been down for a while). It no longer works due to the way Garmin handles logins. It could be updated, but I decided to rewrite everything in Python for the latest version. + +@moderation forked the original from @kjkjava when the various endpoints stopped working and the original repo wasn't been updated. This fork is primarily designed for my use which is cycling. It has not well been tested against other activity types. In the latest updates (April 2018) I've deprecated the Python 2 version (renamed to gcexport2.py) and this script now requires Python 3. The code has been linted using [pylint3](https://packages.debian.org/sid/pylint3). Contributions ------------- diff --git a/gcexport.py b/gcexport.py deleted file mode 100755 index 2c6fec7..0000000 --- a/gcexport.py +++ /dev/null @@ -1,316 +0,0 @@ -#!/usr/bin/python - -""" -File: gcexport.py -Author: Kyle Krafka (https://github.com/kjkjava/) -Date: April 28, 2015 - -Description: Use this script to export your fitness data from Garmin Connect. - See README.md for more information. -""" - -from urllib import urlencode -from datetime import datetime -from getpass import getpass -from sys import argv -from os.path import isdir -from os.path import isfile -from os import mkdir -from os import remove -from xml.dom.minidom import parseString - -import urllib2, cookielib, json -from fileinput import filename - -import argparse -import zipfile - -script_version = '1.0.0' -current_date = datetime.now().strftime('%Y-%m-%d') -activities_directory = './' + current_date + '_garmin_connect_export' - -parser = argparse.ArgumentParser() - -# TODO: Implement verbose and/or quiet options. -# parser.add_argument('-v', '--verbose', help="increase output verbosity", action="store_true") -parser.add_argument('--version', help="print version and exit", action="store_true") -parser.add_argument('--username', help="your Garmin Connect username (otherwise, you will be prompted)", nargs='?') -parser.add_argument('--password', help="your Garmin Connect password (otherwise, you will be prompted)", nargs='?') - -parser.add_argument('-c', '--count', nargs='?', default="1", - help="number of recent activities to download, or 'all' (default: 1)") - -parser.add_argument('-f', '--format', nargs='?', choices=['gpx', 'tcx', 'original'], default="gpx", - help="export format; can be 'gpx', 'tcx', or 'original' (default: 'gpx')") - -parser.add_argument('-d', '--directory', nargs='?', default=activities_directory, - help="the directory to export to (default: './YYYY-MM-DD_garmin_connect_export')") - -parser.add_argument('-u', '--unzip', - help="if downloading ZIP files (format: 'original'), unzip the file and removes the ZIP file", - action="store_true") - -args = parser.parse_args() - -if args.version: - print argv[0] + ", version " + script_version - exit(0) - -cookie_jar = cookielib.CookieJar() -opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) - -# url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. -def http_req(url, post=None, headers={}): - request = urllib2.Request(url) - request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/1337 Safari/537.36') # Tell Garmin we're some supported browser. - for header_key, header_value in headers.iteritems(): - request.add_header(header_key, header_value) - if post: - post = urlencode(post) # Convert dictionary to POST parameter string. - response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError. - - # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below. - if response.getcode() != 200: - raise Exception('Bad return code (' + response.getcode() + ') for: ' + url) - - return response.read() - -print 'Welcome to Garmin Connect Exporter!' - -# Create directory for data files. -if isdir(args.directory): - print 'Warning: Output directory already exists. Will skip already-downloaded files and append to the CSV file.' - -username = args.username if args.username else raw_input('Username: ') -password = args.password if args.password else getpass() - -# Maximum number of activities you can request at once. Set and enforced by Garmin. -limit_maximum = 100 - -# URLs for various services. -url_gc_login = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' -url_gc_post_auth = 'https://connect.garmin.com/post-auth/login?' -url_gc_search = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?' -url_gc_gpx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/' -url_gc_tcx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/' -url_gc_original_activity = 'http://connect.garmin.com/proxy/download-service/files/activity/' - -# Initially, we need to get a valid session cookie, so we pull the login page. -http_req(url_gc_login) - -# Now we'll actually login. -post_data = {'username': username, 'password': password, 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} # Fields that are passed in a typical Garmin login. -http_req(url_gc_login, post_data) - -# Get the key. -# TODO: Can we do this without iterating? -login_ticket = None -for cookie in cookie_jar: - if cookie.name == 'CASTGC': - login_ticket = cookie.value - break - -if not login_ticket: - raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') - -# Chop of 'TGT-' off the beginning, prepend 'ST-0'. -login_ticket = 'ST-0' + login_ticket[4:] - -http_req(url_gc_post_auth + 'ticket=' + login_ticket) - -# We should be logged in now. -if not isdir(args.directory): - mkdir(args.directory) - -csv_filename = args.directory + '/activities.csv' -csv_existed = isfile(csv_filename) - -csv_file = open(csv_filename, 'a') - -# Write header to CSV file -if not csv_existed: - csv_file.write('Activity ID,Activity Name,Description,Begin Timestamp,Begin Timestamp (Raw Milliseconds),End Timestamp,End Timestamp (Raw Milliseconds),Device,Activity Parent,Activity Type,Event Type,Activity Time Zone,Max. Elevation,Max. Elevation (Raw),Begin Latitude (Decimal Degrees Raw),Begin Longitude (Decimal Degrees Raw),End Latitude (Decimal Degrees Raw),End Longitude (Decimal Degrees Raw),Average Moving Speed,Average Moving Speed (Raw),Max. Heart Rate (bpm),Average Heart Rate (bpm),Max. Speed,Max. Speed (Raw),Calories,Calories (Raw),Duration (h:m:s),Duration (Raw Seconds),Moving Duration (h:m:s),Moving Duration (Raw Seconds),Average Speed,Average Speed (Raw),Distance,Distance (Raw),Max. Heart Rate (bpm),Min. Elevation,Min. Elevation (Raw),Elevation Gain,Elevation Gain (Raw),Elevation Loss,Elevation Loss (Raw)\n') - -download_all = False -if args.count == 'all': - # If the user wants to download all activities, first download one, - # then the result of that request will tell us how many are available - # so we will modify the variables then. - total_to_download = 1 - download_all = True -else: - total_to_download = int(args.count) -total_downloaded = 0 - -# This while loop will download data from the server in multiple chunks, if necessary. -while total_downloaded < total_to_download: - # Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100. - if total_to_download - total_downloaded > 100: - num_to_download = 100 - else: - num_to_download = total_to_download - total_downloaded - - search_params = {'start': total_downloaded, 'limit': num_to_download} - # Query Garmin Connect - result = http_req(url_gc_search + urlencode(search_params)) - json_results = json.loads(result) # TODO: Catch possible exceptions here. - - - search = json_results['results']['search'] - - if download_all: - # Modify total_to_download based on how many activities the server reports. - total_to_download = int(search['totalFound']) - # Do it only once. - download_all = False - - # Pull out just the list of activities. - activities = json_results['results']['activities'] - - # Process each activity. - for a in activities: - # Display which entry we're working on. - print 'Garmin Connect activity: [' + a['activity']['activityId'] + ']', - print a['activity']['activityName']['value'] - print '\t' + a['activity']['beginTimestamp']['display'] + ',', - if 'sumElapsedDuration' in a['activity']: - print a['activity']['sumElapsedDuration']['display'] + ',', - else: - print '??:??:??,', - if 'sumDistance' in a['activity']: - print a['activity']['sumDistance']['withUnit'] - else: - print '0.00 Miles' - - if args.format == 'gpx': - data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.gpx' - download_url = url_gc_gpx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'w' - elif args.format == 'tcx': - data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.tcx' - download_url = url_gc_tcx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'w' - elif args.format == 'original': - data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.zip' - fit_filename = args.directory + '/' + a['activity']['activityId'] + '.fit' - download_url = url_gc_original_activity + a['activity']['activityId'] - file_mode = 'wb' - else: - raise Exception('Unrecognized format.') - - if isfile(data_filename): - print '\tData file already exists; skipping...' - continue - if args.format == 'original' and isfile(fit_filename): # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists. - print '\tFIT data file already exists; skipping...' - continue - - # Download the data file from Garmin Connect. - # If the download fails (e.g., due to timeout), this script will die, but nothing - # will have been written to disk about this activity, so just running it again - # should pick up where it left off. - print '\tDownloading file...', - - try: - data = http_req(download_url) - except urllib2.HTTPError as e: - # Handle expected (though unfortunate) error codes; die on unexpected ones. - if e.code == 500 and args.format == 'tcx': - # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. - # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. - # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, - # as I believe Garmin provides a GPX file for every activity. - print 'Writing empty file since Garmin did not generate a TCX file for this activity...', - data = '' - elif e.code == 404 and args.format == 'original': - # For manual activities (i.e., entered in online without a file upload), there is no original file. - # Write an empty file to prevent redownloading it. - print 'Writing empty file since there was no original activity data...', - data = '' - else: - raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') - - save_file = open(data_filename, file_mode) - save_file.write(data) - save_file.close() - - # Write stats to CSV. - empty_record = '"",' - - csv_record = '' - - csv_record += empty_record if 'activityId' not in a['activity'] else '"' + a['activity']['activityId'].replace('"', '""') + '",' - csv_record += empty_record if 'activityName' not in a['activity'] else '"' + a['activity']['activityName']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'activityDescription' not in a['activity'] else '"' + a['activity']['activityDescription']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['millis'].replace('"', '""') + '",' - csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['millis'].replace('"', '""') + '",' - csv_record += empty_record if 'device' not in a['activity'] else '"' + a['activity']['device']['display'].replace('"', '""') + ' ' + a['activity']['device']['version'].replace('"', '""') + '",' - csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['parent']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'eventType' not in a['activity'] else '"' + a['activity']['eventType']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'activityTimeZone' not in a['activity'] else '"' + a['activity']['activityTimeZone']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginLatitude' not in a['activity'] else '"' + a['activity']['beginLatitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginLongitude' not in a['activity'] else '"' + a['activity']['beginLongitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'endLatitude' not in a['activity'] else '"' + a['activity']['endLatitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'endLongitude' not in a['activity'] else '"' + a['activity']['endLongitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'maxHeartRate' not in a['activity'] else '"' + a['activity']['maxHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanHeartRate' not in a['activity'] else '"' + a['activity']['weightedMeanHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'minHeartRate' not in a['activity'] else '"' + a['activity']['minHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['value'].replace('"', '""') + '"' - csv_record += '\n' - - csv_file.write(csv_record.encode('utf8')) - - if args.format == 'gpx': - # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), - # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. - # N.B. You can omit the XML parse (and the associated log messages) to speed things up. - gpx = parseString(data) - gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 - - if gpx_data_exists: - print 'Done. GPX data saved.' - else: - print 'Done. No track points found.' - elif args.format == 'original': - if args.unzip and data_filename[-3:].lower() == 'zip': # Even manual upload of a GPX file is zipped, but we'll validate the extension. - print "Unzipping and removing original files...", - zip_file = open(data_filename, 'rb') - z = zipfile.ZipFile(zip_file) - for name in z.namelist(): - z.extract(name, args.directory) - zip_file.close() - remove(data_filename) - print 'Done.' - else: - # TODO: Consider validating other formats. - print 'Done.' - total_downloaded += num_to_download -# End while loop for multiple chunks. - -csv_file.close() - -print 'Done!' diff --git a/gcexport2.py b/gcexport2.py new file mode 100755 index 0000000..b92aea7 --- /dev/null +++ b/gcexport2.py @@ -0,0 +1,488 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +File: gcexport.py +Original author: Kyle Krafka (https://github.com/kjkjava/) +Date: April 28, 2015 +Fork author: Michae P (https://github.com/moderation/) +Date: February 21, 2016 + +Description: Use this script to export your fitness data from Garmin Connect. + See README.md for more information. +""" + +from urllib import urlencode +from datetime import datetime +from getpass import getpass +from sys import argv +from os.path import isdir +from os.path import isfile +from os import mkdir +from os import remove +from os import stat +from xml.dom.minidom import parseString +from subprocess import call + +import urllib, urllib2, cookielib, json +from fileinput import filename + +import argparse +import zipfile + +script_version = '1.0.0' +current_date = datetime.now().strftime('%Y-%m-%d') +activities_directory = './' + current_date + '_garmin_connect_export' + +parser = argparse.ArgumentParser() + +# TODO: Implement verbose and/or quiet options. +# parser.add_argument('-v', '--verbose', help="increase output verbosity", action="store_true") +parser.add_argument('--version', help="print version and exit", action="store_true") +parser.add_argument('--username', help="your Garmin Connect username (otherwise, you will be prompted)", nargs='?') +parser.add_argument('--password', help="your Garmin Connect password (otherwise, you will be prompted)", nargs='?') + +parser.add_argument('-c', '--count', nargs='?', default="1", + help="number of recent activities to download, or 'all' (default: 1)") + +parser.add_argument('-f', '--format', nargs='?', choices=['gpx', 'tcx', 'original'], default="gpx", + help="export format; can be 'gpx', 'tcx', or 'original' (default: 'gpx')") + +parser.add_argument('-d', '--directory', nargs='?', default=activities_directory, + help="the directory to export to (default: './YYYY-MM-DD_garmin_connect_export')") + +parser.add_argument('-u', '--unzip', + help="if downloading ZIP files (format: 'original'), unzip the file and removes the ZIP file", + action="store_true") + +args = parser.parse_args() + +if args.version: + print argv[0] + ", version " + script_version + exit(0) + +cookie_jar = cookielib.CookieJar() +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) +# print cookie_jar + +# url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. +def http_req(url, post=None, headers={}): + request = urllib2.Request(url) + # request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/1337 Safari/537.36') # Tell Garmin we're some supported browser. + request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2816.0 Safari/537.36') # Tell Garmin we're some supported browser. + for header_key, header_value in headers.iteritems(): + request.add_header(header_key, header_value) + if post: + # print "POSTING" + post = urlencode(post) # Convert dictionary to POST parameter string. + # print request.headers + # print cookie_jar + # print post + # print request + response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError. + + # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below. + # print response.getcode() + if response.getcode() == 204: + # For activities without GPS coordinates, there is no GPX download (204 = no content). + # Write an empty file to prevent redownloading it. + print 'Writing empty file since there was no GPX activity data...', + return '' + elif response.getcode() != 200: + raise Exception('Bad return code (' + str(response.getcode()) + ') for: ' + url) + + return response.read() + +print 'Welcome to Garmin Connect Exporter!' + +# Create directory for data files. +if isdir(args.directory): + print 'Warning: Output directory already exists. Will skip already-downloaded files and append to the CSV file.' + +username = args.username if args.username else raw_input('Username: ') +password = args.password if args.password else getpass() + +# Maximum number of activities you can request at once. Set and enforced by Garmin. +limit_maximum = 100 + +hostname_url = http_req('http://connect.garmin.com/gauth/hostname') +# print hostname_url +hostname = json.loads(hostname_url)['host'] + +REDIRECT = "https://connect.garmin.com/post-auth/login" +BASE_URL = "http://connect.garmin.com/en-US/signin" +GAUTH = "http://connect.garmin.com/gauth/hostname" +SSO = "https://sso.garmin.com/sso" +CSS = "https://static.garmincdn.com/com.garmin.connect/ui/css/gauth-custom-v1.1-min.css" + +data = {'service': REDIRECT, + 'webhost': hostname, + 'source': BASE_URL, + 'redirectAfterAccountLoginUrl': REDIRECT, + 'redirectAfterAccountCreationUrl': REDIRECT, + 'gauthHost': SSO, + 'locale': 'en_US', + 'id': 'gauth-widget', + 'cssUrl': CSS, + 'clientId': 'GarminConnect', + 'rememberMeShown': 'true', + 'rememberMeChecked': 'false', + 'createAccountShown': 'true', + 'openCreateAccount': 'false', + 'usernameShown': 'false', + 'displayNameShown': 'false', + 'consumeServiceTicket': 'false', + 'initialFocus': 'true', + 'embedWidget': 'false', + 'generateExtraServiceTicket': 'false'} + +print urllib.urlencode(data) + +# URLs for various services. +url_gc_login = 'https://sso.garmin.com/sso/login?' + urllib.urlencode(data) +url_gc_post_auth = 'https://connect.garmin.com/post-auth/login?' +url_gc_search = 'http://connect.garmin.com/proxy/activity-search-service-1.2/json/activities?' +url_gc_gpx_activity = 'https://connect.garmin.com/modern/proxy/download-service/export/gpx/activity/' +url_gc_tcx_activity = 'https://connect.garmin.com/modern/proxy/download-service/export/tcx/activity/' +url_gc_original_activity = 'http://connect.garmin.com/proxy/download-service/files/activity/' + +# Initially, we need to get a valid session cookie, so we pull the login page. +print 'Request login page' +http_req(url_gc_login) +print 'Finish login page' + +# Now we'll actually login. +post_data = {'username': username, 'password': password, 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} # Fields that are passed in a typical Garmin login. +print 'Post login data' +http_req(url_gc_login, post_data) +print 'Finish login post' + +# Get the key. +# TODO: Can we do this without iterating? +login_ticket = None +print "-------COOKIE" +for cookie in cookie_jar: + print cookie.name + ": " + cookie.value + if cookie.name == 'CASTGC': + login_ticket = cookie.value + print login_ticket + print cookie.value + break +print "-------COOKIE" + +if not login_ticket: + raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') + +# Chop of 'TGT-' off the beginning, prepend 'ST-0'. +login_ticket = 'ST-0' + login_ticket[4:] +# print login_ticket + +print 'Request authentication' +# print url_gc_post_auth + 'ticket=' + login_ticket +http_req(url_gc_post_auth + 'ticket=' + login_ticket) +print 'Finished authentication' + +# https://github.com/kjkjava/garmin-connect-export/issues/18#issuecomment-243859319 +print "Call modern" +http_req("http://connect.garmin.com/modern") +print "Finish modern" +print "Call legacy session" +http_req("https://connect.garmin.com/legacy/session") +print "Finish legacy session" + +# We should be logged in now. +if not isdir(args.directory): + mkdir(args.directory) + +csv_filename = args.directory + '/activities.csv' +csv_existed = isfile(csv_filename) + +csv_file = open(csv_filename, 'a') + +# Write header to CSV file +if not csv_existed: + csv_file.write('Activity name,\ +Description,\ +Begin timestamp,\ +Duration (h:m:s),\ +Moving duration (h:m:s),\ +Distance (km),\ +Average speed (km/h),\ +Average moving speed (km/h),\ +Max. speed (km/h),\ +Elevation loss uncorrected (m),\ +Elevation gain uncorrected (m),\ +Elevation min. uncorrected (m),\ +Elevation max. uncorrected (m),\ +Min. heart rate (bpm),\ +Max. heart rate (bpm),\ +Average heart rate (bpm),\ +Calories,\ +Avg. cadence (rpm),\ +Max. cadence (rpm),\ +Strokes,\ +Avg. temp (°C),\ +Min. temp (°C),\ +Max. temp (°C),\ +Map,\ +End timestamp,\ +Begin timestamp (ms),\ +End timestamp (ms),\ +Device,\ +Activity type,\ +Event type,\ +Time zone,\ +Begin latitude (°DD),\ +Begin longitude (°DD),\ +End latitude (°DD),\ +End longitude (°DD),\ +Elevation gain corrected (m),\ +Elevation loss corrected (m),\ +Elevation max. corrected (m),\ +Elevation min. corrected (m),\ +Sample count\n') + + +# Max. Elevation,\ +# Average Moving Speed,\ +# Max. Speed,\ +# Calories,\ +# Duration (Raw Seconds),\ +# Moving Duration (Raw Seconds),\ +# Average Speed,\ +# Distance,\ +# Min. Elevation,\ +# Elevation Gain,\ +# Elevation Loss,\ +# Avg Cadence,\ +# Max Cadence,\ +# Avg Temp,\ +# Min Temp,\ +# Max Temp,\ +# Min. elevation (m),\ +# Max. elevation (m),\ +# Activity parent,\ + +download_all = False +if args.count == 'all': + # If the user wants to download all activities, first download one, + # then the result of that request will tell us how many are available + # so we will modify the variables then. + total_to_download = 1 + download_all = True +else: + total_to_download = int(args.count) +total_downloaded = 0 + +# This while loop will download data from the server in multiple chunks, if necessary. +while total_downloaded < total_to_download: + # Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100. + if total_to_download - total_downloaded > 100: + num_to_download = 100 + else: + num_to_download = total_to_download - total_downloaded + + search_params = {'start': total_downloaded, 'limit': num_to_download} + # Query Garmin Connect + print "Making activity request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + print url_gc_search + urlencode(search_params) + result = http_req(url_gc_search + urlencode(search_params)) + print "Finished activity request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + + # Persist JSON + json_filename = args.directory + '/activities.json' + json_file = open(json_filename, 'a') + json_file.write(result) + json_file.close() + + json_results = json.loads(result) # TODO: Catch possible exceptions here. + + # search = json_results['results']['search'] + + if download_all: + # Modify total_to_download based on how many activities the server reports. + total_to_download = int(json_results['results']['totalFound']) + + # Do it only once. + download_all = False + + # Pull out just the list of activities. + activities = json_results['results']['activities'] + + # Process each activity. + for a in activities: + # Display which entry we're working on. + print 'Garmin Connect activity: [' + str(a['activity']['activityId']) + ']', + print a['activity']['activityName'] + print '\t' + a['activity']['activitySummary']['BeginTimestamp']['display'] + ',', + if 'SumElapsedDuration' in a['activity']['activitySummary']: + print a['activity']['activitySummary']['SumElapsedDuration']['display'] + ',', + else: + print '??:??:??,', + if 'SumDistance' in a['activity']['activitySummary']: + print a['activity']['activitySummary']['SumDistance']['withUnit'] + else: + print '0.00 Miles' + + if args.format == 'gpx': + data_filename = args.directory + '/activity_' + str(a['activity']['activityId']) + '.gpx' + download_url = url_gc_gpx_activity + str(a['activity']['activityId']) + '?full=true' + # download_url = url_gc_gpx_activity + str(a['activity']['activityId']) + '?full=true' + '&original=true' + print download_url + file_mode = 'w' + elif args.format == 'tcx': + data_filename = args.directory + '/activity_' + str(a['activity']['activityId']) + '.tcx' + download_url = url_gc_tcx_activity + str(a['activity']['activityId']) + '?full=true' + file_mode = 'w' + elif args.format == 'original': + data_filename = args.directory + '/activity_' + str(a['activity']['activityId']) + '.zip' + fit_filename = args.directory + '/' + str(a['activity']['activityId']) + '.fit' + download_url = url_gc_original_activity + str(a['activity']['activityId']) + file_mode = 'wb' + else: + raise Exception('Unrecognized format.') + + if isfile(data_filename): + print '\tData file already exists; skipping...' + continue + if args.format == 'original' and isfile(fit_filename): # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists. + print '\tFIT data file already exists; skipping...' + continue + + # Download the data file from Garmin Connect. + # If the download fails (e.g., due to timeout), this script will die, but nothing + # will have been written to disk about this activity, so just running it again + # should pick up where it left off. + print '\tDownloading file...', + + try: + data = http_req(download_url) + except urllib2.HTTPError as e: + # Handle expected (though unfortunate) error codes; die on unexpected ones. + if e.code == 500 and args.format == 'tcx': + # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. + # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. + # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, + # as I believe Garmin provides a GPX file for every activity. + print 'Writing empty file since Garmin did not generate a TCX file for this activity...', + data = '' + elif e.code == 404 and args.format == 'original': + # For manual activities (i.e., entered in online without a file upload), there is no original file. + # Write an empty file to prevent redownloading it. + print 'Writing empty file since there was no original activity data...', + data = '' + else: + raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + download_url +').') + + save_file = open(data_filename, file_mode) + save_file.write(data) + save_file.close() + + # Write stats to CSV. + empty_record = '"",' + + csv_record = '' + + csv_record += empty_record if 'activityName' not in a['activity'] else '"' + a['activity']['activityName'].replace('"', '""') + '",' + csv_record += empty_record if 'activityDescription' not in a['activity'] else '"' + a['activity']['activityDescription'].replace('"', '""') + '",' + csv_record += empty_record if 'BeginTimestamp' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['BeginTimestamp']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'SumElapsedDuration' not in a['activity']['activitySummary'] else a['activity']['activitySummary']['SumElapsedDuration']['display'].replace('"', '""') + ',' + csv_record += empty_record if 'SumMovingDuration' not in a['activity']['activitySummary'] else a['activity']['activitySummary']['SumMovingDuration']['display'].replace('"', '""') + ',' + csv_record += empty_record if 'SumDistance' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['SumDistance']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'WeightedMeanSpeed' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['WeightedMeanSpeed']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'WeightedMeanMovingSpeed' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['WeightedMeanMovingSpeed']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'MaxSpeed' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['MaxSpeed']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'LossUncorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['LossUncorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'GainUncorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['GainUncorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'MinUncorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['MinUncorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'MaxUncorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['MaxUncorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'MinHeartRate' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['MinHeartRate']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'MaxHeartRate' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['MaxHeartRate']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'WeightedMeanHeartRate' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['WeightedMeanHeartRate']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'SumEnergy' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['SumEnergy']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'WeightedMeanBikeCadence' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['WeightedMeanBikeCadence']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'MaxBikeCadence' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['MaxBikeCadence']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'SumStrokes' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['SumStrokes']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'WeightedMeanAirTemperature' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['WeightedMeanAirTemperature']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'MinAirTemperature' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['MinAirTemperature']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'MaxAirTemperature' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['MaxAirTemperature']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'activityId' not in a['activity'] else '"https://connect.garmin.com/modern/activity/' + str(a['activity']['activityId']).replace('"', '""') + '",' + csv_record += empty_record if 'EndTimestamp' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['EndTimestamp']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'BeginTimestamp' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['BeginTimestamp']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'EndTimestamp' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['EndTimestamp']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'device' not in a['activity'] else '"' + a['activity']['device']['display'].replace('"', '""') + ' ' + a['activity']['device']['version'].replace('"', '""') + '",' + csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'eventType' not in a['activity'] else '"' + a['activity']['eventType']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'activityTimeZone' not in a['activity'] else '"' + a['activity']['activityTimeZone']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'BeginLatitude' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['BeginLatitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'BeginLongitude' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['BeginLongitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'EndLatitude' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['EndLatitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'EndLongitude' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['EndLongitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'GainCorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['GainCorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'LossCorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['LossCorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'MaxCorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['MaxCorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'MinCorrectedElevation' not in a['activity']['activitySummary'] else '"' + str(float(a['activity']['activitySummary']['MinCorrectedElevation']['value'])/100) + '",' + csv_record += empty_record if 'SumSampleCountDuration' not in a['activity']['activitySummary'] else '"' + a['activity']['activitySummary']['SumSampleCountDuration']['value'].replace('"', '""') + '"' + csv_record += '\n' + +# csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['value'].replace('"', '""') + '",' +# csv_record += empty_record if 'minElevation' not in a['activity'] else '"' + a['activity']['minElevation']['value'].replace('"', '""') + '",' +# csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",' +# csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",' +# csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" +# csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" +# csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['display'].replace('"', '""') + '",' +# csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['value'].replace('"', '""') + '",' +# csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['value'].replace('"', '""') + '",' +# csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['withUnit'].replace('"', '""') + '",' +# csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['withUnit'].replace('"', '""') + '",' +# csv_record += empty_record if 'minElevation' not in a['activity'] else '"' + a['activity']['minElevation']['withUnit'].replace('"', '""') + '",' +# csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['withUnit'].replace('"', '""') + '",' +# csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['withUnit'].replace('"', '""') + '",' +# csv_record += empty_record if 'weightedMeanBikeCadence' not in a['activity'] else '"' + a['activity']['weightedMeanBikeCadence']['withUnitAbbr'].replace('"', '""') + '",' +# csv_record += empty_record if 'maxBikeCadence' not in a['activity'] else '"' + a['activity']['maxBikeCadence']['withUnitAbbr'].replace('"', '""') + '",' +# csv_record += empty_record if 'weightedMeanAirTemperature' not in a['activity'] else '"' + a['activity']['weightedMeanAirTemperature']['withUnitAbbr'].replace('"', '""') + '",' +# csv_record += empty_record if 'minAirTemperature' not in a['activity'] else '"' + a['activity']['minAirTemperature']['withUnitAbbr'].replace('"', '""') + '",' +# csv_record += empty_record if 'maxAirTemperature' not in a['activity'] else '"' + a['activity']['maxAirTemperature']['withUnitAbbr'].replace('"', '""') + '",' +# csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['parent']['display'].replace('"', '""') + '",' + + csv_file.write(csv_record.encode('utf8')) + + if args.format == 'gpx' and data: + # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), + # Garmin Connect still kicks out a GPX (sometimes), but there is only activity information, no GPS data. + # N.B. You can omit the XML parse (and the associated log messages) to speed things up. + gpx = parseString(data) + gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 + + if gpx_data_exists: + print 'Done. GPX data saved.' + else: + print 'Done. No track points found.' + elif args.format == 'original': + if args.unzip and data_filename[-3:].lower() == 'zip': # Even manual upload of a GPX file is zipped, but we'll validate the extension. + print "Unzipping and removing original files...", + print 'Filesize is: ' + str(stat(data_filename).st_size) + if stat(data_filename).st_size > 0: + zip_file = open(data_filename, 'rb') + z = zipfile.ZipFile(zip_file) + for name in z.namelist(): + z.extract(name, args.directory) + zip_file.close() + else: + print 'Skipping 0Kb zip file.' + remove(data_filename) + print 'Done.' + else: + # TODO: Consider validating other formats. + print 'Done.' + total_downloaded += num_to_download +# End while loop for multiple chunks. + +csv_file.close() + +print 'Open CSV output.' +print csv_filename +# call(["open", csv_filename]) + +print 'Done!' diff --git a/gcexport3.py b/gcexport3.py new file mode 100755 index 0000000..582ee1e --- /dev/null +++ b/gcexport3.py @@ -0,0 +1,772 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +""" +File: gcexport.py +Original author: Kyle Krafka (https://github.com/kjkjava/) +Date: April 28, 2015 +Fork author: Michael P (https://github.com/moderation/) +Date: August 25, 2018 + +Description: Use this script to export your fitness data from Garmin Connect. + See README.md for more information. + +Activity & event types: + https://connect.garmin.com/modern/main/js/properties/event_types/event_types.properties + https://connect.garmin.com/modern/main/js/properties/activity_types/activity_types.properties +""" + +from datetime import datetime, timedelta +from getpass import getpass +from os import mkdir, remove, stat +from os.path import isdir, isfile +from subprocess import call +from sys import argv +from xml.dom.minidom import parseString + +import argparse +import http.cookiejar +import json +import re +import urllib.error +import urllib.parse +import urllib.request +import zipfile + +SCRIPT_VERSION = "2.0.0" +CURRENT_DATE = datetime.now().strftime("%Y-%m-%d") +ACTIVITIES_DIRECTORY = "./" + CURRENT_DATE + "_garmin_connect_export" + +PARSER = argparse.ArgumentParser() + +# TODO: Implement verbose and/or quiet options. +# PARSER.add_argument('-v', '--verbose', help="increase output verbosity", action="store_true") +PARSER.add_argument("--version", help="print version and exit", action="store_true") +PARSER.add_argument( + "--username", + help="your Garmin Connect username (otherwise, you will be prompted)", + nargs="?", +) +PARSER.add_argument( + "--password", + help="your Garmin Connect password (otherwise, you will be prompted)", + nargs="?", +) +PARSER.add_argument( + "-c", + "--count", + nargs="?", + default="1", + help="number of recent activities to download, or 'all' (default: 1)", +) +PARSER.add_argument( + "-e", + "--external", + nargs="?", + default="", + help="path to external program to pass CSV file too (default: )", +) +PARSER.add_argument( + "-a", + "--args", + nargs="?", + default="", + help="additional arguments to pass to external program (default: )", +) +PARSER.add_argument( + "-f", + "--format", + nargs="?", + choices=["gpx", "tcx", "original"], + default="gpx", + help="export format; can be 'gpx', 'tcx', or 'original' (default: 'gpx')", +) +PARSER.add_argument( + "-d", + "--directory", + nargs="?", + default=ACTIVITIES_DIRECTORY, + help="the directory to export to (default: './YYYY-MM-DD_garmin_connect_export')", +) +PARSER.add_argument( + "-u", + "--unzip", + help=( + "if downloading ZIP files (format: 'original'), unzip the file and removes the" + " ZIP file" + ), + action="store_true", +) + +ARGS = PARSER.parse_args() + +if ARGS.version: + print(argv[0] + ", version " + SCRIPT_VERSION) + exit(0) + +COOKIE_JAR = http.cookiejar.CookieJar() +OPENER = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(COOKIE_JAR)) +# print(COOKIE_JAR) + + +def hhmmss_from_seconds(sec): + """Helper function that converts seconds to HH:MM:SS time format.""" + if isinstance(sec, float): + formatted_time = str(timedelta(seconds=int(sec))).zfill(8) + else: + formatted_time = "0.000" + return formatted_time + + +def kmh_from_mps(mps): + """Helper function that converts meters per second (mps) to km/h.""" + return str(mps * 3.6) + + +def write_to_file(filename, content, mode): + """Helper function that persists content to file.""" + write_file = open(filename, mode) + write_file.write(content) + write_file.close() + + +def decoding_decider(data): + """Helper function that decides if a decoding should happen or not.""" + if ARGS.format == "original": + # An original file (ZIP file) is binary and not UTF-8 encoded + data = data + elif data: + # GPX and TCX are textfiles and UTF-8 encoded + data = data.decode() + + return data + + +# url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. +def http_req(url, post=None, headers=None): + """Helper function that makes the HTTP requests.""" + request = urllib.request.Request(url) + # Tell Garmin we're some supported browser. + request.add_header( + "User-Agent", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/54.0.2816.0 Safari/537.36", + ) + if headers: + for header_key, header_value in headers.items(): + request.add_header(header_key, header_value) + if post: + post = urllib.parse.urlencode(post) + post = post.encode("utf-8") # Convert dictionary to POST parameter string. + # print("request.headers: " + str(request.headers) + " COOKIE_JAR: " + str(COOKIE_JAR)) + # print("post: " + str(post) + "request: " + str(request)) + response = OPENER.open(request, data=post) + + if response.getcode() == 204: + # For activities without GPS coordinates, there is no GPX download (204 = no content). + # Write an empty file to prevent redownloading it. + print("Writing empty file since there was no GPX activity data...") + return "" + elif response.getcode() != 200: + raise Exception("Bad return code (" + str(response.getcode()) + ") for: " + url) + # print(response.getcode()) + + return response.read() + + +print("Welcome to Garmin Connect Exporter!") + +# Create directory for data files. +if isdir(ARGS.directory): + print( + "Warning: Output directory already exists. Will skip already-downloaded files" + " and append to the CSV file." + ) + +USERNAME = ARGS.username if ARGS.username else input("Username: ") +PASSWORD = ARGS.password if ARGS.password else getpass() + +# Maximum number of activities you can request at once. Set and enforced by Garmin. +LIMIT_MAXIMUM = 1000 + +WEBHOST = "https://connect.garmin.com" +REDIRECT = "https://connect.garmin.com/modern/" +BASE_URL = "https://connect.garmin.com/en-US/signin" +SSO = "https://sso.garmin.com/sso" +CSS = "https://static.garmincdn.com/com.garmin.connect/ui/css/gauth-custom-v1.2-min.css" + +DATA = { + "service": REDIRECT, + "webhost": WEBHOST, + "source": BASE_URL, + "redirectAfterAccountLoginUrl": REDIRECT, + "redirectAfterAccountCreationUrl": REDIRECT, + "gauthHost": SSO, + "locale": "en_US", + "id": "gauth-widget", + "cssUrl": CSS, + "clientId": "GarminConnect", + "rememberMeShown": "true", + "rememberMeChecked": "false", + "createAccountShown": "true", + "openCreateAccount": "false", + "displayNameShown": "false", + "consumeServiceTicket": "false", + "initialFocus": "true", + "embedWidget": "false", + "generateExtraServiceTicket": "true", + "generateTwoExtraServiceTickets": "false", + "generateNoServiceTicket": "false", + "globalOptInShown": "true", + "globalOptInChecked": "false", + "mobile": "false", + "connectLegalTerms": "true", + "locationPromptShown": "true", + "showPassword": "true", +} + +print(urllib.parse.urlencode(DATA)) + +# URLs for various services. +URL_GC_LOGIN = "https://sso.garmin.com/sso/signin?" + urllib.parse.urlencode(DATA) +URL_GC_POST_AUTH = "https://connect.garmin.com/modern/activities?" +URL_GC_PROFILE = "https://connect.garmin.com/modern/profile" +URL_GC_USERSTATS = ( + "https://connect.garmin.com/modern/proxy/userstats-service/statistics/" +) +URL_GC_LIST = "https://connect.garmin.com/modern/proxy/activitylist-service/activities/search/activities?" +URL_GC_ACTIVITY = "https://connect.garmin.com/modern/proxy/activity-service/activity/" +URL_GC_GPX_ACTIVITY = ( + "https://connect.garmin.com/modern/proxy/download-service/export/gpx/activity/" +) +URL_GC_TCX_ACTIVITY = ( + "https://connect.garmin.com/modern/proxy/download-service/export/tcx/activity/" +) +URL_GC_ORIGINAL_ACTIVITY = ( + "http://connect.garmin.com/proxy/download-service/files/activity/" +) +URL_DEVICE_DETAIL = ( + "https://connect.garmin.com/modern/proxy/device-service/deviceservice/app-info/" +) +URL_GEAR_DETAIL = ( + "https://connect.garmin.com/modern/proxy/gear-service/gear/filterGear?" +) +# Initially, we need to get a valid session cookie, so we pull the login page. +print("Request login page") +http_req(URL_GC_LOGIN) +print("Finish login page") + +# Now we'll actually login. +# Fields that are passed in a typical Garmin login. +POST_DATA = { + "username": USERNAME, + "password": PASSWORD, + "embed": "false", + "rememberme": "on", +} + +HEADERS = {"referer": URL_GC_LOGIN} + +print("Post login data") +LOGIN_RESPONSE = http_req(URL_GC_LOGIN + "#", POST_DATA, HEADERS).decode() +print("Finish login post") + +# extract the ticket from the login response +PATTERN = re.compile(r".*\?ticket=([-\w]+)\";.*", re.MULTILINE | re.DOTALL) +MATCH = PATTERN.match(LOGIN_RESPONSE) +if not MATCH: + raise Exception( + "Did not get a ticket in the login response. Cannot log in. Did you enter the" + " correct username and password?" + ) +LOGIN_TICKET = MATCH.group(1) +print("Login ticket=" + LOGIN_TICKET) + +print("Request authentication URL: " + URL_GC_POST_AUTH + "ticket=" + LOGIN_TICKET) +http_req(URL_GC_POST_AUTH + "ticket=" + LOGIN_TICKET) +print("Finished authentication") + +# We should be logged in now. +if not isdir(ARGS.directory): + mkdir(ARGS.directory) + +CSV_FILENAME = ARGS.directory + "/activities.csv" +CSV_EXISTED = isfile(CSV_FILENAME) + +CSV_FILE = open(CSV_FILENAME, "a") + +# Write header to CSV file +if not CSV_EXISTED: + CSV_FILE.write( + "Activity name,Description,Bike,Begin timestamp,Duration (h:m:s),Moving" + " duration (h:m:s),Distance (km),Average speed (km/h),Average moving speed" + " (km/h),Max. speed (km/h),Elevation loss uncorrected (m),Elevation gain" + " uncorrected (m),Elevation min. uncorrected (m),Elevation max. uncorrected" + " (m),Min. heart rate (bpm),Max. heart rate (bpm),Average heart rate" + " (bpm),Calories,Avg. cadence (rpm),Max. cadence (rpm),Strokes,Avg. temp" + " (°C),Min. temp (°C),Max. temp (°C),Map,End timestamp,Begin timestamp (ms),End" + " timestamp (ms),Device,Activity type,Event type,Time zone,Begin latitude" + " (°DD),Begin longitude (°DD),End latitude (°DD),End longitude (°DD),Elevation" + " gain corrected (m),Elevation loss corrected (m),Elevation max. corrected" + " (m),Elevation min. corrected (m),Sample count\n" + ) + +DOWNLOAD_ALL = False +if ARGS.count == "all": + # If the user wants to download all activities, query the userstats + # on the profile page to know how many are available + print("Getting display name and user stats via: " + URL_GC_PROFILE) + PROFILE_PAGE = http_req(URL_GC_PROFILE).decode() + # write_to_file(args.directory + '/profile.html', profile_page, 'a') + + # extract the display name from the profile page, it should be in there as + # \"displayName\":\"eschep\" + PATTERN = re.compile( + r".*\\\"displayName\\\":\\\"([-.\w]+)\\\".*", re.MULTILINE | re.DOTALL + ) + MATCH = PATTERN.match(PROFILE_PAGE) + if not MATCH: + raise Exception("Did not find the display name in the profile page.") + DISPLAY_NAME = MATCH.group(1) + print("displayName=" + DISPLAY_NAME) + + print(URL_GC_USERSTATS + DISPLAY_NAME) + USER_STATS = http_req(URL_GC_USERSTATS + DISPLAY_NAME) + print("Finished display name and user stats ~~~~~~~~~~~~~~~~~~~~~~~~~~~") + + # Persist JSON + write_to_file(ARGS.directory + "/userstats.json", USER_STATS.decode(), "a") + + # Modify total_to_download based on how many activities the server reports. + JSON_USER = json.loads(USER_STATS) + TOTAL_TO_DOWNLOAD = int(JSON_USER["userMetrics"][0]["totalActivities"]) +else: + TOTAL_TO_DOWNLOAD = int(ARGS.count) + +TOTAL_DOWNLOADED = 0 +print("Total to download: " + str(TOTAL_TO_DOWNLOAD)) + +# This while loop will download data from the server in multiple chunks, if necessary. +while TOTAL_DOWNLOADED < TOTAL_TO_DOWNLOAD: + # Maximum chunk size 'limit_maximum' ... 400 return status if over maximum. So download + # maximum or whatever remains if less than maximum. + # As of 2018-03-06 I get return status 500 if over maximum + if TOTAL_TO_DOWNLOAD - TOTAL_DOWNLOADED > LIMIT_MAXIMUM: + NUM_TO_DOWNLOAD = LIMIT_MAXIMUM + else: + NUM_TO_DOWNLOAD = TOTAL_TO_DOWNLOAD - TOTAL_DOWNLOADED + + SEARCH_PARAMS = {"start": TOTAL_DOWNLOADED, "limit": NUM_TO_DOWNLOAD} + + # Query Garmin Connect + print("Activity list URL: " + URL_GC_LIST + urllib.parse.urlencode(SEARCH_PARAMS)) + ACTIVITY_LIST = http_req(URL_GC_LIST + urllib.parse.urlencode(SEARCH_PARAMS)) + write_to_file(ARGS.directory + "/activity_list.json", ACTIVITY_LIST.decode(), "a") + LIST = json.loads(ACTIVITY_LIST) + # print(LIST) + + # Process each activity. + for a in LIST: + # Display which entry we're working on. + print("Garmin Connect activity: [" + str(a["activityId"]) + "]", end=" ") + print(a["activityName"]) + # print("\t" + a["uploadDate"]["display"] + ",", end=" ") + if ARGS.format == "gpx": + data_filename = ( + ARGS.directory + "/" + str(a["activityId"]) + "_activity.gpx" + ) + download_url = URL_GC_GPX_ACTIVITY + str(a["activityId"]) + "?full=true" + print(download_url) + file_mode = "w" + elif ARGS.format == "tcx": + data_filename = ( + ARGS.directory + "/" + str(a["activityId"]) + "_activity.tcx" + ) + download_url = URL_GC_TCX_ACTIVITY + str(a["activityId"]) + "?full=true" + file_mode = "w" + elif ARGS.format == "original": + data_filename = ( + ARGS.directory + "/" + str(a["activityId"]) + "_activity.zip" + ) + fit_filename = ARGS.directory + "/" + str(a["activityId"]) + "_activity.fit" + download_url = URL_GC_ORIGINAL_ACTIVITY + str(a["activityId"]) + file_mode = "wb" + else: + raise Exception("Unrecognized format.") + + if isfile(data_filename): + print("\tData file already exists; skipping...") + continue + # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists. + if ARGS.format == "original" and isfile(fit_filename): + print("\tFIT data file already exists; skipping...") + continue + + # Download the data file from Garmin Connect. If the download fails (e.g., due to timeout), + # this script will die, but nothing will have been written to disk about this activity, so + # just running it again should pick up where it left off. + print("\tDownloading file...", end=" ") + + try: + data = http_req(download_url) + except urllib.error.HTTPError as errs: + # Handle expected (though unfortunate) error codes; die on unexpected ones. + if errs.code == 500 and ARGS.format == "tcx": + # Garmin will give an internal server error (HTTP 500) when downloading TCX files + # if the original was a manual GPX upload. Writing an empty file prevents this file + # from being redownloaded, similar to the way GPX files are saved even when there + # are no tracks. One could be generated here, but that's a bit much. Use the GPX + # format if you want actual data in every file, as I believe Garmin provides a GPX + # file for every activity. + print( + "Writing empty file since Garmin did not generate a TCX file for" + " this activity...", + end=" ", + ) + data = "" + elif errs.code == 404 and ARGS.format == "original": + # For manual activities (i.e., entered in online without a file upload), there is + # no original file. # Write an empty file to prevent redownloading it. + print( + "Writing empty file since there was no original activity data...", + end=" ", + ) + data = "" + else: + raise Exception( + "Failed. Got an unexpected HTTP error (" + + str(errs.code) + + download_url + + ")." + ) + + # Persist file + write_to_file(data_filename, decoding_decider(data), file_mode) + + print("Activity summary URL: " + URL_GC_ACTIVITY + str(a["activityId"])) + ACTIVITY_SUMMARY = http_req(URL_GC_ACTIVITY + str(a["activityId"])) + write_to_file( + ARGS.directory + "/" + str(a["activityId"]) + "_activity_summary.json", + ACTIVITY_SUMMARY.decode(), + "a", + ) + JSON_SUMMARY = json.loads(ACTIVITY_SUMMARY) + # print(JSON_SUMMARY) + + print( + "Device detail URL: " + + URL_DEVICE_DETAIL + + str(JSON_SUMMARY["metadataDTO"]["deviceApplicationInstallationId"]) + ) + DEVICE_DETAIL = http_req( + URL_DEVICE_DETAIL + + str(JSON_SUMMARY["metadataDTO"]["deviceApplicationInstallationId"]) + ) + if DEVICE_DETAIL: + write_to_file( + ARGS.directory + "/" + str(a["activityId"]) + "_app_info.json", + DEVICE_DETAIL.decode(), + "a", + ) + JSON_DEVICE = json.loads(DEVICE_DETAIL) + # print(JSON_DEVICE) + else: + print("Retrieving Device Details failed.") + JSON_DEVICE = None + + print( + "Activity details URL: " + + URL_GC_ACTIVITY + + str(a["activityId"]) + + "/details" + ) + try: + ACTIVITY_DETAIL = http_req( + URL_GC_ACTIVITY + str(a["activityId"]) + "/details" + ) + write_to_file( + ARGS.directory + "/" + str(a["activityId"]) + "_activity_detail.json", + ACTIVITY_DETAIL.decode(), + "a", + ) + JSON_DETAIL = json.loads(ACTIVITY_DETAIL) + # print(JSON_DETAIL) + except: + print("Retrieving Activity Details failed.") + JSON_DETAIL = None + + print( + "Gear details URL: " + + URL_GEAR_DETAIL + + "activityId=" + + str(a["activityId"]) + ) + try: + GEAR_DETAIL = http_req( + URL_GEAR_DETAIL + "activityId=" + str(a["activityId"]) + ) + write_to_file( + ARGS.directory + "/" + str(a["activityId"]) + "_gear_detail.json", + GEAR_DETAIL.decode(), + "a", + ) + JSON_GEAR = json.loads(GEAR_DETAIL) + # print(JSON_GEAR) + except: + print("Retrieving Gear Details failed.") + # JSON_GEAR = None + + # Write stats to CSV. + empty_record = "," + csv_record = "" + + csv_record += ( + empty_record + if "activityName" not in a or not a["activityName"] + else '"' + a["activityName"].replace('"', '""') + '",' + ) + + # maybe a more elegant way of coding this but need to handle description as null + if "description" not in a: + csv_record += empty_record + elif a["description"] is not None: + csv_record += '"' + a["description"].replace('"', '""') + '",' + else: + csv_record += empty_record + + # Gear detail returned as an array so pick the first one + csv_record += ( + empty_record + if not JSON_GEAR or "customMakeModel" not in JSON_GEAR[0] + else JSON_GEAR[0]["customMakeModel"] + "," + ) + csv_record += ( + empty_record + if "startTimeLocal" not in JSON_SUMMARY["summaryDTO"] + else '"' + JSON_SUMMARY["summaryDTO"]["startTimeLocal"] + '",' + ) + csv_record += ( + empty_record + if "elapsedDuration" not in JSON_SUMMARY["summaryDTO"] + else hhmmss_from_seconds(JSON_SUMMARY["summaryDTO"]["elapsedDuration"]) + + "," + ) + csv_record += ( + empty_record + if "movingDuration" not in JSON_SUMMARY["summaryDTO"] + else hhmmss_from_seconds(JSON_SUMMARY["summaryDTO"]["movingDuration"]) + "," + ) + csv_record += ( + empty_record + if "distance" not in JSON_SUMMARY["summaryDTO"] + else "{0:.5f}".format(JSON_SUMMARY["summaryDTO"]["distance"] / 1000) + "," + ) + csv_record += ( + empty_record + if "averageSpeed" not in JSON_SUMMARY["summaryDTO"] + else kmh_from_mps(JSON_SUMMARY["summaryDTO"]["averageSpeed"]) + "," + ) + csv_record += ( + empty_record + if "averageMovingSpeed" not in JSON_SUMMARY["summaryDTO"] + else kmh_from_mps(JSON_SUMMARY["summaryDTO"]["averageMovingSpeed"]) + "," + ) + csv_record += ( + empty_record + if "maxSpeed" not in JSON_SUMMARY["summaryDTO"] + else kmh_from_mps(JSON_SUMMARY["summaryDTO"]["maxSpeed"]) + "," + ) + csv_record += ( + empty_record + if "elevationLoss" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["elevationLoss"]) + "," + ) + csv_record += ( + empty_record + if "elevationGain" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["elevationGain"]) + "," + ) + csv_record += ( + empty_record + if "minElevation" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["minElevation"]) + "," + ) + csv_record += ( + empty_record + if "maxElevation" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["maxElevation"]) + "," + ) + csv_record += empty_record if "minHR" not in JSON_SUMMARY["summaryDTO"] else "," + csv_record += ( + empty_record + if "maxHR" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["maxHR"]) + "," + ) + csv_record += ( + empty_record + if "averageHR" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["averageHR"]) + "," + ) + csv_record += ( + empty_record + if "calories" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["calories"]) + "," + ) + csv_record += ( + empty_record + if "averageBikeCadence" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["averageBikeCadence"]) + "," + ) + csv_record += ( + empty_record + if "maxBikeCadence" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["maxBikeCadence"]) + "," + ) + csv_record += ( + empty_record + if "totalNumberOfStrokes" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["totalNumberOfStrokes"]) + "," + ) + csv_record += ( + empty_record + if "averageTemperature" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["averageTemperature"]) + "," + ) + csv_record += ( + empty_record + if "minTemperature" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["minTemperature"]) + "," + ) + csv_record += ( + empty_record + if "maxTemperature" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["maxTemperature"]) + "," + ) + csv_record += ( + empty_record + if "activityId" not in a + else '"https://connect.garmin.com/modern/activity/' + + str(a["activityId"]) + + '",' + ) + csv_record += ( + empty_record if "endTimestamp" not in JSON_SUMMARY["summaryDTO"] else "," + ) + csv_record += ( + empty_record if "beginTimestamp" not in JSON_SUMMARY["summaryDTO"] else "," + ) + csv_record += ( + empty_record if "endTimestamp" not in JSON_SUMMARY["summaryDTO"] else "," + ) + csv_record += ( + empty_record + if not JSON_DEVICE or "productDisplayName" not in JSON_DEVICE + else JSON_DEVICE["productDisplayName"] + + " " + + JSON_DEVICE["versionString"] + + "," + ) + csv_record += ( + empty_record + if "activityType" not in a + else a["activityType"]["typeKey"].title() + "," + ) + csv_record += ( + empty_record + if "eventType" not in a + else a["eventType"]["typeKey"].title() + "," + ) + csv_record += ( + empty_record + if "timeZoneUnitDTO" not in JSON_SUMMARY + else JSON_SUMMARY["timeZoneUnitDTO"]["timeZone"] + "," + ) + csv_record += ( + empty_record + if "startLatitude" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["startLatitude"]) + "," + ) + csv_record += ( + empty_record + if "startLongitude" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["startLongitude"]) + "," + ) + csv_record += ( + empty_record + if "endLatitude" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["endLatitude"]) + "," + ) + csv_record += ( + empty_record + if "endLongitude" not in JSON_SUMMARY["summaryDTO"] + else str(JSON_SUMMARY["summaryDTO"]["endLongitude"]) + "," + ) + csv_record += ( + empty_record + if "gainCorrectedElevation" not in JSON_SUMMARY["summaryDTO"] + else "," + ) + csv_record += ( + empty_record + if "lossCorrectedElevation" not in JSON_SUMMARY["summaryDTO"] + else "," + ) + csv_record += ( + empty_record + if "maxCorrectedElevation" not in JSON_SUMMARY["summaryDTO"] + else "," + ) + csv_record += ( + empty_record + if "minCorrectedElevation" not in JSON_SUMMARY["summaryDTO"] + else "," + ) + csv_record += ( + empty_record + if not JSON_DETAIL or "metricsCount" not in JSON_DETAIL + else str(JSON_DETAIL["metricsCount"]) + "," + ) + csv_record += "\n" + + CSV_FILE.write(csv_record) + + if ARGS.format == "gpx" and data: + # Validate GPX data. If we have an activity without GPS data (e.g., running on a + # treadmill), Garmin Connect still kicks out a GPX (sometimes), but there is only + # activity information, no GPS data. N.B. You can omit the XML parse (and the + # associated log messages) to speed things up. + gpx = parseString(data) + if gpx.getElementsByTagName("trkpt"): + print("Done. GPX data saved.") + else: + print("Done. No track points found.") + elif ARGS.format == "original": + # Even manual upload of a GPX file is zipped, but we'll validate the extension. + if ARGS.unzip and data_filename[-3:].lower() == "zip": + print("Unzipping and removing original files...", end=" ") + print("Filesize is: " + str(stat(data_filename).st_size)) + if stat(data_filename).st_size > 0: + zip_file = open(data_filename, "rb") + z = zipfile.ZipFile(zip_file) + for name in z.namelist(): + z.extract(name, ARGS.directory) + zip_file.close() + else: + print("Skipping 0Kb zip file.") + remove(data_filename) + print("Done.") + else: + # TODO: Consider validating other formats. + print("Done.") + TOTAL_DOWNLOADED += NUM_TO_DOWNLOAD +# End while loop for multiple chunks. + +CSV_FILE.close() + +if len(ARGS.external): + print("Open CSV output.") + print(CSV_FILENAME) + # open CSV file. Comment this line out if you don't want this behavior + call([ARGS.external, "--" + ARGS.args, CSV_FILENAME]) + +print("Done!") diff --git a/old/garmin-connect-export.php b/old/garmin-connect-export.php deleted file mode 100644 index ab55c3d..0000000 --- a/old/garmin-connect-export.php +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/php - 1 && ( is_numeric( $argv[1] ) ) ) { - $total_to_download = $argv[1]; -} else if ( $argc > 1 && strcasecmp($argv[1], "all") == 0 ) { - // If the user wants to download all activities, first download one, - // then the result of that request will tell us how many are available - // so we will modify the variables then. - $total_to_download = 1; - $download_all = true; -} else { - $total_to_download = 1; -} -$total_downloaded = 0; - -// This while loop will download data from the server in multiple chunks, if necessary -while( $total_downloaded < $total_to_download ) { - $num_to_download = ($total_to_download - $total_downloaded > 100) ? 100 : ($total_to_download - $total_downloaded); // Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100. - - // Query Garmin Connect - $search_opts = array( - 'start' => $total_downloaded, - 'limit' => $num_to_download - ); - - $result = curl( $urlGCSearch . http_build_query( $search_opts ) ); - $json = json_decode( $result ); - - if ( ! $json ) { - echo "Error: "; - switch(json_last_error()) { - case JSON_ERROR_DEPTH: - echo ' - Maximum stack depth exceeded'; - break; - case JSON_ERROR_CTRL_CHAR: - echo ' - Unexpected control character found'; - break; - case JSON_ERROR_SYNTAX: - echo ' - Syntax error, malformed JSON'; - break; - } - echo PHP_EOL; - var_dump( $result ); - die(); - } - - $search = $json->{'results'}->{'search'}; - - if ( $download_all ) { - // Modify $total_to_download based on how many activities the server reports - $total_to_download = intval( $search->{'totalFound'} ); - // Do it only once - $download_all = false; - } - - // Pull out just the list of activities - $activities = $json->{'results'}->{'activities'}; - - // Process each activity. - foreach ( $activities as $a ) { - // Display which entry we're working on. - print "Garmin Connect activity: [" . $a->{'activity'}->{'activityId'} . "] "; - print $a->{'activity'}->{'beginTimestamp'}->{'display'} . ": "; - print $a->{'activity'}->{'activityName'}->{'value'} . "\n"; - - // Write data to CSV - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityId'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityName'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityDescription'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginTimestamp'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginTimestamp'}->{'millis'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endTimestamp'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endTimestamp'}->{'millis'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'device'}->{'display'} . " " . $a->{'activity'}->{'device'}->{'version'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityType'}->{'parent'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityType'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'eventType'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityTimeZone'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginLatitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginLongitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endLatitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endLongitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanMovingSpeed'}->{'display'}) . "\"," ); // The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanMovingSpeed'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxHeartRate'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanHeartRate'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxSpeed'}->{'display'}) . "\"," ); // The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxSpeed'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumEnergy'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumEnergy'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumElapsedDuration'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumElapsedDuration'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumMovingDuration'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumMovingDuration'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanSpeed'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanSpeed'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumDistance'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumDistance'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'minHeartRate'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'gainElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'gainElevation'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'lossElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'lossElevation'}->{'value'}) . "\""); - fwrite( $csv_file, "\n"); - - // Download the GPX file from Garmin Connect - // TODO: Consider using TCX files? Does Garmin Connect include heart rate data in TCX downloads? - print "\tDownloading GPX file... "; - - $gpx_filename = $activities_directory . '/activity_' . $a->{'activity'}->{'activityId'} . '.gpx'; - $save_file = fopen( $gpx_filename, 'w+' ); - $curl_opts = array( - CURLOPT_FILE => $save_file - ); - curl( $urlGCActivity . $a->{'activity'}->{'activityId'} . '?full=true', array(), array(), $curl_opts ); - fclose( $save_file ); - - // Validate the GPX data. If we have an activity without GPS data (e.g. running on a treadmill), - // Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. - $gpx = simplexml_load_file( $gpx_filename, 'SimpleXMLElement', LIBXML_NOCDATA ); - $gpxdataexists = ( count( $gpx->trk->trkseg->trkpt ) > 0); - - if ( $gpxdataexists ) { - print "Done. GPX data saved.\n"; - } else { - print "Done. No track points found.\n"; - } - } - - $total_downloaded += $num_to_download; - -// End while loop for multiple chunks -} - -fclose($csv_file); - -print "Done!\n\n"; -// End - -function curl( $url, $post = array(), $head = array(), $opts = array() ) -{ - $cookie_file = '/tmp/cookies.txt'; - $ch = curl_init(); - - //curl_setopt( $ch, CURLOPT_VERBOSE, 1 ); - curl_setopt( $ch, CURLOPT_URL, $url ); - curl_setopt( $ch, CURLOPT_RETURNTRANSFER, 1 ); - curl_setopt( $ch, CURLOPT_ENCODING, "gzip" ); - curl_setopt( $ch, CURLOPT_COOKIEFILE, $cookie_file ); - curl_setopt( $ch, CURLOPT_COOKIEJAR, $cookie_file ); - curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1 ); - - foreach ( $opts as $k => $v ) { - curl_setopt( $ch, $k, $v ); - } - - if ( count( $post ) > 0 ) { - // POST mode - curl_setopt( $ch, CURLOPT_POST, 1 ); - curl_setopt( $ch, CURLOPT_POSTFIELDS, $post ); - } - else { - curl_setopt( $ch, CURLOPT_HTTPHEADER, $head ); - curl_setopt( $ch, CURLOPT_CRLF, 1 ); - } - - $success = curl_exec( $ch ); - - if ( curl_errno( $ch ) !== 0 ) { - throw new Exception( sprintf( '%s: CURL Error %d: %s', __CLASS__, curl_errno( $ch ), curl_error( $ch ) ) ); - } - - if ( curl_getinfo( $ch, CURLINFO_HTTP_CODE ) !== 200 ) { - if ( curl_getinfo( $ch, CURLINFO_HTTP_CODE ) !== 201 ) { - throw new Exception( sprintf( 'Bad return code(%1$d) for: %2$s', curl_getinfo( $ch, CURLINFO_HTTP_CODE ), $url ) ); - } - } - - curl_close( $ch ); - return $success; -} - -?>