Use a Garth OAuth implementation fixes #95

README.md

@@ -44,6 +44,7 @@ If you have many activities, you may find that this script crashes with an "Oper
 - If you're comfortable using Git, just clone the repo from github
 - Otherwise get the latest `zip` (or `tar.gz`) from the [releases page](https://github.com/pe-st/garmin-connect-export/releases)
   and unpack it where it suits you.
+- Install the dependencies: `pip install -r requirements.txt`
 
 ## Usage
 

gcexport.py

@@ -43,6 +43,7 @@
 from urllib.error import HTTPError, URLError
 from urllib.parse import urlencode
 from urllib.request import Request
+import garth
 
 # Local application/library specific imports
 from filtering import read_exclude, update_download_stats
@@ -88,63 +89,28 @@
 # Maximum number of activities you can request at once.
 # Used to be 100 and enforced by Garmin for older endpoints; for the current endpoint 'URL_GC_LIST'
 # the limit is not known (I have less than 1000 activities and could get them all in one go)
-LIMIT_MAXIMUM = 1000
+SINGLE_REQUEST_LIMIT = 1000
+# The absolute limit for activities is 10.000 on server side
+TOTAL_ACTIVITY_LIMIT = 10_000
 
 MAX_TRIES = 3
 
 CSV_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "csv_header_default.properties")
 
-WEBHOST = "https://connect.garmin.com"
-REDIRECT = "https://connect.garmin.com/modern/"
-BASE_URL = "https://connect.garmin.com/en-US/signin"
-SSO = "https://sso.garmin.com/sso"
-CSS = "https://static.garmincdn.com/com.garmin.connect/ui/css/gauth-custom-v1.2-min.css"
-
-DATA = {
-    'service': REDIRECT,
-    'webhost': WEBHOST,
-    'source': BASE_URL,
-    'redirectAfterAccountLoginUrl': REDIRECT,
-    'redirectAfterAccountCreationUrl': REDIRECT,
-    'gauthHost': SSO,
-    'locale': 'en_US',
-    'id': 'gauth-widget',
-    'cssUrl': CSS,
-    'clientId': 'GarminConnect',
-    'rememberMeShown': 'true',
-    'rememberMeChecked': 'false',
-    'createAccountShown': 'true',
-    'openCreateAccount': 'false',
-    'displayNameShown': 'false',
-    'consumeServiceTicket': 'false',
-    'initialFocus': 'true',
-    'embedWidget': 'false',
-    'generateExtraServiceTicket': 'true',
-    'generateTwoExtraServiceTickets': 'false',
-    'generateNoServiceTicket': 'false',
-    'globalOptInShown': 'true',
-    'globalOptInChecked': 'false',
-    'mobile': 'false',
-    'connectLegalTerms': 'true',
-    'locationPromptShown': 'true',
-    'showPassword': 'true',
-}
+GARMIN_BASE_URL = "https://connect.garmin.com"
 
 # URLs for various services.
-
-URL_GC_LOGIN = 'https://sso.garmin.com/sso/signin?' + urlencode(DATA)
-URL_GC_POST_AUTH = 'https://connect.garmin.com/modern/activities?'
-URL_GC_PROFILE = 'https://connect.garmin.com/modern/profile'
-URL_GC_USERSTATS = 'https://connect.garmin.com/modern/proxy/userstats-service/statistics/'
-URL_GC_LIST = 'https://connect.garmin.com/modern/proxy/activitylist-service/activities/search/activities?'
-URL_GC_ACTIVITY = 'https://connect.garmin.com/modern/proxy/activity-service/activity/'
-URL_GC_DEVICE = 'https://connect.garmin.com/modern/proxy/device-service/deviceservice/app-info/'
-URL_GC_GEAR = 'https://connect.garmin.com/modern/proxy/gear-service/gear/filterGear?activityId='
-URL_GC_ACT_PROPS = 'https://connect.garmin.com/modern/main/js/properties/activity_types/activity_types.properties'
-URL_GC_EVT_PROPS = 'https://connect.garmin.com/modern/main/js/properties/event_types/event_types.properties'
-URL_GC_GPX_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/gpx/activity/'
-URL_GC_TCX_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/tcx/activity/'
-URL_GC_ORIGINAL_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/'
+URL_GC_USER = f'{GARMIN_BASE_URL}/userprofile-service/socialProfile'
+URL_GC_USERSTATS = f'{GARMIN_BASE_URL}/userstats-service/statistics/'
+URL_GC_LIST = f'{GARMIN_BASE_URL}/activitylist-service/activities/search/activities?'
+URL_GC_ACTIVITY = f'{GARMIN_BASE_URL}/activity-service/activity/'
+URL_GC_DEVICE = f'{GARMIN_BASE_URL}/device-service/deviceservice/app-info/'
+URL_GC_GEAR = f'{GARMIN_BASE_URL}/gear-service/gear/filterGear?activityId='
+URL_GC_ACT_PROPS = f'{GARMIN_BASE_URL}/modern/main/js/properties/activity_types/activity_types.properties'
+URL_GC_EVT_PROPS = f'{GARMIN_BASE_URL}/modern/main/js/properties/event_types/event_types.properties'
+URL_GC_GPX_ACTIVITY = f'{GARMIN_BASE_URL}/download-service/export/gpx/activity/'
+URL_GC_TCX_ACTIVITY = f'{GARMIN_BASE_URL}/download-service/export/tcx/activity/'
+URL_GC_ORIGINAL_ACTIVITY = f'{GARMIN_BASE_URL}/download-service/files/activity/'
 
 
 class GarminException(Exception):
@@ -232,6 +198,8 @@ def http_req(url, post=None, headers=None):
         'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2816.0 Safari/537.36',
     )
     request.add_header('nk', 'NT')  # necessary since 2021-02-23 to avoid http error code 402
+    request.add_header('authorization', str(garth.client.oauth2_token))
+    request.add_header('di-backend', 'connectapi.garmin.com')
     if headers:
         for header_key, header_value in headers.items():
             request.add_header(header_key, header_value)
@@ -513,51 +481,11 @@ def login_to_garmin_connect(args):
     username = args.username if args.username else input('Username: ')
     password = args.password if args.password else getpass()
 
-    logging.debug("Login params: %s", urlencode(DATA))
-
-    # Initially, we need to get a valid session cookie, so we pull the login page.
-    print('Connecting to Garmin Connect...', end='')
-    logging.info('Connecting to %s', URL_GC_LOGIN)
-    connect_response = http_req_as_string(URL_GC_LOGIN)
-    if args.verbosity > 0:
-        write_to_file(os.path.join(args.directory, 'connect_response.html'), connect_response, 'w')
-    for cookie in COOKIE_JAR:
-        logging.debug("Cookie %s : %s", cookie.name, cookie.value)
-    print(' Done.')
-
-    # Now we'll actually login.
-    # Fields that are passed in a typical Garmin login.
-    post_data = {
-        'username': username,
-        'password': password,
-        'embed': 'false',
-        'rememberme': 'on',
-    }
-
-    headers = {'referer': URL_GC_LOGIN}
-
-    print('Requesting Login ticket...', end='')
-    logging.info('Requesting Login ticket')
-    login_response = http_req_as_string(f'{URL_GC_LOGIN}#', post_data, headers)
-
-    for cookie in COOKIE_JAR:
-        logging.debug("Cookie %s : %s", cookie.name, cookie.value)
-    if args.verbosity > 0:
-        write_to_file(os.path.join(args.directory, 'login_response.html'), login_response, 'w')
-
-    # extract the ticket from the login response
-    pattern = re.compile(r".*\?ticket=([-\w]+)\";.*", re.MULTILINE | re.DOTALL)
-    match = pattern.match(login_response)
-    if not match:
-        raise GarminException(
-            'Couldn\'t find ticket in the login response. Cannot log in. Did you enter the correct username and password?'
-        )
-    login_ticket = match.group(1)
-    print(' Done. Ticket=', login_ticket, sep='')
-
-    print("Authenticating...", end='')
-    logging.info('Authentication URL %s', f'{URL_GC_POST_AUTH}ticket={login_ticket}')
-    http_req(f'{URL_GC_POST_AUTH}ticket={login_ticket}')
+    print('Authenticating using OAuth...', end=' ')
+    try:
+        garth.login(username, password)
+    except Exception as ex:
+        raise GarminException(f'Authentication failure ({ex}). Did you enter correct credentials?') from ex
     print(' Done.')
 
 
@@ -582,7 +510,7 @@ def csv_write_record(csv_filter, extract, actvty, details, activity_type_name, e
 
     # fmt: off
     csv_filter.set_column('id', str(actvty['activityId']))
-    csv_filter.set_column('url', 'https://connect.garmin.com/modern/activity/' + str(actvty['activityId']))
+    csv_filter.set_column('url', f'{GARMIN_BASE_URL}/modern/activity/' + str(actvty['activityId']))
     csv_filter.set_column('activityName', actvty['activityName'] if present('activityName', actvty) else None)
     csv_filter.set_column('description', actvty['description'] if present('description', actvty) else None)
     csv_filter.set_column('startTimeIso', extract['start_time_with_offset'].isoformat())
@@ -944,12 +872,12 @@ def fetch_userstats(args):
     :return:        json with user statistics
     """
     print('Getting display name...', end='')
-    logging.info('Profile page %s', URL_GC_PROFILE)
-    profile_page = http_req_as_string(URL_GC_PROFILE)
+    logging.info('Profile page %s', URL_GC_USER)
+    profile_page = http_req_as_string(URL_GC_USER)
     if args.verbosity > 0:
-        write_to_file(os.path.join(args.directory, 'profile.html'), profile_page, 'w')
+        write_to_file(os.path.join(args.directory, 'user.json'), profile_page, 'w')
 
-    display_name = extract_display_name(profile_page)
+    display_name = json.loads(profile_page)['displayName']
     print(' Done. displayName=', display_name, sep='')
 
     print('Fetching user stats...', end='')
@@ -963,50 +891,28 @@ def fetch_userstats(args):
     return json.loads(result)
 
 
-def extract_display_name(profile_page):
-    """
-    Extract the display name from the profile page HTML document
-    :param profile_page: HTML document
-    :return:             the display name
-    """
-    # the display name should be in the HTML document as
-    # "displayName":"John.Doe"
-    pattern = re.compile(r".*\"displayName\":\"(.+?)\".*", re.MULTILINE | re.DOTALL)
-    match = pattern.match(profile_page)
-    if not match:
-        raise GarminException('Did not find the display name in the profile page.')
-    display_name = match.group(1)
-    return display_name
-
-
-def fetch_activity_list(args, total_to_download):
+def fetch_activity_list(args):
     """
     Fetch the first 'total_to_download' activity summaries; as a side effect save them in json format.
     :param args:              command-line arguments (for args.directory etc)
-    :param total_to_download: number of activities to download
     :return:                  List of activity summaries
     """
 
-    # This while loop will download data from the server in multiple chunks, if necessary.
-    activities = []
-
-    total_downloaded = 0
-    while total_downloaded < total_to_download:
-        # Maximum chunk size 'LIMIT_MAXIMUM' ... 400 return status if over maximum.  So download
-        # maximum or whatever remains if less than maximum.
-        # As of 2018-03-06 I get return status 500 if over maximum
-        if total_to_download - total_downloaded > LIMIT_MAXIMUM:
-            num_to_download = LIMIT_MAXIMUM
-        else:
-            num_to_download = total_to_download - total_downloaded
+    if args.count == 'all':
+        total_to_download = TOTAL_ACTIVITY_LIMIT
+    else:
+        total_to_download = min(int(args.count), TOTAL_ACTIVITY_LIMIT)
 
-        chunk = fetch_activity_chunk(args, num_to_download, total_downloaded)
+    activities = []
+    # This while loop will download data from the server in multiple chunks, if necessary.
+    while len(activities) <= TOTAL_ACTIVITY_LIMIT - SINGLE_REQUEST_LIMIT:
+        num_downloaded = len(activities)
+        num_to_download = min(SINGLE_REQUEST_LIMIT, total_to_download - num_downloaded)
+        chunk = fetch_activity_chunk(args, num_to_download, num_downloaded)
         activities.extend(chunk)
-        total_downloaded += num_to_download
+        if len(chunk) != SINGLE_REQUEST_LIMIT:
+            break
 
-    # it seems that parent multisport activities are not counted in userstats
-    if len(activities) != total_to_download:
-        logging.info('Expected %s activities, got %s.', total_to_download, len(activities))
     return activities
 
 
@@ -1296,18 +1202,10 @@ def main(argv):
 
     login_to_garmin_connect(args)
 
-    # Query the userstats (activities totals on the profile page). Needed for
-    # filtering and for downloading 'all' to know how many activities are available
-    userstats_json = fetch_userstats(args)
-
-    if args.count == 'all':
-        total_to_download = int(userstats_json['userMetrics'][0]['totalActivities'])
-    else:
-        total_to_download = int(args.count)
-
-    device_dict = {}
+    # Get user stats
+    fetch_userstats(args)
 
-    # load some dictionaries with lookup data from REST services
+    # Load some dictionaries with lookup data from REST services
     activity_type_props = http_req_as_string(URL_GC_ACT_PROPS)
     if args.verbosity > 0:
         write_to_file(os.path.join(args.directory, 'activity_types.properties'), activity_type_props, 'w')
@@ -1317,12 +1215,13 @@ def main(argv):
         write_to_file(os.path.join(args.directory, 'event_types.properties'), event_type_props, 'w')
     event_type_name = load_properties(event_type_props)
 
-    activities = fetch_activity_list(args, total_to_download)
+    activities = fetch_activity_list(args)
     action_list = annotate_activity_list(activities, args.start_activity_no, exclude_list)
 
     csv_filename = os.path.join(args.directory, 'activities.csv')
     csv_existed = os.path.isfile(csv_filename)
 
+    device_dict = {}
     with open(csv_filename, mode='a', encoding='utf-8') as csv_file:
         csv_filter = CsvFilter(csv_file, args.template)
 

requirements.txt

@@ -0,0 +1 @@
+garth==0.4.28