From 2379ff20f69b8542f67d2578f61a2eb0e31ddb15 Mon Sep 17 00:00:00 2001 From: Jakub Tymejczyk Date: Thu, 5 Oct 2023 21:13:02 +0200 Subject: [PATCH 1/4] Add support for shared library Based on https://github.com/icloud-photos-downloader/icloud_photos_downloader/pull/678 --- icloudpy/services/photos.py | 215 ++++++++++++++++++++++-------------- 1 file changed, 133 insertions(+), 82 deletions(-) diff --git a/icloudpy/services/photos.py b/icloudpy/services/photos.py index 5e9c14b..30aa66f 100644 --- a/icloudpy/services/photos.py +++ b/icloudpy/services/photos.py @@ -14,9 +14,11 @@ # fmt: on -class PhotosService: - """The 'Photos' iCloud service.""" +class PhotoLibrary(object): + """Represents a library in the user's photos. + This provides access to all the albums as well as the photos. + """ SMART_FOLDERS = { "All Photos": { "obj_type": "CPLAssetByAddedDate", @@ -128,110 +130,152 @@ class PhotosService: }, } - def __init__(self, service_root, session, params): - self.session = session - self.params = dict(params) - self._service_root = service_root - self.service_endpoint = ( - f"{self._service_root}/database/1/com.apple.photos.cloud/production/private" - ) + def __init__(self, service, zone_id): + self.service = service + self.zone_id = zone_id self._albums = None - self.params.update({"remapEnums": True, "getCurrentSyncToken": True}) + url = ('%s/records/query?%s' % + (self.service._service_endpoint, urlencode(self.service.params))) + json_data = json.dumps({ + "query": {"recordType":"CheckIndexingState"}, + "zoneID": self.zone_id, + }) - url = f"{self.service_endpoint}/records/query?{urlencode(self.params)}" - json_data = ( - '{"query":{"recordType":"CheckIndexingState"},' - '"zoneID":{"zoneName":"PrimarySync"}}' - ) - request = self.session.post( - url, data=json_data, headers={"Content-type": "text/plain"} + request = self.service.session.post( + url, + data=json_data, + headers={'Content-type': 'text/plain'} ) response = request.json() - indexing_state = response["records"][0]["fields"]["state"]["value"] - if indexing_state != "FINISHED": - raise ICloudPyServiceNotActivatedException( - "iCloud Photo Library not finished indexing. " - "Please try again in a few minutes." - ) - - # TODO: Does syncToken ever change? # pylint: disable=fixme - # self.params.update({ - # 'syncToken': response['syncToken'], - # 'clientInstanceId': self.params.pop('clientId') - # }) - - self._photo_assets = {} + indexing_state = response['records'][0]['fields']['state']['value'] + if indexing_state != 'FINISHED': + raise PyiCloudServiceNotActivatedErrror( + ('iCloud Photo Library not finished indexing. Please try ' + 'again in a few minutes'), None) @property def albums(self): - """Returns photo albums.""" if not self._albums: - self._albums = {} + self._albums = { + name: PhotoAlbum(self.service, name, zone_id=self.zone_id, **props) + for (name, props) in self.SMART_FOLDERS.items() + } for folder in self._fetch_folders(): - - # Skipping albums having null name, that can happen sometime - if "albumNameEnc" not in folder["fields"]: + # FIXME: Handle subfolders + if folder['recordName'] in ('----Root-Folder----', + '----Project-Root-Folder----') or \ + (folder['fields'].get('isDeleted') and + folder['fields']['isDeleted']['value']): continue - if folder["recordName"] == "----Root-Folder----" or ( - folder["fields"].get("isDeleted") - and folder["fields"]["isDeleted"]["value"] - ): - continue - - folder_id = folder["recordName"] - folder_obj_type = ( - f"CPLContainerRelationNotDeletedByAssetDate:{folder_id}" - ) + folder_id = folder['recordName'] + folder_obj_type = \ + "CPLContainerRelationNotDeletedByAssetDate:%s" % folder_id folder_name = base64.b64decode( - folder["fields"]["albumNameEnc"]["value"] - ).decode("utf-8") - - query_filter = [ - { - "fieldName": "parentId", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": folder_id}, + folder['fields']['albumNameEnc']['value']).decode('utf-8') + query_filter = [{ + "fieldName": "parentId", + "comparator": "EQUALS", + "fieldValue": { + "type": "STRING", + "value": folder_id } - ] + }] - album = PhotoAlbum( - self, - name=folder_name, - list_type="CPLContainerRelationLiveByAssetDate", - obj_type=folder_obj_type, - direction="ASCENDING", - query_filter=query_filter, - folder_id=folder_id, - ) + album = PhotoAlbum(self.service, folder_name, + 'CPLContainerRelationLiveByAssetDate', + folder_obj_type, 'ASCENDING', query_filter, + zone_id=self.zone_id) self._albums[folder_name] = album - for (name, props) in self.SMART_FOLDERS.items(): - self._albums[name] = PhotoAlbum(self, name, **props) - return self._albums def _fetch_folders(self): - url = f"{self.service_endpoint}/records/query?{urlencode(self.params)}" - json_data = ( - '{"query":{"recordType":"CPLAlbumByPositionLive"},' - '"zoneID":{"zoneName":"PrimarySync"}}' - ) + url = ('%s/records/query?%s' % + (self.service._service_endpoint, urlencode(self.service.params))) + json_data = json.dumps({ + "query": {"recordType":"CPLAlbumByPositionLive"}, + "zoneID": self.zone_id, + }) - request = self.session.post( - url, data=json_data, headers={"Content-type": "text/plain"} + request = self.service.session.post( + url, + data=json_data, + headers={'Content-type': 'text/plain'} ) response = request.json() - return response["records"] + return response['records'] @property def all(self): - """Returns all photos.""" - return self.albums["All Photos"] + return self.albums['All Photos'] + + +class PhotosService(PhotoLibrary): + """The 'Photos' iCloud service. + + This also acts as a way to access the user's primary library. + """ + def __init__(self, service_root, session, params): + self.session = session + self.params = dict(params) + self._service_root = service_root + self._service_endpoint = \ + ('%s/database/1/com.apple.photos.cloud/production/private' + % self._service_root) + + self._libraries = None + + self.params.update({ + 'remapEnums': True, + 'getCurrentSyncToken': True + }) + + # TODO: Does syncToken ever change? + # self.params.update({ + # 'syncToken': response['syncToken'], + # 'clientInstanceId': self.params.pop('clientId') + # }) + + self._photo_assets = {} + + super(PhotosService, self).__init__( + service=self, zone_id={u'zoneName': u'PrimarySync'}) + + @property + def libraries(self): + if not self._libraries: + try: + url = ('%s/changes/database' % + (self._service_endpoint, )) + request = self.session.post( + url, + data='{}', + headers={'Content-type': 'text/plain'} + ) + response = request.json() + zones = response['zones'] + except Exception as e: + logger.error("library exception: %s" % str(e)) + + libraries = {} + for zone in zones: + if not zone.get('deleted'): + zone_name = zone['zoneID']['zoneName'] + libraries[zone_name] = PhotoLibrary( + self, zone_id=zone['zoneID']) + # obj_type='CPLAssetByAssetDateWithoutHiddenOrDeleted', + # list_type="CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted", + # direction="ASCENDING", query_filter=None, + # zone_id=zone['zoneID']) + + self._libraries = libraries + + return self._libraries class PhotoAlbum: @@ -247,6 +291,7 @@ def __init__( query_filter=None, page_size=100, folder_id=None, + zone_id=None, ): self.name = name self.service = service @@ -257,6 +302,11 @@ def __init__( self.page_size = page_size self.folder_id = folder_id + if zone_id: + self._zone_id = zone_id + else: + self._zone_id = 'PrimarySync' + self._len = None self._subalbums = {} @@ -291,7 +341,7 @@ def __len__(self): "recordType": "HyperionIndexCountLookup", }, "zoneWide": True, - "zoneID": {"zoneName": "PrimarySync"}, + "zoneID": {"zoneName": self._zone_id}, } ] } @@ -326,10 +376,11 @@ def _fetch_subalbums(self): ] }}, "zoneID": {{ - "zoneName":"PrimarySync" + "zoneName":"{}" }} }}""".format( - self.folder_id + self.folder_id, + self._zone_id ) json_data = query request = self.service.session.post( @@ -388,7 +439,7 @@ def photos(self): offset = 0 while True: - url = (f"{self.service.service_endpoint}/records/query?") + urlencode( + url = (f"{self.service._service_endpoint}/records/query?") + urlencode( self.service.params ) request = self.service.session.post( @@ -543,7 +594,7 @@ def _list_query_gen(self, offset, list_type, direction, query_filter=None): "position", "isKeyAsset", ], - "zoneID": {"zoneName": "PrimarySync"}, + "zoneID": self._zone_id, } if query_filter: From 4eb82e0b1494d9be305e4f9f36bfb6e38e87d4be Mon Sep 17 00:00:00 2001 From: Jakub Tymejczyk Date: Sat, 7 Oct 2023 14:21:29 +0200 Subject: [PATCH 2/4] fix support for subalbums --- icloudpy/services/photos.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/icloudpy/services/photos.py b/icloudpy/services/photos.py index 30aa66f..9cf3314 100644 --- a/icloudpy/services/photos.py +++ b/icloudpy/services/photos.py @@ -188,7 +188,7 @@ def albums(self): album = PhotoAlbum(self.service, folder_name, 'CPLContainerRelationLiveByAssetDate', folder_obj_type, 'ASCENDING', query_filter, - zone_id=self.zone_id) + folder_id=folder_id, zone_id=self.zone_id) self._albums[folder_name] = album return self._albums @@ -305,7 +305,7 @@ def __init__( if zone_id: self._zone_id = zone_id else: - self._zone_id = 'PrimarySync' + self._zone_id = "PrimarySync" self._len = None @@ -357,7 +357,7 @@ def __len__(self): return self._len def _fetch_subalbums(self): - url = (f"{self.service.service_endpoint}/records/query?") + urlencode( + url = (f"{self.service._service_endpoint}/records/query?") + urlencode( self.service.params ) # pylint: disable=consider-using-f-string @@ -380,7 +380,7 @@ def _fetch_subalbums(self): }} }}""".format( self.folder_id, - self._zone_id + self._zone_id["zoneName"] ) json_data = query request = self.service.session.post( @@ -426,6 +426,7 @@ def subalbums(self): direction="ASCENDING", query_filter=query_filter, folder_id=folder_id, + zone_id=self._zone_id ) self._subalbums[folder_name] = album return self._subalbums From 32ca775f1a5d778473987286efbce3133b458ee6 Mon Sep 17 00:00:00 2001 From: Mandar Patil Date: Mon, 9 Oct 2023 07:47:25 -0700 Subject: [PATCH 3/4] Linting and other fixes --- icloudpy/services/photos.py | 153 +++++++++++++++++++----------------- pylintrc | 3 + 2 files changed, 83 insertions(+), 73 deletions(-) diff --git a/icloudpy/services/photos.py b/icloudpy/services/photos.py index 9cf3314..697d5cc 100644 --- a/icloudpy/services/photos.py +++ b/icloudpy/services/photos.py @@ -1,6 +1,7 @@ """Photo service.""" import base64 import json +import logging from datetime import datetime # fmt: off @@ -9,16 +10,18 @@ from pytz import UTC from six import PY2 +# fmt: on from icloudpy.exceptions import ICloudPyServiceNotActivatedException -# fmt: on +LOGGER = logging.getLogger(__name__) -class PhotoLibrary(object): +class PhotoLibrary: """Represents a library in the user's photos. This provides access to all the albums as well as the photos. """ + SMART_FOLDERS = { "All Photos": { "obj_type": "CPLAssetByAddedDate", @@ -136,24 +139,27 @@ def __init__(self, service, zone_id): self._albums = None - url = ('%s/records/query?%s' % - (self.service._service_endpoint, urlencode(self.service.params))) - json_data = json.dumps({ - "query": {"recordType":"CheckIndexingState"}, - "zoneID": self.zone_id, - }) + url = f"{self.service._service_endpoint}/records/query?{urlencode(self.service.params)}" + json_data = json.dumps( + { + "query": {"recordType": "CheckIndexingState"}, + "zoneID": self.zone_id, + } + ) request = self.service.session.post( - url, - data=json_data, - headers={'Content-type': 'text/plain'} + url, data=json_data, headers={"Content-type": "text/plain"} ) response = request.json() - indexing_state = response['records'][0]['fields']['state']['value'] - if indexing_state != 'FINISHED': - raise PyiCloudServiceNotActivatedErrror( - ('iCloud Photo Library not finished indexing. Please try ' - 'again in a few minutes'), None) + indexing_state = response["records"][0]["fields"]["state"]["value"] + if indexing_state != "FINISHED": + raise ICloudPyServiceNotActivatedException( + ( + "iCloud Photo Library not finished indexing. Please try " + "again in a few minutes" + ), + None, + ) @property def albums(self): @@ -165,54 +171,63 @@ def albums(self): for folder in self._fetch_folders(): # FIXME: Handle subfolders - if folder['recordName'] in ('----Root-Folder----', - '----Project-Root-Folder----') or \ - (folder['fields'].get('isDeleted') and - folder['fields']['isDeleted']['value']): + if folder["recordName"] in ( + "----Root-Folder----", + "----Project-Root-Folder----", + ) or ( + folder["fields"].get("isDeleted") + and folder["fields"]["isDeleted"]["value"] + ): continue - folder_id = folder['recordName'] - folder_obj_type = \ - "CPLContainerRelationNotDeletedByAssetDate:%s" % folder_id + folder_id = folder["recordName"] + folder_obj_type = ( + f"CPLContainerRelationNotDeletedByAssetDate:{folder_id}" + ) folder_name = base64.b64decode( - folder['fields']['albumNameEnc']['value']).decode('utf-8') - query_filter = [{ - "fieldName": "parentId", - "comparator": "EQUALS", - "fieldValue": { - "type": "STRING", - "value": folder_id + folder["fields"]["albumNameEnc"]["value"] + ).decode("utf-8") + query_filter = [ + { + "fieldName": "parentId", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": folder_id}, } - }] + ] - album = PhotoAlbum(self.service, folder_name, - 'CPLContainerRelationLiveByAssetDate', - folder_obj_type, 'ASCENDING', query_filter, - folder_id=folder_id, zone_id=self.zone_id) + album = PhotoAlbum( + self.service, + folder_name, + "CPLContainerRelationLiveByAssetDate", + folder_obj_type, + "ASCENDING", + query_filter, + folder_id=folder_id, + zone_id=self.zone_id, + ) self._albums[folder_name] = album return self._albums def _fetch_folders(self): - url = ('%s/records/query?%s' % - (self.service._service_endpoint, urlencode(self.service.params))) - json_data = json.dumps({ - "query": {"recordType":"CPLAlbumByPositionLive"}, - "zoneID": self.zone_id, - }) + url = f"{self.service._service_endpoint}/records/query?{urlencode(self.service.params)}" + json_data = json.dumps( + { + "query": {"recordType": "CPLAlbumByPositionLive"}, + "zoneID": self.zone_id, + } + ) request = self.service.session.post( - url, - data=json_data, - headers={'Content-type': 'text/plain'} + url, data=json_data, headers={"Content-type": "text/plain"} ) response = request.json() - return response['records'] + return response["records"] @property def all(self): - return self.albums['All Photos'] + return self.albums["All Photos"] class PhotosService(PhotoLibrary): @@ -220,20 +235,18 @@ class PhotosService(PhotoLibrary): This also acts as a way to access the user's primary library. """ + def __init__(self, service_root, session, params): self.session = session self.params = dict(params) self._service_root = service_root - self._service_endpoint = \ - ('%s/database/1/com.apple.photos.cloud/production/private' - % self._service_root) + self._service_endpoint = ( + f"{self._service_root}/database/1/com.apple.photos.cloud/production/private" + ) self._libraries = None - self.params.update({ - 'remapEnums': True, - 'getCurrentSyncToken': True - }) + self.params.update({"remapEnums": True, "getCurrentSyncToken": True}) # TODO: Does syncToken ever change? # self.params.update({ @@ -243,35 +256,30 @@ def __init__(self, service_root, session, params): self._photo_assets = {} - super(PhotosService, self).__init__( - service=self, zone_id={u'zoneName': u'PrimarySync'}) + super().__init__(service=self, zone_id={"zoneName": "PrimarySync"}) @property def libraries(self): if not self._libraries: try: - url = ('%s/changes/database' % - (self._service_endpoint, )) + url = f"{self._service_endpoint}/changes/database" request = self.session.post( - url, - data='{}', - headers={'Content-type': 'text/plain'} + url, data="{}", headers={"Content-type": "text/plain"} ) response = request.json() - zones = response['zones'] + zones = response["zones"] except Exception as e: - logger.error("library exception: %s" % str(e)) + LOGGER.error(f"library exception: {str(e)}") libraries = {} for zone in zones: - if not zone.get('deleted'): - zone_name = zone['zoneID']['zoneName'] - libraries[zone_name] = PhotoLibrary( - self, zone_id=zone['zoneID']) - # obj_type='CPLAssetByAssetDateWithoutHiddenOrDeleted', - # list_type="CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted", - # direction="ASCENDING", query_filter=None, - # zone_id=zone['zoneID']) + if not zone.get("deleted"): + zone_name = zone["zoneID"]["zoneName"] + libraries[zone_name] = PhotoLibrary(self, zone_id=zone["zoneID"]) + # obj_type='CPLAssetByAssetDateWithoutHiddenOrDeleted', + # list_type="CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted", + # direction="ASCENDING", query_filter=None, + # zone_id=zone['zoneID']) self._libraries = libraries @@ -379,8 +387,7 @@ def _fetch_subalbums(self): "zoneName":"{}" }} }}""".format( - self.folder_id, - self._zone_id["zoneName"] + self.folder_id, self._zone_id["zoneName"] ) json_data = query request = self.service.session.post( @@ -426,7 +433,7 @@ def subalbums(self): direction="ASCENDING", query_filter=query_filter, folder_id=folder_id, - zone_id=self._zone_id + zone_id=self._zone_id, ) self._subalbums[folder_name] = album return self._subalbums diff --git a/pylintrc b/pylintrc index 18e12d4..f38bd9d 100644 --- a/pylintrc +++ b/pylintrc @@ -150,6 +150,9 @@ disable=abstract-method, wrong-import-order, xrange-builtin, zip-builtin-not-iterating, + protected-access, + logging-not-lazy, + logging-fstring-interpolation [REPORTS] From bd7cf1344db3a57e1b0eac29a4aba1dab984b307 Mon Sep 17 00:00:00 2001 From: Mandar Patil Date: Wed, 11 Oct 2023 19:34:03 -0700 Subject: [PATCH 4/4] Updated endpoint --- icloudpy/services/photos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icloudpy/services/photos.py b/icloudpy/services/photos.py index 697d5cc..69a20d2 100644 --- a/icloudpy/services/photos.py +++ b/icloudpy/services/photos.py @@ -262,7 +262,7 @@ def __init__(self, service_root, session, params): def libraries(self): if not self._libraries: try: - url = f"{self._service_endpoint}/changes/database" + url = f"{self._service_endpoint}/zones/list" request = self.session.post( url, data="{}", headers={"Content-type": "text/plain"} )