Library and filesystem providers optimizations (#1234)

music_assistant/common/models/media_items.py

@@ -213,7 +213,9 @@ def update(
             if new_val is None:
                 continue
             cur_val = getattr(self, fld.name)
-            if isinstance(cur_val, list) and isinstance(new_val, list):
+            if allow_overwrite and new_val:
+                setattr(self, fld.name, new_val)
+            elif isinstance(cur_val, list) and isinstance(new_val, list):
                 new_val = merge_lists(cur_val, new_val)
                 setattr(self, fld.name, new_val)
             elif isinstance(cur_val, set) and isinstance(new_val, list):
@@ -330,11 +332,21 @@ class ItemMapping(_MediaItemBase):
     """Representation of a minimized item object."""
 
     available: bool = True
+    image: MediaItemImage | None = None
 
     @classmethod
     def from_item(cls, item: MediaItem) -> ItemMapping:
         """Create ItemMapping object from regular item."""
-        return cls.from_dict(item.to_dict())
+        if isinstance(item, ItemMapping):
+            return item
+        thumb_image = None
+        if item.metadata and item.metadata.images:
+            for img in item.metadata.images:
+                if img.type != ImageType.THUMB:
+                    continue
+                thumb_image = img
+                break
+        return cls.from_dict({**item.to_dict(), "image": thumb_image.to_dict()})
 
 
 @dataclass(kw_only=True)
@@ -364,6 +376,9 @@ class Track(MediaItem):
     version: str = ""
     artists: list[Artist | ItemMapping] = field(default_factory=list)
     album: Album | ItemMapping | None = None  # optional
+    disc_number: int | None = None  # required for album tracks
+    track_number: int | None = None  # required for album tracks
+    position: int | None = None  # required for playlist tracks
 
     def __hash__(self):
         """Return custom hash."""
@@ -394,18 +409,43 @@ def artist_str(self) -> str:
 
 @dataclass(kw_only=True)
 class AlbumTrack(Track):
-    """Model for a track on an album."""
+    """
+    Model for a track on an album.
+
+    Same as regular Track but with explicit and required definitions of
+    album, disc_number and track_number
+    """
 
-    album: Album | ItemMapping  # required
-    disc_number: int = 0
-    track_number: int = 0
+    album: Album
+    disc_number: int
+    track_number: int
+
+    @classmethod
+    def from_track(cls: Self, track: Track, album: Album | None = None) -> Self:
+        """Cast Track to AlbumTrack."""
+        if album:
+            track.album = album
+        assert isinstance(track.album, Album)
+        assert track.disc_number is not None
+        assert track.track_number is not None
+        return cast(AlbumTrack, track)
 
 
 @dataclass(kw_only=True)
 class PlaylistTrack(Track):
-    """Model for a track on a playlist."""
+    """
+    Model for a track on a playlist.
 
-    position: int  # required
+    Same as regular Track but with explicit and required definition of position.
+    """
+
+    position: int
+
+    @classmethod
+    def from_track(cls: Self, track: Track) -> Self:
+        """Cast Track to PlaylistTrack."""
+        assert track.position is not None
+        return cast(AlbumTrack, track)
 
 
 @dataclass(kw_only=True)

music_assistant/constants.py

@@ -5,7 +5,7 @@
 
 API_SCHEMA_VERSION: Final[int] = 24
 MIN_SCHEMA_VERSION: Final[int] = 24
-DB_SCHEMA_VERSION: Final[int] = 29
+DB_SCHEMA_VERSION: Final[int] = 30
 
 MASS_LOGGER_NAME: Final[str] = "music_assistant"
 
@@ -76,6 +76,8 @@
 DB_TABLE_ALBUMS: Final[str] = "albums"
 DB_TABLE_TRACKS: Final[str] = "tracks"
 DB_TABLE_ALBUM_TRACKS: Final[str] = "albumtracks"
+DB_TABLE_TRACK_ARTISTS: Final[str] = "trackartists"
+DB_TABLE_ALBUM_ARTISTS: Final[str] = "albumartists"
 DB_TABLE_PLAYLISTS: Final[str] = "playlists"
 DB_TABLE_RADIOS: Final[str] = "radios"
 DB_TABLE_CACHE: Final[str] = "cache"

music_assistant/server/controllers/media/albums.py

@@ -20,11 +20,19 @@
     Album,
     AlbumTrack,
     AlbumType,
+    Artist,
     ItemMapping,
     MediaType,
     Track,
 )
-from music_assistant.constants import DB_TABLE_ALBUM_TRACKS, DB_TABLE_ALBUMS, DB_TABLE_TRACKS
+from music_assistant.constants import (
+    DB_TABLE_ALBUM_ARTISTS,
+    DB_TABLE_ALBUM_TRACKS,
+    DB_TABLE_ALBUMS,
+    DB_TABLE_ARTISTS,
+    DB_TABLE_PROVIDER_MAPPINGS,
+    DB_TABLE_TRACKS,
+)
 from music_assistant.server.controllers.media.base import MediaControllerBase
 from music_assistant.server.helpers.compare import (
     compare_album,
@@ -47,6 +55,35 @@ def __init__(self, *args, **kwargs) -> None:
         """Initialize class."""
         super().__init__(*args, **kwargs)
         self._db_add_lock = asyncio.Lock()
+        self.base_query = f"""
+                SELECT
+                    {self.db_table}.*,
+                    {DB_TABLE_ARTISTS}.sort_name AS sort_artist,
+                    json_group_array(
+                        json_object(
+                            'item_id', {DB_TABLE_PROVIDER_MAPPINGS}.provider_item_id,
+                            'provider_domain', {DB_TABLE_PROVIDER_MAPPINGS}.provider_domain,
+                            'provider_instance', {DB_TABLE_PROVIDER_MAPPINGS}.provider_instance,
+                            'available', {DB_TABLE_PROVIDER_MAPPINGS}.available,
+                            'url', {DB_TABLE_PROVIDER_MAPPINGS}.url,
+                            'audio_format', json({DB_TABLE_PROVIDER_MAPPINGS}.audio_format),
+                            'details', {DB_TABLE_PROVIDER_MAPPINGS}.details
+                        )) filter ( where {DB_TABLE_PROVIDER_MAPPINGS}.item_id is not null) as {DB_TABLE_PROVIDER_MAPPINGS},
+                    json_group_array(
+                        json_object(
+                            'item_id', {DB_TABLE_ARTISTS}.item_id,
+                            'provider', 'library',
+                            'name', {DB_TABLE_ARTISTS}.name,
+                            'sort_name', {DB_TABLE_ARTISTS}.sort_name,
+                            'media_type', 'artist'
+                        )) filter ( where {DB_TABLE_ARTISTS}.name is not null) as {DB_TABLE_ARTISTS}
+                FROM {self.db_table}
+                LEFT JOIN {DB_TABLE_ALBUM_ARTISTS} on {DB_TABLE_ALBUM_ARTISTS}.album_id = {self.db_table}.item_id
+                LEFT JOIN {DB_TABLE_ARTISTS} on {DB_TABLE_ARTISTS}.item_id = {DB_TABLE_ALBUM_ARTISTS}.artist_id
+                LEFT JOIN {DB_TABLE_PROVIDER_MAPPINGS}
+                    ON {self.db_table}.item_id = {DB_TABLE_PROVIDER_MAPPINGS}.item_id
+                    AND {DB_TABLE_PROVIDER_MAPPINGS}.media_type == '{self.media_type.value}'
+        """  # noqa: E501
         # register api handlers
         self.mass.register_api_command("music/albums/library_items", self.library_items)
         self.mass.register_api_command(
@@ -102,6 +139,7 @@ async def add_item_to_library(
         self,
         item: Album,
         metadata_lookup: bool = True,
+        overwrite_existing: bool = False,
         add_album_tracks: bool = False,
     ) -> Album:
         """Add album to library and return the database item."""
@@ -118,16 +156,22 @@ async def add_item_to_library(
         library_item = None
         if cur_item := await self.get_library_item_by_prov_id(item.item_id, item.provider):
             # existing item match by provider id
-            library_item = await self.update_item_in_library(cur_item.item_id, item)
+            library_item = await self.update_item_in_library(
+                cur_item.item_id, item, overwrite=overwrite_existing
+            )
         elif cur_item := await self.get_library_item_by_external_ids(item.external_ids):
             # existing item match by external id
-            library_item = await self.update_item_in_library(cur_item.item_id, item)
+            library_item = await self.update_item_in_library(
+                cur_item.item_id, item, overwrite=overwrite_existing
+            )
         else:
             # search by name
             async for db_item in self.iter_library_items(search=item.name):
                 if compare_album(db_item, item):
                     # existing item found: update it
-                    library_item = await self.update_item_in_library(db_item.item_id, item)
+                    library_item = await self.update_item_in_library(
+                        db_item.item_id, item, overwrite=overwrite_existing
+                    )
                     break
         if not library_item:
             # actually add a new item in the library db
@@ -139,6 +183,7 @@ async def add_item_to_library(
             await self._match(library_item)
             library_item = await self.get_library_item(library_item.item_id)
         # also add album tracks
+        # TODO: make this configurable
         if add_album_tracks and item.provider != "library":
             async with asyncio.TaskGroup() as tg:
                 for track in await self._get_provider_album_tracks(item.item_id, item.provider):
@@ -159,36 +204,35 @@ async def update_item_in_library(
         """Update existing record in the database."""
         db_id = int(item_id)  # ensure integer
         cur_item = await self.get_library_item(db_id)
-        metadata = cur_item.metadata.update(getattr(update, "metadata", None), overwrite)
-        provider_mappings = self._get_provider_mappings(cur_item, update, overwrite)
-        album_artists = await self._get_artist_mappings(cur_item, update, overwrite)
+        metadata = cur_item.metadata.update(update.metadata, overwrite)
         if getattr(update, "album_type", AlbumType.UNKNOWN) != AlbumType.UNKNOWN:
             album_type = update.album_type
         else:
             album_type = cur_item.album_type
-        sort_artist = album_artists[0].sort_name
         cur_item.external_ids.update(update.external_ids)
         await self.mass.music.database.update(
             self.db_table,
             {"item_id": db_id},
             {
                 "name": update.name if overwrite else cur_item.name,
-                "sort_name": update.sort_name if overwrite else cur_item.sort_name,
-                "sort_artist": sort_artist,
+                "sort_name": update.sort_name
+                if overwrite
+                else cur_item.sort_name or update.sort_name,
                 "version": update.version if overwrite else cur_item.version,
                 "year": update.year if overwrite else cur_item.year or update.year,
                 "album_type": album_type.value,
-                "artists": serialize_to_json(album_artists),
                 "metadata": serialize_to_json(metadata),
-                "provider_mappings": serialize_to_json(provider_mappings),
                 "external_ids": serialize_to_json(
                     update.external_ids if overwrite else cur_item.external_ids
                 ),
                 "timestamp_modified": int(utc_timestamp()),
             },
         )
         # update/set provider_mappings table
-        await self._set_provider_mappings(db_id, provider_mappings)
+        await self._set_provider_mappings(db_id, update.provider_mappings, overwrite=overwrite)
+        # set album artist(s)
+        await self._set_album_artists(db_id, update.artists, overwrite=overwrite)
+
         self.logger.debug("updated %s in database: %s", update.name, db_id)
         # get full created object
         library_item = await self.get_library_item(db_id)
@@ -250,8 +294,6 @@ async def versions(
 
     async def _add_library_item(self, item: Album) -> Album:
         """Add a new record to the database."""
-        album_artists = await self._get_artist_mappings(item)
-        sort_artist = album_artists[0].sort_name if album_artists else ""
         new_item = await self.mass.music.database.insert(
             self.db_table,
             {
@@ -262,9 +304,6 @@ async def _add_library_item(self, item: Album) -> Album:
                 "album_type": item.album_type,
                 "year": item.year,
                 "metadata": serialize_to_json(item.metadata),
-                "provider_mappings": serialize_to_json(item.provider_mappings),
-                "artists": serialize_to_json(album_artists),
-                "sort_artist": sort_artist,
                 "external_ids": serialize_to_json(item.external_ids),
                 "timestamp_added": int(utc_timestamp()),
                 "timestamp_modified": int(utc_timestamp()),
@@ -273,6 +312,8 @@ async def _add_library_item(self, item: Album) -> Album:
         db_id = new_item["item_id"]
         # update/set provider_mappings table
         await self._set_provider_mappings(db_id, item.provider_mappings)
+        # set album artist(s)
+        await self._set_album_artists(db_id, item.artists)
         self.logger.debug("added %s to database", item.name)
         # return the full item we just added
         return await self.get_library_item(db_id)
@@ -356,20 +397,15 @@ async def _get_db_album_tracks(
         """Return in-database album tracks for the given database album."""
         db_id = int(item_id)  # ensure integer
         db_album = await self.get_library_item(db_id)
-        result: list[AlbumTrack] = []
-        query = (
-            f"SELECT * FROM {DB_TABLE_TRACKS} INNER JOIN albumtracks "
-            "ON albumtracks.track_id = tracks.item_id WHERE albumtracks.album_id = :album_id"
+        subquery = f"SELECT track_id FROM {DB_TABLE_ALBUM_TRACKS} WHERE album_id = {item_id}"
+        query = f"WHERE {DB_TABLE_TRACKS}.item_id in ({subquery})"
+        return sorted(
+            [
+                AlbumTrack.from_track(track, db_album)
+                async for track in self.mass.music.tracks.iter_library_items(extra_query=query)
+            ],
+            key=lambda x: (x.disc_number, x.track_number),
         )
-        track_rows = await self.mass.music.database.get_rows_from_query(query, {"album_id": db_id})
-        for album_track_row in track_rows:
-            album_track = AlbumTrack.from_dict(
-                self._parse_db_row({**album_track_row, "album": db_album.to_dict()})
-            )
-            if db_album.metadata.images:
-                album_track.metadata.images = db_album.metadata.images
-            result.append(album_track)
-        return sorted(result, key=lambda x: (x.disc_number, x.track_number))
 
     async def _match(self, db_album: Album) -> None:
         """Try to find match on all (streaming) providers for the provided (database) album.
@@ -431,3 +467,57 @@ async def find_prov_match(provider: MusicProvider):
                     db_album.name,
                     provider.name,
                 )
+
+    async def _set_album_artists(
+        self, db_id: int, artists: list[Artist | ItemMapping], overwrite: bool = False
+    ) -> None:
+        """Store Album Artists."""
+        if overwrite:
+            # on overwrite, clear the album_artists table first
+            await self.mass.music.database.delete(
+                DB_TABLE_ALBUM_ARTISTS,
+                {
+                    "album_id": db_id,
+                },
+            )
+        for artist in artists:
+            await self._set_album_artist(db_id, artist=artist, overwrite=overwrite)
+
+    async def _set_album_artist(
+        self, db_id: int, artist: Artist | ItemMapping, overwrite: bool = False
+    ) -> None:
+        """Store Album Artist info."""
+        db_artist: Album | ItemMapping = None
+        if artist.provider == "library":
+            db_artist = artist
+        elif existing := await self.mass.music.artists.get_library_item_by_prov_id(
+            artist.item_id, artist.provider
+        ):
+            db_artist = existing
+        else:
+            # not an existing artist, we need to fetch before we can add it to the library
+            if isinstance(artist, ItemMapping):
+                artist = await self.mass.music.artists.get_provider_item(
+                    artist.item_id, artist.provider, fallback=artist
+                )
+            with contextlib.suppress(MediaNotFoundError, AssertionError, InvalidDataError):
+                db_artist = await self.mass.music.artists.add_item_to_library(
+                    artist, metadata_lookup=False, overwrite_existing=overwrite
+                )
+        if not db_artist:
+            # this should not happen but streaming providers can be awful sometimes
+            self.logger.warning(
+                "Unable to resolve Artist %s for album %s, "
+                "album will be added to the library without this artist!",
+                artist.uri,
+                db_id,
+            )
+            return
+        # write (or update) record in album_artists table
+        await self.mass.music.database.insert_or_replace(
+            DB_TABLE_ALBUM_ARTISTS,
+            {
+                "album_id": db_id,
+                "artist_id": int(db_artist.item_id),
+            },
+        )