rewrite db stuff with scalars and generators

+ dump mixes with less than x=4 artists
+ try: disable pragma mmap_size

app/api/album.py

@@ -92,7 +92,6 @@ def get_album_tracks_and_info(body: GetAlbumInfoBody):
         og_album_title=album.og_title,
     )
 
-
     more_from_albums = get_more_from_artist(more_from_data)
     other_versions = get_album_versions(other_versions_data)
 
@@ -217,6 +216,9 @@ def get_similar_albums(query: GetSimilarAlbumsQuery):
         return []
 
     artisthashes = similar_artists.get_artist_hash_set()
+
+    del similar_artists
+
     artists = ArtistStore.get_artists_by_hashes(artisthashes)
     albums = AlbumStore.get_albums_by_artisthashes([a.artisthash for a in artists])
     sample = random.sample(albums, min(len(albums), limit))

app/api/pages.py

@@ -56,7 +56,7 @@ def get_pages():
     """
     Get all pages.
     """
-    return PageTable.get_all()
+    return [page for page in PageTable.get_all()]
 
 
 class AddPageItemBody(BaseModel):

app/api/playlist.py

@@ -101,6 +101,11 @@ def send_all_playlists(query: SendAllPlaylistsQuery):
     Gets all the playlists.
     """
     playlists = PlaylistTable.get_all()
+    playlists = sorted(
+        playlists,
+        key=lambda p: datetime.strptime(p.last_updated, "%Y-%m-%d %H:%M:%S"),
+        reverse=True,
+    )
 
     for playlist in playlists:
         if not playlist.has_image:
@@ -110,10 +115,10 @@ def send_all_playlists(query: SendAllPlaylistsQuery):
 
         playlist.clear_lists()
 
-    playlists.sort(
-        key=lambda p: datetime.strptime(p.last_updated, "%Y-%m-%d %H:%M:%S"),
-        reverse=True,
-    )
+    # playlists.sort(
+    #     key=lambda p: datetime.strptime(p.last_updated, "%Y-%m-%d %H:%M:%S"),
+    #     reverse=True,
+    # )
 
     return {"data": playlists}
 
@@ -175,7 +180,11 @@ def add_item_to_playlist(path: PlaylistIDPath, body: AddItemToPlaylistBody):
     if itemtype == "tracks":
         trackhashes = itemhash.split(",")
     elif itemtype == "folder":
-        trackhashes = get_path_trackhashes(itemhash, sortoptions.get("tracksortby") or 'default', sortoptions.get("tracksortreverse") or False)
+        trackhashes = get_path_trackhashes(
+            itemhash,
+            sortoptions.get("tracksortby") or "default",
+            sortoptions.get("tracksortreverse") or False,
+        )
     elif itemtype == "album":
         trackhashes = get_album_trackhashes(itemhash)
     elif itemtype == "artist":
@@ -408,7 +417,11 @@ def save_item_as_playlist(body: SavePlaylistAsItemBody):
     if itemtype == "tracks":
         trackhashes = itemhash.split(",")
     elif itemtype == "folder":
-        trackhashes = get_path_trackhashes(itemhash, sortoptions.get("tracksortby") or 'default', sortoptions.get("tracksortreverse") or False)
+        trackhashes = get_path_trackhashes(
+            itemhash,
+            sortoptions.get("tracksortby") or "default",
+            sortoptions.get("tracksortreverse") or False,
+        )
     elif itemtype == "album":
         trackhashes = get_album_trackhashes(itemhash)
     elif itemtype == "artist":

app/api/settings.py

@@ -95,14 +95,14 @@ def get_all_settings():
     Get all settings
     """
     config = asdict(UserConfig())
-    plugins = PluginTable.get_all()
-    config["plugins"] = plugins
+    config["plugins"] = [p for p in PluginTable.get_all()]
     config["version"] = Info.SWINGMUSIC_APP_VERSION
 
     # hide lastfmSessionKeys for other users
     current_user = get_current_userid()
     config["lastfmSessionKey"] = config["lastfmSessionKeys"].get(str(current_user), "")
     del config["lastfmSessionKeys"]
+
     return config
 
 

app/db/__init__.py

@@ -21,14 +21,19 @@ class Base(MappedAsDataclass, DeclarativeBase):
     @classmethod
     def execute(cls, stmt: Any, commit: bool = False):
         with DbEngine.manager(commit=commit) as session:
-            return session.execute(stmt)
+            result = session.execute(stmt.execution_options(yield_per=100))
+
+            if commit:
+                session.commit()
+
+            yield result
 
     @classmethod
     def insert_many(cls, items: list[dict[str, Any]]):
         """
         Inserts multiple items into the database.
         """
-        return cls.execute(insert(cls).values(items), commit=True)
+        return next(cls.execute(insert(cls).values(items), commit=True))
 
     @classmethod
     def insert_one(cls, item: dict[str, Any]):
@@ -39,19 +44,19 @@ class Base(MappedAsDataclass, DeclarativeBase):
 
     @classmethod
     def remove_all(cls):
-        return cls.execute(delete(cls), commit=True)
+        return next(cls.execute(delete(cls), commit=True))
 
     @classmethod
     def remove_one(cls, id: int):
-        return cls.execute(delete(cls).where(cls.id == id), commit=True)
+        return next(cls.execute(delete(cls).where(cls.id == id), commit=True))
 
     @classmethod
     def all(cls):
-        return cls.execute(select(cls))
+        return next(cls.execute(select(cls).execution_options(yield_per=100)))
 
     @classmethod
     def count(cls):
-        return cls.execute(select(func.count()).select_from(cls)).scalar()
+        return next(cls.execute(select(func.count()).select_from(cls))).scalar()
 
 
 def create_all_tables():

app/db/engine.py

@@ -1,5 +1,6 @@
 from contextlib import contextmanager
 from sqlalchemy import Engine, event
+from sqlalchemy.orm import sessionmaker
 
 
 @event.listens_for(Engine, "connect")
@@ -9,8 +10,8 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
     cursor.execute("PRAGMA synchronous=NORMAL")
     cursor.execute("PRAGMA cache_size=10000")
     cursor.execute("PRAGMA foreign_keys=ON")
-    cursor.execute("PRAGMA temp_store=MEMORY")
-    cursor.execute("PRAGMA mmap_size=30000000000")
+    cursor.execute("PRAGMA temp_store=FILE")
+    cursor.execute("PRAGMA mmap_size=0")
     cursor.close()
 
 
@@ -31,16 +32,23 @@ class DbEngine:
 
         If the `commit` parameter is set to `True`, the context manager will commit the transaction when it exits.
         """
-        conn = cls.engine.connect()
+        Session = sessionmaker(cls.engine)
 
         try:
-            yield conn.execution_options(preserve_rowcount=True)
-            if commit:
-                conn.commit()
+            with Session() as session:
+                yield session
+
+                if commit:
+                    session.commit()
+                # yield session.execution_options(preserve_rowcount=True, yield_per=100)
+            # yield conn.execution_options(preserve_rowcount=True, yield_per=100)
         except Exception as e:
-            conn.rollback()
+            session.rollback()
             raise e
         finally:
-            conn.close()
-            del conn
-            cls.engine.clear_compiled_cache()
+            if commit:
+                session.commit()
+
+            session.close()
+            # del conn
+            # cls.engine.clear_compiled_cache()

app/db/libdata.py

@@ -1,5 +1,6 @@
+from app.config import UserConfig
 from app.db import Base
-from app.db.utils import tracks_to_dataclasses
+from app.db.utils import track_to_dataclass, tracks_to_dataclasses
 from app.db.engine import DbEngine
 from sqlalchemy import JSON, Integer, String, delete, select
 from sqlalchemy.orm import Mapped, mapped_column
@@ -38,8 +39,14 @@ class TrackTable(Base):
     @classmethod
     def get_all(cls):
         with DbEngine.manager() as conn:
-            result = conn.execute(select(cls))
-            return tracks_to_dataclasses(result.fetchall())
+            config = UserConfig()
+            result = conn.execute(select(cls).execution_options(yield_per=100))
+
+            for i in result.scalars():
+                d = i.__dict__
+                del d["_sa_instance_state"]
+
+                yield track_to_dataclass(d, config)
 
     @classmethod
     def get_tracks_by_filepaths(cls, filepaths: list[str]):

app/db/userdata.py

@@ -1,6 +1,6 @@
 from dataclasses import asdict
 import datetime
-from typing import Any, Literal
+from typing import Any, Iterable, Literal
 from sqlalchemy import (
     JSON,
     Boolean,
@@ -15,21 +15,17 @@ from sqlalchemy import (
     update,
 )
 
-from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.orm import Mapped, mapped_column, sessionmaker
 
 from app.db.engine import DbEngine
 from app.db.utils import (
+    favorite_to_dataclass,
     favorites_to_dataclass,
     playlist_to_dataclass,
-    playlists_to_dataclasses,
     plugin_to_dataclass,
-    plugin_to_dataclasses,
     similar_artist_to_dataclass,
-    similar_artists_to_dataclass,
     tracklog_to_dataclass,
-    tracklog_to_dataclasses,
     user_to_dataclass,
-    user_to_dataclasses,
 )
 
 from app.db import Base
@@ -52,7 +48,9 @@ class UserTable(Base):
     @classmethod
     def get_all(cls):
         result = cls.execute(select(cls))
-        return user_to_dataclasses(result.fetchall())
+
+        for i in next(result).scalars():
+            yield user_to_dataclass(i)
 
     @classmethod
     def insert_default_user(cls):
@@ -76,30 +74,40 @@ class UserTable(Base):
 
     @classmethod
     def get_by_id(cls, id: int):
-        with DbEngine.manager() as conn:
-            result = conn.execute(select(cls).where(cls.id == id))
-            res = result.fetchone()
+        result = cls.execute(select(cls).where(cls.id == id))
+        res = next(result).scalar()
 
-            if res:
-                return user_to_dataclass(res)
+        if res:
+            return user_to_dataclass(res)
 
     @classmethod
     def get_by_username(cls, username: str):
-        with DbEngine.manager() as conn:
-            result = conn.execute(select(cls).where(cls.username == username))
-            res = result.fetchone()
+        # with DbEngine.manager() as conn:
+        #     result = conn.execute(select(cls).where(cls.username == username))
+        #     res = result.fetchone()
 
-            if res:
-                return user_to_dataclass(res)
+        #     if res:
+        #         return user_to_dataclass(res)
+
+        res = cls.execute(select(cls).where(cls.username == username))
+        res = next(res).scalar()
+
+        if res:
+            return user_to_dataclass(res)
 
     @classmethod
     def update_one(cls, user: dict[str, Any]):
-        with DbEngine.manager(commit=True) as conn:
-            conn.execute(update(cls).where(cls.id == user["id"]).values(user))
+        return next(
+            cls.execute(
+                update(cls).where(cls.id == user["id"]).values(user), commit=True
+            )
+        )
 
     @classmethod
     def remove_by_username(cls, username: str):
-        return cls.execute(delete(cls).where(cls.username == username), commit=True)
+        return next(
+            cls.execute(delete(cls).where(cls.username == username), commit=True)
+        )
 
 
 class PluginTable(Base):
@@ -113,23 +121,34 @@ class PluginTable(Base):
 
     @classmethod
     def get_all(cls):
-        return plugin_to_dataclasses(cls.all())
+        result = cls.execute(select(cls))
+
+        for i in next(result).scalars():
+            yield plugin_to_dataclass(i)
 
     @classmethod
     def activate(cls, name: str, value: bool):
-        return cls.execute(
-            update(cls).where(cls.name == name).values(active=value), commit=True
+        return next(
+            cls.execute(
+                update(cls).where(cls.name == name).values(active=value), commit=True
+            )
         )
 
     @classmethod
     def get_by_name(cls, name: str):
         result = cls.execute(select(cls).where(cls.name == name))
-        return plugin_to_dataclass(result.fetchone())
+        res = next(result).scalar()
+
+        if res:
+            return plugin_to_dataclass(res)
 
     @classmethod
     def update_settings(cls, name: str, settings: dict[str, Any]):
-        return cls.execute(
-            update(cls).where(cls.name == name).values(settings=settings), commit=True
+        return next(
+            cls.execute(
+                update(cls).where(cls.name == name).values(settings=settings),
+                commit=True,
+            )
         )
 
 
@@ -142,11 +161,10 @@ class SimilarArtistTable(Base):
 
     @classmethod
     def get_all(cls):
-        with DbEngine.manager() as conn:
-            result = conn.execute(
-                select(cls.artisthash), execution_options={"stream_results": True}
-            )
-            return result.scalars().all()
+        result = cls.execute(select(cls).execution_options(yield_per=100))
+
+        for i in next(result).scalars():
+            yield similar_artist_to_dataclass(i)
 
     @classmethod
     def exists(cls, artisthash: str):
@@ -156,7 +174,9 @@ class SimilarArtistTable(Base):
 
         with DbEngine.manager() as conn:
             result = conn.execute(
-                select(cls.artisthash).where(cls.artisthash == artisthash)
+                select(cls.artisthash)
+                .where(cls.artisthash == artisthash)
+                .execution_options(yield_per=100)
             )
 
             return len(result.scalars().all()) > 0
@@ -166,13 +186,11 @@ class SimilarArtistTable(Base):
         """
         Get a single artist by hash.
         """
+        result = cls.execute(select(cls).where(cls.artisthash == artisthash))
+        res = next(result).scalar()
 
-        with DbEngine.manager() as conn:
-            result = conn.execute(select(cls).where(cls.artisthash == artisthash))
-            result = result.fetchone()
-
-            if result:
-                return similar_artist_to_dataclass(result)
+        if res:
+            return similar_artist_to_dataclass(res)
 
 
 class FavoritesTable(Base):
@@ -198,29 +216,31 @@ class FavoritesTable(Base):
                 )
             else:
                 result = conn.execute(select(cls))
-            return favorites_to_dataclass(result.fetchall())
+
+            for i in result.scalars():
+                yield favorite_to_dataclass(i)
 
     @classmethod
     def insert_item(cls, item: dict[str, Any]):
         item["timestamp"] = int(datetime.datetime.now().timestamp())
         item["userid"] = get_current_userid()
 
-        with DbEngine.manager(commit=True) as conn:
-            conn.execute(insert(cls).values(item))
+        return next(cls.execute(insert(cls).values(item), commit=True))
 
     @classmethod
     def remove_item(cls, item: dict[str, Any]):
-        with DbEngine.manager(commit=True) as conn:
-            conn.execute(
+        return next(
+            cls.execute(
                 delete(cls).where(
                     (cls.hash == item["hash"]) & (cls.type == item["type"])
                 )
             )
+        )
 
     @classmethod
     def check_exists(cls, hash: str, type: str):
         result = cls.execute(select(cls).where((cls.hash == hash) & (cls.type == type)))
-        return result.fetchone() is not None
+        return next(result).scalar() is not None
 
     @classmethod
     def get_all_of_type(cls, type: str, start: int, limit: int):
@@ -234,7 +254,7 @@ class FavoritesTable(Base):
             .limit(limit if start != 0 else None)
         )
 
-        res = result.fetchall()
+        res = next(result).scalars().all()
 
         if start == 0:
             # if limit == -1, return all
@@ -268,10 +288,10 @@ class FavoritesTable(Base):
             .where(and_(cls.timestamp >= start_time, cls.timestamp <= end_time))
         )
 
-        result = result.fetchone()
+        res = next(result).scalar()
 
-        if result:
-            return result[0]
+        if res:
+            return res[0]
 
         return 0
 
@@ -304,9 +324,11 @@ class ScrobbleTable(Base):
             .order_by(cls.timestamp.desc())
             .offset(start)
             .limit(limit)
+            .execution_options(yield_per=100)
         )
 
-        return tracklog_to_dataclasses(result.fetchall())
+        for i in next(result).scalars():
+            yield tracklog_to_dataclass(i)
 
     @classmethod
     def get_all_in_period(cls, start_time: int, end_time: int, userid: int | None):
@@ -320,15 +342,21 @@ class ScrobbleTable(Base):
             .where(cls.userid == userid)
             .where(and_(cls.timestamp >= start_time, cls.timestamp <= end_time))
             .order_by(cls.timestamp.desc())
+            .execution_options(yield_per=100)
         )
-        return tracklog_to_dataclasses(result.fetchall())
+
+        for i in next(result).scalars():
+            yield tracklog_to_dataclass(i)
 
     @classmethod
     def get_last_entry(cls, userid: int):
         result = cls.execute(
             select(cls).where(cls.userid == userid).order_by(cls.timestamp.desc())
         )
-        return tracklog_to_dataclass(result.fetchone())
+        res = next(result).scalar()
+
+        if res:
+            return tracklog_to_dataclass(res)
 
 
 class PlaylistTable(Base):
@@ -350,24 +378,30 @@ class PlaylistTable(Base):
     @classmethod
     def get_all(cls, current_user: bool = True):
         if current_user:
-            result = cls.execute(select(cls).where(cls.userid == get_current_userid()))
+            result = cls.execute(
+                select(cls)
+                .where(cls.userid == get_current_userid())
+                .execution_options(yield_per=100)
+            )
         else:
-            result = cls.execute(select(cls))
+            result = cls.execute(select(cls).execution_options(yield_per=100))
 
-        return playlists_to_dataclasses(result)
+        for i in next(result).scalars():
+            yield playlist_to_dataclass(i)
 
     @classmethod
     def add_one(cls, playlist: dict[str, Any]):
         playlist["userid"] = get_current_userid()
         result = cls.insert_one(playlist)
-        return result.lastrowid
+
+        return next(result).lastrowid
 
     @classmethod
     def check_exists_by_name(cls, name: str):
         result = cls.execute(
             select(cls).where((cls.name == name) & (cls.userid == get_current_userid()))
         )
-        return result.fetchone() is not None
+        return next(result).scalar() is not None
 
     @classmethod
     def append_to_playlist(cls, id: int, trackhashes: list[str]):
@@ -375,11 +409,13 @@ class PlaylistTable(Base):
         if not dbtrackhashes:
             dbtrackhashes = []
 
-        return cls.execute(
-            update(cls)
-            .where((cls.id == id) & (cls.userid == get_current_userid()))
-            .values(trackhashes=dbtrackhashes + trackhashes),
-            commit=True,
+        return next(
+            cls.execute(
+                update(cls)
+                .where((cls.id == id) & (cls.userid == get_current_userid()))
+                .values(trackhashes=dbtrackhashes + trackhashes),
+                commit=True,
+            )
         )
 
     @classmethod
@@ -389,9 +425,7 @@ class PlaylistTable(Base):
                 (cls.id == id) & (cls.userid == get_current_userid())
             )
         )
-        result = result.fetchone()
-        if result:
-            return result[0]
+        return next(result).scalar()
 
     @classmethod
     def remove_from_playlist(cls, id: int, trackhashes: list[dict[str, Any]]):
@@ -402,11 +436,13 @@ class PlaylistTable(Base):
                 if dbtrackhashes.index(item["trackhash"]) == item["index"]:
                     dbtrackhashes.remove(item["trackhash"])
 
-            return cls.execute(
-                update(cls)
-                .where((cls.id == id) & (cls.userid == get_current_userid()))
-                .values(trackhashes=dbtrackhashes),
-                commit=True,
+            return next(
+                cls.execute(
+                    update(cls)
+                    .where((cls.id == id) & (cls.userid == get_current_userid()))
+                    .values(trackhashes=dbtrackhashes),
+                    commit=True,
+                )
             )
 
     @classmethod
@@ -414,35 +450,42 @@ class PlaylistTable(Base):
         result = cls.execute(
             select(cls).where((cls.id == id) & (cls.userid == get_current_userid()))
         )
-        result = result.fetchone()
+        result = next(result).scalar()
+
         if result:
             return playlist_to_dataclass(result)
 
     @classmethod
     def update_one(cls, id: int, playlist: dict[str, Any]):
-        return cls.execute(
-            update(cls)
-            .where((cls.id == id) & (cls.userid == get_current_userid()))
-            .values(playlist),
-            commit=True,
+        return next(
+            cls.execute(
+                update(cls)
+                .where((cls.id == id) & (cls.userid == get_current_userid()))
+                .values(playlist),
+                commit=True,
+            )
         )
 
     @classmethod
     def update_settings(cls, id: int, settings: dict[str, Any]):
-        return cls.execute(
-            update(cls)
-            .where((cls.id == id) & (cls.userid == get_current_userid()))
-            .values(settings=settings),
-            commit=True,
+        return next(
+            cls.execute(
+                update(cls)
+                .where((cls.id == id) & (cls.userid == get_current_userid()))
+                .values(settings=settings),
+                commit=True,
+            )
         )
 
     @classmethod
     def remove_image(cls, id: int):
-        return cls.execute(
-            update(cls)
-            .where((cls.id == id) & (cls.userid == get_current_userid()))
-            .values(image=None),
-            commit=True,
+        return next(
+            cls.execute(
+                update(cls)
+                .where((cls.id == id) & (cls.userid == get_current_userid()))
+                .values(image=None),
+                commit=True,
+            )
         )
 
 
@@ -461,8 +504,10 @@ class LibDataTable(Base):
 
     @classmethod
     def update_one(cls, hash: str, data: dict[str, Any]):
-        return cls.execute(
-            update(cls).where(cls.itemhash == hash).values(data), commit=True
+        return next(
+            cls.execute(
+                update(cls).where(cls.itemhash == hash).values(data), commit=True
+            )
         )
 
     @classmethod
@@ -470,17 +515,20 @@ class LibDataTable(Base):
         result = cls.execute(
             select(cls).where((cls.itemhash == type + hash) & (cls.itemtype == type))
         )
-        return result.fetchone()
+        return next(result).scalar()
 
     @classmethod
-    def get_all_colors(cls, type: str) -> list[dict[str, str]]:
+    def get_all_colors(cls, type: str) -> Iterable[dict[str, str]]:
         result = cls.execute(
             select(cls.itemhash, cls.color).where(cls.itemtype == type)
         )
-        return [
-            {"itemhash": r[0].replace(type, ""), "color": r[1]}
-            for r in result.fetchall()
-        ]
+        # return [
+        #     {"itemhash": r[0].replace(type, ""), "color": r[1]}
+        #     for r in result.fetchall()
+        # ]
+
+        for i in next(result).scalars():
+            yield {"itemhash": i[0].replace(type, ""), "color": i[1]}
 
 
 class MixTable(Base):
@@ -512,20 +560,23 @@ class MixTable(Base):
         else:
             result = cls.execute(select(cls).order_by(cls.timestamp.desc()))
 
-        return Mix.mixes_to_dataclasses(result.fetchall())
+        for i in next(result).scalars():
+            yield Mix.mix_to_dataclass(i)
 
     @classmethod
     def get_by_sourcehash(cls, sourcehash: str):
         result = cls.execute(select(cls).where(cls.sourcehash == sourcehash))
 
-        res = result.fetchone()
+        res = next(result).scalar()
+
         if res:
             return Mix.mix_to_dataclass(res)
 
     @classmethod
     def get_by_mixid(cls, mixid: str):
         result = cls.execute(select(cls).where(cls.mixid == mixid))
-        res = result.fetchone()
+        res = next(result).scalar()
+
         if res:
             return Mix.mix_to_dataclass(res)
 
@@ -535,7 +586,7 @@ class MixTable(Base):
         mixdict["mixid"] = mix.id
         del mixdict["id"]
 
-        return cls.execute(insert(cls).values(mixdict), commit=True)
+        return next(cls.execute(insert(cls).values(mixdict), commit=True))
 
     @classmethod
     def update_one(cls, mixid: str, mix: Mix):
@@ -543,17 +594,19 @@ class MixTable(Base):
         mixdict["mixid"] = mix.id
         del mixdict["id"]
 
-        return cls.execute(
-            update(cls)
-            .where(
-                and_(
-                    cls.mixid == mixid,
-                    cls.sourcehash == mix.sourcehash,
-                    cls.userid == get_current_userid(),
+        return next(
+            cls.execute(
+                update(cls)
+                .where(
+                    and_(
+                        cls.mixid == mixid,
+                        cls.sourcehash == mix.sourcehash,
+                        cls.userid == get_current_userid(),
+                    )
                 )
+                .values(mixdict),
+                commit=True,
             )
-            .values(mixdict),
-            commit=True,
         )
 
     @classmethod
@@ -579,7 +632,10 @@ class MixTable(Base):
         """
 
         result = cls.execute(select(cls).where(cls.extra.c.trackmix_saved == True))
-        return Mix.mixes_to_dataclasses(result.fetchall())
+        # return Mix.mixes_to_dataclasses(result.fetchall())
+
+        for i in next(result).scalars():
+            yield Mix.mix_to_dataclass(i)
 
     @classmethod
     def save_track_mix(cls, sourcehash: str):
@@ -612,41 +668,58 @@ class PageTable(Base):
 
     @classmethod
     def to_dict(cls, entry: Any) -> dict[str, Any]:
-        return entry._asdict()
+        d = entry.__dict__
+        del d["_sa_instance_state"]
+        return d
 
     @classmethod
     def get_all(cls):
         result = cls.execute(select(cls).where(cls.userid == get_current_userid()))
-        return [cls.to_dict(entry) for entry in result.fetchall()]
+
+        for i in next(result).scalars():
+            yield cls.to_dict(i)
 
     @classmethod
     def get_by_id(cls, id: int):
         result = cls.execute(
             select(cls).where(and_(cls.id == id, cls.userid == get_current_userid()))
         )
-        return cls.to_dict(result.fetchone())
+        res = next(result).scalar()
+
+        if res:
+            return cls.to_dict(res)
 
     @classmethod
     def delete_by_id(cls, id: int):
-        return cls.execute(
-            delete(cls).where(and_(cls.id == id, cls.userid == get_current_userid())),
-            commit=True,
+        return next(
+            cls.execute(
+                delete(cls).where(
+                    and_(cls.id == id, cls.userid == get_current_userid())
+                ),
+                commit=True,
+            )
         )
 
     @classmethod
     def update_items(cls, id: int, items: list[dict[str, Any]]):
-        return cls.execute(
-            update(cls)
-            .where(and_(cls.id == id, cls.userid == get_current_userid()))
-            .values(items=items),
-            commit=True,
+        return next(
+            cls.execute(
+                update(cls)
+                .where(and_(cls.id == id, cls.userid == get_current_userid()))
+                .values(items=items),
+                commit=True,
+            )
         )
 
     @classmethod
     def update_one(cls, payload: dict[str, Any]):
-        return cls.execute(
-            update(cls)
-            .where(and_(cls.id == payload["id"], cls.userid == get_current_userid()))
-            .values(payload),
-            commit=True,
+        return next(
+            cls.execute(
+                update(cls)
+                .where(
+                    and_(cls.id == payload["id"], cls.userid == get_current_userid())
+                )
+                .values(payload),
+                commit=True,
+            )
         )

app/db/utils.py

@@ -10,8 +10,14 @@ from app.models.plugins import Plugin
 from app.models.user import User
 
 
-def track_to_dataclass(track: Any, config: UserConfig):
-    return TrackModel(**track._asdict(), config=config)
+def row_to_dict(row: Any):
+    d = row.__dict__
+    del d["_sa_instance_state"]
+    return d
+
+
+def track_to_dataclass(track: dict, config: UserConfig):
+    return TrackModel(**track, config=config)
 
 
 def tracks_to_dataclasses(tracks: Any):
@@ -35,10 +41,8 @@ def artists_to_dataclasses(artists: Any):
 
 
 # SECTION: User data helpers
-
-
 def similar_artist_to_dataclass(entry: Any):
-    entry_dict = entry._asdict()
+    entry_dict = row_to_dict(entry)
     del entry_dict["id"]
 
     return SimilarArtist(**entry_dict)
@@ -49,7 +53,7 @@ def similar_artists_to_dataclass(entries: Any):
 
 
 def favorite_to_dataclass(entry: Any):
-    entry_dict = entry._asdict()
+    entry_dict = row_to_dict(entry)
     del entry_dict["id"]
 
     return Favorite(**entry_dict)
@@ -60,16 +64,15 @@ def favorites_to_dataclass(entries: Any):
 
 
 def user_to_dataclass(entry: Any):
-    entry_dict = entry._asdict()
-    return User(**entry_dict)
+    return User(**row_to_dict(entry))
 
 
-def user_to_dataclasses(entries: Any):
-    return [user_to_dataclass(entry) for entry in entries]
+# def user_to_dataclasses(entries: Any):
+#     return [user_to_dataclass(entry) for entry in entries]
 
 
 def plugin_to_dataclass(entry: Any):
-    entry_dict = entry._asdict()
+    entry_dict = row_to_dict(entry)
     del entry_dict["id"]
     return Plugin(**entry_dict)
 
@@ -79,8 +82,7 @@ def plugin_to_dataclasses(entries: Any):
 
 
 def tracklog_to_dataclass(entry: Any):
-    entry_dict = entry._asdict()
-    return TrackLog(**entry_dict)
+    return TrackLog(**row_to_dict(entry))
 
 
 def tracklog_to_dataclasses(entries: Any):
@@ -88,7 +90,7 @@ def tracklog_to_dataclasses(entries: Any):
 
 
 def playlist_to_dataclass(entry: Any):
-    entry_dict = entry._asdict()
+    entry_dict = row_to_dict(entry)
     return Playlist(**entry_dict)
 
 

app/lib/mapstuff.py

@@ -33,16 +33,22 @@ def map_scrobble_data():
         if track is None:
             continue
 
-        track.increment_playcount(data["playduration"], data["lastplayed"], data["playcount"])
+        track.increment_playcount(
+            data["playduration"], data["lastplayed"], data["playcount"]
+        )
 
         album = AlbumStore.albummap.get(track.tracks[0].albumhash)
         if album:
-            album.increment_playcount(data["playduration"], data["lastplayed"], data["playcount"])
+            album.increment_playcount(
+                data["playduration"], data["lastplayed"], data["playcount"]
+            )
 
         for artisthash in track.tracks[0].artisthashes:
             artist = ArtistStore.artistmap.get(artisthash)
             if artist:
-                artist.increment_playcount(data["playduration"], data["lastplayed"], data["playcount"])
+                artist.increment_playcount(
+                    data["playduration"], data["lastplayed"], data["playcount"]
+                )
 
 
 def map_favorites():

app/lib/playlistlib.py

@@ -28,7 +28,7 @@ def create_thumbnail(image: Any, img_path: str) -> str:
 
     new_w = round(250 * aspect_ratio)
 
-    thumb = image.resize((new_w, 250), Image.ANTIALIAS)
+    thumb = image.resize((new_w, 250), Image.Resampling.LANCZOS)
     thumb.save(full_thumb_path, "webp")
 
     return thumb_path
@@ -50,7 +50,7 @@ def create_gif_thumbnail(image: Any, img_path: str):
 
         new_w = round(250 * aspect_ratio)
 
-        thumb = frame.resize((new_w, 250), Image.ANTIALIAS)
+        thumb = frame.resize((new_w, 250), Image.Resampling.LANCZOS)
         frames.append(thumb)
 
     frames[0].save(full_thumb_path, save_all=True, append_images=frames[1:])

app/lib/populate.py

@@ -169,8 +169,7 @@ class FetchSimilarArtistsLastFM:
 
     def __init__(self, instance_key: str) -> None:
         # read all artists from db
-        processed = SimilarArtistTable.get_all()
-        processed = ".".join(a for a in processed)
+        processed = set(a.artisthash for a in SimilarArtistTable.get_all())
 
         # filter out artists that already have similar artists
         artists = filter(

app/models/mix.py

@@ -2,6 +2,7 @@ import time
 from dataclasses import asdict, dataclass, field
 from typing import Any
 
+from app.db.utils import row_to_dict
 from app.lib.playlistlib import get_first_4_images
 from app.serializers.track import serialize_tracks
 from app.store.tracks import TrackStore
@@ -60,7 +61,8 @@ class Mix:
 
     @classmethod
     def mix_to_dataclass(cls, entry: Any):
-        entry_dict = entry._asdict()
+        entry_dict = row_to_dict(entry)
+
         entry_dict["id"] = entry_dict["mixid"]
         del entry_dict["mixid"]
 
@@ -69,4 +71,3 @@ class Mix:
     @classmethod
     def mixes_to_dataclasses(cls, entries: Any):
         return [cls.mix_to_dataclass(entry) for entry in entries]
-

app/plugins/mixes.py

@@ -33,6 +33,7 @@ class MixAlreadyExists(Exception):
 class MixesPlugin(Plugin):
     MAX_TRACKS_TO_FETCH = 5
     MIN_TRACK_MIX_LENGTH = 15
+    MIN_ARTISTS_PER_MIX = 4
     MIX_TRACKS_LENGTH = 40
 
     MIN_DAY_LISTEN_DURATION = 3 * 60  # 3 minutes
@@ -291,6 +292,10 @@ class MixesPlugin(Plugin):
         if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH:
             return None
 
+        # INFO: Dump mixes with no variety
+        if len(set(t.artisthashes[0] for t in mix_tracks)) < self.MIN_ARTISTS_PER_MIX:
+            return None
+
         # try downloading artist image
         mix_image = {"image": _artist.artist.image, "color": _artist.artist.color}
         image = self.download_artist_image(_artist.artist)

db.py

@@ -1,74 +0,0 @@
-from sqlalchemy import create_engine, text, Table, Column, Integer, String, MetaData, select
-from sqlalchemy.orm import DeclarativeBase
-
-from typing import List, Optional
-
-from sqlalchemy.orm import Mapped, mapped_column, relationship
-
-fullpath = "/home/cwilvx/temp/swingmusic/swing.db"
-engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True)
-
-class Base(DeclarativeBase):
-    pass
-
-class Tracks(Base):
-    __tablename__ = "tracks"
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    album: Mapped[str] = mapped_column(String())
-    albumartist: Mapped[str] = mapped_column(String())
-    copyright: Mapped[Optional[str]]
-
-    def __repr__(self):
-        return f"<Tracks(album={self.album}, albumartist={self.albumartist})>"
-
-stmt = select(Tracks.album, Tracks.copyright).where(Tracks.album == "RAVAGE")
-print(stmt)
-
-with engine.connect() as conn:
-    result = conn.execute(stmt)
-    for row in result:
-        print(row)
-
-# Base.metadata.create_all(engine)
-
-# metadata = MetaData()
-# track_table = Table(
-#     "tracks",
-#     metadata,
-#     Column("id", Integer, primary_key=True, autoincrement=True),
-#     Column("album", String),
-#     Column("albumartist", String),
-#     Column("albumhash", String),
-#     Column("artist", String),
-#     Column("bitrate", Integer),
-#     Column("copyright", String),
-#     Column("date", Integer),
-#     Column("disc", Integer),
-#     Column("duration", Integer),
-#     Column("filepath", String),
-#     Column("folder", String),
-#     Column("genre", String),
-#     Column("title", String),
-#     Column("track", Integer),
-#     Column("trackhash", String),
-#     Column("last_mod", Integer),
-# )
-
-# metadata.create_all(engine)
-
-
-
-
-
-
-
-# with engine.connect() as conn:
-#     result = conn.execute(
-#         text("SELECT * FROM tracks where trackhash = :trackhash"),
-#         {"trackhash": "93acbea22b"},
-#     )
-#     # print(result.all())
-
-#     for r in result.mappings():
-#         print(r["trackhash"])

pyproject.toml

@@ -12,7 +12,7 @@ dependencies = [
     "requests>=2.27.1",
     "colorgram.py>=1.2.0",
     "tqdm>=4.65.0",
-    "rapidfuzz>=2.13.7",
+    "rapidfuzz==3.12.1",
     "tinytag>=2.0.0",
     "Unidecode>=1.3.6",
     "psutil>=5.9.4",

uv.lock

@@ -1252,7 +1252,7 @@ requires-dist = [
     { name = "pendulum", specifier = ">=3.0.0" },
     { name = "pillow", specifier = ">=11.1.0" },
     { name = "psutil", specifier = ">=5.9.4" },
-    { name = "rapidfuzz", specifier = ">=2.13.7" },
+    { name = "rapidfuzz", specifier = "==3.12.1" },
     { name = "requests", specifier = ">=2.27.1" },
     { name = "schedule", specifier = ">=1.2.2" },
     { name = "setproctitle", specifier = ">=1.3.2" },