start: rewrite the database layer using a freaking ORM

+ start ditching in-mem stores
+ move main db table to a new name
+ experiments!

.github/changelog.md

@@ -12,3 +12,16 @@
 - 
 
 ## Development
+
+
+## THE BIG ONE API CHANGES
+
+- genre is no longer a string, but a struct:
+
+```ts
+interface Genre {
+    name: str;
+    genrehash: str;
+}
+```
+

TODO.md

@@ -1,6 +1,11 @@
 # TODO
 - Migrations:
     1. Move userdata to new hashing algorithm
+        - favorites ✅
+        - playlists
+        - scrobble
+        - images
+        - remove image colors
 
 - Package jsoni and publish on PyPi
 - Rewrite stores to use dictionaries instead of list pools
@@ -8,6 +13,8 @@
 - Disable the watchdog by default, and mark it as experimental
 - rename userid to server id in config file
 - Look into seeding jwts using user password + server id
+- Recreate album hash if featured artists are discover
+- Implement checking if is clean install and skip migrations!
 
 # DONE
 - Support auth headers

app/api/album.py

@@ -2,6 +2,7 @@
 Contains all the album routes.
 """
 
+from itertools import groupby
 import random
 
 from flask_jwt_extended import current_user
@@ -10,13 +11,15 @@ from flask_openapi3 import Tag
 from flask_openapi3 import APIBlueprint
 from app.api.apischemas import AlbumHashSchema, AlbumLimitSchema, ArtistHashSchema
 
+from app.config import UserConfig
+from app.db import AlbumTable as AlbumDb, TrackTable as TrackDb
 from app.settings import Defaults
 from app.models import FavType, Track
 from app.store.albums import AlbumStore
 from app.store.tracks import TrackStore
 from app.utils.hashing import create_hash
 from app.lib.albumslib import sort_by_track_no
-from app.serializers.album import serialize_for_card
+from app.serializers.album import serialize_for_card, serialize_for_card_many
 from app.serializers.track import serialize_track
 from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb
 from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
@@ -38,47 +41,20 @@ def get_album_tracks_and_info(body: AlbumHashSchema):
     Returns album info and tracks for the given albumhash.
     """
     albumhash = body.albumhash
-
-    error_msg = {"error": "Album not created yet."}
-    album = AlbumStore.get_album_by_hash(albumhash)
+    album = AlbumDb.get_album_by_albumhash(albumhash)
 
     if album is None:
-        return error_msg, 404
+        return {"error": "Album not found"}, 404
 
-    tracks = TrackStore.get_tracks_by_albumhash(albumhash)
-
-    if tracks is None:
-        return error_msg, 404
-
-    if len(tracks) == 0:
-        return error_msg, 404
-
-    def get_album_genres(tracks: list[Track]):
-        genres = set()
-
-        for track in tracks:
-            if track.genre is not None:
-                genres.update(track.genre)
-
-        return list(genres)
-
-    album.genres = get_album_genres(tracks)
-    album.count = len(tracks)
-
-    album.get_date_from_tracks(tracks)
+    tracks = TrackDb.get_tracks_by_albumhash(albumhash)
+    album.trackcount = len(tracks)
     album.duration = sum(t.duration for t in tracks)
+    album.type = album.check_type(
+        tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles
+    )
+    album.populate_versions()
 
-    album.check_is_single(tracks)
-
-    if not album.is_single:
-        album.check_type()
-
-    album.is_favorite = check_is_fav(albumhash, FavType.album)
-
-    return {
-        "tracks": [serialize_track(t, remove_disc=False) for t in tracks],
-        "info": album,
-    }
+    return {"info": album, "tracks": tracks}
 
 
 @api.get("/<albumhash>/tracks")
@@ -89,16 +65,16 @@ def get_album_tracks(path: AlbumHashSchema):
     Returns all the tracks in the given album, sorted by disc and track number.
     NOTE: No album info is returned.
     """
-    tracks = TrackStore.get_tracks_by_albumhash(path.albumhash)
+    tracks = TrackDb.get_tracks_by_albumhash(path.albumhash)
     tracks = sort_by_track_no(tracks)
 
     return tracks
 
 
 class GetMoreFromArtistsBody(AlbumLimitSchema):
-    albumartists: str = Field(
+    albumartists: list = Field(
         description="The artist hashes to get more albums from",
-        example=Defaults.API_ARTISTHASH,
+        example='[{"name": "Khalid", "artisthash": "94ca2dba1c"}]',
     )
 
     base_title: str = Field(
@@ -119,29 +95,25 @@ def get_more_from_artist(body: GetMoreFromArtistsBody):
     limit = body.limit
     base_title = body.base_title
 
-    albumartists: list[str] = albumartists.split(",")
+    all_albums = AlbumDb.get_albums_by_artisthashes(albumartists)
 
-    albums = [
-        {
-            "artisthash": a,
-            "albums": AlbumStore.get_albums_by_albumartist(
-                a, limit, exclude=base_title
-            ),
-        }
-        for a in albumartists
+    # filter out albums with the same base title
+    all_albums = filter(
+        lambda a: create_hash(a.base_title) != create_hash(base_title), all_albums
+    )
+    all_albums = list(all_albums)
+
+    if not len(all_albums):
+        return []
+
+    # group by first albumartist's artisthash
+    groups = groupby(all_albums, lambda a: a.albumartists[0]["artisthash"])
+
+    return [
+        {"artisthash": g[0], "albums": serialize_for_card_many(list(g[1])[:limit])}
+        for g in groups
     ]
 
-    albums = [
-        {
-            "artisthash": a["artisthash"],
-            "albums": [serialize_for_card(a_) for a_ in (a["albums"])],
-        }
-        for a in albums
-        if len(a["albums"]) > 0
-    ]
-
-    return albums
-
 
 class GetAlbumVersionsBody(ArtistHashSchema):
     og_album_title: str = Field(
@@ -165,18 +137,29 @@ def get_album_versions(body: GetAlbumVersionsBody):
     base_title = body.base_title
     artisthash = body.artisthash
 
-    albums = AlbumStore.get_albums_by_artisthash(artisthash)
-
+    albums = AlbumDb.get_albums_by_base_title(base_title)
+    print(albums)
     albums = [
         a
         for a in albums
-        if create_hash(a.base_title) == create_hash(base_title)
-        and create_hash(og_album_title) != create_hash(a.og_title)
+        if a.og_title != og_album_title
+        and artisthash in {a["artisthash"] for a in a.albumartists}
     ]
 
-    for a in albums:
-        tracks = TrackStore.get_tracks_by_albumhash(a.albumhash)
-        a.get_date_from_tracks(tracks)
+    print(albums)
+
+    # albums = AlbumStore.get_albums_by_artisthash(artisthash)
+
+    # albums = [
+    #     a
+    #     for a in albums
+    #     if create_hash(a.base_title) == create_hash(base_title)
+    #     and create_hash(og_album_title) != create_hash(a.og_title)
+    # ]
+
+    # for a in albums:
+    #     tracks = TrackStore.get_tracks_by_albumhash(a.albumhash)
+    #     a.get_date_from_tracks(tracks)
 
     return albums
 

app/api/artist.py

@@ -133,7 +133,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery):
             AlbumStore.remove_album_by_hash(a.albumhash)
             continue
 
-        a.check_is_single(album_tracks)
+        a.is_single(album_tracks)
 
     all_albums = sorted(all_albums, key=lambda a: str(a.date), reverse=True)
 

app/api/folder.py

@@ -66,7 +66,9 @@ def get_folder_tree(body: FolderTree):
     else:
         req_dir = "/" + req_dir if not req_dir.startswith("/") else req_dir
 
+    print('stuff!')
     res = GetFilesAndDirs(req_dir, tracks_only=tracks_only)()
+    print(res['folders'])
     res["folders"] = sorted(res["folders"], key=lambda i: i.name)
 
     return res

app/api/getall/__init__.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field
 
 from datetime import datetime
 from app.api.apischemas import GenericLimitSchema
+from app.db import AlbumTable, ArtistTable
 from app.store.albums import AlbumStore
 from app.store.artists import ArtistStore
 
@@ -59,17 +60,19 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery):
     is_albums = path.itemtype == "albums"
     is_artists = path.itemtype == "artists"
 
-    items = AlbumStore.albums
+    if is_albums:
+        items = AlbumTable.get_all(query.start, query.limit)
+    elif is_artists:
+        items = ArtistTable.get_all(query.start, query.limit)
 
-    if is_artists:
-        items = ArtistStore.artists
+    print(items)
 
     start = query.start
     limit = query.limit
     sort = query.sortby
     reverse = query.reverse == "1"
 
-    sort_is_count = sort == "count"
+    sort_is_count = sort == "trackcount"
     sort_is_duration = sort == "duration"
     sort_is_create_date = sort == "created_date"
 
@@ -81,7 +84,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery):
 
     lambda_sort = lambda x: getattr(x, sort)
     if sort_is_artist:
-        lambda_sort = lambda x: getattr(x, sort)[0].name
+        lambda_sort = lambda x: getattr(x, sort)[0]["name"]
 
     sorted_items = sorted(items, key=lambda_sort, reverse=reverse)
     items = sorted_items[start : start + limit]
@@ -101,7 +104,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery):
 
         if sort_is_count:
             item_dict["help_text"] = (
-                f"{format_number(item.count)} track{'' if item.count == 1 else 's'}"
+                f"{format_number(item.trackcount)} track{'' if item.trackcount == 1 else 's'}"
             )
 
         if sort_is_duration:
@@ -114,7 +117,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery):
 
         if sort_is_artist_albumcount:
             item_dict["help_text"] = (
-                f"{format_number(item.albumcount)} album{'' if item.albumcount == 1 else 's'}"
+                f"{format_number(item['albumcount'])} album{'' if item['albumcount'] == 1 else 's'}"
             )
 
         album_list.append(item_dict)

app/config.py

@@ -21,6 +21,7 @@ class UserConfig:
     rootDirs: list[str] = field(default_factory=list)
     excludeDirs: list[str] = field(default_factory=list)
     artistSeparators: set[str] = field(default_factory=list)
+    genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"})
 
     # tracks
     extractFeaturedArtists: bool = True

app/db/__init__.py

@@ -0,0 +1,232 @@
+import json
+from pprint import pprint
+from typing import Any, Optional
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Integer,
+    Row,
+    String,
+    Tuple,
+    create_engine,
+    insert,
+    select,
+)
+from sqlalchemy.orm import (
+    Mapped,
+    mapped_column,
+    DeclarativeBase,
+    MappedAsDataclass,
+    sessionmaker,
+)
+
+from app.models import Track as TrackModel
+from app.models import Album as AlbumModel
+from app.utils.remove_duplicates import remove_duplicates
+
+
+fullpath = "/home/cwilvx/temp/swingmusic/swing.db"
+engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=False)
+
+
+def todict(track: Any):
+    return track._asdict()
+
+
+def todicts(tracks: list[Any]):
+    return [todict(track) for track in tracks]
+
+
+class DbManager:
+    def __init__(self):
+        self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True)
+        self.conn = self.engine.connect()
+
+    def __enter__(self):
+        return self.conn.execution_options(preserve_rowcount=True)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.conn.commit()
+        self.conn.close()
+
+
+class Base(MappedAsDataclass, DeclarativeBase):
+    @classmethod
+    def insert_many(cls, items: list[dict[str, Any]]):
+        """
+        Inserts multiple items into the database.
+        """
+        with DbManager() as conn:
+            conn.execute(insert(cls).values(items))
+
+    @classmethod
+    def insert_one(cls, item: dict[str, Any]):
+        """
+        Inserts a single item into the database.
+        """
+        return cls.insert_many([item])
+
+    @classmethod
+    def get_all(cls):
+        """
+        Returns all the items from the database.
+        """
+        with DbManager() as conn:
+            result = conn.execute(select(cls))
+            return result.fetchall()
+
+
+class ArtistTable(Base):
+    __tablename__ = "artist"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    albumcount: Mapped[int] = mapped_column(Integer())
+    artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True)
+    created_date: Mapped[int] = mapped_column(Integer())
+    date: Mapped[int] = mapped_column(Integer())
+    duration: Mapped[int] = mapped_column(Integer())
+    genres: Mapped[str] = mapped_column(JSON())
+    name: Mapped[str] = mapped_column(String(), index=True)
+    trackcount: Mapped[int] = mapped_column(Integer())
+    is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean())
+
+    @classmethod
+    def get_all(cls, start: int, limit: int):
+        with DbManager() as conn:
+            result = conn.execute(select(cls).offset(start).limit(limit))
+            return albums_to_dataclasses(result.fetchall())
+
+
+class AlbumTable(Base):
+    __tablename__ = "album"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True)
+    albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True)
+    base_title: Mapped[str] = mapped_column(String())
+    color: Mapped[Optional[str]] = mapped_column(String())
+    created_date: Mapped[int] = mapped_column(Integer())
+    date: Mapped[int] = mapped_column(Integer())
+    duration: Mapped[int] = mapped_column(Integer())
+    genres: Mapped[str] = mapped_column(JSON())
+    og_title: Mapped[str] = mapped_column(String())
+    title: Mapped[str] = mapped_column(String())
+    trackcount: Mapped[int] = mapped_column(Integer())
+
+    @classmethod
+    def get_album_by_albumhash(cls, hash: str):
+        with DbManager() as conn:
+            result = conn.execute(
+                select(AlbumTable).where(AlbumTable.albumhash == hash)
+            )
+            album = result.fetchone()
+
+            if album:
+                return album_to_dataclass(album)
+
+    @classmethod
+    def get_all(cls, start: int, limit: int):
+        with DbManager() as conn:
+            result = conn.execute(select(AlbumTable).offset(start).limit(limit))
+            return albums_to_dataclasses(result.fetchall())
+
+    @classmethod
+    def get_albums_by_artisthashes(cls, artisthashes: list[dict[str, str]]):
+        with DbManager() as conn:
+            albums: list[AlbumModel] = []
+
+            for artist in artisthashes:
+                result = conn.execute(
+                    # NOTE: The artist dict keys need to in the same order they appear in the db for this to work!
+                    select(AlbumTable).where(AlbumTable.albumartists.contains(artist))
+                )
+                albums.extend(albums_to_dataclasses(result.fetchall()))
+
+            print(albums)
+            return albums
+
+    @classmethod
+    def get_albums_by_base_title(cls, base_title: str):
+        with DbManager() as conn:
+            result = conn.execute(
+                select(AlbumTable).where(AlbumTable.base_title == base_title)
+            )
+            return albums_to_dataclasses(result.fetchall())
+
+
+class TrackTable(Base):
+    __tablename__ = "track"
+
+    id: Mapped[int] = mapped_column(init=False, primary_key=True)
+    album: Mapped[str] = mapped_column(String())
+    albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON())
+    albumhash: Mapped[str] = mapped_column(String(), index=True)
+    artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True)
+    bitrate: Mapped[int] = mapped_column(Integer())
+    copyright: Mapped[Optional[str]] = mapped_column(String())
+    date: Mapped[int] = mapped_column(Integer())
+    disc: Mapped[int] = mapped_column(Integer())
+    duration: Mapped[int] = mapped_column(Integer())
+    filepath: Mapped[str] = mapped_column(String(), unique=True)
+    folder: Mapped[str] = mapped_column(String(), index=True)
+    genre: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON())
+    last_mod: Mapped[float] = mapped_column(Integer())
+    og_album: Mapped[str] = mapped_column(String())
+    og_title: Mapped[str] = mapped_column(String())
+    title: Mapped[str] = mapped_column(String())
+    track: Mapped[int] = mapped_column(Integer())
+    trackhash: Mapped[str] = mapped_column(String(), index=True)
+
+    @classmethod
+    def get_tracks_by_filepaths(cls, filepaths: list[str]):
+        print(filepaths[0])
+        with DbManager() as conn:
+            result = conn.execute(
+                select(TrackTable).where(TrackTable.filepath.in_(filepaths))
+            )
+            return [dict(r) for r in result.mappings().fetchall()]
+
+    @classmethod
+    def count_tracks_containing_paths(cls, paths: list[str]):
+        results: list[dict[str, int | str]] = []
+
+        with DbManager() as conn:
+            for path in paths:
+                result = conn.execute(
+                    select(TrackTable).where(TrackTable.filepath.contains(path))
+                )
+                results.append({"path": path, "trackcount": result.all().__len__()})
+
+        return results
+
+    @classmethod
+    def get_tracks_by_albumhash(cls, albumhash: str):
+        with DbManager() as conn:
+            result = conn.execute(
+                select(TrackTable).where(TrackTable.albumhash == albumhash)
+            )
+            tracks = tracks_to_dataclasses(result.fetchall())
+            return remove_duplicates(tracks, is_album_tracks=True)
+
+
+# SECTION: HELPER FUNCTIONS
+
+
+def album_to_dataclass(album: Row[AlbumTable]):
+    return AlbumModel(**album._asdict())
+
+
+def albums_to_dataclasses(albums: list[Row[AlbumTable]]):
+    return [album_to_dataclass(album) for album in albums]
+
+
+def track_to_dataclass(track: Row[TrackTable]):
+    return TrackModel(**track._asdict())
+
+
+def tracks_to_dataclasses(tracks: list[Row[TrackTable]]):
+    return [track_to_dataclass(track) for track in tracks]
+
+
+Base().metadata.create_all(engine)

app/db/sqlite/auth.py

@@ -75,7 +75,6 @@ class SQLiteAuthMethods:
         {', '.join([f"{key} = :{key}" for key in keys if key != 'id'])}
         WHERE id = :id
         """
-        print(sql, user)
 
         with SQLiteManager(userdata_db=True) as cur:
             cur.execute(sql, user)
@@ -140,7 +139,6 @@ class SQLiteAuthMethods:
         Delete a user by username.
         """
         sql = "DELETE FROM users WHERE id = ?"
-        print("deleting user: ", username)
         with SQLiteManager(userdata_db=True) as cur:
             cur.execute(sql, (3,))
             cur.close()

app/db/sqlite/logger/tracks.py

@@ -1,6 +1,7 @@
 from flask_jwt_extended import current_user
 from app.db.sqlite.utils import SQLiteManager
 from app.models.logger import TrackLog as TrackLog
+from app.utils.auth import get_current_userid
 
 
 class SQLiteTrackLogger:
@@ -10,6 +11,7 @@ class SQLiteTrackLogger:
         Inserts a track play record into the database
         """
 
+        userid = get_current_userid()
         with SQLiteManager(userdata_db=True) as cur:
             sql = """INSERT OR REPLACE INTO track_logger(
                 trackhash,
@@ -21,7 +23,7 @@ class SQLiteTrackLogger:
                 """
 
             cur.execute(
-                sql, (trackhash, duration, timestamp, source, current_user["id"])
+                sql, (trackhash, duration, timestamp, source, userid)
             )
             lastrowid = cur.lastrowid
 
@@ -34,7 +36,8 @@ class SQLiteTrackLogger:
         """
 
         with SQLiteManager(userdata_db=True) as cur:
-            sql = f"""SELECT * FROM track_logger WHERE userid = {current_user['id']} ORDER BY timestamp DESC"""
+            userid = get_current_userid()
+            sql = f"""SELECT * FROM track_logger WHERE userid = {userid} ORDER BY timestamp DESC"""
 
             cur.execute(sql)
             rows = cur.fetchall()

app/db/sqlite/playlists.py

@@ -60,7 +60,15 @@ class SQLitePlaylistMethods:
     @staticmethod
     def get_all_playlists():
         with SQLiteManager(userdata_db=True) as cur:
-            cur.execute(f"SELECT * FROM playlists WHERE userid = {current_user['id']}")
+            userid = 1
+
+            try:
+                userid = current_user["id"]
+            except RuntimeError:
+                # Catch this error raised during migration execution
+                pass
+
+            cur.execute(f"SELECT * FROM playlists WHERE userid = {userid}")
             playlists = cur.fetchall()
             cur.close()
 
@@ -92,7 +100,15 @@ class SQLitePlaylistMethods:
         Adds a string item to a json dumped list using a playlist id and field name.
         Takes the playlist ID, a field name, an item to add to the field.
         """
-        sql = f"SELECT {field} FROM playlists WHERE id = ? and userid = {current_user['id']}"
+        userid = 1
+
+        try:
+            userid = current_user["id"]
+        except RuntimeError:
+            # Catch this error raised during migration execution
+            pass
+
+        sql = f"SELECT {field} FROM playlists WHERE id = ? and userid = {userid}"
 
         with SQLiteManager(userdata_db=True) as cur:
             cur.execute(sql, (playlist_id,))
@@ -173,10 +189,17 @@ class SQLitePlaylistMethods:
         """
 
         sql = """UPDATE playlists SET trackhashes = ? WHERE id = ?"""
+        userid = 1
+
+        try:
+            userid = current_user["id"]
+        except RuntimeError:
+            # Catch this error raised during migration execution
+            pass
 
         with SQLiteManager(userdata_db=True) as cur:
             cur.execute(
-                f"SELECT trackhashes FROM playlists WHERE id = ? and userid = {current_user['id']}",
+                f"SELECT trackhashes FROM playlists WHERE id = ? and userid = {userid}",
                 (playlistid,),
             )
             data = cur.fetchone()
@@ -185,17 +208,20 @@ class SQLitePlaylistMethods:
                 return
 
             trackhashes: list[str] = json.loads(data[0])
+            to_remove = []
 
             for track in tracks:
                 # {
                 #    trackhash: str;
                 #    index: int;
                 # }
-
                 index = trackhashes.index(track["trackhash"])
 
                 if index == track["index"]:
-                    trackhashes.remove(track["trackhash"])
+                    to_remove.append(track["trackhash"])
+
+            for trackhash in to_remove:
+                trackhashes.remove(trackhash)
 
             cur.execute(sql, (json.dumps(trackhashes), playlistid))
 

app/db/sqlite/tracks.py

@@ -99,6 +99,20 @@ class SQLiteTrackMethods:
 
             return None
         
+    @staticmethod
+    def get_track_by_albumhash(albumhash: str):
+        """
+        Gets a track using its albumhash. Returns a Track object or None.
+        """
+        with SQLiteManager() as cur:
+            cur.execute("SELECT * FROM tracks WHERE albumhash=?", (albumhash,))
+            row = cur.fetchone()
+
+            if row is not None:
+                return tuple_to_track(row)
+
+            return None
+
     @staticmethod
     def remove_tracks_by_filepaths(filepaths: str | set[str]):
         """

app/lib/folderslib.py

@@ -8,6 +8,7 @@ from app.settings import SUPPORTED_FILES
 from app.utils.wintools import win_replace_slash
 
 from app.store.tracks import TrackStore
+from app.db import TrackTable as TrackDB
 
 
 def create_folder(path: str, trackcount=0, foldercount=0) -> Folder:
@@ -37,44 +38,52 @@ def get_first_child_from_path(root: str, maybe_child: str):
 
     return os.path.join(root, first)
 
+
 def get_folders(paths: list[str]):
     """
     Filters out folders that don't have any tracks and
     returns a list of folder objects.
     """
-    count_dict = {
-        "tracks": {path: 0 for path in paths},
-        # folders are immediate children of the root folder
-        "folders": {path: set() for path in paths},
-    }
-
-    for track in TrackStore.tracks:
-        for path in paths:
-
-            # a child path should be longer than the root path
-            if len(track.folder) >= len(path) and track.folder.startswith(path):
-                count_dict["tracks"][path] += 1
-
-                # counting subfolders
-                p = get_first_child_from_path(path, track.folder)
-
-                if p:
-                    count_dict["folders"][path].add(p)
-
-    folders = [
-        {
-            "path": path,
-            "trackcount": count_dict["tracks"][path],
-            "foldercount": len(count_dict["folders"][path]),
-        }
-        for path in paths
-    ]
+    folders = TrackDB.count_tracks_containing_paths(paths)
 
     return [
-        create_folder(f["path"], f["trackcount"], f["foldercount"])
+        create_folder(f["path"], f["trackcount"], foldercount=0)
         for f in folders
         if f["trackcount"] > 0
     ]
+    # count_dict = {
+    #     "tracks": {path: 0 for path in paths},
+    #     # folders are immediate children of the root folder
+    #     "folders": {path: set() for path in paths},
+    # }
+
+    # for track in TrackStore.tracks:
+    #     for path in paths:
+
+    #         # a child path should be longer than the root path
+    #         if len(track.folder) >= len(path) and track.folder.startswith(path):
+    #             count_dict["tracks"][path] += 1
+
+    #             # counting subfolders
+    #             p = get_first_child_from_path(path, track.folder)
+
+    #             if p:
+    #                 count_dict["folders"][path].add(p)
+
+    # folders = [
+    #     {
+    #         "path": path,
+    #         "trackcount": count_dict["tracks"][path],
+    #         "foldercount": len(count_dict["folders"][path]),
+    #     }
+    #     for path in paths
+    # ]
+
+    # return [
+    #     create_folder(f["path"], f["trackcount"], f["foldercount"])
+    #     for f in folders
+    #     if f["trackcount"] > 0
+    # ]
 
 
 class GetFilesAndDirs:
@@ -131,7 +140,13 @@ class GetFilesAndDirs:
         files_.sort(key=lambda f: f["time"])
         files = [f["path"] for f in files_]
 
-        tracks = TrackStore.get_tracks_by_filepaths(files)
+        tracks = []
+        if files:
+            tracks = TrackDB.get_tracks_by_filepaths(files)
+            print("printing files")
+            print(tracks)
+
+        # tracks = TrackStore.get_tracks_by_filepaths(files)
 
         folders = []
         if not self.tracks_only:
@@ -145,7 +160,7 @@ class GetFilesAndDirs:
 
         return {
             "path": path,
-            "tracks": serialize_tracks(tracks),
+            "tracks": tracks,
             "folders": folders,
         }
 

app/lib/populate.py

@@ -7,6 +7,7 @@ from requests import ConnectionError as RequestConnectionError
 from requests import ReadTimeout
 
 from app import settings
+from app.db import TrackTable
 from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
 from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
 from app.db.sqlite.settings import SettingsSQLMethods as sdb
@@ -121,14 +122,14 @@ class Populate:
                 return
 
     @staticmethod
-    def remove_modified(tracks: Generator[Track, None, None]):
+    def remove_modified(tracks: Generator[TrackTable, None, None]):
         """
         Removes tracks from the database that have been modified
         since they were added to the database.
         """
 
         unmodified_paths = set()
-        modified_tracks: list[Track] = []
+        modified_tracks: list[TrackTable] = []
         modified_paths = set()
 
         for track in tracks:
@@ -151,18 +152,6 @@ class Populate:
 
     @staticmethod
     def tag_untagged(untagged: set[str], key: str):
-        log.info("Found %s new tracks", len(untagged))
-        tagged_tracks: deque[dict] = deque()
-        tagged_count = 0
-
-        favs = favdb.get_fav_tracks()
-        records = dict()
-
-        for fav in favs:
-            r = records.setdefault(fav[1], set())
-            r.add(fav[4])
-
-
         for file in tqdm(untagged, desc="Reading files"):
             if POPULATE_KEY != key:
                 log.warning("'Populate.tag_untagged': Populate key changed")
@@ -171,36 +160,49 @@ class Populate:
             tags = get_tags(file)
 
             if tags is not None:
-                tagged_tracks.append(tags)
-                track = Track(**tags)
+                TrackTable.insert_one(tags)
 
-                track.fav_userids = list(records.get(track.trackhash, set()))
+        # log.info("Found %s new tracks", len(untagged))
+        # # tagged_tracks: deque[dict] = deque()
+        # # tagged_count = 0
 
-                TrackStore.add_track(track)
+        # favs = favdb.get_fav_tracks()
+        # records = dict()
 
-                if not AlbumStore.album_exists(track.albumhash):
-                    AlbumStore.add_album(AlbumStore.create_album(track))
+        # for fav in favs:
+        #     r = records.setdefault(fav[1], set())
+        #     r.add(fav[4])
 
-                for artist in track.artists:
-                    if not ArtistStore.artist_exists(artist.artisthash):
-                        ArtistStore.add_artist(Artist(artist.name))
+            #     tagged_tracks.append(tags)
+            #     track = Track(**tags)
 
-                for artist in track.albumartists:
-                    if not ArtistStore.artist_exists(artist.artisthash):
-                        ArtistStore.add_artist(Artist(artist.name))
+            #     track.fav_userids = list(records.get(track.trackhash, set()))
 
-                tagged_count += 1
-            else:
-                log.warning("Could not read file: %s", file)
+            #     TrackStore.add_track(track)
 
-        if len(tagged_tracks) > 0:
-            log.info("Adding %s tracks to database", len(tagged_tracks))
-            insert_many_tracks(tagged_tracks)
+            #     if not AlbumStore.album_exists(track.albumhash):
+            #         AlbumStore.add_album(AlbumStore.create_album(track))
 
-        log.info("Added %s/%s tracks", tagged_count, len(untagged))
+            #     for artist in track.artists:
+            #         if not ArtistStore.artist_exists(artist.artisthash):
+            #             ArtistStore.add_artist(Artist(artist.name))
+
+            #     for artist in track.albumartists:
+            #         if not ArtistStore.artist_exists(artist.artisthash):
+            #             ArtistStore.add_artist(Artist(artist.name))
+
+            #     tagged_count += 1
+            # else:
+            #     log.warning("Could not read file: %s", file)
+
+        # if len(tagged_tracks) > 0:
+        #     log.info("Adding %s tracks to database", len(tagged_tracks))
+        #     insert_many_tracks(tagged_tracks)
+
+        # log.info("Added %s/%s tracks", tagged_count, len(untagged))
 
     @staticmethod
-    def extract_thumb_with_overwrite(tracks: list[Track]):
+    def extract_thumb_with_overwrite(tracks: list[TrackTable]):
         """
         Extracts the thumbnail from a list of filepaths,
         overwriting the existing thumbnail if it exists,

app/lib/searchlib.py

@@ -195,7 +195,7 @@ class TopResults:
             except AttributeError:
                 item.duration = 0
 
-            item.check_is_single(tracks)
+            item.is_single(tracks)
 
             if not item.is_single:
                 item.check_type()

app/lib/tagger.py

@@ -0,0 +1,154 @@
+from pprint import pprint
+from app.db import AlbumTable, ArtistTable, TrackTable
+from app.lib.taglib import get_tags
+from app.utils.filesystem import run_fast_scandir
+from app.utils.parsers import get_base_album_title
+from app.utils.progressbar import tqdm
+
+
+class IndexTracks:
+    def __init__(self) -> None:
+        dirs_to_scan = ["/home/cwilvx/Music"]
+
+        files = set()
+
+        for _dir in dirs_to_scan:
+            files = files.union(run_fast_scandir(_dir, full=True)[1])
+
+        self.tag_untagged(files)
+        # unmodified, modified_tracks = self.remove_modified(tracks)
+        # untagged = files - unmodified
+
+    def tag_untagged(self, files: set[str]):
+        for file in tqdm(files, desc="Reading files"):
+            # if POPULATE_KEY != key:
+            #     log.warning("'Populate.tag_untagged': Populate key changed")
+            #     return
+
+            tags = get_tags(file)
+
+            if tags is not None:
+                TrackTable.insert_one(tags)
+
+
+class IndexAlbums:
+    def __init__(self) -> None:
+        albums = dict()
+
+        all_tracks: list[TrackTable] = TrackTable.get_all()
+
+        for track in all_tracks:
+            if track.albumhash not in albums:
+                albums[track.albumhash] = {
+                    "albumartists": track.albumartists,
+                    "albumhash": track.albumhash,
+                    "base_title": None,
+                    "color": None,
+                    "created_date": None,
+                    "date": None,
+                    "duration": track.duration,
+                    "genres": [*track.genre] if track.genre else [],
+                    "og_title": track.og_album,
+                    "title": track.album,
+                    "trackcount": 1,
+                    "dates": [track.date],
+                    "created_dates": [track.last_mod],
+                }
+            else:
+                album = albums[track.albumhash]
+                album["trackcount"] += 1
+                album["duration"] += track.duration
+                album["dates"].append(track.date)
+                album["created_dates"].append(track.last_mod)
+
+                if track.genre:
+                    album["genres"].append(track.genre)
+
+        for album in albums.values():
+            album["date"] = min(album["dates"])
+            album["created_date"] = min(album["created_dates"])
+
+            genres = []
+
+            for genre in album["genres"]:
+                if genre not in genres:
+                    genres.append(genre)
+
+            album["genres"] = genres
+            album["base_title"], _ = get_base_album_title(album["og_title"])
+
+            del album["dates"]
+            del album["created_dates"]
+
+        pprint(albums)
+
+        AlbumTable.insert_many(list(albums.values()))
+
+
+class IndexArtists:
+    def __init__(self) -> None:
+        all_tracks: list[TrackTable] = TrackTable.get_all()
+        artists = dict()
+
+        for track in all_tracks:
+            this_artists = track.artists
+
+            for a in track.albumartists:
+                if a not in this_artists:
+                    this_artists.append(a)
+
+            for artist in this_artists:
+                if artist["artisthash"] not in artists:
+                    artists[artist["artisthash"]] = {
+                        "albumcount": None,
+                        "albums": {track.albumhash},
+                        "artisthash": artist["artisthash"],
+                        "created_dates": [track.last_mod],
+                        "dates": [track.date],
+                        "date": None,
+                        "duration": track.duration,
+                        "genres": [*track.genre] if track.genre else [],
+                        "name": artist["name"],
+                        "trackcount": None,
+                        "tracks": {track.trackhash},
+                    }
+                else:
+                    artist = artists[artist["artisthash"]]
+                    artist["duration"] += track.duration
+                    artist["albums"].add(track.albumhash)
+                    artist["tracks"].add(track.trackhash)
+                    artist["dates"].append(track.date)
+                    artist["created_dates"].append(track.last_mod)
+
+                    if track.genre:
+                        artist["genres"].append(track.genre)
+
+
+        for artist in artists.values():
+            artist["albumcount"] = len(artist["albums"])
+            artist["trackcount"] = len(artist["tracks"])
+            artist["date"] = min(artist["dates"])
+            artist["created_date"] = min(artist["created_dates"])
+
+            genres = []
+
+            for genre in artist["genres"]:
+                if genre not in genres:
+                    genres.append(genre)
+
+            artist["genres"] = genres
+
+            del artist["tracks"]
+            del artist["albums"]
+            del artist["dates"]
+            del artist["created_dates"]
+
+        pprint(artists)
+        ArtistTable.insert_many(list(artists.values()))
+
+class IndexEverything:
+    def __init__(self) -> None:
+        # IndexTracks()
+        # IndexAlbums()
+        # IndexArtists()
+        pass

app/lib/taglib.py

@@ -8,9 +8,16 @@ import pendulum
 from PIL import Image, UnidentifiedImageError
 from tinytag import TinyTag
 
+from app.config import UserConfig
 from app.settings import Defaults, Paths
 from app.utils.hashing import create_hash
-from app.utils.parsers import split_artists
+from app.utils.parsers import (
+    clean_title,
+    get_base_title_and_versions,
+    parse_feat_from_title,
+    remove_prod,
+    split_artists,
+)
 from app.utils.wintools import win_replace_slash
 
 
@@ -206,9 +213,7 @@ def get_tags(filepath: str):
     except KeyError:
         tags.copyright = None
 
-    tags.albumhash = create_hash(tags.album, tags.albumartist)
-    tags.trackhash = create_hash(tags.artist, tags.album, tags.title)
-    tags.image = f"{tags.albumhash}.webp"
+    # tags.image = f"{tags.albumhash}.webp"
     tags.folder = win_replace_slash(os.path.dirname(filepath))
 
     tags.date = parse_date(tags.year) or int(last_mod)
@@ -218,9 +223,100 @@ def get_tags(filepath: str):
     tags.artists = tags.artist
     tags.albumartists = tags.albumartist
 
+    split_artist = split_artists(tags.artist)
+    split_albumartists = split_artists(tags.albumartist)
+    new_title = tags.title
+
+    # TODO: Figure out which is the best spot to create these hashes
+    # create albumhash using og_album
+    tags.albumhash = create_hash(tags.album or "", tags.albumartist)
+
+    config = UserConfig()
+
+    # extract featured artists
+    if config.extractFeaturedArtists:
+        feat, new_title = parse_feat_from_title(tags.title)
+        original_lower = "-".join([create_hash(a) for a in split_artist])
+        split_artist.extend(a for a in feat if create_hash(a) not in original_lower)
+
+    # if no albumartist, assign to the first artist
+    if not tags.albumartist:
+        tags.albumartist = split_artist[:1]
+
+    # create json objects for artists and albumartists
+    tags.artists = [
+        {
+            "artisthash": create_hash(a, decode=True),
+            "name": a,
+        }
+        for a in split_artist
+    ]
+
+    tags.albumartists = [
+        {
+            "artisthash": create_hash(a, decode=True),
+            "name": a,
+        }
+        for a in split_albumartists
+    ]
+
+    # remove prod by
+    if config.removeProdBy:
+        new_title = remove_prod(new_title)
+
+    # if track is a single, ie.
+    # if og_title == album, rename album to new_title
+    if tags.title == tags.album:
+        tags.album = new_title
+
+    # remove remaster from track title
+    if config.removeRemasterInfo:
+        new_title = clean_title(new_title)
+
+    # save final title
+    tags.og_title = tags.title
+    tags.title = new_title
+    tags.og_album = tags.album
+
+    # clean album title
+    if config.cleanAlbumTitle:
+        tags.album, _ = get_base_title_and_versions(tags.album, get_versions=False)
+
+    # merge album versions
+    if config.mergeAlbums:
+        tags.albumhash = create_hash(
+            tags.album, *(a["name"] for a in tags.albumartists)
+        )
+
+    # process genres
+    if tags.genre:
+        tags.genre = tags.genre.lower()
+        # separators = {"/", ";", "&"}
+        separators = set(config.genreSeparators)
+
+        contains_rnb = "r&b" in tags.genre
+        contains_rock = "rock & roll" in tags.genre
+
+        if contains_rnb:
+            tags.genre = tags.genre.replace("r&b", "RnB")
+
+        if contains_rock:
+            tags.genre = tags.genre.replace("rock & roll", "rock")
+
+        for s in separators:
+            tags.genre = tags.genre.replace(s, ",")
+
+        tags.genre = tags.genre.split(",")
+        tags.genre = [
+            {"name": g.strip(), "genrehash": create_hash(g.strip())} for g in tags.genre
+        ]
+
     # sub underscore with space
     tags.title = tags.title.replace("_", " ")
     tags.album = tags.album.replace("_", " ")
+    tags.trackhash = create_hash(
+        *[a["name"] for a in tags.artists], tags.album, tags.title
+    )
 
     tags = tags.__dict__
 

app/migrations/__init__.py

@@ -5,6 +5,7 @@ Reads and applies the latest database migrations.
 """
 
 import inspect
+import sys
 from types import ModuleType
 from app.db.sqlite.migrations import MigrationManager
 from app.logger import log
@@ -55,11 +56,12 @@ def apply_migrations():
         to_apply = all_migrations[index:]
 
     for migration in to_apply:
-        try:
+        # try:
             migration.migrate()
             log.info("Applied migration: %s", migration.__name__)
-        except Exception as e:
-            log.error("Failed to run migration: %s", migration.__name__)
-            log.error(e)
+        # except Exception as e:
+        #     log.error("Failed to run migration: %s", migration.__name__)
+        #     log.error(e)
 
+    # sys.exit(0)
     MigrationManager.set_index(len(all_migrations))

app/migrations/v1_4_9/__init__.py

@@ -1,9 +1,90 @@
 import os
 import shutil
+import sqlite3
+from time import time
 from app.db.sqlite.utils import SQLiteManager
 from app.migrations.base import Migration
 from app.settings import Paths
 
+import hashlib
+from unidecode import unidecode
+
+from app.db.sqlite.tracks import SQLiteTrackMethods as tdb
+from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb
+from app.db.sqlite.logger.tracks import SQLiteTrackLogger as ldb
+from app.utils.hashing import create_hash
+
+
+def create_sha256_hash(*args: str, decode=False, limit=10) -> str:
+    """
+    This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments.
+
+    Example use case:
+        - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID.
+
+    :param args: The arguments to hash.
+    :param decode: Whether to decode the arguments before hashing.
+    :param limit: The number of characters to return.
+
+    :return: The hash.
+    """
+
+    def remove_non_alnum(token: str) -> str:
+        token = token.lower().strip().replace(" ", "")
+        t = "".join(t for t in token if t.isalnum())
+
+        if t == "":
+            return token
+
+        return t
+
+    str_ = "".join(remove_non_alnum(t) for t in args)
+
+    if decode:
+        str_ = unidecode(str_)
+
+    str_ = str_.encode("utf-8")
+    str_ = hashlib.sha256(str_).hexdigest()
+    return str_[-limit:]
+
+
+def create_sha1_hash(*args: str, decode=False, limit=10) -> str:
+    """
+    This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments.
+
+    Example use case:
+        - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID.
+
+    :param args: The arguments to hash.
+    :param decode: Whether to decode the arguments before hashing.
+    :param limit: The number of characters to return.
+
+    :return: The hash.
+    """
+
+    def remove_non_alnum(token: str) -> str:
+        token = token.lower().strip().replace(" ", "")
+        t = "".join(t for t in token if t.isalnum())
+
+        if t == "":
+            return token
+
+        return t
+
+    str_ = "".join(remove_non_alnum(t) for t in args)
+
+    if decode:
+        str_ = unidecode(str_)
+
+    str_ = str_.encode("utf-8")
+    str_ = hashlib.sha1(str_).hexdigest()
+
+    return (
+        str_[: limit // 2] + str_[-limit // 2 :]
+        if limit % 2 == 0
+        else str_[: limit // 2] + str_[-limit // 2 - 1 :]
+    )
+
 
 class _1AddTimestampToFavoritesTable(Migration):
     """
@@ -13,37 +94,23 @@ class _1AddTimestampToFavoritesTable(Migration):
     @staticmethod
     def migrate():
         # INFO: add timestamp column with automatic current timestamp
-        sql = f"ALTER TABLE favorites ADD COLUMN IF NOT EXISTS timestamp INTEGER NOT NULL DEFAULT 0"
+        sql = f"ALTER TABLE favorites ADD COLUMN timestamp INTEGER NOT NULL DEFAULT 0"
 
         # INFO: execute the sql
         with SQLiteManager(userdata_db=True) as cur:
-            try:
-                # INFO: Add the timestamp column to the favorites table
-                cur.execute(sql)
+            table_exists = cur.execute(
+                "select count(*) from pragma_table_info('favorites') where name = 'timestamp'"
+            )
 
-                # INFO: Set all the timestamps to the current time
-                cur.execute("UPDATE favorites SET timestamp = strftime('%s', 'now')")
-            except Exception as e:
-                # INFO: timestamp column already exists
-                pass
-            finally:
-                cur.close()
+            table_exists = table_exists.fetchone()
 
+            if table_exists[0] == 1:
+                return
 
-class _4MoveHashesToSha1(Migration):
-    """
-    Moves the 10 bit item hashes from sha256 to sha1 which is
-    faster and more lenient on less powerful devices.
-
-    Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution.
-    """
-
-    enabled: bool = False
-
-    pass
-
-    # INFO: Apparentlly, every single table is affected by this migration.
-    # NOTE: Use generators to avoid memory issues.
+            # INFO: Add the timestamp column to the favorites table
+            timestamp = int(time())
+            cur.execute(sql)
+            cur.execute(f"UPDATE favorites SET timestamp = {timestamp}")
 
 
 class _2DeleteOriginalThumbnails(Migration):
@@ -175,3 +242,164 @@ class _5AddUserIdToPlaylistsTable(Migration):
 
             # INFO: Execute the sql
             cur.executescript(sql)
+
+
+class _6MoveHashesToSha1(Migration):
+    """
+    Moves the 10 bit item hashes from sha256 to sha1 which is
+    faster and more lenient on less powerful devices.
+
+    Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution.
+    """
+
+    # enabled: bool = False
+
+    # pass
+
+    # INFO: Apparentlly, every single table is affected by this migration.
+    # NOTE: Use generators to avoid memory issues.
+
+    @classmethod
+    def port_track(cls, trackhash: str):
+        # get the track with the track hash
+        track = tdb.get_track_by_trackhash(trackhash)
+
+        if track is None:
+            return
+
+        title = track.og_title
+        if track.trackhash != trackhash:
+            # raise ValueError("Track hash mismatch")
+            print("Track hash mismatch")
+            title = track.title
+        else:
+            print("Porting track: ", track.title)
+
+        # return the new hash
+        finalhash = create_sha1_hash(
+            ", ".join(a.name for a in track.artists),
+            track.og_album,
+            title,
+        )
+
+        if finalhash != create_hash(
+            ", ".join(a.name for a in track.artists), track.og_album, title
+        ):
+            raise ValueError("Hash mismatch")
+
+    @classmethod
+    def port_album(cls, albumhash: str):
+        # get the first track with the album hash
+        track = tdb.get_track_by_albumhash(albumhash)
+
+        if track is None:
+            return
+
+        # return the new hash
+        return create_sha1_hash(
+            track.og_album,
+            ", ".join(a.name for a in track.albumartists),
+        )
+
+    @classmethod
+    def port_artist(cls, artisthash: str):
+        # find all tracks with the artist hash
+        tracks = [t for t in cls.tracks if artisthash in t.artist_hashes]
+
+        if len(tracks) == 0:
+            return
+
+        # find the artist name
+        artist = [
+            a.name
+            for a in tracks[0].artists
+            if create_sha256_hash(a.name, decode=True) == artisthash
+        ][0]
+
+        # return the new hash
+        return create_sha1_hash(artist, decode=True)
+
+    @classmethod
+    def migrate_favorites(cls):
+        with SQLiteManager(userdata_db=True) as cur:
+            # read all favorites
+            data = cur.execute("SELECT * FROM favorites")
+            data = data.fetchall()
+
+            for track in cls.tracks:
+                track.artist_hashes = "-".join(
+                    [create_sha256_hash(a.name, decode=True) for a in track.artists]
+                )
+
+            for entry in data:
+                # hash is the 2nd column in the table
+                hash = entry[1]
+
+                # entry type is the 3rd column in the table
+                if entry[2] == "track":
+                    newhash = cls.port_track(hash)
+
+                    if newhash:
+                        cur.execute(
+                            f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'track'"
+                        )
+
+                elif entry[2] == "album":
+                    newhash = cls.port_album(hash)
+
+                    if newhash:
+                        cur.execute(
+                            f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'album'"
+                        )
+
+                elif entry[2] == "artist":
+                    newhash = cls.port_artist(hash)
+
+                    if newhash:
+                        cur.execute(
+                            f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'artist'"
+                        )
+
+    @classmethod
+    def migrate_playlists(cls):
+        playlists = pdb.get_all_playlists()
+
+        for playlist in playlists:
+            # remove previous hashes
+            to_remove = [
+                {"trackhash": trackhash, "index": index}
+                for index, trackhash in enumerate(playlist.trackhashes)
+            ]
+            pdb.remove_tracks_from_playlist(playlist.id, to_remove)
+
+            # add new hashes
+            newhashes = [
+                cls.port_track(trackhash) for trackhash in playlist.trackhashes
+            ]
+            newhashes = [h for h in newhashes if h is not None]
+            pdb.add_tracks_to_playlist(playlist.id, newhashes)
+
+            print("Ported playlist: ", playlist.name)
+            print("Total tracks: ", len(newhashes))
+
+    @classmethod
+    def migrate_scrobble(cls):
+        # read all logs
+        logs = ldb.get_all()
+
+        with SQLiteManager(userdata_db=True) as cur:
+            # for each log, port the hash
+            for log in logs:
+                newhash = cls.port_track(log[1])
+
+                if newhash:
+                    cur.execute(
+                        f"UPDATE track_logger SET trackhash = '{newhash}' WHERE trackhash = '{log[1]}'"
+                    )
+
+    @classmethod
+    def migrate(cls):
+        cls.tracks = list(tdb.get_all_tracks())
+        cls.migrate_favorites()
+        # cls.migrate_playlists()
+        # cls.migrate_scrobble()

app/models/album.py

@@ -2,6 +2,7 @@ import dataclasses
 import datetime
 from dataclasses import dataclass
 
+from app.config import UserConfig
 from app.settings import SessionVarKeys, get_flag
 
 from ..utils.hashing import create_hash
@@ -16,94 +17,111 @@ class Album:
     Creates an album object
     """
 
+    id: int
+    albumartists: list[dict[str, str]]
     albumhash: str
+    base_title: str
+    color: str
+    created_date: int
+    date: int
+    duration: int
+    genres: list[dict[str, str]]
+    og_title: str
     title: str
-    albumartists: list[Artist]
+    trackcount: int
 
-    albumartists_hashes: str = ""
-    image: str = ""
-    count: int = 0
-    duration: int = 0
-    colors: list[str] = dataclasses.field(default_factory=list)
-    date: str = ""
-
-    created_date: int = 0
-    og_title: str = ""
-    base_title: str = ""
-    is_soundtrack: bool = False
-    is_compilation: bool = False
-    is_single: bool = False
-    is_EP: bool = False
-    is_favorite: bool = False
-    is_live: bool = False
-
-    genres: list[str] = dataclasses.field(default_factory=list)
+    type: str = "album"
     versions: list[str] = dataclasses.field(default_factory=list)
 
     def __post_init__(self):
-        self.title = self.title.strip()
-        self.og_title = self.title
-        self.image = self.albumhash + ".webp"
+        self.date = datetime.datetime.fromtimestamp(self.date).year
 
-        # Fetch album artists from title
-        if get_flag(SessionVarKeys.EXTRACT_FEAT):
-            featured, self.title = parse_feat_from_title(self.title)
+    # albumhash: str
+    # title: str
+    # albumartists: list[Artist]
 
-            if len(featured) > 0:
-                original_lower = "-".join([a.name.lower() for a in self.albumartists])
-                self.albumartists.extend(
-                    [Artist(a) for a in featured if a.lower() not in original_lower]
-                )
+    # albumartists_hashes: str = ""
+    # image: str = ""
+    # count: int = 0
+    # duration: int = 0
+    # colors: list[str] = dataclasses.field(default_factory=list)
+    # date: str = ""
 
-                from ..store.tracks import TrackStore
+    # created_date: int = 0
+    # og_title: str = ""
+    # base_title: str = ""
+    # is_soundtrack: bool = False
+    # is_compilation: bool = False
+    # is_single: bool = False
+    # is_EP: bool = False
+    # is_favorite: bool = False
+    # is_live: bool = False
 
-                TrackStore.append_track_artists(self.albumhash, featured, self.title)
+    # genres: list[str] = dataclasses.field(default_factory=list)
 
-        # Handle album version data
-        if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE):
-            get_versions = not get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS)
+    # def __post_init__(self):
+    #     self.title = self.title.strip()
+    #     self.og_title = self.title
+    #     self.image = self.albumhash + ".webp"
 
-            self.title, self.versions = get_base_title_and_versions(
-                self.title, get_versions=get_versions
-            )
-            self.base_title = self.title
+    #     # Fetch album artists from title
+    #     if get_flag(SessionVarKeys.EXTRACT_FEAT):
+    #         featured, self.title = parse_feat_from_title(self.title)
 
-            if "super_deluxe" in self.versions:
-                self.versions.remove("deluxe")
+    #         if len(featured) > 0:
+    #             original_lower = "-".join([a.name.lower() for a in self.albumartists])
+    #             self.albumartists.extend(
+    #                 [Artist(a) for a in featured if a.lower() not in original_lower]
+    #             )
 
-            if "original" in self.versions and self.check_is_soundtrack():
-                self.versions.remove("original")
+    #             from ..store.tracks import TrackStore
 
-            self.versions = [v.replace("_", " ") for v in self.versions]
-        else:
-            self.base_title = get_base_title_and_versions(
-                self.title, get_versions=False
-            )[0]
+    #             TrackStore.append_track_artists(self.albumhash, featured, self.title)
 
-        self.albumartists_hashes = "-".join(a.artisthash for a in self.albumartists)
+    #     # Handle album version data
+    #     else:
+    #         self.base_title = get_base_title_and_versions(
+    #             self.title, get_versions=False
+    #         )[0]
 
-    def set_colors(self, colors: list[str]):
-        self.colors = colors
+    #     self.albumartists_hashes = "-".join(a.artisthash for a in self.albumartists)
 
-    def check_type(self):
+    # # def set_colors(self, colors: list[str]):
+    # #     self.colors = colors
+    def populate_versions(self):
+        _, self.versions = get_base_title_and_versions(self.og_title, get_versions=True)
+
+        if "super_deluxe" in self.versions:
+            self.versions.remove("deluxe")
+
+        # at this point, we should know the type of album
+        if "original" in self.versions and self.type == "soundtrack":
+            self.versions.remove("original")
+
+        self.versions = [v.replace("_", " ") for v in self.versions]
+
+    def check_type(self, tracks: list[Track], singleTrackAsSingle: bool):
         """
         Runs all the checks to determine the type of album.
         """
-        self.is_soundtrack = self.check_is_soundtrack()
-        if self.is_soundtrack:
-            return
+        if self.is_single(tracks, singleTrackAsSingle):
+            return "single"
 
-        self.is_live = self.check_is_live_album()
-        if self.is_live:
-            return
+        if self.is_soundtrack():
+            return "soundtrack"
 
-        self.is_compilation = self.check_is_compilation()
-        if self.is_compilation:
-            return
+        if self.is_live_album():
+            return "live album"
 
-        self.is_EP = self.check_is_ep()
+        if self.is_compilation():
+            return "compilation"
 
-    def check_is_soundtrack(self) -> bool:
+        if self.is_ep():
+            return "ep"
+
+        return "album"
+
+    def is_soundtrack(self) -> bool:
         """
         Checks if the album is a soundtrack.
         """
@@ -114,11 +132,11 @@ class Album:
 
         return False
 
-    def check_is_compilation(self) -> bool:
+    def is_compilation(self) -> bool:
         """
         Checks if the album is a compilation.
         """
-        artists = [a.name for a in self.albumartists]
+        artists = [a["name"] for a in self.albumartists]
         artists = "".join(artists).lower()
 
         if "various artists" in artists:
@@ -137,7 +155,7 @@ class Album:
             "biggest hits",
             "the hits",
             "the ultimate",
-            "compilation"
+            "compilation",
         }
 
         for substring in substrings:
@@ -146,7 +164,7 @@ class Album:
 
         return False
 
-    def check_is_live_album(self):
+    def is_live_album(self):
         """
         Checks if the album is a live album.
         """
@@ -157,7 +175,7 @@ class Album:
 
         return False
 
-    def check_is_ep(self) -> bool:
+    def is_ep(self) -> bool:
         """
         Checks if the album is an EP.
         """
@@ -165,22 +183,22 @@ class Album:
 
         # TODO: check against number of tracks
 
-    def check_is_single(self, tracks: list[Track]):
+    def is_single(self, tracks: list[Track], singleTrackAsSingle: bool):
         """
         Checks if the album is a single.
         """
         keywords = ["single version", "- single"]
 
-        show_albums_as_singles = get_flag(SessionVarKeys.SHOW_ALBUMS_AS_SINGLES)
+        # show_albums_as_singles = get_flag(SessionVarKeys.SHOW_ALBUMS_AS_SINGLES)
 
         for keyword in keywords:
             if keyword in self.og_title.lower():
-                self.is_single = True
-                return
+                return True
 
-        if show_albums_as_singles and len(tracks) == 1:
-            self.is_single = True
-            return
+        # REVIEW: Reading from the config file in a for loop will be slow
+        # TODO: Find a
+        if singleTrackAsSingle and len(tracks) == 1:
+            return True
 
         if (
             len(tracks) == 1
@@ -192,29 +210,29 @@ class Album:
             # and tracks[0].disc == 1
             # TODO: Review -> Are the above commented checks necessary?
         ):
-            self.is_single = True
+            return True
 
-    def get_date_from_tracks(self, tracks: list[Track]):
-        """
-        Gets the date of the album its tracks.
+    # def get_date_from_tracks(self, tracks: list[Track]):
+    #     """
+    #     Gets the date of the album its tracks.
 
-        Args:
-            tracks (list[Track]): The tracks of the album.
-        """
-        if self.date:
-            return
+    #     Args:
+    #         tracks (list[Track]): The tracks of the album.
+    #     """
+    #     if self.date:
+    #         return
 
-        dates = (int(t.date) for t in tracks if t.date)
-        try:
-            self.date = datetime.datetime.fromtimestamp(min(dates)).year
-        except:
-            self.date = datetime.datetime.now().year
+    #     dates = (int(t.date) for t in tracks if t.date)
+    #     try:
+    #         self.date = datetime.datetime.fromtimestamp(min(dates)).year
+    #     except:
+    #         self.date = datetime.datetime.now().year
 
-    def set_count(self, count: int):
-        self.count = count
+    # def set_count(self, count: int):
+    #     self.count = count
 
-    def set_duration(self, duration: int):
-        self.duration = duration
+    # def set_duration(self, duration: int):
+    #     self.duration = duration
 
-    def set_created_date(self, created_date: int):
-        self.created_date = created_date
+    # def set_created_date(self, created_date: int):
+    #     self.created_date = created_date

app/models/artist.py

@@ -23,6 +23,12 @@ class ArtistMinimal:
         if self.artisthash == "5a37d5315e":
             self.name = "Juice WRLD"
 
+    def to_json(self):
+        return {
+            "name": self.name,
+            "artisthash": self.artisthash,
+        }
+
 
 @dataclass(slots=True)
 class Artist(ArtistMinimal):

app/models/track.py

@@ -4,7 +4,6 @@ from pathlib import Path
 
 from flask_jwt_extended import current_user
 
-
 from app.settings import SessionVarKeys, get_flag
 from app.utils.hashing import create_hash
 from app.utils.parsers import (
@@ -24,10 +23,11 @@ class Track:
     Track class
     """
 
+    id: int
     album: str
-    albumartists: str | list[ArtistMinimal]
+    albumartists: list[dict[str, str]]
     albumhash: str
-    artists: str | list[ArtistMinimal]
+    artists: str
     bitrate: int
     copyright: str
     date: int
@@ -35,152 +35,174 @@ class Track:
     duration: int
     filepath: str
     folder: str
-    genre: str | list[str]
+    genre: list[dict[str, str]]
+    last_mod: int
+    og_album: str
+    og_title: str
     title: str
     track: int
     trackhash: str
-    last_mod: str | int
 
-    image: str = ""
-    artist_hashes: str = ""
+    _pos: int = 0
+    _ati: str = ""
 
-    fav_userids: list = field(default_factory=list)
-    """
-    A string of user ids separated by commas.
-    """
-    # is_favorite: bool = False
+    # album: str
+    # albumartists: str | list[ArtistMinimal]
+    # albumhash: str
+    # artists: str | list[ArtistMinimal]
+    # bitrate: int
+    # copyright: str
+    # date: int
+    # disc: int
+    # duration: int
+    # filepath: str
+    # folder: str
+    # genre: str | list[str]
+    # title: str
+    # track: int
+    # trackhash: str
+    # last_mod: str | int
 
-    @property
-    def is_favorite(self):
-        return current_user['id'] in self.fav_userids
+    # image: str = ""
+    # artist_hashes: str = ""
 
-    # temporary attributes
-    _pos: int = 0  # for sorting tracks by disc and track number
-    _ati: str = (
-        ""  # (album track identifier) for removing duplicates when merging album versions
-    )
+    # fav_userids: list = field(default_factory=list)
+    # """
+    # A string of user ids separated by commas.
+    # """
+    # # is_favorite: bool = False
 
-    og_title: str = ""
-    og_album: str = ""
-    created_date: float = 0.0
+    # @property
+    # def is_favorite(self):
+    #     return current_user["id"] in self.fav_userids
 
-    def set_created_date(self):
-        try:
-            self.created_date = Path(self.filepath).stat().st_ctime
-        except FileNotFoundError:
-            pass
+    # # temporary attributes
+    # _pos: int = 0  # for sorting tracks by disc and track number
+    # _ati: str = (
+    #     ""  # (album track identifier) for removing duplicates when merging album versions
+    # )
 
-    def __post_init__(self):
-        self.og_title = self.title
-        self.og_album = self.album
-        self.last_mod = int(self.last_mod)
-        self.date = int(self.date)
+    # og_title: str = ""
+    # og_album: str = ""
+    # created_date: float = 0.0
 
-        # add a trailing slash to the folder path
-        # to avoid matching a folder starting with the same name as the root path
-        # eg. .../Music and .../Music Videos
-        self.folder = os.path.join(self.folder, "")
+    # def set_created_date(self):
+    #     try:
+    #         self.created_date = Path(self.filepath).stat().st_ctime
+    #     except FileNotFoundError:
+    #         pass
 
-        if self.artists is not None:
-            artists = split_artists(self.artists)
-            new_title = self.title
+    # def __post_init__(self):
+    #     self.og_title = self.title
+    #     self.og_album = self.album
+    #     self.last_mod = int(self.last_mod)
+    #     self.date = int(self.date)
 
-            if get_flag(SessionVarKeys.EXTRACT_FEAT):
-                featured, new_title = parse_feat_from_title(self.title)
-                original_lower = "-".join([create_hash(a) for a in artists])
-                artists.extend(
-                    a for a in featured if create_hash(a) not in original_lower
-                )
+    #     # add a trailing slash to the folder path
+    #     # to avoid matching a folder starting with the same name as the root path
+    #     # eg. .../Music and .../Music Videos
+    #     self.folder = os.path.join(self.folder, "")
 
-            self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists)
-            self.artists = [ArtistMinimal(a) for a in artists]
+    #     if self.artists is not None:
+    #         artists = split_artists(self.artists)
+    #         new_title = self.title
 
-            albumartists = split_artists(self.albumartists)
+    # if get_flag(SessionVarKeys.EXTRACT_FEAT):
+    #     featured, new_title = parse_feat_from_title(self.title)
+    #     original_lower = "-".join([create_hash(a) for a in artists])
+    #     artists.extend(
+    #         a for a in featured if create_hash(a) not in original_lower
+    #     )
 
-            if not albumartists:
-                self.albumartists = self.artists[:1]
-            else:
-                self.albumartists = [ArtistMinimal(a) for a in albumartists]
+    # self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists)
+    # self.artists = [ArtistMinimal(a) for a in artists]
 
-            if get_flag(SessionVarKeys.REMOVE_PROD):
-                new_title = remove_prod(new_title)
+    # albumartists = split_artists(self.albumartists)
 
-            # if track is a single
-            if self.og_title == self.album:
-                self.rename_album(new_title)
+    # if not albumartists:
+    #     self.albumartists = self.artists[:1]
+    # else:
+    #     self.albumartists = [ArtistMinimal(a) for a in albumartists]
 
-            if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK):
-                new_title = clean_title(new_title)
+    # if get_flag(SessionVarKeys.REMOVE_PROD):
+    #     new_title = remove_prod(new_title)
 
-            self.title = new_title
+    # if track is a single
+    # if self.og_title == self.album:
+    #     self.rename_album(new_title)
 
-            if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE):
-                self.album, _ = get_base_title_and_versions(
-                    self.album, get_versions=False
-                )
+    # if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK):
+    #     new_title = clean_title(new_title)
 
-            if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS):
-                self.recreate_albumhash()
+    # self.title = new_title
 
-        self.image = self.albumhash + ".webp"
+    # if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE):
+    #     self.album, _ = get_base_title_and_versions(
+    #         self.album, get_versions=False
+    #     )
 
-        if self.genre is not None and self.genre != "":
-            self.genre = self.genre.lower()
-            separators = {"/", ";", "&"}
+    # if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS):
+    #     self.recreate_albumhash()
 
-            contains_rnb = "r&b" in self.genre
-            contains_rock = "rock & roll" in self.genre
+    # self.image = self.albumhash + ".webp"
 
-            if contains_rnb:
-                self.genre = self.genre.replace("r&b", "RnB")
+    # if self.genre is not None and self.genre != "":
+    #     self.genre = self.genre.lower()
+    #     separators = {"/", ";", "&"}
 
-            if contains_rock:
-                self.genre = self.genre.replace("rock & roll", "rock")
+    #     contains_rnb = "r&b" in self.genre
+    #     contains_rock = "rock & roll" in self.genre
 
-            for s in separators:
-                self.genre: str = self.genre.replace(s, ",")
+    #     if contains_rnb:
+    #         self.genre = self.genre.replace("r&b", "RnB")
 
-            self.genre = self.genre.split(",")
-            self.genre = [g.strip() for g in self.genre]
+    #     if contains_rock:
+    #         self.genre = self.genre.replace("rock & roll", "rock")
 
-        self.recreate_hash()
-        self.set_created_date()
+    #     for s in separators:
+    #         self.genre: str = self.genre.replace(s, ",")
 
-    def recreate_hash(self):
-        """
-        Recreates a track hash if the track title was altered
-        to prevent duplicate tracks having different hashes.
-        """
-        if self.og_title == self.title and self.og_album == self.album:
-            return
+    #     self.genre = self.genre.split(",")
+    #     self.genre = [g.strip() for g in self.genre]
 
-        self.trackhash = create_hash(
-            ", ".join(a.name for a in self.artists), self.og_album, self.title
-        )
+    #     self.recreate_hash()
+    #     self.set_created_date()
 
-    def recreate_artists_hash(self):
-        """
-        Recreates a track's artist hashes if the artist list was altered
-        """
-        self.artist_hashes = "-".join(a.artisthash for a in self.artists)
+    # def recreate_hash(self):
+    #     """
+    #     Recreates a track hash if the track title was altered
+    #     to prevent duplicate tracks having different hashes.
+    #     """
+    #     if self.og_title == self.title and self.og_album == self.album:
+    #         return
 
-    def recreate_albumhash(self):
-        """
-        Recreates an albumhash of a track to merge all versions of an album.
-        """
-        albumartists = (a.name for a in self.albumartists)
-        self.albumhash = create_hash(self.album, *albumartists)
+    #     self.trackhash = create_hash(
+    #         ", ".join(a.name for a in self.artists), self.og_album, self.title
+    #     )
 
-    def rename_album(self, new_album: str):
-        """
-        Renames an album
-        """
-        self.album = new_album
+    # def recreate_artists_hash(self):
+    #     """
+    #     Recreates a track's artist hashes if the artist list was altered
+    #     """
+    #     self.artist_hashes = "-".join(a.artisthash for a in self.artists)
 
-    def add_artists(self, artists: list[str], new_album_title: str):
-        for artist in artists:
-            if create_hash(artist, decode=True) not in self.artist_hashes:
-                self.artists.append(ArtistMinimal(artist))
+    # def recreate_albumhash(self):
+    #     """
+    #     Recreates an albumhash of a track to merge all versions of an album.
+    #     """
+    #     albumartists = (a.name for a in self.albumartists)
+    #     self.albumhash = create_hash(self.album, *albumartists)
 
-        self.recreate_artists_hash()
-        self.rename_album(new_album_title)
+    # def rename_album(self, new_album: str):
+    #     """
+    #     Renames an album
+    #     """
+    #     self.album = new_album
+
+    # def add_artists(self, artists: list[str], new_album_title: str):
+    #     for artist in artists:
+    #         if create_hash(artist, decode=True) not in self.artist_hashes:
+    #             self.artists.append(ArtistMinimal(artist))
+
+    #     self.recreate_artists_hash()
+    #     self.rename_album(new_album_title)

app/utils/auth.py

@@ -1,6 +1,8 @@
 import hmac
 import hashlib
 
+from flask_jwt_extended import current_user
+
 from app.config import UserConfig
 
 
@@ -29,3 +31,14 @@ def check_password(password: str, hashed: str) -> bool:
     """
 
     return hmac.compare_digest(hash_password(password), hashed)
+
+
+def get_current_userid() -> int:
+    """
+    Get the current session user.
+    """
+    try:
+        return current_user["id"]
+    except RuntimeError:
+        # Catch this error raised during migration execution
+        return 1

db.py

@@ -0,0 +1,74 @@
+from sqlalchemy import create_engine, text, Table, Column, Integer, String, MetaData, select
+from sqlalchemy.orm import DeclarativeBase
+
+from typing import List, Optional
+
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+fullpath = "/home/cwilvx/temp/swingmusic/swing.db"
+engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True)
+
+class Base(DeclarativeBase):
+    pass
+
+class Tracks(Base):
+    __tablename__ = "tracks"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    album: Mapped[str] = mapped_column(String())
+    albumartist: Mapped[str] = mapped_column(String())
+    copyright: Mapped[Optional[str]]
+
+    def __repr__(self):
+        return f"<Tracks(album={self.album}, albumartist={self.albumartist})>"
+
+stmt = select(Tracks.album, Tracks.copyright).where(Tracks.album == "RAVAGE")
+print(stmt)
+
+with engine.connect() as conn:
+    result = conn.execute(stmt)
+    for row in result:
+        print(row)
+
+# Base.metadata.create_all(engine)
+
+# metadata = MetaData()
+# track_table = Table(
+#     "tracks",
+#     metadata,
+#     Column("id", Integer, primary_key=True, autoincrement=True),
+#     Column("album", String),
+#     Column("albumartist", String),
+#     Column("albumhash", String),
+#     Column("artist", String),
+#     Column("bitrate", Integer),
+#     Column("copyright", String),
+#     Column("date", Integer),
+#     Column("disc", Integer),
+#     Column("duration", Integer),
+#     Column("filepath", String),
+#     Column("folder", String),
+#     Column("genre", String),
+#     Column("title", String),
+#     Column("track", Integer),
+#     Column("trackhash", String),
+#     Column("last_mod", Integer),
+# )
+
+# metadata.create_all(engine)
+
+
+
+
+
+
+
+# with engine.connect() as conn:
+#     result = conn.execute(
+#         text("SELECT * FROM tracks where trackhash = :trackhash"),
+#         {"trackhash": "93acbea22b"},
+#     )
+#     # print(result.all())
+
+#     for r in result.mappings():
+#         print(r["trackhash"])

manage.py

@@ -21,6 +21,7 @@ import setproctitle
 
 from app.api import create_api
 from app.arg_handler import ProcessArgs
+from app.lib.tagger import IndexEverything
 from app.lib.watchdogg import Watcher as WatchDog
 from app.periodic_scan import run_periodic_scans
 from app.plugins.register import register_plugins
@@ -49,10 +50,11 @@ werkzeug.setLevel(logging.ERROR)
 
 
 # Background tasks
-# @background
-# def bg_run_setup():
-#     pass
+@background
+def bg_run_setup():
+    pass
     # run_periodic_scans()
+    IndexEverything()
 
 
 # @background
@@ -63,7 +65,7 @@ werkzeug.setLevel(logging.ERROR)
 @background
 def run_swingmusic():
     log_startup_info()
-    # bg_run_setup()
+    bg_run_setup()
     register_plugins()
 
     # start_watchdog()

poetry.lock

@@ -2164,6 +2164,93 @@ files = [
     {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"},
 ]
 
+[[package]]
+name = "sqlalchemy"
+version = "2.0.31"
+description = "Database Abstraction Library"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"},
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"},
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"},
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"},
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"},
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"},
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"},
+    {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"},
+    {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"},
+    {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"},
+    {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"},
+    {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"},
+    {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"},
+    {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"},
+    {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"},
+    {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"},
+    {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"},
+    {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"},
+    {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"},
+    {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"},
+    {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"},
+]
+
+[package.dependencies]
+greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"}
+typing-extensions = ">=4.6.0"
+
+[package.extras]
+aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"]
+aioodbc = ["aioodbc", "greenlet (!=0.4.17)"]
+aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"]
+asyncio = ["greenlet (!=0.4.17)"]
+asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"]
+mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"]
+mssql = ["pyodbc"]
+mssql-pymssql = ["pymssql"]
+mssql-pyodbc = ["pyodbc"]
+mypy = ["mypy (>=0.910)"]
+mysql = ["mysqlclient (>=1.4.0)"]
+mysql-connector = ["mysql-connector-python"]
+oracle = ["cx_oracle (>=8)"]
+oracle-oracledb = ["oracledb (>=1.0.1)"]
+postgresql = ["psycopg2 (>=2.7)"]
+postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
+postgresql-pg8000 = ["pg8000 (>=1.29.1)"]
+postgresql-psycopg = ["psycopg (>=3.0.7)"]
+postgresql-psycopg2binary = ["psycopg2-binary"]
+postgresql-psycopg2cffi = ["psycopg2cffi"]
+postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"]
+pymysql = ["pymysql"]
+sqlcipher = ["sqlcipher3_binary"]
+
 [[package]]
 name = "tabulate"
 version = "0.9.0"
@@ -2515,4 +2602,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.12"
-content-hash = "5722346cbfc224340877e337d4cee2c8e8a3a7ea68f9a64d9c5806e0ebcf919a"
+content-hash = "333baa055ac4a32ed914fb46025a48559575806dafba7db5aac97a3878ade23c"

pyproject.toml

@@ -26,6 +26,7 @@ watchdog = "^4.0.0"
 pendulum = "^3.0.0"
 flask-openapi3 = "^3.0.2"
 flask-jwt-extended = "^4.6.0"
+sqlalchemy = "^2.0.31"
 
 [tool.poetry.dev-dependencies]
 pylint = "^2.15.5"