From c42ec4dcded20171bcca3905275c46088cd2b4fe Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 24 Jun 2024 00:26:47 +0300 Subject: [PATCH 01/44] start: rewrite the database layer using a freaking ORM + start ditching in-mem stores + move main db table to a new name + experiments! --- .github/changelog.md | 13 ++ TODO.md | 7 + app/api/album.py | 117 ++++++------- app/api/artist.py | 2 +- app/api/folder.py | 2 + app/api/getall/__init__.py | 17 +- app/config.py | 1 + app/db/__init__.py | 232 +++++++++++++++++++++++++ app/db/sqlite/auth.py | 2 - app/db/sqlite/logger/tracks.py | 7 +- app/db/sqlite/playlists.py | 36 +++- app/db/sqlite/tracks.py | 14 ++ app/lib/folderslib.py | 75 ++++---- app/lib/populate.py | 70 ++++---- app/lib/searchlib.py | 2 +- app/lib/tagger.py | 154 +++++++++++++++++ app/lib/taglib.py | 104 ++++++++++- app/migrations/__init__.py | 10 +- app/migrations/v1_4_9/__init__.py | 278 +++++++++++++++++++++++++++--- app/models/album.py | 208 ++++++++++++---------- app/models/artist.py | 6 + app/models/track.py | 252 +++++++++++++++------------ app/utils/auth.py | 13 ++ db.py | 74 ++++++++ manage.py | 10 +- poetry.lock | 89 +++++++++- pyproject.toml | 1 + 27 files changed, 1399 insertions(+), 397 deletions(-) create mode 100644 app/lib/tagger.py create mode 100644 db.py diff --git a/.github/changelog.md b/.github/changelog.md index 0a151692..ec423361 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -12,3 +12,16 @@ - ## Development + + +## THE BIG ONE API CHANGES + +- genre is no longer a string, but a struct: + +```ts +interface Genre { + name: str; + genrehash: str; +} +``` + diff --git a/TODO.md b/TODO.md index 0812b840..7f5cc591 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,11 @@ # TODO - Migrations: 1. Move userdata to new hashing algorithm + - favorites ✅ + - playlists + - scrobble + - images + - remove image colors - Package jsoni and publish on PyPi - Rewrite stores to use dictionaries instead of list pools @@ -8,6 +13,8 @@ - Disable the watchdog by default, and mark it as experimental - rename userid to server id in config file - Look into seeding jwts using user password + server id +- Recreate album hash if featured artists are discover +- Implement checking if is clean install and skip migrations! # DONE - Support auth headers diff --git a/app/api/album.py b/app/api/album.py index 139e1621..b915612c 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -2,6 +2,7 @@ Contains all the album routes. """ +from itertools import groupby import random from flask_jwt_extended import current_user @@ -10,13 +11,15 @@ from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from app.api.apischemas import AlbumHashSchema, AlbumLimitSchema, ArtistHashSchema +from app.config import UserConfig +from app.db import AlbumTable as AlbumDb, TrackTable as TrackDb from app.settings import Defaults from app.models import FavType, Track from app.store.albums import AlbumStore from app.store.tracks import TrackStore from app.utils.hashing import create_hash from app.lib.albumslib import sort_by_track_no -from app.serializers.album import serialize_for_card +from app.serializers.album import serialize_for_card, serialize_for_card_many from app.serializers.track import serialize_track from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb @@ -38,47 +41,20 @@ def get_album_tracks_and_info(body: AlbumHashSchema): Returns album info and tracks for the given albumhash. """ albumhash = body.albumhash - - error_msg = {"error": "Album not created yet."} - album = AlbumStore.get_album_by_hash(albumhash) + album = AlbumDb.get_album_by_albumhash(albumhash) if album is None: - return error_msg, 404 + return {"error": "Album not found"}, 404 - tracks = TrackStore.get_tracks_by_albumhash(albumhash) - - if tracks is None: - return error_msg, 404 - - if len(tracks) == 0: - return error_msg, 404 - - def get_album_genres(tracks: list[Track]): - genres = set() - - for track in tracks: - if track.genre is not None: - genres.update(track.genre) - - return list(genres) - - album.genres = get_album_genres(tracks) - album.count = len(tracks) - - album.get_date_from_tracks(tracks) + tracks = TrackDb.get_tracks_by_albumhash(albumhash) + album.trackcount = len(tracks) album.duration = sum(t.duration for t in tracks) + album.type = album.check_type( + tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles + ) + album.populate_versions() - album.check_is_single(tracks) - - if not album.is_single: - album.check_type() - - album.is_favorite = check_is_fav(albumhash, FavType.album) - - return { - "tracks": [serialize_track(t, remove_disc=False) for t in tracks], - "info": album, - } + return {"info": album, "tracks": tracks} @api.get("//tracks") @@ -89,16 +65,16 @@ def get_album_tracks(path: AlbumHashSchema): Returns all the tracks in the given album, sorted by disc and track number. NOTE: No album info is returned. """ - tracks = TrackStore.get_tracks_by_albumhash(path.albumhash) + tracks = TrackDb.get_tracks_by_albumhash(path.albumhash) tracks = sort_by_track_no(tracks) return tracks class GetMoreFromArtistsBody(AlbumLimitSchema): - albumartists: str = Field( + albumartists: list = Field( description="The artist hashes to get more albums from", - example=Defaults.API_ARTISTHASH, + example='[{"name": "Khalid", "artisthash": "94ca2dba1c"}]', ) base_title: str = Field( @@ -119,29 +95,25 @@ def get_more_from_artist(body: GetMoreFromArtistsBody): limit = body.limit base_title = body.base_title - albumartists: list[str] = albumartists.split(",") + all_albums = AlbumDb.get_albums_by_artisthashes(albumartists) - albums = [ - { - "artisthash": a, - "albums": AlbumStore.get_albums_by_albumartist( - a, limit, exclude=base_title - ), - } - for a in albumartists + # filter out albums with the same base title + all_albums = filter( + lambda a: create_hash(a.base_title) != create_hash(base_title), all_albums + ) + all_albums = list(all_albums) + + if not len(all_albums): + return [] + + # group by first albumartist's artisthash + groups = groupby(all_albums, lambda a: a.albumartists[0]["artisthash"]) + + return [ + {"artisthash": g[0], "albums": serialize_for_card_many(list(g[1])[:limit])} + for g in groups ] - albums = [ - { - "artisthash": a["artisthash"], - "albums": [serialize_for_card(a_) for a_ in (a["albums"])], - } - for a in albums - if len(a["albums"]) > 0 - ] - - return albums - class GetAlbumVersionsBody(ArtistHashSchema): og_album_title: str = Field( @@ -165,18 +137,29 @@ def get_album_versions(body: GetAlbumVersionsBody): base_title = body.base_title artisthash = body.artisthash - albums = AlbumStore.get_albums_by_artisthash(artisthash) - + albums = AlbumDb.get_albums_by_base_title(base_title) + print(albums) albums = [ a for a in albums - if create_hash(a.base_title) == create_hash(base_title) - and create_hash(og_album_title) != create_hash(a.og_title) + if a.og_title != og_album_title + and artisthash in {a["artisthash"] for a in a.albumartists} ] - for a in albums: - tracks = TrackStore.get_tracks_by_albumhash(a.albumhash) - a.get_date_from_tracks(tracks) + print(albums) + + # albums = AlbumStore.get_albums_by_artisthash(artisthash) + + # albums = [ + # a + # for a in albums + # if create_hash(a.base_title) == create_hash(base_title) + # and create_hash(og_album_title) != create_hash(a.og_title) + # ] + + # for a in albums: + # tracks = TrackStore.get_tracks_by_albumhash(a.albumhash) + # a.get_date_from_tracks(tracks) return albums diff --git a/app/api/artist.py b/app/api/artist.py index 94c8f6ef..2d69e542 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -133,7 +133,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): AlbumStore.remove_album_by_hash(a.albumhash) continue - a.check_is_single(album_tracks) + a.is_single(album_tracks) all_albums = sorted(all_albums, key=lambda a: str(a.date), reverse=True) diff --git a/app/api/folder.py b/app/api/folder.py index bec377d0..71e8ac5b 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -66,7 +66,9 @@ def get_folder_tree(body: FolderTree): else: req_dir = "/" + req_dir if not req_dir.startswith("/") else req_dir + print('stuff!') res = GetFilesAndDirs(req_dir, tracks_only=tracks_only)() + print(res['folders']) res["folders"] = sorted(res["folders"], key=lambda i: i.name) return res diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index b3199ca9..607fa08a 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field from datetime import datetime from app.api.apischemas import GenericLimitSchema +from app.db import AlbumTable, ArtistTable from app.store.albums import AlbumStore from app.store.artists import ArtistStore @@ -59,17 +60,19 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): is_albums = path.itemtype == "albums" is_artists = path.itemtype == "artists" - items = AlbumStore.albums + if is_albums: + items = AlbumTable.get_all(query.start, query.limit) + elif is_artists: + items = ArtistTable.get_all(query.start, query.limit) - if is_artists: - items = ArtistStore.artists + print(items) start = query.start limit = query.limit sort = query.sortby reverse = query.reverse == "1" - sort_is_count = sort == "count" + sort_is_count = sort == "trackcount" sort_is_duration = sort == "duration" sort_is_create_date = sort == "created_date" @@ -81,7 +84,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): lambda_sort = lambda x: getattr(x, sort) if sort_is_artist: - lambda_sort = lambda x: getattr(x, sort)[0].name + lambda_sort = lambda x: getattr(x, sort)[0]["name"] sorted_items = sorted(items, key=lambda_sort, reverse=reverse) items = sorted_items[start : start + limit] @@ -101,7 +104,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): if sort_is_count: item_dict["help_text"] = ( - f"{format_number(item.count)} track{'' if item.count == 1 else 's'}" + f"{format_number(item.trackcount)} track{'' if item.trackcount == 1 else 's'}" ) if sort_is_duration: @@ -114,7 +117,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): if sort_is_artist_albumcount: item_dict["help_text"] = ( - f"{format_number(item.albumcount)} album{'' if item.albumcount == 1 else 's'}" + f"{format_number(item['albumcount'])} album{'' if item['albumcount'] == 1 else 's'}" ) album_list.append(item_dict) diff --git a/app/config.py b/app/config.py index 00e6cde5..2d477ea0 100644 --- a/app/config.py +++ b/app/config.py @@ -21,6 +21,7 @@ class UserConfig: rootDirs: list[str] = field(default_factory=list) excludeDirs: list[str] = field(default_factory=list) artistSeparators: set[str] = field(default_factory=list) + genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) # tracks extractFeaturedArtists: bool = True diff --git a/app/db/__init__.py b/app/db/__init__.py index e69de29b..dbea0ad6 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -0,0 +1,232 @@ +import json +from pprint import pprint +from typing import Any, Optional + +from sqlalchemy import ( + JSON, + Boolean, + Integer, + Row, + String, + Tuple, + create_engine, + insert, + select, +) +from sqlalchemy.orm import ( + Mapped, + mapped_column, + DeclarativeBase, + MappedAsDataclass, + sessionmaker, +) + +from app.models import Track as TrackModel +from app.models import Album as AlbumModel +from app.utils.remove_duplicates import remove_duplicates + + +fullpath = "/home/cwilvx/temp/swingmusic/swing.db" +engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=False) + + +def todict(track: Any): + return track._asdict() + + +def todicts(tracks: list[Any]): + return [todict(track) for track in tracks] + + +class DbManager: + def __init__(self): + self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) + self.conn = self.engine.connect() + + def __enter__(self): + return self.conn.execution_options(preserve_rowcount=True) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.conn.commit() + self.conn.close() + + +class Base(MappedAsDataclass, DeclarativeBase): + @classmethod + def insert_many(cls, items: list[dict[str, Any]]): + """ + Inserts multiple items into the database. + """ + with DbManager() as conn: + conn.execute(insert(cls).values(items)) + + @classmethod + def insert_one(cls, item: dict[str, Any]): + """ + Inserts a single item into the database. + """ + return cls.insert_many([item]) + + @classmethod + def get_all(cls): + """ + Returns all the items from the database. + """ + with DbManager() as conn: + result = conn.execute(select(cls)) + return result.fetchall() + + +class ArtistTable(Base): + __tablename__ = "artist" + + id: Mapped[int] = mapped_column(primary_key=True) + albumcount: Mapped[int] = mapped_column(Integer()) + artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genres: Mapped[str] = mapped_column(JSON()) + name: Mapped[str] = mapped_column(String(), index=True) + trackcount: Mapped[int] = mapped_column(Integer()) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + + @classmethod + def get_all(cls, start: int, limit: int): + with DbManager() as conn: + result = conn.execute(select(cls).offset(start).limit(limit)) + return albums_to_dataclasses(result.fetchall()) + + +class AlbumTable(Base): + __tablename__ = "album" + + id: Mapped[int] = mapped_column(primary_key=True) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) + base_title: Mapped[str] = mapped_column(String()) + color: Mapped[Optional[str]] = mapped_column(String()) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genres: Mapped[str] = mapped_column(JSON()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + trackcount: Mapped[int] = mapped_column(Integer()) + + @classmethod + def get_album_by_albumhash(cls, hash: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.albumhash == hash) + ) + album = result.fetchone() + + if album: + return album_to_dataclass(album) + + @classmethod + def get_all(cls, start: int, limit: int): + with DbManager() as conn: + result = conn.execute(select(AlbumTable).offset(start).limit(limit)) + return albums_to_dataclasses(result.fetchall()) + + @classmethod + def get_albums_by_artisthashes(cls, artisthashes: list[dict[str, str]]): + with DbManager() as conn: + albums: list[AlbumModel] = [] + + for artist in artisthashes: + result = conn.execute( + # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! + select(AlbumTable).where(AlbumTable.albumartists.contains(artist)) + ) + albums.extend(albums_to_dataclasses(result.fetchall())) + + print(albums) + return albums + + @classmethod + def get_albums_by_base_title(cls, base_title: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.base_title == base_title) + ) + return albums_to_dataclasses(result.fetchall()) + + +class TrackTable(Base): + __tablename__ = "track" + + id: Mapped[int] = mapped_column(init=False, primary_key=True) + album: Mapped[str] = mapped_column(String()) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) + albumhash: Mapped[str] = mapped_column(String(), index=True) + artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + bitrate: Mapped[int] = mapped_column(Integer()) + copyright: Mapped[Optional[str]] = mapped_column(String()) + date: Mapped[int] = mapped_column(Integer()) + disc: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + filepath: Mapped[str] = mapped_column(String(), unique=True) + folder: Mapped[str] = mapped_column(String(), index=True) + genre: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) + last_mod: Mapped[float] = mapped_column(Integer()) + og_album: Mapped[str] = mapped_column(String()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + track: Mapped[int] = mapped_column(Integer()) + trackhash: Mapped[str] = mapped_column(String(), index=True) + + @classmethod + def get_tracks_by_filepaths(cls, filepaths: list[str]): + print(filepaths[0]) + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.filepath.in_(filepaths)) + ) + return [dict(r) for r in result.mappings().fetchall()] + + @classmethod + def count_tracks_containing_paths(cls, paths: list[str]): + results: list[dict[str, int | str]] = [] + + with DbManager() as conn: + for path in paths: + result = conn.execute( + select(TrackTable).where(TrackTable.filepath.contains(path)) + ) + results.append({"path": path, "trackcount": result.all().__len__()}) + + return results + + @classmethod + def get_tracks_by_albumhash(cls, albumhash: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.albumhash == albumhash) + ) + tracks = tracks_to_dataclasses(result.fetchall()) + return remove_duplicates(tracks, is_album_tracks=True) + + +# SECTION: HELPER FUNCTIONS + + +def album_to_dataclass(album: Row[AlbumTable]): + return AlbumModel(**album._asdict()) + + +def albums_to_dataclasses(albums: list[Row[AlbumTable]]): + return [album_to_dataclass(album) for album in albums] + + +def track_to_dataclass(track: Row[TrackTable]): + return TrackModel(**track._asdict()) + + +def tracks_to_dataclasses(tracks: list[Row[TrackTable]]): + return [track_to_dataclass(track) for track in tracks] + + +Base().metadata.create_all(engine) diff --git a/app/db/sqlite/auth.py b/app/db/sqlite/auth.py index e79dfe78..dede16b0 100644 --- a/app/db/sqlite/auth.py +++ b/app/db/sqlite/auth.py @@ -75,7 +75,6 @@ class SQLiteAuthMethods: {', '.join([f"{key} = :{key}" for key in keys if key != 'id'])} WHERE id = :id """ - print(sql, user) with SQLiteManager(userdata_db=True) as cur: cur.execute(sql, user) @@ -140,7 +139,6 @@ class SQLiteAuthMethods: Delete a user by username. """ sql = "DELETE FROM users WHERE id = ?" - print("deleting user: ", username) with SQLiteManager(userdata_db=True) as cur: cur.execute(sql, (3,)) cur.close() diff --git a/app/db/sqlite/logger/tracks.py b/app/db/sqlite/logger/tracks.py index 7f43e8b2..c95c123b 100644 --- a/app/db/sqlite/logger/tracks.py +++ b/app/db/sqlite/logger/tracks.py @@ -1,6 +1,7 @@ from flask_jwt_extended import current_user from app.db.sqlite.utils import SQLiteManager from app.models.logger import TrackLog as TrackLog +from app.utils.auth import get_current_userid class SQLiteTrackLogger: @@ -10,6 +11,7 @@ class SQLiteTrackLogger: Inserts a track play record into the database """ + userid = get_current_userid() with SQLiteManager(userdata_db=True) as cur: sql = """INSERT OR REPLACE INTO track_logger( trackhash, @@ -21,7 +23,7 @@ class SQLiteTrackLogger: """ cur.execute( - sql, (trackhash, duration, timestamp, source, current_user["id"]) + sql, (trackhash, duration, timestamp, source, userid) ) lastrowid = cur.lastrowid @@ -34,7 +36,8 @@ class SQLiteTrackLogger: """ with SQLiteManager(userdata_db=True) as cur: - sql = f"""SELECT * FROM track_logger WHERE userid = {current_user['id']} ORDER BY timestamp DESC""" + userid = get_current_userid() + sql = f"""SELECT * FROM track_logger WHERE userid = {userid} ORDER BY timestamp DESC""" cur.execute(sql) rows = cur.fetchall() diff --git a/app/db/sqlite/playlists.py b/app/db/sqlite/playlists.py index bfed7185..b4cca33e 100644 --- a/app/db/sqlite/playlists.py +++ b/app/db/sqlite/playlists.py @@ -60,7 +60,15 @@ class SQLitePlaylistMethods: @staticmethod def get_all_playlists(): with SQLiteManager(userdata_db=True) as cur: - cur.execute(f"SELECT * FROM playlists WHERE userid = {current_user['id']}") + userid = 1 + + try: + userid = current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + pass + + cur.execute(f"SELECT * FROM playlists WHERE userid = {userid}") playlists = cur.fetchall() cur.close() @@ -92,7 +100,15 @@ class SQLitePlaylistMethods: Adds a string item to a json dumped list using a playlist id and field name. Takes the playlist ID, a field name, an item to add to the field. """ - sql = f"SELECT {field} FROM playlists WHERE id = ? and userid = {current_user['id']}" + userid = 1 + + try: + userid = current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + pass + + sql = f"SELECT {field} FROM playlists WHERE id = ? and userid = {userid}" with SQLiteManager(userdata_db=True) as cur: cur.execute(sql, (playlist_id,)) @@ -173,10 +189,17 @@ class SQLitePlaylistMethods: """ sql = """UPDATE playlists SET trackhashes = ? WHERE id = ?""" + userid = 1 + + try: + userid = current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + pass with SQLiteManager(userdata_db=True) as cur: cur.execute( - f"SELECT trackhashes FROM playlists WHERE id = ? and userid = {current_user['id']}", + f"SELECT trackhashes FROM playlists WHERE id = ? and userid = {userid}", (playlistid,), ) data = cur.fetchone() @@ -185,17 +208,20 @@ class SQLitePlaylistMethods: return trackhashes: list[str] = json.loads(data[0]) + to_remove = [] for track in tracks: # { # trackhash: str; # index: int; # } - index = trackhashes.index(track["trackhash"]) if index == track["index"]: - trackhashes.remove(track["trackhash"]) + to_remove.append(track["trackhash"]) + + for trackhash in to_remove: + trackhashes.remove(trackhash) cur.execute(sql, (json.dumps(trackhashes), playlistid)) diff --git a/app/db/sqlite/tracks.py b/app/db/sqlite/tracks.py index c8bed28c..0324641b 100644 --- a/app/db/sqlite/tracks.py +++ b/app/db/sqlite/tracks.py @@ -98,6 +98,20 @@ class SQLiteTrackMethods: return tuple_to_track(row) return None + + @staticmethod + def get_track_by_albumhash(albumhash: str): + """ + Gets a track using its albumhash. Returns a Track object or None. + """ + with SQLiteManager() as cur: + cur.execute("SELECT * FROM tracks WHERE albumhash=?", (albumhash,)) + row = cur.fetchone() + + if row is not None: + return tuple_to_track(row) + + return None @staticmethod def remove_tracks_by_filepaths(filepaths: str | set[str]): diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index 112a72d7..ccf1c499 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -8,6 +8,7 @@ from app.settings import SUPPORTED_FILES from app.utils.wintools import win_replace_slash from app.store.tracks import TrackStore +from app.db import TrackTable as TrackDB def create_folder(path: str, trackcount=0, foldercount=0) -> Folder: @@ -37,44 +38,52 @@ def get_first_child_from_path(root: str, maybe_child: str): return os.path.join(root, first) + def get_folders(paths: list[str]): """ Filters out folders that don't have any tracks and returns a list of folder objects. """ - count_dict = { - "tracks": {path: 0 for path in paths}, - # folders are immediate children of the root folder - "folders": {path: set() for path in paths}, - } - - for track in TrackStore.tracks: - for path in paths: - - # a child path should be longer than the root path - if len(track.folder) >= len(path) and track.folder.startswith(path): - count_dict["tracks"][path] += 1 - - # counting subfolders - p = get_first_child_from_path(path, track.folder) - - if p: - count_dict["folders"][path].add(p) - - folders = [ - { - "path": path, - "trackcount": count_dict["tracks"][path], - "foldercount": len(count_dict["folders"][path]), - } - for path in paths - ] + folders = TrackDB.count_tracks_containing_paths(paths) return [ - create_folder(f["path"], f["trackcount"], f["foldercount"]) + create_folder(f["path"], f["trackcount"], foldercount=0) for f in folders if f["trackcount"] > 0 ] + # count_dict = { + # "tracks": {path: 0 for path in paths}, + # # folders are immediate children of the root folder + # "folders": {path: set() for path in paths}, + # } + + # for track in TrackStore.tracks: + # for path in paths: + + # # a child path should be longer than the root path + # if len(track.folder) >= len(path) and track.folder.startswith(path): + # count_dict["tracks"][path] += 1 + + # # counting subfolders + # p = get_first_child_from_path(path, track.folder) + + # if p: + # count_dict["folders"][path].add(p) + + # folders = [ + # { + # "path": path, + # "trackcount": count_dict["tracks"][path], + # "foldercount": len(count_dict["folders"][path]), + # } + # for path in paths + # ] + + # return [ + # create_folder(f["path"], f["trackcount"], f["foldercount"]) + # for f in folders + # if f["trackcount"] > 0 + # ] class GetFilesAndDirs: @@ -131,7 +140,13 @@ class GetFilesAndDirs: files_.sort(key=lambda f: f["time"]) files = [f["path"] for f in files_] - tracks = TrackStore.get_tracks_by_filepaths(files) + tracks = [] + if files: + tracks = TrackDB.get_tracks_by_filepaths(files) + print("printing files") + print(tracks) + + # tracks = TrackStore.get_tracks_by_filepaths(files) folders = [] if not self.tracks_only: @@ -145,7 +160,7 @@ class GetFilesAndDirs: return { "path": path, - "tracks": serialize_tracks(tracks), + "tracks": tracks, "folders": folders, } diff --git a/app/lib/populate.py b/app/lib/populate.py index ceab17d8..4ed73060 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -7,6 +7,7 @@ from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout from app import settings +from app.db import TrackTable from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb from app.db.sqlite.settings import SettingsSQLMethods as sdb @@ -121,14 +122,14 @@ class Populate: return @staticmethod - def remove_modified(tracks: Generator[Track, None, None]): + def remove_modified(tracks: Generator[TrackTable, None, None]): """ Removes tracks from the database that have been modified since they were added to the database. """ unmodified_paths = set() - modified_tracks: list[Track] = [] + modified_tracks: list[TrackTable] = [] modified_paths = set() for track in tracks: @@ -151,18 +152,6 @@ class Populate: @staticmethod def tag_untagged(untagged: set[str], key: str): - log.info("Found %s new tracks", len(untagged)) - tagged_tracks: deque[dict] = deque() - tagged_count = 0 - - favs = favdb.get_fav_tracks() - records = dict() - - for fav in favs: - r = records.setdefault(fav[1], set()) - r.add(fav[4]) - - for file in tqdm(untagged, desc="Reading files"): if POPULATE_KEY != key: log.warning("'Populate.tag_untagged': Populate key changed") @@ -171,36 +160,49 @@ class Populate: tags = get_tags(file) if tags is not None: - tagged_tracks.append(tags) - track = Track(**tags) + TrackTable.insert_one(tags) - track.fav_userids = list(records.get(track.trackhash, set())) + # log.info("Found %s new tracks", len(untagged)) + # # tagged_tracks: deque[dict] = deque() + # # tagged_count = 0 - TrackStore.add_track(track) + # favs = favdb.get_fav_tracks() + # records = dict() - if not AlbumStore.album_exists(track.albumhash): - AlbumStore.add_album(AlbumStore.create_album(track)) + # for fav in favs: + # r = records.setdefault(fav[1], set()) + # r.add(fav[4]) - for artist in track.artists: - if not ArtistStore.artist_exists(artist.artisthash): - ArtistStore.add_artist(Artist(artist.name)) + # tagged_tracks.append(tags) + # track = Track(**tags) - for artist in track.albumartists: - if not ArtistStore.artist_exists(artist.artisthash): - ArtistStore.add_artist(Artist(artist.name)) + # track.fav_userids = list(records.get(track.trackhash, set())) - tagged_count += 1 - else: - log.warning("Could not read file: %s", file) + # TrackStore.add_track(track) - if len(tagged_tracks) > 0: - log.info("Adding %s tracks to database", len(tagged_tracks)) - insert_many_tracks(tagged_tracks) + # if not AlbumStore.album_exists(track.albumhash): + # AlbumStore.add_album(AlbumStore.create_album(track)) - log.info("Added %s/%s tracks", tagged_count, len(untagged)) + # for artist in track.artists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) + + # for artist in track.albumartists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) + + # tagged_count += 1 + # else: + # log.warning("Could not read file: %s", file) + + # if len(tagged_tracks) > 0: + # log.info("Adding %s tracks to database", len(tagged_tracks)) + # insert_many_tracks(tagged_tracks) + + # log.info("Added %s/%s tracks", tagged_count, len(untagged)) @staticmethod - def extract_thumb_with_overwrite(tracks: list[Track]): + def extract_thumb_with_overwrite(tracks: list[TrackTable]): """ Extracts the thumbnail from a list of filepaths, overwriting the existing thumbnail if it exists, diff --git a/app/lib/searchlib.py b/app/lib/searchlib.py index 8bca6420..a23d67fa 100644 --- a/app/lib/searchlib.py +++ b/app/lib/searchlib.py @@ -195,7 +195,7 @@ class TopResults: except AttributeError: item.duration = 0 - item.check_is_single(tracks) + item.is_single(tracks) if not item.is_single: item.check_type() diff --git a/app/lib/tagger.py b/app/lib/tagger.py new file mode 100644 index 00000000..38101f80 --- /dev/null +++ b/app/lib/tagger.py @@ -0,0 +1,154 @@ +from pprint import pprint +from app.db import AlbumTable, ArtistTable, TrackTable +from app.lib.taglib import get_tags +from app.utils.filesystem import run_fast_scandir +from app.utils.parsers import get_base_album_title +from app.utils.progressbar import tqdm + + +class IndexTracks: + def __init__(self) -> None: + dirs_to_scan = ["/home/cwilvx/Music"] + + files = set() + + for _dir in dirs_to_scan: + files = files.union(run_fast_scandir(_dir, full=True)[1]) + + self.tag_untagged(files) + # unmodified, modified_tracks = self.remove_modified(tracks) + # untagged = files - unmodified + + def tag_untagged(self, files: set[str]): + for file in tqdm(files, desc="Reading files"): + # if POPULATE_KEY != key: + # log.warning("'Populate.tag_untagged': Populate key changed") + # return + + tags = get_tags(file) + + if tags is not None: + TrackTable.insert_one(tags) + + +class IndexAlbums: + def __init__(self) -> None: + albums = dict() + + all_tracks: list[TrackTable] = TrackTable.get_all() + + for track in all_tracks: + if track.albumhash not in albums: + albums[track.albumhash] = { + "albumartists": track.albumartists, + "albumhash": track.albumhash, + "base_title": None, + "color": None, + "created_date": None, + "date": None, + "duration": track.duration, + "genres": [*track.genre] if track.genre else [], + "og_title": track.og_album, + "title": track.album, + "trackcount": 1, + "dates": [track.date], + "created_dates": [track.last_mod], + } + else: + album = albums[track.albumhash] + album["trackcount"] += 1 + album["duration"] += track.duration + album["dates"].append(track.date) + album["created_dates"].append(track.last_mod) + + if track.genre: + album["genres"].append(track.genre) + + for album in albums.values(): + album["date"] = min(album["dates"]) + album["created_date"] = min(album["created_dates"]) + + genres = [] + + for genre in album["genres"]: + if genre not in genres: + genres.append(genre) + + album["genres"] = genres + album["base_title"], _ = get_base_album_title(album["og_title"]) + + del album["dates"] + del album["created_dates"] + + pprint(albums) + + AlbumTable.insert_many(list(albums.values())) + + +class IndexArtists: + def __init__(self) -> None: + all_tracks: list[TrackTable] = TrackTable.get_all() + artists = dict() + + for track in all_tracks: + this_artists = track.artists + + for a in track.albumartists: + if a not in this_artists: + this_artists.append(a) + + for artist in this_artists: + if artist["artisthash"] not in artists: + artists[artist["artisthash"]] = { + "albumcount": None, + "albums": {track.albumhash}, + "artisthash": artist["artisthash"], + "created_dates": [track.last_mod], + "dates": [track.date], + "date": None, + "duration": track.duration, + "genres": [*track.genre] if track.genre else [], + "name": artist["name"], + "trackcount": None, + "tracks": {track.trackhash}, + } + else: + artist = artists[artist["artisthash"]] + artist["duration"] += track.duration + artist["albums"].add(track.albumhash) + artist["tracks"].add(track.trackhash) + artist["dates"].append(track.date) + artist["created_dates"].append(track.last_mod) + + if track.genre: + artist["genres"].append(track.genre) + + + for artist in artists.values(): + artist["albumcount"] = len(artist["albums"]) + artist["trackcount"] = len(artist["tracks"]) + artist["date"] = min(artist["dates"]) + artist["created_date"] = min(artist["created_dates"]) + + genres = [] + + for genre in artist["genres"]: + if genre not in genres: + genres.append(genre) + + artist["genres"] = genres + + del artist["tracks"] + del artist["albums"] + del artist["dates"] + del artist["created_dates"] + + pprint(artists) + ArtistTable.insert_many(list(artists.values())) + +class IndexEverything: + def __init__(self) -> None: + # IndexTracks() + # IndexAlbums() + # IndexArtists() + pass diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 3d7e9f6d..68cfd519 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -8,9 +8,16 @@ import pendulum from PIL import Image, UnidentifiedImageError from tinytag import TinyTag +from app.config import UserConfig from app.settings import Defaults, Paths from app.utils.hashing import create_hash -from app.utils.parsers import split_artists +from app.utils.parsers import ( + clean_title, + get_base_title_and_versions, + parse_feat_from_title, + remove_prod, + split_artists, +) from app.utils.wintools import win_replace_slash @@ -206,9 +213,7 @@ def get_tags(filepath: str): except KeyError: tags.copyright = None - tags.albumhash = create_hash(tags.album, tags.albumartist) - tags.trackhash = create_hash(tags.artist, tags.album, tags.title) - tags.image = f"{tags.albumhash}.webp" + # tags.image = f"{tags.albumhash}.webp" tags.folder = win_replace_slash(os.path.dirname(filepath)) tags.date = parse_date(tags.year) or int(last_mod) @@ -218,9 +223,100 @@ def get_tags(filepath: str): tags.artists = tags.artist tags.albumartists = tags.albumartist + split_artist = split_artists(tags.artist) + split_albumartists = split_artists(tags.albumartist) + new_title = tags.title + + # TODO: Figure out which is the best spot to create these hashes + # create albumhash using og_album + tags.albumhash = create_hash(tags.album or "", tags.albumartist) + + config = UserConfig() + + # extract featured artists + if config.extractFeaturedArtists: + feat, new_title = parse_feat_from_title(tags.title) + original_lower = "-".join([create_hash(a) for a in split_artist]) + split_artist.extend(a for a in feat if create_hash(a) not in original_lower) + + # if no albumartist, assign to the first artist + if not tags.albumartist: + tags.albumartist = split_artist[:1] + + # create json objects for artists and albumartists + tags.artists = [ + { + "artisthash": create_hash(a, decode=True), + "name": a, + } + for a in split_artist + ] + + tags.albumartists = [ + { + "artisthash": create_hash(a, decode=True), + "name": a, + } + for a in split_albumartists + ] + + # remove prod by + if config.removeProdBy: + new_title = remove_prod(new_title) + + # if track is a single, ie. + # if og_title == album, rename album to new_title + if tags.title == tags.album: + tags.album = new_title + + # remove remaster from track title + if config.removeRemasterInfo: + new_title = clean_title(new_title) + + # save final title + tags.og_title = tags.title + tags.title = new_title + tags.og_album = tags.album + + # clean album title + if config.cleanAlbumTitle: + tags.album, _ = get_base_title_and_versions(tags.album, get_versions=False) + + # merge album versions + if config.mergeAlbums: + tags.albumhash = create_hash( + tags.album, *(a["name"] for a in tags.albumartists) + ) + + # process genres + if tags.genre: + tags.genre = tags.genre.lower() + # separators = {"/", ";", "&"} + separators = set(config.genreSeparators) + + contains_rnb = "r&b" in tags.genre + contains_rock = "rock & roll" in tags.genre + + if contains_rnb: + tags.genre = tags.genre.replace("r&b", "RnB") + + if contains_rock: + tags.genre = tags.genre.replace("rock & roll", "rock") + + for s in separators: + tags.genre = tags.genre.replace(s, ",") + + tags.genre = tags.genre.split(",") + tags.genre = [ + {"name": g.strip(), "genrehash": create_hash(g.strip())} for g in tags.genre + ] + # sub underscore with space tags.title = tags.title.replace("_", " ") tags.album = tags.album.replace("_", " ") + tags.trackhash = create_hash( + *[a["name"] for a in tags.artists], tags.album, tags.title + ) tags = tags.__dict__ diff --git a/app/migrations/__init__.py b/app/migrations/__init__.py index 9e473e92..570cc115 100644 --- a/app/migrations/__init__.py +++ b/app/migrations/__init__.py @@ -5,6 +5,7 @@ Reads and applies the latest database migrations. """ import inspect +import sys from types import ModuleType from app.db.sqlite.migrations import MigrationManager from app.logger import log @@ -55,11 +56,12 @@ def apply_migrations(): to_apply = all_migrations[index:] for migration in to_apply: - try: + # try: migration.migrate() log.info("Applied migration: %s", migration.__name__) - except Exception as e: - log.error("Failed to run migration: %s", migration.__name__) - log.error(e) + # except Exception as e: + # log.error("Failed to run migration: %s", migration.__name__) + # log.error(e) + # sys.exit(0) MigrationManager.set_index(len(all_migrations)) diff --git a/app/migrations/v1_4_9/__init__.py b/app/migrations/v1_4_9/__init__.py index b5729c4f..be10c97b 100644 --- a/app/migrations/v1_4_9/__init__.py +++ b/app/migrations/v1_4_9/__init__.py @@ -1,9 +1,90 @@ import os import shutil +import sqlite3 +from time import time from app.db.sqlite.utils import SQLiteManager from app.migrations.base import Migration from app.settings import Paths +import hashlib +from unidecode import unidecode + +from app.db.sqlite.tracks import SQLiteTrackMethods as tdb +from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb +from app.db.sqlite.logger.tracks import SQLiteTrackLogger as ldb +from app.utils.hashing import create_hash + + +def create_sha256_hash(*args: str, decode=False, limit=10) -> str: + """ + This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments. + + Example use case: + - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID. + + :param args: The arguments to hash. + :param decode: Whether to decode the arguments before hashing. + :param limit: The number of characters to return. + + :return: The hash. + """ + + def remove_non_alnum(token: str) -> str: + token = token.lower().strip().replace(" ", "") + t = "".join(t for t in token if t.isalnum()) + + if t == "": + return token + + return t + + str_ = "".join(remove_non_alnum(t) for t in args) + + if decode: + str_ = unidecode(str_) + + str_ = str_.encode("utf-8") + str_ = hashlib.sha256(str_).hexdigest() + return str_[-limit:] + + +def create_sha1_hash(*args: str, decode=False, limit=10) -> str: + """ + This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments. + + Example use case: + - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID. + + :param args: The arguments to hash. + :param decode: Whether to decode the arguments before hashing. + :param limit: The number of characters to return. + + :return: The hash. + """ + + def remove_non_alnum(token: str) -> str: + token = token.lower().strip().replace(" ", "") + t = "".join(t for t in token if t.isalnum()) + + if t == "": + return token + + return t + + str_ = "".join(remove_non_alnum(t) for t in args) + + if decode: + str_ = unidecode(str_) + + str_ = str_.encode("utf-8") + str_ = hashlib.sha1(str_).hexdigest() + + return ( + str_[: limit // 2] + str_[-limit // 2 :] + if limit % 2 == 0 + else str_[: limit // 2] + str_[-limit // 2 - 1 :] + ) + class _1AddTimestampToFavoritesTable(Migration): """ @@ -13,37 +94,23 @@ class _1AddTimestampToFavoritesTable(Migration): @staticmethod def migrate(): # INFO: add timestamp column with automatic current timestamp - sql = f"ALTER TABLE favorites ADD COLUMN IF NOT EXISTS timestamp INTEGER NOT NULL DEFAULT 0" + sql = f"ALTER TABLE favorites ADD COLUMN timestamp INTEGER NOT NULL DEFAULT 0" # INFO: execute the sql with SQLiteManager(userdata_db=True) as cur: - try: - # INFO: Add the timestamp column to the favorites table - cur.execute(sql) + table_exists = cur.execute( + "select count(*) from pragma_table_info('favorites') where name = 'timestamp'" + ) - # INFO: Set all the timestamps to the current time - cur.execute("UPDATE favorites SET timestamp = strftime('%s', 'now')") - except Exception as e: - # INFO: timestamp column already exists - pass - finally: - cur.close() + table_exists = table_exists.fetchone() + if table_exists[0] == 1: + return -class _4MoveHashesToSha1(Migration): - """ - Moves the 10 bit item hashes from sha256 to sha1 which is - faster and more lenient on less powerful devices. - - Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution. - """ - - enabled: bool = False - - pass - - # INFO: Apparentlly, every single table is affected by this migration. - # NOTE: Use generators to avoid memory issues. + # INFO: Add the timestamp column to the favorites table + timestamp = int(time()) + cur.execute(sql) + cur.execute(f"UPDATE favorites SET timestamp = {timestamp}") class _2DeleteOriginalThumbnails(Migration): @@ -175,3 +242,164 @@ class _5AddUserIdToPlaylistsTable(Migration): # INFO: Execute the sql cur.executescript(sql) + + +class _6MoveHashesToSha1(Migration): + """ + Moves the 10 bit item hashes from sha256 to sha1 which is + faster and more lenient on less powerful devices. + + Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution. + """ + + # enabled: bool = False + + # pass + + # INFO: Apparentlly, every single table is affected by this migration. + # NOTE: Use generators to avoid memory issues. + + @classmethod + def port_track(cls, trackhash: str): + # get the track with the track hash + track = tdb.get_track_by_trackhash(trackhash) + + if track is None: + return + + title = track.og_title + if track.trackhash != trackhash: + # raise ValueError("Track hash mismatch") + print("Track hash mismatch") + title = track.title + else: + print("Porting track: ", track.title) + + # return the new hash + finalhash = create_sha1_hash( + ", ".join(a.name for a in track.artists), + track.og_album, + title, + ) + + if finalhash != create_hash( + ", ".join(a.name for a in track.artists), track.og_album, title + ): + raise ValueError("Hash mismatch") + + @classmethod + def port_album(cls, albumhash: str): + # get the first track with the album hash + track = tdb.get_track_by_albumhash(albumhash) + + if track is None: + return + + # return the new hash + return create_sha1_hash( + track.og_album, + ", ".join(a.name for a in track.albumartists), + ) + + @classmethod + def port_artist(cls, artisthash: str): + # find all tracks with the artist hash + tracks = [t for t in cls.tracks if artisthash in t.artist_hashes] + + if len(tracks) == 0: + return + + # find the artist name + artist = [ + a.name + for a in tracks[0].artists + if create_sha256_hash(a.name, decode=True) == artisthash + ][0] + + # return the new hash + return create_sha1_hash(artist, decode=True) + + @classmethod + def migrate_favorites(cls): + with SQLiteManager(userdata_db=True) as cur: + # read all favorites + data = cur.execute("SELECT * FROM favorites") + data = data.fetchall() + + for track in cls.tracks: + track.artist_hashes = "-".join( + [create_sha256_hash(a.name, decode=True) for a in track.artists] + ) + + for entry in data: + # hash is the 2nd column in the table + hash = entry[1] + + # entry type is the 3rd column in the table + if entry[2] == "track": + newhash = cls.port_track(hash) + + if newhash: + cur.execute( + f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'track'" + ) + + elif entry[2] == "album": + newhash = cls.port_album(hash) + + if newhash: + cur.execute( + f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'album'" + ) + + elif entry[2] == "artist": + newhash = cls.port_artist(hash) + + if newhash: + cur.execute( + f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'artist'" + ) + + @classmethod + def migrate_playlists(cls): + playlists = pdb.get_all_playlists() + + for playlist in playlists: + # remove previous hashes + to_remove = [ + {"trackhash": trackhash, "index": index} + for index, trackhash in enumerate(playlist.trackhashes) + ] + pdb.remove_tracks_from_playlist(playlist.id, to_remove) + + # add new hashes + newhashes = [ + cls.port_track(trackhash) for trackhash in playlist.trackhashes + ] + newhashes = [h for h in newhashes if h is not None] + pdb.add_tracks_to_playlist(playlist.id, newhashes) + + print("Ported playlist: ", playlist.name) + print("Total tracks: ", len(newhashes)) + + @classmethod + def migrate_scrobble(cls): + # read all logs + logs = ldb.get_all() + + with SQLiteManager(userdata_db=True) as cur: + # for each log, port the hash + for log in logs: + newhash = cls.port_track(log[1]) + + if newhash: + cur.execute( + f"UPDATE track_logger SET trackhash = '{newhash}' WHERE trackhash = '{log[1]}'" + ) + + @classmethod + def migrate(cls): + cls.tracks = list(tdb.get_all_tracks()) + cls.migrate_favorites() + # cls.migrate_playlists() + # cls.migrate_scrobble() diff --git a/app/models/album.py b/app/models/album.py index 36d1629e..63949b17 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -2,6 +2,7 @@ import dataclasses import datetime from dataclasses import dataclass +from app.config import UserConfig from app.settings import SessionVarKeys, get_flag from ..utils.hashing import create_hash @@ -16,94 +17,111 @@ class Album: Creates an album object """ + id: int + albumartists: list[dict[str, str]] albumhash: str + base_title: str + color: str + created_date: int + date: int + duration: int + genres: list[dict[str, str]] + og_title: str title: str - albumartists: list[Artist] + trackcount: int - albumartists_hashes: str = "" - image: str = "" - count: int = 0 - duration: int = 0 - colors: list[str] = dataclasses.field(default_factory=list) - date: str = "" - - created_date: int = 0 - og_title: str = "" - base_title: str = "" - is_soundtrack: bool = False - is_compilation: bool = False - is_single: bool = False - is_EP: bool = False - is_favorite: bool = False - is_live: bool = False - - genres: list[str] = dataclasses.field(default_factory=list) + type: str = "album" versions: list[str] = dataclasses.field(default_factory=list) def __post_init__(self): - self.title = self.title.strip() - self.og_title = self.title - self.image = self.albumhash + ".webp" + self.date = datetime.datetime.fromtimestamp(self.date).year - # Fetch album artists from title - if get_flag(SessionVarKeys.EXTRACT_FEAT): - featured, self.title = parse_feat_from_title(self.title) + # albumhash: str + # title: str + # albumartists: list[Artist] - if len(featured) > 0: - original_lower = "-".join([a.name.lower() for a in self.albumartists]) - self.albumartists.extend( - [Artist(a) for a in featured if a.lower() not in original_lower] - ) + # albumartists_hashes: str = "" + # image: str = "" + # count: int = 0 + # duration: int = 0 + # colors: list[str] = dataclasses.field(default_factory=list) + # date: str = "" - from ..store.tracks import TrackStore + # created_date: int = 0 + # og_title: str = "" + # base_title: str = "" + # is_soundtrack: bool = False + # is_compilation: bool = False + # is_single: bool = False + # is_EP: bool = False + # is_favorite: bool = False + # is_live: bool = False - TrackStore.append_track_artists(self.albumhash, featured, self.title) + # genres: list[str] = dataclasses.field(default_factory=list) - # Handle album version data - if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): - get_versions = not get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS) + # def __post_init__(self): + # self.title = self.title.strip() + # self.og_title = self.title + # self.image = self.albumhash + ".webp" - self.title, self.versions = get_base_title_and_versions( - self.title, get_versions=get_versions - ) - self.base_title = self.title + # # Fetch album artists from title + # if get_flag(SessionVarKeys.EXTRACT_FEAT): + # featured, self.title = parse_feat_from_title(self.title) - if "super_deluxe" in self.versions: - self.versions.remove("deluxe") + # if len(featured) > 0: + # original_lower = "-".join([a.name.lower() for a in self.albumartists]) + # self.albumartists.extend( + # [Artist(a) for a in featured if a.lower() not in original_lower] + # ) - if "original" in self.versions and self.check_is_soundtrack(): - self.versions.remove("original") + # from ..store.tracks import TrackStore - self.versions = [v.replace("_", " ") for v in self.versions] - else: - self.base_title = get_base_title_and_versions( - self.title, get_versions=False - )[0] + # TrackStore.append_track_artists(self.albumhash, featured, self.title) - self.albumartists_hashes = "-".join(a.artisthash for a in self.albumartists) + # # Handle album version data + # else: + # self.base_title = get_base_title_and_versions( + # self.title, get_versions=False + # )[0] - def set_colors(self, colors: list[str]): - self.colors = colors + # self.albumartists_hashes = "-".join(a.artisthash for a in self.albumartists) - def check_type(self): + # # def set_colors(self, colors: list[str]): + # # self.colors = colors + def populate_versions(self): + _, self.versions = get_base_title_and_versions(self.og_title, get_versions=True) + + if "super_deluxe" in self.versions: + self.versions.remove("deluxe") + + # at this point, we should know the type of album + if "original" in self.versions and self.type == "soundtrack": + self.versions.remove("original") + + self.versions = [v.replace("_", " ") for v in self.versions] + + def check_type(self, tracks: list[Track], singleTrackAsSingle: bool): """ Runs all the checks to determine the type of album. """ - self.is_soundtrack = self.check_is_soundtrack() - if self.is_soundtrack: - return + if self.is_single(tracks, singleTrackAsSingle): + return "single" - self.is_live = self.check_is_live_album() - if self.is_live: - return + if self.is_soundtrack(): + return "soundtrack" - self.is_compilation = self.check_is_compilation() - if self.is_compilation: - return + if self.is_live_album(): + return "live album" - self.is_EP = self.check_is_ep() + if self.is_compilation(): + return "compilation" - def check_is_soundtrack(self) -> bool: + if self.is_ep(): + return "ep" + + return "album" + + def is_soundtrack(self) -> bool: """ Checks if the album is a soundtrack. """ @@ -114,11 +132,11 @@ class Album: return False - def check_is_compilation(self) -> bool: + def is_compilation(self) -> bool: """ Checks if the album is a compilation. """ - artists = [a.name for a in self.albumartists] + artists = [a["name"] for a in self.albumartists] artists = "".join(artists).lower() if "various artists" in artists: @@ -137,7 +155,7 @@ class Album: "biggest hits", "the hits", "the ultimate", - "compilation" + "compilation", } for substring in substrings: @@ -146,7 +164,7 @@ class Album: return False - def check_is_live_album(self): + def is_live_album(self): """ Checks if the album is a live album. """ @@ -157,7 +175,7 @@ class Album: return False - def check_is_ep(self) -> bool: + def is_ep(self) -> bool: """ Checks if the album is an EP. """ @@ -165,22 +183,22 @@ class Album: # TODO: check against number of tracks - def check_is_single(self, tracks: list[Track]): + def is_single(self, tracks: list[Track], singleTrackAsSingle: bool): """ Checks if the album is a single. """ keywords = ["single version", "- single"] - show_albums_as_singles = get_flag(SessionVarKeys.SHOW_ALBUMS_AS_SINGLES) + # show_albums_as_singles = get_flag(SessionVarKeys.SHOW_ALBUMS_AS_SINGLES) for keyword in keywords: if keyword in self.og_title.lower(): - self.is_single = True - return + return True - if show_albums_as_singles and len(tracks) == 1: - self.is_single = True - return + # REVIEW: Reading from the config file in a for loop will be slow + # TODO: Find a + if singleTrackAsSingle and len(tracks) == 1: + return True if ( len(tracks) == 1 @@ -192,29 +210,29 @@ class Album: # and tracks[0].disc == 1 # TODO: Review -> Are the above commented checks necessary? ): - self.is_single = True + return True - def get_date_from_tracks(self, tracks: list[Track]): - """ - Gets the date of the album its tracks. + # def get_date_from_tracks(self, tracks: list[Track]): + # """ + # Gets the date of the album its tracks. - Args: - tracks (list[Track]): The tracks of the album. - """ - if self.date: - return + # Args: + # tracks (list[Track]): The tracks of the album. + # """ + # if self.date: + # return - dates = (int(t.date) for t in tracks if t.date) - try: - self.date = datetime.datetime.fromtimestamp(min(dates)).year - except: - self.date = datetime.datetime.now().year + # dates = (int(t.date) for t in tracks if t.date) + # try: + # self.date = datetime.datetime.fromtimestamp(min(dates)).year + # except: + # self.date = datetime.datetime.now().year - def set_count(self, count: int): - self.count = count + # def set_count(self, count: int): + # self.count = count - def set_duration(self, duration: int): - self.duration = duration + # def set_duration(self, duration: int): + # self.duration = duration - def set_created_date(self, created_date: int): - self.created_date = created_date + # def set_created_date(self, created_date: int): + # self.created_date = created_date diff --git a/app/models/artist.py b/app/models/artist.py index c8b6eb10..1e0446a7 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -23,6 +23,12 @@ class ArtistMinimal: if self.artisthash == "5a37d5315e": self.name = "Juice WRLD" + def to_json(self): + return { + "name": self.name, + "artisthash": self.artisthash, + } + @dataclass(slots=True) class Artist(ArtistMinimal): diff --git a/app/models/track.py b/app/models/track.py index ba928cc5..d4c638bc 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -4,7 +4,6 @@ from pathlib import Path from flask_jwt_extended import current_user - from app.settings import SessionVarKeys, get_flag from app.utils.hashing import create_hash from app.utils.parsers import ( @@ -24,10 +23,11 @@ class Track: Track class """ + id: int album: str - albumartists: str | list[ArtistMinimal] + albumartists: list[dict[str, str]] albumhash: str - artists: str | list[ArtistMinimal] + artists: str bitrate: int copyright: str date: int @@ -35,152 +35,174 @@ class Track: duration: int filepath: str folder: str - genre: str | list[str] + genre: list[dict[str, str]] + last_mod: int + og_album: str + og_title: str title: str track: int trackhash: str - last_mod: str | int - image: str = "" - artist_hashes: str = "" + _pos: int = 0 + _ati: str = "" - fav_userids: list = field(default_factory=list) - """ - A string of user ids separated by commas. - """ - # is_favorite: bool = False + # album: str + # albumartists: str | list[ArtistMinimal] + # albumhash: str + # artists: str | list[ArtistMinimal] + # bitrate: int + # copyright: str + # date: int + # disc: int + # duration: int + # filepath: str + # folder: str + # genre: str | list[str] + # title: str + # track: int + # trackhash: str + # last_mod: str | int - @property - def is_favorite(self): - return current_user['id'] in self.fav_userids + # image: str = "" + # artist_hashes: str = "" - # temporary attributes - _pos: int = 0 # for sorting tracks by disc and track number - _ati: str = ( - "" # (album track identifier) for removing duplicates when merging album versions - ) + # fav_userids: list = field(default_factory=list) + # """ + # A string of user ids separated by commas. + # """ + # # is_favorite: bool = False - og_title: str = "" - og_album: str = "" - created_date: float = 0.0 + # @property + # def is_favorite(self): + # return current_user["id"] in self.fav_userids - def set_created_date(self): - try: - self.created_date = Path(self.filepath).stat().st_ctime - except FileNotFoundError: - pass + # # temporary attributes + # _pos: int = 0 # for sorting tracks by disc and track number + # _ati: str = ( + # "" # (album track identifier) for removing duplicates when merging album versions + # ) - def __post_init__(self): - self.og_title = self.title - self.og_album = self.album - self.last_mod = int(self.last_mod) - self.date = int(self.date) + # og_title: str = "" + # og_album: str = "" + # created_date: float = 0.0 - # add a trailing slash to the folder path - # to avoid matching a folder starting with the same name as the root path - # eg. .../Music and .../Music Videos - self.folder = os.path.join(self.folder, "") + # def set_created_date(self): + # try: + # self.created_date = Path(self.filepath).stat().st_ctime + # except FileNotFoundError: + # pass - if self.artists is not None: - artists = split_artists(self.artists) - new_title = self.title + # def __post_init__(self): + # self.og_title = self.title + # self.og_album = self.album + # self.last_mod = int(self.last_mod) + # self.date = int(self.date) - if get_flag(SessionVarKeys.EXTRACT_FEAT): - featured, new_title = parse_feat_from_title(self.title) - original_lower = "-".join([create_hash(a) for a in artists]) - artists.extend( - a for a in featured if create_hash(a) not in original_lower - ) + # # add a trailing slash to the folder path + # # to avoid matching a folder starting with the same name as the root path + # # eg. .../Music and .../Music Videos + # self.folder = os.path.join(self.folder, "") - self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists) - self.artists = [ArtistMinimal(a) for a in artists] + # if self.artists is not None: + # artists = split_artists(self.artists) + # new_title = self.title - albumartists = split_artists(self.albumartists) + # if get_flag(SessionVarKeys.EXTRACT_FEAT): + # featured, new_title = parse_feat_from_title(self.title) + # original_lower = "-".join([create_hash(a) for a in artists]) + # artists.extend( + # a for a in featured if create_hash(a) not in original_lower + # ) - if not albumartists: - self.albumartists = self.artists[:1] - else: - self.albumartists = [ArtistMinimal(a) for a in albumartists] + # self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists) + # self.artists = [ArtistMinimal(a) for a in artists] - if get_flag(SessionVarKeys.REMOVE_PROD): - new_title = remove_prod(new_title) + # albumartists = split_artists(self.albumartists) - # if track is a single - if self.og_title == self.album: - self.rename_album(new_title) + # if not albumartists: + # self.albumartists = self.artists[:1] + # else: + # self.albumartists = [ArtistMinimal(a) for a in albumartists] - if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK): - new_title = clean_title(new_title) + # if get_flag(SessionVarKeys.REMOVE_PROD): + # new_title = remove_prod(new_title) - self.title = new_title + # if track is a single + # if self.og_title == self.album: + # self.rename_album(new_title) - if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): - self.album, _ = get_base_title_and_versions( - self.album, get_versions=False - ) + # if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK): + # new_title = clean_title(new_title) - if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS): - self.recreate_albumhash() + # self.title = new_title - self.image = self.albumhash + ".webp" + # if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): + # self.album, _ = get_base_title_and_versions( + # self.album, get_versions=False + # ) - if self.genre is not None and self.genre != "": - self.genre = self.genre.lower() - separators = {"/", ";", "&"} + # if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS): + # self.recreate_albumhash() - contains_rnb = "r&b" in self.genre - contains_rock = "rock & roll" in self.genre + # self.image = self.albumhash + ".webp" - if contains_rnb: - self.genre = self.genre.replace("r&b", "RnB") + # if self.genre is not None and self.genre != "": + # self.genre = self.genre.lower() + # separators = {"/", ";", "&"} - if contains_rock: - self.genre = self.genre.replace("rock & roll", "rock") + # contains_rnb = "r&b" in self.genre + # contains_rock = "rock & roll" in self.genre - for s in separators: - self.genre: str = self.genre.replace(s, ",") + # if contains_rnb: + # self.genre = self.genre.replace("r&b", "RnB") - self.genre = self.genre.split(",") - self.genre = [g.strip() for g in self.genre] + # if contains_rock: + # self.genre = self.genre.replace("rock & roll", "rock") - self.recreate_hash() - self.set_created_date() + # for s in separators: + # self.genre: str = self.genre.replace(s, ",") - def recreate_hash(self): - """ - Recreates a track hash if the track title was altered - to prevent duplicate tracks having different hashes. - """ - if self.og_title == self.title and self.og_album == self.album: - return + # self.genre = self.genre.split(",") + # self.genre = [g.strip() for g in self.genre] - self.trackhash = create_hash( - ", ".join(a.name for a in self.artists), self.og_album, self.title - ) + # self.recreate_hash() + # self.set_created_date() - def recreate_artists_hash(self): - """ - Recreates a track's artist hashes if the artist list was altered - """ - self.artist_hashes = "-".join(a.artisthash for a in self.artists) + # def recreate_hash(self): + # """ + # Recreates a track hash if the track title was altered + # to prevent duplicate tracks having different hashes. + # """ + # if self.og_title == self.title and self.og_album == self.album: + # return - def recreate_albumhash(self): - """ - Recreates an albumhash of a track to merge all versions of an album. - """ - albumartists = (a.name for a in self.albumartists) - self.albumhash = create_hash(self.album, *albumartists) + # self.trackhash = create_hash( + # ", ".join(a.name for a in self.artists), self.og_album, self.title + # ) - def rename_album(self, new_album: str): - """ - Renames an album - """ - self.album = new_album + # def recreate_artists_hash(self): + # """ + # Recreates a track's artist hashes if the artist list was altered + # """ + # self.artist_hashes = "-".join(a.artisthash for a in self.artists) - def add_artists(self, artists: list[str], new_album_title: str): - for artist in artists: - if create_hash(artist, decode=True) not in self.artist_hashes: - self.artists.append(ArtistMinimal(artist)) + # def recreate_albumhash(self): + # """ + # Recreates an albumhash of a track to merge all versions of an album. + # """ + # albumartists = (a.name for a in self.albumartists) + # self.albumhash = create_hash(self.album, *albumartists) - self.recreate_artists_hash() - self.rename_album(new_album_title) + # def rename_album(self, new_album: str): + # """ + # Renames an album + # """ + # self.album = new_album + + # def add_artists(self, artists: list[str], new_album_title: str): + # for artist in artists: + # if create_hash(artist, decode=True) not in self.artist_hashes: + # self.artists.append(ArtistMinimal(artist)) + + # self.recreate_artists_hash() + # self.rename_album(new_album_title) diff --git a/app/utils/auth.py b/app/utils/auth.py index 87f20c75..a19656a3 100644 --- a/app/utils/auth.py +++ b/app/utils/auth.py @@ -1,6 +1,8 @@ import hmac import hashlib +from flask_jwt_extended import current_user + from app.config import UserConfig @@ -29,3 +31,14 @@ def check_password(password: str, hashed: str) -> bool: """ return hmac.compare_digest(hash_password(password), hashed) + + +def get_current_userid() -> int: + """ + Get the current session user. + """ + try: + return current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + return 1 diff --git a/db.py b/db.py new file mode 100644 index 00000000..5ab99a23 --- /dev/null +++ b/db.py @@ -0,0 +1,74 @@ +from sqlalchemy import create_engine, text, Table, Column, Integer, String, MetaData, select +from sqlalchemy.orm import DeclarativeBase + +from typing import List, Optional + +from sqlalchemy.orm import Mapped, mapped_column, relationship + +fullpath = "/home/cwilvx/temp/swingmusic/swing.db" +engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) + +class Base(DeclarativeBase): + pass + +class Tracks(Base): + __tablename__ = "tracks" + + id: Mapped[int] = mapped_column(primary_key=True) + album: Mapped[str] = mapped_column(String()) + albumartist: Mapped[str] = mapped_column(String()) + copyright: Mapped[Optional[str]] + + def __repr__(self): + return f"" + +stmt = select(Tracks.album, Tracks.copyright).where(Tracks.album == "RAVAGE") +print(stmt) + +with engine.connect() as conn: + result = conn.execute(stmt) + for row in result: + print(row) + +# Base.metadata.create_all(engine) + +# metadata = MetaData() +# track_table = Table( +# "tracks", +# metadata, +# Column("id", Integer, primary_key=True, autoincrement=True), +# Column("album", String), +# Column("albumartist", String), +# Column("albumhash", String), +# Column("artist", String), +# Column("bitrate", Integer), +# Column("copyright", String), +# Column("date", Integer), +# Column("disc", Integer), +# Column("duration", Integer), +# Column("filepath", String), +# Column("folder", String), +# Column("genre", String), +# Column("title", String), +# Column("track", Integer), +# Column("trackhash", String), +# Column("last_mod", Integer), +# ) + +# metadata.create_all(engine) + + + + + + + +# with engine.connect() as conn: +# result = conn.execute( +# text("SELECT * FROM tracks where trackhash = :trackhash"), +# {"trackhash": "93acbea22b"}, +# ) +# # print(result.all()) + +# for r in result.mappings(): +# print(r["trackhash"]) diff --git a/manage.py b/manage.py index ddf13c76..62ddf2e1 100644 --- a/manage.py +++ b/manage.py @@ -21,6 +21,7 @@ import setproctitle from app.api import create_api from app.arg_handler import ProcessArgs +from app.lib.tagger import IndexEverything from app.lib.watchdogg import Watcher as WatchDog from app.periodic_scan import run_periodic_scans from app.plugins.register import register_plugins @@ -49,10 +50,11 @@ werkzeug.setLevel(logging.ERROR) # Background tasks -# @background -# def bg_run_setup(): -# pass +@background +def bg_run_setup(): + pass # run_periodic_scans() + IndexEverything() # @background @@ -63,7 +65,7 @@ werkzeug.setLevel(logging.ERROR) @background def run_swingmusic(): log_startup_info() - # bg_run_setup() + bg_run_setup() register_plugins() # start_watchdog() diff --git a/poetry.lock b/poetry.lock index f03244a1..2444e038 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2164,6 +2164,93 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "sqlalchemy" +version = "2.0.31" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, + {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, + {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + [[package]] name = "tabulate" version = "0.9.0" @@ -2515,4 +2602,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "5722346cbfc224340877e337d4cee2c8e8a3a7ea68f9a64d9c5806e0ebcf919a" +content-hash = "333baa055ac4a32ed914fb46025a48559575806dafba7db5aac97a3878ade23c" diff --git a/pyproject.toml b/pyproject.toml index 2cd8ede6..d521a0a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ watchdog = "^4.0.0" pendulum = "^3.0.0" flask-openapi3 = "^3.0.2" flask-jwt-extended = "^4.6.0" +sqlalchemy = "^2.0.31" [tool.poetry.dev-dependencies] pylint = "^2.15.5" From 3593b205ebc9620b858c80c59d1a764c501cb7c4 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 24 Jun 2024 20:48:13 +0300 Subject: [PATCH 02/44] save extra tags + port: streaming --- TODO.md | 4 +++ app/api/stream.py | 66 +++++++++------------------------------------ app/db/__init__.py | 42 ++++++++++++++++++++++++----- app/lib/tagger.py | 2 +- app/lib/taglib.py | 17 +++++++++++- app/models/track.py | 1 + 6 files changed, 69 insertions(+), 63 deletions(-) diff --git a/TODO.md b/TODO.md index 7f5cc591..76547381 100644 --- a/TODO.md +++ b/TODO.md @@ -16,6 +16,10 @@ - Recreate album hash if featured artists are discover - Implement checking if is clean install and skip migrations! + + + + # DONE - Support auth headers - Add recently played playlist diff --git a/app/api/stream.py b/app/api/stream.py index d5a79535..3eaee2a6 100644 --- a/app/api/stream.py +++ b/app/api/stream.py @@ -10,7 +10,8 @@ from pydantic import BaseModel, Field from app.api.apischemas import TrackHashSchema from app.lib.trackslib import get_silence_paddings -from app.store.tracks import TrackStore +# from app.store.tracks import TrackStore +from app.db import TrackTable from app.utils.files import guess_mime_type bp_tag = Tag(name="File", description="Audio files") @@ -34,36 +35,12 @@ def send_track_file_legacy(path: TrackHashSchema, query: SendTrackFileQuery): filepath = query.filepath msg = {"msg": "File Not Found"} - def get_mime(filename: str) -> str: - ext = filename.rsplit(".", maxsplit=1)[-1] - return f"audio/{ext}" + track = TrackTable.get_track_by_trackhash(trackhash, filepath) + track_exists = track is not None and os.path.exists(track.filepath) - # If filepath is provide, try to send that - if filepath is not None: - try: - track = TrackStore.get_tracks_by_filepaths([filepath])[0] - except IndexError: - track = None - - track_exists = track is not None and os.path.exists(track.filepath) - - if track_exists: - audio_type = get_mime(filepath) - return send_file(filepath, mimetype=audio_type, conditional=True) - - # Else, find file by trackhash - tracks = TrackStore.get_tracks_by_trackhashes([trackhash]) - - for track in tracks: - if track is None: - return msg, 404 - - audio_type = get_mime(track.filepath) - - try: - return send_file(track.filepath, mimetype=audio_type, conditional=True) - except (FileNotFoundError, OSError) as e: - return msg, 404 + if track_exists: + audio_type = guess_mime_type(filepath) + return send_file(filepath, mimetype=audio_type, conditional=True) return msg, 404 @@ -80,31 +57,12 @@ def send_track_file(path: TrackHashSchema, query: SendTrackFileQuery): msg = {"msg": "File Not Found"} # If filepath is provided, try to send that - if filepath is not None: - try: - track = TrackStore.get_tracks_by_filepaths([filepath])[0] - except IndexError: - track = None + track = TrackTable.get_track_by_trackhash(trackhash, filepath) + track_exists = track is not None and os.path.exists(track.filepath) - track_exists = track is not None and os.path.exists(track.filepath) - - if track_exists: - audio_type = guess_mime_type(filepath) - return send_file_as_chunks(track.filepath, audio_type) - - # Else, find file by trackhash - tracks = TrackStore.get_tracks_by_trackhashes([trackhash]) - - for track in tracks: - if track is None: - return msg, 404 - - audio_type = guess_mime_type(track.filepath) - - try: - return send_file_as_chunks(track.filepath, audio_type) - except (FileNotFoundError, OSError) as e: - return msg, 404 + if track_exists: + audio_type = guess_mime_type(filepath) + return send_file_as_chunks(track.filepath, audio_type) return msg, 404 diff --git a/app/db/__init__.py b/app/db/__init__.py index dbea0ad6..24adab64 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -9,6 +9,7 @@ from sqlalchemy import ( Row, String, Tuple, + and_, create_engine, insert, select, @@ -39,7 +40,8 @@ def todicts(tracks: list[Any]): class DbManager: - def __init__(self): + def __init__(self, commit: bool = False): + self.commit = commit self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) self.conn = self.engine.connect() @@ -47,7 +49,8 @@ class DbManager: return self.conn.execution_options(preserve_rowcount=True) def __exit__(self, exc_type, exc_val, exc_tb): - self.conn.commit() + if self.commit: + self.conn.commit() self.conn.close() @@ -57,7 +60,7 @@ class Base(MappedAsDataclass, DeclarativeBase): """ Inserts multiple items into the database. """ - with DbManager() as conn: + with DbManager(commit=True) as conn: conn.execute(insert(cls).values(items)) @classmethod @@ -177,6 +180,7 @@ class TrackTable(Base): title: Mapped[str] = mapped_column(String()) track: Mapped[int] = mapped_column(Integer()) trackhash: Mapped[str] = mapped_column(String(), index=True) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) @classmethod def get_tracks_by_filepaths(cls, filepaths: list[str]): @@ -209,23 +213,47 @@ class TrackTable(Base): tracks = tracks_to_dataclasses(result.fetchall()) return remove_duplicates(tracks, is_album_tracks=True) + @classmethod + def get_track_by_trackhash(cls, hash: str, filepath: str = ""): + with DbManager() as conn: + if filepath: + result = conn.execute( + select(TrackTable) + .where( + and_( + TrackTable.trackhash == hash, + TrackTable.filepath == filepath, + ) + ) + .order_by(TrackTable.bitrate.desc()) + ) + else: + result = conn.execute( + select(TrackTable).where(TrackTable.trackhash == hash) + ) + + track = result.fetchone() + + if track: + return track_to_dataclass(track) + # SECTION: HELPER FUNCTIONS -def album_to_dataclass(album: Row[AlbumTable]): +def album_to_dataclass(album: Any): return AlbumModel(**album._asdict()) -def albums_to_dataclasses(albums: list[Row[AlbumTable]]): +def albums_to_dataclasses(albums: Any): return [album_to_dataclass(album) for album in albums] -def track_to_dataclass(track: Row[TrackTable]): +def track_to_dataclass(track: Any): return TrackModel(**track._asdict()) -def tracks_to_dataclasses(tracks: list[Row[TrackTable]]): +def tracks_to_dataclasses(tracks: Any): return [track_to_dataclass(track) for track in tracks] diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 38101f80..95e1a510 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -62,7 +62,7 @@ class IndexAlbums: album["created_dates"].append(track.last_mod) if track.genre: - album["genres"].append(track.genre) + album["genres"].extend(track.genre) for album in albums.values(): album["date"] = min(album["dates"]) diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 68cfd519..6c1bc9c5 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -2,7 +2,9 @@ from dataclasses import dataclass import os from io import BytesIO from pathlib import Path +from pprint import pprint import re +import sys import pendulum from PIL import Image, UnidentifiedImageError @@ -318,6 +320,20 @@ def get_tags(filepath: str): *[a["name"] for a in tags.artists], tags.album, tags.title ) + more_extra = { + "audio_offset": tags.audio_offset, + "bitdepth": tags.bitdepth, + "composer": tags.composer, + "channels": tags.channels, + "comment": tags.comment, + "disc_total": tags.disc_total, + "filesize": tags.filesize, + "samplerate": tags.samplerate, + "track_total": tags.track_total, + } + + tags.extra = {**tags.extra, **more_extra} + tags = tags.__dict__ # delete all tag properties that start with _ (tinytag internals) @@ -332,7 +348,6 @@ def get_tags(filepath: str): "comment", "composer", "disc_total", - "extra", "samplerate", "track_total", "year", diff --git a/app/models/track.py b/app/models/track.py index d4c638bc..a2261190 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -42,6 +42,7 @@ class Track: title: str track: int trackhash: str + extra: dict _pos: int = 0 _ati: str = "" From 54a1b85d8b2ebf3fb19c8eb668560a87e9a8472e Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 24 Jun 2024 22:08:05 +0300 Subject: [PATCH 03/44] port: artist page --- app/api/artist.py | 213 +++++++++++++++++++++++++------------------ app/db/__init__.py | 43 +++++++++ app/lib/tagger.py | 5 +- app/lib/taglib.py | 3 + app/models/album.py | 1 + app/models/artist.py | 38 +++----- app/models/track.py | 3 + 7 files changed, 187 insertions(+), 119 deletions(-) diff --git a/app/api/artist.py b/app/api/artist.py index 2d69e542..03a01874 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -2,6 +2,7 @@ Contains all the artist(s) routes. """ +from itertools import groupby import math import random from datetime import datetime @@ -9,17 +10,26 @@ from datetime import datetime from flask_jwt_extended import current_user from flask_openapi3 import APIBlueprint, Tag from pydantic import Field -from app.api.apischemas import AlbumLimitSchema, ArtistHashSchema, ArtistLimitSchema, TrackLimitSchema +from app.api.apischemas import ( + AlbumLimitSchema, + ArtistHashSchema, + ArtistLimitSchema, + TrackLimitSchema, +) +from app.config import UserConfig +from app.db import AlbumTable, ArtistTable, TrackTable from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as fmdb from app.models import Album, FavType from app.serializers.album import serialize_for_card_many from app.serializers.track import serialize_tracks + from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore + bp_tag = Tag(name="Artist", description="Single artist") api = APIBlueprint("artist", __name__, url_prefix="/artist", abp_tags=[bp_tag]) @@ -34,35 +44,22 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): artisthash = path.artisthash limit = query.limit - artist = ArtistStore.get_artist_by_hash(artisthash) + artist = ArtistTable.get_artist_by_hash(artisthash) + print(artist) if artist is None: return {"error": "Artist not found"}, 404 - tracks = TrackStore.get_tracks_by_artisthash(artisthash) + tracks = TrackTable.get_tracks_by_artisthash(artisthash) tcount = len(tracks) - acount = AlbumStore.count_albums_by_artisthash(artisthash) - if acount == 0 and tcount < 10: + if artist.albumcount == 0 and tcount < 10: limit = tcount - artist.set_trackcount(tcount) - artist.set_albumcount(acount) - artist.set_duration(sum(t.duration for t in tracks)) - - artist.is_favorite = favdb.check_is_favorite(artisthash, FavType.artist) - - genres = set() - - for t in tracks: - if t.genre is not None: - genres = genres.union(t.genre) - - genres = list(genres) + # artist.is_favorite = favdb.check_is_favorite(artisthash, FavType.artist) try: - min_stamp = min(t.date for t in tracks) - year = datetime.fromtimestamp(min_stamp).year + year = datetime.fromtimestamp(artist.date).year except ValueError: year = 0 @@ -73,12 +70,11 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): decade = str(decade)[2:] + "s" if decade: - genres.insert(0, decade) + artist.genres.insert(0, {"name": decade, "genrehash": decade}) return { "artist": artist, "tracks": serialize_tracks(tracks[:limit]), - "genres": genres, } @@ -98,83 +94,118 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): limit = query.limit - all_albums = AlbumStore.get_albums_by_artisthash(artisthash) - # start: check for missing albums. ie. compilations and features - all_tracks = TrackStore.get_tracks_by_artisthash(artisthash) - - track_albums = set(t.albumhash for t in all_tracks) - missing_album_hashes = track_albums.difference(set(a.albumhash for a in all_albums)) - - if len(missing_album_hashes) > 0: - missing_albums = AlbumStore.get_albums_by_hashes(list(missing_album_hashes)) - all_albums.extend(missing_albums) - - # end check - - def get_album_tracks(albumhash: str): - tracks = [t for t in all_tracks if t.albumhash == albumhash] - - if len(tracks) > 0: - return tracks - - return TrackStore.get_tracks_by_albumhash(albumhash) - - for a in all_albums: - a.check_type() - - album_tracks = get_album_tracks(a.albumhash) - - if len(album_tracks) == 0: - continue - - a.get_date_from_tracks(album_tracks) - - if a.date == 0: - AlbumStore.remove_album_by_hash(a.albumhash) - continue - - a.is_single(album_tracks) - - all_albums = sorted(all_albums, key=lambda a: str(a.date), reverse=True) - - singles = [a for a in all_albums if a.is_single] - eps = [a for a in all_albums if a.is_EP] - - def remove_EPs_and_singles(albums_: list[Album]): - albums_ = [a for a in albums_ if not a.is_single] - albums_ = [a for a in albums_ if not a.is_EP] - return albums_ - - albums = filter(lambda a: artisthash in a.albumartists_hashes, all_albums) - albums = list(albums) - albums = remove_EPs_and_singles(albums) - - compilations = [a for a in albums if a.is_compilation] - for c in compilations: - albums.remove(c) - - appearances = filter(lambda a: artisthash not in a.albumartists_hashes, all_albums) - appearances = list(appearances) - - appearances = remove_EPs_and_singles(appearances) - - artist = ArtistStore.get_artist_by_hash(artisthash) + artist = ArtistTable.get_artist_by_hash(artisthash) if artist is None: return {"error": "Artist not found"}, 404 + albums = AlbumTable.get_albums_by_artisthash(artisthash) + tracks = TrackTable.get_tracks_by_artisthash(artisthash) + + missing_albumhashes = { + t.albumhash for t in tracks if t.albumhash not in {a.albumhash for a in albums} + } + + albums.extend(AlbumTable.get_albums_by_hash(missing_albumhashes)) + albumdict = {a.albumhash: a for a in albums} + + config = UserConfig() + albumgroups = groupby(tracks, key=lambda t: t.albumhash) + for albumhash, tracks in albumgroups: + album = albumdict.get(albumhash) + + if album: + album.check_type(list(tracks), config.showAlbumsAsSingles) + + # all_albums = AlbumStore.get_albums_by_artisthash(artisthash) + # start: check for missing albums. ie. compilations and features + # all_tracks = TrackStore.get_tracks_by_artisthash(artisthash) + + # track_albums = set(t.albumhash for t in all_tracks) + # missing_album_hashes = track_albums.difference(set(a.albumhash for a in all_albums)) + + # if len(missing_album_hashes) > 0: + # missing_albums = AlbumStore.get_albums_by_hashes(list(missing_album_hashes)) + # all_albums.extend(missing_albums) + + # end check + + # def get_album_tracks(albumhash: str): + # tracks = [t for t in all_tracks if t.albumhash == albumhash] + + # if len(tracks) > 0: + # return tracks + + # return TrackStore.get_tracks_by_albumhash(albumhash) + + # for a in all_albums: + # a.check_type() + + # album_tracks = get_album_tracks(a.albumhash) + + # if len(album_tracks) == 0: + # continue + + # a.get_date_from_tracks(album_tracks) + + # if a.date == 0: + # AlbumStore.remove_album_by_hash(a.albumhash) + # continue + + # a.is_single(album_tracks) + + albums = [a for a in albumdict.values()] + all_albums = sorted(albums, key=lambda a: str(a.date), reverse=True) + + res = { + "albums": [], + "appearances": [], + "compilations": [], + "singles_and_eps": [], + } + + for album in all_albums: + if album.type == "single" or album.type == "ep": + res["singles_and_eps"].append(album) + elif album.type == "compilation": + res["compilations"].append(album) + elif album.albumhash in missing_albumhashes: + res["appearances"].append(album) + else: + res["albums"].append(album) + + # def remove_EPs_and_singles(albums_: list[Album]): + # albums_ = [a for a in albums_ if not a.type == "single"] + # albums_ = [a for a in albums_ if not a.type == "ep"] + # return albums_ + + # albums = filter(lambda a: artisthash in missing_albumhashes, all_albums) + # albums = list(albums) + # albums = remove_EPs_and_singles(albums) + + # compilations = [a for a in albums if a.is_compilation] + # for c in compilations: + # albums.remove(c) + + # appearances = filter(lambda a: artisthash not in a.albumartists_hashes, all_albums) + # appearances = list(appearances) + + # appearances = remove_EPs_and_singles(appearances) + + # artist = ArtistStore.get_artist_by_hash(artisthash) + + # if artist is None: + # return {"error": "Artist not found"}, 404 + if return_all: limit = len(all_albums) - singles_and_eps = singles + eps + # loop through the res dict and serialize the albums + for key, value in res.items(): + res[key] = serialize_for_card_many(value[:limit]) - return { - "artistname": artist.name, - "albums": serialize_for_card_many(albums[:limit]), - "singles_and_eps": serialize_for_card_many(singles_and_eps[:limit]), - "appearances": serialize_for_card_many(appearances[:limit]), - "compilations": serialize_for_card_many(compilations[:limit]), - } + res["artistname"] = artist.name + return res @api.get("//tracks") diff --git a/app/db/__init__.py b/app/db/__init__.py index 24adab64..df2132b6 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -24,6 +24,7 @@ from sqlalchemy.orm import ( from app.models import Track as TrackModel from app.models import Album as AlbumModel +from app.models import Artist as ArtistModel from app.utils.remove_duplicates import remove_duplicates @@ -100,12 +101,21 @@ class ArtistTable(Base): result = conn.execute(select(cls).offset(start).limit(limit)) return albums_to_dataclasses(result.fetchall()) + @classmethod + def get_artist_by_hash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(ArtistTable).where(ArtistTable.artisthash == artisthash) + ) + return artist_to_dataclass(result.fetchone()) + class AlbumTable(Base): __tablename__ = "album" id: Mapped[int] = mapped_column(primary_key=True) albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) base_title: Mapped[str] = mapped_column(String()) color: Mapped[Optional[str]] = mapped_column(String()) @@ -128,6 +138,14 @@ class AlbumTable(Base): if album: return album_to_dataclass(album) + @classmethod + def get_albums_by_hash(cls, hashes: set[str]): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)) + ) + return albums_to_dataclasses(result.fetchall()) + @classmethod def get_all(cls, start: int, limit: int): with DbManager() as conn: @@ -157,6 +175,14 @@ class AlbumTable(Base): ) return albums_to_dataclasses(result.fetchall()) + @classmethod + def get_albums_by_artisthash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash)) + ) + return albums_to_dataclasses(result.all()) + class TrackTable(Base): __tablename__ = "track" @@ -165,6 +191,7 @@ class TrackTable(Base): album: Mapped[str] = mapped_column(String()) albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) albumhash: Mapped[str] = mapped_column(String(), index=True) + artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) bitrate: Mapped[int] = mapped_column(Integer()) copyright: Mapped[Optional[str]] = mapped_column(String()) @@ -237,10 +264,26 @@ class TrackTable(Base): if track: return track_to_dataclass(track) + @classmethod + def get_tracks_by_artisthash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.artists.contains(artisthash)) + ) + return tracks_to_dataclasses(result.fetchall()) + # SECTION: HELPER FUNCTIONS +def artist_to_dataclass(artist: Any): + return ArtistModel(**artist._asdict()) + + +def artists_to_dataclasses(artists: Any): + return [artist_to_dataclass(artist) for artist in artists] + + def album_to_dataclass(album: Any): return AlbumModel(**album._asdict()) diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 95e1a510..9d783b62 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -41,6 +41,7 @@ class IndexAlbums: if track.albumhash not in albums: albums[track.albumhash] = { "albumartists": track.albumartists, + "artisthashes": [a['artisthash'] for a in track.albumartists], "albumhash": track.albumhash, "base_title": None, "color": None, @@ -107,7 +108,7 @@ class IndexArtists: "dates": [track.date], "date": None, "duration": track.duration, - "genres": [*track.genre] if track.genre else [], + "genres": track.genre if track.genre else [], "name": artist["name"], "trackcount": None, "tracks": {track.trackhash}, @@ -121,7 +122,7 @@ class IndexArtists: artist["created_dates"].append(track.last_mod) if track.genre: - artist["genres"].append(track.genre) + artist["genres"].extend(track.genre) for artist in artists.values(): diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 6c1bc9c5..03cafc1c 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -262,6 +262,9 @@ def get_tags(filepath: str): for a in split_albumartists ] + tags.artisthashes = list({a["artisthash"] for a in tags.artists + tags.albumartists}) + + # remove prod by if config.removeProdBy: new_title = remove_prod(new_title) diff --git a/app/models/album.py b/app/models/album.py index 63949b17..337f24cf 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -20,6 +20,7 @@ class Album: id: int albumartists: list[dict[str, str]] albumhash: str + artisthashes: list[str] base_title: str color: str created_date: int diff --git a/app/models/artist.py b/app/models/artist.py index 1e0446a7..0c50a60d 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -31,33 +31,19 @@ class ArtistMinimal: @dataclass(slots=True) -class Artist(ArtistMinimal): +class Artist: """ Artist class """ - name: str = "" - trackcount: int = 0 - albumcount: int = 0 - duration: int = 0 - colors: list[str] = dataclasses.field(default_factory=list) - is_favorite: bool = False - created_date: float = 0.0 - - def __post_init__(self): - super(Artist, self).__init__(self.name) - - def set_trackcount(self, count: int): - self.trackcount = count - - def set_albumcount(self, count: int): - self.albumcount = count - - def set_duration(self, duration: int): - self.duration = duration - - def set_colors(self, colors: list[str]): - self.colors = colors - - def set_created_date(self, created_date: float): - self.created_date = created_date + id: str + name: str + albumcount: int + artisthash: str + created_date: int + date: int + duration: int + genres: list[dict[str, str]] + name: str + trackcount: int + is_favorite: bool diff --git a/app/models/track.py b/app/models/track.py index a2261190..0a853b3e 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -27,6 +27,7 @@ class Track: album: str albumartists: list[dict[str, str]] albumhash: str + artisthashes: list[str] artists: str bitrate: int copyright: str @@ -44,9 +45,11 @@ class Track: trackhash: str extra: dict + is_favorite: bool = False _pos: int = 0 _ati: str = "" + # album: str # albumartists: str | list[ArtistMinimal] # albumhash: str From 1a66194c6ca68332212d6f07a328c551620a9a3d Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 27 Jun 2024 00:02:08 +0300 Subject: [PATCH 04/44] fix: slow folder track count + etc --- app/api/__init__.py | 2 +- app/api/folder.py | 8 +- app/api/getall/__init__.py | 11 +-- app/config.py | 6 +- app/db/__init__.py | 150 ++++++++++++++++++++++++++++++------- app/lib/folderslib.py | 42 +---------- app/lib/populate.py | 19 +++-- app/lib/tagger.py | 9 ++- app/setup/__init__.py | 4 +- app/utils/auth.py | 2 +- manage.py | 8 +- poetry.lock | 16 +++- pyproject.toml | 1 + 13 files changed, 183 insertions(+), 95 deletions(-) diff --git a/app/api/__init__.py b/app/api/__init__.py index 108e7513..f6eab130 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -64,7 +64,7 @@ def create_api(): app = OpenAPI(__name__, info=api_info, doc_prefix="/docs") # JWT CONFIGS - app.config["JWT_SECRET_KEY"] = UserConfig().userId + app.config["JWT_SECRET_KEY"] = UserConfig().serverId app.config["JWT_TOKEN_LOCATION"] = ["cookies", "headers"] app.config["JWT_COOKIE_CSRF_PROTECT"] = False app.config["JWT_SESSION_COOKIE"] = False diff --git a/app/api/folder.py b/app/api/folder.py index 71e8ac5b..c2d1cc85 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -10,12 +10,13 @@ from pydantic import BaseModel, Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from showinfm import show_in_file_manager +from memory_profiler import profile from app import settings +from app.db import TrackTable from app.db.sqlite.settings import SettingsSQLMethods as db from app.lib.folderslib import GetFilesAndDirs, get_folders from app.serializers.track import serialize_track -from app.store.tracks import TrackStore as store from app.utils.wintools import is_windows, win_replace_slash tag = Tag(name="Folders", description="Get folders and tracks in a directory") @@ -66,9 +67,7 @@ def get_folder_tree(body: FolderTree): else: req_dir = "/" + req_dir if not req_dir.startswith("/") else req_dir - print('stuff!') res = GetFilesAndDirs(req_dir, tracks_only=tracks_only)() - print(res['folders']) res["folders"] = sorted(res["folders"], key=lambda i: i.name) return res @@ -183,8 +182,7 @@ def get_tracks_in_path(query: GetTracksInPathQuery): Used when adding tracks to the queue. """ - tracks = store.get_tracks_in_path(query.path) - tracks = sorted(tracks, key=lambda i: i.last_mod) + tracks = TrackTable.get_tracks_in_path(query.path) tracks = (serialize_track(t) for t in tracks if Path(t.filepath).exists()) return { diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index 607fa08a..31c72b27 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -61,11 +61,11 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): is_artists = path.itemtype == "artists" if is_albums: - items = AlbumTable.get_all(query.start, query.limit) + items, total = AlbumTable.get_all(query.start, query.limit) elif is_artists: - items = ArtistTable.get_all(query.start, query.limit) + items, total = ArtistTable.get_all(query.start, query.limit) - print(items) + # print(items) start = query.start limit = query.limit @@ -93,6 +93,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): for item in items: item_dict = serialize_album(item) if is_albums else serialize_artist(item) + print(item_dict) if sort_is_date: item_dict["help_text"] = item.date @@ -117,9 +118,9 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): if sort_is_artist_albumcount: item_dict["help_text"] = ( - f"{format_number(item['albumcount'])} album{'' if item['albumcount'] == 1 else 's'}" + f"{format_number(item.albumcount)} album{'' if item.albumcount == 1 else 's'}" ) album_list.append(item_dict) - return {"items": album_list, "total": len(sorted_items)} + return {"items": album_list, "total": total} diff --git a/app/config.py b/app/config.py index 2d477ea0..b751f129 100644 --- a/app/config.py +++ b/app/config.py @@ -14,13 +14,13 @@ class UserConfig: # auth stuff # NOTE: Don't expose the userId via the API - userId: str = "" + serverId: str = "" usersOnLogin: bool = True # lists rootDirs: list[str] = field(default_factory=list) excludeDirs: list[str] = field(default_factory=list) - artistSeparators: set[str] = field(default_factory=list) + artistSeparators: set[str] = field(default_factory=set) genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) # tracks @@ -80,7 +80,7 @@ class UserConfig: settings = {k: v for k, v in settings.items() if not k.startswith("_")} with open(self._config_path, "w") as f: - json.dump(settings, f, indent=4) + json.dump(settings, f, indent=4, default=list) def __setattr__(self, key: str, value: Any) -> None: """ diff --git a/app/db/__init__.py b/app/db/__init__.py index df2132b6..c7321c80 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -1,7 +1,11 @@ +from concurrent.futures import ThreadPoolExecutor import json +import os +from pathlib import Path from pprint import pprint from typing import Any, Optional +from memory_profiler import profile from sqlalchemy import ( JSON, Boolean, @@ -27,32 +31,83 @@ from app.models import Album as AlbumModel from app.models import Artist as ArtistModel from app.utils.remove_duplicates import remove_duplicates - fullpath = "/home/cwilvx/temp/swingmusic/swing.db" -engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=False) +engine = create_engine( + f"sqlite+pysqlite:///{fullpath}", + echo=False, + max_overflow=0, + pool_size=5, +) + +if not os.path.exists(fullpath): + os.makedirs(Path(fullpath).parent) + +connection = engine.connect() +all_filepaths = list() -def todict(track: Any): - return track._asdict() +def getIndexOfFirstMatch(strings: list[str], prefix: str): + """ + Find the index of the first path that starts with the given path. + + Uses a binary search algorithm to find the index. + """ + + left = 0 + right = len(strings) - 1 + + while left <= right: + mid = (left + right) // 2 + + if strings[mid].startswith(prefix): + if mid == 0 or not strings[mid - 1].startswith(prefix): + return mid + right = mid - 1 + elif strings[mid] < prefix: + left = mid + 1 + else: + right = mid - 1 + + return -1 -def todicts(tracks: list[Any]): - return [todict(track) for track in tracks] +def countFilepathsInDir(dirpath: str): + """ + Return all the filepaths in a directory. + """ + global all_filepaths + index = getIndexOfFirstMatch(all_filepaths, dirpath) + + if index == -1: + return 0 + + paths: list[str] = [] + + for path in all_filepaths[index:]: + if path.startswith(dirpath): + paths.append(path) + else: + break + + return len(paths) class DbManager: def __init__(self, commit: bool = False): self.commit = commit - self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) - self.conn = self.engine.connect() + # self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) + # self.conn = self.engine.connect() + # pass def __enter__(self): - return self.conn.execution_options(preserve_rowcount=True) + # return self.conn.execution_options(preserve_rowcount=True) + return connection def __exit__(self, exc_type, exc_val, exc_tb): if self.commit: - self.conn.commit() - self.conn.close() + connection.commit() + + # self.conn.close() class Base(MappedAsDataclass, DeclarativeBase): @@ -98,8 +153,13 @@ class ArtistTable(Base): @classmethod def get_all(cls, start: int, limit: int): with DbManager() as conn: - result = conn.execute(select(cls).offset(start).limit(limit)) - return albums_to_dataclasses(result.fetchall()) + if start == 0: + result = conn.execute(select(cls)) + else: + result = conn.execute(select(cls).offset(start).limit(limit)) + + all = result.fetchall() + return artists_to_dataclasses(all), len(all) @classmethod def get_artist_by_hash(cls, artisthash: str): @@ -149,8 +209,14 @@ class AlbumTable(Base): @classmethod def get_all(cls, start: int, limit: int): with DbManager() as conn: - result = conn.execute(select(AlbumTable).offset(start).limit(limit)) - return albums_to_dataclasses(result.fetchall()) + if start == 0: + result = conn.execute(select(AlbumTable)) + else: + result = conn.execute(select(AlbumTable).offset(start).limit(limit)) + + all = result.fetchall() + + return albums_to_dataclasses(all)[:limit], len(all) @classmethod def get_albums_by_artisthashes(cls, artisthashes: list[dict[str, str]]): @@ -164,7 +230,6 @@ class AlbumTable(Base): ) albums.extend(albums_to_dataclasses(result.fetchall())) - print(albums) return albums @classmethod @@ -198,7 +263,7 @@ class TrackTable(Base): date: Mapped[int] = mapped_column(Integer()) disc: Mapped[int] = mapped_column(Integer()) duration: Mapped[int] = mapped_column(Integer()) - filepath: Mapped[str] = mapped_column(String(), unique=True) + filepath: Mapped[str] = mapped_column(String(), index=True, unique=True) folder: Mapped[str] = mapped_column(String(), index=True) genre: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) last_mod: Mapped[float] = mapped_column(Integer()) @@ -211,23 +276,21 @@ class TrackTable(Base): @classmethod def get_tracks_by_filepaths(cls, filepaths: list[str]): - print(filepaths[0]) with DbManager() as conn: result = conn.execute( select(TrackTable).where(TrackTable.filepath.in_(filepaths)) ) - return [dict(r) for r in result.mappings().fetchall()] + return tracks_to_dataclasses(result.fetchall()) @classmethod def count_tracks_containing_paths(cls, paths: list[str]): results: list[dict[str, int | str]] = [] - with DbManager() as conn: - for path in paths: - result = conn.execute( - select(TrackTable).where(TrackTable.filepath.contains(path)) - ) - results.append({"path": path, "trackcount": result.all().__len__()}) + with ThreadPoolExecutor() as executor: + res = executor.map(countFilepathsInDir, paths) + results = [ + {"path": path, "trackcount": count} for path, count in zip(paths, res) + ] return results @@ -272,6 +335,43 @@ class TrackTable(Base): ) return tracks_to_dataclasses(result.fetchall()) + @classmethod + def get_tracks_in_path(cls, path: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable) + .where(TrackTable.filepath.contains(path)) + .order_by(TrackTable.last_mod) + ) + return tracks_to_dataclasses(result.fetchall()) + + +all_tracks = TrackTable.get_all() + +for track in all_tracks: + all_filepaths.append(track.filepath) + +all_filepaths.sort() + +# print("files in path: ",getFilepathsInDir("/home/cwilvx/Music/").__len__()) + + +# SECTION: Userdata database +class UserTable(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column(String(), unique=True) + firstname: Mapped[Optional[str]] = mapped_column(String()) + lastname: Mapped[Optional[str]] = mapped_column(String()) + password: Mapped[str] = mapped_column(String()) + email: Mapped[Optional[str]] = mapped_column(String()) + image: Mapped[Optional[str]] = mapped_column(String()) + roles: Mapped[list[str]] = mapped_column(JSON(), default_factory=lambda: ["user"]) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSON(), default_factory=dict + ) + # SECTION: HELPER FUNCTIONS diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index ccf1c499..0f3340c2 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -2,12 +2,11 @@ import os from pathlib import Path from app.logger import log -from app.models import Folder, Track +from app.models import Folder from app.serializers.track import serialize_tracks from app.settings import SUPPORTED_FILES from app.utils.wintools import win_replace_slash -from app.store.tracks import TrackStore from app.db import TrackTable as TrackDB @@ -51,39 +50,6 @@ def get_folders(paths: list[str]): for f in folders if f["trackcount"] > 0 ] - # count_dict = { - # "tracks": {path: 0 for path in paths}, - # # folders are immediate children of the root folder - # "folders": {path: set() for path in paths}, - # } - - # for track in TrackStore.tracks: - # for path in paths: - - # # a child path should be longer than the root path - # if len(track.folder) >= len(path) and track.folder.startswith(path): - # count_dict["tracks"][path] += 1 - - # # counting subfolders - # p = get_first_child_from_path(path, track.folder) - - # if p: - # count_dict["folders"][path].add(p) - - # folders = [ - # { - # "path": path, - # "trackcount": count_dict["tracks"][path], - # "foldercount": len(count_dict["folders"][path]), - # } - # for path in paths - # ] - - # return [ - # create_folder(f["path"], f["trackcount"], f["foldercount"]) - # for f in folders - # if f["trackcount"] > 0 - # ] class GetFilesAndDirs: @@ -143,10 +109,6 @@ class GetFilesAndDirs: tracks = [] if files: tracks = TrackDB.get_tracks_by_filepaths(files) - print("printing files") - print(tracks) - - # tracks = TrackStore.get_tracks_by_filepaths(files) folders = [] if not self.tracks_only: @@ -160,7 +122,7 @@ class GetFilesAndDirs: return { "path": path, - "tracks": tracks, + "tracks": serialize_tracks(tracks), "folders": folders, } diff --git a/app/lib/populate.py b/app/lib/populate.py index 4ed73060..593e0282 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -45,6 +45,8 @@ class Populate: """ def __init__(self, instance_key: str) -> None: + return + global POPULATE_KEY POPULATE_KEY = instance_key @@ -152,15 +154,18 @@ class Populate: @staticmethod def tag_untagged(untagged: set[str], key: str): - for file in tqdm(untagged, desc="Reading files"): - if POPULATE_KEY != key: - log.warning("'Populate.tag_untagged': Populate key changed") - return + pass + # for file in tqdm(untagged, desc="Reading files"): + # if POPULATE_KEY != key: + # log.warning("'Populate.tag_untagged': Populate key changed") + # return - tags = get_tags(file) + # tags = get_tags(file) - if tags is not None: - TrackTable.insert_one(tags) + # if tags is not None: + # TrackTable.insert_one(tags) + + # ============================================= # log.info("Found %s new tracks", len(untagged)) # # tagged_tracks: deque[dict] = deque() diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 9d783b62..b9ee9d04 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -30,6 +30,7 @@ class IndexTracks: if tags is not None: TrackTable.insert_one(tags) + del tags class IndexAlbums: def __init__(self) -> None: @@ -84,6 +85,7 @@ class IndexAlbums: pprint(albums) AlbumTable.insert_many(list(albums.values())) + del albums class IndexArtists: @@ -146,10 +148,11 @@ class IndexArtists: pprint(artists) ArtistTable.insert_many(list(artists.values())) + del artists class IndexEverything: def __init__(self) -> None: - # IndexTracks() - # IndexAlbums() - # IndexArtists() + IndexTracks() + IndexAlbums() + IndexArtists() pass diff --git a/app/setup/__init__.py b/app/setup/__init__.py index 36e7e356..c3d37f61 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -23,8 +23,8 @@ def run_setup(): config = UserConfig() config.setup_config_file() - if not config.userId: - config.userId = str(uuid.uuid4()) + if not config.serverId: + config.serverId = str(uuid.uuid4()) setup_sqlite() run_migrations() diff --git a/app/utils/auth.py b/app/utils/auth.py index a19656a3..983cd83f 100644 --- a/app/utils/auth.py +++ b/app/utils/auth.py @@ -16,7 +16,7 @@ def hash_password(password: str) -> str: """ return hashlib.pbkdf2_hmac( - "sha256", password.encode("utf-8"), UserConfig().userId.encode("utf-8"), 100000 + "sha256", password.encode("utf-8"), UserConfig().serverId.encode("utf-8"), 100000 ).hex() diff --git a/manage.py b/manage.py index 62ddf2e1..20eda458 100644 --- a/manage.py +++ b/manage.py @@ -45,8 +45,12 @@ mimetypes.add_type("image/gif", ".gif") mimetypes.add_type("font/woff", ".woff") mimetypes.add_type("application/manifest+json", ".webmanifest") -werkzeug = logging.getLogger("werkzeug") -werkzeug.setLevel(logging.ERROR) +logging.disable(logging.CRITICAL) +# werkzeug = logging.getLogger("werkzeug") +# werkzeug.setLevel(logging.ERROR) + +# # logging.basicConfig() +# logging.getLogger("sqlalchemy.engine").setLevel(logging.ERROR) # Background tasks diff --git a/poetry.lock b/poetry.lock index 2444e038..914b502c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1120,6 +1120,20 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "memory-profiler" +version = "0.61.0" +description = "A module for monitoring memory usage of a python program" +optional = false +python-versions = ">=3.5" +files = [ + {file = "memory_profiler-0.61.0-py3-none-any.whl", hash = "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84"}, + {file = "memory_profiler-0.61.0.tar.gz", hash = "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0"}, +] + +[package.dependencies] +psutil = "*" + [[package]] name = "msgpack" version = "1.0.7" @@ -2602,4 +2616,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "333baa055ac4a32ed914fb46025a48559575806dafba7db5aac97a3878ade23c" +content-hash = "9c7ba20671a6a3b59dbb120e3e56ded7e4dfcbf2de14418bdef41059233cdcb1" diff --git a/pyproject.toml b/pyproject.toml index d521a0a7..de3dece2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ pendulum = "^3.0.0" flask-openapi3 = "^3.0.2" flask-jwt-extended = "^4.6.0" sqlalchemy = "^2.0.31" +memory-profiler = "^0.61.0" [tool.poetry.dev-dependencies] pylint = "^2.15.5" From 4a9f804e706ad046599eba5311945e9098a886db Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 30 Jun 2024 15:06:33 +0300 Subject: [PATCH 05/44] combine userdata and swing db into one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit + port populate to new db interface + add genrehashes and hash info to tracks + properly structure new db table files + move helpers to dedicated utils file + move settings from db to config file + move artists, albums, auth and favorites endpoint to new db interface + use folder store to index filepaths + paginate favorite pages + 56 moretiny changes 😅 --- .github/changelog.md | 5 + TODO.md | 35 ++- app/api/__init__.py | 8 +- app/api/album.py | 67 ++--- app/api/artist.py | 103 +------ app/api/auth.py | 96 +++--- app/api/favorites.py | 170 +++++------ app/api/folder.py | 16 +- app/api/getall/__init__.py | 9 +- app/api/settings.py | 122 ++++---- app/api/stream.py | 2 +- app/arg_handler.py | 7 +- app/config.py | 10 +- app/db/__init__.py | 381 ++---------------------- app/db/libdata.py | 312 +++++++++++++++++++ app/db/metadata.py | 33 ++ app/db/sqlite/lastfm/similar_artists.py | 2 +- app/db/sqlite/plugins/__init__.py | 10 - app/db/sqlite/queries.py | 7 - app/db/sqlite/settings.py | 217 +++++++------- app/db/userdata.py | 240 +++++++++++++++ app/db/utils.py | 75 +++++ app/lib/albumslib.py | 54 +--- app/lib/artistlib.py | 119 +------- app/lib/colorlib.py | 66 ++-- app/lib/folderslib.py | 6 +- app/lib/populate.py | 276 ++++++++--------- app/lib/tagger.py | 196 ++++++++++-- app/lib/taglib.py | 66 ++-- app/lib/trackslib.py | 10 - app/lib/watchdogg.py | 10 +- app/migrations/__init__.py | 35 ++- app/models/album.py | 11 +- app/models/artist.py | 7 + app/models/favorite.py | 11 + app/models/lastfm.py | 19 +- app/models/plugins.py | 2 +- app/models/track.py | 31 +- app/models/user.py | 14 +- app/periodic_scan.py | 12 +- app/plugins/register.py | 17 +- app/requests/artists.py | 17 +- app/settings.py | 11 - app/setup/__init__.py | 21 +- app/setup/sqlite.py | 22 +- app/store/artists.py | 1 - app/store/folder.py | 95 ++++++ app/utils/dates.py | 2 +- app/utils/hashing.py | 1 + app/utils/parsers.py | 8 +- manage.py | 2 +- poetry.lock | 2 +- pyproject.toml | 1 + 53 files changed, 1719 insertions(+), 1353 deletions(-) create mode 100644 app/db/libdata.py create mode 100644 app/db/metadata.py create mode 100644 app/db/userdata.py create mode 100644 app/db/utils.py create mode 100644 app/models/favorite.py create mode 100644 app/store/folder.py diff --git a/.github/changelog.md b/.github/changelog.md index ec423361..d2993bf1 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -25,3 +25,8 @@ interface Genre { } ``` +- Pairing via QR Code has been split into 2 endpoint: + 1. `/getpaircode` + 2. `/pair` + +- \ No newline at end of file diff --git a/TODO.md b/TODO.md index 76547381..e1ed9eee 100644 --- a/TODO.md +++ b/TODO.md @@ -1,11 +1,13 @@ # TODO + - Migrations: - 1. Move userdata to new hashing algorithm - - favorites ✅ - - playlists - - scrobble - - images - - remove image colors + + 1. Move userdata to new hashing algorithm + - favorites ✅ + - playlists + - scrobble + - images + - remove image colors - Package jsoni and publish on PyPi - Rewrite stores to use dictionaries instead of list pools @@ -16,15 +18,30 @@ - Recreate album hash if featured artists are discover - Implement checking if is clean install and skip migrations! - # DONE + - Support auth headers - Add recently played playlist - Move user track logs to user zero - Move future logs to appropriate user id - Store (and read) from the correct user account: - 1. Playlists - 2. Favorites \ No newline at end of file + 1. Playlists + 2. Favorites + +# THE BIG ONE + +- Updating settings +- Cleaning out commented code +- Watchdog +- Periodic scans +- Remove legacy db methods +- Remove all stores +- Review: We don't need server side image colors +- Clean up main db and userdata modules +- Move plugins to a config file +- What about our migrations? +- Add userid to queries +- Remove duplicates on artist page (test with Hanson) \ No newline at end of file diff --git a/app/api/__init__.py b/app/api/__init__.py index f6eab130..b9eacf49 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -11,9 +11,9 @@ from flask_openapi3 import OpenAPI from flask_jwt_extended import JWTManager from app.config import UserConfig +from app.db.userdata import UserTable from app.settings import Info as AppInfo from .plugins import lyrics as lyrics_plugin -from app.db.sqlite.auth import SQLiteAuthMethods as authdb from app.api import ( album, artist, @@ -92,8 +92,10 @@ def create_api(): def user_lookup_callback(_jwt_header, jwt_data): identity = jwt_data["sub"] userid = identity["id"] - user = authdb.get_user_by_id(userid) - return user.todict() + user = UserTable.get_by_id(userid) + + if user: + return user.todict() # Register all the API blueprints with app.app_context(): diff --git a/app/api/album.py b/app/api/album.py index b915612c..6deb7f7c 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -12,15 +12,14 @@ from flask_openapi3 import APIBlueprint from app.api.apischemas import AlbumHashSchema, AlbumLimitSchema, ArtistHashSchema from app.config import UserConfig -from app.db import AlbumTable as AlbumDb, TrackTable as TrackDb +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable as AlbumDb, TrackTable as TrackDb +from app.db.userdata import SimilarArtistTable from app.settings import Defaults -from app.models import FavType, Track -from app.store.albums import AlbumStore -from app.store.tracks import TrackStore from app.utils.hashing import create_hash from app.lib.albumslib import sort_by_track_no from app.serializers.album import serialize_for_card, serialize_for_card_many -from app.serializers.track import serialize_track +from app.serializers.track import serialize_track, serialize_tracks from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb @@ -52,9 +51,22 @@ def get_album_tracks_and_info(body: AlbumHashSchema): album.type = album.check_type( tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles ) - album.populate_versions() - return {"info": album, "tracks": tracks} + track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks}) + + return { + "info": album, + "extra": { + # INFO: track_total is the sum of a set of track_total values from each track + # ASSUMPTIONS + # 1. All the tracks have the correct track totals + # 2. Tracks with the same track total are from the same disc + "track_total": track_total, + "avg_bitrate": sum(t.bitrate for t in tracks) // len(tracks), + }, + "copyright": tracks[0].copyright, + "tracks": serialize_tracks(tracks, remove_disc=False), + } @api.get("//tracks") @@ -68,7 +80,7 @@ def get_album_tracks(path: AlbumHashSchema): tracks = TrackDb.get_tracks_by_albumhash(path.albumhash) tracks = sort_by_track_no(tracks) - return tracks + return serialize_tracks(tracks) class GetMoreFromArtistsBody(AlbumLimitSchema): @@ -138,30 +150,14 @@ def get_album_versions(body: GetAlbumVersionsBody): artisthash = body.artisthash albums = AlbumDb.get_albums_by_base_title(base_title) - print(albums) albums = [ a for a in albums if a.og_title != og_album_title and artisthash in {a["artisthash"] for a in a.albumartists} ] - print(albums) - - # albums = AlbumStore.get_albums_by_artisthash(artisthash) - - # albums = [ - # a - # for a in albums - # if create_hash(a.base_title) == create_hash(base_title) - # and create_hash(og_album_title) != create_hash(a.og_title) - # ] - - # for a in albums: - # tracks = TrackStore.get_tracks_by_albumhash(a.albumhash) - # a.get_date_from_tracks(tracks) - - return albums + return serialize_for_card_many(albums) class GetSimilarAlbumsQuery(ArtistHashSchema, AlbumLimitSchema): @@ -178,24 +174,15 @@ def get_similar_albums(query: GetSimilarAlbumsQuery): artisthash = query.artisthash limit = query.limit - similar_artists = lastfmdb.get_similar_artists_for(artisthash) + similar_artists = SimilarArtistTable.get_by_hash(artisthash) if similar_artists is None: - return {"albums": []} + return [] artisthashes = similar_artists.get_artist_hash_set() + artists = ArtistTable.get_artists_by_artisthashes(artisthashes) - if len(artisthashes) == 0: - return {"albums": []} + albums = AlbumDb.get_albums_by_artisthashes([a.artisthash for a in artists]) + sample = random.sample(albums, min(len(albums), limit)) - albums = [AlbumStore.get_albums_by_artisthash(a) for a in artisthashes] - - albums = [a for sublist in albums for a in sublist] - albums = list({a.albumhash: a for a in albums}.values()) - - try: - albums = random.sample(albums, min(len(albums), limit)) - except ValueError: - pass - - return [serialize_for_card(a) for a in albums[:limit]] + return serialize_for_card_many(sample[:limit]) diff --git a/app/api/artist.py b/app/api/artist.py index 03a01874..65bc6798 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -2,12 +2,11 @@ Contains all the artist(s) routes. """ -from itertools import groupby import math import random from datetime import datetime +from itertools import groupby -from flask_jwt_extended import current_user from flask_openapi3 import APIBlueprint, Tag from pydantic import Field from app.api.apischemas import ( @@ -18,18 +17,13 @@ from app.api.apischemas import ( ) from app.config import UserConfig -from app.db import AlbumTable, ArtistTable, TrackTable -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as fmdb -from app.models import Album, FavType +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable +from app.db.userdata import SimilarArtistTable + from app.serializers.album import serialize_for_card_many from app.serializers.track import serialize_tracks -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore -from app.store.tracks import TrackStore - - bp_tag = Tag(name="Artist", description="Single artist") api = APIBlueprint("artist", __name__, url_prefix="/artist", abp_tags=[bp_tag]) @@ -45,8 +39,6 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): limit = query.limit artist = ArtistTable.get_artist_by_hash(artisthash) - print(artist) - if artist is None: return {"error": "Artist not found"}, 404 @@ -56,8 +48,6 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): if artist.albumcount == 0 and tcount < 10: limit = tcount - # artist.is_favorite = favdb.check_is_favorite(artisthash, FavType.artist) - try: year = datetime.fromtimestamp(artist.date).year except ValueError: @@ -106,7 +96,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): t.albumhash for t in tracks if t.albumhash not in {a.albumhash for a in albums} } - albums.extend(AlbumTable.get_albums_by_hash(missing_albumhashes)) + albums.extend(AlbumTable.get_albums_by_albumhashes(missing_albumhashes)) albumdict = {a.albumhash: a for a in albums} config = UserConfig() @@ -117,43 +107,6 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): if album: album.check_type(list(tracks), config.showAlbumsAsSingles) - # all_albums = AlbumStore.get_albums_by_artisthash(artisthash) - # start: check for missing albums. ie. compilations and features - # all_tracks = TrackStore.get_tracks_by_artisthash(artisthash) - - # track_albums = set(t.albumhash for t in all_tracks) - # missing_album_hashes = track_albums.difference(set(a.albumhash for a in all_albums)) - - # if len(missing_album_hashes) > 0: - # missing_albums = AlbumStore.get_albums_by_hashes(list(missing_album_hashes)) - # all_albums.extend(missing_albums) - - # end check - - # def get_album_tracks(albumhash: str): - # tracks = [t for t in all_tracks if t.albumhash == albumhash] - - # if len(tracks) > 0: - # return tracks - - # return TrackStore.get_tracks_by_albumhash(albumhash) - - # for a in all_albums: - # a.check_type() - - # album_tracks = get_album_tracks(a.albumhash) - - # if len(album_tracks) == 0: - # continue - - # a.get_date_from_tracks(album_tracks) - - # if a.date == 0: - # AlbumStore.remove_album_by_hash(a.albumhash) - # continue - - # a.is_single(album_tracks) - albums = [a for a in albumdict.values()] all_albums = sorted(albums, key=lambda a: str(a.date), reverse=True) @@ -174,29 +127,6 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): else: res["albums"].append(album) - # def remove_EPs_and_singles(albums_: list[Album]): - # albums_ = [a for a in albums_ if not a.type == "single"] - # albums_ = [a for a in albums_ if not a.type == "ep"] - # return albums_ - - # albums = filter(lambda a: artisthash in missing_albumhashes, all_albums) - # albums = list(albums) - # albums = remove_EPs_and_singles(albums) - - # compilations = [a for a in albums if a.is_compilation] - # for c in compilations: - # albums.remove(c) - - # appearances = filter(lambda a: artisthash not in a.albumartists_hashes, all_albums) - # appearances = list(appearances) - - # appearances = remove_EPs_and_singles(appearances) - - # artist = ArtistStore.get_artist_by_hash(artisthash) - - # if artist is None: - # return {"error": "Artist not found"}, 404 - if return_all: limit = len(all_albums) @@ -215,8 +145,8 @@ def get_all_artist_tracks(path: ArtistHashSchema): Returns all artists by a given artist. """ - tracks = TrackStore.get_tracks_by_artisthash(path.artisthash) - + # tracks = TrackStore.get_tracks_by_artisthash(path.artisthash) + tracks = TrackTable.get_tracks_by_artisthash(path.artisthash) return serialize_tracks(tracks) @@ -226,23 +156,14 @@ def get_similar_artists(path: ArtistHashSchema, query: ArtistLimitSchema): Get similar artists. """ limit = query.limit - - artist = ArtistStore.get_artist_by_hash(path.artisthash) - - if artist is None: - return {"error": "Artist not found"}, 404 - - result = fmdb.get_similar_artists_for(artist.artisthash) + result = SimilarArtistTable.get_by_hash(path.artisthash) if result is None: - return {"artists": []} + return [] - similar = ArtistStore.get_artists_by_hashes(result.get_artist_hash_set()) + similar = ArtistTable.get_artists_by_artisthashes(result.get_artist_hash_set()) if len(similar) > limit: - similar = random.sample(similar, limit) + similar = random.sample(similar, min(limit, len(similar))) return similar[:limit] - - -# TODO: Rewrite this file using generators where possible diff --git a/app/api/auth.py b/app/api/auth.py index 4f36250d..0cf41bf1 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -14,7 +14,8 @@ from pydantic import BaseModel, Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint -from app.db.sqlite.auth import SQLiteAuthMethods as authdb +# from app.db.sqlite.auth import SQLiteAuthMethods as authdb +from app.db.userdata import UserTable from app.utils.auth import check_password, hash_password from app.config import UserConfig @@ -65,7 +66,7 @@ def login(body: LoginBody): Authenticate using username and password """ - user = authdb.get_user_by_username(body.username) + user = UserTable.get_by_username(body.username) if user is None: return {"msg": "User not found"}, 404 @@ -87,42 +88,41 @@ def login(body: LoginBody): pair_token = dict() +@api.get("/getpaircode") +def get_pair(): + """ + Get a new pair code to log in to thee Swing Music mobile app + """ + # INFO: if user is already logged in, create a new pair code + token = create_new_token(get_jwt_identity()) + key = token["accesstoken"][-6:] + + global pair_token + pair_token = { + key: token, + } + + return {"code": key} + + class PairDeviceQuery(BaseModel): code: str = Field("", description="The code") -@api.get("/pair") +@api.post("/pair") @jwt_required(optional=True) -def pair_device(query: PairDeviceQuery): +def pair_with_code(body: PairDeviceQuery): """ - Pair the Swing Music mobile app with this server - - Send a code to get an access token. Send an authenticated request without the code to generate a new token. + Get an access token by sending a pair code. NOTE: A code can only be used once! """ - # INFO: if user is already logged in, create a new pair code - if current_user: - token = create_new_token(get_jwt_identity()) - key = token["accesstoken"][-6:] + global pair_token + token = pair_token.get(body.code, None) - global pair_token - pair_token = { - key: token, - } + if token: + pair_token = {} + return token - return {"code": key} - - # INFO: if there's a pair code, return the token - if query.code: - token = pair_token.get(query.code, None) - - if token: - # INFO: reset pair_token - pair_token = {} - return token - - return {"msg": "Invalid code"}, 400 - - return {"msg": "No code provided"}, 400 + return {"msg": "Invalid code"}, 400 @api.post("/refresh") @@ -133,6 +133,8 @@ def refresh(): >>> Headers: >>> Authorization: Bearer + + Won't work with cookies!!! """ user = get_jwt_identity() return create_new_token(user) @@ -153,7 +155,6 @@ def update_profile(body: UpdateProfileBody): """ user = { "id": body.id, - "email": body.email, "username": body.username, "password": body.password, "roles": body.roles, @@ -172,7 +173,8 @@ def update_profile(body: UpdateProfileBody): if "admin" not in current_user["roles"]: return {"msg": "Only admins can update roles"}, 403 - all_users = authdb.get_all_users() + # all_users = authdb.get_all_users() + all_users = UserTable.get_all() if "admin" not in body.roles: # check if we're removing the last admin admins = [user for user in all_users if "admin" in user.roles] @@ -195,7 +197,9 @@ def update_profile(body: UpdateProfileBody): clean_user = {k: v for k, v in user.items() if v} try: - return authdb.update_user(clean_user) + # return authdb.update_user(clean_user) + UserTable.update_one(clean_user) + return UserTable.get_by_id(user["id"]).todict() except sqlite3.IntegrityError: return {"msg": "Username already exists"}, 400 @@ -216,11 +220,18 @@ def create_user(body: UpdateProfileBody): } # check if user already exists - if authdb.get_user_by_username(user["username"]): + if UserTable.get_by_username(user["username"]): return {"msg": "Username already exists"}, 400 - userid = authdb.insert_user(user) - return authdb.get_user_by_id(userid).todict() + UserTable.insert_one(user) + user = UserTable.get_by_username(user["username"]) + + if user: + return user.todict() + + return { + "msg": "Failed to create user", + }, 500 @api.post("/profile/guest/create") @@ -230,14 +241,14 @@ def create_guest_user(): Create a guest user """ # check if guest user already exists - guest_user = authdb.get_user_by_username("guest") + guest_user = UserTable.get_by_username("guest") if guest_user: return { "msg": "Guest user already exists", }, 400 - userid = authdb.insert_guest_user() + userid = UserTable.insert_guest_user() if userid: return { @@ -264,12 +275,12 @@ def delete_user(body: DeleteUseBody): return {"msg": "Sorry! you cannot delete yourselfu"}, 400 # prevent deleting the only admin - users = authdb.get_all_users() + users = UserTable.get_all() admins = [user for user in users if "admin" in user.roles] if len(admins) == 1 and admins[0].username == body.username: return {"msg": "Cannot delete the only admin"}, 400 - authdb.delete_user_by_username(body.username) + UserTable.remove_by_username(body.username) return {"msg": f"User {body.username} deleted"} @@ -308,8 +319,7 @@ def get_all_users(query: GetAllUsersQuery): "users": [], } - users = authdb.get_all_users() - + users = UserTable.get_all() is_admin = current_user and "admin" in current_user["roles"] settings["enableGuest"] = [ user for user in users if user.username == "guest" @@ -355,8 +365,8 @@ def get_all_users(query: GetAllUsersQuery): if query.simplified: res["users"] = [user.todict_simplified() for user in users] - - res["users"] = [user.todict() for user in users] + else: + res["users"] = [user.todict() for user in users] return res diff --git a/app/api/favorites.py b/app/api/favorites.py index 94fa57b7..c6127a5c 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -6,18 +6,19 @@ from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from app.api.apischemas import GenericLimitSchema +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable +from app.db.userdata import FavoritesTable from app.models import FavType from app.settings import Defaults from app.utils.bisection import use_bisection -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.serializers.track import serialize_track, serialize_tracks -from app.serializers.artist import serialize_for_card as serialize_artist, serialize_for_cards -from app.serializers.album import serialize_for_card, serialize_for_card_many - -from app.store.albums import AlbumStore -from app.store.tracks import TrackStore -from app.store.artists import ArtistStore +from app.serializers.artist import ( + serialize_for_card as serialize_artist, + serialize_for_cards, +) from app.utils.dates import timestamp_to_time_passed +from app.serializers.album import serialize_for_card, serialize_for_card_many bp_tag = Tag(name="Favorites", description="Your favorite items") api = APIBlueprint("favorites", __name__, url_prefix="/favorites", abp_tags=[bp_tag]) @@ -41,17 +42,18 @@ class FavoritesAddBody(BaseModel): @api.post("/add") -def add_favorite(body: FavoritesAddBody): +def toggle_favorite(body: FavoritesAddBody): """ Adds a favorite to the database. """ - itemhash = body.hash - itemtype = body.type + FavoritesTable.insert_item({"hash": body.hash, "type": body.type}) - favdb.insert_one_favorite(itemtype, itemhash) - - if itemtype == FavType.track: - TrackStore.make_track_fav(itemhash) + if body.type == FavType.track: + TrackTable.set_is_favorite(body.hash, True) + elif body.type == FavType.album: + AlbumTable.set_is_favorite(body.hash, True) + elif body.type == FavType.artist: + ArtistTable.set_is_favorite(body.hash, True) return {"msg": "Added to favorites"} @@ -61,80 +63,62 @@ def remove_favorite(body: FavoritesAddBody): """ Removes a favorite from the database. """ - itemhash = body.hash - itemtype = body.type + FavoritesTable.remove_item({"hash": body.hash, "type": body.type}) - favdb.delete_favorite(itemtype, itemhash) - - if itemtype == FavType.track: - TrackStore.remove_track_from_fav(itemhash) + if body.type == FavType.track: + TrackTable.set_is_favorite(body.hash, False) + elif body.type == FavType.album: + AlbumTable.set_is_favorite(body.hash, False) + elif body.type == FavType.artist: + ArtistTable.set_is_favorite(body.hash, False) return {"msg": "Removed from favorites"} +class GetAllOfTypeQuery(GenericLimitSchema): + """ + Extending this class will give you a model with the `limit` field + """ + + start: int = Field( + description="Where to start from", + example=Defaults.API_CARD_LIMIT, + default=Defaults.API_CARD_LIMIT, + ) + + @api.get("/albums") -def get_favorite_albums(query: GenericLimitSchema): +def get_favorite_albums(query: GetAllOfTypeQuery): """ Get favorite albums """ - limit = query.limit - albums = favdb.get_fav_albums() - albumhashes = [a[1] for a in albums] - albumhashes.reverse() + fav_albums, total = FavoritesTable.get_fav_albums(query.start, query.limit) + fav_albums.reverse() - src_albums = sorted(AlbumStore.albums, key=lambda x: x.albumhash) - - fav_albums = use_bisection(src_albums, "albumhash", albumhashes) - fav_albums = remove_none(fav_albums) - - if limit == 0: - limit = len(albums) - - return {"albums": serialize_for_card_many(fav_albums[:limit])} + return {"albums": serialize_for_card_many(fav_albums), "total": total} @api.get("/tracks") -def get_favorite_tracks(query: GenericLimitSchema): +def get_favorite_tracks(query: GetAllOfTypeQuery): """ Get favorite tracks """ - limit = query.limit - userid = current_user['id'] - - tracks = favdb.get_fav_tracks(userid) - trackhashes = [t[1] for t in tracks] - trackhashes.reverse() - src_tracks = sorted(TrackStore.tracks, key=lambda x: x.trackhash) - - tracks = use_bisection(src_tracks, "trackhash", trackhashes) - tracks = remove_none(tracks) - - if limit == 0: - limit = len(tracks) - - return {"tracks": serialize_tracks(tracks[:limit])} + tracks, total = FavoritesTable.get_fav_tracks(query.start, query.limit) + return {"tracks": serialize_tracks(tracks), "total": total} @api.get("/artists") -def get_favorite_artists(query: GenericLimitSchema): +def get_favorite_artists(query: GetAllOfTypeQuery): """ Get favorite artists """ - limit = query.limit + artists, total = FavoritesTable.get_fav_artists( + start=query.start, + limit=query.limit, + ) + artists.reverse() - artists = favdb.get_fav_artists() - artisthashes = [a[1] for a in artists] - artisthashes.reverse() - - src_artists = sorted(ArtistStore.artists, key=lambda x: x.artisthash) - - artists = use_bisection(src_artists, "artisthash", artisthashes) - artists = remove_none(artists) - - if limit == 0: - limit = len(artists) - - return {"artists": artists[:limit]} + return {"artists": [serialize_artist(a) for a in artists], "total": total} class GetAllFavoritesQuery(BaseModel): @@ -173,27 +157,29 @@ def get_all_favorites(query: GetAllFavoritesQuery): # largest is x2 to accound for broken hashes if any largest = max(track_limit, album_limit, artist_limit) - favs = favdb.get_all() + favs = FavoritesTable.get_all() favs.reverse() tracks = [] albums = [] artists = [] - track_master_hash = set(t.trackhash for t in TrackStore.tracks) - album_master_hash = set(a.albumhash for a in AlbumStore.albums) - artist_master_hash = set(a.artisthash for a in ArtistStore.artists) + track_master_hash = TrackTable.get_all_hashes() + album_master_hash = AlbumTable.get_all_hashes() + artist_master_hash = ArtistTable.get_all_hashes() + # INFO: Filter out invalid hashes (file not found or tags edited) for fav in favs: - # INFO: hash is [1], type is [2], timestamp is [3] - hash = fav[1] - if fav[2] == FavType.track: + hash = fav.hash + type = fav.type + + if type == FavType.track: tracks.append(hash) if hash in track_master_hash else None - if fav[2] == FavType.artist: + if type == FavType.artist: artists.append(hash) if hash in artist_master_hash else None - if fav[2] == FavType.album: + if type == FavType.album: albums.append(hash) if hash in album_master_hash else None count = { @@ -202,35 +188,26 @@ def get_all_favorites(query: GetAllFavoritesQuery): "artists": len(artists), } - src_tracks = sorted(TrackStore.tracks, key=lambda x: x.trackhash) - src_albums = sorted(AlbumStore.albums, key=lambda x: x.albumhash) - src_artists = sorted(ArtistStore.artists, key=lambda x: x.artisthash) - - tracks = use_bisection(src_tracks, "trackhash", tracks, limit=track_limit) - albums = use_bisection(src_albums, "albumhash", albums, limit=album_limit) - artists = use_bisection(src_artists, "artisthash", artists, limit=artist_limit) - - tracks = remove_none(tracks) - albums = remove_none(albums) - artists = remove_none(artists) + tracks = TrackTable.get_tracks_by_trackhashes(tracks, limit=track_limit) + albums = AlbumTable.get_albums_by_albumhashes(albums, limit=album_limit) + artists = ArtistTable.get_artists_by_artisthashes(artists, limit=artist_limit) recents = [] # first_n = favs for fav in favs: - # INFO: hash is [1], type is [2], timestamp is [3] if len(recents) >= largest: break - if fav[2] == FavType.album: - album = next((a for a in albums if a.albumhash == fav[1]), None) + if fav.type == FavType.album: + album = next((a for a in albums if a.albumhash == fav.hash), None) if album is None: continue album = serialize_for_card(album) album["help_text"] = "album" - album["time"] = timestamp_to_time_passed(fav[3]) + album["time"] = timestamp_to_time_passed(fav.timestamp) recents.append( { @@ -239,15 +216,15 @@ def get_all_favorites(query: GetAllFavoritesQuery): } ) - if fav[2] == FavType.artist: - artist = next((a for a in artists if a.artisthash == fav[1]), None) + if fav.type == FavType.artist: + artist = next((a for a in artists if a.artisthash == fav.hash), None) if artist is None: continue artist = serialize_artist(artist) artist["help_text"] = "artist" - artist["time"] = timestamp_to_time_passed(fav[3]) + artist["time"] = timestamp_to_time_passed(fav.timestamp) recents.append( { @@ -256,15 +233,15 @@ def get_all_favorites(query: GetAllFavoritesQuery): } ) - if fav[2] == FavType.track: - track = next((t for t in tracks if t.trackhash == fav[1]), None) + if fav.type == FavType.track: + track = next((t for t in tracks if t.trackhash == fav.hash), None) if track is None: continue track = serialize_track(track) track["help_text"] = "track" - track["time"] = timestamp_to_time_passed(fav[3]) + track["time"] = timestamp_to_time_passed(fav.timestamp) recents.append({"type": "track", "item": track}) @@ -284,6 +261,5 @@ def check_favorite(query: FavoritesAddBody): """ itemhash = query.hash itemtype = query.type - exists = favdb.check_is_favorite(itemhash, itemtype) - return {"is_favorite": exists} + return {"is_favorite": FavoritesTable.check_exists(itemhash, itemtype)} diff --git a/app/api/folder.py b/app/api/folder.py index c2d1cc85..098742af 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -10,11 +10,10 @@ from pydantic import BaseModel, Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from showinfm import show_in_file_manager -from memory_profiler import profile from app import settings -from app.db import TrackTable -from app.db.sqlite.settings import SettingsSQLMethods as db +from app.config import UserConfig +from app.db.libdata import TrackTable from app.lib.folderslib import GetFilesAndDirs, get_folders from app.serializers.track import serialize_track from app.utils.wintools import is_windows, win_replace_slash @@ -40,8 +39,9 @@ def get_folder_tree(body: FolderTree): req_dir = body.folder tracks_only = body.tracks_only - root_dirs = db.get_root_dirs() - root_dirs.sort() + + config = UserConfig() + root_dirs = config.rootDirs try: if req_dir == "$home" and root_dirs[0] == "$home": @@ -72,12 +72,6 @@ def get_folder_tree(body: FolderTree): return res - # return { - # "path": req_dir, - # "tracks": tracks, - # "folders": sorted(folders, key=lambda i: i.name), - # } - def get_all_drives(is_win: bool = False): """ diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index 31c72b27..f1fe2a35 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -6,7 +6,8 @@ from pydantic import BaseModel, Field from datetime import datetime from app.api.apischemas import GenericLimitSchema -from app.db import AlbumTable, ArtistTable +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable from app.store.albums import AlbumStore from app.store.artists import ArtistStore @@ -61,11 +62,11 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): is_artists = path.itemtype == "artists" if is_albums: - items, total = AlbumTable.get_all(query.start, query.limit) + items = AlbumTable.get_all() elif is_artists: - items, total = ArtistTable.get_all(query.start, query.limit) + items = ArtistTable.get_all() - # print(items) + total = len(items) start = query.start limit = query.limit diff --git a/app/api/settings.py b/app/api/settings.py index 4931bc52..3f1a3047 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -1,3 +1,4 @@ +from dataclasses import asdict from typing import Any from flask import request from flask_openapi3 import Tag @@ -6,12 +7,13 @@ from pydantic import BaseModel, Field from app.api.auth import admin_required from app.db.sqlite.plugins import PluginsMethods as pdb -from app.db.sqlite.settings import SettingsSQLMethods as sdb from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb +from app.db.userdata import PluginTable from app.lib import populate +from app.lib.tagger import index_everything from app.lib.watchdogg import Watcher as WatchDog from app.logger import log -from app.settings import Info, Paths, SessionVarKeys, set_flag +from app.settings import Info, Paths, SessionVarKeys from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore @@ -48,42 +50,43 @@ def reload_everything(instance_key: str): except Exception as e: log.error(e) +# CHECKPOINT: TEST SETTINGS API ENDPOINTS -@background -def rebuild_store(db_dirs: list[str]): - """ - Restarts watchdog and rebuilds the music library. - """ - instance_key = get_random_str() +# @background +# def rebuild_store(db_dirs: list[str]): +# """ +# Restarts watchdog and rebuilds the music library. +# """ +# instance_key = get_random_str() - log.info("Rebuilding library...") - trackdb.remove_tracks_not_in_folders(db_dirs) - reload_everything(instance_key) +# log.info("Rebuilding library...") +# trackdb.remove_tracks_not_in_folders(db_dirs) +# reload_everything(instance_key) - try: - populate.Populate(instance_key=instance_key) - except populate.PopulateCancelledError as e: - print(e) - reload_everything(instance_key) - return +# try: +# populate.Populate(instance_key=instance_key) +# except populate.PopulateCancelledError as e: +# print(e) +# reload_everything(instance_key) +# return - WatchDog().restart() +# WatchDog().restart() - log.info("Rebuilding library... ✅") +# log.info("Rebuilding library... ✅") -# I freaking don't know what this function does anymore -def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]): - """ - Params: - new_: will be added to the database - removed_: will be removed from the database - db_dirs_: will be used to remove tracks that - are outside these directories from the database and store. - """ - sdb.remove_root_dirs(removed_) - sdb.add_root_dirs(new_) - rebuild_store(db_dirs_) +# # I freaking don't know what this function does anymore +# def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]): +# """ +# Params: +# new_: will be added to the database +# removed_: will be removed from the database +# db_dirs_: will be used to remove tracks that +# are outside these directories from the database and store. +# """ +# sdb.remove_root_dirs(removed_) +# sdb.add_root_dirs(new_) +# rebuild_store(db_dirs_) class AddRootDirsBody(BaseModel): @@ -106,7 +109,8 @@ def add_root_dirs(body: AddRootDirsBody): new_dirs = body.new_dirs removed_dirs = body.removed - db_dirs = sdb.get_root_dirs() + config = UserConfig() + db_dirs = config.rootDirs home = "$home" db_home = any([d == home for d in db_dirs]) # if $home is in db @@ -114,13 +118,16 @@ def add_root_dirs(body: AddRootDirsBody): # handle $home case if db_home and incoming_home: - return {"msg": "Not changed!"} + return {"msg": "Not changed!"}, 304 + # if $home is the current root dir or the incoming root dir + # is $home, remove all root dirs if db_home or incoming_home: - sdb.remove_root_dirs(db_dirs) + config.rootDirs = [] if incoming_home: - finalize([home], [], [Paths.USER_HOME_DIR]) + config.rootDirs = [home] + index_everything() return {"root_dirs": [home]} # --- @@ -136,11 +143,10 @@ def add_root_dirs(body: AddRootDirsBody): pass db_dirs.extend(new_dirs) - db_dirs = [dir_ for dir_ in db_dirs if dir_ != home] + config.rootDirs = [dir_ for dir_ in db_dirs if dir_ != home] - finalize(new_dirs, removed_dirs, db_dirs) - - return {"root_dirs": db_dirs} + index_everything() + return {"root_dirs": config.rootDirs} @api.get("/get-root-dirs") @@ -148,9 +154,7 @@ def get_root_dirs(): """ Get root directories """ - dirs = sdb.get_root_dirs() - - return {"dirs": dirs} + return {"dirs": UserConfig().rootDirs} # maps settings to their parser flags @@ -170,35 +174,12 @@ def get_all_settings(): """ Get all settings """ + config = asdict(UserConfig()) + plugins = PluginTable.get_all() + config["plugins"] = plugins + config["version"] = Info.SWINGMUSIC_APP_VERSION - settings = sdb.get_all_settings() - plugins = pdb.get_all_plugins() - - key_list = list(mapp.keys()) - s = {} - - for key in key_list: - val_index = key_list.index(key) - - try: - s[key] = settings[val_index] - - if type(s[key]) == int: - s[key] = bool(s[key]) - if type(s[key]) == str: - s[key] = str(s[key]).split(",") - - except IndexError: - s[key] = None - - root_dirs = sdb.get_root_dirs() - s["root_dirs"] = root_dirs - s["plugins"] = plugins - s["version"] = Info.SWINGMUSIC_APP_VERSION - - return { - "settings": s, - } + return config @background @@ -245,7 +226,6 @@ def set_setting(body: SetSettingBody): value = str(value).split(",") value = set(value) - set_flag(flag, value) reload_all_for_set_setting() # if value is a set, convert it to a string diff --git a/app/api/stream.py b/app/api/stream.py index 3eaee2a6..4993539a 100644 --- a/app/api/stream.py +++ b/app/api/stream.py @@ -11,7 +11,7 @@ from app.api.apischemas import TrackHashSchema from app.lib.trackslib import get_silence_paddings # from app.store.tracks import TrackStore -from app.db import TrackTable +from app.db.libdata import TrackTable from app.utils.files import guess_mime_type bp_tag = Tag(name="File", description="Audio files") diff --git a/app/arg_handler.py b/app/arg_handler.py index b74acfe4..249ef062 100644 --- a/app/arg_handler.py +++ b/app/arg_handler.py @@ -9,6 +9,7 @@ import sys import PyInstaller.__main__ as bundler from app import settings +from app.config import UserConfig from app.logger import log from app.print_help import HELP_MESSAGE from app.utils.auth import hash_password @@ -160,7 +161,7 @@ class ProcessArgs: @staticmethod def handle_periodic_scan(): if any((a in ARGS for a in ALLARGS.no_periodic_scan)): - settings.SessionVars.DO_PERIODIC_SCANS = False + UserConfig().enablePeriodicScans = False @staticmethod def handle_periodic_scan_interval(): @@ -182,10 +183,10 @@ class ProcessArgs: sys.exit(0) if psi < 0: - print("WADAFUCK ARE YOU TRYING?") + print("WHAT ARE YOU TRYING?") sys.exit(0) - settings.SessionVars.PERIODIC_SCAN_INTERVAL = psi + UserConfig().scanInterval = psi @staticmethod def handle_help(): diff --git a/app/config.py b/app/config.py index b751f129..bca9c69e 100644 --- a/app/config.py +++ b/app/config.py @@ -6,6 +6,7 @@ from .settings import Paths # TODO: Publish this on PyPi + @dataclass class UserConfig: _config_path: str = "" @@ -20,7 +21,7 @@ class UserConfig: # lists rootDirs: list[str] = field(default_factory=list) excludeDirs: list[str] = field(default_factory=list) - artistSeparators: set[str] = field(default_factory=set) + artistSeparators: set[str] = field(default_factory=lambda: {";", "/"}) genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) # tracks @@ -33,6 +34,13 @@ class UserConfig: cleanAlbumTitle: bool = True showAlbumsAsSingles: bool = False + # misc + enablePeriodicScans: bool = False + scanInterval: int = 60 * 10 # 10 minutes + + # plugins + enablePlugins: bool = True + def __post_init__(self): """ Loads the config file and sets the values to this instance diff --git a/app/db/__init__.py b/app/db/__init__.py index c7321c80..ebc6d776 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -1,36 +1,21 @@ -from concurrent.futures import ThreadPoolExecutor -import json -import os -from pathlib import Path -from pprint import pprint -from typing import Any, Optional +from typing import Any -from memory_profiler import profile from sqlalchemy import ( - JSON, - Boolean, - Integer, - Row, - String, - Tuple, - and_, create_engine, + delete, insert, select, ) + +from sqlalchemy.engine import Engine +from sqlalchemy import event from sqlalchemy.orm import ( - Mapped, - mapped_column, DeclarativeBase, MappedAsDataclass, - sessionmaker, ) -from app.models import Track as TrackModel -from app.models import Album as AlbumModel -from app.models import Artist as ArtistModel -from app.utils.remove_duplicates import remove_duplicates - +# ============================================================ +# TODO: Make sure the database is created before we run this. fullpath = "/home/cwilvx/temp/swingmusic/swing.db" engine = create_engine( f"sqlite+pysqlite:///{fullpath}", @@ -39,85 +24,46 @@ engine = create_engine( pool_size=5, ) -if not os.path.exists(fullpath): - os.makedirs(Path(fullpath).parent) - -connection = engine.connect() -all_filepaths = list() +# connection = engine.connect() -def getIndexOfFirstMatch(strings: list[str], prefix: str): - """ - Find the index of the first path that starts with the given path. - - Uses a binary search algorithm to find the index. - """ - - left = 0 - right = len(strings) - 1 - - while left <= right: - mid = (left + right) // 2 - - if strings[mid].startswith(prefix): - if mid == 0 or not strings[mid - 1].startswith(prefix): - return mid - right = mid - 1 - elif strings[mid] < prefix: - left = mid + 1 - else: - right = mid - 1 - - return -1 - - -def countFilepathsInDir(dirpath: str): - """ - Return all the filepaths in a directory. - """ - global all_filepaths - index = getIndexOfFirstMatch(all_filepaths, dirpath) - - if index == -1: - return 0 - - paths: list[str] = [] - - for path in all_filepaths[index:]: - if path.startswith(dirpath): - paths.append(path) - else: - break - - return len(paths) +@event.listens_for(Engine, "connect") +def set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() class DbManager: def __init__(self, commit: bool = False): self.commit = commit - # self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) - # self.conn = self.engine.connect() - # pass + self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) + self.conn = self.engine.connect() def __enter__(self): - # return self.conn.execution_options(preserve_rowcount=True) - return connection + return self.conn.execution_options(preserve_rowcount=True) + # return connection def __exit__(self, exc_type, exc_val, exc_tb): if self.commit: - connection.commit() + self.conn.commit() - # self.conn.close() + self.conn.close() class Base(MappedAsDataclass, DeclarativeBase): + @classmethod + def execute(cls, stmt: Any, commit: bool = False): + with DbManager(commit=commit) as conn: + return conn.execute(stmt) + @classmethod def insert_many(cls, items: list[dict[str, Any]]): """ Inserts multiple items into the database. """ with DbManager(commit=True) as conn: - conn.execute(insert(cls).values(items)) + return conn.execute(insert(cls).values(items)) @classmethod def insert_one(cls, item: dict[str, Any]): @@ -127,277 +73,14 @@ class Base(MappedAsDataclass, DeclarativeBase): return cls.insert_many([item]) @classmethod - def get_all(cls): - """ - Returns all the items from the database. - """ - with DbManager() as conn: - result = conn.execute(select(cls)) - return result.fetchall() - - -class ArtistTable(Base): - __tablename__ = "artist" - - id: Mapped[int] = mapped_column(primary_key=True) - albumcount: Mapped[int] = mapped_column(Integer()) - artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True) - created_date: Mapped[int] = mapped_column(Integer()) - date: Mapped[int] = mapped_column(Integer()) - duration: Mapped[int] = mapped_column(Integer()) - genres: Mapped[str] = mapped_column(JSON()) - name: Mapped[str] = mapped_column(String(), index=True) - trackcount: Mapped[int] = mapped_column(Integer()) - is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + def remove_all(cls): + with DbManager(commit=True) as conn: + conn.execute(delete(cls)) @classmethod - def get_all(cls, start: int, limit: int): - with DbManager() as conn: - if start == 0: - result = conn.execute(select(cls)) - else: - result = conn.execute(select(cls).offset(start).limit(limit)) - - all = result.fetchall() - return artists_to_dataclasses(all), len(all) - - @classmethod - def get_artist_by_hash(cls, artisthash: str): - with DbManager() as conn: - result = conn.execute( - select(ArtistTable).where(ArtistTable.artisthash == artisthash) - ) - return artist_to_dataclass(result.fetchone()) + def all(cls): + return cls.execute(select(cls)) -class AlbumTable(Base): - __tablename__ = "album" - - id: Mapped[int] = mapped_column(primary_key=True) - albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) - artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) - albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) - base_title: Mapped[str] = mapped_column(String()) - color: Mapped[Optional[str]] = mapped_column(String()) - created_date: Mapped[int] = mapped_column(Integer()) - date: Mapped[int] = mapped_column(Integer()) - duration: Mapped[int] = mapped_column(Integer()) - genres: Mapped[str] = mapped_column(JSON()) - og_title: Mapped[str] = mapped_column(String()) - title: Mapped[str] = mapped_column(String()) - trackcount: Mapped[int] = mapped_column(Integer()) - - @classmethod - def get_album_by_albumhash(cls, hash: str): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.albumhash == hash) - ) - album = result.fetchone() - - if album: - return album_to_dataclass(album) - - @classmethod - def get_albums_by_hash(cls, hashes: set[str]): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)) - ) - return albums_to_dataclasses(result.fetchall()) - - @classmethod - def get_all(cls, start: int, limit: int): - with DbManager() as conn: - if start == 0: - result = conn.execute(select(AlbumTable)) - else: - result = conn.execute(select(AlbumTable).offset(start).limit(limit)) - - all = result.fetchall() - - return albums_to_dataclasses(all)[:limit], len(all) - - @classmethod - def get_albums_by_artisthashes(cls, artisthashes: list[dict[str, str]]): - with DbManager() as conn: - albums: list[AlbumModel] = [] - - for artist in artisthashes: - result = conn.execute( - # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! - select(AlbumTable).where(AlbumTable.albumartists.contains(artist)) - ) - albums.extend(albums_to_dataclasses(result.fetchall())) - - return albums - - @classmethod - def get_albums_by_base_title(cls, base_title: str): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.base_title == base_title) - ) - return albums_to_dataclasses(result.fetchall()) - - @classmethod - def get_albums_by_artisthash(cls, artisthash: str): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash)) - ) - return albums_to_dataclasses(result.all()) - - -class TrackTable(Base): - __tablename__ = "track" - - id: Mapped[int] = mapped_column(init=False, primary_key=True) - album: Mapped[str] = mapped_column(String()) - albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) - albumhash: Mapped[str] = mapped_column(String(), index=True) - artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) - artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) - bitrate: Mapped[int] = mapped_column(Integer()) - copyright: Mapped[Optional[str]] = mapped_column(String()) - date: Mapped[int] = mapped_column(Integer()) - disc: Mapped[int] = mapped_column(Integer()) - duration: Mapped[int] = mapped_column(Integer()) - filepath: Mapped[str] = mapped_column(String(), index=True, unique=True) - folder: Mapped[str] = mapped_column(String(), index=True) - genre: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) - last_mod: Mapped[float] = mapped_column(Integer()) - og_album: Mapped[str] = mapped_column(String()) - og_title: Mapped[str] = mapped_column(String()) - title: Mapped[str] = mapped_column(String()) - track: Mapped[int] = mapped_column(Integer()) - trackhash: Mapped[str] = mapped_column(String(), index=True) - extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) - - @classmethod - def get_tracks_by_filepaths(cls, filepaths: list[str]): - with DbManager() as conn: - result = conn.execute( - select(TrackTable).where(TrackTable.filepath.in_(filepaths)) - ) - return tracks_to_dataclasses(result.fetchall()) - - @classmethod - def count_tracks_containing_paths(cls, paths: list[str]): - results: list[dict[str, int | str]] = [] - - with ThreadPoolExecutor() as executor: - res = executor.map(countFilepathsInDir, paths) - results = [ - {"path": path, "trackcount": count} for path, count in zip(paths, res) - ] - - return results - - @classmethod - def get_tracks_by_albumhash(cls, albumhash: str): - with DbManager() as conn: - result = conn.execute( - select(TrackTable).where(TrackTable.albumhash == albumhash) - ) - tracks = tracks_to_dataclasses(result.fetchall()) - return remove_duplicates(tracks, is_album_tracks=True) - - @classmethod - def get_track_by_trackhash(cls, hash: str, filepath: str = ""): - with DbManager() as conn: - if filepath: - result = conn.execute( - select(TrackTable) - .where( - and_( - TrackTable.trackhash == hash, - TrackTable.filepath == filepath, - ) - ) - .order_by(TrackTable.bitrate.desc()) - ) - else: - result = conn.execute( - select(TrackTable).where(TrackTable.trackhash == hash) - ) - - track = result.fetchone() - - if track: - return track_to_dataclass(track) - - @classmethod - def get_tracks_by_artisthash(cls, artisthash: str): - with DbManager() as conn: - result = conn.execute( - select(TrackTable).where(TrackTable.artists.contains(artisthash)) - ) - return tracks_to_dataclasses(result.fetchall()) - - @classmethod - def get_tracks_in_path(cls, path: str): - with DbManager() as conn: - result = conn.execute( - select(TrackTable) - .where(TrackTable.filepath.contains(path)) - .order_by(TrackTable.last_mod) - ) - return tracks_to_dataclasses(result.fetchall()) - - -all_tracks = TrackTable.get_all() - -for track in all_tracks: - all_filepaths.append(track.filepath) - -all_filepaths.sort() - -# print("files in path: ",getFilepathsInDir("/home/cwilvx/Music/").__len__()) - - -# SECTION: Userdata database -class UserTable(Base): - __tablename__ = "user" - - id: Mapped[int] = mapped_column(primary_key=True) - username: Mapped[str] = mapped_column(String(), unique=True) - firstname: Mapped[Optional[str]] = mapped_column(String()) - lastname: Mapped[Optional[str]] = mapped_column(String()) - password: Mapped[str] = mapped_column(String()) - email: Mapped[Optional[str]] = mapped_column(String()) - image: Mapped[Optional[str]] = mapped_column(String()) - roles: Mapped[list[str]] = mapped_column(JSON(), default_factory=lambda: ["user"]) - extra: Mapped[Optional[dict[str, Any]]] = mapped_column( - JSON(), default_factory=dict - ) - - -# SECTION: HELPER FUNCTIONS - - -def artist_to_dataclass(artist: Any): - return ArtistModel(**artist._asdict()) - - -def artists_to_dataclasses(artists: Any): - return [artist_to_dataclass(artist) for artist in artists] - - -def album_to_dataclass(album: Any): - return AlbumModel(**album._asdict()) - - -def albums_to_dataclasses(albums: Any): - return [album_to_dataclass(album) for album in albums] - - -def track_to_dataclass(track: Any): - return TrackModel(**track._asdict()) - - -def tracks_to_dataclasses(tracks: Any): - return [track_to_dataclass(track) for track in tracks] - - -Base().metadata.create_all(engine) +def create_all(): + Base().metadata.create_all(engine) diff --git a/app/db/libdata.py b/app/db/libdata.py new file mode 100644 index 00000000..9d3ebbd6 --- /dev/null +++ b/app/db/libdata.py @@ -0,0 +1,312 @@ +from app.db import ( + Base as MasterBase, + DbManager, +) +from app.db.utils import ( + album_to_dataclass, + albums_to_dataclasses, + artist_to_dataclass, + artists_to_dataclasses, + track_to_dataclass, + tracks_to_dataclasses, +) +from app.models import Album as AlbumModel +from app.utils.remove_duplicates import remove_duplicates +from app.db import engine + +from sqlalchemy import JSON, Boolean, Integer, String, and_, delete, select, update +from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase + + +from typing import Any, Iterable, Optional + + +def create_all(): + """ + Create all the tables defined in this file. + """ + Base.metadata.create_all(engine) + + +class Base(MasterBase, DeclarativeBase): + @classmethod + def get_all_hashes(cls): + with DbManager() as conn: + if cls.__tablename__ == "track": + stmt = select(TrackTable.trackhash) + elif cls.__tablename__ == "album": + stmt = select(AlbumTable.albumhash) + elif cls.__tablename__ == "artist": + stmt = select(ArtistTable.artisthash) + + result = conn.execute(stmt) + return {row[0] for row in result.fetchall()} + + @classmethod + def set_is_favorite(cls, hash: str, is_favorite: bool): + """ + Set the 'is_favorite' flag for a specific hash. + + Args: + hash (str): The hash value. + is_favorite (bool): The value of the 'is_favorite' flag. + """ + with DbManager(commit=True) as conn: + if cls.__tablename__ == "track": + stmt = ( + update(cls) + .where(TrackTable.trackhash == hash) + .values(is_favorite=is_favorite) + ) + elif cls.__tablename__ == "album": + stmt = ( + update(cls) + .where(AlbumTable.albumhash == hash) + .values(is_favorite=is_favorite) + ) + elif cls.__tablename__ == "artist": + stmt = ( + update(cls) + .where(ArtistTable.artisthash == hash) + .values(is_favorite=is_favorite) + ) + + conn.execute(stmt) + + +class TrackTable(Base): + __tablename__ = "track" + + id: Mapped[int] = mapped_column(init=False, primary_key=True) + album: Mapped[str] = mapped_column(String()) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) + albumhash: Mapped[str] = mapped_column(String(), index=True) + artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + bitrate: Mapped[int] = mapped_column(Integer()) + copyright: Mapped[Optional[str]] = mapped_column(String()) + date: Mapped[int] = mapped_column(Integer(), nullable=True) + disc: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + filepath: Mapped[str] = mapped_column(String(), unique=True) + folder: Mapped[str] = mapped_column(String(), index=True) + genrehashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + genres: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) + last_mod: Mapped[float] = mapped_column(Integer()) + og_album: Mapped[str] = mapped_column(String()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + track: Mapped[int] = mapped_column(Integer()) + trackhash: Mapped[str] = mapped_column(String(), index=True) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + playcount: Mapped[int] = mapped_column(Integer()) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_by_filepaths(cls, filepaths: list[str]): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.filepath.in_(filepaths)) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_by_albumhash(cls, albumhash: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.albumhash == albumhash) + ) + tracks = tracks_to_dataclasses(result.fetchall()) + return remove_duplicates(tracks, is_album_tracks=True) + + @classmethod + def get_track_by_trackhash(cls, hash: str, filepath: str = ""): + with DbManager() as conn: + if filepath: + result = conn.execute( + select(TrackTable) + .where( + and_( + TrackTable.trackhash == hash, + TrackTable.filepath == filepath, + ) + ) + .order_by(TrackTable.bitrate.desc()) + ) + else: + result = conn.execute( + select(TrackTable).where(TrackTable.trackhash == hash) + ) + + track = result.fetchone() + + if track: + return track_to_dataclass(track) + + @classmethod + def get_tracks_by_artisthash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.artists.contains(artisthash)) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_in_path(cls, path: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable) + .where(TrackTable.filepath.contains(path)) + .order_by(TrackTable.last_mod) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_by_trackhashes(cls, hashes: Iterable[str], limit: int | None = None): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.trackhash.in_(hashes)).limit(limit) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def remove_tracks_by_filepaths(cls, filepaths: set[str]): + with DbManager(commit=True) as conn: + conn.execute(delete(TrackTable).where(TrackTable.filepath.in_(filepaths))) + + +class AlbumTable(Base): + __tablename__ = "album" + + id: Mapped[int] = mapped_column(primary_key=True) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) + base_title: Mapped[str] = mapped_column(String()) + color: Mapped[Optional[str]] = mapped_column(String()) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True) + genres: Mapped[str] = mapped_column(JSON()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + trackcount: Mapped[int] = mapped_column(Integer()) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(AlbumTable)) + all = result.fetchall() + return albums_to_dataclasses(all) + + @classmethod + def get_album_by_albumhash(cls, hash: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.albumhash == hash) + ) + album = result.fetchone() + + if album: + return album_to_dataclass(album) + + @classmethod + def get_albums_by_albumhashes(cls, hashes: Iterable[str], limit: int | None = None): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)).limit(limit) + ) + return albums_to_dataclasses(result.fetchall()) + + @classmethod + def get_albums_by_artisthashes(cls, artisthashes: list[str]): + with DbManager() as conn: + albums: list[AlbumModel] = [] + + for artist in artisthashes: + result = conn.execute( + # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! + select(AlbumTable).where(AlbumTable.albumartists.contains(artist)) + ) + albums.extend(albums_to_dataclasses(result.fetchall())) + + return albums + + @classmethod + def get_albums_by_base_title(cls, base_title: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.base_title == base_title) + ) + return albums_to_dataclasses(result.fetchall()) + + @classmethod + def get_albums_by_artisthash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash)) + ) + return albums_to_dataclasses(result.all()) + + +class ArtistTable(Base): + __tablename__ = "artist" + + id: Mapped[int] = mapped_column(primary_key=True) + albumcount: Mapped[int] = mapped_column(Integer()) + artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True) + genres: Mapped[str] = mapped_column(JSON()) + name: Mapped[str] = mapped_column(String(), index=True) + trackcount: Mapped[int] = mapped_column(Integer()) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + all = result.fetchall() + return artists_to_dataclasses(all) + + @classmethod + def get_artist_by_hash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(ArtistTable).where(ArtistTable.artisthash == artisthash) + ) + return artist_to_dataclass(result.fetchone()) + + @classmethod + def get_artisthashes_not_in(cls, artisthashes: list[str]): + with DbManager() as conn: + result = conn.execute( + select(ArtistTable.artisthash, ArtistTable.name).where( + ~ArtistTable.artisthash.in_(artisthashes) + ) + ) + return [{"artisthash": row[0], "name": row[1]} for row in result.fetchall()] + + @classmethod + def get_artists_by_artisthashes( + cls, hashes: Iterable[str], limit: int | None = None + ): + with DbManager() as conn: + result = conn.execute( + select(ArtistTable) + .where(ArtistTable.artisthash.in_(hashes)) + .limit(limit) + ) + return artists_to_dataclasses(result.fetchall()) diff --git a/app/db/metadata.py b/app/db/metadata.py new file mode 100644 index 00000000..16976b45 --- /dev/null +++ b/app/db/metadata.py @@ -0,0 +1,33 @@ +from app.db import Base, DbManager + + +from sqlalchemy import Integer, insert, select, update +from sqlalchemy.orm import Mapped, mapped_column + + +class MigrationTable(Base): + __tablename__ = "dbmigration" + + id: Mapped[int] = mapped_column(primary_key=True) + version: Mapped[int] = mapped_column(Integer()) + + @classmethod + def set_version(cls, version: int): + with DbManager(commit=True) as conn: + result = conn.execute( + update(cls).where(cls.id == 1).values(version=version) + ) + + if result.rowcount == 0: + conn.execute(insert(cls).values(id=1, version=version)) + + @classmethod + def get_version(cls): + with DbManager() as conn: + result = conn.execute(select(cls.version).where(cls.id == 1)) + result = result.fetchone() + + if result: + return result[0] + + return -1 \ No newline at end of file diff --git a/app/db/sqlite/lastfm/similar_artists.py b/app/db/sqlite/lastfm/similar_artists.py index 22a5825c..53a45b70 100644 --- a/app/db/sqlite/lastfm/similar_artists.py +++ b/app/db/sqlite/lastfm/similar_artists.py @@ -16,7 +16,7 @@ class SQLiteLastFMSimilarArtists: sql = """INSERT OR REPLACE INTO lastfm_similar_artists(artisthash, similar_artists) VALUES(?,?)""" with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (artist.artisthash, artist.similar_artist_hashes)) + cur.execute(sql, (artist.artisthash, artist.similar_artists)) cur.close() @classmethod diff --git a/app/db/sqlite/plugins/__init__.py b/app/db/sqlite/plugins/__init__.py index 2cfd2345..0afc5d43 100644 --- a/app/db/sqlite/plugins/__init__.py +++ b/app/db/sqlite/plugins/__init__.py @@ -8,7 +8,6 @@ from ..utils import SQLiteManager def plugin_tuple_to_obj(plugin_tuple: tuple) -> Plugin: return Plugin( name=plugin_tuple[1], - description=plugin_tuple[2], active=bool(plugin_tuple[3]), settings=json.loads(plugin_tuple[4]), ) @@ -43,15 +42,6 @@ class PluginsMethods: return lastrowid - @classmethod - def insert_lyrics_plugin(cls): - plugin = Plugin( - name="lyrics_finder", - description="Find lyrics from the internet", - active=False, - settings={"auto_download": False}, - ) - cls.insert_plugin(plugin) @classmethod def get_all_plugins(cls): diff --git a/app/db/sqlite/queries.py b/app/db/sqlite/queries.py index 1e52b272..547b4ffe 100644 --- a/app/db/sqlite/queries.py +++ b/app/db/sqlite/queries.py @@ -14,13 +14,6 @@ CREATE TABLE IF NOT EXISTS playlists ( constraint fk_users foreign key (userid) references users(id) on delete cascade ); -CREATE TABLE IF NOT EXISTS favorites ( - id integer PRIMARY KEY, - hash text not null, - type text not null, - timestamp integer not null default 0 -); - CREATE TABLE IF NOT EXISTS settings ( id integer PRIMARY KEY, root_dirs text NOT NULL, diff --git a/app/db/sqlite/settings.py b/app/db/sqlite/settings.py index a597d1b2..0121fa9f 100644 --- a/app/db/sqlite/settings.py +++ b/app/db/sqlite/settings.py @@ -1,150 +1,151 @@ from pprint import pprint from typing import Any +from app.config import UserConfig from app.db.sqlite.utils import SQLiteManager -from app.settings import SessionVars from app.utils.wintools import win_replace_slash -class SettingsSQLMethods: - """ - Methods for interacting with the settings table. - """ +# class SettingsSQLMethods: +# """ +# Methods for interacting with the settings table. +# """ - @staticmethod - def get_all_settings(): - """ - Gets all settings from the database. - """ +# @staticmethod +# def get_all_settings(): +# """ +# Gets all settings from the database. +# """ - sql = "SELECT * FROM settings WHERE id = 1" +# sql = "SELECT * FROM settings WHERE id = 1" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - settings = cur.fetchone() - cur.close() +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql) +# settings = cur.fetchone() +# cur.close() - # if root_dirs not set - if settings is None: - return [] +# # if root_dirs not set +# if settings is None: +# return [] - # omit id, root_dirs, and exclude_dirs - return settings[3:] +# # omit id, root_dirs, and exclude_dirs +# return settings[3:] - @staticmethod - def get_root_dirs() -> list[str]: - """ - Gets custom root directories from the database. - """ +# @staticmethod +# def get_root_dirs() -> list[str]: +# """ +# Gets custom root directories from the database. +# """ - sql = "SELECT root_dirs FROM settings" +# sql = "SELECT root_dirs FROM settings" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - dirs = cur.fetchall() - cur.close() +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql) +# dirs = cur.fetchall() +# cur.close() - dirs = [_dir[0] for _dir in dirs] - return [win_replace_slash(d) for d in dirs] +# dirs = [_dir[0] for _dir in dirs] +# return [win_replace_slash(d) for d in dirs] - @staticmethod - def add_root_dirs(dirs: list[str]): - """ - Add custom root directories to the database. - """ +# @staticmethod +# def add_root_dirs(dirs: list[str]): +# """ +# Add custom root directories to the database. +# """ - sql = "INSERT INTO settings (root_dirs) VALUES (?)" - existing_dirs = SettingsSQLMethods.get_root_dirs() +# sql = "INSERT INTO settings (root_dirs) VALUES (?)" +# existing_dirs = SettingsSQLMethods.get_root_dirs() - dirs = [_dir for _dir in dirs if _dir not in existing_dirs] +# dirs = [_dir for _dir in dirs if _dir not in existing_dirs] - if len(dirs) == 0: - return +# if len(dirs) == 0: +# return - with SQLiteManager(userdata_db=True) as cur: - for _dir in dirs: - cur.execute(sql, (_dir,)) +# with SQLiteManager(userdata_db=True) as cur: +# for _dir in dirs: +# cur.execute(sql, (_dir,)) - @staticmethod - def remove_root_dirs(dirs: list[str]): - """ - Remove custom root directories from the database. - """ +# @staticmethod +# def remove_root_dirs(dirs: list[str]): +# """ +# Remove custom root directories from the database. +# """ - sql = "DELETE FROM settings WHERE root_dirs = ?" +# sql = "DELETE FROM settings WHERE root_dirs = ?" - with SQLiteManager(userdata_db=True) as cur: - for _dir in dirs: - cur.execute(sql, (_dir,)) +# with SQLiteManager(userdata_db=True) as cur: +# for _dir in dirs: +# cur.execute(sql, (_dir,)) - # Not currently used anywhere, to be used later - @staticmethod - def add_excluded_dirs(dirs: list[str]): - """ - Add custom exclude directories to the database. - """ +# # Not currently used anywhere, to be used later +# @staticmethod +# def add_excluded_dirs(dirs: list[str]): +# """ +# Add custom exclude directories to the database. +# """ - sql = "INSERT INTO settings (exclude_dirs) VALUES (?)" +# sql = "INSERT INTO settings (exclude_dirs) VALUES (?)" - with SQLiteManager(userdata_db=True) as cur: - cur.executemany(sql, dirs) +# with SQLiteManager(userdata_db=True) as cur: +# cur.executemany(sql, dirs) - @staticmethod - def remove_excluded_dirs(dirs: list[str]): - """ - Remove custom exclude directories from the database. - """ +# @staticmethod +# def remove_excluded_dirs(dirs: list[str]): +# """ +# Remove custom exclude directories from the database. +# """ - sql = "DELETE FROM settings WHERE exclude_dirs = ?" +# sql = "DELETE FROM settings WHERE exclude_dirs = ?" - with SQLiteManager(userdata_db=True) as cur: - cur.executemany(sql, dirs) +# with SQLiteManager(userdata_db=True) as cur: +# cur.executemany(sql, dirs) - @staticmethod - def get_excluded_dirs() -> list[str]: - """ - Gets custom exclude directories from the database. - """ +# @staticmethod +# def get_excluded_dirs() -> list[str]: +# """ +# Gets custom exclude directories from the database. +# """ - sql = "SELECT exclude_dirs FROM settings" +# sql = "SELECT exclude_dirs FROM settings" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - dirs = cur.fetchall() - return [_dir[0] for _dir in dirs] +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql) +# dirs = cur.fetchall() +# return [_dir[0] for _dir in dirs] - @staticmethod - def get_settings() -> dict[str, Any]: - pass +# @staticmethod +# def get_settings() -> dict[str, Any]: +# pass - @staticmethod - def set_setting(key: str, value: Any): - sql = f"UPDATE settings SET {key} = :value WHERE id = 1" +# @staticmethod +# def set_setting(key: str, value: Any): +# sql = f"UPDATE settings SET {key} = :value WHERE id = 1" - if type(value) == bool: - value = str(int(value)) +# if type(value) == bool: +# value = str(int(value)) - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, {"value": value}) +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql, {"value": value}) -def load_settings(): - s = SettingsSQLMethods.get_all_settings() +# def load_settings(): +# # s = SettingsSQLMethods.get_all_settings() +# config = UserConfig() - try: - db_separators: str = s[0] - db_separators = db_separators.replace(" ", "") - separators = db_separators.split(",") - separators = set(separators) - except IndexError: - separators = {";", "/"} + # try: + # db_separators: str = s[0] + # db_separators = db_separators.replace(" ", "") + # separators = db_separators.split(",") + # separators = set(separators) + # except IndexError: + # separators = {";", "/"} - SessionVars.ARTIST_SEPARATORS = separators + # SessionVars.ARTIST_SEPARATORS = config.artistSeparators - # boolean settings - SessionVars.EXTRACT_FEAT = bool(s[1]) - SessionVars.REMOVE_PROD = bool(s[2]) - SessionVars.CLEAN_ALBUM_TITLE = bool(s[3]) - SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4]) - SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5]) - SessionVars.SHOW_ALBUMS_AS_SINGLES = bool(s[6]) + # # boolean settings + # SessionVars.EXTRACT_FEAT = bool(s[1]) + # SessionVars.REMOVE_PROD = bool(s[2]) + # SessionVars.CLEAN_ALBUM_TITLE = bool(s[3]) + # SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4]) + # SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5]) + # SessionVars.SHOW_ALBUMS_AS_SINGLES = bool(s[6]) diff --git a/app/db/userdata.py b/app/db/userdata.py new file mode 100644 index 00000000..ce2b6708 --- /dev/null +++ b/app/db/userdata.py @@ -0,0 +1,240 @@ +import datetime +from shlex import join +from typing import Any +from flask_jwt_extended import current_user +from sqlalchemy import ( + JSON, + Boolean, + ForeignKey, + Integer, + String, + and_, + delete, + insert, + select, + update, + join, +) + +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.utils import ( + albums_to_dataclasses, + artists_to_dataclasses, + favorites_to_dataclass, + plugin_to_dataclasses, + similar_artist_to_dataclass, + similar_artists_to_dataclass, + tracks_to_dataclasses, + user_to_dataclass, + user_to_dataclasses, +) + +from app.db import Base, DbManager +from app.utils.auth import hash_password + + +class UserTable(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column(primary_key=True) + image: Mapped[str] = mapped_column(String(), nullable=True) + password: Mapped[str] = mapped_column(String()) + username: Mapped[str] = mapped_column(String(), index=True) + roles: Mapped[list[str]] = mapped_column(JSON(), default_factory=lambda: ["user"]) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) + + @classmethod + def get_all(cls): + result = cls.execute(select(cls)) + return user_to_dataclasses(result.fetchall()) + + @classmethod + def insert_default_user(cls): + user = { + "username": "admin", + "password": hash_password("admin"), + "roles": ["admin"], + } + + return cls.insert_one(user) + + @classmethod + def insert_guest_user(cls): + user = { + "username": "guest", + "password": hash_password("guest"), + "roles": ["guest"], + } + + return cls.insert_one(user) + + @classmethod + def get_by_id(cls, id: int): + with DbManager() as conn: + result = conn.execute(select(cls).where(cls.id == id)) + res = result.fetchone() + + if res: + return user_to_dataclass(res) + + @classmethod + def get_by_username(cls, username: str): + with DbManager() as conn: + result = conn.execute(select(cls).where(cls.username == username)) + res = result.fetchone() + + if res: + return user_to_dataclass(res) + + @classmethod + def update_one(cls, user: dict[str, Any]): + with DbManager(commit=True) as conn: + conn.execute(update(cls).where(cls.id == user["id"]).values(user)) + + @classmethod + def remove_by_username(cls, username: str): + return cls.execute(delete(cls).where(cls.username == username), commit=True) + + +class PluginTable(Base): + __tablename__ = "plugin" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(), unique=True) + active: Mapped[bool] = mapped_column(Boolean()) + settings: Mapped[dict[str, Any]] = mapped_column(JSON()) + extra: Mapped[dict[str, Any]] = mapped_column(JSON(), nullable=True) + + @classmethod + def get_all(cls): + return plugin_to_dataclasses(cls.all()) + + +class SimilarArtistTable(Base): + __tablename__ = "notlastfm_similar_artists" + + id: Mapped[int] = mapped_column(Integer(), primary_key=True) + artisthash: Mapped[str] = mapped_column(String(), index=True) + similar_artists: Mapped[dict[str, str]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + return similar_artists_to_dataclass(result.fetchall()) + + @classmethod + def exists(cls, artisthash: str): + """ + Check whether an artisthash exists in the database. + """ + + with DbManager() as conn: + result = conn.execute( + select(cls.artisthash).where(cls.artisthash == artisthash) + ) + return result.fetchone() is not None + + @classmethod + def get_by_hash(cls, artisthash: str): + """ + Get a single artist by hash. + """ + + with DbManager() as conn: + result = conn.execute(select(cls).where(cls.artisthash == artisthash)) + result = result.fetchone() + + if result: + return similar_artist_to_dataclass(result) + + +class FavoritesTable(Base): + __tablename__ = "favorite" + + id: Mapped[int] = mapped_column(primary_key=True) + hash: Mapped[str] = mapped_column(String()) + type: Mapped[str] = mapped_column(String(), index=True) + timestamp: Mapped[int] = mapped_column(Integer(), index=True) + userid: Mapped[int] = mapped_column( + Integer(), ForeignKey("user.id"), default=1, index=True + ) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + return favorites_to_dataclass(result.fetchall()) + + @classmethod + def insert_item(cls, item: dict[str, Any]): + item["timestamp"] = int(datetime.datetime.now().timestamp()) + item["userid"] = current_user["id"] + + with DbManager(commit=True) as conn: + conn.execute(insert(cls).values(item)) + + @classmethod + def remove_item(cls, item: dict[str, Any]): + with DbManager(commit=True) as conn: + conn.execute( + delete(cls).where( + (cls.hash == item["hash"]) & (cls.type == item["type"]) + ) + ) + + @classmethod + def check_exists(cls, hash: str, type: str): + result = cls.execute(select(cls).where((cls.hash == hash) & (cls.type == type))) + return result.fetchone() is not None + + @classmethod + def get_all_of_type(cls, table: Any, field: Any, type: str, start: int, limit: int): + result = cls.execute( + select(table) + .select_from(join(table, cls, field == cls.hash)) + .where(and_(cls.type == type, cls.userid == current_user["id"])) + .offset(start) + # INFO: If start is 0, fetch all so we can get the total count + .limit(limit if start != 0 else None) + ) + + res = result.fetchall() + + if start == 0: + return res[:limit], len(res) + + return res, -1 + + @classmethod + def get_fav_tracks(cls, start: int, limit: int): + from .libdata import TrackTable + + result, total = cls.get_all_of_type( + TrackTable, TrackTable.trackhash, "track", start, limit + ) + return tracks_to_dataclasses(result), total + + @classmethod + def get_fav_albums(cls, start: int, limit: int): + from .libdata import AlbumTable + + result, total = cls.get_all_of_type( + AlbumTable, AlbumTable.albumhash, "album", start, limit + ) + return albums_to_dataclasses(result), total + + @classmethod + def get_fav_artists(cls, start: int, limit: int): + from .libdata import ArtistTable + + result, total = cls.get_all_of_type( + ArtistTable, ArtistTable.artisthash, "artist", start, limit + ) + return artists_to_dataclasses(result), total diff --git a/app/db/utils.py b/app/db/utils.py new file mode 100644 index 00000000..0df36aab --- /dev/null +++ b/app/db/utils.py @@ -0,0 +1,75 @@ +from typing import Any + +from app.models import Album as AlbumModel, Artist as ArtistModel, Track as TrackModel +from app.models.favorite import Favorite +from app.models.lastfm import SimilarArtist +from app.models.plugins import Plugin +from app.models.user import User + + +def track_to_dataclass(track: Any): + return TrackModel(**track._asdict()) + + +def tracks_to_dataclasses(tracks: Any): + return [track_to_dataclass(track) for track in tracks] + + +def album_to_dataclass(album: Any): + return AlbumModel(**album._asdict()) + + +def albums_to_dataclasses(albums: Any): + return [album_to_dataclass(album) for album in albums] + + +def artist_to_dataclass(artist: Any): + return ArtistModel(**artist._asdict()) + + +def artists_to_dataclasses(artists: Any): + return [artist_to_dataclass(artist) for artist in artists] + + +# SECTION: User data helpers + + +def similar_artist_to_dataclass(entry: Any): + entry_dict = entry._asdict() + del entry_dict["id"] + + return SimilarArtist(**entry_dict) + + +def similar_artists_to_dataclass(entries: Any): + return [similar_artist_to_dataclass(entry) for entry in entries] + + +def favorite_to_dataclass(entry: Any): + entry_dict = entry._asdict() + del entry_dict["id"] + + return Favorite(**entry_dict) + + +def favorites_to_dataclass(entries: Any): + return [favorite_to_dataclass(entry) for entry in entries] + + +def user_to_dataclass(entry: Any): + entry_dict = entry._asdict() + return User(**entry_dict) + + +def user_to_dataclasses(entries: Any): + return [user_to_dataclass(entry) for entry in entries] + + +def plugin_to_dataclass(entry: Any): + entry_dict = entry._asdict() + del entry_dict["id"] + return Plugin(**entry_dict) + + +def plugin_to_dataclasses(entries: Any): + return [plugin_to_dataclass(entry) for entry in entries] diff --git a/app/lib/albumslib.py b/app/lib/albumslib.py index c8f9802e..36034f3a 100644 --- a/app/lib/albumslib.py +++ b/app/lib/albumslib.py @@ -12,63 +12,21 @@ from app.store.albums import AlbumStore from app.store.tracks import TrackStore -def create_albums(): - """ - Creates albums from the tracks in the store. - """ - - # group all tracks by albumhash - tracks = TrackStore.tracks - tracks = sorted(tracks, key=lambda t: t.albumhash) - grouped = groupby(tracks, lambda t: t.albumhash) - - # create albums from the groups - albums: list[Track] = [] - for albumhash, tracks in grouped: - count = len(list(tracks)) - duration = sum(t.duration for t in tracks) - created_date = min(t.created_date for t in tracks) - - album = AlbumStore.create_album(list(tracks)[0]) - album.set_count(count) - album.set_duration(duration) - album.set_created_date(created_date) - - albums.append(album) - - return albums - - -def validate_albums(): - """ - Removes albums that have no tracks. - - Probably albums that were added from incompletely written files. - """ - - album_hashes = {t.albumhash for t in TrackStore.tracks} - albums = AlbumStore.albums - - for album in albums: - if album.albumhash not in album_hashes: - AlbumStore.remove_album(album) - - def remove_duplicate_on_merge_versions(tracks: list[Track]) -> list[Track]: """ Removes duplicate tracks when merging versions of the same album. """ - + # TODO! pass -def sort_by_track_no(tracks: list[Track]) -> list[dict[str, Any]]: - tracks = [asdict(t) for t in tracks] +def sort_by_track_no(tracks: list[Track]): + # tracks = [asdict(t) for t in tracks] for t in tracks: - track = str(t["track"]).zfill(3) - t["_pos"] = int(f"{t['disc']}{track}") + track = str(t.track).zfill(3) + t._pos = int(f"{t.disc}{track}") - tracks = sorted(tracks, key=lambda t: t["_pos"]) + tracks = sorted(tracks, key=lambda t: t._pos) return tracks diff --git a/app/lib/artistlib.py b/app/lib/artistlib.py index b40cb56a..c101b515 100644 --- a/app/lib/artistlib.py +++ b/app/lib/artistlib.py @@ -1,5 +1,3 @@ -from collections import namedtuple -from itertools import groupby import os import urllib from concurrent.futures import ThreadPoolExecutor @@ -12,9 +10,10 @@ from requests.exceptions import ConnectionError as RequestConnectionError from requests.exceptions import ReadTimeout from app import settings -from app.models import Album, Artist, Track -from app.store import artists as artist_store -from app.store.tracks import TrackStore +from app.db.libdata import ArtistTable + +# from app.store import artists as artist_store +# from app.store.tracks import TrackStore from app.utils.hashing import create_hash from app.utils.progressbar import tqdm @@ -107,22 +106,15 @@ class CheckArtistImages: # read all files in the artist image folder path = settings.Paths.get_sm_artist_img_path() - processed = "".join(os.listdir(path)).replace("webp", "") - - # filter out artists that already have an image - artists = filter( - lambda a: a.artisthash not in processed, artist_store.ArtistStore.artists - ) - artists = list(artists) - - # process the rest - key_artist_map = ((instance_key, artist) for artist in artists) + processed = [path.replace(".webp", "") for path in os.listdir(path)] + unprocessed = ArtistTable.get_artisthashes_not_in(processed) + key_artist_map = ((instance_key, artist) for artist in unprocessed) with ThreadPoolExecutor(max_workers=14) as executor: res = list( tqdm( executor.map(self.download_image, key_artist_map), - total=len(artists), + total=len(unprocessed), desc="Downloading missing artist images", ) ) @@ -130,7 +122,7 @@ class CheckArtistImages: list(res) @staticmethod - def download_image(_map: tuple[str, Artist]): + def download_image(_map: tuple[str, dict[str, str]]): """ Checks if an artist image exists and downloads it if not. @@ -142,16 +134,17 @@ class CheckArtistImages: return img_path = ( - Path(settings.Paths.get_sm_artist_img_path()) / f"{artist.artisthash}.webp" + Path(settings.Paths.get_sm_artist_img_path()) + / f"{artist['artisthash']}.webp" ) if img_path.exists(): return - url = get_artist_image_link(artist.name) + url = get_artist_image_link(artist["name"]) if url is not None: - return DownloadImage(url, name=f"{artist.artisthash}.webp") + return DownloadImage(url, name=f"{artist['artisthash']}.webp") # def fetch_album_bio(title: str, albumartist: str) -> str | None: """ Returns the album bio for a given album. """ @@ -183,89 +176,3 @@ class CheckArtistImages: # def __call__(self): # return fetch_album_bio(self.title, self.albumartist) - - -def get_artists_from_tracks(tracks: list[Track]) -> set[str]: - """ - Extracts all artists from a list of tracks. Returns a list of Artists. - """ - artists = set() - - master_artist_list = [[x.name for x in t.artists] for t in tracks] - artists = artists.union(*master_artist_list) - - return artists - - -def get_albumartists(albums: list[Album]) -> set[str]: - artists = set() - - for album in albums: - albumartists = [a.name for a in album.albumartists] - - artists.update(albumartists) - - return artists - - -def get_all_artists(tracks: list[Track], albums: list[Album]) -> list[Artist]: - TrackInfo = namedtuple( - "TrackInfo", - [ - "artisthash", - "albumhash", - "trackhash", - "duration", - "artistname", - "created_date", - ], - ) - src_tracks = TrackStore.tracks - all_tracks: set[TrackInfo] = set() - - for track in src_tracks: - artist_hashes = {(a.name, a.artisthash) for a in track.artists}.union( - (a.name, a.artisthash) for a in track.albumartists - ) - - for artist in artist_hashes: - track_info = TrackInfo( - artistname=artist[0], - artisthash=artist[1], - albumhash=track.albumhash, - trackhash=track.trackhash, - duration=track.duration, - created_date=track.created_date, - # work on created date - ) - - all_tracks.add(track_info) - - all_tracks = sorted(all_tracks, key=lambda x: x.artisthash) - all_tracks = groupby(all_tracks, key=lambda x: x.artisthash) - - artists = [] - - for artisthash, tracks in all_tracks: - tracks: list[TrackInfo] = list(tracks) - - artistname = ( - sorted({t.artistname for t in tracks})[0] - if len(tracks) > 1 - else tracks[0].artistname - ) - - albumcount = len({t.albumhash for t in tracks}) - duration = sum(t.duration for t in tracks) - created_date = min(t.created_date for t in tracks) - - artist = Artist(name=artistname) - - artist.set_trackcount(len(tracks)) - artist.set_albumcount(albumcount) - artist.set_duration(duration) - artist.set_created_date(created_date) - - artists.append(artist) - - return artists diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index 086fcb2b..e71dbb1f 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -52,47 +52,47 @@ def process_color(item_hash: str, is_album=True): return get_image_colors(str(path)) -class ProcessAlbumColors: - """ - Extracts the most dominant color from the album art and saves it to the database. - """ +# class ProcessAlbumColors: +# """ +# Extracts the most dominant color from the album art and saves it to the database. +# """ - def __init__(self, instance_key: str) -> None: - global PROCESS_ALBUM_COLORS_KEY - PROCESS_ALBUM_COLORS_KEY = instance_key +# def __init__(self, instance_key: str) -> None: +# global PROCESS_ALBUM_COLORS_KEY +# PROCESS_ALBUM_COLORS_KEY = instance_key - albums = [ - a - for a in AlbumStore.albums - if a is not None and a.colors is not None and len(a.colors) == 0 - ] +# albums = [ +# a +# for a in AlbumStore.albums +# if a is not None and a.colors is not None and len(a.colors) == 0 +# ] - with SQLiteManager() as cur: - try: - for album in tqdm(albums, desc="Processing missing album colors"): - if PROCESS_ALBUM_COLORS_KEY != instance_key: - raise PopulateCancelledError( - "A newer 'ProcessAlbumColors' instance is running. Stopping this one." - ) +# with SQLiteManager() as cur: +# try: +# for album in tqdm(albums, desc="Processing missing album colors"): +# if PROCESS_ALBUM_COLORS_KEY != instance_key: +# raise PopulateCancelledError( +# "A newer 'ProcessAlbumColors' instance is running. Stopping this one." +# ) - # TODO: Stop hitting the database for every album. - # Instead, fetch all the data from the database and - # check from memory. +# # TODO: Stop hitting the database for every album. +# # Instead, fetch all the data from the database and +# # check from memory. - exists = aldb.exists(album.albumhash, cur=cur) - if exists: - continue +# exists = aldb.exists(album.albumhash, cur=cur) +# if exists: +# continue - colors = process_color(album.albumhash) +# colors = process_color(album.albumhash) - if colors is None: - continue +# if colors is None: +# continue - album.set_colors(colors) - color_str = json.dumps(colors) - aldb.insert_one_album(cur, album.albumhash, color_str) - finally: - cur.close() +# album.set_colors(colors) +# color_str = json.dumps(colors) +# aldb.insert_one_album(cur, album.albumhash, color_str) +# finally: +# cur.close() class ProcessArtistColors: diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index 0f3340c2..85542e11 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -5,9 +5,10 @@ from app.logger import log from app.models import Folder from app.serializers.track import serialize_tracks from app.settings import SUPPORTED_FILES +from app.store.folder import FolderStore from app.utils.wintools import win_replace_slash -from app.db import TrackTable as TrackDB +from app.db.libdata import TrackTable as TrackDB def create_folder(path: str, trackcount=0, foldercount=0) -> Folder: @@ -43,8 +44,7 @@ def get_folders(paths: list[str]): Filters out folders that don't have any tracks and returns a list of folder objects. """ - folders = TrackDB.count_tracks_containing_paths(paths) - + folders = FolderStore.count_tracks_containing_paths(paths) return [ create_folder(f["path"], f["trackcount"], foldercount=0) for f in folders diff --git a/app/lib/populate.py b/app/lib/populate.py index 593e0282..ea1e1afe 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -1,3 +1,4 @@ +from dataclasses import asdict import os from collections import deque from concurrent.futures import ThreadPoolExecutor @@ -7,28 +8,26 @@ from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout from app import settings -from app.db import TrackTable +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb -from app.db.sqlite.settings import SettingsSQLMethods as sdb + +# from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb from app.db.sqlite.tracks import SQLiteTrackMethods -from app.lib.albumslib import validate_albums from app.lib.artistlib import CheckArtistImages -from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors +from app.lib.colorlib import ProcessArtistColors from app.lib.errors import PopulateCancelledError -from app.lib.taglib import extract_thumb, get_tags -from app.lib.trackslib import validate_tracks +from app.lib.taglib import extract_thumb from app.logger import log from app.models import Album, Artist, Track from app.models.lastfm import SimilarArtist from app.requests.artists import fetch_similar_artists -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore -from app.store.tracks import TrackStore from app.utils.filesystem import run_fast_scandir from app.utils.network import has_connection from app.utils.progressbar import tqdm +from app.db.userdata import SimilarArtistTable + get_all_tracks = SQLiteTrackMethods.get_all_tracks insert_many_tracks = SQLiteTrackMethods.insert_many_tracks remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths @@ -44,50 +43,49 @@ class Populate: also checks if the album art exists in the image path, if not tries to extract it. """ - def __init__(self, instance_key: str) -> None: - return + # def __init__(self, instance_key: str) -> None: + # return + # if len(dirs_to_scan) == 0: + # log.warning( + # ( + # "The root directory is not configured. " + # + "Open the app in your webbrowser to configure." + # ) + # ) + # return + + # try: + # if dirs_to_scan[0] == "$home": + # dirs_to_scan = [settings.Paths.USER_HOME_DIR] + # except IndexError: + # pass + + # files = set() + + # for _dir in dirs_to_scan: + # files = files.union(run_fast_scandir(_dir, full=True)[1]) + + # unmodified, modified_tracks = self.remove_modified(tracks) + # untagged = files - unmodified + + # if len(untagged) != 0: + # self.tag_untagged(untagged, instance_key) + + # self.extract_thumb_with_overwrite(modified_tracks) + + +class CordinateMedia: + """ + Cordinates the extracting of thumbnails + """ + + def __init__(self, instance_key: str): global POPULATE_KEY POPULATE_KEY = instance_key - validate_tracks() - validate_albums() - - tracks = get_all_tracks() - - dirs_to_scan = sdb.get_root_dirs() - - if len(dirs_to_scan) == 0: - log.warning( - ( - "The root directory is not configured. " - + "Open the app in your webbrowser to configure." - ) - ) - return - - try: - if dirs_to_scan[0] == "$home": - dirs_to_scan = [settings.Paths.USER_HOME_DIR] - except IndexError: - pass - - files = set() - - for _dir in dirs_to_scan: - files = files.union(run_fast_scandir(_dir, full=True)[1]) - - unmodified, modified_tracks = self.remove_modified(tracks) - untagged = files - unmodified - - if len(untagged) != 0: - self.tag_untagged(untagged, instance_key) - - self.extract_thumb_with_overwrite(modified_tracks) - try: ProcessTrackThumbnails(instance_key) - ProcessAlbumColors(instance_key) ProcessArtistColors(instance_key) except PopulateCancelledError as e: log.warn(e) @@ -95,10 +93,6 @@ class Populate: tried_to_download_new_images = False - ArtistStore.load_artists(instance_key) - AlbumStore.load_albums(instance_key) - TrackStore.load_all_tracks(instance_key) - if has_connection(): tried_to_download_new_images = True try: @@ -123,101 +117,101 @@ class Populate: log.warn(e) return - @staticmethod - def remove_modified(tracks: Generator[TrackTable, None, None]): - """ - Removes tracks from the database that have been modified - since they were added to the database. - """ + # @staticmethod + # def remove_modified(tracks: Generator[TrackTable, None, None]): + # """ + # Removes tracks from the database that have been modified + # since they were added to the database. + # """ - unmodified_paths = set() - modified_tracks: list[TrackTable] = [] - modified_paths = set() + # unmodified_paths = set() + # modified_tracks: list[TrackTable] = [] + # modified_paths = set() - for track in tracks: - try: - if track.last_mod == round(os.path.getmtime(track.filepath)): - unmodified_paths.add(track.filepath) - continue - except (FileNotFoundError, OSError) as e: - log.warning(e) # REVIEW More informations = good - TrackStore.remove_track_obj(track) - remove_tracks_by_filepaths(track.filepath) + # for track in tracks: + # try: + # if track.last_mod == round(os.path.getmtime(track.filepath)): + # unmodified_paths.add(track.filepath) + # continue + # except (FileNotFoundError, OSError) as e: + # log.warning(e) # REVIEW More informations = good + # TrackStore.remove_track_obj(track) + # remove_tracks_by_filepaths(track.filepath) - modified_paths.add(track.filepath) - modified_tracks.append(track) + # modified_paths.add(track.filepath) + # modified_tracks.append(track) - TrackStore.remove_tracks_by_filepaths(modified_paths) - remove_tracks_by_filepaths(modified_paths) + # TrackStore.remove_tracks_by_filepaths(modified_paths) + # remove_tracks_by_filepaths(modified_paths) - return unmodified_paths, modified_tracks + # return unmodified_paths, modified_tracks - @staticmethod - def tag_untagged(untagged: set[str], key: str): - pass - # for file in tqdm(untagged, desc="Reading files"): - # if POPULATE_KEY != key: - # log.warning("'Populate.tag_untagged': Populate key changed") - # return + # @staticmethod + # def tag_untagged(untagged: set[str], key: str): + # pass + # for file in tqdm(untagged, desc="Reading files"): + # if POPULATE_KEY != key: + # log.warning("'Populate.tag_untagged': Populate key changed") + # return - # tags = get_tags(file) + # tags = get_tags(file) - # if tags is not None: - # TrackTable.insert_one(tags) + # if tags is not None: + # TrackTable.insert_one(tags) - # ============================================= + # ============================================= - # log.info("Found %s new tracks", len(untagged)) - # # tagged_tracks: deque[dict] = deque() - # # tagged_count = 0 + # log.info("Found %s new tracks", len(untagged)) + # # tagged_tracks: deque[dict] = deque() + # # tagged_count = 0 - # favs = favdb.get_fav_tracks() - # records = dict() + # favs = favdb.get_fav_tracks() + # records = dict() - # for fav in favs: - # r = records.setdefault(fav[1], set()) - # r.add(fav[4]) + # for fav in favs: + # r = records.setdefault(fav[1], set()) + # r.add(fav[4]) - # tagged_tracks.append(tags) - # track = Track(**tags) + # tagged_tracks.append(tags) + # track = Track(**tags) - # track.fav_userids = list(records.get(track.trackhash, set())) + # track.fav_userids = list(records.get(track.trackhash, set())) - # TrackStore.add_track(track) + # TrackStore.add_track(track) - # if not AlbumStore.album_exists(track.albumhash): - # AlbumStore.add_album(AlbumStore.create_album(track)) + # if not AlbumStore.album_exists(track.albumhash): + # AlbumStore.add_album(AlbumStore.create_album(track)) - # for artist in track.artists: - # if not ArtistStore.artist_exists(artist.artisthash): - # ArtistStore.add_artist(Artist(artist.name)) + # for artist in track.artists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) - # for artist in track.albumartists: - # if not ArtistStore.artist_exists(artist.artisthash): - # ArtistStore.add_artist(Artist(artist.name)) + # for artist in track.albumartists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) - # tagged_count += 1 - # else: - # log.warning("Could not read file: %s", file) + # tagged_count += 1 + # else: + # log.warning("Could not read file: %s", file) - # if len(tagged_tracks) > 0: - # log.info("Adding %s tracks to database", len(tagged_tracks)) - # insert_many_tracks(tagged_tracks) + # if len(tagged_tracks) > 0: + # log.info("Adding %s tracks to database", len(tagged_tracks)) + # insert_many_tracks(tagged_tracks) - # log.info("Added %s/%s tracks", tagged_count, len(untagged)) + # log.info("Added %s/%s tracks", tagged_count, len(untagged)) - @staticmethod - def extract_thumb_with_overwrite(tracks: list[TrackTable]): - """ - Extracts the thumbnail from a list of filepaths, - overwriting the existing thumbnail if it exists, - for modified files. - """ - for track in tracks: - try: - extract_thumb(track.filepath, track.image, overwrite=True) - except FileNotFoundError: - continue + # @staticmethod + # def extract_thumb_with_overwrite(tracks: list[TrackTable]): + # """ + # Extracts the thumbnail from a list of filepaths, + # overwriting the existing thumbnail if it exists, + # for modified files. + # """ + # for track in tracks: + # try: + # extract_thumb(track.filepath, track.image, overwrite=True) + # except FileNotFoundError: + # continue def get_image(_map: tuple[str, Album]): @@ -235,7 +229,8 @@ def get_image(_map: tuple[str, Album]): raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed") matching_tracks = filter( - lambda t: t.albumhash == album.albumhash, TrackStore.tracks + lambda t: t.albumhash == album.albumhash, + TrackTable.get_tracks_by_albumhash(album.albumhash), ) try: @@ -254,8 +249,12 @@ def get_image(_map: tuple[str, Album]): pass -_cpu_count = os.cpu_count() -CPU_COUNT = _cpu_count // 2 if _cpu_count > 2 else _cpu_count +def get_cpu_count(): + """ + Returns the number of CPUs on the machine. + """ + cpu_count = os.cpu_count() or 0 + return cpu_count // 2 if cpu_count > 2 else cpu_count class ProcessTrackThumbnails: @@ -275,14 +274,14 @@ class ProcessTrackThumbnails: # filter out albums that already have thumbnails albums = filter( - lambda album: album.albumhash not in processed, AlbumStore.albums + lambda album: album.albumhash not in processed, AlbumTable.get_all() ) albums = list(albums) # process the rest key_album_map = ((instance_key, album) for album in albums) - with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: + with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor: results = list( tqdm( executor.map(get_image, key_album_map), @@ -307,16 +306,17 @@ def save_similar_artists(_map: tuple[str, Artist]): "'FetchSimilarArtistsLastFM': Populate key changed" ) - if lastfmdb.exists(artist.artisthash): + if SimilarArtistTable.exists(artist.artisthash): return - artist_hashes = fetch_similar_artists(artist.name) - artist_ = SimilarArtist(artist.artisthash, "~".join(artist_hashes)) + artists = fetch_similar_artists(artist.name) - if len(artist_.similar_artist_hashes) == 0: + # INFO: Nones mean there was a connection error + if artists is None: return - lastfmdb.insert_one(artist_) + artist_ = SimilarArtist(artist.artisthash, artists) + SimilarArtistTable.insert_one(asdict(artist_)) class FetchSimilarArtistsLastFM: @@ -326,17 +326,17 @@ class FetchSimilarArtistsLastFM: def __init__(self, instance_key: str) -> None: # read all artists from db - processed = lastfmdb.get_all() + processed = SimilarArtistTable.get_all() processed = ".".join(a.artisthash for a in processed) # filter out artists that already have similar artists - artists = filter(lambda a: a.artisthash not in processed, ArtistStore.artists) + artists = filter(lambda a: a.artisthash not in processed, ArtistTable.get_all()) artists = list(artists) # process the rest key_artist_map = ((instance_key, artist) for artist in artists) - with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: + with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor: try: print("Processing similar artists") results = list( diff --git a/app/lib/tagger.py b/app/lib/tagger.py index b9ee9d04..d8a679a6 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -1,55 +1,165 @@ +import os from pprint import pprint -from app.db import AlbumTable, ArtistTable, TrackTable -from app.lib.taglib import get_tags +from time import time +from typing import Generator +from app import settings +from app.config import UserConfig +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable +from app.lib.populate import CordinateMedia +from app.lib.taglib import extract_thumb, get_tags +from app.models.track import Track +from app.store.folder import FolderStore from app.utils.filesystem import run_fast_scandir from app.utils.parsers import get_base_album_title from app.utils.progressbar import tqdm +from app.logger import log +from app.utils.threading import background + +POPULATE_KEY: float = 0 + class IndexTracks: - def __init__(self) -> None: - dirs_to_scan = ["/home/cwilvx/Music"] + def __init__(self, instance_key: float) -> None: + """ + Indexes all tracks in the database. + + An instance key is used to prevent multiple instances of the + same class from running at the same time. + """ + global POPULATE_KEY + POPULATE_KEY = instance_key + + # dirs_to_scan = sdb.get_root_dirs() + dirs_to_scan = UserConfig().rootDirs + + if len(dirs_to_scan) == 0: + log.warning( + ( + "The root directory is not configured. " + + "Open the app in your webbrowser to configure." + ) + ) + return + + try: + if dirs_to_scan[0] == "$home": + dirs_to_scan = [settings.Paths.USER_HOME_DIR] + except IndexError: + pass files = set() for _dir in dirs_to_scan: files = files.union(run_fast_scandir(_dir, full=True)[1]) - self.tag_untagged(files) - # unmodified, modified_tracks = self.remove_modified(tracks) - # untagged = files - unmodified + unmodified, modified_tracks = self.filter_modded() + untagged = files - unmodified - def tag_untagged(self, files: set[str]): + self.tag_untagged(untagged, instance_key) + self.extract_thumb_with_overwrite(modified_tracks) + + @staticmethod + def extract_thumb_with_overwrite(tracks: list[dict[str, str]]): + """ + Extracts the thumbnail from a list of filepaths, + overwriting the existing thumbnail if it exists, + for modified files. + """ + for track in tracks: + try: + extract_thumb( + track["filepath"], track["trackhash"] + ".webp", overwrite=True + ) + except FileNotFoundError: + continue + + @staticmethod + def filter_modded(): + """ + Removes tracks from the database that have been modified + since they were indexed. + + Returns a tuple of unmodified paths and modified tracks. + Unmodified paths are indexed and the modified tracks are + + """ + + unmodified_paths = set() + modified_tracks: list[dict[str, str]] = [] + + to_remove = set() + + for track in TrackTable.get_all(): + try: + if track.last_mod == round(os.path.getmtime(track.filepath)): + unmodified_paths.add(track.filepath) + continue + except (FileNotFoundError, OSError) as e: + log.warning(e) # REVIEW More informations = good + to_remove.add(track.filepath) + + modified_tracks.append( + { + "filepath": track.filepath, + "trackhash": track.trackhash, + } + ) + + to_remove = to_remove.union(set(t["filepath"] for t in modified_tracks)) + TrackTable.remove_tracks_by_filepaths(to_remove) + + # REVIEW: Remove after testing! + track = TrackTable.get_tracks_by_filepaths(list(to_remove)[:1]) + if track: + raise Exception("Track not removed") + # ============================================================= + + return unmodified_paths, modified_tracks + + def get_untagged(self): + tracks = TrackTable.get_all() + + def tag_untagged(self, files: set[str], key: float): + config = UserConfig() for file in tqdm(files, desc="Reading files"): - # if POPULATE_KEY != key: - # log.warning("'Populate.tag_untagged': Populate key changed") - # return + if POPULATE_KEY != key: + log.warning("'Populate.tag_untagged': Populate key changed") + return - tags = get_tags(file) + tags = get_tags(file, artist_separators=config.artistSeparators) if tags is not None: TrackTable.insert_one(tags) + FolderStore.filepaths.add(tags["filepath"]) del tags + print(f"{len(files)} new files indexed") + print("Done") + + class IndexAlbums: def __init__(self) -> None: albums = dict() + all_tracks: list[Track] = TrackTable.get_all() - all_tracks: list[TrackTable] = TrackTable.get_all() + if len(all_tracks) == 0: + return for track in all_tracks: if track.albumhash not in albums: albums[track.albumhash] = { "albumartists": track.albumartists, - "artisthashes": [a['artisthash'] for a in track.albumartists], + "artisthashes": [a["artisthash"] for a in track.albumartists], "albumhash": track.albumhash, "base_title": None, "color": None, "created_date": None, "date": None, "duration": track.duration, - "genres": [*track.genre] if track.genre else [], + "genres": [*track.genres] if track.genres else [], "og_title": track.og_album, "title": track.album, "trackcount": 1, @@ -63,8 +173,8 @@ class IndexAlbums: album["dates"].append(track.date) album["created_dates"].append(track.last_mod) - if track.genre: - album["genres"].extend(track.genre) + if track.genres: + album["genres"].extend(track.genres) for album in albums.values(): album["date"] = min(album["dates"]) @@ -79,20 +189,23 @@ class IndexAlbums: album["genres"] = genres album["base_title"], _ = get_base_album_title(album["og_title"]) + del genres del album["dates"] del album["created_dates"] - pprint(albums) - + AlbumTable.remove_all() AlbumTable.insert_many(list(albums.values())) del albums class IndexArtists: def __init__(self) -> None: - all_tracks: list[TrackTable] = TrackTable.get_all() + all_tracks: list[Track] = TrackTable.get_all() artists = dict() + if len(all_tracks) == 0: + return + for track in all_tracks: this_artists = track.artists @@ -100,32 +213,33 @@ class IndexArtists: if a not in this_artists: this_artists.append(a) - for artist in this_artists: - if artist["artisthash"] not in artists: - artists[artist["artisthash"]] = { + for thisartist in this_artists: + if thisartist["artisthash"] not in artists: + artists[thisartist["artisthash"]] = { "albumcount": None, "albums": {track.albumhash}, - "artisthash": artist["artisthash"], + "artisthash": thisartist["artisthash"], "created_dates": [track.last_mod], "dates": [track.date], "date": None, "duration": track.duration, - "genres": track.genre if track.genre else [], - "name": artist["name"], + "genres": track.genres if track.genres else [], + "name": None, + "names": {thisartist["name"]}, "trackcount": None, "tracks": {track.trackhash}, } else: - artist = artists[artist["artisthash"]] + artist = artists[thisartist["artisthash"]] artist["duration"] += track.duration artist["albums"].add(track.albumhash) artist["tracks"].add(track.trackhash) artist["dates"].append(track.date) artist["created_dates"].append(track.last_mod) + artist["names"].add(thisartist["name"]) - if track.genre: - artist["genres"].extend(track.genre) - + if track.genres: + artist["genres"].extend(track.genres) for artist in artists.values(): artist["albumcount"] = len(artist["albums"]) @@ -140,19 +254,35 @@ class IndexArtists: genres.append(genre) artist["genres"] = genres + artist["name"] = sorted(artist["names"])[0] + # INFO: Delete temporary keys + del artist["names"] del artist["tracks"] del artist["albums"] del artist["dates"] del artist["created_dates"] - pprint(artists) + # INFO: Delete local variables + del genres + + ArtistTable.remove_all() ArtistTable.insert_many(list(artists.values())) del artists + class IndexEverything: def __init__(self) -> None: - IndexTracks() + IndexTracks(instance_key=time()) IndexAlbums() IndexArtists() - pass + FolderStore.load_filepaths() + + # pass + + CordinateMedia(instance_key=str(time())) + + +@background +def index_everything(): + return IndexEverything() diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 03cafc1c..a987cc17 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -5,6 +5,7 @@ from pathlib import Path from pprint import pprint import re import sys +from typing import Any import pendulum from PIL import Image, UnidentifiedImageError @@ -86,7 +87,7 @@ def extract_thumb(filepath: str, webp_path: str, overwrite=False) -> bool: return False -def parse_date(date_str: str | None) -> int | None: +def parse_date(date_str: str) -> int | None: """ Extracts the date from a string and returns a timestamp. """ @@ -108,12 +109,13 @@ def clean_filename(filename: str): class ParseData: artist: str title: str + artist_separators: set[str] def __post_init__(self): - self.artist = split_artists(self.artist) + self.artist = split_artists(self.artist, self.artist_separators) -def extract_artist_title(filename: str): +def extract_artist_title(filename: str, artist_separators: set[str]): path = Path(filename).with_suffix("") path = clean_filename(str(path)) @@ -121,22 +123,24 @@ def extract_artist_title(filename: str): split_result = [x.strip() for x in split_result] if len(split_result) == 1: - return ParseData("", split_result[0]) + return ParseData("", split_result[0], artist_separators) if len(split_result) > 2: try: int(split_result[0]) - return ParseData(split_result[1], " - ".join(split_result[2:])) + return ParseData( + split_result[1], " - ".join(split_result[2:]), artist_separators + ) except ValueError: pass artist = split_result[0] title = split_result[1] - return ParseData(artist, title) + return ParseData(artist, title, artist_separators) -def get_tags(filepath: str): +def get_tags(filepath: str, artist_separators: set[str]): """ Returns the tags for a given audio file. """ @@ -150,7 +154,7 @@ def get_tags(filepath: str): return None try: - tags = TinyTag.get(filepath) + tags: Any = TinyTag.get(filepath) except: # noqa: E722 return None @@ -169,7 +173,7 @@ def get_tags(filepath: str): for tag in to_filename: p = getattr(tags, tag) if p == "" or p is None: - parse_data = extract_artist_title(filename) + parse_data = extract_artist_title(filename, artist_separators) title = parse_data.title setattr(tags, tag, title) @@ -179,7 +183,7 @@ def get_tags(filepath: str): if p == "" or p is None: if not parse_data: - parse_data = extract_artist_title(filename) + parse_data = extract_artist_title(filename, artist_separators) artist = parse_data.artist @@ -225,8 +229,8 @@ def get_tags(filepath: str): tags.artists = tags.artist tags.albumartists = tags.albumartist - split_artist = split_artists(tags.artist) - split_albumartists = split_artists(tags.albumartist) + split_artist = split_artists(tags.artist, separators=artist_separators) + split_albumartists = split_artists(tags.albumartist, separators=artist_separators) new_title = tags.title # TODO: Figure out which is the best spot to create these hashes @@ -237,7 +241,9 @@ def get_tags(filepath: str): # extract featured artists if config.extractFeaturedArtists: - feat, new_title = parse_feat_from_title(tags.title) + feat, new_title = parse_feat_from_title( + tags.title, separators=artist_separators + ) original_lower = "-".join([create_hash(a) for a in split_artist]) split_artist.extend(a for a in feat if create_hash(a) not in original_lower) @@ -262,8 +268,9 @@ def get_tags(filepath: str): for a in split_albumartists ] - tags.artisthashes = list({a["artisthash"] for a in tags.artists + tags.albumartists}) - + tags.artisthashes = list( + {a["artisthash"] for a in tags.artists + tags.albumartists} + ) # remove prod by if config.removeProdBy: @@ -295,26 +302,32 @@ def get_tags(filepath: str): # process genres if tags.genre: - tags.genre = tags.genre.lower() + src_genres: str = tags.genre + src_genres = src_genres.lower() # separators = {"/", ";", "&"} separators = set(config.genreSeparators) - contains_rnb = "r&b" in tags.genre - contains_rock = "rock & roll" in tags.genre + contains_rnb = "r&b" in src_genres + contains_rock = "rock & roll" in src_genres if contains_rnb: - tags.genre = tags.genre.replace("r&b", "RnB") + src_genres = src_genres.replace("r&b", "RnB") if contains_rock: - tags.genre = tags.genre.replace("rock & roll", "rock") + src_genres = src_genres.replace("rock & roll", "rock") for s in separators: - tags.genre = tags.genre.replace(s, ",") + src_genres = src_genres.replace(s, ",") - tags.genre = tags.genre.split(",") - tags.genre = [ - {"name": g.strip(), "genrehash": create_hash(g.strip())} for g in tags.genre + genres_list: list[str] = src_genres.split(",") + tags.genres = [ + {"name": g.strip(), "genrehash": create_hash(g.strip())} + for g in genres_list ] + tags.genrehashes = [g["genrehash"] for g in tags.genres] + else: + tags.genres = [] + tags.genrehashes = [] # sub underscore with space tags.title = tags.title.replace("_", " ") @@ -333,6 +346,10 @@ def get_tags(filepath: str): "filesize": tags.filesize, "samplerate": tags.samplerate, "track_total": tags.track_total, + "hashinfo": { + "algo": "sha1", + "format": "[:5]+[-5:]", # first 5 + last 5 chars + }, } tags.extra = {**tags.extra, **more_extra} @@ -357,6 +374,7 @@ def get_tags(filepath: str): "bitdepth", "artist", "albumartist", + "genre", ] for tag in to_delete: diff --git a/app/lib/trackslib.py b/app/lib/trackslib.py index 085dda0c..c8a7a74e 100644 --- a/app/lib/trackslib.py +++ b/app/lib/trackslib.py @@ -13,16 +13,6 @@ from app.utils.progressbar import tqdm from app.utils.threading import ThreadWithReturnValue -def validate_tracks() -> None: - """ - Removes track records whose files no longer exist. - """ - for track in tqdm(TrackStore.tracks, desc="Validating tracks"): - if not os.path.exists(track.filepath): - TrackStore.remove_track_obj(track) - trackdb.remove_tracks_by_filepaths(track.filepath) - - def get_leading_silence_end(filepath: str): """ Returns the leading silence of a track. diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py index 0ba670c2..8d7b9955 100644 --- a/app/lib/watchdogg.py +++ b/app/lib/watchdogg.py @@ -11,8 +11,10 @@ from watchdog.events import PatternMatchingEventHandler from watchdog.observers import Observer from app import settings +from app.config import UserConfig from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb -from app.db.sqlite.settings import SettingsSQLMethods as sdb + +# from app.db.sqlite.settings import SettingsSQLMethods as sdb from app.db.sqlite.tracks import SQLiteManager from app.db.sqlite.tracks import SQLiteTrackMethods as db from app.lib.colorlib import process_color @@ -43,7 +45,8 @@ class Watcher: while trials < 10: try: - dirs = sdb.get_root_dirs() + # dirs = sdb.get_root_dirs() + dirs = UserConfig().rootDirs dirs = [rf"{d}" for d in dirs] dir_map = [ @@ -152,7 +155,8 @@ def add_track(filepath: str) -> None: TrackStore.remove_track_by_filepath(filepath) - tags = get_tags(filepath) + config = UserConfig() + tags = get_tags(filepath, artist_separators=config.artistSeparators) # if the track is somehow invalid, return if tags is None or tags["bitrate"] == 0 or tags["duration"] == 0: diff --git a/app/migrations/__init__.py b/app/migrations/__init__.py index 570cc115..54b51bf5 100644 --- a/app/migrations/__init__.py +++ b/app/migrations/__init__.py @@ -7,7 +7,9 @@ Reads and applies the latest database migrations. import inspect import sys from types import ModuleType -from app.db.sqlite.migrations import MigrationManager + +# from app.db.sqlite.migrations import MigrationManager +from app.db.metadata import MigrationTable from app.logger import log from app.migrations import v1_3_0, v1_4_9 from app.migrations.base import Migration @@ -42,26 +44,29 @@ def apply_migrations(): modules = [v1_3_0, v1_4_9] migrations = [get_all_migrations(m) for m in modules] - index = MigrationManager.get_index() + # index = MigrationManager.get_index() + index = MigrationTable.get_version() all_migrations = [migration for sublist in migrations for migration in sublist] to_apply: list[Migration] = [] # if index is from old release, # get migrations from the "migrations" list - if index < 3: - _migrations = migrations[index:] - to_apply = [migration for sublist in _migrations for migration in sublist] - else: - to_apply = all_migrations[index:] + + # if index < 3: + # _migrations = migrations[index:] + # to_apply = [migration for sublist in _migrations for migration in sublist] + # else: + # to_apply = all_migrations[index:] - for migration in to_apply: - # try: - migration.migrate() - log.info("Applied migration: %s", migration.__name__) - # except Exception as e: - # log.error("Failed to run migration: %s", migration.__name__) - # log.error(e) + # for migration in to_apply: + # # try: + # migration.migrate() + # log.info("Applied migration: %s", migration.__name__) + # except Exception as e: + # log.error("Failed to run migration: %s", migration.__name__) + # log.error(e) # sys.exit(0) - MigrationManager.set_index(len(all_migrations)) + # MigrationManager.set_index(len(all_migrations)) + MigrationTable.set_version(len(all_migrations)) diff --git a/app/models/album.py b/app/models/album.py index 337f24cf..aa843e43 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -2,9 +2,6 @@ import dataclasses import datetime from dataclasses import dataclass -from app.config import UserConfig -from app.settings import SessionVarKeys, get_flag - from ..utils.hashing import create_hash from ..utils.parsers import get_base_title_and_versions, parse_feat_from_title from .artist import Artist @@ -27,15 +24,21 @@ class Album: date: int duration: int genres: list[dict[str, str]] + genrehashes: list[str] og_title: str title: str trackcount: int + is_favorite: bool + extra: dict type: str = "album" + image: str = "" versions: list[str] = dataclasses.field(default_factory=list) def __post_init__(self): - self.date = datetime.datetime.fromtimestamp(self.date).year + # self.date = datetime.datetime.fromtimestamp(self.date).year + self.image = self.albumhash + ".webp" + self.populate_versions() # albumhash: str # title: str diff --git a/app/models/artist.py b/app/models/artist.py index 0c50a60d..1ef9aa26 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -44,6 +44,13 @@ class Artist: date: int duration: int genres: list[dict[str, str]] + genrehashes: list[str] name: str trackcount: int is_favorite: bool + extra: dict + + image: str = "" + + def __post_init__(self): + self.image = self.artisthash + ".webp" \ No newline at end of file diff --git a/app/models/favorite.py b/app/models/favorite.py new file mode 100644 index 00000000..ab02d593 --- /dev/null +++ b/app/models/favorite.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import Any, Literal + + +@dataclass +class Favorite: + hash: str + type: Literal["album", "track", "artist"] + timestamp: int + userid: int + extra: dict[str, Any] \ No newline at end of file diff --git a/app/models/lastfm.py b/app/models/lastfm.py index ebd2333c..573ab4fd 100644 --- a/app/models/lastfm.py +++ b/app/models/lastfm.py @@ -1,13 +1,28 @@ from dataclasses import dataclass +from typing import Any + + +@dataclass +class SimilarArtistEntry: + artisthash: str + name: str + weight: float + scrobbles: int + listeners: int @dataclass class SimilarArtist: artisthash: str - similar_artist_hashes: str + similar_artists: list[SimilarArtistEntry] + def get_artist_hash_set(self) -> set[str]: """ Returns a set of similar artists. """ - return set(self.similar_artist_hashes.split("~")) + if not self.similar_artists: + return set() + + # INFO: + return set(a['artisthash'] for a in self.similar_artists) diff --git a/app/models/plugins.py b/app/models/plugins.py index d3b55f88..fac7f6ec 100644 --- a/app/models/plugins.py +++ b/app/models/plugins.py @@ -4,7 +4,7 @@ from dataclasses import dataclass @dataclass class Plugin: name: str - description: str active: bool settings: dict + extra: dict diff --git a/app/models/track.py b/app/models/track.py index 0a853b3e..140df3cd 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -1,20 +1,4 @@ -from dataclasses import dataclass, field -import os -from pathlib import Path - -from flask_jwt_extended import current_user - -from app.settings import SessionVarKeys, get_flag -from app.utils.hashing import create_hash -from app.utils.parsers import ( - clean_title, - get_base_title_and_versions, - parse_feat_from_title, - remove_prod, - split_artists, -) - -from .artist import ArtistMinimal +from dataclasses import dataclass @dataclass(slots=True) @@ -28,7 +12,7 @@ class Track: albumartists: list[dict[str, str]] albumhash: str artisthashes: list[str] - artists: str + artists: list[dict[str, str]] bitrate: int copyright: str date: int @@ -36,7 +20,8 @@ class Track: duration: int filepath: str folder: str - genre: list[dict[str, str]] + genres: list[dict[str, str]] + genrehashes: list[str] last_mod: int og_album: str og_title: str @@ -48,7 +33,15 @@ class Track: is_favorite: bool = False _pos: int = 0 _ati: str = "" + image: str = "" + def __post_init__(self): + self.image = self.albumhash + ".webp" + self.extra = { + "disc_total": self.extra.get("disc_total", 0), + "track_total": self.extra.get("track_total", 0), + "samplerate": self.extra.get("samplerate", -1), + } # album: str # albumartists: str | list[ArtistMinimal] diff --git a/app/models/user.py b/app/models/user.py index b8d05d01..3aca9b95 100644 --- a/app/models/user.py +++ b/app/models/user.py @@ -5,19 +5,15 @@ import json @dataclass(slots=True) class User: id: int - username: str - firstname: str - lastname: str - password: str - email: str image: str + password: str + username: str + roles: list[str] + extra: dict[str, str] = field(default_factory=dict) # NOTE: roles: ['admin', 'user', 'curator'] roles: list[str] = field(default_factory=lambda: ["user"]) - def __post_init__(self): - self.roles = json.loads(self.roles) - def todict(self): this_dict = asdict(self) del this_dict["password"] @@ -28,5 +24,5 @@ class User: return { "id": self.id, "username": self.username, - "firstname": self.firstname, + "firstname": self.extra["firstname"] if self.extra else "", } diff --git a/app/periodic_scan.py b/app/periodic_scan.py index 7a0af7e3..f8b19133 100644 --- a/app/periodic_scan.py +++ b/app/periodic_scan.py @@ -1,14 +1,16 @@ """ This module contains functions for the server """ + import time +from app.config import UserConfig from app.lib.populate import Populate, PopulateCancelledError -from app.settings import SessionVarKeys, get_flag, get_scan_sleep_time from app.utils.generators import get_random_str from app.utils.threading import background from app.logger import log + @background def run_periodic_scans(): """ @@ -23,10 +25,7 @@ def run_periodic_scans(): # ValidateAlbumThumbs() # ValidatePlaylistThumbs() - run_periodic_scan = True - - while run_periodic_scan: - run_periodic_scan = get_flag(SessionVarKeys.DO_PERIODIC_SCANS) + while UserConfig().enablePeriodicScans: try: Populate(instance_key=get_random_str()) @@ -34,5 +33,4 @@ def run_periodic_scans(): log.error("'run_periodic_scans': Periodic scan cancelled.") pass - sleep_time = get_scan_sleep_time() - time.sleep(sleep_time) + time.sleep(UserConfig().scanInterval) diff --git a/app/plugins/register.py b/app/plugins/register.py index 7ace2260..b809722c 100644 --- a/app/plugins/register.py +++ b/app/plugins/register.py @@ -1,5 +1,18 @@ -from app.db.sqlite.plugins import PluginsMethods +from app.db.userdata import PluginTable +from sqlalchemy.exc import IntegrityError def register_plugins(): - PluginsMethods.insert_lyrics_plugin() + try: + PluginTable.insert_one( + { + "name": "lyrics_finder", + "active": False, + "settings": {"auto_download": False}, + "extra": { + "description": "Find lyrics from the internet", + }, + } + ) + except IntegrityError: + pass diff --git a/app/requests/artists.py b/app/requests/artists.py index 05e26336..784c9167 100644 --- a/app/requests/artists.py +++ b/app/requests/artists.py @@ -7,6 +7,7 @@ import urllib.parse import requests from requests import ConnectionError, HTTPError, ReadTimeout +from app.models.lastfm import SimilarArtistEntry from app.utils.hashing import create_hash @@ -20,7 +21,7 @@ def fetch_similar_artists(name: str): response = requests.get(url, timeout=10) response.raise_for_status() except (ConnectionError, ReadTimeout, HTTPError): - return [] + return None data = response.json() @@ -29,5 +30,15 @@ def fetch_similar_artists(name: str): except KeyError: return [] - for artist in artists: - yield create_hash(artist["name"]) + return [ + SimilarArtistEntry( + **{ + "artisthash": create_hash(artist["name"]), + "name": artist["name"], + "weight": artist["weight"], + "listeners": int(artist["listeners"]), + "scrobbles": int(artist["scrobbles"]), + } + ) + for artist in artists + ] diff --git a/app/settings.py b/app/settings.py index c3c308fd..d8a7036f 100644 --- a/app/settings.py +++ b/app/settings.py @@ -244,17 +244,6 @@ class SessionVarKeys: SHOW_ALBUMS_AS_SINGLES = "SHOW_ALBUMS_AS_SINGLES" -def get_flag(key: SessionVarKeys) -> bool: - return getattr(SessionVars, key) - - -def set_flag(key: SessionVarKeys, value: Any): - setattr(SessionVars, key, value) - - -def get_scan_sleep_time() -> int: - return SessionVars.PERIODIC_SCAN_INTERVAL - class TCOLOR: """ diff --git a/app/setup/__init__.py b/app/setup/__init__.py index c3d37f61..60738d68 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -3,11 +3,11 @@ Prepares the server for use. """ import uuid -from app.db.sqlite.settings import load_settings from app.setup.files import create_config_dir from app.setup.sqlite import run_migrations, setup_sqlite from app.store.albums import AlbumStore from app.store.artists import ArtistStore +from app.store.folder import FolderStore from app.store.tracks import TrackStore from app.utils.generators import get_random_str from app.config import UserConfig @@ -29,20 +29,21 @@ def run_setup(): setup_sqlite() run_migrations() - try: - load_settings() - except IndexError: - # settings table is empty - pass + # try: + # load_settings() + # except IndexError: + # # settings table is empty + # pass def load_into_mem(): """ Load all tracks, albums, and artists into memory. """ - instance_key = get_random_str() + # instance_key = get_random_str() # INFO: Load all tracks, albums, and artists into memory - TrackStore.load_all_tracks(instance_key) - AlbumStore.load_albums(instance_key) - ArtistStore.load_artists(instance_key) + # TrackStore.load_all_tracks(instance_key) + # AlbumStore.load_albums(instance_key) + # ArtistStore.load_artists(instance_key) + FolderStore.load_filepaths() \ No newline at end of file diff --git a/app/setup/sqlite.py b/app/setup/sqlite.py index 667d2e4b..8ce8dbe5 100644 --- a/app/setup/sqlite.py +++ b/app/setup/sqlite.py @@ -3,11 +3,15 @@ Module to setup Sqlite databases and tables. Applies migrations. """ +from app.db.userdata import UserTable from app.db.sqlite import create_connection, create_tables, queries from app.db.sqlite.auth import SQLiteAuthMethods as authdb from app.migrations import apply_migrations from app.settings import Db +from app.db import create_all +from app.db.libdata import create_all as create_all_libdata + def run_migrations(): """ @@ -20,18 +24,8 @@ def setup_sqlite(): """ Create Sqlite databases and tables. """ - # if os.path.exists(DB_PATH): - # os.remove(DB_PATH) + create_all() + create_all_libdata() - app_db_conn = create_connection(Db.get_app_db_path()) - user_db_conn = create_connection(Db.get_userdata_db_path()) - - create_tables(app_db_conn, queries.CREATE_APPDB_TABLES) - create_tables(user_db_conn, queries.CREATE_USERDATA_TABLES) - create_tables(app_db_conn, queries.CREATE_MIGRATIONS_TABLE) - - if not authdb.get_all_users(): - authdb.insert_default_user() - - app_db_conn.close() - user_db_conn.close() + if not UserTable.get_all(): + UserTable.insert_default_user() diff --git a/app/store/artists.py b/app/store/artists.py index 84bcb242..82cbf976 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -1,7 +1,6 @@ import json from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb -from app.lib.artistlib import get_all_artists from app.models import Artist from app.utils.bisection import use_bisection from app.utils.customlist import CustomList diff --git a/app/store/folder.py b/app/store/folder.py new file mode 100644 index 00000000..0bfe3ae3 --- /dev/null +++ b/app/store/folder.py @@ -0,0 +1,95 @@ +from sortedcontainers import SortedSet +from concurrent.futures import ThreadPoolExecutor + +from app.db.libdata import TrackTable + + +class FolderStore: + """ + The Folder store is used to hold all the indexed tracks filepaths in memory + for fast count operations when browsing the folder page. + + Counting from the database is super slow, even with a small number of folders to get the count for. Up to 700ms for 10 folders. By using this store, we are able to reduce that to less than 10ms. + """ + + filepaths: SortedSet = SortedSet() + + @classmethod + def load_filepaths(cls): + """ + Load all the filepaths from the database into memory. + + This is needed to speed up the process of counting the number of tracks in the folder page. + """ + cls.filepaths.clear() + + tracks = TrackTable.get_all() + for track in tracks: + cls.filepaths.add(track.filepath) + + + @classmethod + def count_tracks_containing_paths(cls, paths: list[str]): + """ + Count the number of tracks in each directory. + + Uses a ThreadPoolExecutor to count the number of tracks + in each directory for fast execution time. + """ + results: list[dict[str, int | str]] = [] + + with ThreadPoolExecutor() as executor: + res = executor.map(countFilepathsInDir, paths) + results = [ + {"path": path, "trackcount": count} for path, count in zip(paths, res) + ] + + return results + + +def getIndexOfFirstMatch(strings: list[str], prefix: str): + """ + Find the index of the first path that starts with the given path. + + Uses a binary search algorithm to find the index. + """ + + left = 0 + right = len(strings) - 1 + + while left <= right: + mid = (left + right) // 2 + + if strings[mid].startswith(prefix): + if mid == 0 or not strings[mid - 1].startswith(prefix): + return mid + right = mid - 1 + elif strings[mid] < prefix: + left = mid + 1 + else: + right = mid - 1 + + return -1 + + +def countFilepathsInDir(dirpath: str): + """ + Counts the number of filepaths that start with the given directory path. + + Gets the index of the first path that starts with the given directory path, + then checks each path after that to see if it starts with the given directory path. + """ + index = getIndexOfFirstMatch(FolderStore.filepaths, dirpath) + + if index == -1: + return 0 + + paths: list[str] = [] + + for path in FolderStore.filepaths[index:]: + if path.startswith(dirpath): + paths.append(path) + else: + break + + return len(paths) diff --git a/app/utils/dates.py b/app/utils/dates.py index f5495a2d..ebca97f1 100644 --- a/app/utils/dates.py +++ b/app/utils/dates.py @@ -26,7 +26,7 @@ def create_new_date(date: datetime = None) -> str: return date.strftime(_format) -def timestamp_to_time_passed(timestamp: str): +def timestamp_to_time_passed(timestamp: str | int): """ Converts a timestamp to time passed. e.g. 2 minutes ago, 1 hour ago, yesterday, 2 days ago, 2 weeks ago, etc. """ diff --git a/app/utils/hashing.py b/app/utils/hashing.py index 7e4f1b27..98a883ea 100644 --- a/app/utils/hashing.py +++ b/app/utils/hashing.py @@ -34,6 +34,7 @@ def create_hash(*args: str, decode=False, limit=10) -> str: str_ = str_.encode("utf-8") str_ = hashlib.sha1(str_).hexdigest() + # INFO: Return first 5 + last 5 characters return ( str_[: limit // 2] + str_[-limit // 2 :] if limit % 2 == 0 diff --git a/app/utils/parsers.py b/app/utils/parsers.py index 129b7835..f45c871f 100644 --- a/app/utils/parsers.py +++ b/app/utils/parsers.py @@ -1,14 +1,12 @@ import re from app.enums.album_versions import AlbumVersionEnum, get_all_keywords -from app.settings import SessionVarKeys, get_flag -def split_artists(src: str): +def split_artists(src: str, separators: set[str]): """ Splits a string of artists into a list of artists. """ - separators: set = get_flag(SessionVarKeys.ARTIST_SEPARATORS) for sep in separators: src = src.replace(sep, ",") @@ -38,7 +36,7 @@ def remove_prod(title: str) -> str: return title.strip() -def parse_feat_from_title(title: str) -> tuple[list[str], str]: +def parse_feat_from_title(title: str, separators: set[str]) -> tuple[list[str], str]: """ Extracts featured artists from a song title using regex. """ @@ -56,7 +54,7 @@ def parse_feat_from_title(title: str) -> tuple[list[str], str]: return [], title artists = match.group(1) - artists = split_artists(artists) + artists = split_artists(artists, separators) # remove "feat" group from title new_title = re.sub(regex, "", title, flags=re.IGNORECASE) diff --git a/manage.py b/manage.py index 20eda458..e0baabf2 100644 --- a/manage.py +++ b/manage.py @@ -45,7 +45,7 @@ mimetypes.add_type("image/gif", ".gif") mimetypes.add_type("font/woff", ".woff") mimetypes.add_type("application/manifest+json", ".webmanifest") -logging.disable(logging.CRITICAL) +# logging.disable(logging.CRITICAL) # werkzeug = logging.getLogger("werkzeug") # werkzeug.setLevel(logging.ERROR) diff --git a/poetry.lock b/poetry.lock index 914b502c..10c11ade 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2616,4 +2616,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "9c7ba20671a6a3b59dbb120e3e56ded7e4dfcbf2de14418bdef41059233cdcb1" +content-hash = "80cb2755efc6cec2cb20d50cb8927dee554991741283e70e7a2665e6253b895d" diff --git a/pyproject.toml b/pyproject.toml index de3dece2..65574f5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ flask-openapi3 = "^3.0.2" flask-jwt-extended = "^4.6.0" sqlalchemy = "^2.0.31" memory-profiler = "^0.61.0" +sortedcontainers = "^2.4.0" [tool.poetry.dev-dependencies] pylint = "^2.15.5" From b9ad07441a308b78f8eda45d9f23193c59e892f3 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 30 Jun 2024 19:33:13 +0300 Subject: [PATCH 06/44] store playcount and duration on the track table + allow sorting all items with those two + add methods to update scrobble info --- TODO.md | 4 +- app/api/__init__.py | 4 +- app/api/getall/__init__.py | 26 +++++++++- app/api/{logger => scrobble}/__init__.py | 22 ++++---- app/db/__init__.py | 2 +- app/db/libdata.py | 64 ++++++++++++++++++++++-- app/db/userdata.py | 29 +++++++++-- app/lib/tagger.py | 41 ++++++++------- app/models/album.py | 3 ++ app/models/artist.py | 3 ++ app/models/track.py | 3 ++ app/serializers/album.py | 1 + app/serializers/artist.py | 1 + app/serializers/track.py | 1 + app/utils/dates.py | 2 +- 15 files changed, 161 insertions(+), 45 deletions(-) rename app/api/{logger => scrobble}/__init__.py (64%) diff --git a/TODO.md b/TODO.md index e1ed9eee..dc6d5216 100644 --- a/TODO.md +++ b/TODO.md @@ -44,4 +44,6 @@ - Move plugins to a config file - What about our migrations? - Add userid to queries -- Remove duplicates on artist page (test with Hanson) \ No newline at end of file +- Remove duplicates on artist page (test with Hanson) +- Test foreign keys on delete +- Map scrobble info on app start diff --git a/app/api/__init__.py b/app/api/__init__.py index b9eacf49..73a0c663 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -26,7 +26,7 @@ from app.api import ( settings, lyrics, plugins, - logger, + scrobble, home, getall, auth, @@ -116,7 +116,7 @@ def create_api(): app.register_api(lyrics_plugin.api) # Logger - app.register_api(logger.api) + app.register_api(scrobble.api) # Home app.register_api(home.api) diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index f1fe2a35..5489803a 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -18,6 +18,7 @@ from app.utils.dates import ( create_new_date, date_string_to_time_passed, seconds_to_time_string, + timestamp_to_time_passed, ) bp_tag = Tag(name="Get all", description="List all items") @@ -57,6 +58,13 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): Get all items Used to show all albums or artists in the library + + Sort keys: + - + Both albums and artists: `duration`, `created_date`, `playcount`, `playduration`, `lastplayed`, `trackcount` + + Albums only: `title`, `albumartists`, `date` + Artists only: `name`, `albumcount` """ is_albums = path.itemtype == "albums" is_artists = path.itemtype == "artists" @@ -76,6 +84,9 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): sort_is_count = sort == "trackcount" sort_is_duration = sort == "duration" sort_is_create_date = sort == "created_date" + sort_is_playcount = sort == "playcount" + sort_is_playduration = sort == "playduration" + sort_is_lastplayed = sort == "lastplayed" sort_is_date = is_albums and sort == "date" sort_is_artist = is_albums and sort == "albumartists" @@ -94,7 +105,6 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): for item in items: item_dict = serialize_album(item) if is_albums else serialize_artist(item) - print(item_dict) if sort_is_date: item_dict["help_text"] = item.date @@ -122,6 +132,20 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): f"{format_number(item.albumcount)} album{'' if item.albumcount == 1 else 's'}" ) + if sort_is_playcount: + item_dict["help_text"] = ( + f"{format_number(item.playcount)} play{'' if item.playcount == 1 else 's'}" + ) + + if sort_is_lastplayed: + if item.playduration == 0: + item_dict["help_text"] = "Never played" + else: + item_dict["help_text"] = timestamp_to_time_passed(item.lastplayed) + + if sort_is_playduration: + item_dict["help_text"] = seconds_to_time_string(item.playduration) + album_list.append(item_dict) return {"items": album_list, "total": total} diff --git a/app/api/logger/__init__.py b/app/api/scrobble/__init__.py similarity index 64% rename from app/api/logger/__init__.py rename to app/api/scrobble/__init__.py index 40fd4db7..19137a0b 100644 --- a/app/api/logger/__init__.py +++ b/app/api/scrobble/__init__.py @@ -3,7 +3,8 @@ from flask_openapi3 import APIBlueprint from pydantic import Field from app.api.apischemas import TrackHashSchema -from app.db.sqlite.logger.tracks import SQLiteTrackLogger as db +from app.db.libdata import AlbumTable, ArtistTable, TrackTable +from app.db.userdata import ScrobbleTable from app.settings import Defaults bp_tag = Tag(name="Logger", description="Log item plays") @@ -26,19 +27,20 @@ def log_track(body: LogTrackBody): """ Log a track play to the database. """ - trackhash = body.trackhash timestamp = body.timestamp duration = body.duration - source = body.source if not timestamp or duration < 5: return {"msg": "Invalid entry."}, 400 - last_row = db.insert_track( - trackhash=trackhash, - timestamp=timestamp, - duration=duration, - source=source, - ) + track = TrackTable.get_track_by_trackhash(body.trackhash) - return {"total entries": last_row} + if track is None: + return {"msg": "Track not found."}, 404 + + ScrobbleTable.add(dict(body)) + TrackTable.increment_playcount(body.trackhash, duration, timestamp) + AlbumTable.increment_playcount(track.albumhash, duration, timestamp) + ArtistTable.increment_playcount(track.artisthashes, duration, timestamp) + + return {"msg": "recorded"}, 201 diff --git a/app/db/__init__.py b/app/db/__init__.py index ebc6d776..bd857356 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -48,7 +48,7 @@ class DbManager: if self.commit: self.conn.commit() - self.conn.close() + # self.conn.close() class Base(MappedAsDataclass, DeclarativeBase): diff --git a/app/db/libdata.py b/app/db/libdata.py index 9d3ebbd6..9748fe3f 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -73,6 +73,21 @@ class Base(MasterBase, DeclarativeBase): conn.execute(stmt) + @classmethod + def increment_scrobblecount( + cls, table: Any, field: Any, hash: str, duration: int, timestamp: int + ): + cls.execute( + update(table) + .where(field == hash) + .values( + playcount=table.playcount + 1, + playduration=table.playduration + duration, + lastplayed=timestamp, + ), + commit=True, + ) + class TrackTable(Base): __tablename__ = "track" @@ -99,8 +114,12 @@ class TrackTable(Base): track: Mapped[int] = mapped_column(Integer()) trackhash: Mapped[str] = mapped_column(String(), index=True) is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) - playcount: Mapped[int] = mapped_column(Integer()) - extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + lastplayed: Mapped[int] = mapped_column(Integer(), default=0) + playcount: Mapped[int] = mapped_column(Integer(), default=0) + playduration: Mapped[int] = mapped_column(Integer(), default=0) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSON(), default_factory=dict + ) @classmethod def get_all(cls): @@ -180,6 +199,12 @@ class TrackTable(Base): with DbManager(commit=True) as conn: conn.execute(delete(TrackTable).where(TrackTable.filepath.in_(filepaths))) + @classmethod + def increment_playcount(cls, trackhash: str, duration: int, timestamp: int): + cls.increment_scrobblecount( + TrackTable, TrackTable.trackhash, trackhash, duration, timestamp + ) + class AlbumTable(Base): __tablename__ = "album" @@ -199,7 +224,12 @@ class AlbumTable(Base): title: Mapped[str] = mapped_column(String()) trackcount: Mapped[int] = mapped_column(Integer()) is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) - extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + lastplayed: Mapped[int] = mapped_column(Integer(), default=0) + playcount: Mapped[int] = mapped_column(Integer(), default=0) + playduration: Mapped[int] = mapped_column(Integer(), default=0) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSON(), default_factory=dict + ) @classmethod def get_all(cls): @@ -257,6 +287,12 @@ class AlbumTable(Base): ) return albums_to_dataclasses(result.all()) + @classmethod + def increment_playcount(cls, albumhash: str, duration: int, timestamp: int): + return cls.increment_scrobblecount( + AlbumTable, AlbumTable.albumhash, albumhash, duration, timestamp + ) + class ArtistTable(Base): __tablename__ = "artist" @@ -272,7 +308,12 @@ class ArtistTable(Base): name: Mapped[str] = mapped_column(String(), index=True) trackcount: Mapped[int] = mapped_column(Integer()) is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) - extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + lastplayed: Mapped[int] = mapped_column(Integer(), default=0) + playcount: Mapped[int] = mapped_column(Integer(), default=0) + playduration: Mapped[int] = mapped_column(Integer(), default=0) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSON(), default_factory=dict + ) @classmethod def get_all(cls): @@ -310,3 +351,18 @@ class ArtistTable(Base): .limit(limit) ) return artists_to_dataclasses(result.fetchall()) + + @classmethod + def increment_playcount( + cls, artisthashes: list[str], duration: int, timestamp: int + ): + cls.execute( + update(cls) + .where(ArtistTable.artisthash.in_(artisthashes)) + .values( + playcount=ArtistTable.playcount + 1, + playduration=ArtistTable.playduration + duration, + lastplayed=timestamp, + ), + commit=True, + ) diff --git a/app/db/userdata.py b/app/db/userdata.py index ce2b6708..b57cd73f 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -31,7 +31,7 @@ from app.db.utils import ( ) from app.db import Base, DbManager -from app.utils.auth import hash_password +from app.utils.auth import get_current_userid, hash_password class UserTable(Base): @@ -160,7 +160,7 @@ class FavoritesTable(Base): type: Mapped[str] = mapped_column(String(), index=True) timestamp: Mapped[int] = mapped_column(Integer(), index=True) userid: Mapped[int] = mapped_column( - Integer(), ForeignKey("user.id"), default=1, index=True + Integer(), ForeignKey("user.id", ondelete="cascade"), default=1, index=True ) extra: Mapped[dict[str, Any]] = mapped_column( JSON(), nullable=True, default_factory=dict @@ -175,7 +175,7 @@ class FavoritesTable(Base): @classmethod def insert_item(cls, item: dict[str, Any]): item["timestamp"] = int(datetime.datetime.now().timestamp()) - item["userid"] = current_user["id"] + item["userid"] = get_current_userid() with DbManager(commit=True) as conn: conn.execute(insert(cls).values(item)) @@ -199,7 +199,7 @@ class FavoritesTable(Base): result = cls.execute( select(table) .select_from(join(table, cls, field == cls.hash)) - .where(and_(cls.type == type, cls.userid == current_user["id"])) + .where(and_(cls.type == type, cls.userid == get_current_userid())) .offset(start) # INFO: If start is 0, fetch all so we can get the total count .limit(limit if start != 0 else None) @@ -238,3 +238,24 @@ class FavoritesTable(Base): ArtistTable, ArtistTable.artisthash, "artist", start, limit ) return artists_to_dataclasses(result), total + + +class ScrobbleTable(Base): + __tablename__ = "scrobble" + + id: Mapped[int] = mapped_column(primary_key=True) + trackhash: Mapped[str] = mapped_column(String(), index=True) + duration: Mapped[int] = mapped_column(Integer()) + timestamp: Mapped[int] = mapped_column(Integer()) + source: Mapped[str] = mapped_column(String()) + userid: Mapped[int] = mapped_column( + Integer(), ForeignKey("user.id", ondelete="cascade"), index=True + ) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) + + @classmethod + def add(cls, item: dict[str, Any]): + item["userid"] = get_current_userid() + return cls.insert_one(item) diff --git a/app/lib/tagger.py b/app/lib/tagger.py index d8a679a6..2bb7f660 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -156,32 +156,32 @@ class IndexAlbums: "albumhash": track.albumhash, "base_title": None, "color": None, - "created_date": None, - "date": None, + "created_date": track.last_mod, + "date": track.date, "duration": track.duration, "genres": [*track.genres] if track.genres else [], "og_title": track.og_album, + "lastplayed": track.lastplayed, + "playcount": track.playcount, + "playduration": track.playduration, "title": track.album, "trackcount": 1, - "dates": [track.date], - "created_dates": [track.last_mod], } else: album = albums[track.albumhash] album["trackcount"] += 1 + album["playcount"] += track.playcount + album["playduration"] += track.playduration + album["lastplayed"] = max(album["lastplayed"], track.lastplayed) album["duration"] += track.duration - album["dates"].append(track.date) - album["created_dates"].append(track.last_mod) + album["date"] = min(album["date"], track.date) + album["created_date"] = min(album["created_date"], track.last_mod) if track.genres: album["genres"].extend(track.genres) for album in albums.values(): - album["date"] = min(album["dates"]) - album["created_date"] = min(album["created_dates"]) - genres = [] - for genre in album["genres"]: if genre not in genres: genres.append(genre) @@ -190,8 +190,6 @@ class IndexAlbums: album["base_title"], _ = get_base_album_title(album["og_title"]) del genres - del album["dates"] - del album["created_dates"] AlbumTable.remove_all() AlbumTable.insert_many(list(albums.values())) @@ -219,23 +217,28 @@ class IndexArtists: "albumcount": None, "albums": {track.albumhash}, "artisthash": thisartist["artisthash"], - "created_dates": [track.last_mod], - "dates": [track.date], - "date": None, + "created_date": track.last_mod, + "date": track.date, "duration": track.duration, "genres": track.genres if track.genres else [], "name": None, "names": {thisartist["name"]}, + "lastplayed": track.lastplayed, + "playcount": track.playcount, + "playduration": track.playduration, "trackcount": None, "tracks": {track.trackhash}, } else: artist = artists[thisartist["artisthash"]] artist["duration"] += track.duration + artist["playcount"] += track.playcount + artist["playduration"] += track.playduration artist["albums"].add(track.albumhash) artist["tracks"].add(track.trackhash) - artist["dates"].append(track.date) - artist["created_dates"].append(track.last_mod) + artist["date"] = min(artist["date"], track.date) + artist["lastplayed"] = max(artist["lastplayed"], track.lastplayed) + artist["created_date"] = min(artist["created_date"], track.last_mod) artist["names"].add(thisartist["name"]) if track.genres: @@ -244,8 +247,6 @@ class IndexArtists: for artist in artists.values(): artist["albumcount"] = len(artist["albums"]) artist["trackcount"] = len(artist["tracks"]) - artist["date"] = min(artist["dates"]) - artist["created_date"] = min(artist["created_dates"]) genres = [] @@ -260,8 +261,6 @@ class IndexArtists: del artist["names"] del artist["tracks"] del artist["albums"] - del artist["dates"] - del artist["created_dates"] # INFO: Delete local variables del genres diff --git a/app/models/album.py b/app/models/album.py index aa843e43..dfcb7137 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -29,6 +29,9 @@ class Album: title: str trackcount: int is_favorite: bool + lastplayed: int + playcount: int + playduration: int extra: dict type: str = "album" diff --git a/app/models/artist.py b/app/models/artist.py index 1ef9aa26..6f8d8002 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -48,6 +48,9 @@ class Artist: name: str trackcount: int is_favorite: bool + lastplayed: int + playcount: int + playduration: int extra: dict image: str = "" diff --git a/app/models/track.py b/app/models/track.py index 140df3cd..86dc2da1 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -29,6 +29,9 @@ class Track: track: int trackhash: str extra: dict + lastplayed: int + playcount: int + playduration: int is_favorite: bool = False _pos: int = 0 diff --git a/app/serializers/album.py b/app/serializers/album.py index ed3aebbf..2a559866 100644 --- a/app/serializers/album.py +++ b/app/serializers/album.py @@ -27,6 +27,7 @@ def serialize_for_card(album: Album): "og_title", "base_title", "genres", + "playcount" } return album_serializer(album, props_to_remove) diff --git a/app/serializers/artist.py b/app/serializers/artist.py index 82802586..633b0789 100644 --- a/app/serializers/artist.py +++ b/app/serializers/artist.py @@ -14,6 +14,7 @@ def serialize_for_card(artist: Artist): "trackcount", "duration", "albumcount", + "playcount", } for key in props_to_remove: diff --git a/app/serializers/track.py b/app/serializers/track.py index 062b5bd4..10426fbb 100644 --- a/app/serializers/track.py +++ b/app/serializers/track.py @@ -20,6 +20,7 @@ def serialize_track(track: Track, to_remove: set = {}, remove_disc=True) -> dict "artist_hashes", "created_date", "fav_userids", + "playcount", }.union(to_remove) if not remove_disc: diff --git a/app/utils/dates.py b/app/utils/dates.py index ebca97f1..82172946 100644 --- a/app/utils/dates.py +++ b/app/utils/dates.py @@ -63,4 +63,4 @@ def seconds_to_time_string(seconds): if minutes > 0: return f"{minutes} minute{'s' if minutes > 1 else ''}" - return f"{remaining_seconds} second{'s' if remaining_seconds > 1 else ''}" + return f"{remaining_seconds} second{'' if remaining_seconds == 1 else 's'}" From 5759521de0c5c1892a608d0744c9e82c637cbef6 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 30 Jun 2024 21:40:49 +0300 Subject: [PATCH 07/44] fix: recently added --- app/db/libdata.py | 21 +++++- app/lib/home/recentlyadded.py | 137 +++++++++++++++++++++------------- app/models/playlist.py | 4 +- app/utils/__init__.py | 9 +++ 4 files changed, 113 insertions(+), 58 deletions(-) diff --git a/app/db/libdata.py b/app/db/libdata.py index 9748fe3f..0708b7fc 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -30,14 +30,14 @@ def create_all(): class Base(MasterBase, DeclarativeBase): @classmethod - def get_all_hashes(cls): + def get_all_hashes(cls, create_date: int | None = None): with DbManager() as conn: if cls.__tablename__ == "track": - stmt = select(TrackTable.trackhash) + stmt = select(TrackTable.trackhash).where(cls.last_mod < create_date) elif cls.__tablename__ == "album": - stmt = select(AlbumTable.albumhash) + stmt = select(AlbumTable.albumhash).where(cls.created_date < create_date) elif cls.__tablename__ == "artist": - stmt = select(ArtistTable.artisthash) + stmt = select(ArtistTable.artisthash).where(cls.created_date < create_date) result = conn.execute(stmt) return {row[0] for row in result.fetchall()} @@ -194,6 +194,18 @@ class TrackTable(Base): ) return tracks_to_dataclasses(result.fetchall()) + @classmethod + def get_recently_added(cls, start: int, limit: int): + with DbManager() as conn: + result = conn.execute( + select(TrackTable) + .order_by(TrackTable.last_mod.desc()) + .offset(start) + .limit(limit) + ) + + return tracks_to_dataclasses(result.fetchall()) + @classmethod def remove_tracks_by_filepaths(cls, filepaths: set[str]): with DbManager(commit=True) as conn: @@ -238,6 +250,7 @@ class AlbumTable(Base): all = result.fetchall() return albums_to_dataclasses(all) + @classmethod def get_album_by_albumhash(cls, hash: str): with DbManager() as conn: diff --git a/app/lib/home/recentlyadded.py b/app/lib/home/recentlyadded.py index ce9c86cf..ba99c8d1 100644 --- a/app/lib/home/recentlyadded.py +++ b/app/lib/home/recentlyadded.py @@ -1,11 +1,15 @@ from datetime import datetime +from pprint import pprint +from time import time +from app.db.libdata import AlbumTable, ArtistTable, TrackTable from app.lib.playlistlib import get_first_4_images from app.models.playlist import Playlist from app.models.track import Track -from app.store.tracks import TrackStore -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore + +# from app.store.tracks import TrackStore +# from app.store.albums import AlbumStore +# from app.store.artists import ArtistStore from app.serializers.track import serialize_track from app.serializers.album import album_serializer @@ -13,7 +17,12 @@ from app.serializers.artist import serialize_for_card from itertools import groupby -from app.utils.dates import create_new_date, date_string_to_time_passed, timestamp_to_time_passed +from app.utils import flatten +from app.utils.dates import ( + create_new_date, + date_string_to_time_passed, + timestamp_to_time_passed, +) older_albums = set() older_artists = set() @@ -36,7 +45,7 @@ def check_is_album_folder(tracks: list[Track]): def check_is_artist_folder(tracks: list[Track]): # INFO: flatten artist hashes using "-" as a separator - artisthashes = "-".join(t.artist_hashes for t in tracks).split("-") + artisthashes = flatten([t.artisthashes for t in tracks]) return calc_based_on_percent(artisthashes, len(tracks)) @@ -48,27 +57,22 @@ def check_is_track_folder(tracks: list[Track]): return [create_track(t) for t in tracks] -def check_is_new_artist(artisthash: str, timestamp: int): - """ - Checks if an artist already exists in the library. - """ - tracks = filter( - lambda t: t.last_mod < timestamp and artisthash in t.artist_hashes, - TrackStore.tracks, - ) - - return next(tracks, None) is None +# def check_is_new_artist(hashes: set[str], artisthash: str, timestamp: int): +# """ +# Checks if an artist already exists in the library. +# """ +# return artisthash not in hashes -def check_is_new_album(albumhash: str, timestamp: int): - """ - Checks if an album already exists in the library. - """ - tracks = filter( - lambda t: t.last_mod < timestamp and t.albumhash == albumhash, TrackStore.tracks - ) +# def check_is_new_album(albumhash: str, timestamp: int): +# """ +# Checks if an album already exists in the library. +# """ +# tracks = filter( +# lambda t: t.last_mod < timestamp and t.albumhash == albumhash, TrackStore.tracks +# ) - return next(tracks, None) is None +# return next(tracks, None) is None def create_track(t: Track): @@ -88,13 +92,19 @@ def create_track(t: Track): group_type = dict[str, list[Track], float] -def check_folder_type(group_: group_type) -> str: +def check_folder_type(group_: group_type): # check if all tracks in group have the same albumhash # if so, return "album" key = group_["folder"] tracks = group_["tracks"] time = group_["time"] + print(f"Checking folder: {key}") + print(f"Tracks: {len(tracks)}") + + existing_artist_hashes: set[str] = set(ArtistTable.get_all_hashes(time)) + existing_album_hashes: set[str] = set(AlbumTable.get_all_hashes(time)) + if len(tracks) == 1: entry = create_track(tracks[0]) entry["item"]["time"] = timestamp_to_time_passed(time) @@ -102,7 +112,7 @@ def check_folder_type(group_: group_type) -> str: is_album, albumhash, _ = check_is_album_folder(tracks) if is_album: - album = AlbumStore.get_album_by_hash(albumhash) + album = AlbumTable.get_album_by_albumhash(albumhash) if album is None: return None @@ -120,7 +130,7 @@ def check_folder_type(group_: group_type) -> str: }, ) album["help_text"] = ( - "NEW ALBUM" if check_is_new_album(albumhash, time) else "NEW TRACKS" + "NEW ALBUM" if albumhash in existing_album_hashes else "NEW TRACKS" ) album["time"] = timestamp_to_time_passed(time) @@ -131,7 +141,7 @@ def check_folder_type(group_: group_type) -> str: is_artist, artisthash, trackcount = check_is_artist_folder(tracks) if is_artist: - artist = ArtistStore.get_artist_by_hash(artisthash) + artist = ArtistTable.get_artist_by_hash(artisthash) if artist is None: return None @@ -139,7 +149,7 @@ def check_folder_type(group_: group_type) -> str: artist = serialize_for_card(artist) artist["trackcount"] = trackcount artist["help_text"] = ( - "NEW ARTIST" if check_is_new_artist(artisthash, time) else "NEW MUSIC" + "NEW ARTIST" if artisthash not in existing_artist_hashes else "NEW MUSIC" ) artist["time"] = timestamp_to_time_passed(time) @@ -165,40 +175,62 @@ def check_folder_type(group_: group_type) -> str: ) -def group_track_by_folders(tracks: Track): +def group_track_by_folders(tracks: list[Track], groups: dict[str, list[Track]]): """ Groups tracks by folder and returns a list of groups sorted by last modified date. Uses generator expressions to avoid creating intermediate lists. """ - # INFO: sort tracks by folder name, then group by folder name tracks = sorted(tracks, key=lambda t: t.folder) - groups = groupby(tracks, lambda t: t.folder) + thisgroup = groupby(tracks, lambda t: t.folder) - # INFO: sort tracks by last modified date in descending order to get the most recent last modified date - groups = ( - (folder, sorted(tracks, key=lambda t: t.last_mod, reverse=True)) - for folder, tracks in groups - ) + for folder, thistracks in thisgroup: + groups.setdefault(folder, []).extend(thistracks) - # INFO: Return a generator of the groups - groups = ( - {"folder": folder, "tracks": list(tracks), "time": tracks[0].last_mod} - for folder, tracks in groups - ) - - # sort groups by last modified date - return sorted(groups, key=lambda group: group["time"], reverse=True) + return groups def get_recently_added_items(limit: int = 7): - tracks = sorted(TrackStore.tracks, key=lambda t: t.created_date) - groups = group_track_by_folders(tracks) + # tracks = sorted(TrackStore.tracks, key=lambda t: t.created_date) + now = time() + tracks = get_recently_added_tracks(start=0, limit=None) + then = time() + + print(f"Time taken to get tracks: {then - now}") + groups = group_track_by_folders(tracks, {}) + # print(groups) + last_trackcount: int = len(tracks) + + # while len(groups.keys()) < limit and last_trackcount > 0: + # distracks = get_recently_added_tracks(start=len(tracks), limit=100) + # last_trackcount = len(distracks) + + # tracks.extend(distracks) + # groups = group_track_by_folders(tracks, groups) + + grouplist = [] + + # INFO: sort tracks by last modified date in descending order to get the most recent last modified date + for folder, trackgroup in groups.items(): + trackgroup.sort(key=lambda t: t.last_mod, reverse=True) + grouplist.append( + { + "folder": folder, + "len": len(trackgroup), + "tracks": trackgroup, + "time": trackgroup[0].last_mod, + } + ) + + pprint(f"😅😅😅😅😅😅😅😅😅😅😅😅😅😅😅😅 {grouplist[0]['len']}") + + # sort groups by last modified date + grouplist = sorted(grouplist, key=lambda group: group["time"], reverse=True) recent_items = [] - for group in groups: + for group in grouplist: item = check_folder_type(group) if item not in recent_items: @@ -217,7 +249,6 @@ def get_recently_added_items(limit: int = 7): return recent_items - def get_recently_added_playlist(limit: int = 100): playlist = Playlist( id="recentlyadded", @@ -232,7 +263,7 @@ def get_recently_added_playlist(limit: int = 100): try: # Create date to show as last updated - date = datetime.fromtimestamp(tracks[0].created_date) + date = datetime.fromtimestamp(tracks[0].last_mod) except IndexError: return playlist, [] @@ -244,6 +275,8 @@ def get_recently_added_playlist(limit: int = 100): return playlist, tracks -def get_recently_added_tracks(limit: int): - tracks = sorted(TrackStore.tracks, key=lambda t: t.created_date, reverse=True) - return tracks[:limit] \ No newline at end of file + +def get_recently_added_tracks(start: int = 0, limit: int = 100): + # tracks = sorted(TrackStore.tracks, key=lambda t: t.created_date, reverse=True) + return TrackTable.get_recently_added(start, limit) + # return tracks[:limit] diff --git a/app/models/playlist.py b/app/models/playlist.py index a916bb3d..34e95c3f 100644 --- a/app/models/playlist.py +++ b/app/models/playlist.py @@ -10,7 +10,7 @@ from app import settings class Playlist: """Creates playlist objects""" - id: int + id: int | str image: str | None last_updated: str name: str @@ -21,7 +21,7 @@ class Playlist: count: int = 0 duration: int = 0 has_image: bool = False - images: list[str] = dataclasses.field(default_factory=list) + images: list[dict[str, str]] = dataclasses.field(default_factory=list) pinned: bool = False def __post_init__(self): diff --git a/app/utils/__init__.py b/app/utils/__init__.py index 84cb6dd0..7cce8009 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -1,4 +1,7 @@ import locale +from typing import TypeVar + +T = TypeVar("T") # Set to user's default locale: locale.setlocale(locale.LC_ALL, "") @@ -9,3 +12,9 @@ locale.setlocale(locale.LC_ALL, "") def format_number(number: float) -> str: return locale.format_string("%d", number, grouping=True) + + + + +def flatten(list_: list[list[T]]) -> list[T]: + return [item for sublist in list_ for item in sublist] From a3c4558d5214e998c7b1ca0011b892f2b7ec2140 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 30 Jun 2024 23:11:33 +0300 Subject: [PATCH 08/44] port: recent items for homepage --- TODO.md | 1 + app/db/__init__.py | 5 +++ app/db/libdata.py | 14 +++++-- app/db/userdata.py | 15 ++++++++ app/db/utils.py | 8 ++++ app/lib/home/recentlyplayed.py | 69 +++++++++++++++++++++------------- app/lib/playlistlib.py | 12 +++--- app/lib/tagger.py | 2 + app/models/logger.py | 3 +- 9 files changed, 92 insertions(+), 37 deletions(-) diff --git a/TODO.md b/TODO.md index dc6d5216..ef37f2ac 100644 --- a/TODO.md +++ b/TODO.md @@ -47,3 +47,4 @@ - Remove duplicates on artist page (test with Hanson) - Test foreign keys on delete - Map scrobble info on app start +- Make home page recent items faster! \ No newline at end of file diff --git a/app/db/__init__.py b/app/db/__init__.py index bd857356..853a0eac 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -3,6 +3,7 @@ from typing import Any from sqlalchemy import ( create_engine, delete, + func, insert, select, ) @@ -81,6 +82,10 @@ class Base(MappedAsDataclass, DeclarativeBase): def all(cls): return cls.execute(select(cls)) + @classmethod + def count(cls): + return cls.execute(select(func.count()).select_from(cls)).scalar() + def create_all(): Base().metadata.create_all(engine) diff --git a/app/db/libdata.py b/app/db/libdata.py index 0708b7fc..f8750453 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -35,9 +35,13 @@ class Base(MasterBase, DeclarativeBase): if cls.__tablename__ == "track": stmt = select(TrackTable.trackhash).where(cls.last_mod < create_date) elif cls.__tablename__ == "album": - stmt = select(AlbumTable.albumhash).where(cls.created_date < create_date) + stmt = select(AlbumTable.albumhash).where( + cls.created_date < create_date + ) elif cls.__tablename__ == "artist": - stmt = select(ArtistTable.artisthash).where(cls.created_date < create_date) + stmt = select(ArtistTable.artisthash).where( + cls.created_date < create_date + ) result = conn.execute(stmt) return {row[0] for row in result.fetchall()} @@ -206,6 +210,11 @@ class TrackTable(Base): return tracks_to_dataclasses(result.fetchall()) + @classmethod + def get_recently_played(cls, limit: int): + result = cls.execute(select(cls).order_by(cls.lastplayed.desc()).limit(limit)) + return tracks_to_dataclasses(result.fetchall()) + @classmethod def remove_tracks_by_filepaths(cls, filepaths: set[str]): with DbManager(commit=True) as conn: @@ -250,7 +259,6 @@ class AlbumTable(Base): all = result.fetchall() return albums_to_dataclasses(all) - @classmethod def get_album_by_albumhash(cls, hash: str): with DbManager() as conn: diff --git a/app/db/userdata.py b/app/db/userdata.py index b57cd73f..1ca026a5 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -25,6 +25,8 @@ from app.db.utils import ( plugin_to_dataclasses, similar_artist_to_dataclass, similar_artists_to_dataclass, + tracklog_to_dataclass, + tracklog_to_dataclasses, tracks_to_dataclasses, user_to_dataclass, user_to_dataclasses, @@ -166,6 +168,7 @@ class FavoritesTable(Base): JSON(), nullable=True, default_factory=dict ) + @classmethod def get_all(cls): with DbManager() as conn: @@ -259,3 +262,15 @@ class ScrobbleTable(Base): def add(cls, item: dict[str, Any]): item["userid"] = get_current_userid() return cls.insert_one(item) + + @classmethod + def get_all(cls, start: int, limit: int): + result = cls.execute( + select(cls) + .where(cls.userid == get_current_userid()) + .order_by(cls.timestamp.desc()) + .offset(start) + .limit(limit) + ) + + return tracklog_to_dataclasses(result.fetchall()) diff --git a/app/db/utils.py b/app/db/utils.py index 0df36aab..0ed92c2c 100644 --- a/app/db/utils.py +++ b/app/db/utils.py @@ -3,6 +3,7 @@ from typing import Any from app.models import Album as AlbumModel, Artist as ArtistModel, Track as TrackModel from app.models.favorite import Favorite from app.models.lastfm import SimilarArtist +from app.models.logger import TrackLog from app.models.plugins import Plugin from app.models.user import User @@ -73,3 +74,10 @@ def plugin_to_dataclass(entry: Any): def plugin_to_dataclasses(entries: Any): return [plugin_to_dataclass(entry) for entry in entries] + +def tracklog_to_dataclass(entry: Any): + entry_dict = entry._asdict() + return TrackLog(**entry_dict) + +def tracklog_to_dataclasses(entries: Any): + return [tracklog_to_dataclass(entry) for entry in entries] \ No newline at end of file diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index 7dc5066e..a22bd61d 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -1,29 +1,36 @@ from datetime import datetime import os +from app.db.libdata import AlbumTable, ArtistTable, TrackTable +from app.db.userdata import FavoritesTable, ScrobbleTable from app.models.logger import TrackLog -from app.db.sqlite.logger.tracks import SQLiteTrackLogger as db -from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb -from app.db.sqlite.favorite import SQLiteFavoriteMethods as fdb +# from app.db.sqlite.logger.tracks import SQLiteTrackLogger as db +# from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb +# from app.db.sqlite.favorite import SQLiteFavoriteMethods as fdb from app.models.playlist import Playlist from app.serializers.track import serialize_track from app.serializers.album import album_serializer from app.lib.playlistlib import get_first_4_images -from app.utils.dates import create_new_date, date_string_to_time_passed, timestamp_to_time_passed +from app.store.folder import FolderStore +from app.utils.dates import ( + create_new_date, + date_string_to_time_passed, + timestamp_to_time_passed, +) from app.serializers.artist import serialize_for_card from app.serializers.playlist import serialize_for_card as serialize_playlist from app.lib.home.recentlyadded import get_recently_added_playlist -from app.store.albums import AlbumStore -from app.store.tracks import TrackStore -from app.store.artists import ArtistStore - +# from app.store.albums import AlbumStore +# from app.store.tracks import TrackStore +# from app.store.artists import ArtistStore def get_recently_played(limit=7): # TODO: Paginate this - entries = db.get_all() + # entries = db.get_all() + entries = ScrobbleTable.get_all(0, 200) items = [] added = set() @@ -36,7 +43,7 @@ def get_recently_played(limit=7): if len(items) >= limit: break - entry = TrackLog(*entry) + # entry = TrackLog(*entry) if entry.source in added: continue @@ -44,7 +51,8 @@ def get_recently_played(limit=7): added.add(entry.source) if entry.type == "album": - album = AlbumStore.get_album_by_hash(entry.type_src) + # album = AlbumStore.get_album_by_hash(entry.type_src) + album = AlbumTable.get_album_by_albumhash(entry.type_src) if album is None: continue @@ -72,7 +80,8 @@ def get_recently_played(limit=7): continue if entry.type == "artist": - artist = ArtistStore.get_artist_by_hash(entry.type_src) + # artist = ArtistStore.get_artist_by_hash(entry.type_src) + artist = ArtistTable.get_artist_by_hash(entry.type_src) if artist is None: continue @@ -107,13 +116,14 @@ def get_recently_played(limit=7): # print(folder) # folder = os.path.join("/", folder, "") # print(folder) - count = len([t for t in TrackStore.tracks if t.folder == folder]) + # count = len([t for t in TrackStore.tracks if t.folder == folder]) + count = FolderStore.count_tracks_containing_paths([folder]) items.append( { "type": "folder", "item": { "path": folder, - "count": count, + "count": count[0]["trackcount"], "help_text": "folder", "time": timestamp_to_time_passed(entry.timestamp), }, @@ -122,12 +132,15 @@ def get_recently_played(limit=7): continue if entry.type == "playlist": + continue is_custom = entry.type_src in [i["name"] for i in custom_playlists] # is_recently_added = entry.type_src == "recentlyadded" if is_custom: playlist, _ = next( - i["handler"]() for i in custom_playlists if i["name"] == entry.type_src + i["handler"]() + for i in custom_playlists + if i["name"] == entry.type_src ) playlist.images = [i["image"] for i in playlist.images] @@ -175,16 +188,17 @@ def get_recently_played(limit=7): "type": "favorite_tracks", "item": { "help_text": "playlist", - "count": fdb.get_track_count(), + "count": FavoritesTable.count(), "time": timestamp_to_time_passed(entry.timestamp), }, } ) continue - try: - track = TrackStore.get_tracks_by_trackhashes([entry.trackhash])[0] - except IndexError: + # track = TrackStore.get_tracks_by_trackhashes([entry.trackhash])[0] + track = TrackTable.get_track_by_trackhash(entry.trackhash) + + if track is None: continue track = serialize_track(track) @@ -202,10 +216,12 @@ def get_recently_played(limit=7): def get_recently_played_tracks(limit: int): - records = db.get_recently_played(start=0, limit=limit) - last_updated = records[0].timestamp - tracks = TrackStore.get_tracks_by_trackhashes([r.trackhash for r in records]) - return tracks, last_updated + # records = db.get_recently_played(start=0, limit=limit) + # last_updated = records[0].timestamp + # tracks = TrackStore.get_tracks_by_trackhashes([r.trackhash for r in records]) + # return tracks, last_updated + return TrackTable.get_recently_played(limit) + def get_recently_played_playlist(limit: int = 100): playlist = Playlist( @@ -217,13 +233,12 @@ def get_recently_played_playlist(limit: int = 100): trackhashes=[], ) - tracks, timestamp = get_recently_played_tracks(limit) - - date = datetime.fromtimestamp(timestamp) + tracks = get_recently_played_tracks(limit) + date = datetime.fromtimestamp(tracks[0].lastplayed) playlist.last_updated = date_string_to_time_passed(create_new_date(date)) images = get_first_4_images(tracks=tracks) playlist.images = images playlist.set_count(len(tracks)) - return playlist, tracks \ No newline at end of file + return playlist, tracks diff --git a/app/lib/playlistlib.py b/app/lib/playlistlib.py index f9512fe1..5d8f7292 100644 --- a/app/lib/playlistlib.py +++ b/app/lib/playlistlib.py @@ -10,10 +10,8 @@ from typing import Any from PIL import Image, ImageSequence from app import settings +from app.db.libdata import AlbumTable, TrackTable from app.models.track import Track -from app.store.albums import AlbumStore -from app.store.tracks import TrackStore - def create_thumbnail(image: Any, img_path: str) -> str: """ @@ -105,7 +103,8 @@ def get_first_4_images( tracks: list[Track] = [], trackhashes: list[str] = [] ) -> list[dict["str", str]]: if len(trackhashes) > 0: - tracks = TrackStore.get_tracks_by_trackhashes(trackhashes) + # tracks = TrackStore.get_tracks_by_trackhashes(trackhashes) + tracks = TrackTable.get_tracks_by_trackhashes(trackhashes) albums = [] @@ -116,11 +115,12 @@ def get_first_4_images( if len(albums) == 4: break - albums = AlbumStore.get_albums_by_hashes(albums) + # albums = AlbumStore.get_albums_by_hashes(albums) + albums = AlbumTable.get_albums_by_albumhashes(albums) images = [ { "image": album.image, - "color": "".join(album.colors), + "color": album.color, } for album in albums ] diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 2bb7f660..052a7dff 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -1,3 +1,4 @@ +import gc import os from pprint import pprint from time import time @@ -280,6 +281,7 @@ class IndexEverything: # pass CordinateMedia(instance_key=str(time())) + gc.collect() @background diff --git a/app/models/logger.py b/app/models/logger.py index 0f1e92d8..e4b6e368 100644 --- a/app/models/logger.py +++ b/app/models/logger.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Literal +from typing import Any, Literal @dataclass @@ -14,6 +14,7 @@ class TrackLog: timestamp: int source: str userid: int + extra: dict[str, Any] type = "track" type_src = None From ff7343a7be08658ab2b0cf0b7d11c7f513da4666 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 1 Jul 2024 09:22:52 +0300 Subject: [PATCH 09/44] start porting: playlists endpoints --- app/api/playlist.py | 106 +++++++++++++++++++++------------------ app/db/__init__.py | 4 ++ app/db/userdata.py | 109 ++++++++++++++++++++++++++++++++++++++++- app/db/utils.py | 14 +++++- app/models/playlist.py | 25 +++++----- 5 files changed, 196 insertions(+), 62 deletions(-) diff --git a/app/api/playlist.py b/app/api/playlist.py index 1f2720d0..a65c6714 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -12,13 +12,14 @@ from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint, FileStorage from app import models -from app.db.sqlite.playlists import SQLitePlaylistMethods +from app.db.libdata import TrackTable +from app.db.userdata import PlaylistTable from app.lib import playlistlib from app.lib.albumslib import sort_by_track_no from app.lib.home.recentlyadded import get_recently_added_playlist from app.lib.home.recentlyplayed import get_recently_played_playlist +from app.models.playlist import Playlist from app.serializers.playlist import serialize_for_card -from app.store.tracks import TrackStore from app.utils.dates import create_new_date, date_string_to_time_passed from app.utils.remove_duplicates import remove_duplicates from app.settings import Paths @@ -26,8 +27,6 @@ from app.settings import Paths tag = Tag(name="Playlists", description="Get and manage playlists") api = APIBlueprint("playlists", __name__, url_prefix="/playlists", abp_tags=[tag]) -PL = SQLitePlaylistMethods - class SendAllPlaylistsQuery(BaseModel): no_images: bool = Field(False, description="Whether to include images") @@ -38,7 +37,7 @@ def send_all_playlists(query: SendAllPlaylistsQuery): """ Gets all the playlists. """ - playlists = PL.get_all_playlists() + playlists = PlaylistTable.get_all() playlists = list(playlists) for playlist in playlists: @@ -63,18 +62,21 @@ def insert_playlist(name: str, image: str = None): "image": image, "last_updated": create_new_date(), "name": name, - "trackhashes": json.dumps([]), - "settings": json.dumps( - { - "has_gif": False, - "banner_pos": 50, - "square_img": True if image else False, - "pinned": False, - } - ), + "trackhashes": [], + "settings": { + "has_gif": False, + "banner_pos": 50, + "square_img": True if image else False, + "pinned": False, + }, } - return PL.insert_one_playlist(playlist) + rowid = PlaylistTable.add_one(playlist) + if rowid: + playlist["id"] = rowid + return Playlist(**playlist) + + return None class CreatePlaylistBody(BaseModel): @@ -88,9 +90,10 @@ def create_playlist(body: CreatePlaylistBody): Creates a new playlist. Accepts POST method with a JSON body. """ - existing_playlist_count = PL.count_playlist_by_name(body.name) + # existing_playlist_count = PL.count_playlist_by_name(body.name) + exists = PlaylistTable.check_exists_by_name(body.name) - if existing_playlist_count > 0: + if exists: return {"error": "Playlist already exists"}, 409 playlist = insert_playlist(body.name) @@ -105,8 +108,7 @@ def get_path_trackhashes(path: str): """ Returns a list of trackhashes in a folder. """ - tracks = TrackStore.get_tracks_in_path(path) - tracks = sorted(tracks, key=lambda t: t.last_mod) + tracks = TrackTable.get_tracks_in_path(path) return [t.trackhash for t in tracks] @@ -114,17 +116,17 @@ def get_album_trackhashes(albumhash: str): """ Returns a list of trackhashes in an album. """ - tracks = TrackStore.get_tracks_by_albumhash(albumhash) + tracks = TrackTable.get_tracks_by_albumhash(albumhash) tracks = sort_by_track_no(tracks) - return [t["trackhash"] for t in tracks] + return [t.trackhash for t in tracks] def get_artist_trackhashes(artisthash: str): """ Returns a list of trackhashes for an artist. """ - tracks = TrackStore.get_tracks_by_artisthash(artisthash) + tracks = TrackTable.get_tracks_by_artisthash(artisthash) return [t.trackhash for t in tracks] @@ -164,10 +166,11 @@ def add_item_to_playlist(path: PlaylistIDPath, body: AddItemToPlaylistBody): else: trackhashes = [] - insert_count = PL.add_tracks_to_playlist(int(playlist_id), trackhashes) + # insert_count = PL.add_tracks_to_playlist(int(playlist_id), trackhashes) + PlaylistTable.append_to_playlist(int(playlist_id), trackhashes) - if insert_count == 0: - return {"error": "Item already exists in playlist"}, 409 + # if insert_count == 0: + # return {"error": "Item already exists in playlist"}, 409 return {"msg": "Done"}, 200 @@ -179,11 +182,10 @@ class GetPlaylistQuery(BaseModel): def format_custom_playlist(playlist: models.Playlist, tracks: list[models.Track]): duration = sum(t.duration for t in tracks) - playlist.set_duration(duration) - playlist = serialize_for_card(playlist) + playlist.duration = duration return { - "info": playlist, + "info": serialize_for_card(playlist), "tracks": tracks, } @@ -209,19 +211,20 @@ def get_playlist(path: PlaylistIDPath, query: GetPlaylistQuery): playlist, tracks = handler() return format_custom_playlist(playlist, tracks) - playlist = PL.get_playlist_by_id(int(playlistid)) + # playlist = PL.get_playlist_by_id(int(playlistid)) + playlist = PlaylistTable.get_by_id(playlistid) if playlist is None: return {"msg": "Playlist not found"}, 404 - tracks = TrackStore.get_tracks_by_trackhashes(list(playlist.trackhashes)) - + # tracks = TrackStore.get_tracks_by_trackhashes(list(playlist.trackhashes)) + tracks = TrackTable.get_tracks_by_trackhashes(playlist.trackhashes) tracks = remove_duplicates(tracks) + duration = sum(t.duration for t in tracks) playlist.last_updated = date_string_to_time_passed(playlist.last_updated) - playlist.set_duration(duration) - playlist.set_count(len(tracks)) + playlist.duration = duration if not playlist.has_image: playlist.images = playlistlib.get_first_4_images(tracks) @@ -247,7 +250,8 @@ def update_playlist_info(path: PlaylistIDPath, form: UpdatePlaylistForm): Update playlist """ playlistid = path.playlistid - db_playlist = PL.get_playlist_by_id(playlistid) + # db_playlist = PL.get_playlist_by_id(playlistid) + db_playlist = PlaylistTable.get_by_id(playlistid) if db_playlist is None: return {"error": "Playlist not found"}, 404 @@ -286,7 +290,8 @@ def update_playlist_info(path: PlaylistIDPath, form: UpdatePlaylistForm): p_tuple = (*playlist.values(),) - PL.update_playlist(playlistid, playlist) + # PL.update_playlist(playlistid, playlist) + PlaylistTable.update_one(playlistid, playlist) playlist = models.Playlist(*p_tuple) playlist.last_updated = date_string_to_time_passed(playlist.last_updated) @@ -301,7 +306,9 @@ def pin_unpin_playlist(path: PlaylistIDPath): """ Pin playlist. """ - playlist = PL.get_playlist_by_id(path.playlistid) + # playlist = PL.get_playlist_by_id(path.playlistid) + + playlist = PlaylistTable.get_by_id(path.playlistid) if playlist is None: return {"error": "Playlist not found"}, 404 @@ -313,8 +320,8 @@ def pin_unpin_playlist(path: PlaylistIDPath): except KeyError: settings["pinned"] = True - PL.update_settings(path.playlistid, settings) - + # PL.update_settings(path.playlistid, settings) + PlaylistTable.update_settings(path.playlistid, settings) return {"msg": "Done"}, 200 @@ -323,12 +330,14 @@ def remove_playlist_image(path: PlaylistIDPath): """ Clear playlist image. """ - playlist = PL.get_playlist_by_id(path.playlistid) + # playlist = PL.get_playlist_by_id(path.playlistid) + playlist = PlaylistTable.get_by_id(path.playlistid) if playlist is None: return {"error": "Playlist not found"}, 404 - PL.remove_banner(path.playlistid) + # PL.remove_banner(path.playlistid) + PlaylistTable.remove_image(path.playlistid) playlist.image = None playlist.thumb = None @@ -346,7 +355,8 @@ def remove_playlist(path: PlaylistIDPath): """ Delete playlist """ - PL.delete_playlist(path.playlistid) + # PL.delete_playlist(path.playlistid) + PlaylistTable.remove_one(path.playlistid) return {"msg": "Done"}, 200 @@ -368,15 +378,12 @@ def remove_tracks_from_playlist( # index: int; # } - PL.remove_tracks_from_playlist(path.playlistid, body.tracks) + # PL.remove_tracks_from_playlist(path.playlistid, body.tracks) + PlaylistTable.remove_from_playlist(path.playlistid, body.tracks) return {"msg": "Done"}, 200 -def playlist_name_exists(name: str) -> bool: - return PL.count_playlist_by_name(name) > 0 - - class SavePlaylistAsItemBody(BaseModel): itemtype: str = Field(..., description="The type of item", example="tracks") playlist_name: str = Field(..., description="The name of the playlist") @@ -394,7 +401,7 @@ def save_item_as_playlist(body: SavePlaylistAsItemBody): playlist_name = body.playlist_name itemhash = body.itemhash - if playlist_name_exists(playlist_name): + if PlaylistTable.check_exists_by_name(playlist_name): return {"error": "Playlist already exists"}, 409 if itemtype == "tracks": @@ -437,8 +444,9 @@ def save_item_as_playlist(body: SavePlaylistAsItemBody): img, str(playlist.id), "image/webp", filename=filename ) - PL.add_tracks_to_playlist(playlist.id, trackhashes) - playlist.set_count(len(trackhashes)) + # PL.add_tracks_to_playlist(playlist.id, trackhashes) + PlaylistTable.append_to_playlist(playlist.id, trackhashes) + playlist.count = len(trackhashes) images = playlistlib.get_first_4_images(trackhashes=trackhashes) playlist.images = [img["image"] for img in images] diff --git a/app/db/__init__.py b/app/db/__init__.py index 853a0eac..4b1c4366 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -78,6 +78,10 @@ class Base(MappedAsDataclass, DeclarativeBase): with DbManager(commit=True) as conn: conn.execute(delete(cls)) + @classmethod + def remove_one(cls, id: int): + cls.execute(delete(cls).where(cls.id == id), commit=True) + @classmethod def all(cls): return cls.execute(select(cls)) diff --git a/app/db/userdata.py b/app/db/userdata.py index 1ca026a5..1e9407f0 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -22,6 +22,8 @@ from app.db.utils import ( albums_to_dataclasses, artists_to_dataclasses, favorites_to_dataclass, + playlist_to_dataclass, + playlists_to_dataclasses, plugin_to_dataclasses, similar_artist_to_dataclass, similar_artists_to_dataclass, @@ -168,7 +170,6 @@ class FavoritesTable(Base): JSON(), nullable=True, default_factory=dict ) - @classmethod def get_all(cls): with DbManager() as conn: @@ -274,3 +275,109 @@ class ScrobbleTable(Base): ) return tracklog_to_dataclasses(result.fetchall()) + + +class PlaylistTable(Base): + __tablename__ = "playlist" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(), index=True) + last_updated: Mapped[int] = mapped_column(Integer()) + image: Mapped[str] = mapped_column(String(), nullable=True) + userid: Mapped[int] = mapped_column( + Integer(), ForeignKey("user.id", ondelete="cascade") + ) + settings: Mapped[dict[str, Any]] = mapped_column(JSON()) + trackhashes: Mapped[list[str]] = mapped_column(JSON(), default_factory=list) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) + + @classmethod + def get_all(cls): + result = cls.all() + return playlists_to_dataclasses(result) + + @classmethod + def add_one(cls, playlist: dict[str, Any]): + playlist["userid"] = get_current_userid() + result = cls.insert_one(playlist) + return result.lastrowid + + @classmethod + def check_exists_by_name(cls, name: str): + result = cls.execute( + select(cls).where((cls.name == name) & (cls.userid == get_current_userid())) + ) + return result.fetchone() is not None + + @classmethod + def append_to_playlist(cls, id: int, trackhashes: list[str]): + print("type(trackhashes):", type(trackhashes)) + return cls.execute( + update(cls) + .where((cls.id == id) & (cls.userid == get_current_userid())) + .values(trackhashes=cls.trackhashes + trackhashes), + commit=True, + ) + + @classmethod + def remove_from_playlist(cls, id: int, trackhashes: list[dict[str, Any]]): + # CHECKPOINT: Properly remove tracks from a playlist + # Without messing up the order in case of duplicates + tracks = cls.execute( + select(cls.trackhashes).where( + (cls.id == id) & (cls.userid == get_current_userid()) + ) + ) + + results = tracks.fetchone() + if results: + dbhashes: list[str] = results[0] + + for item in trackhashes: + if dbhashes.index(item["trackhash"]) == item["index"]: + dbhashes.remove(item["trackhash"]) + + return cls.execute( + update(cls) + .where((cls.id == id) & (cls.userid == get_current_userid())) + .values(trackhashes=dbhashes), + commit=True, + ) + + @classmethod + def get_by_id(cls, id: int): + result = cls.execute( + select(cls).where((cls.id == id) & (cls.userid == get_current_userid())) + ) + result = result.fetchone() + if result: + return playlist_to_dataclass(result) + + @classmethod + def update_one(cls, id: int, playlist: dict[str, Any]): + return cls.execute( + update(cls) + .where((cls.id == id) & (cls.userid == get_current_userid())) + .values(playlist), + commit=True, + ) + + @classmethod + def update_settings(cls, id: int, settings: dict[str, Any]): + return cls.execute( + update(cls) + .where((cls.id == id) & (cls.userid == get_current_userid())) + .values(settings=settings), + commit=True, + ) + + @classmethod + def remove_image(cls, id: int): + return cls.execute( + update(cls) + .where((cls.id == id) & (cls.userid == get_current_userid())) + .values(image=None), + commit=True, + ) diff --git a/app/db/utils.py b/app/db/utils.py index 0ed92c2c..550a7537 100644 --- a/app/db/utils.py +++ b/app/db/utils.py @@ -4,6 +4,7 @@ from app.models import Album as AlbumModel, Artist as ArtistModel, Track as Trac from app.models.favorite import Favorite from app.models.lastfm import SimilarArtist from app.models.logger import TrackLog +from app.models.playlist import Playlist from app.models.plugins import Plugin from app.models.user import User @@ -75,9 +76,20 @@ def plugin_to_dataclass(entry: Any): def plugin_to_dataclasses(entries: Any): return [plugin_to_dataclass(entry) for entry in entries] + def tracklog_to_dataclass(entry: Any): entry_dict = entry._asdict() return TrackLog(**entry_dict) + def tracklog_to_dataclasses(entries: Any): - return [tracklog_to_dataclass(entry) for entry in entries] \ No newline at end of file + return [tracklog_to_dataclass(entry) for entry in entries] + + +def playlist_to_dataclass(entry: Any): + entry_dict = entry._asdict() + return Playlist(**entry_dict) + + +def playlists_to_dataclasses(entries: Any): + return [playlist_to_dataclass(entry) for entry in entries] diff --git a/app/models/playlist.py b/app/models/playlist.py index 34e95c3f..d413010d 100644 --- a/app/models/playlist.py +++ b/app/models/playlist.py @@ -2,6 +2,7 @@ import dataclasses import json from dataclasses import dataclass from pathlib import Path +from typing import Any from app import settings @@ -14,10 +15,12 @@ class Playlist: image: str | None last_updated: str name: str - settings: str | dict - trackhashes: str | list[str] + settings: dict + userid: int + trackhashes: list[str] + extra: dict[str, Any] = dataclasses.field(default_factory=dict) - thumb: str | None = "" + thumb: str = "" count: int = 0 duration: int = 0 has_image: bool = False @@ -25,11 +28,11 @@ class Playlist: pinned: bool = False def __post_init__(self): - self.trackhashes = json.loads(str(self.trackhashes)) - self.count = len(self.trackhashes) + # self.trackhashes = json.loads(str(self.trackhashes)) + # self.count = len(self.trackhashes) - if isinstance(self.settings, str): - self.settings = dict(json.loads(self.settings)) + # if isinstance(self.settings, str): + # self.settings = dict(json.loads(self.settings)) self.pinned = self.settings.get("pinned", False) self.has_image = ( @@ -42,11 +45,11 @@ class Playlist: self.image = "None" self.thumb = "None" - def set_duration(self, duration: int): - self.duration = duration + # def set_duration(self, duration: int): + # self.duration = duration - def set_count(self, count: int): - self.count = count + # def set_count(self, count: int): + # self.count = count def clear_lists(self): """ From a5634f267f62bee8eedd496fd8c970b571b6ff98 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Wed, 3 Jul 2024 11:12:06 +0300 Subject: [PATCH 10/44] move to xxh3 hashing algorithm + port: search --- .github/changelog.md | 2 + TODO.md | 10 ++- app/api/playlist.py | 18 ----- app/api/search.py | 74 +++++++------------- app/api/stream.py | 2 +- app/db/libdata.py | 70 +++++++++++++------ app/db/userdata.py | 64 ++++++++++++++---- app/lib/home/recentlyplayed.py | 12 +--- app/lib/searchlib.py | 77 +++++++++++---------- app/models/album.py | 17 +++-- app/models/playlist.py | 18 ++--- app/settings.py | 2 +- app/utils/__init__.py | 4 +- app/utils/hashing.py | 14 ++-- poetry.lock | 119 ++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 16 files changed, 322 insertions(+), 182 deletions(-) diff --git a/.github/changelog.md b/.github/changelog.md index d2993bf1..e0348801 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -2,6 +2,7 @@ - Auth +- New artists/albums Sort by: last played, no. of streams, total stream duration ## Improvements - The context menu now doesn't take forever to open up @@ -12,6 +13,7 @@ - ## Development +- Rewritten the whole DB layer to move stores from memory to the database. ## THE BIG ONE API CHANGES diff --git a/TODO.md b/TODO.md index ef37f2ac..057b7d7d 100644 --- a/TODO.md +++ b/TODO.md @@ -47,4 +47,12 @@ - Remove duplicates on artist page (test with Hanson) - Test foreign keys on delete - Map scrobble info on app start -- Make home page recent items faster! \ No newline at end of file +- Make home page recent items faster! +- Normalize playlists table: + - New table to hold playlist entries +- Normalize similar artists: + - New table to hold similar artist entries + - Create 2 way relationships, such that if an artist A is similar to another B with a certain weight, + then artist B is similar to A with the same weight, unless overwritten. +- Figure out how to update album/artist tables instead of deleting all rows when the app starts +- Move get all filtering and sorting operations to the database since all sort keys are table columns \ No newline at end of file diff --git a/app/api/playlist.py b/app/api/playlist.py index a65c6714..3d61d238 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -166,12 +166,7 @@ def add_item_to_playlist(path: PlaylistIDPath, body: AddItemToPlaylistBody): else: trackhashes = [] - # insert_count = PL.add_tracks_to_playlist(int(playlist_id), trackhashes) PlaylistTable.append_to_playlist(int(playlist_id), trackhashes) - - # if insert_count == 0: - # return {"error": "Item already exists in playlist"}, 409 - return {"msg": "Done"}, 200 @@ -211,15 +206,12 @@ def get_playlist(path: PlaylistIDPath, query: GetPlaylistQuery): playlist, tracks = handler() return format_custom_playlist(playlist, tracks) - # playlist = PL.get_playlist_by_id(int(playlistid)) playlist = PlaylistTable.get_by_id(playlistid) if playlist is None: return {"msg": "Playlist not found"}, 404 - # tracks = TrackStore.get_tracks_by_trackhashes(list(playlist.trackhashes)) tracks = TrackTable.get_tracks_by_trackhashes(playlist.trackhashes) - tracks = remove_duplicates(tracks) duration = sum(t.duration for t in tracks) playlist.last_updated = date_string_to_time_passed(playlist.last_updated) @@ -250,7 +242,6 @@ def update_playlist_info(path: PlaylistIDPath, form: UpdatePlaylistForm): Update playlist """ playlistid = path.playlistid - # db_playlist = PL.get_playlist_by_id(playlistid) db_playlist = PlaylistTable.get_by_id(playlistid) if db_playlist is None: @@ -270,7 +261,6 @@ def update_playlist_info(path: PlaylistIDPath, form: UpdatePlaylistForm): "last_updated": create_new_date(), "name": str(form.name).strip(), "settings": settings, - "trackhashes": json.dumps([]), } if image: @@ -290,7 +280,6 @@ def update_playlist_info(path: PlaylistIDPath, form: UpdatePlaylistForm): p_tuple = (*playlist.values(),) - # PL.update_playlist(playlistid, playlist) PlaylistTable.update_one(playlistid, playlist) playlist = models.Playlist(*p_tuple) @@ -306,8 +295,6 @@ def pin_unpin_playlist(path: PlaylistIDPath): """ Pin playlist. """ - # playlist = PL.get_playlist_by_id(path.playlistid) - playlist = PlaylistTable.get_by_id(path.playlistid) if playlist is None: @@ -320,7 +307,6 @@ def pin_unpin_playlist(path: PlaylistIDPath): except KeyError: settings["pinned"] = True - # PL.update_settings(path.playlistid, settings) PlaylistTable.update_settings(path.playlistid, settings) return {"msg": "Done"}, 200 @@ -330,13 +316,11 @@ def remove_playlist_image(path: PlaylistIDPath): """ Clear playlist image. """ - # playlist = PL.get_playlist_by_id(path.playlistid) playlist = PlaylistTable.get_by_id(path.playlistid) if playlist is None: return {"error": "Playlist not found"}, 404 - # PL.remove_banner(path.playlistid) PlaylistTable.remove_image(path.playlistid) playlist.image = None @@ -355,7 +339,6 @@ def remove_playlist(path: PlaylistIDPath): """ Delete playlist """ - # PL.delete_playlist(path.playlistid) PlaylistTable.remove_one(path.playlistid) return {"msg": "Done"}, 200 @@ -378,7 +361,6 @@ def remove_tracks_from_playlist( # index: int; # } - # PL.remove_tracks_from_playlist(path.playlistid, body.tracks) PlaylistTable.remove_from_playlist(path.playlistid, body.tracks) return {"msg": "Done"}, 200 diff --git a/app/api/search.py b/app/api/search.py index 871c3dcf..f08840f3 100644 --- a/app/api/search.py +++ b/app/api/search.py @@ -2,16 +2,17 @@ Contains all the search routes. """ -from flask import request from unidecode import unidecode -from pydantic import BaseModel, Field +from pydantic import Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from app import models +from app.api.apischemas import GenericLimitSchema +from app.db.libdata import TrackTable from app.lib import searchlib from app.settings import Defaults -from app.store.tracks import TrackStore + tag = Tag(name="Search", description="Search for tracks, albums and artists") api = APIBlueprint("search", __name__, url_prefix="/search", abp_tags=[tag]) @@ -20,30 +21,18 @@ SEARCH_COUNT = 30 """The max amount of items to return per request""" -def query_in_quotes(query: str) -> bool: - """ - Returns True if the query is in quotes - """ - try: - return query.startswith('"') and query.endswith('"') - except AttributeError: - return False - - class Search: def __init__(self, query: str) -> None: self.tracks: list[models.Track] = [] self.query = unidecode(query) - def search_tracks(self, in_quotes=False): + def search_tracks(self): """ Calls :class:`SearchTracks` which returns the tracks that fuzzily match the search terms. Then adds them to the `SearchResults` store. """ - self.tracks = TrackStore.tracks - return searchlib.TopResults().search( - self.query, tracks_only=True, in_quotes=in_quotes - ) + self.tracks = TrackTable.get_all() + return searchlib.TopResults().search(self.query, tracks_only=True) def search_artists(self): """Calls :class:`SearchArtists` which returns the artists that fuzzily match @@ -51,25 +40,23 @@ class Search: """ return searchlib.SearchArtists(self.query)() - def search_albums(self, in_quotes=False): + def search_albums(self): """Calls :class:`SearchAlbums` which returns the albums that fuzzily match the search term. Then adds them to the `SearchResults` store. """ - return searchlib.TopResults().search( - self.query, albums_only=True, in_quotes=in_quotes - ) + return searchlib.TopResults().search(self.query, albums_only=True) def get_top_results( self, limit: int, - in_quotes=False, ): finder = searchlib.TopResults() - return finder.search(self.query, in_quotes=in_quotes, limit=limit) + return finder.search(self.query, limit=limit) -class SearchQuery(BaseModel): +class SearchQuery(GenericLimitSchema): q: str = Field(description="The search query", example=Defaults.API_ARTISTNAME) + start: int = Field(description="The index to start from", default=0, example=0) @api.get("/tracks") @@ -77,11 +64,7 @@ def search_tracks(query: SearchQuery): """ Search tracks """ - - query = query.q - in_quotes = query_in_quotes(query) - - tracks = Search(query).search_tracks(in_quotes) + tracks = Search(query.q).search_tracks() return { "tracks": tracks[:SEARCH_COUNT], @@ -90,15 +73,13 @@ def search_tracks(query: SearchQuery): @api.get("/albums") -def search_albums(query: SearchQuery): +def search_albums( + query: SearchQuery, +): """ Search albums. """ - - query = query.q - in_quotes = query_in_quotes(query) - - albums = Search(query).search_albums(in_quotes) + albums = Search(query.q).search_albums() return { "albums": albums[:SEARCH_COUNT], @@ -111,13 +92,10 @@ def search_artists(query: SearchQuery): """ Search artists. """ - - query = query.q - - if not query: + if not query.q: return {"error": "No query provided"}, 400 - artists = Search(query).search_artists() + artists = Search(query.q).search_artists() return { "artists": artists[:SEARCH_COUNT], @@ -138,15 +116,10 @@ def get_top_results(query: TopResultsQuery): Returns the top results for the given query. """ - limit = query.limit - query = query.q - - in_quotes = query_in_quotes(query) - - if not query: + if not query.q: return {"error": "No query provided"}, 400 - return Search(query).get_top_results(in_quotes=in_quotes, limit=limit) + return Search(query.q).get_top_results(limit=query.limit) class SearchLoadMoreQuery(SearchQuery): @@ -166,17 +139,16 @@ def search_load_more(query: SearchLoadMoreQuery): query = query.q item_type = query.type index = query.index - in_quotes = query_in_quotes(query) if item_type == "tracks": - t = Search(query).search_tracks(in_quotes) + t = Search(query).search_tracks() return { "tracks": t[index : index + SEARCH_COUNT], "more": len(t) > index + SEARCH_COUNT, } elif item_type == "albums": - a = Search(query).search_albums(in_quotes) + a = Search(query).search_albums() return { "albums": a[index : index + SEARCH_COUNT], "more": len(a) > index + SEARCH_COUNT, diff --git a/app/api/stream.py b/app/api/stream.py index 4993539a..db6aada2 100644 --- a/app/api/stream.py +++ b/app/api/stream.py @@ -73,7 +73,7 @@ def send_file_as_chunks(filepath: str, audio_type: str) -> Response: """ # NOTE: +1 makes sure the last byte is included in the range. # NOTE: -1 is used to convert the end index to a 0-based index. - chunk_size = 1024 * 360 # 360 KB + chunk_size = 1024 * 512 # 0.5MB # Get file size file_size = os.path.getsize(filepath) diff --git a/app/db/libdata.py b/app/db/libdata.py index f8750453..ecb0f5de 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -14,7 +14,7 @@ from app.models import Album as AlbumModel from app.utils.remove_duplicates import remove_duplicates from app.db import engine -from sqlalchemy import JSON, Boolean, Integer, String, and_, delete, select, update +from sqlalchemy import JSON, Boolean, Integer, String, delete, select, update from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase @@ -32,16 +32,26 @@ class Base(MasterBase, DeclarativeBase): @classmethod def get_all_hashes(cls, create_date: int | None = None): with DbManager() as conn: - if cls.__tablename__ == "track": - stmt = select(TrackTable.trackhash).where(cls.last_mod < create_date) - elif cls.__tablename__ == "album": - stmt = select(AlbumTable.albumhash).where( - cls.created_date < create_date - ) - elif cls.__tablename__ == "artist": - stmt = select(ArtistTable.artisthash).where( - cls.created_date < create_date - ) + if create_date: + if cls.__tablename__ == "track": + stmt = select(TrackTable.trackhash).where( + cls.last_mod < create_date + ) + elif cls.__tablename__ == "album": + stmt = select(AlbumTable.albumhash).where( + cls.created_date < create_date + ) + elif cls.__tablename__ == "artist": + stmt = select(ArtistTable.artisthash).where( + cls.created_date < create_date + ) + else: + if cls.__tablename__ == "track": + stmt = select(TrackTable.trackhash) + elif cls.__tablename__ == "album": + stmt = select(AlbumTable.albumhash) + elif cls.__tablename__ == "artist": + stmt = select(ArtistTable.artisthash) result = conn.execute(stmt) return {row[0] for row in result.fetchall()} @@ -135,7 +145,9 @@ class TrackTable(Base): def get_tracks_by_filepaths(cls, filepaths: list[str]): with DbManager() as conn: result = conn.execute( - select(TrackTable).where(TrackTable.filepath.in_(filepaths)) + select(TrackTable) + .where(TrackTable.filepath.in_(filepaths)) + .order_by(TrackTable.last_mod) ) return tracks_to_dataclasses(result.fetchall()) @@ -155,10 +167,8 @@ class TrackTable(Base): result = conn.execute( select(TrackTable) .where( - and_( - TrackTable.trackhash == hash, - TrackTable.filepath == filepath, - ) + (TrackTable.trackhash == hash) + & (TrackTable.filepath == filepath), ) .order_by(TrackTable.bitrate.desc()) ) @@ -194,9 +204,18 @@ class TrackTable(Base): def get_tracks_by_trackhashes(cls, hashes: Iterable[str], limit: int | None = None): with DbManager() as conn: result = conn.execute( - select(TrackTable).where(TrackTable.trackhash.in_(hashes)).limit(limit) + select(TrackTable) + .where(TrackTable.trackhash.in_(hashes)) + .group_by(TrackTable.trackhash) + .limit(limit) ) - return tracks_to_dataclasses(result.fetchall()) + tracks = tracks_to_dataclasses(result.fetchall()) + + # order the tracks in the same order as the hashes + if type(hashes) == list: + return sorted(tracks, key=lambda x: hashes.index(x.trackhash)) + + return tracks @classmethod def get_recently_added(cls, start: int, limit: int): @@ -212,7 +231,12 @@ class TrackTable(Base): @classmethod def get_recently_played(cls, limit: int): - result = cls.execute(select(cls).order_by(cls.lastplayed.desc()).limit(limit)) + result = cls.execute( + select(cls) + .group_by(cls.trackhash) + .order_by(cls.lastplayed.desc()) + .limit(limit) + ) return tracks_to_dataclasses(result.fetchall()) @classmethod @@ -276,7 +300,13 @@ class AlbumTable(Base): result = conn.execute( select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)).limit(limit) ) - return albums_to_dataclasses(result.fetchall()) + albums = albums_to_dataclasses(result.fetchall()) + + # order the albums in the same order as the hashes + if type(hashes) == list: + return sorted(albums, key=lambda x: hashes.index(x.albumhash)) + + return albums @classmethod def get_albums_by_artisthashes(cls, artisthashes: list[str]): diff --git a/app/db/userdata.py b/app/db/userdata.py index 1e9407f0..7ab3d0db 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -1,7 +1,7 @@ import datetime +import enum from shlex import join from typing import Any -from flask_jwt_extended import current_user from sqlalchemy import ( JSON, Boolean, @@ -313,36 +313,41 @@ class PlaylistTable(Base): @classmethod def append_to_playlist(cls, id: int, trackhashes: list[str]): - print("type(trackhashes):", type(trackhashes)) + dbtrackhashes = cls.get_trackhashes(id) + if not dbtrackhashes: + dbtrackhashes = [] + return cls.execute( update(cls) .where((cls.id == id) & (cls.userid == get_current_userid())) - .values(trackhashes=cls.trackhashes + trackhashes), + .values(trackhashes=dbtrackhashes + trackhashes), commit=True, ) @classmethod - def remove_from_playlist(cls, id: int, trackhashes: list[dict[str, Any]]): - # CHECKPOINT: Properly remove tracks from a playlist - # Without messing up the order in case of duplicates - tracks = cls.execute( + def get_trackhashes(cls, id: int) -> list[str]: + result = cls.execute( select(cls.trackhashes).where( (cls.id == id) & (cls.userid == get_current_userid()) ) ) + result = result.fetchone() + if result: + return result[0] - results = tracks.fetchone() - if results: - dbhashes: list[str] = results[0] - + @classmethod + def remove_from_playlist(cls, id: int, trackhashes: list[dict[str, Any]]): + # INFO: Get db trackhashes + dbtrackhashes = cls.get_trackhashes(id) + if dbtrackhashes: for item in trackhashes: - if dbhashes.index(item["trackhash"]) == item["index"]: - dbhashes.remove(item["trackhash"]) + if dbtrackhashes.index(item["trackhash"]) == item["index"]: + dbtrackhashes.remove(item["trackhash"]) return cls.execute( update(cls) .where((cls.id == id) & (cls.userid == get_current_userid())) - .values(trackhashes=dbhashes), + .values(trackhashes=dbtrackhashes), commit=True, ) @@ -381,3 +386,34 @@ class PlaylistTable(Base): .values(image=None), commit=True, ) + + +# class PlaylistTrackTable(Base): +# __tablename__ = "playlisttrack" + +# id: Mapped[int] = mapped_column(primary_key=True) +# trackhash: Mapped[str] = mapped_column(String(), index=True) +# playlistid: Mapped[int] = mapped_column( +# Integer(), ForeignKey("playlist.id", ondelete="cascade") +# ) +# index: Mapped[int] = mapped_column(Integer()) +# userid: Mapped[int] = mapped_column( +# Integer(), ForeignKey("user.id", ondelete="cascade") +# ) + +# @classmethod +# def count_by_playlist() + +# @classmethod +# def insert_many(cls, playlistid: int, trackhashes: list[str]): +# userid = get_current_userid() +# items = [ +# { +# "index": index, +# "userid": userid, +# "trackhash": trackhash, +# "playlistid": playlistid, +# } +# for index, trackhash in enumerate(trackhashes) +# ] +# return cls.execute(insert(cls).values(items), commit=True) diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index a22bd61d..a89cb92e 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -215,14 +215,6 @@ def get_recently_played(limit=7): return items -def get_recently_played_tracks(limit: int): - # records = db.get_recently_played(start=0, limit=limit) - # last_updated = records[0].timestamp - # tracks = TrackStore.get_tracks_by_trackhashes([r.trackhash for r in records]) - # return tracks, last_updated - return TrackTable.get_recently_played(limit) - - def get_recently_played_playlist(limit: int = 100): playlist = Playlist( id="recentlyplayed", @@ -233,12 +225,12 @@ def get_recently_played_playlist(limit: int = 100): trackhashes=[], ) - tracks = get_recently_played_tracks(limit) + tracks = TrackTable.get_recently_played(limit) date = datetime.fromtimestamp(tracks[0].lastplayed) playlist.last_updated = date_string_to_time_passed(create_new_date(date)) images = get_first_4_images(tracks=tracks) playlist.images = images - playlist.set_count(len(tracks)) + playlist.count = len(tracks) return playlist, tracks diff --git a/app/lib/searchlib.py b/app/lib/searchlib.py index a23d67fa..3d6cb1cb 100644 --- a/app/lib/searchlib.py +++ b/app/lib/searchlib.py @@ -8,16 +8,21 @@ from rapidfuzz import process, utils from unidecode import unidecode from app import models -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb +from app.config import UserConfig +from app.db.libdata import AlbumTable, ArtistTable, TrackTable + +# from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.models.enums import FavType from app.models.track import Track from app.serializers.album import serialize_for_card as serialize_album from app.serializers.album import serialize_for_card_many as serialize_albums from app.serializers.artist import serialize_for_cards from app.serializers.track import serialize_track, serialize_tracks -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore -from app.store.tracks import TrackStore + +# from app.store.albums import AlbumStore +# from app.store.artists import ArtistStore +# from app.store.tracks import TrackStore + from app.utils.remove_duplicates import remove_duplicates # ratio = fuzz.ratio @@ -49,7 +54,8 @@ class Limit: class SearchTracks: def __init__(self, query: str) -> None: self.query = query - self.tracks = TrackStore.tracks + # self.tracks = TrackStore.tracks + self.tracks = TrackTable.get_all() def __call__(self) -> List[models.Track]: """ @@ -72,7 +78,8 @@ class SearchTracks: class SearchArtists: def __init__(self, query: str) -> None: self.query = query - self.artists = ArtistStore.artists + # self.artists = ArtistStore.artists + self.artists = ArtistTable.get_all() def __call__(self): """ @@ -94,7 +101,8 @@ class SearchArtists: class SearchAlbums: def __init__(self, query: str) -> None: self.query = query - self.albums = AlbumStore.albums + # self.albums = AlbumStore.albums + self.albums = AlbumTable.get_all() def __call__(self) -> List[models.Album]: """ @@ -137,7 +145,7 @@ _S2 = TypeVar("_S2") _ResultType = int | float -def get_titles(items: _type): +def get_titles(items: list[_type]): for item in items: if isinstance(item, models.Track): text = item.og_title @@ -161,9 +169,9 @@ class TopResults: def collect_all(): all_items: list[_type] = [] - all_items.extend(ArtistStore.artists) - all_items.extend(TrackStore.tracks) - all_items.extend(AlbumStore.albums) + all_items.extend(ArtistTable.get_all()) + all_items.extend(TrackTable.get_all()) + all_items.extend(AlbumTable.get_all()) return all_items, get_titles(all_items) @@ -186,22 +194,16 @@ class TopResults: return {"type": "track", "item": item} if isinstance(item, models.Album): - tracks = TrackStore.get_tracks_by_albumhash(item.albumhash) + tracks = TrackTable.get_tracks_by_albumhash(item.albumhash) tracks = remove_duplicates(tracks) - item.get_date_from_tracks(tracks) try: item.duration = sum((t.duration for t in tracks)) except AttributeError: item.duration = 0 - item.is_single(tracks) - - if not item.is_single: - item.check_type() - - item.is_favorite = favdb.check_is_favorite( - item.albumhash, fav_type=FavType.album + item.check_type( + tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles ) return {"type": "album", "item": item} @@ -210,15 +212,18 @@ class TopResults: track_count = 0 duration = 0 - for track in TrackStore.get_tracks_by_artisthash(item.artisthash): + tracks = TrackTable.get_tracks_by_artisthash(item.artisthash) + tracks = remove_duplicates(tracks) + + for track in tracks: track_count += 1 duration += track.duration - album_count = AlbumStore.count_albums_by_artisthash(item.artisthash) + # album_count = AlbumStore.count_albums_by_artisthash(item.artisthash) - item.set_trackcount(track_count) - item.set_albumcount(album_count) - item.set_duration(duration) + # item.set_trackcount(track_count) + # item.set_albumcount(album_count) + # item.set_duration(duration) return {"type": "artist", "item": item} @@ -230,7 +235,8 @@ class TopResults: tracks.extend(SearchTracks(query)()) if item["type"] == "album": - t = TrackStore.get_tracks_by_albumhash(item["item"].albumhash) + t = TrackTable.get_tracks_by_albumhash(item["item"].albumhash) + # t = TrackStore.get_tracks_by_albumhash(item["item"].albumhash) t.sort(key=lambda x: x.last_mod) # if there are less than the limit, get more tracks @@ -242,7 +248,8 @@ class TopResults: tracks.extend(t) if item["type"] == "artist": - t = TrackStore.get_tracks_by_artisthash(item["item"].artisthash) + # t = TrackStore.get_tracks_by_artisthash(item["item"].artisthash) + t = TrackTable.get_tracks_by_artisthash(item["item"].artisthash) # if there are less than the limit, get more tracks if len(t) < limit: @@ -263,7 +270,8 @@ class TopResults: return SearchAlbums(query)()[:limit] if item["type"] == "artist": - albums = AlbumStore.get_albums_by_artisthash(item["item"].artisthash) + # albums = AlbumStore.get_albums_by_artisthash(item["item"].artisthash) + albums = AlbumTable.get_albums_by_artisthash(item["item"].artisthash) # if there are less than the limit, get more albums if len(albums) < limit: @@ -279,7 +287,6 @@ class TopResults: limit: int = None, albums_only=False, tracks_only=False, - in_quotes=False, ): items, titles = TopResults.collect_all() results = TopResults.get_results(titles, query) @@ -307,21 +314,13 @@ class TopResults: result = TopResults.map_with_type(result) - if in_quotes: - top_tracks = SearchTracks(query)()[:tracks_limit] - else: - top_tracks = TopResults.get_track_items(result, query, limit=tracks_limit) - + top_tracks = TopResults.get_track_items(result, query, limit=tracks_limit) top_tracks = serialize_tracks(top_tracks) if tracks_only: return top_tracks - if in_quotes: - albums = SearchAlbums(query)()[:albums_limit] - else: - albums = TopResults.get_album_items(result, query, limit=albums_limit) - + albums = TopResults.get_album_items(result, query, limit=albums_limit) albums = serialize_albums(albums) if albums_only: diff --git a/app/models/album.py b/app/models/album.py index dfcb7137..f6262b8a 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -112,21 +112,26 @@ class Album: Runs all the checks to determine the type of album. """ if self.is_single(tracks, singleTrackAsSingle): - return "single" + self.type = "single" + return if self.is_soundtrack(): - return "soundtrack" + self.type = "soundtrack" + return if self.is_live_album(): - return "live album" + self.type = "live album" + return if self.is_compilation(): - return "compilation" + self.type = "compilation" + return if self.is_ep(): - return "ep" + self.type = "ep" + return - return "album" + self.type = "album" def is_soundtrack(self) -> bool: """ diff --git a/app/models/playlist.py b/app/models/playlist.py index d413010d..16282f67 100644 --- a/app/models/playlist.py +++ b/app/models/playlist.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any from app import settings +from app.utils.auth import get_current_userid @dataclass(slots=True) @@ -16,10 +17,10 @@ class Playlist: last_updated: str name: str settings: dict - userid: int - trackhashes: list[str] + trackhashes: list[str] = dataclasses.field(default_factory=list) extra: dict[str, Any] = dataclasses.field(default_factory=dict) + userid: int | None = None thumb: str = "" count: int = 0 duration: int = 0 @@ -28,11 +29,10 @@ class Playlist: pinned: bool = False def __post_init__(self): - # self.trackhashes = json.loads(str(self.trackhashes)) - # self.count = len(self.trackhashes) + self.count = len(self.trackhashes) - # if isinstance(self.settings, str): - # self.settings = dict(json.loads(self.settings)) + if self.userid is None: + self.userid = get_current_userid() self.pinned = self.settings.get("pinned", False) self.has_image = ( @@ -45,12 +45,6 @@ class Playlist: self.image = "None" self.thumb = "None" - # def set_duration(self, duration: int): - # self.duration = duration - - # def set_count(self, count: int): - # self.count = count - def clear_lists(self): """ Removes data from lists to make it lighter for sending diff --git a/app/settings.py b/app/settings.py index d8a7036f..32c36bde 100644 --- a/app/settings.py +++ b/app/settings.py @@ -126,7 +126,7 @@ class Defaults: SM_ARTIST_IMG_SIZE = 128 MD_ARTIST_IMG_SIZE = 256 - HASH_LENGTH = 10 + HASH_LENGTH = 16 API_ALBUMHASH = "bfe300e966" API_ARTISTHASH = "cae59f1fc5" API_TRACKHASH = "0853280a12" diff --git a/app/utils/__init__.py b/app/utils/__init__.py index 7cce8009..c432160c 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -1,5 +1,5 @@ import locale -from typing import TypeVar +from typing import Iterable, TypeVar T = TypeVar("T") @@ -16,5 +16,5 @@ def format_number(number: float) -> str: -def flatten(list_: list[list[T]]) -> list[T]: +def flatten(list_: Iterable[list[T]]) -> list[T]: return [item for sublist in list_ for item in sublist] diff --git a/app/utils/hashing.py b/app/utils/hashing.py index 98a883ea..7671012e 100644 --- a/app/utils/hashing.py +++ b/app/utils/hashing.py @@ -1,4 +1,5 @@ import hashlib +import xxhash from unidecode import unidecode @@ -32,11 +33,12 @@ def create_hash(*args: str, decode=False, limit=10) -> str: str_ = unidecode(str_) str_ = str_.encode("utf-8") - str_ = hashlib.sha1(str_).hexdigest() + return xxhash.xxh3_64(str_).hexdigest() + # str_ = hashlib.sha1(str_).hexdigest() # INFO: Return first 5 + last 5 characters - return ( - str_[: limit // 2] + str_[-limit // 2 :] - if limit % 2 == 0 - else str_[: limit // 2] + str_[-limit // 2 - 1 :] - ) + # return ( + # str_[: limit // 2] + str_[-limit // 2 :] + # if limit % 2 == 0 + # else str_[: limit // 2] + str_[-limit // 2 - 1 :] + # ) diff --git a/poetry.lock b/poetry.lock index 10c11ade..06d77703 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2542,6 +2542,123 @@ files = [ {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] +[[package]] +name = "xxhash" +version = "3.4.1" +description = "Python binding for xxHash" +optional = false +python-versions = ">=3.7" +files = [ + {file = "xxhash-3.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91dbfa55346ad3e18e738742236554531a621042e419b70ad8f3c1d9c7a16e7f"}, + {file = "xxhash-3.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:665a65c2a48a72068fcc4d21721510df5f51f1142541c890491afc80451636d2"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb11628470a6004dc71a09fe90c2f459ff03d611376c1debeec2d648f44cb693"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bef2a7dc7b4f4beb45a1edbba9b9194c60a43a89598a87f1a0226d183764189"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c0f7b2d547d72c7eda7aa817acf8791f0146b12b9eba1d4432c531fb0352228"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00f2fdef6b41c9db3d2fc0e7f94cb3db86693e5c45d6de09625caad9a469635b"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23cfd9ca09acaf07a43e5a695143d9a21bf00f5b49b15c07d5388cadf1f9ce11"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6a9ff50a3cf88355ca4731682c168049af1ca222d1d2925ef7119c1a78e95b3b"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f1d7c69a1e9ca5faa75546fdd267f214f63f52f12692f9b3a2f6467c9e67d5e7"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:672b273040d5d5a6864a36287f3514efcd1d4b1b6a7480f294c4b1d1ee1b8de0"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4178f78d70e88f1c4a89ff1ffe9f43147185930bb962ee3979dba15f2b1cc799"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9804b9eb254d4b8cc83ab5a2002128f7d631dd427aa873c8727dba7f1f0d1c2b"}, + {file = "xxhash-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c09c49473212d9c87261d22c74370457cfff5db2ddfc7fd1e35c80c31a8c14ce"}, + {file = "xxhash-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ebbb1616435b4a194ce3466d7247df23499475c7ed4eb2681a1fa42ff766aff6"}, + {file = "xxhash-3.4.1-cp310-cp310-win_arm64.whl", hash = "sha256:25dc66be3db54f8a2d136f695b00cfe88018e59ccff0f3b8f545869f376a8a46"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58c49083801885273e262c0f5bbeac23e520564b8357fbb18fb94ff09d3d3ea5"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b526015a973bfbe81e804a586b703f163861da36d186627e27524f5427b0d520"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36ad4457644c91a966f6fe137d7467636bdc51a6ce10a1d04f365c70d6a16d7e"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:248d3e83d119770f96003271fe41e049dd4ae52da2feb8f832b7a20e791d2920"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2070b6d5bbef5ee031666cf21d4953c16e92c2f8a24a94b5c240f8995ba3b1d0"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2746035f518f0410915e247877f7df43ef3372bf36cfa52cc4bc33e85242641"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a8ba6181514681c2591840d5632fcf7356ab287d4aff1c8dea20f3c78097088"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0aac5010869240e95f740de43cd6a05eae180c59edd182ad93bf12ee289484fa"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4cb11d8debab1626181633d184b2372aaa09825bde709bf927704ed72765bed1"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b29728cff2c12f3d9f1d940528ee83918d803c0567866e062683f300d1d2eff3"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a15cbf3a9c40672523bdb6ea97ff74b443406ba0ab9bca10ceccd9546414bd84"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e66df260fed01ed8ea790c2913271641c58481e807790d9fca8bfd5a3c13844"}, + {file = "xxhash-3.4.1-cp311-cp311-win32.whl", hash = "sha256:e867f68a8f381ea12858e6d67378c05359d3a53a888913b5f7d35fbf68939d5f"}, + {file = "xxhash-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:200a5a3ad9c7c0c02ed1484a1d838b63edcf92ff538770ea07456a3732c577f4"}, + {file = "xxhash-3.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:1d03f1c0d16d24ea032e99f61c552cb2b77d502e545187338bea461fde253583"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c4bbba9b182697a52bc0c9f8ec0ba1acb914b4937cd4a877ad78a3b3eeabefb3"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9fd28a9da300e64e434cfc96567a8387d9a96e824a9be1452a1e7248b7763b78"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6066d88c9329ab230e18998daec53d819daeee99d003955c8db6fc4971b45ca3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93805bc3233ad89abf51772f2ed3355097a5dc74e6080de19706fc447da99cd3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64da57d5ed586ebb2ecdde1e997fa37c27fe32fe61a656b77fabbc58e6fbff6e"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a97322e9a7440bf3c9805cbaac090358b43f650516486746f7fa482672593df"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbe750d512982ee7d831838a5dee9e9848f3fb440e4734cca3f298228cc957a6"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fd79d4087727daf4d5b8afe594b37d611ab95dc8e29fe1a7517320794837eb7d"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:743612da4071ff9aa4d055f3f111ae5247342931dedb955268954ef7201a71ff"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b41edaf05734092f24f48c0958b3c6cbaaa5b7e024880692078c6b1f8247e2fc"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:a90356ead70d715fe64c30cd0969072de1860e56b78adf7c69d954b43e29d9fa"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac56eebb364e44c85e1d9e9cc5f6031d78a34f0092fea7fc80478139369a8b4a"}, + {file = "xxhash-3.4.1-cp312-cp312-win32.whl", hash = "sha256:911035345932a153c427107397c1518f8ce456f93c618dd1c5b54ebb22e73747"}, + {file = "xxhash-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:f31ce76489f8601cc7b8713201ce94b4bd7b7ce90ba3353dccce7e9e1fee71fa"}, + {file = "xxhash-3.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:b5beb1c6a72fdc7584102f42c4d9df232ee018ddf806e8c90906547dfb43b2da"}, + {file = "xxhash-3.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6d42b24d1496deb05dee5a24ed510b16de1d6c866c626c2beb11aebf3be278b9"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b685fab18876b14a8f94813fa2ca80cfb5ab6a85d31d5539b7cd749ce9e3624"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:419ffe34c17ae2df019a4685e8d3934d46b2e0bbe46221ab40b7e04ed9f11137"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e041ce5714f95251a88670c114b748bca3bf80cc72400e9f23e6d0d59cf2681"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc860d887c5cb2f524899fb8338e1bb3d5789f75fac179101920d9afddef284b"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:312eba88ffe0a05e332e3a6f9788b73883752be63f8588a6dc1261a3eaaaf2b2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e01226b6b6a1ffe4e6bd6d08cfcb3ca708b16f02eb06dd44f3c6e53285f03e4f"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9f3025a0d5d8cf406a9313cd0d5789c77433ba2004b1c75439b67678e5136537"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:6d3472fd4afef2a567d5f14411d94060099901cd8ce9788b22b8c6f13c606a93"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:43984c0a92f06cac434ad181f329a1445017c33807b7ae4f033878d860a4b0f2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a55e0506fdb09640a82ec4f44171273eeabf6f371a4ec605633adb2837b5d9d5"}, + {file = "xxhash-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:faec30437919555b039a8bdbaba49c013043e8f76c999670aef146d33e05b3a0"}, + {file = "xxhash-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:c9e1b646af61f1fc7083bb7b40536be944f1ac67ef5e360bca2d73430186971a"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:961d948b7b1c1b6c08484bbce3d489cdf153e4122c3dfb07c2039621243d8795"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:719a378930504ab159f7b8e20fa2aa1896cde050011af838af7e7e3518dd82de"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fb5cb9406ccd7c4dd917f16630d2e5e8cbbb02fc2fca4e559b2a47a64f4940"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5dab508ac39e0ab988039bc7f962c6ad021acd81fd29145962b068df4148c476"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c59f3e46e7daf4c589e8e853d700ef6607afa037bfad32c390175da28127e8c"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc07256eff0795e0f642df74ad096f8c5d23fe66bc138b83970b50fc7f7f6c5"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9f749999ed80f3955a4af0eb18bb43993f04939350b07b8dd2f44edc98ffee9"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7688d7c02149a90a3d46d55b341ab7ad1b4a3f767be2357e211b4e893efbaaf6"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a8b4977963926f60b0d4f830941c864bed16aa151206c01ad5c531636da5708e"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8106d88da330f6535a58a8195aa463ef5281a9aa23b04af1848ff715c4398fb4"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4c76a77dbd169450b61c06fd2d5d436189fc8ab7c1571d39265d4822da16df22"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:11f11357c86d83e53719c592021fd524efa9cf024dc7cb1dfb57bbbd0d8713f2"}, + {file = "xxhash-3.4.1-cp38-cp38-win32.whl", hash = "sha256:0c786a6cd74e8765c6809892a0d45886e7c3dc54de4985b4a5eb8b630f3b8e3b"}, + {file = "xxhash-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:aabf37fb8fa27430d50507deeab2ee7b1bcce89910dd10657c38e71fee835594"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6127813abc1477f3a83529b6bbcfeddc23162cece76fa69aee8f6a8a97720562"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef2e194262f5db16075caea7b3f7f49392242c688412f386d3c7b07c7733a70a"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71be94265b6c6590f0018bbf73759d21a41c6bda20409782d8117e76cd0dfa8b"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10e0a619cdd1c0980e25eb04e30fe96cf8f4324758fa497080af9c21a6de573f"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa122124d2e3bd36581dd78c0efa5f429f5220313479fb1072858188bc2d5ff1"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17032f5a4fea0a074717fe33477cb5ee723a5f428de7563e75af64bfc1b1e10"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca7783b20e3e4f3f52f093538895863f21d18598f9a48211ad757680c3bd006f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d77d09a1113899fad5f354a1eb4f0a9afcf58cefff51082c8ad643ff890e30cf"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:21287bcdd299fdc3328cc0fbbdeaa46838a1c05391264e51ddb38a3f5b09611f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:dfd7a6cc483e20b4ad90224aeb589e64ec0f31e5610ab9957ff4314270b2bf31"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:543c7fcbc02bbb4840ea9915134e14dc3dc15cbd5a30873a7a5bf66039db97ec"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fe0a98d990e433013f41827b62be9ab43e3cf18e08b1483fcc343bda0d691182"}, + {file = "xxhash-3.4.1-cp39-cp39-win32.whl", hash = "sha256:b9097af00ebf429cc7c0e7d2fdf28384e4e2e91008130ccda8d5ae653db71e54"}, + {file = "xxhash-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:d699b921af0dcde50ab18be76c0d832f803034d80470703700cb7df0fbec2832"}, + {file = "xxhash-3.4.1-cp39-cp39-win_arm64.whl", hash = "sha256:2be491723405e15cc099ade1280133ccfbf6322d2ef568494fb7d07d280e7eee"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:431625fad7ab5649368c4849d2b49a83dc711b1f20e1f7f04955aab86cd307bc"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc6dbd5fc3c9886a9e041848508b7fb65fd82f94cc793253990f81617b61fe49"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ff8dbd0ec97aec842476cb8ccc3e17dd288cd6ce3c8ef38bff83d6eb927817"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef73a53fe90558a4096e3256752268a8bdc0322f4692ed928b6cd7ce06ad4fe3"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:450401f42bbd274b519d3d8dcf3c57166913381a3d2664d6609004685039f9d3"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a162840cf4de8a7cd8720ff3b4417fbc10001eefdd2d21541a8226bb5556e3bb"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b736a2a2728ba45017cb67785e03125a79d246462dfa892d023b827007412c52"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d0ae4c2e7698adef58710d6e7a32ff518b66b98854b1c68e70eee504ad061d8"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6322c4291c3ff174dcd104fae41500e75dad12be6f3085d119c2c8a80956c51"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:dd59ed668801c3fae282f8f4edadf6dc7784db6d18139b584b6d9677ddde1b6b"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92693c487e39523a80474b0394645b393f0ae781d8db3474ccdcead0559ccf45"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4603a0f642a1e8d7f3ba5c4c25509aca6a9c1cc16f85091004a7028607ead663"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa45e8cbfbadb40a920fe9ca40c34b393e0b067082d94006f7f64e70c7490a6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:595b252943b3552de491ff51e5bb79660f84f033977f88f6ca1605846637b7c6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:562d8b8f783c6af969806aaacf95b6c7b776929ae26c0cd941d54644ea7ef51e"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:41ddeae47cf2828335d8d991f2d2b03b0bdc89289dc64349d712ff8ce59d0647"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c44d584afdf3c4dbb3277e32321d1a7b01d6071c1992524b6543025fb8f4206f"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd7bddb3a5b86213cc3f2c61500c16945a1b80ecd572f3078ddbbe68f9dabdfb"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ecb6c987b62437c2f99c01e97caf8d25660bf541fe79a481d05732e5236719c"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:696b4e18b7023527d5c50ed0626ac0520edac45a50ec7cf3fc265cd08b1f4c03"}, + {file = "xxhash-3.4.1.tar.gz", hash = "sha256:0379d6cf1ff987cd421609a264ce025e74f346e3e145dd106c0cc2e3ec3f99a9"}, +] + [[package]] name = "zope-event" version = "5.0" @@ -2616,4 +2733,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "80cb2755efc6cec2cb20d50cb8927dee554991741283e70e7a2665e6253b895d" +content-hash = "184c1c56051131473212b74de5719e2629c630555b13cd4cd2de371b3a2fb195" diff --git a/pyproject.toml b/pyproject.toml index 65574f5d..695cc857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ flask-jwt-extended = "^4.6.0" sqlalchemy = "^2.0.31" memory-profiler = "^0.61.0" sortedcontainers = "^2.4.0" +xxhash = "^3.4.1" [tool.poetry.dev-dependencies] pylint = "^2.15.5" From 2e63aa4a4119105cfc94fac0b57d7f95acf07bf4 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Wed, 3 Jul 2024 15:57:05 +0300 Subject: [PATCH 11/44] fix: artist track count + fix: album type on artits page + sort by release date help text on get all --- app/api/album.py | 2 +- app/api/getall/__init__.py | 2 +- app/lib/tagger.py | 12 ++++++++++-- app/lib/taglib.py | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/app/api/album.py b/app/api/album.py index 6deb7f7c..d03b3025 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -48,7 +48,7 @@ def get_album_tracks_and_info(body: AlbumHashSchema): tracks = TrackDb.get_tracks_by_albumhash(albumhash) album.trackcount = len(tracks) album.duration = sum(t.duration for t in tracks) - album.type = album.check_type( + album.check_type( tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles ) diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index 5489803a..44e0f6f1 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -107,7 +107,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): item_dict = serialize_album(item) if is_albums else serialize_artist(item) if sort_is_date: - item_dict["help_text"] = item.date + item_dict["help_text"] = datetime.fromtimestamp(item.date).year if sort_is_create_date: date = create_new_date(datetime.fromtimestamp(item.created_date)) diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 052a7dff..46926aeb 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -210,6 +210,7 @@ class IndexArtists: for a in track.albumartists: if a not in this_artists: + a["in_track"] = False this_artists.append(a) for thisartist in this_artists: @@ -228,7 +229,11 @@ class IndexArtists: "playcount": track.playcount, "playduration": track.playduration, "trackcount": None, - "tracks": {track.trackhash}, + "tracks": ( + {track.trackhash} + if thisartist.get("in_track", True) + else set() + ), } else: artist = artists[thisartist["artisthash"]] @@ -236,12 +241,15 @@ class IndexArtists: artist["playcount"] += track.playcount artist["playduration"] += track.playduration artist["albums"].add(track.albumhash) - artist["tracks"].add(track.trackhash) artist["date"] = min(artist["date"], track.date) artist["lastplayed"] = max(artist["lastplayed"], track.lastplayed) artist["created_date"] = min(artist["created_date"], track.last_mod) artist["names"].add(thisartist["name"]) + + if thisartist.get("in_track", True): + artist["tracks"].add(track.trackhash) + if track.genres: artist["genres"].extend(track.genres) diff --git a/app/lib/taglib.py b/app/lib/taglib.py index a987cc17..e0a00ae1 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -269,7 +269,7 @@ def get_tags(filepath: str, artist_separators: set[str]): ] tags.artisthashes = list( - {a["artisthash"] for a in tags.artists + tags.albumartists} + {a["artisthash"] for a in tags.artists} ) # remove prod by From 678eed3ab6280b1ace3ceac2b25041535aa5db4e Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 4 Jul 2024 11:47:08 +0300 Subject: [PATCH 12/44] remove comments from album model --- .github/changelog.md | 18 ++++++---- app/models/album.py | 78 -------------------------------------------- 2 files changed, 11 insertions(+), 85 deletions(-) diff --git a/.github/changelog.md b/.github/changelog.md index e0348801..5132e9bf 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -1,20 +1,23 @@ # What's New? + - Auth - New artists/albums Sort by: last played, no. of streams, total stream duration +- Option to show now playing track info on tab title. Go to Settings > Appearance to enable ## Improvements + - The context menu now doesn't take forever to open up - Merged "Save as Playlist" with "Add to Playlist" > "New Playlist" ## Bug fixes + - Add to queue adding to last index -1 -- ## Development -- Rewritten the whole DB layer to move stores from memory to the database. +- Rewritten the whole DB layer to move stores from memory to the database. ## THE BIG ONE API CHANGES @@ -22,13 +25,14 @@ ```ts interface Genre { - name: str; - genrehash: str; + name: str; + genrehash: str; } ``` - Pairing via QR Code has been split into 2 endpoint: - 1. `/getpaircode` - 2. `/pair` -- \ No newline at end of file + 1. `/getpaircode` + 2. `/pair` + +- diff --git a/app/models/album.py b/app/models/album.py index f6262b8a..fb3cda97 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -39,62 +39,9 @@ class Album: versions: list[str] = dataclasses.field(default_factory=list) def __post_init__(self): - # self.date = datetime.datetime.fromtimestamp(self.date).year self.image = self.albumhash + ".webp" self.populate_versions() - # albumhash: str - # title: str - # albumartists: list[Artist] - - # albumartists_hashes: str = "" - # image: str = "" - # count: int = 0 - # duration: int = 0 - # colors: list[str] = dataclasses.field(default_factory=list) - # date: str = "" - - # created_date: int = 0 - # og_title: str = "" - # base_title: str = "" - # is_soundtrack: bool = False - # is_compilation: bool = False - # is_single: bool = False - # is_EP: bool = False - # is_favorite: bool = False - # is_live: bool = False - - # genres: list[str] = dataclasses.field(default_factory=list) - - # def __post_init__(self): - # self.title = self.title.strip() - # self.og_title = self.title - # self.image = self.albumhash + ".webp" - - # # Fetch album artists from title - # if get_flag(SessionVarKeys.EXTRACT_FEAT): - # featured, self.title = parse_feat_from_title(self.title) - - # if len(featured) > 0: - # original_lower = "-".join([a.name.lower() for a in self.albumartists]) - # self.albumartists.extend( - # [Artist(a) for a in featured if a.lower() not in original_lower] - # ) - - # from ..store.tracks import TrackStore - - # TrackStore.append_track_artists(self.albumhash, featured, self.title) - - # # Handle album version data - # else: - # self.base_title = get_base_title_and_versions( - # self.title, get_versions=False - # )[0] - - # self.albumartists_hashes = "-".join(a.artisthash for a in self.albumartists) - - # # def set_colors(self, colors: list[str]): - # # self.colors = colors def populate_versions(self): _, self.versions = get_base_title_and_versions(self.og_title, get_versions=True) @@ -223,28 +170,3 @@ class Album: # TODO: Review -> Are the above commented checks necessary? ): return True - - # def get_date_from_tracks(self, tracks: list[Track]): - # """ - # Gets the date of the album its tracks. - - # Args: - # tracks (list[Track]): The tracks of the album. - # """ - # if self.date: - # return - - # dates = (int(t.date) for t in tracks if t.date) - # try: - # self.date = datetime.datetime.fromtimestamp(min(dates)).year - # except: - # self.date = datetime.datetime.now().year - - # def set_count(self, count: int): - # self.count = count - - # def set_duration(self, duration: int): - # self.duration = duration - - # def set_created_date(self, created_date: int): - # self.created_date = created_date From a76e91cf5a11ef82bfdb7a479446257905de5eef Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 5 Jul 2024 04:43:39 +0300 Subject: [PATCH 13/44] fix: duplication of artist albums on album/from-artist + remove more fields from artist, album and artist models on serializers --- app/api/album.py | 42 ++++++++++++++++++++------------------- app/api/artist.py | 3 ++- app/db/libdata.py | 6 +++--- app/serializers/album.py | 11 +++++++++- app/serializers/artist.py | 9 +++++++++ app/serializers/track.py | 9 +++++++-- 6 files changed, 53 insertions(+), 27 deletions(-) diff --git a/app/api/album.py b/app/api/album.py index d03b3025..6c8ab65f 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -2,10 +2,8 @@ Contains all the album routes. """ -from itertools import groupby import random -from flask_jwt_extended import current_user from pydantic import Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint @@ -16,10 +14,11 @@ from app.db.libdata import ArtistTable from app.db.libdata import AlbumTable as AlbumDb, TrackTable as TrackDb from app.db.userdata import SimilarArtistTable from app.settings import Defaults +from app.utils import flatten from app.utils.hashing import create_hash from app.lib.albumslib import sort_by_track_no -from app.serializers.album import serialize_for_card, serialize_for_card_many -from app.serializers.track import serialize_track, serialize_tracks +from app.serializers.album import serialize_for_card_many +from app.serializers.track import serialize_tracks from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb @@ -86,7 +85,6 @@ def get_album_tracks(path: AlbumHashSchema): class GetMoreFromArtistsBody(AlbumLimitSchema): albumartists: list = Field( description="The artist hashes to get more albums from", - example='[{"name": "Khalid", "artisthash": "94ca2dba1c"}]', ) base_title: str = Field( @@ -108,23 +106,26 @@ def get_more_from_artist(body: GetMoreFromArtistsBody): base_title = body.base_title all_albums = AlbumDb.get_albums_by_artisthashes(albumartists) + seen_hashes = set() - # filter out albums with the same base title - all_albums = filter( - lambda a: create_hash(a.base_title) != create_hash(base_title), all_albums - ) - all_albums = list(all_albums) + for artisthash, albums in all_albums.items(): + albums = [ + a + for a in albums + # INFO: filter out albums added to other artists + if a.albumhash not in seen_hashes + # INFO: filter out albums with the same base title + and create_hash(a.base_title) != create_hash(base_title) + ] + all_albums[artisthash] = serialize_for_card_many( + [a for a in albums if create_hash(a.base_title) != create_hash(base_title)][ + :limit + ] + ) + # INFO: record albums added to other artists + seen_hashes.update([a.albumhash for a in albums][:limit]) - if not len(all_albums): - return [] - - # group by first albumartist's artisthash - groups = groupby(all_albums, lambda a: a.albumartists[0]["artisthash"]) - - return [ - {"artisthash": g[0], "albums": serialize_for_card_many(list(g[1])[:limit])} - for g in groups - ] + return all_albums class GetAlbumVersionsBody(ArtistHashSchema): @@ -183,6 +184,7 @@ def get_similar_albums(query: GetSimilarAlbumsQuery): artists = ArtistTable.get_artists_by_artisthashes(artisthashes) albums = AlbumDb.get_albums_by_artisthashes([a.artisthash for a in artists]) + albums = flatten(albums.values()) sample = random.sample(albums, min(len(albums), limit)) return serialize_for_card_many(sample[:limit]) diff --git a/app/api/artist.py b/app/api/artist.py index 65bc6798..d2b7e2af 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -22,6 +22,7 @@ from app.db.libdata import AlbumTable, TrackTable from app.db.userdata import SimilarArtistTable from app.serializers.album import serialize_for_card_many +from app.serializers.artist import serialize_for_cards from app.serializers.track import serialize_tracks bp_tag = Tag(name="Artist", description="Single artist") @@ -166,4 +167,4 @@ def get_similar_artists(path: ArtistHashSchema, query: ArtistLimitSchema): if len(similar) > limit: similar = random.sample(similar, min(limit, len(similar))) - return similar[:limit] + return serialize_for_cards(similar[:limit]) diff --git a/app/db/libdata.py b/app/db/libdata.py index ecb0f5de..c769eb2e 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -311,14 +311,14 @@ class AlbumTable(Base): @classmethod def get_albums_by_artisthashes(cls, artisthashes: list[str]): with DbManager() as conn: - albums: list[AlbumModel] = [] + albums: dict[str, list[AlbumModel]] = {} for artist in artisthashes: result = conn.execute( # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! - select(AlbumTable).where(AlbumTable.albumartists.contains(artist)) + select(AlbumTable).where(AlbumTable.artisthashes.contains(artist)) ) - albums.extend(albums_to_dataclasses(result.fetchall())) + albums[artist] = (albums_to_dataclasses(result.fetchall())) return albums diff --git a/app/serializers/album.py b/app/serializers/album.py index 2a559866..4bec0b4e 100644 --- a/app/serializers/album.py +++ b/app/serializers/album.py @@ -23,11 +23,20 @@ def serialize_for_card(album: Album): props_to_remove = { "duration", "count", + "artisthashes", "albumartists_hashes", + "created_date", "og_title", "base_title", "genres", - "playcount" + "playcount", + "trackcount", + "type", + "playduration", + "genrehashes", + "extra", + "id", + "lastplayed", } return album_serializer(album, props_to_remove) diff --git a/app/serializers/artist.py b/app/serializers/artist.py index 633b0789..dc39e72e 100644 --- a/app/serializers/artist.py +++ b/app/serializers/artist.py @@ -15,6 +15,15 @@ def serialize_for_card(artist: Artist): "duration", "albumcount", "playcount", + "playduration", + "playcount", + "lastplayed", + "id", + "genres", + "genrehashes", + "extra", + "created_date", + "date", } for key in props_to_remove: diff --git a/app/serializers/track.py b/app/serializers/track.py index 10426fbb..49618050 100644 --- a/app/serializers/track.py +++ b/app/serializers/track.py @@ -3,7 +3,7 @@ from dataclasses import asdict from app.models.track import Track -def serialize_track(track: Track, to_remove: set = {}, remove_disc=True) -> dict: +def serialize_track(track: Track, to_remove: set = set(), remove_disc=True) -> dict: album_dict = asdict(track) # is_favorite @property is not included in asdict album_dict["is_favorite"] = track.is_favorite @@ -21,6 +21,11 @@ def serialize_track(track: Track, to_remove: set = {}, remove_disc=True) -> dict "created_date", "fav_userids", "playcount", + "genrehashes", + "id", + "lastplayed", + "playduration", + "genres", }.union(to_remove) if not remove_disc: @@ -42,6 +47,6 @@ def serialize_track(track: Track, to_remove: set = {}, remove_disc=True) -> dict def serialize_tracks( - tracks: list[Track], _remove: set = {}, remove_disc=True + tracks: list[Track], _remove: set = set(), remove_disc=True ) -> list[dict]: return [serialize_track(t, _remove, remove_disc) for t in tracks] From 727dea748cc2db0cdfa83862b7828b6671bbfe86 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 5 Jul 2024 05:00:28 +0300 Subject: [PATCH 14/44] remove hardcoded db location + move Db engine to module --- app/db/__init__.py | 20 +++----------------- app/db/engine.py | 5 +++++ app/db/libdata.py | 5 ++--- app/setup/sqlite.py | 10 ++++++++++ pyproject.toml | 1 + 5 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 app/db/engine.py diff --git a/app/db/__init__.py b/app/db/__init__.py index 4b1c4366..4f7fb524 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -1,7 +1,6 @@ from typing import Any from sqlalchemy import ( - create_engine, delete, func, insert, @@ -15,18 +14,7 @@ from sqlalchemy.orm import ( MappedAsDataclass, ) -# ============================================================ -# TODO: Make sure the database is created before we run this. -fullpath = "/home/cwilvx/temp/swingmusic/swing.db" -engine = create_engine( - f"sqlite+pysqlite:///{fullpath}", - echo=False, - max_overflow=0, - pool_size=5, -) - -# connection = engine.connect() - +from app.db.engine import DbEngine @event.listens_for(Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): @@ -38,12 +26,10 @@ def set_sqlite_pragma(dbapi_connection, connection_record): class DbManager: def __init__(self, commit: bool = False): self.commit = commit - self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) - self.conn = self.engine.connect() + self.conn = DbEngine.engine.connect() def __enter__(self): return self.conn.execution_options(preserve_rowcount=True) - # return connection def __exit__(self, exc_type, exc_val, exc_tb): if self.commit: @@ -92,4 +78,4 @@ class Base(MappedAsDataclass, DeclarativeBase): def create_all(): - Base().metadata.create_all(engine) + Base().metadata.create_all(DbEngine.engine) diff --git a/app/db/engine.py b/app/db/engine.py new file mode 100644 index 00000000..233ba004 --- /dev/null +++ b/app/db/engine.py @@ -0,0 +1,5 @@ +from sqlalchemy import Engine + + +class DbEngine: + engine: Engine = None diff --git a/app/db/libdata.py b/app/db/libdata.py index c769eb2e..f68aa95c 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -12,8 +12,7 @@ from app.db.utils import ( ) from app.models import Album as AlbumModel from app.utils.remove_duplicates import remove_duplicates -from app.db import engine - +from app.db.engine import DbEngine from sqlalchemy import JSON, Boolean, Integer, String, delete, select, update from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase @@ -25,7 +24,7 @@ def create_all(): """ Create all the tables defined in this file. """ - Base.metadata.create_all(engine) + Base.metadata.create_all(DbEngine.engine) class Base(MasterBase, DeclarativeBase): diff --git a/app/setup/sqlite.py b/app/setup/sqlite.py index 8ce8dbe5..81d60b7e 100644 --- a/app/setup/sqlite.py +++ b/app/setup/sqlite.py @@ -3,6 +3,7 @@ Module to setup Sqlite databases and tables. Applies migrations. """ +from sqlalchemy import create_engine from app.db.userdata import UserTable from app.db.sqlite import create_connection, create_tables, queries from app.db.sqlite.auth import SQLiteAuthMethods as authdb @@ -12,6 +13,8 @@ from app.settings import Db from app.db import create_all from app.db.libdata import create_all as create_all_libdata +from app.db.engine import DbEngine + def run_migrations(): """ @@ -24,6 +27,13 @@ def setup_sqlite(): """ Create Sqlite databases and tables. """ + DbEngine.engine = create_engine( + f"sqlite+pysqlite:///{Db.get_app_db_path()}", + echo=False, + max_overflow=0, + pool_size=25, + ) + create_all() create_all_libdata() diff --git a/pyproject.toml b/pyproject.toml index 695cc857..8275feb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ name = "Swing music player" version = "0.1.0" description = "" authors = ["geoffrey45 "] +package-mode = false [tool.poetry.dependencies] python = ">=3.10,<3.12" From 09d67c866079e3f10484c271fb8f2a88505ccd2c Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 5 Jul 2024 05:02:42 +0300 Subject: [PATCH 15/44] increase db connection pool limit --- app/setup/sqlite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/setup/sqlite.py b/app/setup/sqlite.py index 81d60b7e..7b2a82c4 100644 --- a/app/setup/sqlite.py +++ b/app/setup/sqlite.py @@ -30,8 +30,8 @@ def setup_sqlite(): DbEngine.engine = create_engine( f"sqlite+pysqlite:///{Db.get_app_db_path()}", echo=False, - max_overflow=0, - pool_size=25, + max_overflow=20, + pool_size=10, ) create_all() From 104ef4c34663dd5482bf2706b81061c05d1bbc4a Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 5 Jul 2024 05:33:01 +0300 Subject: [PATCH 16/44] index filepaths --- app/db/libdata.py | 2 +- app/setup/sqlite.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/db/libdata.py b/app/db/libdata.py index f68aa95c..23e93af9 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -116,7 +116,7 @@ class TrackTable(Base): date: Mapped[int] = mapped_column(Integer(), nullable=True) disc: Mapped[int] = mapped_column(Integer()) duration: Mapped[int] = mapped_column(Integer()) - filepath: Mapped[str] = mapped_column(String(), unique=True) + filepath: Mapped[str] = mapped_column(String(), index=True, unique=True) folder: Mapped[str] = mapped_column(String(), index=True) genrehashes: Mapped[list[str]] = mapped_column(JSON(), index=True) genres: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) diff --git a/app/setup/sqlite.py b/app/setup/sqlite.py index 7b2a82c4..22f52c5e 100644 --- a/app/setup/sqlite.py +++ b/app/setup/sqlite.py @@ -5,7 +5,6 @@ Applies migrations. from sqlalchemy import create_engine from app.db.userdata import UserTable -from app.db.sqlite import create_connection, create_tables, queries from app.db.sqlite.auth import SQLiteAuthMethods as authdb from app.migrations import apply_migrations from app.settings import Db From 32a2684ea26bbd4f514a4866a7e00a8c563cc0ba Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 6 Jul 2024 23:44:24 +0300 Subject: [PATCH 17/44] close a connection --- TODO.md | 7 ++++++- app/db/__init__.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 057b7d7d..44b864d0 100644 --- a/TODO.md +++ b/TODO.md @@ -55,4 +55,9 @@ - Create 2 way relationships, such that if an artist A is similar to another B with a certain weight, then artist B is similar to A with the same weight, unless overwritten. - Figure out how to update album/artist tables instead of deleting all rows when the app starts -- Move get all filtering and sorting operations to the database since all sort keys are table columns \ No newline at end of file +- Move get all filtering and sorting operations to the database since all sort keys are table columns + +- Paginate the following endpoints: + 1. Folder tracks + 2. Playlist tracks + \ No newline at end of file diff --git a/app/db/__init__.py b/app/db/__init__.py index 4f7fb524..e6d1cc54 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -12,6 +12,7 @@ from sqlalchemy import event from sqlalchemy.orm import ( DeclarativeBase, MappedAsDataclass, + Session ) from app.db.engine import DbEngine @@ -27,6 +28,8 @@ class DbManager: def __init__(self, commit: bool = False): self.commit = commit self.conn = DbEngine.engine.connect() + with Session(DbEngine.engine) as session: + session.connection def __enter__(self): return self.conn.execution_options(preserve_rowcount=True) @@ -35,7 +38,7 @@ class DbManager: if self.commit: self.conn.commit() - # self.conn.close() + self.conn.close() class Base(MappedAsDataclass, DeclarativeBase): From 2ba5d6c1d7169d7409466d7f8c3dac0f5cf6c5e8 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 7 Jul 2024 16:07:27 +0300 Subject: [PATCH 18/44] document + rename stuff --- TODO.md | 2 +- app/api/__init__.py | 5 +---- app/db/__init__.py | 32 ++++++++++++++++++++------------ app/db/engine.py | 27 ++++++++++++++++++++++++++- app/db/libdata.py | 5 ++++- app/db/sqlite/utils.py | 4 ++-- app/lib/albumslib.py | 10 +--------- app/lib/artistlib.py | 31 ------------------------------- app/settings.py | 2 +- app/setup/sqlite.py | 14 ++++++-------- manage.py | 12 ++++++++++-- 11 files changed, 72 insertions(+), 72 deletions(-) diff --git a/TODO.md b/TODO.md index 44b864d0..e54eebbf 100644 --- a/TODO.md +++ b/TODO.md @@ -56,8 +56,8 @@ then artist B is similar to A with the same weight, unless overwritten. - Figure out how to update album/artist tables instead of deleting all rows when the app starts - Move get all filtering and sorting operations to the database since all sort keys are table columns +- Replace the DbManager class with cls.execute() - Paginate the following endpoints: 1. Folder tracks 2. Playlist tracks - \ No newline at end of file diff --git a/app/api/__init__.py b/app/api/__init__.py index 73a0c663..b883ca7a 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -76,6 +76,7 @@ def create_api(): CORS(app, origins="*", supports_credentials=True) # RESPONSE COMPRESSION + # Only compress JSON responses Compress(app) app.config["COMPRESS_MIMETYPES"] = [ "application/json", @@ -84,10 +85,6 @@ def create_api(): # JWT jwt = JWTManager(app) - # @jwt.user_identity_loader - # def user_identity_lookup(user): - # return user - @jwt.user_lookup_loader def user_lookup_callback(_jwt_header, jwt_data): identity = jwt_data["sub"] diff --git a/app/db/__init__.py b/app/db/__init__.py index e6d1cc54..42fcb287 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -9,14 +9,12 @@ from sqlalchemy import ( from sqlalchemy.engine import Engine from sqlalchemy import event -from sqlalchemy.orm import ( - DeclarativeBase, - MappedAsDataclass, - Session -) +from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass, Session from app.db.engine import DbEngine + +# Enable foreign key constraints for SQLite @event.listens_for(Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() @@ -25,9 +23,12 @@ def set_sqlite_pragma(dbapi_connection, connection_record): class DbManager: + """ """ + def __init__(self, commit: bool = False): self.commit = commit self.conn = DbEngine.engine.connect() + with Session(DbEngine.engine) as session: session.connection @@ -42,9 +43,15 @@ class DbManager: class Base(MappedAsDataclass, DeclarativeBase): + """ + Base class for all database models. + + It has methods common to all tables. eg. `insert_one`, `insert_many`, `remove_all`, `remove_one`, `all`, `count`. + """ + @classmethod def execute(cls, stmt: Any, commit: bool = False): - with DbManager(commit=commit) as conn: + with DbEngine.manager(commit=commit) as conn: return conn.execute(stmt) @classmethod @@ -52,8 +59,7 @@ class Base(MappedAsDataclass, DeclarativeBase): """ Inserts multiple items into the database. """ - with DbManager(commit=True) as conn: - return conn.execute(insert(cls).values(items)) + return cls.execute(insert(cls).values(items), commit=True) @classmethod def insert_one(cls, item: dict[str, Any]): @@ -64,12 +70,11 @@ class Base(MappedAsDataclass, DeclarativeBase): @classmethod def remove_all(cls): - with DbManager(commit=True) as conn: - conn.execute(delete(cls)) + return cls.execute(delete(cls), commit=True) @classmethod def remove_one(cls, id: int): - cls.execute(delete(cls).where(cls.id == id), commit=True) + return cls.execute(delete(cls).where(cls.id == id), commit=True) @classmethod def all(cls): @@ -80,5 +85,8 @@ class Base(MappedAsDataclass, DeclarativeBase): return cls.execute(select(func.count()).select_from(cls)).scalar() -def create_all(): +def create_all_tables(): + """ + Creates all the tables that build on the Base class. + """ Base().metadata.create_all(DbEngine.engine) diff --git a/app/db/engine.py b/app/db/engine.py index 233ba004..841702de 100644 --- a/app/db/engine.py +++ b/app/db/engine.py @@ -1,5 +1,30 @@ +from contextlib import contextmanager from sqlalchemy import Engine class DbEngine: - engine: Engine = None + """ + The database engine instance. + """ + + engine: Engine + + @classmethod + @contextmanager + def manager(cls, commit: bool): + """ + This context manager manages access to the database. + + When the context manager is entered, it returns a connection object that can be used to execute SQL statements. + + If the `commit` parameter is set to `True`, the context manager will commit the transaction when it exits. + """ + + try: + conn = cls.engine.connect() + yield conn.execution_options(preserve_rowcount=True) + + if commit: + conn.commit() + finally: + conn.close() diff --git a/app/db/libdata.py b/app/db/libdata.py index 23e93af9..caa0b3ab 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -23,6 +23,9 @@ from typing import Any, Iterable, Optional def create_all(): """ Create all the tables defined in this file. + + NOTE: We need this function because the MasterBase does not collect + the tables defined here (as they are grand-children of the MasterBase) """ Base.metadata.create_all(DbEngine.engine) @@ -317,7 +320,7 @@ class AlbumTable(Base): # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! select(AlbumTable).where(AlbumTable.artisthashes.contains(artist)) ) - albums[artist] = (albums_to_dataclasses(result.fetchall())) + albums[artist] = albums_to_dataclasses(result.fetchall()) return albums diff --git a/app/db/sqlite/utils.py b/app/db/sqlite/utils.py index 27535d25..06a45640 100644 --- a/app/db/sqlite/utils.py +++ b/app/db/sqlite/utils.py @@ -90,10 +90,10 @@ class SQLiteManager: if self.test_db_path: db_path = self.test_db_path else: - db_path = settings.Db.get_app_db_path() + db_path = settings.DbPaths.get_app_db_path() if self.userdata_db: - db_path = settings.Db.get_userdata_db_path() + db_path = settings.DbPaths.get_userdata_db_path() self.conn = sqlite3.connect( db_path, diff --git a/app/lib/albumslib.py b/app/lib/albumslib.py index 36034f3a..afa28101 100644 --- a/app/lib/albumslib.py +++ b/app/lib/albumslib.py @@ -2,17 +2,11 @@ Contains methods relating to albums. """ -from dataclasses import asdict -from typing import Any -from itertools import groupby - from app.models.track import Track -from app.store.albums import AlbumStore -from app.store.tracks import TrackStore -def remove_duplicate_on_merge_versions(tracks: list[Track]) -> list[Track]: +def remove_duplicate_on_merge_versions(tracks: list[Track]): """ Removes duplicate tracks when merging versions of the same album. """ @@ -21,8 +15,6 @@ def remove_duplicate_on_merge_versions(tracks: list[Track]) -> list[Track]: def sort_by_track_no(tracks: list[Track]): - # tracks = [asdict(t) for t in tracks] - for t in tracks: track = str(t.track).zfill(3) t._pos = int(f"{t.disc}{track}") diff --git a/app/lib/artistlib.py b/app/lib/artistlib.py index c101b515..f4664e84 100644 --- a/app/lib/artistlib.py +++ b/app/lib/artistlib.py @@ -145,34 +145,3 @@ class CheckArtistImages: if url is not None: return DownloadImage(url, name=f"{artist['artisthash']}.webp") - - -# def fetch_album_bio(title: str, albumartist: str) -> str | None: """ Returns the album bio for a given album. """ -# last_fm_url = "http://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key={}&artist={}&album={ -# }&format=json".format( settings.Paths.LAST_FM_API_KEY, albumartist, title ) - -# try: -# response = requests.get(last_fm_url) -# data = response.json() -# except: -# return None - -# try: -# bio = data["album"]["wiki"]["summary"].split('= 0 and user_agent.find("Chrome") < 0 if is_safari: From c1169579825307b4392a31636583a6df093dcc25 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 7 Jul 2024 16:52:18 +0300 Subject: [PATCH 19/44] try hashmap trackstore --- app/db/libdata.py | 6 +- app/lib/folderslib.py | 2 +- app/models/album.py | 2 +- app/models/artist.py | 2 +- app/models/track.py | 15 ++- app/setup/__init__.py | 1 + app/store/folder.py | 16 +++ app/store/tracks.py | 300 ++++++++++++++++++++++++++++++++---------- 8 files changed, 263 insertions(+), 81 deletions(-) diff --git a/app/db/libdata.py b/app/db/libdata.py index caa0b3ab..ac89ebab 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -129,7 +129,7 @@ class TrackTable(Base): title: Mapped[str] = mapped_column(String()) track: Mapped[int] = mapped_column(Integer()) trackhash: Mapped[str] = mapped_column(String(), index=True) - is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + # is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) lastplayed: Mapped[int] = mapped_column(Integer(), default=0) playcount: Mapped[int] = mapped_column(Integer(), default=0) playduration: Mapped[int] = mapped_column(Integer(), default=0) @@ -270,7 +270,7 @@ class AlbumTable(Base): og_title: Mapped[str] = mapped_column(String()) title: Mapped[str] = mapped_column(String()) trackcount: Mapped[int] = mapped_column(Integer()) - is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + # is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) lastplayed: Mapped[int] = mapped_column(Integer(), default=0) playcount: Mapped[int] = mapped_column(Integer(), default=0) playduration: Mapped[int] = mapped_column(Integer(), default=0) @@ -360,7 +360,7 @@ class ArtistTable(Base): genres: Mapped[str] = mapped_column(JSON()) name: Mapped[str] = mapped_column(String(), index=True) trackcount: Mapped[int] = mapped_column(Integer()) - is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + # is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) lastplayed: Mapped[int] = mapped_column(Integer(), default=0) playcount: Mapped[int] = mapped_column(Integer(), default=0) playduration: Mapped[int] = mapped_column(Integer(), default=0) diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index 85542e11..7ff5205a 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -108,7 +108,7 @@ class GetFilesAndDirs: tracks = [] if files: - tracks = TrackDB.get_tracks_by_filepaths(files) + tracks = list(FolderStore.get_tracks_by_filepaths(files)) folders = [] if not self.tracks_only: diff --git a/app/models/album.py b/app/models/album.py index fb3cda97..425c91e6 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -28,7 +28,7 @@ class Album: og_title: str title: str trackcount: int - is_favorite: bool + # is_favorite: bool lastplayed: int playcount: int playduration: int diff --git a/app/models/artist.py b/app/models/artist.py index 6f8d8002..ee706866 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -47,7 +47,7 @@ class Artist: genrehashes: list[str] name: str trackcount: int - is_favorite: bool + # is_favorite: bool lastplayed: int playcount: int playduration: int diff --git a/app/models/track.py b/app/models/track.py index 86dc2da1..fa6984e5 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -1,4 +1,6 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field + +from app.utils.auth import get_current_userid @dataclass(slots=True) @@ -33,10 +35,14 @@ class Track: playcount: int playduration: int - is_favorite: bool = False _pos: int = 0 _ati: str = "" image: str = "" + fav_userids: set = field(default_factory=set) + + @property + def is_favorite(self): + return get_current_userid() in self.fav_userids def __post_init__(self): self.image = self.albumhash + ".webp" @@ -66,16 +72,11 @@ class Track: # image: str = "" # artist_hashes: str = "" - # fav_userids: list = field(default_factory=list) # """ # A string of user ids separated by commas. # """ # # is_favorite: bool = False - # @property - # def is_favorite(self): - # return current_user["id"] in self.fav_userids - # # temporary attributes # _pos: int = 0 # for sorting tracks by disc and track number # _ati: str = ( diff --git a/app/setup/__init__.py b/app/setup/__init__.py index 60738d68..098afe02 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -46,4 +46,5 @@ def load_into_mem(): # TrackStore.load_all_tracks(instance_key) # AlbumStore.load_albums(instance_key) # ArtistStore.load_artists(instance_key) + TrackStore.load_all_tracks(get_random_str()) FolderStore.load_filepaths() \ No newline at end of file diff --git a/app/store/folder.py b/app/store/folder.py index 0bfe3ae3..10fd60fe 100644 --- a/app/store/folder.py +++ b/app/store/folder.py @@ -2,6 +2,7 @@ from sortedcontainers import SortedSet from concurrent.futures import ThreadPoolExecutor from app.db.libdata import TrackTable +from app.store.tracks import TrackStore class FolderStore: @@ -13,6 +14,10 @@ class FolderStore: """ filepaths: SortedSet = SortedSet() + map: dict[str, str] = {} + """ + The map above is a dictionary that maps the folder path to the track hash, which can be used to fetch the track from the track store (a dict of track hashes to track objects). + """ @classmethod def load_filepaths(cls): @@ -26,7 +31,18 @@ class FolderStore: tracks = TrackTable.get_all() for track in tracks: cls.filepaths.add(track.filepath) + cls.map[track.filepath] = track.trackhash + @classmethod + def get_tracks_by_filepaths(cls, filepaths: list[str]): + for filepath in filepaths: + trackhash = cls.map.get(filepath) + + if trackhash: + track = TrackStore.trackhashmap.get(trackhash) + + if track: + yield [t for t in track.tracks if t.filepath == filepath][0] @classmethod def count_tracks_containing_paths(cls, paths: list[str]): diff --git a/app/store/tracks.py b/app/store/tracks.py index 21a65d66..a3b158aa 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -1,18 +1,99 @@ # from tqdm import tqdm +import itertools +import sys +from typing import Callable from flask_jwt_extended import current_user +from app.db.libdata import TrackTable from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb + +# from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb +from app.db.userdata import FavoritesTable from app.models import Track -from app.utils.bisection import use_bisection -from app.utils.customlist import CustomList from app.utils.remove_duplicates import remove_duplicates TRACKS_LOAD_KEY = "" +class TrackGroup: + """ + Tracks grouped under the same trackhash. + """ + + def __init__(self, tracks: list[Track]): + self.tracks = tracks + + def append(self, track: Track): + """ + Adds a track to the group. + """ + self.tracks.append(track) + + def remove(self, track: Track): + """ + Removes a track from the group. + """ + self.tracks.remove(track) + + def set_fav_userids(self, userids: set[int]): + """ + Sets the favorite userids. + """ + for track in self.tracks: + track.fav_userids = userids + + def get_best(self): + """ + Returns the track with higest bitrate. + """ + return max(self.tracks, key=lambda x: x.bitrate) + + def toggle_favorite(self, remove: bool = False): + """ + Adds a track to the favorites. + """ + + userids = set(self.tracks[0].fav_userids) + + if remove: + userids.remove(current_user["id"]) + else: + userids.add(current_user["id"]) + + for track in self.tracks: + track.fav_userids = userids + + def __len__(self): + return len(self.tracks) + + +class classproperty(property): + """ + A class property decorator. + """ + + def __get__(self, owner_self, owner_cls): + return self.fget(owner_cls) + + class TrackStore: - tracks: list[Track] = CustomList() + # {'trackhash': Track[]} + trackhashmap: dict[str, TrackGroup] = dict() + + @classproperty + def tracks(cls) -> list[Track]: + return cls.get_flat_list() + + @classmethod + def get_flat_list(cls): + """ + Returns a flat list of all tracks. + """ + return list( + itertools.chain.from_iterable( + [group.tracks for group in cls.trackhashmap.values()] + ) + ) @classmethod def load_all_tracks(cls, instance_key: str): @@ -24,32 +105,60 @@ class TrackStore: global TRACKS_LOAD_KEY TRACKS_LOAD_KEY = instance_key - cls.tracks = CustomList(trackdb.get_all_tracks()) + cls.trackhashmap = dict() + tracks = TrackTable.get_all() - favs = favdb.get_fav_tracks() - - records = dict() - - for fav in favs: - r = records.setdefault(fav[1], set()) - r.add(fav[4]) - - for track in cls.tracks: + # INFO: Load all tracks into the dict store + for track in tracks: if instance_key != TRACKS_LOAD_KEY: return - userids = records.get(track.trackhash, set()) - track.fav_userids = list(userids) + exists = cls.trackhashmap.get(track.trackhash, None) + if not exists: + cls.trackhashmap[track.trackhash] = TrackGroup([track]) + else: + cls.trackhashmap[track.trackhash].append(track) - print("Done!") + # favs = favdb.get_fav_tracks() + favs = FavoritesTable.get_all() + records: dict[str, set[int]] = dict() + + # convert records: {trackhash: {userid, userid, ...}} + for fav in favs: + if fav.hash not in records: + # if trackhash not in dict, add it + # and set the value to a set containing the userid + records[fav.hash] = {fav.userid} + + # if trackhash is in dict, add the userid to the set + records[fav.hash].add(fav.userid) + + for record in records: + if instance_key != TRACKS_LOAD_KEY: + return + + group = cls.trackhashmap.get(record, None) + + if not group: + continue + + group.set_fav_userids(records.get(record, set())) + + # print("Done!") + # print(cls.trackhashmap.get("0d6b22c19c").tracks[0].fav_userids) + # sys.exit(0) @classmethod def add_track(cls, track: Track): """ Adds a single track to the store. """ + group = cls.trackhashmap.get(track.trackhash, None) - cls.tracks.append(track) + if group: + return group.append(track) + + cls.trackhashmap[track.trackhash] = TrackGroup([track]) @classmethod def add_tracks(cls, tracks: list[Track]): @@ -57,17 +166,21 @@ class TrackStore: Adds multiple tracks to the store. """ - cls.tracks.extend(tracks) + for track in tracks: + cls.add_track(track) @classmethod - def remove_track_obj(cls, track: Track): + def remove_track(cls, track: Track): """ Removes a single track from the store. """ - try: - cls.tracks.remove(track) - except ValueError: - pass + group = cls.trackhashmap.get(track.trackhash, None) + + if group: + group.remove(track) + + if len(group) == 0: + del cls.trackhashmap[track.trackhash] @classmethod def remove_track_by_filepath(cls, filepath: str): @@ -75,10 +188,7 @@ class TrackStore: Removes a track from the store by its filepath. """ - for track in cls.tracks: - if track.filepath == filepath: - cls.remove_track_obj(track) - break + return cls.remove_tracks_by_filepaths({filepath}) @classmethod def remove_tracks_by_filepaths(cls, filepaths: set[str]): @@ -86,47 +196,47 @@ class TrackStore: Removes multiple tracks from the store by their filepaths. """ - for track in cls.tracks: - if track.filepath in filepaths: - cls.remove_track_obj(track) + filecount = len(filepaths) + + for trackhash in cls.trackhashmap: + group = cls.trackhashmap[trackhash] + + for track in group.tracks: + if track.filepath in filepaths: + group.remove(track) + + if len(group) == 0: + del cls.trackhashmap[trackhash] + + filecount -= 1 + + if filecount == 0: + break @classmethod def count_tracks_by_trackhash(cls, trackhash: str) -> int: """ Counts the number of tracks with a specific trackhash. """ - return sum(1 for track in cls.tracks if track.trackhash == trackhash) + return len(cls.trackhashmap.get(trackhash, [])) @classmethod - def make_track_fav(cls, trackhash: str): + def toggle_favorite(cls, trackhash: str, remove: bool = False): """ Adds a track to the favorites. """ - for track in cls.tracks: - if track.trackhash == trackhash: - if current_user["id"] not in track.fav_userids: - track.fav_userids.append(current_user["id"]) + group = cls.trackhashmap.get(trackhash) + + if group: + group.toggle_favorite(remove=remove) @classmethod def remove_track_from_fav(cls, trackhash: str): """ Removes a track from the favorites. """ - - for track in cls.tracks: - if track.trackhash == trackhash: - if current_user["id"] in track.fav_userids: - track.fav_userids.remove(current_user["id"]) - - @classmethod - def append_track_artists( - cls, albumhash: str, artists: list[str], new_album_title: str - ): - tracks = cls.get_tracks_by_albumhash(albumhash) - - for track in tracks: - track.add_artists(artists, new_album_title) + return cls.toggle_favorite(trackhash, remove=True) # ================================================ # ================== GETTERS ===================== @@ -138,16 +248,16 @@ class TrackStore: Returns a list of tracks by their hashes. """ hash_set = set(trackhashes) - set_len = len(hash_set) - tracks = [] - for track in cls.tracks: - if track.trackhash in hash_set: + tracks: list[Track] = [] + + for trackhash in hash_set: + group = cls.trackhashmap.get(trackhash, None) + + if group: + track = group.get_best() tracks.append(track) - if len(tracks) == set_len: - break - # sort the tracks in the order of the given trackhashes tracks.sort(key=lambda t: trackhashes.index(t.trackhash)) return tracks @@ -156,32 +266,86 @@ class TrackStore: def get_tracks_by_filepaths(cls, paths: list[str]) -> list[Track]: """ Returns all tracks matching the given paths. + + ⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔ """ - tracks = sorted(cls.tracks, key=lambda x: x.filepath) - tracks = use_bisection(tracks, "filepath", paths) - return [track for track in tracks if track is not None] + # tracks = sorted(cls.trackhashmap, key=lambda x: x.filepath) + # tracks = use_bisection(tracks, "filepath", paths) + # return [track for track in tracks if track is not None] + # return cls.find_tracks_by(key="filepath", value=paths) + + tracks: list[Track] = [] + + for trackhash in cls.trackhashmap: + group = cls.trackhashmap.get(trackhash) + + if not group: + continue + + for track in group.tracks: + if track.filepath in paths: + tracks.append(track) + + return tracks + + @classmethod + def find_tracks_by( + cls, + key: str, + value: str, + predicate: Callable = lambda prop_value, value: prop_value == value, + including_duplicates: bool = False, + ): + """ + Find all tracks by a specific key. + """ + tracks: list[Track] = [] + + for trackhash in cls.trackhashmap: + group = cls.trackhashmap.get(trackhash, None) + + if not group: + continue + + for track in group.tracks: + prop_value = getattr(track, key) + if predicate(prop_value, value): + tracks.append(track) + + if including_duplicates: + return tracks + + return remove_duplicates(tracks) @classmethod def get_tracks_by_albumhash(cls, album_hash: str) -> list[Track]: """ Returns all tracks matching the given album hash. """ - tracks = [t for t in cls.tracks if t.albumhash == album_hash] - return remove_duplicates(tracks, is_album_tracks=True) + return cls.find_tracks_by(key="albumhash", value=album_hash) @classmethod def get_tracks_by_artisthash(cls, artisthash: str): """ Returns all tracks matching the given artist. Duplicate tracks are removed. """ - tracks = [t for t in cls.tracks if artisthash in t.artist_hashes] - tracks = remove_duplicates(tracks) - tracks.sort(key=lambda x: x.last_mod) - return tracks + predicate = lambda artisthashes, artisthash: artisthash in artisthashes + return cls.find_tracks_by( + key="artist_hashes", value=artisthash, predicate=predicate + ) @classmethod def get_tracks_in_path(cls, path: str): """ Returns all tracks in the given path. """ - return (t for t in cls.tracks if t.folder.startswith(path)) + predicate: Callable[[str, str], bool] = ( + lambda track_folder, path: track_folder.startswith(path) + ) + + return cls.find_tracks_by( + key="folder", + value=path, + predicate=predicate, + including_duplicates=True, + ) From e07d8db8e03b338a0aa3ff16bd6551477614fcd1 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 7 Jul 2024 16:59:10 +0300 Subject: [PATCH 20/44] try sending the first song in a group --- app/lib/folderslib.py | 2 +- app/store/folder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index 7ff5205a..ff8bb9c2 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -8,7 +8,7 @@ from app.settings import SUPPORTED_FILES from app.store.folder import FolderStore from app.utils.wintools import win_replace_slash -from app.db.libdata import TrackTable as TrackDB +# from app.db.libdata import TrackTable as TrackDB def create_folder(path: str, trackcount=0, foldercount=0) -> Folder: diff --git a/app/store/folder.py b/app/store/folder.py index 10fd60fe..34555ccb 100644 --- a/app/store/folder.py +++ b/app/store/folder.py @@ -42,7 +42,7 @@ class FolderStore: track = TrackStore.trackhashmap.get(trackhash) if track: - yield [t for t in track.tracks if t.filepath == filepath][0] + yield track.tracks[0] @classmethod def count_tracks_containing_paths(cls, paths: list[str]): From 83e105a198377879fac7e89d280f44859f2d8053 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 13 Jul 2024 12:37:08 +0300 Subject: [PATCH 21/44] paginate folder tracks endpoint --- app/api/folder.py | 9 ++- app/lib/folderslib.py | 138 +++++++++++++++++++++--------------------- 2 files changed, 75 insertions(+), 72 deletions(-) diff --git a/app/api/folder.py b/app/api/folder.py index 098742af..2972c3e8 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -14,7 +14,7 @@ from showinfm import show_in_file_manager from app import settings from app.config import UserConfig from app.db.libdata import TrackTable -from app.lib.folderslib import GetFilesAndDirs, get_folders +from app.lib.folderslib import get_files_and_dirs, get_folders from app.serializers.track import serialize_track from app.utils.wintools import is_windows, win_replace_slash @@ -26,6 +26,8 @@ class FolderTree(BaseModel): folder: str = Field( "$home", example="$home", description="The folder to things from" ) + start: int = Field(0, description="The start index") + end: int = Field(50, description="The end index") tracks_only: bool = Field(False, description="Whether to only get tracks") @@ -39,7 +41,6 @@ def get_folder_tree(body: FolderTree): req_dir = body.folder tracks_only = body.tracks_only - config = UserConfig() root_dirs = config.rootDirs @@ -67,7 +68,9 @@ def get_folder_tree(body: FolderTree): else: req_dir = "/" + req_dir if not req_dir.startswith("/") else req_dir - res = GetFilesAndDirs(req_dir, tracks_only=tracks_only)() + res = get_files_and_dirs( + req_dir, start=body.start, end=body.end, tracks_only=tracks_only + ) res["folders"] = sorted(res["folders"], key=lambda i: i.name) return res diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index ff8bb9c2..ef7b7510 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -52,79 +52,79 @@ def get_folders(paths: list[str]): ] -class GetFilesAndDirs: +def get_files_and_dirs( + path: str, start: int, end: int, tracks_only: bool = False, skip_empty_folders=True +): """ - Get files and folders from a directory. + Given a path, returns a list of tracks and folders in that immediate path. + + Can recursively call itself to skip through empty folders. """ - def __init__(self, path: str, tracks_only=False) -> None: - self.path = path - self.tracks_only = tracks_only - - def get_files_and_dirs(self, path: str, skip_empty_folders=True): - """ - Given a path, returns a list of tracks and folders in that immediate path. - - Can recursively call itself to skip through empty folders. - """ - try: - entries = os.scandir(path) - except FileNotFoundError: - return { - "path": path, - "tracks": [], - "folders": [], - } - - dirs, files = [], [] - - for entry in entries: - ext = os.path.splitext(entry.name)[1].lower() - - if entry.is_dir() and not entry.name.startswith("."): - dir = win_replace_slash(entry.path) - # add a trailing slash to the folder path - # to avoid matching a folder starting with the same name as the root path - # eg. .../Music and .../Music VideosI - dirs.append(os.path.join(dir, "")) - elif entry.is_file() and ext in SUPPORTED_FILES: - files.append(win_replace_slash(entry.path)) - - files_ = [] - - for file in files: - try: - files_.append( - { - "path": file, - "time": os.path.getmtime(file), - } - ) - except OSError as e: - log.error(e) - - files_.sort(key=lambda f: f["time"]) - files = [f["path"] for f in files_] - - tracks = [] - if files: - tracks = list(FolderStore.get_tracks_by_filepaths(files)) - - folders = [] - if not self.tracks_only: - folders = get_folders(dirs) - - if skip_empty_folders and len(folders) == 1 and len(tracks) == 0: - # INFO: When we only have one folder and no tracks, - # skip through empty folders. - # Call recursively with the first folder in the list. - return self.get_files_and_dirs(folders[0].path) - + try: + entries = os.scandir(path) + except FileNotFoundError: return { "path": path, - "tracks": serialize_tracks(tracks), - "folders": folders, + "tracks": [], + "folders": [], } - def __call__(self): - return self.get_files_and_dirs(self.path) + dirs, files = [], [] + + for entry in entries: + ext = os.path.splitext(entry.name)[1].lower() + + if entry.is_dir() and not entry.name.startswith("."): + dir = win_replace_slash(entry.path) + # add a trailing slash to the folder path + # to avoid matching a folder starting with the same name as the root path + # eg. .../Music and .../Music VideosI + dirs.append(os.path.join(dir, "")) + elif entry.is_file() and ext in SUPPORTED_FILES: + files.append(win_replace_slash(entry.path)) + + files_ = [] + + for file in files: + try: + files_.append( + { + "path": file, + "time": os.path.getmtime(file), + } + ) + except OSError as e: + log.error(e) + + files_.sort(key=lambda f: f["time"]) + files = [f["path"] for f in files_] + + tracks = [] + if files: + if end == -1: + end = len(files) + + tracks = list(FolderStore.get_tracks_by_filepaths(files[start:end])) + + folders = [] + if not tracks_only: + folders = get_folders(dirs) + + if skip_empty_folders and len(folders) == 1 and len(tracks) == 0: + # INFO: When we only have one folder and no tracks, + # skip through empty folders. + # Call recursively with the first folder in the list. + return get_files_and_dirs( + folders[0].path, + start=start, + end=end, + tracks_only=tracks_only, + skip_empty_folders=True, + ) + + return { + "path": path, + "tracks": serialize_tracks(tracks), + "folders": folders, + } From 58c90d95b1a51aa0ab8f6c5f15eef02c105e01f1 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 15 Jul 2024 00:26:56 +0300 Subject: [PATCH 22/44] rewrite album and artist stores using in-mem hashmap stores --- app/api/album.py | 63 ++++++----- app/api/artist.py | 31 +++--- app/db/libdata.py | 1 - app/lib/colorlib.py | 5 +- app/lib/tagger.py | 243 +++++++++++++++++++++--------------------- app/models/album.py | 2 +- app/models/artist.py | 2 +- app/setup/__init__.py | 2 + app/store/albums.py | 114 +++++++++++--------- app/store/artists.py | 62 ++++++++--- app/store/tracks.py | 9 +- 11 files changed, 297 insertions(+), 237 deletions(-) diff --git a/app/api/album.py b/app/api/album.py index 6c8ab65f..8c762f1f 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -4,26 +4,24 @@ Contains all the album routes. import random -from pydantic import Field +from pydantic import BaseModel, Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from app.api.apischemas import AlbumHashSchema, AlbumLimitSchema, ArtistHashSchema from app.config import UserConfig -from app.db.libdata import ArtistTable -from app.db.libdata import AlbumTable as AlbumDb, TrackTable as TrackDb from app.db.userdata import SimilarArtistTable +from app.models.album import Album from app.settings import Defaults -from app.utils import flatten +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore from app.utils.hashing import create_hash from app.lib.albumslib import sort_by_track_no from app.serializers.album import serialize_for_card_many from app.serializers.track import serialize_tracks -from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb -get_albums_by_albumartist = adb.get_albums_by_albumartist check_is_fav = favdb.check_is_favorite bp_tag = Tag(name="Album", description="Single album") @@ -39,12 +37,14 @@ def get_album_tracks_and_info(body: AlbumHashSchema): Returns album info and tracks for the given albumhash. """ albumhash = body.albumhash - album = AlbumDb.get_album_by_albumhash(albumhash) + # album = AlbumDb.get_album_by_albumhash(albumhash) + albumentry = AlbumStore.albummap.get(albumhash) - if album is None: + if albumentry is None: return {"error": "Album not found"}, 404 - tracks = TrackDb.get_tracks_by_albumhash(albumhash) + album = albumentry.album + tracks = TrackStore.get_tracks_by_trackhashes(albumentry.trackhashes) album.trackcount = len(tracks) album.duration = sum(t.duration for t in tracks) album.check_type( @@ -52,6 +52,7 @@ def get_album_tracks_and_info(body: AlbumHashSchema): ) track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks}) + avg_bitrate = sum(t.bitrate for t in tracks) // (len(tracks) or 1) return { "info": album, @@ -61,7 +62,7 @@ def get_album_tracks_and_info(body: AlbumHashSchema): # 1. All the tracks have the correct track totals # 2. Tracks with the same track total are from the same disc "track_total": track_total, - "avg_bitrate": sum(t.bitrate for t in tracks) // len(tracks), + "avg_bitrate": avg_bitrate, }, "copyright": tracks[0].copyright, "tracks": serialize_tracks(tracks, remove_disc=False), @@ -76,7 +77,7 @@ def get_album_tracks(path: AlbumHashSchema): Returns all the tracks in the given album, sorted by disc and track number. NOTE: No album info is returned. """ - tracks = TrackDb.get_tracks_by_albumhash(path.albumhash) + tracks = AlbumStore.get_album_tracks(path.albumhash) tracks = sort_by_track_no(tracks) return serialize_tracks(tracks) @@ -105,7 +106,11 @@ def get_more_from_artist(body: GetMoreFromArtistsBody): limit = body.limit base_title = body.base_title - all_albums = AlbumDb.get_albums_by_artisthashes(albumartists) + all_albums: dict[str, list[Album]] = {} + + for artisthash in albumartists: + all_albums[artisthash] = AlbumStore.get_albums_by_artisthash(artisthash) + seen_hashes = set() for artisthash, albums in all_albums.items(): @@ -128,14 +133,15 @@ def get_more_from_artist(body: GetMoreFromArtistsBody): return all_albums -class GetAlbumVersionsBody(ArtistHashSchema): +class GetAlbumVersionsBody(BaseModel): og_album_title: str = Field( description="The original album title (album.og_title)", example=Defaults.API_ALBUMNAME, ) - base_title: str = Field( - description="The base title of the album to exclude from the results.", - example=Defaults.API_ALBUMNAME, + + albumhash: str = Field( + description="The album hash of the album to exclude from the results.", + example=Defaults.API_ALBUMHASH, ) @@ -146,18 +152,23 @@ def get_album_versions(body: GetAlbumVersionsBody): Returns other versions of the given album. """ - og_album_title = body.og_album_title - base_title = body.base_title - artisthash = body.artisthash + albumhash = body.albumhash - albums = AlbumDb.get_albums_by_base_title(base_title) + album = AlbumStore.albummap.get(albumhash) + if not album: + return [] + artisthash = album.album.artisthashes[0] + albums = AlbumStore.get_albums_by_artisthash(artisthash) + + basetitle = album.basetitle albums = [ a for a in albums - if a.og_title != og_album_title + if a.og_title != album.album.og_title + if a.base_title == basetitle and artisthash in {a["artisthash"] for a in a.albumartists} ] - print(albums) + return serialize_for_card_many(albums) @@ -181,10 +192,8 @@ def get_similar_albums(query: GetSimilarAlbumsQuery): return [] artisthashes = similar_artists.get_artist_hash_set() - artists = ArtistTable.get_artists_by_artisthashes(artisthashes) - - albums = AlbumDb.get_albums_by_artisthashes([a.artisthash for a in artists]) - albums = flatten(albums.values()) + artists = ArtistStore.get_artists_by_hashes(artisthashes) + albums = AlbumStore.get_albums_by_artisthashes([a.artisthash for a in artists]) sample = random.sample(albums, min(len(albums), limit)) return serialize_for_card_many(sample[:limit]) diff --git a/app/api/artist.py b/app/api/artist.py index d2b7e2af..eb119183 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -17,14 +17,16 @@ from app.api.apischemas import ( ) from app.config import UserConfig -from app.db.libdata import ArtistTable -from app.db.libdata import AlbumTable, TrackTable from app.db.userdata import SimilarArtistTable from app.serializers.album import serialize_for_card_many from app.serializers.artist import serialize_for_cards from app.serializers.track import serialize_tracks +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore + bp_tag = Tag(name="Artist", description="Single artist") api = APIBlueprint("artist", __name__, url_prefix="/artist", abp_tags=[bp_tag]) @@ -39,13 +41,15 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): artisthash = path.artisthash limit = query.limit - artist = ArtistTable.get_artist_by_hash(artisthash) - if artist is None: + entry = ArtistStore.artistmap.get(artisthash) + + if entry is None: return {"error": "Artist not found"}, 404 - tracks = TrackTable.get_tracks_by_artisthash(artisthash) + tracks = TrackStore.get_tracks_by_trackhashes(entry.trackhashes) tcount = len(tracks) + artist = entry.artist if artist.albumcount == 0 and tcount < 10: limit = tcount @@ -85,19 +89,19 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): limit = query.limit - artist = ArtistTable.get_artist_by_hash(artisthash) + entry = ArtistStore.artistmap.get(artisthash) - if artist is None: + if entry is None: return {"error": "Artist not found"}, 404 - albums = AlbumTable.get_albums_by_artisthash(artisthash) - tracks = TrackTable.get_tracks_by_artisthash(artisthash) + albums = AlbumStore.get_albums_by_hashes(entry.albumhashes) + tracks = TrackStore.get_tracks_by_trackhashes(entry.trackhashes) missing_albumhashes = { t.albumhash for t in tracks if t.albumhash not in {a.albumhash for a in albums} } - albums.extend(AlbumTable.get_albums_by_albumhashes(missing_albumhashes)) + albums.extend(AlbumStore.get_albums_by_hashes(missing_albumhashes)) albumdict = {a.albumhash: a for a in albums} config = UserConfig() @@ -135,7 +139,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): for key, value in res.items(): res[key] = serialize_for_card_many(value[:limit]) - res["artistname"] = artist.name + res["artistname"] = entry.artist.name return res @@ -146,8 +150,7 @@ def get_all_artist_tracks(path: ArtistHashSchema): Returns all artists by a given artist. """ - # tracks = TrackStore.get_tracks_by_artisthash(path.artisthash) - tracks = TrackTable.get_tracks_by_artisthash(path.artisthash) + tracks = ArtistStore.get_artist_tracks(path.artisthash) return serialize_tracks(tracks) @@ -162,7 +165,7 @@ def get_similar_artists(path: ArtistHashSchema, query: ArtistLimitSchema): if result is None: return [] - similar = ArtistTable.get_artists_by_artisthashes(result.get_artist_hash_set()) + similar = ArtistStore.get_artists_by_hashes(result.get_artist_hash_set()) if len(similar) > limit: similar = random.sample(similar, min(limit, len(similar))) diff --git a/app/db/libdata.py b/app/db/libdata.py index ac89ebab..acf4c510 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -317,7 +317,6 @@ class AlbumTable(Base): for artist in artisthashes: result = conn.execute( - # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! select(AlbumTable).where(AlbumTable.artisthashes.contains(artist)) ) albums[artist] = albums_to_dataclasses(result.fetchall()) diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index e71dbb1f..cb360da5 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -12,8 +12,7 @@ from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb from app.db.sqlite.artistcolors import SQLiteArtistMethods as adb from app.db.sqlite.utils import SQLiteManager -from app.store.artists import ArtistStore -from app.store.albums import AlbumStore +# from app.store.artists import ArtistStore from app.logger import log from app.lib.errors import PopulateCancelledError from app.utils.progressbar import tqdm @@ -101,7 +100,7 @@ class ProcessArtistColors: """ def __init__(self, instance_key: str) -> None: - all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0] + # all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0] global PROCESS_ARTIST_COLORS_KEY PROCESS_ARTIST_COLORS_KEY = instance_key diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 46926aeb..0564f94a 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -2,13 +2,14 @@ import gc import os from pprint import pprint from time import time -from typing import Generator from app import settings from app.config import UserConfig from app.db.libdata import ArtistTable -from app.db.libdata import AlbumTable, TrackTable +from app.db.libdata import TrackTable from app.lib.populate import CordinateMedia from app.lib.taglib import extract_thumb, get_tags +from app.models.album import Album +from app.models.artist import Artist from app.models.track import Track from app.store.folder import FolderStore from app.utils.filesystem import run_fast_scandir @@ -141,154 +142,152 @@ class IndexTracks: print("Done") -class IndexAlbums: - def __init__(self) -> None: - albums = dict() - all_tracks: list[Track] = TrackTable.get_all() +# class IndexAlbums: +def create_albums(): + albums = dict() + all_tracks: list[Track] = TrackTable.get_all() - if len(all_tracks) == 0: - return + for track in all_tracks: + if track.albumhash not in albums: + albums[track.albumhash] = { + "albumartists": track.albumartists, + "artisthashes": [a["artisthash"] for a in track.albumartists], + "albumhash": track.albumhash, + "base_title": None, + "color": None, + "created_date": track.last_mod, + "date": track.date, + "duration": track.duration, + "genres": [*track.genres] if track.genres else [], + "og_title": track.og_album, + "lastplayed": track.lastplayed, + "playcount": track.playcount, + "playduration": track.playduration, + "title": track.album, + "trackcount": 1, + "extra": {} + } + else: + album = albums[track.albumhash] + album["trackcount"] += 1 + album["playcount"] += track.playcount + album["playduration"] += track.playduration + album["lastplayed"] = max(album["lastplayed"], track.lastplayed) + album["duration"] += track.duration + album["date"] = min(album["date"], track.date) + album["created_date"] = min(album["created_date"], track.last_mod) - for track in all_tracks: - if track.albumhash not in albums: - albums[track.albumhash] = { - "albumartists": track.albumartists, - "artisthashes": [a["artisthash"] for a in track.albumartists], - "albumhash": track.albumhash, - "base_title": None, - "color": None, + if track.genres: + album["genres"].extend(track.genres) + + for album in albums.values(): + genres = [] + for genre in album["genres"]: + if genre not in genres: + genres.append(genre) + + album["genres"] = genres + album["genrehashes"] = " ".join([g['genrehash'] for g in genres]) + album["base_title"], _ = get_base_album_title(album["og_title"]) + + del genres + + # AlbumTable.remove_all() + # AlbumTable.insert_many(list(albums.values())) + return [Album(**album) for album in albums.values()] + + +# class IndexArtists: +def create_artists(): + all_tracks: list[Track] = TrackTable.get_all() + artists = dict() + + for track in all_tracks: + this_artists = track.artists + + for a in track.albumartists: + if a not in this_artists: + a["in_track"] = False + this_artists.append(a) + + for thisartist in this_artists: + if thisartist["artisthash"] not in artists: + artists[thisartist["artisthash"]] = { + "albumcount": None, + "albums": {track.albumhash}, + "artisthash": thisartist["artisthash"], "created_date": track.last_mod, "date": track.date, "duration": track.duration, - "genres": [*track.genres] if track.genres else [], - "og_title": track.og_album, + "genres": track.genres if track.genres else [], + "name": None, + "names": {thisartist["name"]}, "lastplayed": track.lastplayed, "playcount": track.playcount, "playduration": track.playduration, - "title": track.album, - "trackcount": 1, + "trackcount": None, + "tracks": ( + {track.trackhash} + if thisartist.get("in_track", True) + else set() + ), + "extra": {}, } else: - album = albums[track.albumhash] - album["trackcount"] += 1 - album["playcount"] += track.playcount - album["playduration"] += track.playduration - album["lastplayed"] = max(album["lastplayed"], track.lastplayed) - album["duration"] += track.duration - album["date"] = min(album["date"], track.date) - album["created_date"] = min(album["created_date"], track.last_mod) + artist = artists[thisartist["artisthash"]] + artist["duration"] += track.duration + artist["playcount"] += track.playcount + artist["playduration"] += track.playduration + artist["albums"].add(track.albumhash) + artist["date"] = min(artist["date"], track.date) + artist["lastplayed"] = max(artist["lastplayed"], track.lastplayed) + artist["created_date"] = min(artist["created_date"], track.last_mod) + artist["names"].add(thisartist["name"]) + + if thisartist.get("in_track", True): + artist["tracks"].add(track.trackhash) if track.genres: - album["genres"].extend(track.genres) + artist["genres"].extend(track.genres) - for album in albums.values(): - genres = [] - for genre in album["genres"]: - if genre not in genres: - genres.append(genre) + for artist in artists.values(): + artist["albumcount"] = len(artist["albums"]) + artist["trackcount"] = len(artist["tracks"]) - album["genres"] = genres - album["base_title"], _ = get_base_album_title(album["og_title"]) + genres = [] - del genres + for genre in artist["genres"]: + if genre not in genres: + genres.append(genre) - AlbumTable.remove_all() - AlbumTable.insert_many(list(albums.values())) - del albums + artist["genres"] = genres + artist["genrehashes"] = " ".join([g['genrehash'] for g in genres]) + artist["name"] = sorted(artist["names"])[0] + # INFO: Delete temporary keys + del artist["names"] + del artist["tracks"] + del artist["albums"] -class IndexArtists: - def __init__(self) -> None: - all_tracks: list[Track] = TrackTable.get_all() - artists = dict() + # INFO: Delete local variables + del genres - if len(all_tracks) == 0: - return - - for track in all_tracks: - this_artists = track.artists - - for a in track.albumartists: - if a not in this_artists: - a["in_track"] = False - this_artists.append(a) - - for thisartist in this_artists: - if thisartist["artisthash"] not in artists: - artists[thisartist["artisthash"]] = { - "albumcount": None, - "albums": {track.albumhash}, - "artisthash": thisartist["artisthash"], - "created_date": track.last_mod, - "date": track.date, - "duration": track.duration, - "genres": track.genres if track.genres else [], - "name": None, - "names": {thisartist["name"]}, - "lastplayed": track.lastplayed, - "playcount": track.playcount, - "playduration": track.playduration, - "trackcount": None, - "tracks": ( - {track.trackhash} - if thisartist.get("in_track", True) - else set() - ), - } - else: - artist = artists[thisartist["artisthash"]] - artist["duration"] += track.duration - artist["playcount"] += track.playcount - artist["playduration"] += track.playduration - artist["albums"].add(track.albumhash) - artist["date"] = min(artist["date"], track.date) - artist["lastplayed"] = max(artist["lastplayed"], track.lastplayed) - artist["created_date"] = min(artist["created_date"], track.last_mod) - artist["names"].add(thisartist["name"]) - - - if thisartist.get("in_track", True): - artist["tracks"].add(track.trackhash) - - if track.genres: - artist["genres"].extend(track.genres) - - for artist in artists.values(): - artist["albumcount"] = len(artist["albums"]) - artist["trackcount"] = len(artist["tracks"]) - - genres = [] - - for genre in artist["genres"]: - if genre not in genres: - genres.append(genre) - - artist["genres"] = genres - artist["name"] = sorted(artist["names"])[0] - - # INFO: Delete temporary keys - del artist["names"] - del artist["tracks"] - del artist["albums"] - - # INFO: Delete local variables - del genres - - ArtistTable.remove_all() - ArtistTable.insert_many(list(artists.values())) - del artists + # ArtistTable.remove_all() + # ArtistTable.insert_many(list(artists.values())) + # del artists + return [Artist(**artist) for artist in artists.values()] class IndexEverything: def __init__(self) -> None: IndexTracks(instance_key=time()) - IndexAlbums() - IndexArtists() + # IndexAlbums() + # IndexArtists() FolderStore.load_filepaths() # pass - CordinateMedia(instance_key=str(time())) + # CordinateMedia(instance_key=str(time())) gc.collect() diff --git a/app/models/album.py b/app/models/album.py index 425c91e6..5b247e8a 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -14,7 +14,6 @@ class Album: Creates an album object """ - id: int albumartists: list[dict[str, str]] albumhash: str artisthashes: list[str] @@ -34,6 +33,7 @@ class Album: playduration: int extra: dict + id: int = -1 type: str = "album" image: str = "" versions: list[str] = dataclasses.field(default_factory=list) diff --git a/app/models/artist.py b/app/models/artist.py index ee706866..f7813f56 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -36,7 +36,6 @@ class Artist: Artist class """ - id: str name: str albumcount: int artisthash: str @@ -53,6 +52,7 @@ class Artist: playduration: int extra: dict + id: int = -1 image: str = "" def __post_init__(self): diff --git a/app/setup/__init__.py b/app/setup/__init__.py index 098afe02..85e6b334 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -47,4 +47,6 @@ def load_into_mem(): # AlbumStore.load_albums(instance_key) # ArtistStore.load_artists(instance_key) TrackStore.load_all_tracks(get_random_str()) + AlbumStore.load_albums('a') + ArtistStore.load_artists('a') FolderStore.load_filepaths() \ No newline at end of file diff --git a/app/store/albums.py b/app/store/albums.py index 50424102..69ef981e 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -1,9 +1,13 @@ from itertools import groupby import json +from pprint import pprint import random +from typing import Iterable from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb +from app.lib.tagger import create_albums from app.models import Album, Track +from app.store.artists import ArtistStore from app.utils.customlist import CustomList from app.utils.remove_duplicates import remove_duplicates @@ -14,8 +18,19 @@ from app.utils.progressbar import tqdm ALBUM_LOAD_KEY = "" +class AlbumMapEntry: + def __init__(self, album: Album) -> None: + self.album = album + self.trackhashes: set[str] = set() + + @property + def basetitle(self): + return self.album.base_title + + class AlbumStore: albums: list[Album] = CustomList() + albummap: dict[str, AlbumMapEntry] = {} @staticmethod def create_album(track: Track): @@ -36,43 +51,28 @@ class AlbumStore: global ALBUM_LOAD_KEY ALBUM_LOAD_KEY = instance_key - cls.albums = CustomList() - print("Loading albums... ", end="") - tracks = remove_duplicates(TrackStore.tracks) + + cls.albummap = { + album.albumhash: AlbumMapEntry(album=album) for album in create_albums() + } + tracks = remove_duplicates(TrackStore.get_flat_list()) tracks = sorted(tracks, key=lambda t: t.albumhash) grouped = groupby(tracks, lambda t: t.albumhash) for albumhash, tracks in grouped: - tracks = list(tracks) - sample = tracks[0] + cls.albummap[albumhash].trackhashes = {t.trackhash for t in tracks} - if sample is None: - continue + # db_albums: list[tuple] = aldb.get_all_albums() - count = len(list(tracks)) - duration = sum(t.duration for t in tracks) - created_date = min(t.created_date for t in tracks) + # for album in db_albums: + # albumhash = album[1] + # colors = json.loads(album[2]) - album = AlbumStore.create_album(sample) - - album.get_date_from_tracks(tracks) - album.set_count(count) - album.set_duration(duration) - album.set_created_date(created_date) - - cls.albums.append(album) - - db_albums: list[tuple] = aldb.get_all_albums() - - for album in db_albums: - albumhash = album[1] - colors = json.loads(album[2]) - - for _al in cls.albums: - if _al.albumhash == albumhash: - _al.set_colors(colors) - break + # for _al in cls.albums: + # if _al.albumhash == albumhash: + # _al.set_colors(colors) + # break print("Done!") @@ -98,9 +98,7 @@ class AlbumStore: Returns N albums by the given albumartist, excluding the specified album. """ - albums = [ - album for album in cls.albums if artisthash in album.albumartists_hashes - ] + albums = [album for album in cls.albums if artisthash in album.artisthashes] albums = [ album @@ -126,25 +124,11 @@ class AlbumStore: return None @classmethod - def get_albums_by_hashes(cls, albumhashes: list[str]) -> list[Album]: + def get_albums_by_hashes(cls, albumhashes: Iterable[str]) -> list[Album]: """ Returns albums by their hashes. """ - albums_str = "-".join(albumhashes) - albums = [a for a in cls.albums if a.albumhash in albums_str] - - # sort albums by the order of the hashes - albums.sort(key=lambda x: albumhashes.index(x.albumhash)) - return albums - - @classmethod - def get_albums_by_artisthash(cls, artisthash: str) -> list[Album]: - """ - Returns all albums by the given artist. - """ - return [ - album for album in cls.albums if artisthash in album.albumartists_hashes - ] + return [cls.albummap[albumhash].album for albumhash in albumhashes] @classmethod def count_albums_by_artisthash(cls, artisthash: str): @@ -174,3 +158,37 @@ class AlbumStore: Removes an album from the store. """ cls.albums = CustomList(a for a in cls.albums if a.albumhash != albumhash) + + @classmethod + def get_albums_by_artisthash(cls, hash: str): + """ + Returns all albums by the given artist hash. + """ + artist = ArtistStore.artistmap.get(hash) + + if not artist: + return [] + + return [cls.albummap[albumhash].album for albumhash in artist.albumhashes] + + @classmethod + def get_albums_by_artisthashes(cls, hashes: Iterable[str]): + """ + Returns all albums by the given artist hashes. + """ + albums = [] + for hash in hashes: + albums.extend(cls.get_albums_by_artisthash(hash)) + + return albums + + @classmethod + def get_album_tracks(cls, albumhash: str) -> list[Track]: + """ + Returns all tracks for the given album hash. + """ + album = cls.albummap.get(albumhash) + if not album: + return [] + + return TrackStore.get_tracks_by_trackhashes(album.trackhashes) diff --git a/app/store/artists.py b/app/store/artists.py index 82cbf976..c3875e0f 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -1,19 +1,30 @@ import json +from typing import Iterable from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb +from app.lib.tagger import create_artists from app.models import Artist from app.utils.bisection import use_bisection from app.utils.customlist import CustomList from app.utils.progressbar import tqdm +from .tracks import TrackStore -from .albums import AlbumStore +# from .albums import AlbumStore from .tracks import TrackStore ARTIST_LOAD_KEY = "" +class ArtistMapEntry: + def __init__(self, artist: Artist) -> None: + self.artist = artist + self.albumhashes: set[str] = set() + self.trackhashes: set[str] = set() + + class ArtistStore: artists: list[Artist] = CustomList() + artistmap: dict[str, ArtistMapEntry] = {} @classmethod def load_artists(cls, instance_key: str): @@ -24,15 +35,27 @@ class ArtistStore: ARTIST_LOAD_KEY = instance_key print("Loading artists... ", end="") - cls.artists.clear() + cls.artistmap.clear() - cls.artists.extend(get_all_artists(TrackStore.tracks, AlbumStore.albums)) - print("Done!") - for artist in ardb.get_all_artists(): + cls.artistmap = { + artist.artisthash: ArtistMapEntry(artist=artist) + for artist in create_artists() + } + + for track in TrackStore.get_flat_list(): if instance_key != ARTIST_LOAD_KEY: return - cls.map_artist_color(artist) + for hash in track.artisthashes: + cls.artistmap[hash].trackhashes.add(track.trackhash) + cls.artistmap[hash].albumhashes.add(track.albumhash) + + print("Done!") + # for artist in ardb.get_all_artists(): + # if instance_key != ARTIST_LOAD_KEY: + # return + + # cls.map_artist_color(artist) @classmethod def map_artist_color(cls, artist_tuple: tuple): @@ -65,24 +88,20 @@ class ArtistStore: cls.artists.append(artist) @classmethod - def get_artist_by_hash(cls, artisthash: str) -> Artist: + def get_artist_by_hash(cls, artisthash: str): """ Returns an artist by its hash.P """ - artists = sorted(cls.artists, key=lambda x: x.artisthash) - try: - artist = use_bisection(artists, "artisthash", [artisthash])[0] - return artist - except IndexError: - return None + entry = cls.artistmap.get(artisthash, None) + if entry is not None: + return entry.artist @classmethod - def get_artists_by_hashes(cls, artisthashes: list[str]) -> list[Artist]: + def get_artists_by_hashes(cls, artisthashes: Iterable[str]): """ Returns artists by their hashes. """ - artists = sorted(cls.artists, key=lambda x: x.artisthash) - artists = use_bisection(artists, "artisthash", artisthashes) + artists = [cls.get_artist_by_hash(hash) for hash in artisthashes] return [a for a in artists if a is not None] @classmethod @@ -113,3 +132,14 @@ class ArtistStore: Removes an artist from the store. """ cls.artists = CustomList(a for a in cls.artists if a.artisthash != artisthash) + + @classmethod + def get_artist_tracks(cls, artisthash: str): + """ + Returns all tracks by the given artist hash. + """ + entry = cls.artistmap.get(artisthash) + if entry is not None: + return TrackStore.get_tracks_by_trackhashes(entry.trackhashes) + + return [] diff --git a/app/store/tracks.py b/app/store/tracks.py index a3b158aa..65e5a1fb 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -1,8 +1,7 @@ # from tqdm import tqdm import itertools -import sys -from typing import Callable +from typing import Callable, Iterable from flask_jwt_extended import current_user from app.db.libdata import TrackTable from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb @@ -243,7 +242,7 @@ class TrackStore: # ================================================ @classmethod - def get_tracks_by_trackhashes(cls, trackhashes: list[str]) -> list[Track]: + def get_tracks_by_trackhashes(cls, trackhashes: Iterable[str]) -> list[Track]: """ Returns a list of tracks by their hashes. """ @@ -259,7 +258,9 @@ class TrackStore: tracks.append(track) # sort the tracks in the order of the given trackhashes - tracks.sort(key=lambda t: trackhashes.index(t.trackhash)) + if type(trackhashes) == list: + tracks.sort(key=lambda t: trackhashes.index(t.trackhash)) + return tracks @classmethod From 88a72763dfdf3ca26db7db8fd5c9ba5fd179bd07 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 15 Jul 2024 00:50:18 +0300 Subject: [PATCH 23/44] rewrite get all endpoint with stores --- app/api/getall/__init__.py | 19 ++++++++++--------- app/store/albums.py | 8 ++++++++ app/store/artists.py | 8 ++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index 44e0f6f1..b4801a59 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -1,13 +1,9 @@ -from flask import Blueprint - from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from datetime import datetime from app.api.apischemas import GenericLimitSchema -from app.db.libdata import ArtistTable -from app.db.libdata import AlbumTable from app.store.albums import AlbumStore from app.store.artists import ArtistStore @@ -70,9 +66,9 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): is_artists = path.itemtype == "artists" if is_albums: - items = AlbumTable.get_all() + items = AlbumStore.get_flat_list() elif is_artists: - items = ArtistTable.get_all() + items = ArtistStore.get_flat_list() total = len(items) @@ -95,12 +91,17 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): sort_is_artist_albumcount = is_artists and sort == "albumcount" lambda_sort = lambda x: getattr(x, sort) + lambda_sort_casefold = lambda x: getattr(x, sort).casefold() + if sort_is_artist: - lambda_sort = lambda x: getattr(x, sort)[0]["name"] + lambda_sort = lambda x: getattr(x, sort)[0]["name"].casefold() + + try: + sorted_items = sorted(items, key=lambda_sort_casefold, reverse=reverse) + except AttributeError: + sorted_items = sorted(items, key=lambda_sort, reverse=reverse) - sorted_items = sorted(items, key=lambda_sort, reverse=reverse) items = sorted_items[start : start + limit] - album_list = [] for item in items: diff --git a/app/store/albums.py b/app/store/albums.py index 69ef981e..76ab76c9 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -8,6 +8,7 @@ from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb from app.lib.tagger import create_albums from app.models import Album, Track from app.store.artists import ArtistStore +from app.utils import flatten from app.utils.customlist import CustomList from app.utils.remove_duplicates import remove_duplicates @@ -76,6 +77,13 @@ class AlbumStore: print("Done!") + @classmethod + def get_flat_list(cls): + """ + Returns a flat list of all albums. + """ + return [a.album for a in cls.albummap.values()] + @classmethod def add_album(cls, album: Album): """ diff --git a/app/store/artists.py b/app/store/artists.py index c3875e0f..5ae6c725 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -4,6 +4,7 @@ from typing import Iterable from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb from app.lib.tagger import create_artists from app.models import Artist +from app.utils import flatten from app.utils.bisection import use_bisection from app.utils.customlist import CustomList from app.utils.progressbar import tqdm @@ -57,6 +58,13 @@ class ArtistStore: # cls.map_artist_color(artist) + @classmethod + def get_flat_list(cls): + """ + Returns a flat list of all artists. + """ + return [a.artist for a in cls.artistmap.values()] + @classmethod def map_artist_color(cls, artist_tuple: tuple): """ From c8c21dc01a29f0aa717e0c8e42aa907232163be5 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 15 Jul 2024 20:11:18 +0300 Subject: [PATCH 24/44] implement playlist store --- TODO.md | 3 + app/api/playlist.py | 151 +++++++++++++++++++---------------------- app/models/playlist.py | 2 +- app/setup/__init__.py | 2 + app/store/playlists.py | 50 ++++++++++++++ 5 files changed, 127 insertions(+), 81 deletions(-) create mode 100644 app/store/playlists.py diff --git a/TODO.md b/TODO.md index e54eebbf..820ca4e2 100644 --- a/TODO.md +++ b/TODO.md @@ -61,3 +61,6 @@ - Paginate the following endpoints: 1. Folder tracks 2. Playlist tracks + + +- When you update a playlist, update the store as well! \ No newline at end of file diff --git a/app/api/playlist.py b/app/api/playlist.py index 3d61d238..f1da68d2 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -12,6 +12,7 @@ from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint, FileStorage from app import models +from app.api.apischemas import GenericLimitSchema from app.db.libdata import TrackTable from app.db.userdata import PlaylistTable from app.lib import playlistlib @@ -20,6 +21,9 @@ from app.lib.home.recentlyadded import get_recently_added_playlist from app.lib.home.recentlyplayed import get_recently_played_playlist from app.models.playlist import Playlist from app.serializers.playlist import serialize_for_card +from app.serializers.track import serialize_tracks +from app.store.playlists import PlaylistStore +from app.store.tracks import TrackStore from app.utils.dates import create_new_date, date_string_to_time_passed from app.utils.remove_duplicates import remove_duplicates from app.settings import Paths @@ -28,35 +32,6 @@ tag = Tag(name="Playlists", description="Get and manage playlists") api = APIBlueprint("playlists", __name__, url_prefix="/playlists", abp_tags=[tag]) -class SendAllPlaylistsQuery(BaseModel): - no_images: bool = Field(False, description="Whether to include images") - - -@api.get("") -def send_all_playlists(query: SendAllPlaylistsQuery): - """ - Gets all the playlists. - """ - playlists = PlaylistTable.get_all() - playlists = list(playlists) - - for playlist in playlists: - if not query.no_images: - playlist.images = playlistlib.get_first_4_images( - trackhashes=playlist.trackhashes - ) - playlist.images = [img["image"] for img in playlist.images] - - playlist.clear_lists() - - playlists.sort( - key=lambda p: datetime.strptime(p.last_updated, "%Y-%m-%d %H:%M:%S"), - reverse=True, - ) - - return {"data": playlists} - - def insert_playlist(name: str, image: str = None): playlist = { "image": image, @@ -79,31 +54,6 @@ def insert_playlist(name: str, image: str = None): return None -class CreatePlaylistBody(BaseModel): - name: str = Field(..., description="The name of the playlist") - - -@api.post("/new") -def create_playlist(body: CreatePlaylistBody): - """ - New playlist - - Creates a new playlist. Accepts POST method with a JSON body. - """ - # existing_playlist_count = PL.count_playlist_by_name(body.name) - exists = PlaylistTable.check_exists_by_name(body.name) - - if exists: - return {"error": "Playlist already exists"}, 409 - - playlist = insert_playlist(body.name) - - if playlist is None: - return {"error": "Playlist could not be created"}, 500 - - return {"playlist": playlist}, 201 - - def get_path_trackhashes(path: str): """ Returns a list of trackhashes in a folder. @@ -130,9 +80,63 @@ def get_artist_trackhashes(artisthash: str): return [t.trackhash for t in tracks] +def format_custom_playlist(playlist: models.Playlist, tracks: list[models.Track]): + duration = sum(t.duration for t in tracks) + + playlist.duration = duration + + return { + "info": serialize_for_card(playlist), + "tracks": tracks, + } + + +class SendAllPlaylistsQuery(BaseModel): + no_images: bool = Field(False, description="Whether to include images") + + +@api.get("") +def send_all_playlists(query: SendAllPlaylistsQuery): + """ + Gets all the playlists. + """ + playlists = PlaylistStore.get_flat_list() + playlists.sort( + key=lambda p: datetime.strptime(p.last_updated, "%Y-%m-%d %H:%M:%S"), + reverse=True, + ) + + return {"data": playlists} + + +class CreatePlaylistBody(BaseModel): + name: str = Field(..., description="The name of the playlist") + + +@api.post("/new") +def create_playlist(body: CreatePlaylistBody): + """ + New playlist + + Creates a new playlist. Accepts POST method with a JSON body. + """ + exists = PlaylistTable.check_exists_by_name(body.name) + + if exists: + return {"error": "Playlist already exists"}, 409 + + playlist = insert_playlist(body.name) + + if playlist is None: + return {"error": "Playlist could not be created"}, 500 + + PlaylistStore.add_playlist(playlist) + return {"playlist": playlist}, 201 + + class PlaylistIDPath(BaseModel): # INFO: playlistid string examples: "recentlyadded" - playlistid: int | str = Field(..., description="The ID of the playlist") + playlistid: str = Field(..., description="The ID of the playlist") class AddItemToPlaylistBody(BaseModel): @@ -170,19 +174,9 @@ def add_item_to_playlist(path: PlaylistIDPath, body: AddItemToPlaylistBody): return {"msg": "Done"}, 200 -class GetPlaylistQuery(BaseModel): +class GetPlaylistQuery(GenericLimitSchema): no_tracks: bool = Field(False, description="Whether to include tracks") - - -def format_custom_playlist(playlist: models.Playlist, tracks: list[models.Track]): - duration = sum(t.duration for t in tracks) - - playlist.duration = duration - - return { - "info": serialize_for_card(playlist), - "tracks": tracks, - } + start: int = Field(0, description="The start index of the tracks") @api.get("/") @@ -206,24 +200,22 @@ def get_playlist(path: PlaylistIDPath, query: GetPlaylistQuery): playlist, tracks = handler() return format_custom_playlist(playlist, tracks) - playlist = PlaylistTable.get_by_id(playlistid) + entry = PlaylistStore.playlistmap.get(playlistid) - if playlist is None: + if entry is None: return {"msg": "Playlist not found"}, 404 - tracks = TrackTable.get_tracks_by_trackhashes(playlist.trackhashes) + playlist = entry.playlist + tracks = PlaylistStore.get_playlist_tracks(playlistid, query.start, query.limit) duration = sum(t.duration for t in tracks) - playlist.last_updated = date_string_to_time_passed(playlist.last_updated) - + playlist._last_updated = date_string_to_time_passed(playlist.last_updated) playlist.duration = duration - if not playlist.has_image: - playlist.images = playlistlib.get_first_4_images(tracks) - - playlist.clear_lists() - - return {"info": playlist, "tracks": tracks if not no_tracks else []} + return { + "info": playlist, + "tracks": serialize_tracks(tracks) if not no_tracks else [], + } class UpdatePlaylistForm(BaseModel): @@ -340,7 +332,7 @@ def remove_playlist(path: PlaylistIDPath): Delete playlist """ PlaylistTable.remove_one(path.playlistid) - + PlaylistStore.playlistmap.pop(path.playlistid, None) return {"msg": "Done"}, 200 @@ -426,7 +418,6 @@ def save_item_as_playlist(body: SavePlaylistAsItemBody): img, str(playlist.id), "image/webp", filename=filename ) - # PL.add_tracks_to_playlist(playlist.id, trackhashes) PlaylistTable.append_to_playlist(playlist.id, trackhashes) playlist.count = len(trackhashes) diff --git a/app/models/playlist.py b/app/models/playlist.py index 16282f67..47423f29 100644 --- a/app/models/playlist.py +++ b/app/models/playlist.py @@ -1,5 +1,4 @@ import dataclasses -import json from dataclasses import dataclass from pathlib import Path from typing import Any @@ -20,6 +19,7 @@ class Playlist: trackhashes: list[str] = dataclasses.field(default_factory=list) extra: dict[str, Any] = dataclasses.field(default_factory=dict) + _last_updated: str = "" userid: int | None = None thumb: str = "" count: int = 0 diff --git a/app/setup/__init__.py b/app/setup/__init__.py index 85e6b334..70c5a1aa 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -8,6 +8,7 @@ from app.setup.sqlite import run_migrations, setup_sqlite from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.folder import FolderStore +from app.store.playlists import PlaylistStore from app.store.tracks import TrackStore from app.utils.generators import get_random_str from app.config import UserConfig @@ -49,4 +50,5 @@ def load_into_mem(): TrackStore.load_all_tracks(get_random_str()) AlbumStore.load_albums('a') ArtistStore.load_artists('a') + PlaylistStore.load_playlists() FolderStore.load_filepaths() \ No newline at end of file diff --git a/app/store/playlists.py b/app/store/playlists.py new file mode 100644 index 00000000..11151706 --- /dev/null +++ b/app/store/playlists.py @@ -0,0 +1,50 @@ +from app.db.userdata import PlaylistTable +from app.lib.playlistlib import get_first_4_images +from app.models.playlist import Playlist +from app.store.tracks import TrackStore + + +class PlaylistEntry: + def __init__(self, playlist: Playlist) -> None: + self.playlist = playlist + self.trackhashes: list[str] = playlist.trackhashes + self.playlist.clear_lists() + + if not playlist.has_image: + self.playlist.images = get_first_4_images( + TrackStore.get_tracks_by_trackhashes(self.trackhashes) + ) + + +class PlaylistStore: + playlistmap: dict[str, PlaylistEntry] = {} + + @classmethod + def load_playlists(cls): + """ + Loads all playlists into the store. + """ + cls.playlistmap = {str(p.id): PlaylistEntry(p) for p in PlaylistTable.get_all()} + print(cls.playlistmap) + + @classmethod + def get_playlist_tracks(cls, playlist_id: str, start: int, limit: int): + """ + Returns the trackhashes for a playlist. + """ + + entry = cls.playlistmap.get(playlist_id) + if entry is None: + return [] + + return TrackStore.get_tracks_by_trackhashes( + entry.trackhashes[start : start + limit] + ) + + @classmethod + def get_flat_list(cls): + return [p.playlist for p in cls.playlistmap.values()] + + @classmethod + def add_playlist(cls, playlist: Playlist): + cls.playlistmap[str(playlist.id)] = PlaylistEntry(playlist) From 2a1f178da29a68adad5baa7010715202c2b39983 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 18 Jul 2024 23:18:03 +0300 Subject: [PATCH 25/44] fix: album sort order on artist page --- app/api/artist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/artist.py b/app/api/artist.py index eb119183..96d7ad74 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -113,7 +113,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): album.check_type(list(tracks), config.showAlbumsAsSingles) albums = [a for a in albumdict.values()] - all_albums = sorted(albums, key=lambda a: str(a.date), reverse=True) + all_albums = sorted(albums, key=lambda a: a.date, reverse=True) res = { "albums": [], From 8f592a4636d19e61370cd7daec44745db66d79d6 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 19 Jul 2024 23:07:22 +0300 Subject: [PATCH 26/44] port recents endpoints to use stores --- TODO.md | 6 ++-- app/api/playlist.py | 8 +++--- app/lib/home/recentlyadded.py | 51 +++++++++++++++------------------- app/lib/home/recentlyplayed.py | 38 +++++++++---------------- app/store/albums.py | 8 ++---- app/store/playlists.py | 30 ++++++++++++++++++-- app/store/tracks.py | 17 ++++++++++++ 7 files changed, 90 insertions(+), 68 deletions(-) diff --git a/TODO.md b/TODO.md index 820ca4e2..21050db2 100644 --- a/TODO.md +++ b/TODO.md @@ -59,8 +59,8 @@ - Replace the DbManager class with cls.execute() - Paginate the following endpoints: + 1. Folder tracks - 2. Playlist tracks + 2. Playlist tracks ⭐ - -- When you update a playlist, update the store as well! \ No newline at end of file +- When you update a playlist, update the store as well! diff --git a/app/api/playlist.py b/app/api/playlist.py index f1da68d2..fe382785 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -81,13 +81,12 @@ def get_artist_trackhashes(artisthash: str): def format_custom_playlist(playlist: models.Playlist, tracks: list[models.Track]): - duration = sum(t.duration for t in tracks) - - playlist.duration = duration + playlist.duration = sum(t.duration for t in tracks) + playlist.count = len(tracks) return { "info": serialize_for_card(playlist), - "tracks": tracks, + "tracks": serialize_tracks(tracks), } @@ -354,6 +353,7 @@ def remove_tracks_from_playlist( # } PlaylistTable.remove_from_playlist(path.playlistid, body.tracks) + PlaylistStore.remove_from_playlist(path.playlistid, body.tracks) return {"msg": "Done"}, 200 diff --git a/app/lib/home/recentlyadded.py b/app/lib/home/recentlyadded.py index ba99c8d1..a3216830 100644 --- a/app/lib/home/recentlyadded.py +++ b/app/lib/home/recentlyadded.py @@ -1,15 +1,13 @@ from datetime import datetime -from pprint import pprint from time import time -from app.db.libdata import AlbumTable, ArtistTable, TrackTable from app.lib.playlistlib import get_first_4_images from app.models.playlist import Playlist from app.models.track import Track -# from app.store.tracks import TrackStore -# from app.store.albums import AlbumStore -# from app.store.artists import ArtistStore +from app.store.tracks import TrackStore +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore from app.serializers.track import serialize_track from app.serializers.album import album_serializer @@ -89,21 +87,21 @@ def create_track(t: Track): # INFO: Keys: folder, tracks, time (timestamp) -group_type = dict[str, list[Track], float] +# group_type = dict[str, str | list[Track] | float] -def check_folder_type(group_: group_type): +def check_folder_type(group_: dict): # check if all tracks in group have the same albumhash # if so, return "album" - key = group_["folder"] - tracks = group_["tracks"] - time = group_["time"] + key: str = group_["folder"] + tracks: list[Track] = group_["tracks"] + time: float = group_["time"] print(f"Checking folder: {key}") print(f"Tracks: {len(tracks)}") - existing_artist_hashes: set[str] = set(ArtistTable.get_all_hashes(time)) - existing_album_hashes: set[str] = set(AlbumTable.get_all_hashes(time)) + existing_artist_hashes: set[str] = set(ArtistStore.artistmap.keys()) + existing_album_hashes: set[str] = set(AlbumStore.albummap.keys()) if len(tracks) == 1: entry = create_track(tracks[0]) @@ -112,13 +110,14 @@ def check_folder_type(group_: group_type): is_album, albumhash, _ = check_is_album_folder(tracks) if is_album: - album = AlbumTable.get_album_by_albumhash(albumhash) + # album = AlbumTable.get_album_by_albumhash(albumhash) + entry = AlbumStore.albummap.get(albumhash) - if album is None: + if entry is None: return None album = album_serializer( - album, + entry.album, to_remove={ "genres", "og_title", @@ -141,12 +140,12 @@ def check_folder_type(group_: group_type): is_artist, artisthash, trackcount = check_is_artist_folder(tracks) if is_artist: - artist = ArtistTable.get_artist_by_hash(artisthash) + entry = ArtistStore.artistmap.get(artisthash) - if artist is None: + if entry is None: return None - artist = serialize_for_card(artist) + artist = serialize_for_card(entry.artist) artist["trackcount"] = trackcount artist["help_text"] = ( "NEW ARTIST" if artisthash not in existing_artist_hashes else "NEW MUSIC" @@ -200,7 +199,7 @@ def get_recently_added_items(limit: int = 7): print(f"Time taken to get tracks: {then - now}") groups = group_track_by_folders(tracks, {}) # print(groups) - last_trackcount: int = len(tracks) + # last_trackcount: int = len(tracks) # while len(groups.keys()) < limit and last_trackcount > 0: # distracks = get_recently_added_tracks(start=len(tracks), limit=100) @@ -223,8 +222,6 @@ def get_recently_added_items(limit: int = 7): } ) - pprint(f"😅😅😅😅😅😅😅😅😅😅😅😅😅😅😅😅 {grouplist[0]['len']}") - # sort groups by last modified date grouplist = sorted(grouplist, key=lambda group: group["time"], reverse=True) @@ -267,16 +264,14 @@ def get_recently_added_playlist(limit: int = 100): except IndexError: return playlist, [] - playlist.last_updated = date_string_to_time_passed(create_new_date(date)) - + playlist._last_updated = date_string_to_time_passed(create_new_date(date)) images = get_first_4_images(tracks=tracks) playlist.images = images - playlist.set_count(len(tracks)) + playlist.duration = sum(t.duration for t in tracks) + playlist.count = len(tracks) return playlist, tracks -def get_recently_added_tracks(start: int = 0, limit: int = 100): - # tracks = sorted(TrackStore.tracks, key=lambda t: t.created_date, reverse=True) - return TrackTable.get_recently_added(start, limit) - # return tracks[:limit] +def get_recently_added_tracks(start: int = 0, limit: int | None = 100): + return TrackStore.get_recently_added(start, limit) diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index a89cb92e..1f45761c 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -1,18 +1,13 @@ from datetime import datetime import os -from app.db.libdata import AlbumTable, ArtistTable, TrackTable from app.db.userdata import FavoritesTable, ScrobbleTable -from app.models.logger import TrackLog - -# from app.db.sqlite.logger.tracks import SQLiteTrackLogger as db -# from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb -# from app.db.sqlite.favorite import SQLiteFavoriteMethods as fdb from app.models.playlist import Playlist from app.serializers.track import serialize_track from app.serializers.album import album_serializer from app.lib.playlistlib import get_first_4_images from app.store.folder import FolderStore +from app.store.playlists import PlaylistStore from app.utils.dates import ( create_new_date, date_string_to_time_passed, @@ -22,14 +17,13 @@ from app.serializers.artist import serialize_for_card from app.serializers.playlist import serialize_for_card as serialize_playlist from app.lib.home.recentlyadded import get_recently_added_playlist -# from app.store.albums import AlbumStore -# from app.store.tracks import TrackStore -# from app.store.artists import ArtistStore +from app.store.albums import AlbumStore +from app.store.tracks import TrackStore +from app.store.artists import ArtistStore def get_recently_played(limit=7): # TODO: Paginate this - # entries = db.get_all() entries = ScrobbleTable.get_all(0, 200) items = [] added = set() @@ -43,16 +37,13 @@ def get_recently_played(limit=7): if len(items) >= limit: break - # entry = TrackLog(*entry) - if entry.source in added: continue added.add(entry.source) if entry.type == "album": - # album = AlbumStore.get_album_by_hash(entry.type_src) - album = AlbumTable.get_album_by_albumhash(entry.type_src) + album = AlbumStore.get_album_by_hash(entry.type_src) if album is None: continue @@ -80,8 +71,7 @@ def get_recently_played(limit=7): continue if entry.type == "artist": - # artist = ArtistStore.get_artist_by_hash(entry.type_src) - artist = ArtistTable.get_artist_by_hash(entry.type_src) + artist = ArtistStore.get_artist_by_hash(entry.type_src) if artist is None: continue @@ -132,7 +122,6 @@ def get_recently_played(limit=7): continue if entry.type == "playlist": - continue is_custom = entry.type_src in [i["name"] for i in custom_playlists] # is_recently_added = entry.type_src == "recentlyadded" @@ -159,7 +148,8 @@ def get_recently_played(limit=7): ) continue - playlist = pdb.get_playlist_by_id(entry.type_src) + # playlist = pdb.get_playlist_by_id(entry.type_src) + playlist = PlaylistStore.get_playlist_by_id(entry.type_src) if playlist is None: continue @@ -195,13 +185,12 @@ def get_recently_played(limit=7): ) continue - # track = TrackStore.get_tracks_by_trackhashes([entry.trackhash])[0] - track = TrackTable.get_track_by_trackhash(entry.trackhash) + t = TrackStore.trackhashmap.get(entry.trackhash) - if track is None: + if t is None: continue - track = serialize_track(track) + track = serialize_track(t.get_best()) track["help_text"] = "track" track["time"] = timestamp_to_time_passed(entry.timestamp) @@ -225,12 +214,11 @@ def get_recently_played_playlist(limit: int = 100): trackhashes=[], ) - tracks = TrackTable.get_recently_played(limit) + tracks = TrackStore.get_recently_played(limit) date = datetime.fromtimestamp(tracks[0].lastplayed) - playlist.last_updated = date_string_to_time_passed(create_new_date(date)) + playlist._last_updated = date_string_to_time_passed(create_new_date(date)) images = get_first_4_images(tracks=tracks) playlist.images = images - playlist.count = len(tracks) return playlist, tracks diff --git a/app/store/albums.py b/app/store/albums.py index 76ab76c9..b7e97d16 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -125,11 +125,9 @@ class AlbumStore: """ Returns an album by its hash. """ - for album in cls.albums: - if album.albumhash == albumhash: - return album - - return None + entry = cls.albummap.get(albumhash) + if entry is not None: + return entry.album @classmethod def get_albums_by_hashes(cls, albumhashes: Iterable[str]) -> list[Album]: diff --git a/app/store/playlists.py b/app/store/playlists.py index 11151706..c74827f8 100644 --- a/app/store/playlists.py +++ b/app/store/playlists.py @@ -1,3 +1,4 @@ +from operator import index from app.db.userdata import PlaylistTable from app.lib.playlistlib import get_first_4_images from app.models.playlist import Playlist @@ -11,9 +12,12 @@ class PlaylistEntry: self.playlist.clear_lists() if not playlist.has_image: - self.playlist.images = get_first_4_images( - TrackStore.get_tracks_by_trackhashes(self.trackhashes) - ) + self.rebuild_images() + + def rebuild_images(self): + self.playlist.images = get_first_4_images( + TrackStore.get_tracks_by_trackhashes(self.trackhashes) + ) class PlaylistStore: @@ -48,3 +52,23 @@ class PlaylistStore: @classmethod def add_playlist(cls, playlist: Playlist): cls.playlistmap[str(playlist.id)] = PlaylistEntry(playlist) + + @classmethod + def get_playlist_by_id(cls, id: str): + entry = cls.playlistmap.get(id) + + if entry is not None: + return entry.playlist + + @classmethod + def remove_from_playlist(cls, pid: str, tracks: list[dict[str, str]]): + playlist = cls.playlistmap.get(pid) + + if not playlist: + return + + for track in tracks: + if playlist.trackhashes.index(track["trackhash"]) == track["index"]: + playlist.trackhashes.remove(track["trackhash"]) + + playlist.rebuild_images() \ No newline at end of file diff --git a/app/store/tracks.py b/app/store/tracks.py index 65e5a1fb..a5905e15 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -350,3 +350,20 @@ class TrackStore: predicate=predicate, including_duplicates=True, ) + + @classmethod + def get_recently_added(cls, start: int, limit: int | None): + """ + Returns the most recently added tracks. + """ + tracks = cls.get_flat_list() + + if limit is None: + return sorted(tracks, key=lambda x: x.last_mod, reverse=True)[start:] + + return sorted(tracks, key=lambda x: x.last_mod, reverse=True)[start:limit] + + @classmethod + def get_recently_played(cls, limit: int): + tracks = cls.get_flat_list() + return sorted(tracks, key=lambda x: x.lastplayed, reverse=True)[:limit] From 5d32536758268d9c7c15fc3c0670bc2317eeea87 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 19 Jul 2024 23:46:28 +0300 Subject: [PATCH 27/44] implement getting all playlist tracks --- app/api/playlist.py | 3 +++ app/store/playlists.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/api/playlist.py b/app/api/playlist.py index fe382785..7e9aa3a3 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -205,6 +205,9 @@ def get_playlist(path: PlaylistIDPath, query: GetPlaylistQuery): return {"msg": "Playlist not found"}, 404 playlist = entry.playlist + if query.limit == -1: + query.limit = None + tracks = PlaylistStore.get_playlist_tracks(playlistid, query.start, query.limit) duration = sum(t.duration for t in tracks) diff --git a/app/store/playlists.py b/app/store/playlists.py index c74827f8..a575e11e 100644 --- a/app/store/playlists.py +++ b/app/store/playlists.py @@ -32,7 +32,7 @@ class PlaylistStore: print(cls.playlistmap) @classmethod - def get_playlist_tracks(cls, playlist_id: str, start: int, limit: int): + def get_playlist_tracks(cls, playlist_id: str, start: int, limit: int | None): """ Returns the trackhashes for a playlist. """ @@ -41,6 +41,9 @@ class PlaylistStore: if entry is None: return [] + if limit is None: + return TrackStore.get_tracks_by_trackhashes(entry.trackhashes[start:]) + return TrackStore.get_tracks_by_trackhashes( entry.trackhashes[start : start + limit] ) @@ -71,4 +74,4 @@ class PlaylistStore: if playlist.trackhashes.index(track["trackhash"]) == track["index"]: playlist.trackhashes.remove(track["trackhash"]) - playlist.rebuild_images() \ No newline at end of file + playlist.rebuild_images() From b0e904c84f34cd99f20c3c70777df10ecf4efdea Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 27 Jul 2024 21:44:33 +0300 Subject: [PATCH 28/44] port search to stores + fix favorites --- app/api/album.py | 4 + app/api/favorites.py | 48 +++++++--- app/api/scrobble/__init__.py | 29 ++++-- app/api/search.py | 16 ++-- app/api/settings.py | 5 +- app/db/__init__.py | 37 +------- app/db/engine.py | 25 ++++- app/db/libdata.py | 48 +++++----- app/db/metadata.py | 8 +- app/db/userdata.py | 107 +++++++++------------ app/lib/colorlib.py | 42 ++++----- app/lib/index.py | 25 +++++ app/lib/mapstuff.py | 68 ++++++++++++++ app/lib/playlistlib.py | 6 +- app/lib/populate.py | 176 +++-------------------------------- app/lib/searchlib.py | 41 ++++---- app/lib/tagger.py | 40 ++------ app/models/album.py | 23 ++++- app/models/artist.py | 20 +++- app/models/track.py | 168 +++------------------------------ app/periodic_scan.py | 39 ++++---- app/store/albums.py | 12 +++ app/store/artists.py | 12 +++ app/store/tracks.py | 90 ++++-------------- manage.py | 5 +- 25 files changed, 428 insertions(+), 666 deletions(-) create mode 100644 app/lib/index.py create mode 100644 app/lib/mapstuff.py diff --git a/app/api/album.py b/app/api/album.py index 8c762f1f..13dee219 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -2,6 +2,7 @@ Contains all the album routes. """ +from pprint import pprint import random from pydantic import BaseModel, Field @@ -54,6 +55,9 @@ def get_album_tracks_and_info(body: AlbumHashSchema): track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks}) avg_bitrate = sum(t.bitrate for t in tracks) // (len(tracks) or 1) + album.fav_userids = [1] + pprint(album) + return { "info": album, "extra": { diff --git a/app/api/favorites.py b/app/api/favorites.py index c6127a5c..481bbf66 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -11,7 +11,11 @@ from app.db.libdata import AlbumTable, TrackTable from app.db.userdata import FavoritesTable from app.models import FavType from app.settings import Defaults -from app.utils.bisection import use_bisection + +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore + from app.serializers.track import serialize_track, serialize_tracks from app.serializers.artist import ( serialize_for_card as serialize_artist, @@ -46,14 +50,27 @@ def toggle_favorite(body: FavoritesAddBody): """ Adds a favorite to the database. """ - FavoritesTable.insert_item({"hash": body.hash, "type": body.type}) + + try: + FavoritesTable.insert_item({"hash": body.hash, "type": body.type}) + except: + return {"msg": "Failed! An error occured"}, 500 if body.type == FavType.track: - TrackTable.set_is_favorite(body.hash, True) + entry = TrackStore.trackhashmap.get(body.hash) + if entry is not None: + entry.toggle_favorite_user() + elif body.type == FavType.album: - AlbumTable.set_is_favorite(body.hash, True) + entry = AlbumStore.albummap.get(body.hash) + + if entry is not None: + entry.toggle_favorite_user() elif body.type == FavType.artist: - ArtistTable.set_is_favorite(body.hash, True) + entry = ArtistStore.artistmap.get(body.hash) + + if entry is not None: + entry.toggle_favorite_user() return {"msg": "Added to favorites"} @@ -95,7 +112,8 @@ def get_favorite_albums(query: GetAllOfTypeQuery): fav_albums, total = FavoritesTable.get_fav_albums(query.start, query.limit) fav_albums.reverse() - return {"albums": serialize_for_card_many(fav_albums), "total": total} + albums = AlbumStore.get_albums_by_hashes(a.hash for a in fav_albums) + return {"albums": serialize_for_card_many(albums), "total": total} @api.get("/tracks") @@ -104,6 +122,10 @@ def get_favorite_tracks(query: GetAllOfTypeQuery): Get favorite tracks """ tracks, total = FavoritesTable.get_fav_tracks(query.start, query.limit) + + tracks.reverse() + tracks = TrackTable.get_tracks_by_trackhashes([t.hash for t in tracks]) + return {"tracks": serialize_tracks(tracks), "total": total} @@ -118,6 +140,7 @@ def get_favorite_artists(query: GetAllOfTypeQuery): ) artists.reverse() + artists = ArtistStore.get_artists_by_hashes(a.hash for a in artists) return {"artists": [serialize_artist(a) for a in artists], "total": total} @@ -164,9 +187,9 @@ def get_all_favorites(query: GetAllFavoritesQuery): albums = [] artists = [] - track_master_hash = TrackTable.get_all_hashes() - album_master_hash = AlbumTable.get_all_hashes() - artist_master_hash = ArtistTable.get_all_hashes() + track_master_hash = TrackStore.trackhashmap.keys() + album_master_hash = AlbumStore.albummap.keys() + artist_master_hash = ArtistStore.artistmap.keys() # INFO: Filter out invalid hashes (file not found or tags edited) for fav in favs: @@ -188,12 +211,11 @@ def get_all_favorites(query: GetAllFavoritesQuery): "artists": len(artists), } - tracks = TrackTable.get_tracks_by_trackhashes(tracks, limit=track_limit) - albums = AlbumTable.get_albums_by_albumhashes(albums, limit=album_limit) - artists = ArtistTable.get_artists_by_artisthashes(artists, limit=artist_limit) + tracks = TrackStore.get_tracks_by_trackhashes(tracks[:track_limit]) + albums = AlbumStore.get_albums_by_hashes(albums[:album_limit]) + artists = ArtistStore.get_artists_by_hashes(artists[:artist_limit]) recents = [] - # first_n = favs for fav in favs: if len(recents) >= largest: diff --git a/app/api/scrobble/__init__.py b/app/api/scrobble/__init__.py index 19137a0b..36400c2b 100644 --- a/app/api/scrobble/__init__.py +++ b/app/api/scrobble/__init__.py @@ -3,9 +3,11 @@ from flask_openapi3 import APIBlueprint from pydantic import Field from app.api.apischemas import TrackHashSchema -from app.db.libdata import AlbumTable, ArtistTable, TrackTable from app.db.userdata import ScrobbleTable from app.settings import Defaults +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore bp_tag = Tag(name="Logger", description="Log item plays") api = APIBlueprint("logger", __name__, url_prefix="/logger", abp_tags=[bp_tag]) @@ -33,14 +35,27 @@ def log_track(body: LogTrackBody): if not timestamp or duration < 5: return {"msg": "Invalid entry."}, 400 - track = TrackTable.get_track_by_trackhash(body.trackhash) - - if track is None: + trackentry = TrackStore.trackhashmap.get(body.trackhash) + if trackentry is None: return {"msg": "Track not found."}, 404 ScrobbleTable.add(dict(body)) - TrackTable.increment_playcount(body.trackhash, duration, timestamp) - AlbumTable.increment_playcount(track.albumhash, duration, timestamp) - ArtistTable.increment_playcount(track.artisthashes, duration, timestamp) + + # Update play data on the in-memory stores + track = trackentry.tracks[0] + album = AlbumStore.albummap.get(track.albumhash) + + if album: + album.increment_playcount(duration, timestamp) + + for hash in track.artisthashes: + artist = ArtistStore.artistmap.get(hash) + + if artist: + artist.increment_playcount(duration, timestamp) + + track = TrackStore.trackhashmap.get(body.trackhash) + if track: + track.increment_playcount(duration, timestamp) return {"msg": "recorded"}, 201 diff --git a/app/api/search.py b/app/api/search.py index f08840f3..19265185 100644 --- a/app/api/search.py +++ b/app/api/search.py @@ -9,9 +9,9 @@ from flask_openapi3 import APIBlueprint from app import models from app.api.apischemas import GenericLimitSchema -from app.db.libdata import TrackTable from app.lib import searchlib from app.settings import Defaults +from app.store.tracks import TrackStore tag = Tag(name="Search", description="Search for tracks, albums and artists") @@ -31,7 +31,7 @@ class Search: Calls :class:`SearchTracks` which returns the tracks that fuzzily match the search terms. Then adds them to the `SearchResults` store. """ - self.tracks = TrackTable.get_all() + self.tracks = TrackStore.get_flat_list() return searchlib.TopResults().search(self.query, tracks_only=True) def search_artists(self): @@ -124,7 +124,7 @@ def get_top_results(query: TopResultsQuery): class SearchLoadMoreQuery(SearchQuery): type: str = Field(description="The type of search", example="tracks") - index: int = Field(description="The index to start from", default=0) + start: int = Field(description="The index to start from", default=0) @api.get("/loadmore") @@ -136,26 +136,26 @@ def search_load_more(query: SearchLoadMoreQuery): NOTE: You must first initiate a search using the `/search` endpoint. """ - query = query.q + q = query.q item_type = query.type - index = query.index + index = query.start if item_type == "tracks": - t = Search(query).search_tracks() + t = Search(q).search_tracks() return { "tracks": t[index : index + SEARCH_COUNT], "more": len(t) > index + SEARCH_COUNT, } elif item_type == "albums": - a = Search(query).search_albums() + a = Search(q).search_albums() return { "albums": a[index : index + SEARCH_COUNT], "more": len(a) > index + SEARCH_COUNT, } elif item_type == "artists": - a = Search(query).search_artists() + a = Search(q).search_artists() return { "artists": a[index : index + SEARCH_COUNT], "more": len(a) > index + SEARCH_COUNT, diff --git a/app/api/settings.py b/app/api/settings.py index 3f1a3047..b333a34b 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -10,7 +10,7 @@ from app.db.sqlite.plugins import PluginsMethods as pdb from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb from app.db.userdata import PluginTable from app.lib import populate -from app.lib.tagger import index_everything +from app.lib.index import index_everything from app.lib.watchdogg import Watcher as WatchDog from app.logger import log from app.settings import Info, Paths, SessionVarKeys @@ -238,7 +238,8 @@ def set_setting(body: SetSettingBody): @background def run_populate(): - populate.Populate(instance_key=get_random_str()) + # populate.Populate(instance_key=get_random_str()) + pass @api.get("/trigger-scan") diff --git a/app/db/__init__.py b/app/db/__init__.py index 42fcb287..4f9bcad9 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -7,41 +7,10 @@ from sqlalchemy import ( select, ) -from sqlalchemy.engine import Engine -from sqlalchemy import event -from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass, Session - +from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass from app.db.engine import DbEngine -# Enable foreign key constraints for SQLite -@event.listens_for(Engine, "connect") -def set_sqlite_pragma(dbapi_connection, connection_record): - cursor = dbapi_connection.cursor() - cursor.execute("PRAGMA foreign_keys=ON") - cursor.close() - - -class DbManager: - """ """ - - def __init__(self, commit: bool = False): - self.commit = commit - self.conn = DbEngine.engine.connect() - - with Session(DbEngine.engine) as session: - session.connection - - def __enter__(self): - return self.conn.execution_options(preserve_rowcount=True) - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.commit: - self.conn.commit() - - self.conn.close() - - class Base(MappedAsDataclass, DeclarativeBase): """ Base class for all database models. @@ -51,8 +20,8 @@ class Base(MappedAsDataclass, DeclarativeBase): @classmethod def execute(cls, stmt: Any, commit: bool = False): - with DbEngine.manager(commit=commit) as conn: - return conn.execute(stmt) + with DbEngine.manager(commit=commit) as session: + return session.execute(stmt) @classmethod def insert_many(cls, items: list[dict[str, Any]]): diff --git a/app/db/engine.py b/app/db/engine.py index 841702de..055f5b42 100644 --- a/app/db/engine.py +++ b/app/db/engine.py @@ -1,5 +1,18 @@ from contextlib import contextmanager -from sqlalchemy import Engine +import gc +from sqlalchemy import Engine, event + + +@event.listens_for(Engine, "connect") +def set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA synchronous=NORMAL") + cursor.execute("PRAGMA cache_size=10000") + cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA temp_store=MEMORY") + cursor.execute("PRAGMA mmap_size=30000000000") + cursor.close() class DbEngine: @@ -11,20 +24,22 @@ class DbEngine: @classmethod @contextmanager - def manager(cls, commit: bool): + def manager(cls, commit: bool = False): """ This context manager manages access to the database. - When the context manager is entered, it returns a connection object that can be used to execute SQL statements. + When the context manager is entered, it returns a session object that can be used to execute SQL statements. If the `commit` parameter is set to `True`, the context manager will commit the transaction when it exits. """ + conn = cls.engine.connect() try: - conn = cls.engine.connect() yield conn.execution_options(preserve_rowcount=True) - if commit: conn.commit() + except Exception as e: + conn.rollback() + raise e finally: conn.close() diff --git a/app/db/libdata.py b/app/db/libdata.py index acf4c510..c6db3f15 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -1,6 +1,5 @@ from app.db import ( Base as MasterBase, - DbManager, ) from app.db.utils import ( album_to_dataclass, @@ -13,7 +12,7 @@ from app.db.utils import ( from app.models import Album as AlbumModel from app.utils.remove_duplicates import remove_duplicates from app.db.engine import DbEngine -from sqlalchemy import JSON, Boolean, Integer, String, delete, select, update +from sqlalchemy import JSON, Integer, String, delete, select, update from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase @@ -33,7 +32,7 @@ def create_all(): class Base(MasterBase, DeclarativeBase): @classmethod def get_all_hashes(cls, create_date: int | None = None): - with DbManager() as conn: + with DbEngine.manager() as conn: if create_date: if cls.__tablename__ == "track": stmt = select(TrackTable.trackhash).where( @@ -67,7 +66,7 @@ class Base(MasterBase, DeclarativeBase): hash (str): The hash value. is_favorite (bool): The value of the 'is_favorite' flag. """ - with DbManager(commit=True) as conn: + with DbEngine.manager(commit=True) as conn: if cls.__tablename__ == "track": stmt = ( update(cls) @@ -129,7 +128,6 @@ class TrackTable(Base): title: Mapped[str] = mapped_column(String()) track: Mapped[int] = mapped_column(Integer()) trackhash: Mapped[str] = mapped_column(String(), index=True) - # is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) lastplayed: Mapped[int] = mapped_column(Integer(), default=0) playcount: Mapped[int] = mapped_column(Integer(), default=0) playduration: Mapped[int] = mapped_column(Integer(), default=0) @@ -139,13 +137,13 @@ class TrackTable(Base): @classmethod def get_all(cls): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls)) return tracks_to_dataclasses(result.fetchall()) @classmethod def get_tracks_by_filepaths(cls, filepaths: list[str]): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(TrackTable) .where(TrackTable.filepath.in_(filepaths)) @@ -155,7 +153,7 @@ class TrackTable(Base): @classmethod def get_tracks_by_albumhash(cls, albumhash: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(TrackTable).where(TrackTable.albumhash == albumhash) ) @@ -164,7 +162,7 @@ class TrackTable(Base): @classmethod def get_track_by_trackhash(cls, hash: str, filepath: str = ""): - with DbManager() as conn: + with DbEngine.manager() as conn: if filepath: result = conn.execute( select(TrackTable) @@ -186,7 +184,7 @@ class TrackTable(Base): @classmethod def get_tracks_by_artisthash(cls, artisthash: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(TrackTable).where(TrackTable.artists.contains(artisthash)) ) @@ -194,7 +192,7 @@ class TrackTable(Base): @classmethod def get_tracks_in_path(cls, path: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(TrackTable) .where(TrackTable.filepath.contains(path)) @@ -204,7 +202,7 @@ class TrackTable(Base): @classmethod def get_tracks_by_trackhashes(cls, hashes: Iterable[str], limit: int | None = None): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(TrackTable) .where(TrackTable.trackhash.in_(hashes)) @@ -221,7 +219,7 @@ class TrackTable(Base): @classmethod def get_recently_added(cls, start: int, limit: int): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(TrackTable) .order_by(TrackTable.last_mod.desc()) @@ -243,7 +241,7 @@ class TrackTable(Base): @classmethod def remove_tracks_by_filepaths(cls, filepaths: set[str]): - with DbManager(commit=True) as conn: + with DbEngine.manager(commit=True) as conn: conn.execute(delete(TrackTable).where(TrackTable.filepath.in_(filepaths))) @classmethod @@ -270,7 +268,6 @@ class AlbumTable(Base): og_title: Mapped[str] = mapped_column(String()) title: Mapped[str] = mapped_column(String()) trackcount: Mapped[int] = mapped_column(Integer()) - # is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) lastplayed: Mapped[int] = mapped_column(Integer(), default=0) playcount: Mapped[int] = mapped_column(Integer(), default=0) playduration: Mapped[int] = mapped_column(Integer(), default=0) @@ -280,14 +277,14 @@ class AlbumTable(Base): @classmethod def get_all(cls): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(AlbumTable)) all = result.fetchall() return albums_to_dataclasses(all) @classmethod def get_album_by_albumhash(cls, hash: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(AlbumTable).where(AlbumTable.albumhash == hash) ) @@ -298,7 +295,7 @@ class AlbumTable(Base): @classmethod def get_albums_by_albumhashes(cls, hashes: Iterable[str], limit: int | None = None): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)).limit(limit) ) @@ -312,7 +309,7 @@ class AlbumTable(Base): @classmethod def get_albums_by_artisthashes(cls, artisthashes: list[str]): - with DbManager() as conn: + with DbEngine.manager() as conn: albums: dict[str, list[AlbumModel]] = {} for artist in artisthashes: @@ -325,7 +322,7 @@ class AlbumTable(Base): @classmethod def get_albums_by_base_title(cls, base_title: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(AlbumTable).where(AlbumTable.base_title == base_title) ) @@ -333,7 +330,7 @@ class AlbumTable(Base): @classmethod def get_albums_by_artisthash(cls, artisthash: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash)) ) @@ -359,7 +356,6 @@ class ArtistTable(Base): genres: Mapped[str] = mapped_column(JSON()) name: Mapped[str] = mapped_column(String(), index=True) trackcount: Mapped[int] = mapped_column(Integer()) - # is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) lastplayed: Mapped[int] = mapped_column(Integer(), default=0) playcount: Mapped[int] = mapped_column(Integer(), default=0) playduration: Mapped[int] = mapped_column(Integer(), default=0) @@ -369,14 +365,14 @@ class ArtistTable(Base): @classmethod def get_all(cls): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls)) all = result.fetchall() return artists_to_dataclasses(all) @classmethod def get_artist_by_hash(cls, artisthash: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(ArtistTable).where(ArtistTable.artisthash == artisthash) ) @@ -384,7 +380,7 @@ class ArtistTable(Base): @classmethod def get_artisthashes_not_in(cls, artisthashes: list[str]): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(ArtistTable.artisthash, ArtistTable.name).where( ~ArtistTable.artisthash.in_(artisthashes) @@ -396,7 +392,7 @@ class ArtistTable(Base): def get_artists_by_artisthashes( cls, hashes: Iterable[str], limit: int | None = None ): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(ArtistTable) .where(ArtistTable.artisthash.in_(hashes)) diff --git a/app/db/metadata.py b/app/db/metadata.py index 16976b45..3e5d37cf 100644 --- a/app/db/metadata.py +++ b/app/db/metadata.py @@ -1,9 +1,11 @@ -from app.db import Base, DbManager +from app.db import Base from sqlalchemy import Integer, insert, select, update from sqlalchemy.orm import Mapped, mapped_column +from app.db.engine import DbEngine + class MigrationTable(Base): __tablename__ = "dbmigration" @@ -13,7 +15,7 @@ class MigrationTable(Base): @classmethod def set_version(cls, version: int): - with DbManager(commit=True) as conn: + with DbEngine.manager(commit=True) as conn: result = conn.execute( update(cls).where(cls.id == 1).values(version=version) ) @@ -23,7 +25,7 @@ class MigrationTable(Base): @classmethod def get_version(cls): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls.version).where(cls.id == 1)) result = result.fetchone() diff --git a/app/db/userdata.py b/app/db/userdata.py index 7ab3d0db..57e87433 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -18,6 +18,7 @@ from sqlalchemy import ( from sqlalchemy.orm import Mapped, mapped_column +from app.db.engine import DbEngine from app.db.utils import ( albums_to_dataclasses, artists_to_dataclasses, @@ -27,14 +28,13 @@ from app.db.utils import ( plugin_to_dataclasses, similar_artist_to_dataclass, similar_artists_to_dataclass, - tracklog_to_dataclass, tracklog_to_dataclasses, tracks_to_dataclasses, user_to_dataclass, user_to_dataclasses, ) -from app.db import Base, DbManager +from app.db import Base from app.utils.auth import get_current_userid, hash_password @@ -77,7 +77,7 @@ class UserTable(Base): @classmethod def get_by_id(cls, id: int): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls).where(cls.id == id)) res = result.fetchone() @@ -86,7 +86,7 @@ class UserTable(Base): @classmethod def get_by_username(cls, username: str): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls).where(cls.username == username)) res = result.fetchone() @@ -95,7 +95,7 @@ class UserTable(Base): @classmethod def update_one(cls, user: dict[str, Any]): - with DbManager(commit=True) as conn: + with DbEngine.manager(commit=True) as conn: conn.execute(update(cls).where(cls.id == user["id"]).values(user)) @classmethod @@ -126,7 +126,7 @@ class SimilarArtistTable(Base): @classmethod def get_all(cls): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls)) return similar_artists_to_dataclass(result.fetchall()) @@ -136,7 +136,7 @@ class SimilarArtistTable(Base): Check whether an artisthash exists in the database. """ - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute( select(cls.artisthash).where(cls.artisthash == artisthash) ) @@ -148,7 +148,7 @@ class SimilarArtistTable(Base): Get a single artist by hash. """ - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls).where(cls.artisthash == artisthash)) result = result.fetchone() @@ -160,7 +160,7 @@ class FavoritesTable(Base): __tablename__ = "favorite" id: Mapped[int] = mapped_column(primary_key=True) - hash: Mapped[str] = mapped_column(String()) + hash: Mapped[str] = mapped_column(String(), unique=True) type: Mapped[str] = mapped_column(String(), index=True) timestamp: Mapped[int] = mapped_column(Integer(), index=True) userid: Mapped[int] = mapped_column( @@ -172,7 +172,7 @@ class FavoritesTable(Base): @classmethod def get_all(cls): - with DbManager() as conn: + with DbEngine.manager() as conn: result = conn.execute(select(cls)) return favorites_to_dataclass(result.fetchall()) @@ -181,12 +181,12 @@ class FavoritesTable(Base): item["timestamp"] = int(datetime.datetime.now().timestamp()) item["userid"] = get_current_userid() - with DbManager(commit=True) as conn: + with DbEngine.manager(commit=True) as conn: conn.execute(insert(cls).values(item)) @classmethod def remove_item(cls, item: dict[str, Any]): - with DbManager(commit=True) as conn: + with DbEngine.manager(commit=True) as conn: conn.execute( delete(cls).where( (cls.hash == item["hash"]) & (cls.type == item["type"]) @@ -199,12 +199,13 @@ class FavoritesTable(Base): return result.fetchone() is not None @classmethod - def get_all_of_type(cls, table: Any, field: Any, type: str, start: int, limit: int): + def get_all_of_type(cls, type: str, start: int, limit: int): result = cls.execute( - select(table) - .select_from(join(table, cls, field == cls.hash)) - .where(and_(cls.type == type, cls.userid == get_current_userid())) - .offset(start) + select(cls) + # .select_from(join(table, cls, field == cls.hash)) + .where(and_(cls.type == type, cls.userid == get_current_userid())).offset( + start + ) # INFO: If start is 0, fetch all so we can get the total count .limit(limit if start != 0 else None) ) @@ -218,30 +219,18 @@ class FavoritesTable(Base): @classmethod def get_fav_tracks(cls, start: int, limit: int): - from .libdata import TrackTable - - result, total = cls.get_all_of_type( - TrackTable, TrackTable.trackhash, "track", start, limit - ) - return tracks_to_dataclasses(result), total + result, total = cls.get_all_of_type("track", start, limit) + return favorites_to_dataclass(result), total @classmethod def get_fav_albums(cls, start: int, limit: int): - from .libdata import AlbumTable - - result, total = cls.get_all_of_type( - AlbumTable, AlbumTable.albumhash, "album", start, limit - ) - return albums_to_dataclasses(result), total + result, total = cls.get_all_of_type("album", start, limit) + return favorites_to_dataclass(result), total @classmethod def get_fav_artists(cls, start: int, limit: int): - from .libdata import ArtistTable - - result, total = cls.get_all_of_type( - ArtistTable, ArtistTable.artisthash, "artist", start, limit - ) - return artists_to_dataclasses(result), total + result, total = cls.get_all_of_type("artist", start, limit) + return favorites_to_dataclass(result), total class ScrobbleTable(Base): @@ -265,7 +254,7 @@ class ScrobbleTable(Base): return cls.insert_one(item) @classmethod - def get_all(cls, start: int, limit: int): + def get_all(cls, start: int, limit: int | None): result = cls.execute( select(cls) .where(cls.userid == get_current_userid()) @@ -325,7 +314,7 @@ class PlaylistTable(Base): ) @classmethod - def get_trackhashes(cls, id: int) -> list[str]: + def get_trackhashes(cls, id: int): result = cls.execute( select(cls.trackhashes).where( (cls.id == id) & (cls.userid == get_current_userid()) @@ -388,32 +377,24 @@ class PlaylistTable(Base): ) -# class PlaylistTrackTable(Base): -# __tablename__ = "playlisttrack" +class ArtistData(Base): + __tablename__ = "artistdata" -# id: Mapped[int] = mapped_column(primary_key=True) -# trackhash: Mapped[str] = mapped_column(String(), index=True) -# playlistid: Mapped[int] = mapped_column( -# Integer(), ForeignKey("playlist.id", ondelete="cascade") -# ) -# index: Mapped[int] = mapped_column(Integer()) -# userid: Mapped[int] = mapped_column( -# Integer(), ForeignKey("user.id", ondelete="cascade") -# ) + id: Mapped[int] = mapped_column(primary_key=True) + artisthash: Mapped[str] = mapped_column(String(), index=True) + color: Mapped[str] = mapped_column(String(), nullable=True) + bio: Mapped[str] = mapped_column(String(), nullable=True) + info: Mapped[dict[str, Any]] = mapped_column(JSON(), nullable=True) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) -# @classmethod -# def count_by_playlist() + @classmethod + def find_one(cls, artisthash: str): + result = cls.execute(select(cls).where(cls.artisthash == artisthash)) + return result.fetchone() -# @classmethod -# def insert_many(cls, playlistid: int, trackhashes: list[str]): -# userid = get_current_userid() -# items = [ -# { -# "index": index, -# "userid": userid, -# "trackhash": trackhash, -# "playlistid": playlistid, -# } -# for index, trackhash in enumerate(trackhashes) -# ] -# return cls.execute(insert(cls).values(items), commit=True) + @classmethod + def get_all_colors(cls) -> dict[str, str]: + result = cls.execute(select(cls.artisthash, cls.color)) + return dict(result.fetchall()) diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index cb360da5..8b73dfcf 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -12,9 +12,10 @@ from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb from app.db.sqlite.artistcolors import SQLiteArtistMethods as adb from app.db.sqlite.utils import SQLiteManager -# from app.store.artists import ArtistStore +from app.db.userdata import ArtistData from app.logger import log from app.lib.errors import PopulateCancelledError +from app.store.artists import ArtistStore from app.utils.progressbar import tqdm PROCESS_ALBUM_COLORS_KEY = "" @@ -100,32 +101,29 @@ class ProcessArtistColors: """ def __init__(self, instance_key: str) -> None: - # all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0] + all_artists = ArtistStore.get_flat_list() global PROCESS_ARTIST_COLORS_KEY PROCESS_ARTIST_COLORS_KEY = instance_key - with SQLiteManager() as cur: - try: - for artist in tqdm( - all_artists, desc="Processing missing artist colors" - ): - if PROCESS_ARTIST_COLORS_KEY != instance_key: - raise PopulateCancelledError( - "A newer 'ProcessArtistColors' instance is running. Stopping this one." - ) + try: + for artist in tqdm(all_artists, desc="Processing missing artist colors"): + if PROCESS_ARTIST_COLORS_KEY != instance_key: + raise PopulateCancelledError( + "A newer 'ProcessArtistColors' instance is running. Stopping this one." + ) - exists = adb.exists(artist.artisthash, cur=cur) + # exists = adb.exists(artist.artisthash, cur=cur) + artist = ArtistData.find_one(artist.artisthash) + if artist and artist.color is not None: + continue - if exists: - continue + colors = process_color(artist.artisthash, is_album=False) - colors = process_color(artist.artisthash, is_album=False) + if colors is None: + continue - if colors is None: - continue - - artist.set_colors(colors) - adb.insert_one_artist(cur, artist.artisthash, colors) - finally: - cur.close() + artist.set_colors(colors) + adb.insert_one_artist(cur, artist.artisthash, colors) + finally: + cur.close() diff --git a/app/lib/index.py b/app/lib/index.py new file mode 100644 index 00000000..4749ba7e --- /dev/null +++ b/app/lib/index.py @@ -0,0 +1,25 @@ +from app.lib.mapstuff import map_favorites, map_scrobble_data +from app.lib.populate import CordinateMedia +from app.lib.tagger import IndexTracks +from app.store.folder import FolderStore + + +import gc +from time import time + +from app.utils.threading import background + + +class IndexEverything: + def __init__(self) -> None: + IndexTracks(instance_key=time()) + FolderStore.load_filepaths() + map_scrobble_data() + map_favorites() + # CordinateMedia(instance_key=str(time())) + gc.collect() + + +@background +def index_everything(): + return IndexEverything() diff --git a/app/lib/mapstuff.py b/app/lib/mapstuff.py new file mode 100644 index 00000000..658e3b8a --- /dev/null +++ b/app/lib/mapstuff.py @@ -0,0 +1,68 @@ +from app.db.userdata import FavoritesTable, ScrobbleTable +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore + + +from typing import Any + + +def map_scrobble_data(): + """ + Maps scrobble data to the in-memory stores. + + The scrobble data is loaded from the database and grouped by trackhash. + The album and artist scrobble data (for those tracks) are then incremented based on the data. + """ + records = ScrobbleTable.get_all(0, None) + + # group records by trackhash + grouped: dict[str, dict[str, Any]] = {} + + for record in records: + # aggregate playcount, playduration and lastplayed + item = grouped.setdefault(record.trackhash, {}) + item["playcount"] = item.get("playcount", 0) + 1 + item["playduration"] = item.get("playduration", 0) + record.duration + item["lastplayed"] = max(item.get("lastplayed", 0), record.timestamp) + + # increment playcount, playduration and lastplayed for albums and artists + for trackhash, data in grouped.items(): + track = TrackStore.trackhashmap.get(trackhash) + + if track is None: + continue + + track.increment_playcount(data["playduration"], data["lastplayed"]) + + album = AlbumStore.albummap.get(track.tracks[0].albumhash) + if album: + album.increment_playcount(data["playduration"], data["lastplayed"]) + + for artisthash in track.tracks[0].artisthashes: + artist = ArtistStore.artistmap.get(artisthash) + if artist: + artist.increment_playcount(data["playduration"], data["lastplayed"]) + + +def map_favorites(): + """ + Maps favorites data to the in-memory stores. + """ + favorites = FavoritesTable.get_all() + + for entry in favorites: + if entry.type == "album": + album = AlbumStore.albummap.get(entry.hash) + if album: + album.toggle_favorite_user(entry.userid) + + elif entry.type == "artist": + artist = ArtistStore.artistmap.get(entry.hash) + if artist: + artist.toggle_favorite_user(entry.userid) + + elif entry.type == "track": + track = TrackStore.trackhashmap.get(entry.hash) + if track: + track.toggle_favorite_user(entry.userid) diff --git a/app/lib/playlistlib.py b/app/lib/playlistlib.py index 5d8f7292..ea62af18 100644 --- a/app/lib/playlistlib.py +++ b/app/lib/playlistlib.py @@ -10,8 +10,9 @@ from typing import Any from PIL import Image, ImageSequence from app import settings -from app.db.libdata import AlbumTable, TrackTable +from app.db.libdata import TrackTable from app.models.track import Track +from app.store.albums import AlbumStore def create_thumbnail(image: Any, img_path: str) -> str: """ @@ -115,8 +116,7 @@ def get_first_4_images( if len(albums) == 4: break - # albums = AlbumStore.get_albums_by_hashes(albums) - albums = AlbumTable.get_albums_by_albumhashes(albums) + albums = AlbumStore.get_albums_by_hashes(albums) images = [ { "image": album.image, diff --git a/app/lib/populate.py b/app/lib/populate.py index ea1e1afe..5506741e 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -1,28 +1,22 @@ from dataclasses import asdict import os -from collections import deque from concurrent.futures import ThreadPoolExecutor -from typing import Generator from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout from app import settings -from app.db.libdata import ArtistTable -from app.db.libdata import AlbumTable, TrackTable -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb - -# from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb from app.db.sqlite.tracks import SQLiteTrackMethods from app.lib.artistlib import CheckArtistImages from app.lib.colorlib import ProcessArtistColors from app.lib.errors import PopulateCancelledError from app.lib.taglib import extract_thumb from app.logger import log -from app.models import Album, Artist, Track +from app.models import Album, Artist from app.models.lastfm import SimilarArtist from app.requests.artists import fetch_similar_artists -from app.utils.filesystem import run_fast_scandir +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore from app.utils.network import has_connection from app.utils.progressbar import tqdm @@ -35,46 +29,6 @@ remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths POPULATE_KEY = "" -class Populate: - """ - Populates the database with all songs in the music directory - - checks if the song is in the database, if not, it adds it - also checks if the album art exists in the image path, if not tries to extract it. - """ - - # def __init__(self, instance_key: str) -> None: - # return - - # if len(dirs_to_scan) == 0: - # log.warning( - # ( - # "The root directory is not configured. " - # + "Open the app in your webbrowser to configure." - # ) - # ) - # return - - # try: - # if dirs_to_scan[0] == "$home": - # dirs_to_scan = [settings.Paths.USER_HOME_DIR] - # except IndexError: - # pass - - # files = set() - - # for _dir in dirs_to_scan: - # files = files.union(run_fast_scandir(_dir, full=True)[1]) - - # unmodified, modified_tracks = self.remove_modified(tracks) - # untagged = files - unmodified - - # if len(untagged) != 0: - # self.tag_untagged(untagged, instance_key) - - # self.extract_thumb_with_overwrite(modified_tracks) - - class CordinateMedia: """ Cordinates the extracting of thumbnails @@ -117,102 +71,6 @@ class CordinateMedia: log.warn(e) return - # @staticmethod - # def remove_modified(tracks: Generator[TrackTable, None, None]): - # """ - # Removes tracks from the database that have been modified - # since they were added to the database. - # """ - - # unmodified_paths = set() - # modified_tracks: list[TrackTable] = [] - # modified_paths = set() - - # for track in tracks: - # try: - # if track.last_mod == round(os.path.getmtime(track.filepath)): - # unmodified_paths.add(track.filepath) - # continue - # except (FileNotFoundError, OSError) as e: - # log.warning(e) # REVIEW More informations = good - # TrackStore.remove_track_obj(track) - # remove_tracks_by_filepaths(track.filepath) - - # modified_paths.add(track.filepath) - # modified_tracks.append(track) - - # TrackStore.remove_tracks_by_filepaths(modified_paths) - # remove_tracks_by_filepaths(modified_paths) - - # return unmodified_paths, modified_tracks - - # @staticmethod - # def tag_untagged(untagged: set[str], key: str): - # pass - # for file in tqdm(untagged, desc="Reading files"): - # if POPULATE_KEY != key: - # log.warning("'Populate.tag_untagged': Populate key changed") - # return - - # tags = get_tags(file) - - # if tags is not None: - # TrackTable.insert_one(tags) - - # ============================================= - - # log.info("Found %s new tracks", len(untagged)) - # # tagged_tracks: deque[dict] = deque() - # # tagged_count = 0 - - # favs = favdb.get_fav_tracks() - # records = dict() - - # for fav in favs: - # r = records.setdefault(fav[1], set()) - # r.add(fav[4]) - - # tagged_tracks.append(tags) - # track = Track(**tags) - - # track.fav_userids = list(records.get(track.trackhash, set())) - - # TrackStore.add_track(track) - - # if not AlbumStore.album_exists(track.albumhash): - # AlbumStore.add_album(AlbumStore.create_album(track)) - - # for artist in track.artists: - # if not ArtistStore.artist_exists(artist.artisthash): - # ArtistStore.add_artist(Artist(artist.name)) - - # for artist in track.albumartists: - # if not ArtistStore.artist_exists(artist.artisthash): - # ArtistStore.add_artist(Artist(artist.name)) - - # tagged_count += 1 - # else: - # log.warning("Could not read file: %s", file) - - # if len(tagged_tracks) > 0: - # log.info("Adding %s tracks to database", len(tagged_tracks)) - # insert_many_tracks(tagged_tracks) - - # log.info("Added %s/%s tracks", tagged_count, len(untagged)) - - # @staticmethod - # def extract_thumb_with_overwrite(tracks: list[TrackTable]): - # """ - # Extracts the thumbnail from a list of filepaths, - # overwriting the existing thumbnail if it exists, - # for modified files. - # """ - # for track in tracks: - # try: - # extract_thumb(track.filepath, track.image, overwrite=True) - # except FileNotFoundError: - # continue - def get_image(_map: tuple[str, Album]): """ @@ -228,25 +86,11 @@ def get_image(_map: tuple[str, Album]): if POPULATE_KEY != instance_key: raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed") - matching_tracks = filter( - lambda t: t.albumhash == album.albumhash, - TrackTable.get_tracks_by_albumhash(album.albumhash), - ) + matching_tracks = AlbumStore.get_album_tracks(album.albumhash) - try: - track = next(matching_tracks) - extracted = extract_thumb(track.filepath, track.image) - - while not extracted: - try: - track = next(matching_tracks) - extracted = extract_thumb(track.filepath, track.image) - except StopIteration: - break - - return - except StopIteration: - pass + for track in matching_tracks: + if extract_thumb(track.filepath, track.image): + break def get_cpu_count(): @@ -274,7 +118,7 @@ class ProcessTrackThumbnails: # filter out albums that already have thumbnails albums = filter( - lambda album: album.albumhash not in processed, AlbumTable.get_all() + lambda album: album.albumhash not in processed, AlbumStore.get_flat_list() ) albums = list(albums) @@ -330,7 +174,9 @@ class FetchSimilarArtistsLastFM: processed = ".".join(a.artisthash for a in processed) # filter out artists that already have similar artists - artists = filter(lambda a: a.artisthash not in processed, ArtistTable.get_all()) + artists = filter( + lambda a: a.artisthash not in processed, ArtistStore.get_flat_list() + ) artists = list(artists) # process the rest diff --git a/app/lib/searchlib.py b/app/lib/searchlib.py index 3d6cb1cb..d2bf65b9 100644 --- a/app/lib/searchlib.py +++ b/app/lib/searchlib.py @@ -9,7 +9,8 @@ from unidecode import unidecode from app import models from app.config import UserConfig -from app.db.libdata import AlbumTable, ArtistTable, TrackTable + +# from app.db.libdata import AlbumTable, ArtistTable, TrackTable # from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.models.enums import FavType @@ -19,9 +20,9 @@ from app.serializers.album import serialize_for_card_many as serialize_albums from app.serializers.artist import serialize_for_cards from app.serializers.track import serialize_track, serialize_tracks -# from app.store.albums import AlbumStore -# from app.store.artists import ArtistStore -# from app.store.tracks import TrackStore +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore from app.utils.remove_duplicates import remove_duplicates @@ -54,8 +55,7 @@ class Limit: class SearchTracks: def __init__(self, query: str) -> None: self.query = query - # self.tracks = TrackStore.tracks - self.tracks = TrackTable.get_all() + self.tracks = TrackStore.get_flat_list() def __call__(self) -> List[models.Track]: """ @@ -78,8 +78,7 @@ class SearchTracks: class SearchArtists: def __init__(self, query: str) -> None: self.query = query - # self.artists = ArtistStore.artists - self.artists = ArtistTable.get_all() + self.artists = ArtistStore.get_flat_list() def __call__(self): """ @@ -101,8 +100,7 @@ class SearchArtists: class SearchAlbums: def __init__(self, query: str) -> None: self.query = query - # self.albums = AlbumStore.albums - self.albums = AlbumTable.get_all() + self.albums = AlbumStore.get_flat_list() def __call__(self) -> List[models.Album]: """ @@ -169,9 +167,9 @@ class TopResults: def collect_all(): all_items: list[_type] = [] - all_items.extend(ArtistTable.get_all()) - all_items.extend(TrackTable.get_all()) - all_items.extend(AlbumTable.get_all()) + all_items.extend(ArtistStore.get_flat_list()) + all_items.extend(TrackStore.get_flat_list()) + all_items.extend(TrackStore.get_flat_list()) return all_items, get_titles(all_items) @@ -194,7 +192,7 @@ class TopResults: return {"type": "track", "item": item} if isinstance(item, models.Album): - tracks = TrackTable.get_tracks_by_albumhash(item.albumhash) + tracks = TrackStore.get_tracks_by_albumhash(item.albumhash) tracks = remove_duplicates(tracks) try: @@ -212,19 +210,13 @@ class TopResults: track_count = 0 duration = 0 - tracks = TrackTable.get_tracks_by_artisthash(item.artisthash) + tracks = TrackStore.get_tracks_by_artisthash(item.artisthash) tracks = remove_duplicates(tracks) for track in tracks: track_count += 1 duration += track.duration - # album_count = AlbumStore.count_albums_by_artisthash(item.artisthash) - - # item.set_trackcount(track_count) - # item.set_albumcount(album_count) - # item.set_duration(duration) - return {"type": "artist", "item": item} @staticmethod @@ -235,8 +227,7 @@ class TopResults: tracks.extend(SearchTracks(query)()) if item["type"] == "album": - t = TrackTable.get_tracks_by_albumhash(item["item"].albumhash) - # t = TrackStore.get_tracks_by_albumhash(item["item"].albumhash) + t = TrackStore.get_tracks_by_albumhash(item["item"].albumhash) t.sort(key=lambda x: x.last_mod) # if there are less than the limit, get more tracks @@ -249,7 +240,7 @@ class TopResults: if item["type"] == "artist": # t = TrackStore.get_tracks_by_artisthash(item["item"].artisthash) - t = TrackTable.get_tracks_by_artisthash(item["item"].artisthash) + t = TrackStore.get_tracks_by_artisthash(item["item"].artisthash) # if there are less than the limit, get more tracks if len(t) < limit: @@ -271,7 +262,7 @@ class TopResults: if item["type"] == "artist": # albums = AlbumStore.get_albums_by_artisthash(item["item"].artisthash) - albums = AlbumTable.get_albums_by_artisthash(item["item"].artisthash) + albums = AlbumStore.get_albums_by_artisthash(item["item"].artisthash) # if there are less than the limit, get more albums if len(albums) < limit: diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 0564f94a..b7613125 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -1,12 +1,9 @@ -import gc import os -from pprint import pprint -from time import time from app import settings from app.config import UserConfig -from app.db.libdata import ArtistTable from app.db.libdata import TrackTable -from app.lib.populate import CordinateMedia + +# from app.lib.populate import CordinateMedia from app.lib.taglib import extract_thumb, get_tags from app.models.album import Album from app.models.artist import Artist @@ -17,7 +14,6 @@ from app.utils.parsers import get_base_album_title from app.utils.progressbar import tqdm from app.logger import log -from app.utils.threading import background POPULATE_KEY: float = 0 @@ -142,7 +138,6 @@ class IndexTracks: print("Done") -# class IndexAlbums: def create_albums(): albums = dict() all_tracks: list[Track] = TrackTable.get_all() @@ -165,7 +160,7 @@ def create_albums(): "playduration": track.playduration, "title": track.album, "trackcount": 1, - "extra": {} + "extra": {}, } else: album = albums[track.albumhash] @@ -187,13 +182,11 @@ def create_albums(): genres.append(genre) album["genres"] = genres - album["genrehashes"] = " ".join([g['genrehash'] for g in genres]) + album["genrehashes"] = " ".join([g["genrehash"] for g in genres]) album["base_title"], _ = get_base_album_title(album["og_title"]) del genres - # AlbumTable.remove_all() - # AlbumTable.insert_many(list(albums.values())) return [Album(**album) for album in albums.values()] @@ -227,9 +220,7 @@ def create_artists(): "playduration": track.playduration, "trackcount": None, "tracks": ( - {track.trackhash} - if thisartist.get("in_track", True) - else set() + {track.trackhash} if thisartist.get("in_track", True) else set() ), "extra": {}, } @@ -261,7 +252,7 @@ def create_artists(): genres.append(genre) artist["genres"] = genres - artist["genrehashes"] = " ".join([g['genrehash'] for g in genres]) + artist["genrehashes"] = " ".join([g["genrehash"] for g in genres]) artist["name"] = sorted(artist["names"])[0] # INFO: Delete temporary keys @@ -272,25 +263,6 @@ def create_artists(): # INFO: Delete local variables del genres - # ArtistTable.remove_all() - # ArtistTable.insert_many(list(artists.values())) - # del artists return [Artist(**artist) for artist in artists.values()] -class IndexEverything: - def __init__(self) -> None: - IndexTracks(instance_key=time()) - # IndexAlbums() - # IndexArtists() - FolderStore.load_filepaths() - - # pass - - # CordinateMedia(instance_key=str(time())) - gc.collect() - - -@background -def index_everything(): - return IndexEverything() diff --git a/app/models/album.py b/app/models/album.py index 5b247e8a..0dc4322f 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -1,11 +1,10 @@ import dataclasses -import datetime from dataclasses import dataclass -from ..utils.hashing import create_hash -from ..utils.parsers import get_base_title_and_versions, parse_feat_from_title -from .artist import Artist from .track import Track +from ..utils.hashing import create_hash +from app.utils.auth import get_current_userid +from ..utils.parsers import get_base_title_and_versions @dataclass(slots=True) @@ -27,7 +26,6 @@ class Album: og_title: str title: str trackcount: int - # is_favorite: bool lastplayed: int playcount: int playduration: int @@ -37,6 +35,21 @@ class Album: type: str = "album" image: str = "" versions: list[str] = dataclasses.field(default_factory=list) + fav_userids: list[int] = dataclasses.field(default_factory=list) + + @property + def is_favorite(self): + return get_current_userid() in self.fav_userids + + def toggle_favorite_user(self, userid: int): + """ + Adds or removes the given user from the list of users + who have favorited the album. + """ + if userid in self.fav_userids: + self.fav_userids.remove(userid) + else: + self.fav_userids.append(userid) def __post_init__(self): self.image = self.albumhash + ".webp" diff --git a/app/models/artist.py b/app/models/artist.py index f7813f56..540098a7 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -1,6 +1,7 @@ import dataclasses from dataclasses import dataclass +from app.utils.auth import get_current_userid from app.utils.hashing import create_hash @@ -46,7 +47,6 @@ class Artist: genrehashes: list[str] name: str trackcount: int - # is_favorite: bool lastplayed: int playcount: int playduration: int @@ -55,5 +55,21 @@ class Artist: id: int = -1 image: str = "" + fav_userids: list[int] = dataclasses.field(default_factory=list) + + @property + def is_favorite(self): + return get_current_userid() in self.fav_userids + + def toggle_favorite_user(self, userid: int): + """ + Adds or removes the given user from the list of users + who have favorited this artist. + """ + if userid in self.fav_userids: + self.fav_userids.remove(userid) + else: + self.fav_userids.append(userid) + def __post_init__(self): - self.image = self.artisthash + ".webp" \ No newline at end of file + self.image = self.artisthash + ".webp" diff --git a/app/models/track.py b/app/models/track.py index fa6984e5..37436105 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -38,12 +38,22 @@ class Track: _pos: int = 0 _ati: str = "" image: str = "" - fav_userids: set = field(default_factory=set) + fav_userids: list[int] = field(default_factory=list) @property def is_favorite(self): return get_current_userid() in self.fav_userids + def toggle_favorite_user(self, userid: int): + """ + Adds or removes the given user from the list of users + who have favorited the track. + """ + if userid in self.fav_userids: + self.fav_userids.remove(userid) + else: + self.fav_userids.append(userid) + def __post_init__(self): self.image = self.albumhash + ".webp" self.extra = { @@ -51,159 +61,3 @@ class Track: "track_total": self.extra.get("track_total", 0), "samplerate": self.extra.get("samplerate", -1), } - - # album: str - # albumartists: str | list[ArtistMinimal] - # albumhash: str - # artists: str | list[ArtistMinimal] - # bitrate: int - # copyright: str - # date: int - # disc: int - # duration: int - # filepath: str - # folder: str - # genre: str | list[str] - # title: str - # track: int - # trackhash: str - # last_mod: str | int - - # image: str = "" - # artist_hashes: str = "" - - # """ - # A string of user ids separated by commas. - # """ - # # is_favorite: bool = False - - # # temporary attributes - # _pos: int = 0 # for sorting tracks by disc and track number - # _ati: str = ( - # "" # (album track identifier) for removing duplicates when merging album versions - # ) - - # og_title: str = "" - # og_album: str = "" - # created_date: float = 0.0 - - # def set_created_date(self): - # try: - # self.created_date = Path(self.filepath).stat().st_ctime - # except FileNotFoundError: - # pass - - # def __post_init__(self): - # self.og_title = self.title - # self.og_album = self.album - # self.last_mod = int(self.last_mod) - # self.date = int(self.date) - - # # add a trailing slash to the folder path - # # to avoid matching a folder starting with the same name as the root path - # # eg. .../Music and .../Music Videos - # self.folder = os.path.join(self.folder, "") - - # if self.artists is not None: - # artists = split_artists(self.artists) - # new_title = self.title - - # if get_flag(SessionVarKeys.EXTRACT_FEAT): - # featured, new_title = parse_feat_from_title(self.title) - # original_lower = "-".join([create_hash(a) for a in artists]) - # artists.extend( - # a for a in featured if create_hash(a) not in original_lower - # ) - - # self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists) - # self.artists = [ArtistMinimal(a) for a in artists] - - # albumartists = split_artists(self.albumartists) - - # if not albumartists: - # self.albumartists = self.artists[:1] - # else: - # self.albumartists = [ArtistMinimal(a) for a in albumartists] - - # if get_flag(SessionVarKeys.REMOVE_PROD): - # new_title = remove_prod(new_title) - - # if track is a single - # if self.og_title == self.album: - # self.rename_album(new_title) - - # if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK): - # new_title = clean_title(new_title) - - # self.title = new_title - - # if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): - # self.album, _ = get_base_title_and_versions( - # self.album, get_versions=False - # ) - - # if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS): - # self.recreate_albumhash() - - # self.image = self.albumhash + ".webp" - - # if self.genre is not None and self.genre != "": - # self.genre = self.genre.lower() - # separators = {"/", ";", "&"} - - # contains_rnb = "r&b" in self.genre - # contains_rock = "rock & roll" in self.genre - - # if contains_rnb: - # self.genre = self.genre.replace("r&b", "RnB") - - # if contains_rock: - # self.genre = self.genre.replace("rock & roll", "rock") - - # for s in separators: - # self.genre: str = self.genre.replace(s, ",") - - # self.genre = self.genre.split(",") - # self.genre = [g.strip() for g in self.genre] - - # self.recreate_hash() - # self.set_created_date() - - # def recreate_hash(self): - # """ - # Recreates a track hash if the track title was altered - # to prevent duplicate tracks having different hashes. - # """ - # if self.og_title == self.title and self.og_album == self.album: - # return - - # self.trackhash = create_hash( - # ", ".join(a.name for a in self.artists), self.og_album, self.title - # ) - - # def recreate_artists_hash(self): - # """ - # Recreates a track's artist hashes if the artist list was altered - # """ - # self.artist_hashes = "-".join(a.artisthash for a in self.artists) - - # def recreate_albumhash(self): - # """ - # Recreates an albumhash of a track to merge all versions of an album. - # """ - # albumartists = (a.name for a in self.albumartists) - # self.albumhash = create_hash(self.album, *albumartists) - - # def rename_album(self, new_album: str): - # """ - # Renames an album - # """ - # self.album = new_album - - # def add_artists(self, artists: list[str], new_album_title: str): - # for artist in artists: - # if create_hash(artist, decode=True) not in self.artist_hashes: - # self.artists.append(ArtistMinimal(artist)) - - # self.recreate_artists_hash() - # self.rename_album(new_album_title) diff --git a/app/periodic_scan.py b/app/periodic_scan.py index f8b19133..931e8779 100644 --- a/app/periodic_scan.py +++ b/app/periodic_scan.py @@ -5,32 +5,31 @@ This module contains functions for the server import time from app.config import UserConfig -from app.lib.populate import Populate, PopulateCancelledError +from app.lib.populate import PopulateCancelledError from app.utils.generators import get_random_str from app.utils.threading import background from app.logger import log -@background -def run_periodic_scans(): - """ - Runs periodic scans. +# @background +# def run_periodic_scans(): +# """ +# Runs periodic scans. - Periodic scans are checks that run every few minutes - in the background to do stuff like: - - checking for new music - - delete deleted entries - - downloading artist images, and other data. - """ - # ValidateAlbumThumbs() - # ValidatePlaylistThumbs() +# Periodic scans are checks that run every few minutes +# in the background to do stuff like: +# - checking for new music +# - delete deleted entries +# - downloading artist images, and other data. +# """ +# # ValidateAlbumThumbs() +# # ValidatePlaylistThumbs() - while UserConfig().enablePeriodicScans: +# while UserConfig().enablePeriodicScans: - try: - Populate(instance_key=get_random_str()) - except PopulateCancelledError: - log.error("'run_periodic_scans': Periodic scan cancelled.") - pass +# try: +# except PopulateCancelledError: +# log.error("'run_periodic_scans': Periodic scan cancelled.") +# pass - time.sleep(UserConfig().scanInterval) +# time.sleep(UserConfig().scanInterval) diff --git a/app/store/albums.py b/app/store/albums.py index b7e97d16..14cc0086 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -9,6 +9,7 @@ from app.lib.tagger import create_albums from app.models import Album, Track from app.store.artists import ArtistStore from app.utils import flatten +from app.utils.auth import get_current_userid from app.utils.customlist import CustomList from app.utils.remove_duplicates import remove_duplicates @@ -28,6 +29,17 @@ class AlbumMapEntry: def basetitle(self): return self.album.base_title + def increment_playcount(self, duration: int, timestamp: int): + self.album.lastplayed = timestamp + self.album.playduration += duration + self.album.playcount += 1 + + def toggle_favorite_user(self, userid: int | None = None): + if userid is None: + userid = get_current_userid() + + self.album.toggle_favorite_user(userid) + class AlbumStore: albums: list[Album] = CustomList() diff --git a/app/store/artists.py b/app/store/artists.py index 5ae6c725..b4d71ae2 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -5,6 +5,7 @@ from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb from app.lib.tagger import create_artists from app.models import Artist from app.utils import flatten +from app.utils.auth import get_current_userid from app.utils.bisection import use_bisection from app.utils.customlist import CustomList from app.utils.progressbar import tqdm @@ -22,6 +23,17 @@ class ArtistMapEntry: self.albumhashes: set[str] = set() self.trackhashes: set[str] = set() + def increment_playcount(self, duration: int, timestamp: int): + self.artist.lastplayed = timestamp + self.artist.playduration += duration + self.artist.playcount += 1 + + def toggle_favorite_user(self, userid: int | None = None): + if userid is None: + userid = get_current_userid() + + self.artist.toggle_favorite_user(userid) + class ArtistStore: artists: list[Artist] = CustomList() diff --git a/app/store/tracks.py b/app/store/tracks.py index a5905e15..21e56759 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -2,13 +2,10 @@ import itertools from typing import Callable, Iterable -from flask_jwt_extended import current_user from app.db.libdata import TrackTable -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -# from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb -from app.db.userdata import FavoritesTable from app.models import Track +from app.utils.auth import get_current_userid from app.utils.remove_duplicates import remove_duplicates TRACKS_LOAD_KEY = "" @@ -34,12 +31,24 @@ class TrackGroup: """ self.tracks.remove(track) - def set_fav_userids(self, userids: set[int]): + def increment_playcount(self, duration: int, timestamp: int): """ - Sets the favorite userids. + Increments the playcount of all tracks in the group. """ for track in self.tracks: - track.fav_userids = userids + track.playcount += 1 + track.lastplayed = timestamp + track.playduration += duration + + def toggle_favorite_user(self, userid: int | None = None): + """ + Adds or removes a user from the list of users who have favorited the track. + """ + if userid is None: + userid = get_current_userid() + + for track in self.tracks: + track.toggle_favorite_user(userid) def get_best(self): """ @@ -47,21 +56,6 @@ class TrackGroup: """ return max(self.tracks, key=lambda x: x.bitrate) - def toggle_favorite(self, remove: bool = False): - """ - Adds a track to the favorites. - """ - - userids = set(self.tracks[0].fav_userids) - - if remove: - userids.remove(current_user["id"]) - else: - userids.add(current_user["id"]) - - for track in self.tracks: - track.fav_userids = userids - def __len__(self): return len(self.tracks) @@ -72,7 +66,8 @@ class classproperty(property): """ def __get__(self, owner_self, owner_cls): - return self.fget(owner_cls) + if self.fget: + return self.fget(owner_cls) class TrackStore: @@ -118,35 +113,6 @@ class TrackStore: else: cls.trackhashmap[track.trackhash].append(track) - # favs = favdb.get_fav_tracks() - favs = FavoritesTable.get_all() - records: dict[str, set[int]] = dict() - - # convert records: {trackhash: {userid, userid, ...}} - for fav in favs: - if fav.hash not in records: - # if trackhash not in dict, add it - # and set the value to a set containing the userid - records[fav.hash] = {fav.userid} - - # if trackhash is in dict, add the userid to the set - records[fav.hash].add(fav.userid) - - for record in records: - if instance_key != TRACKS_LOAD_KEY: - return - - group = cls.trackhashmap.get(record, None) - - if not group: - continue - - group.set_fav_userids(records.get(record, set())) - - # print("Done!") - # print(cls.trackhashmap.get("0d6b22c19c").tracks[0].fav_userids) - # sys.exit(0) - @classmethod def add_track(cls, track: Track): """ @@ -219,24 +185,6 @@ class TrackStore: """ return len(cls.trackhashmap.get(trackhash, [])) - @classmethod - def toggle_favorite(cls, trackhash: str, remove: bool = False): - """ - Adds a track to the favorites. - """ - - group = cls.trackhashmap.get(trackhash) - - if group: - group.toggle_favorite(remove=remove) - - @classmethod - def remove_track_from_fav(cls, trackhash: str): - """ - Removes a track from the favorites. - """ - return cls.toggle_favorite(trackhash, remove=True) - # ================================================ # ================== GETTERS ===================== # ================================================ @@ -332,7 +280,7 @@ class TrackStore: """ predicate = lambda artisthashes, artisthash: artisthash in artisthashes return cls.find_tracks_by( - key="artist_hashes", value=artisthash, predicate=predicate + key="artisthashes", value=artisthash, predicate=predicate ) @classmethod diff --git a/manage.py b/manage.py index aaa7e25d..f2f11639 100644 --- a/manage.py +++ b/manage.py @@ -21,7 +21,8 @@ import setproctitle from app.api import create_api from app.arg_handler import ProcessArgs -from app.lib.tagger import IndexEverything +from app.lib.mapstuff import map_favorites, map_scrobble_data +from app.lib.index import IndexEverything from app.lib.watchdogg import Watcher as WatchDog from app.plugins.register import register_plugins from app.settings import FLASKVARS, TCOLOR, Info @@ -65,6 +66,8 @@ def bg_run_setup(): pass # run_periodic_scans() IndexEverything() + # map_scrobble_data() + # map_favorites() # @background From 881e1d6581847fbfc3d107f3a7518272706771e9 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 27 Jul 2024 22:23:02 +0300 Subject: [PATCH 29/44] fix playlists --- app/api/playlist.py | 31 ++++++++------ app/lib/home/recentlyplayed.py | 6 +-- app/lib/playlistlib.py | 11 ++++- app/setup/__init__.py | 7 +--- app/store/playlists.py | 77 ---------------------------------- 5 files changed, 31 insertions(+), 101 deletions(-) delete mode 100644 app/store/playlists.py diff --git a/app/api/playlist.py b/app/api/playlist.py index 7e9aa3a3..a437bf8f 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -22,10 +22,9 @@ from app.lib.home.recentlyplayed import get_recently_played_playlist from app.models.playlist import Playlist from app.serializers.playlist import serialize_for_card from app.serializers.track import serialize_tracks -from app.store.playlists import PlaylistStore + from app.store.tracks import TrackStore from app.utils.dates import create_new_date, date_string_to_time_passed -from app.utils.remove_duplicates import remove_duplicates from app.settings import Paths tag = Tag(name="Playlists", description="Get and manage playlists") @@ -99,7 +98,16 @@ def send_all_playlists(query: SendAllPlaylistsQuery): """ Gets all the playlists. """ - playlists = PlaylistStore.get_flat_list() + playlists = PlaylistTable.get_all() + + for playlist in playlists: + if not playlist.has_image: + playlist.images = playlistlib.get_first_4_images( + trackhashes=playlist.trackhashes + ) + + playlist.clear_lists() + playlists.sort( key=lambda p: datetime.strptime(p.last_updated, "%Y-%m-%d %H:%M:%S"), reverse=True, @@ -129,7 +137,6 @@ def create_playlist(body: CreatePlaylistBody): if playlist is None: return {"error": "Playlist could not be created"}, 500 - PlaylistStore.add_playlist(playlist) return {"playlist": playlist}, 201 @@ -199,20 +206,22 @@ def get_playlist(path: PlaylistIDPath, query: GetPlaylistQuery): playlist, tracks = handler() return format_custom_playlist(playlist, tracks) - entry = PlaylistStore.playlistmap.get(playlistid) + playlist = PlaylistTable.get_by_id(int(playlistid)) - if entry is None: + if playlist is None: return {"msg": "Playlist not found"}, 404 - playlist = entry.playlist if query.limit == -1: - query.limit = None - - tracks = PlaylistStore.get_playlist_tracks(playlistid, query.start, query.limit) + query.limit = len(playlist.trackhashes) - 1 + tracks = TrackStore.get_tracks_by_trackhashes( + playlist.trackhashes[query.start : query.start + query.limit] + ) duration = sum(t.duration for t in tracks) playlist._last_updated = date_string_to_time_passed(playlist.last_updated) playlist.duration = duration + playlist.images = playlistlib.get_first_4_images(tracks) + playlist.clear_lists() return { "info": playlist, @@ -334,7 +343,6 @@ def remove_playlist(path: PlaylistIDPath): Delete playlist """ PlaylistTable.remove_one(path.playlistid) - PlaylistStore.playlistmap.pop(path.playlistid, None) return {"msg": "Done"}, 200 @@ -356,7 +364,6 @@ def remove_tracks_from_playlist( # } PlaylistTable.remove_from_playlist(path.playlistid, body.tracks) - PlaylistStore.remove_from_playlist(path.playlistid, body.tracks) return {"msg": "Done"}, 200 diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index 1f45761c..6a133af0 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -1,13 +1,12 @@ from datetime import datetime import os -from app.db.userdata import FavoritesTable, ScrobbleTable +from app.db.userdata import FavoritesTable, PlaylistTable, ScrobbleTable from app.models.playlist import Playlist from app.serializers.track import serialize_track from app.serializers.album import album_serializer from app.lib.playlistlib import get_first_4_images from app.store.folder import FolderStore -from app.store.playlists import PlaylistStore from app.utils.dates import ( create_new_date, date_string_to_time_passed, @@ -148,8 +147,7 @@ def get_recently_played(limit=7): ) continue - # playlist = pdb.get_playlist_by_id(entry.type_src) - playlist = PlaylistStore.get_playlist_by_id(entry.type_src) + playlist = PlaylistTable.get_by_id(entry.type_src) if playlist is None: continue diff --git a/app/lib/playlistlib.py b/app/lib/playlistlib.py index ea62af18..4757b323 100644 --- a/app/lib/playlistlib.py +++ b/app/lib/playlistlib.py @@ -13,6 +13,8 @@ from app import settings from app.db.libdata import TrackTable from app.models.track import Track from app.store.albums import AlbumStore +from app.store.tracks import TrackStore + def create_thumbnail(image: Any, img_path: str) -> str: """ @@ -103,9 +105,14 @@ def duplicate_images(images: list): def get_first_4_images( tracks: list[Track] = [], trackhashes: list[str] = [] ) -> list[dict["str", str]]: + """ + Returns images of the first 4 albums that appear in the track list. + + When tracks are not passed, trackhashes need to be passed. + Tracks are then resolved from the store. + """ if len(trackhashes) > 0: - # tracks = TrackStore.get_tracks_by_trackhashes(trackhashes) - tracks = TrackTable.get_tracks_by_trackhashes(trackhashes) + tracks = TrackStore.get_tracks_by_trackhashes(trackhashes) albums = [] diff --git a/app/setup/__init__.py b/app/setup/__init__.py index 70c5a1aa..7aa5e766 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -8,7 +8,6 @@ from app.setup.sqlite import run_migrations, setup_sqlite from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.folder import FolderStore -from app.store.playlists import PlaylistStore from app.store.tracks import TrackStore from app.utils.generators import get_random_str from app.config import UserConfig @@ -43,12 +42,8 @@ def load_into_mem(): """ # instance_key = get_random_str() - # INFO: Load all tracks, albums, and artists into memory - # TrackStore.load_all_tracks(instance_key) - # AlbumStore.load_albums(instance_key) - # ArtistStore.load_artists(instance_key) + # INFO: Load all tracks, albums, and artists data into memory TrackStore.load_all_tracks(get_random_str()) AlbumStore.load_albums('a') ArtistStore.load_artists('a') - PlaylistStore.load_playlists() FolderStore.load_filepaths() \ No newline at end of file diff --git a/app/store/playlists.py b/app/store/playlists.py deleted file mode 100644 index a575e11e..00000000 --- a/app/store/playlists.py +++ /dev/null @@ -1,77 +0,0 @@ -from operator import index -from app.db.userdata import PlaylistTable -from app.lib.playlistlib import get_first_4_images -from app.models.playlist import Playlist -from app.store.tracks import TrackStore - - -class PlaylistEntry: - def __init__(self, playlist: Playlist) -> None: - self.playlist = playlist - self.trackhashes: list[str] = playlist.trackhashes - self.playlist.clear_lists() - - if not playlist.has_image: - self.rebuild_images() - - def rebuild_images(self): - self.playlist.images = get_first_4_images( - TrackStore.get_tracks_by_trackhashes(self.trackhashes) - ) - - -class PlaylistStore: - playlistmap: dict[str, PlaylistEntry] = {} - - @classmethod - def load_playlists(cls): - """ - Loads all playlists into the store. - """ - cls.playlistmap = {str(p.id): PlaylistEntry(p) for p in PlaylistTable.get_all()} - print(cls.playlistmap) - - @classmethod - def get_playlist_tracks(cls, playlist_id: str, start: int, limit: int | None): - """ - Returns the trackhashes for a playlist. - """ - - entry = cls.playlistmap.get(playlist_id) - if entry is None: - return [] - - if limit is None: - return TrackStore.get_tracks_by_trackhashes(entry.trackhashes[start:]) - - return TrackStore.get_tracks_by_trackhashes( - entry.trackhashes[start : start + limit] - ) - - @classmethod - def get_flat_list(cls): - return [p.playlist for p in cls.playlistmap.values()] - - @classmethod - def add_playlist(cls, playlist: Playlist): - cls.playlistmap[str(playlist.id)] = PlaylistEntry(playlist) - - @classmethod - def get_playlist_by_id(cls, id: str): - entry = cls.playlistmap.get(id) - - if entry is not None: - return entry.playlist - - @classmethod - def remove_from_playlist(cls, pid: str, tracks: list[dict[str, str]]): - playlist = cls.playlistmap.get(pid) - - if not playlist: - return - - for track in tracks: - if playlist.trackhashes.index(track["trackhash"]) == track["index"]: - playlist.trackhashes.remove(track["trackhash"]) - - playlist.rebuild_images() From 56da0acd85d5b84399292d7829552d3288a6d831 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 28 Jul 2024 17:02:58 +0300 Subject: [PATCH 30/44] fix: is_favorite missing from album and artist page --- app/api/album.py | 8 +++++++- app/api/artist.py | 3 ++- app/api/settings.py | 1 - 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/api/album.py b/app/api/album.py index 13dee219..aa84e7eb 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -2,6 +2,7 @@ Contains all the album routes. """ +from dataclasses import asdict from pprint import pprint import random @@ -52,6 +53,8 @@ def get_album_tracks_and_info(body: AlbumHashSchema): tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles ) + print("is_favorite", album.is_favorite) + track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks}) avg_bitrate = sum(t.bitrate for t in tracks) // (len(tracks) or 1) @@ -59,7 +62,10 @@ def get_album_tracks_and_info(body: AlbumHashSchema): pprint(album) return { - "info": album, + "info": { + **asdict(album), + "is_favorite": album.is_favorite, + }, "extra": { # INFO: track_total is the sum of a set of track_total values from each track # ASSUMPTIONS diff --git a/app/api/artist.py b/app/api/artist.py index 96d7ad74..a43ad880 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -2,6 +2,7 @@ Contains all the artist(s) routes. """ +from dataclasses import asdict import math import random from datetime import datetime @@ -68,7 +69,7 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): artist.genres.insert(0, {"name": decade, "genrehash": decade}) return { - "artist": artist, + "artist": {**asdict(artist), "is_favorite": artist.is_favorite}, "tracks": serialize_tracks(tracks[:limit]), } diff --git a/app/api/settings.py b/app/api/settings.py index b333a34b..1e65f3d8 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -9,7 +9,6 @@ from app.api.auth import admin_required from app.db.sqlite.plugins import PluginsMethods as pdb from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb from app.db.userdata import PluginTable -from app.lib import populate from app.lib.index import index_everything from app.lib.watchdogg import Watcher as WatchDog from app.logger import log From 16db3e1ad226f00c1be7dafe33781068abc93e0a Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 28 Jul 2024 17:12:00 +0300 Subject: [PATCH 31/44] fix: wrong albums appearing as appearances in artist page --- TODO.md | 13 ------------- app/api/artist.py | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/TODO.md b/TODO.md index 21050db2..a7a80a75 100644 --- a/TODO.md +++ b/TODO.md @@ -38,7 +38,6 @@ - Watchdog - Periodic scans - Remove legacy db methods -- Remove all stores - Review: We don't need server side image colors - Clean up main db and userdata modules - Move plugins to a config file @@ -46,21 +45,9 @@ - Add userid to queries - Remove duplicates on artist page (test with Hanson) - Test foreign keys on delete -- Map scrobble info on app start -- Make home page recent items faster! - Normalize playlists table: - New table to hold playlist entries - Normalize similar artists: - New table to hold similar artist entries - Create 2 way relationships, such that if an artist A is similar to another B with a certain weight, then artist B is similar to A with the same weight, unless overwritten. -- Figure out how to update album/artist tables instead of deleting all rows when the app starts -- Move get all filtering and sorting operations to the database since all sort keys are table columns -- Replace the DbManager class with cls.execute() - -- Paginate the following endpoints: - - 1. Folder tracks - 2. Playlist tracks ⭐ - -- When you update a playlist, update the store as well! diff --git a/app/api/artist.py b/app/api/artist.py index a43ad880..762ddca9 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -128,7 +128,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): res["singles_and_eps"].append(album) elif album.type == "compilation": res["compilations"].append(album) - elif album.albumhash in missing_albumhashes: + elif album.albumhash in missing_albumhashes or artisthash not in album.artisthashes: res["appearances"].append(album) else: res["albums"].append(album) From 0463c80070b149c021b5d8ff95d12956d1d19e50 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 2 Aug 2024 12:25:55 +0300 Subject: [PATCH 32/44] fix: album favorite state, artist and album colors + fix: unserialized artist result + misc --- TODO.md | 4 ++ app/api/album.py | 10 +-- app/api/artist.py | 16 ++++- app/api/favorites.py | 55 ++++++++-------- app/db/userdata.py | 31 +++++---- app/lib/colorlib.py | 115 +++++++++++++++++----------------- app/lib/home/recentlyadded.py | 4 -- app/lib/index.py | 6 +- app/lib/mapstuff.py | 22 ++++++- app/lib/populate.py | 3 +- app/lib/tagger.py | 25 +++++--- app/models/artist.py | 1 + app/serializers/album.py | 1 + app/store/albums.py | 29 +++------ app/store/artists.py | 36 +++++------ manage.py | 2 - 16 files changed, 195 insertions(+), 165 deletions(-) diff --git a/TODO.md b/TODO.md index a7a80a75..ee8a7879 100644 --- a/TODO.md +++ b/TODO.md @@ -51,3 +51,7 @@ - New table to hold similar artist entries - Create 2 way relationships, such that if an artist A is similar to another B with a certain weight, then artist B is similar to A with the same weight, unless overwritten. + +# Bug fixes + +- Duplicates on search \ No newline at end of file diff --git a/app/api/album.py b/app/api/album.py index aa84e7eb..1eb3660b 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -3,7 +3,6 @@ Contains all the album routes. """ from dataclasses import asdict -from pprint import pprint import random from pydantic import BaseModel, Field @@ -39,7 +38,6 @@ def get_album_tracks_and_info(body: AlbumHashSchema): Returns album info and tracks for the given albumhash. """ albumhash = body.albumhash - # album = AlbumDb.get_album_by_albumhash(albumhash) albumentry = AlbumStore.albummap.get(albumhash) if albumentry is None: @@ -53,14 +51,9 @@ def get_album_tracks_and_info(body: AlbumHashSchema): tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles ) - print("is_favorite", album.is_favorite) - track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks}) avg_bitrate = sum(t.bitrate for t in tracks) // (len(tracks) or 1) - album.fav_userids = [1] - pprint(album) - return { "info": { **asdict(album), @@ -128,10 +121,11 @@ def get_more_from_artist(body: GetMoreFromArtistsBody): a for a in albums # INFO: filter out albums added to other artists - if a.albumhash not in seen_hashes + if a.albumhash not in seen_hashes and artisthash in a.artisthashes # INFO: filter out albums with the same base title and create_hash(a.base_title) != create_hash(base_title) ] + all_albums[artisthash] = serialize_for_card_many( [a for a in albums if create_hash(a.base_title) != create_hash(base_title)][ :limit diff --git a/app/api/artist.py b/app/api/artist.py index 762ddca9..150796cd 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -21,7 +21,7 @@ from app.config import UserConfig from app.db.userdata import SimilarArtistTable from app.serializers.album import serialize_for_card_many -from app.serializers.artist import serialize_for_cards +from app.serializers.artist import serialize_for_cards, serialize_for_card from app.serializers.track import serialize_tracks from app.store.albums import AlbumStore @@ -69,7 +69,14 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): artist.genres.insert(0, {"name": decade, "genrehash": decade}) return { - "artist": {**asdict(artist), "is_favorite": artist.is_favorite}, + "artist": { + **serialize_for_card(artist), + "duration": sum(t.duration for t in tracks) if tracks else 0, + "trackcount": tcount, + "albumcount": artist.albumcount, + "genres": artist.genres, + "is_favorite": artist.is_favorite, + }, "tracks": serialize_tracks(tracks[:limit]), } @@ -128,7 +135,10 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): res["singles_and_eps"].append(album) elif album.type == "compilation": res["compilations"].append(album) - elif album.albumhash in missing_albumhashes or artisthash not in album.artisthashes: + elif ( + album.albumhash in missing_albumhashes + or artisthash not in album.artisthashes + ): res["appearances"].append(album) else: res["albums"].append(album) diff --git a/app/api/favorites.py b/app/api/favorites.py index 481bbf66..7e8aaaae 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -1,13 +1,11 @@ from typing import List, TypeVar -from flask_jwt_extended import current_user from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from app.api.apischemas import GenericLimitSchema -from app.db.libdata import ArtistTable -from app.db.libdata import AlbumTable, TrackTable +from app.db.libdata import TrackTable from app.db.userdata import FavoritesTable from app.models import FavType from app.settings import Defaults @@ -45,6 +43,29 @@ class FavoritesAddBody(BaseModel): type: str = Field(description="The type of the item", example=FavType.album) +def toggle_fav(type: str, hash: str): + """ + Toggles a favorite item. + """ + if type == FavType.track: + entry = TrackStore.trackhashmap.get(hash) + if entry is not None: + entry.toggle_favorite_user() + + elif type == FavType.album: + entry = AlbumStore.albummap.get(hash) + + if entry is not None: + entry.toggle_favorite_user() + elif type == FavType.artist: + entry = ArtistStore.artistmap.get(hash) + + if entry is not None: + entry.toggle_favorite_user() + + return {"msg": "Added to favorites"} + + @api.post("/add") def toggle_favorite(body: FavoritesAddBody): """ @@ -56,21 +77,7 @@ def toggle_favorite(body: FavoritesAddBody): except: return {"msg": "Failed! An error occured"}, 500 - if body.type == FavType.track: - entry = TrackStore.trackhashmap.get(body.hash) - if entry is not None: - entry.toggle_favorite_user() - - elif body.type == FavType.album: - entry = AlbumStore.albummap.get(body.hash) - - if entry is not None: - entry.toggle_favorite_user() - elif body.type == FavType.artist: - entry = ArtistStore.artistmap.get(body.hash) - - if entry is not None: - entry.toggle_favorite_user() + toggle_fav(body.type, body.hash) return {"msg": "Added to favorites"} @@ -80,14 +87,12 @@ def remove_favorite(body: FavoritesAddBody): """ Removes a favorite from the database. """ - FavoritesTable.remove_item({"hash": body.hash, "type": body.type}) + try: + FavoritesTable.remove_item({"hash": body.hash, "type": body.type}) + except: + return {"msg": "Failed! An error occured"}, 500 - if body.type == FavType.track: - TrackTable.set_is_favorite(body.hash, False) - elif body.type == FavType.album: - AlbumTable.set_is_favorite(body.hash, False) - elif body.type == FavType.artist: - ArtistTable.set_is_favorite(body.hash, False) + toggle_fav(body.type, body.hash) return {"msg": "Removed from favorites"} diff --git a/app/db/userdata.py b/app/db/userdata.py index 57e87433..9f3bea2f 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -1,6 +1,4 @@ import datetime -import enum -from shlex import join from typing import Any from sqlalchemy import ( JSON, @@ -13,15 +11,12 @@ from sqlalchemy import ( insert, select, update, - join, ) from sqlalchemy.orm import Mapped, mapped_column from app.db.engine import DbEngine from app.db.utils import ( - albums_to_dataclasses, - artists_to_dataclasses, favorites_to_dataclass, playlist_to_dataclass, playlists_to_dataclasses, @@ -29,7 +24,6 @@ from app.db.utils import ( similar_artist_to_dataclass, similar_artists_to_dataclass, tracklog_to_dataclasses, - tracks_to_dataclasses, user_to_dataclass, user_to_dataclasses, ) @@ -377,11 +371,12 @@ class PlaylistTable(Base): ) -class ArtistData(Base): +class LibDataTable(Base): __tablename__ = "artistdata" id: Mapped[int] = mapped_column(primary_key=True) - artisthash: Mapped[str] = mapped_column(String(), index=True) + itemhash: Mapped[str] = mapped_column(String(), unique=True, index=True) + itemtype: Mapped[str] = mapped_column(String()) color: Mapped[str] = mapped_column(String(), nullable=True) bio: Mapped[str] = mapped_column(String(), nullable=True) info: Mapped[dict[str, Any]] = mapped_column(JSON(), nullable=True) @@ -390,11 +385,21 @@ class ArtistData(Base): ) @classmethod - def find_one(cls, artisthash: str): - result = cls.execute(select(cls).where(cls.artisthash == artisthash)) + def update_one(cls, hash: str, data: dict[str, Any]): + return cls.execute( + update(cls).where(cls.itemhash == hash).values(data), commit=True + ) + + @classmethod + def find_one(cls, hash: str, type: str): + result = cls.execute( + select(cls).where((cls.itemhash == hash) & (cls.itemtype == type)) + ) return result.fetchone() @classmethod - def get_all_colors(cls) -> dict[str, str]: - result = cls.execute(select(cls.artisthash, cls.color)) - return dict(result.fetchall()) + def get_all_colors(cls, type: str) -> list[dict[str, str]]: + result = cls.execute( + select(cls.itemhash, cls.color).where(cls.itemtype == type) + ) + return [{"itemhash": r[0], "color": r[1]} for r in result.fetchall()] diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index 8b73dfcf..5b56dbe0 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -2,19 +2,16 @@ Contains everything that deals with image color extraction. """ -import json from pathlib import Path import colorgram from app import settings -from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb -from app.db.sqlite.artistcolors import SQLiteArtistMethods as adb -from app.db.sqlite.utils import SQLiteManager -from app.db.userdata import ArtistData +from app.db.userdata import LibDataTable from app.logger import log from app.lib.errors import PopulateCancelledError +from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.utils.progressbar import tqdm @@ -52,47 +49,45 @@ def process_color(item_hash: str, is_album=True): return get_image_colors(str(path)) -# class ProcessAlbumColors: -# """ -# Extracts the most dominant color from the album art and saves it to the database. -# """ +class ProcessAlbumColors: + """ + Extracts the most dominant color from the album art and saves it to the database. + """ -# def __init__(self, instance_key: str) -> None: -# global PROCESS_ALBUM_COLORS_KEY -# PROCESS_ALBUM_COLORS_KEY = instance_key + def __init__(self, instance_key: str) -> None: + global PROCESS_ALBUM_COLORS_KEY + PROCESS_ALBUM_COLORS_KEY = instance_key -# albums = [ -# a -# for a in AlbumStore.albums -# if a is not None and a.colors is not None and len(a.colors) == 0 -# ] + albums = [a for a in AlbumStore.get_flat_list() if not a.color] -# with SQLiteManager() as cur: -# try: -# for album in tqdm(albums, desc="Processing missing album colors"): -# if PROCESS_ALBUM_COLORS_KEY != instance_key: -# raise PopulateCancelledError( -# "A newer 'ProcessAlbumColors' instance is running. Stopping this one." -# ) + for album in tqdm(albums, desc="Processing missing album colors"): + albumhash = album.albumhash + if PROCESS_ALBUM_COLORS_KEY != instance_key: + raise PopulateCancelledError( + "A newer 'ProcessAlbumColors' instance is running. Stopping this one." + ) -# # TODO: Stop hitting the database for every album. -# # Instead, fetch all the data from the database and -# # check from memory. + albumrecord = LibDataTable.find_one(albumhash, type="album") + if albumrecord is not None and albumrecord.color is not None: + continue -# exists = aldb.exists(album.albumhash, cur=cur) -# if exists: -# continue + colors = process_color(albumhash) -# colors = process_color(album.albumhash) + if colors is None: + continue -# if colors is None: -# continue + album = AlbumStore.albummap.get(albumhash) -# album.set_colors(colors) -# color_str = json.dumps(colors) -# aldb.insert_one_album(cur, album.albumhash, color_str) -# finally: -# cur.close() + if album: + album.set_color(colors[0]) + + # INFO: Write to the database. + if albumrecord is None: + LibDataTable.insert_one( + {"itemhash": albumhash, "color": colors[0], "itemtype": "album"} + ) + else: + LibDataTable.update_one(albumhash, {"color": colors[0]}) class ProcessArtistColors: @@ -101,29 +96,37 @@ class ProcessArtistColors: """ def __init__(self, instance_key: str) -> None: - all_artists = ArtistStore.get_flat_list() + all_artists = [a for a in ArtistStore.get_flat_list() if not a.color] global PROCESS_ARTIST_COLORS_KEY PROCESS_ARTIST_COLORS_KEY = instance_key - try: - for artist in tqdm(all_artists, desc="Processing missing artist colors"): - if PROCESS_ARTIST_COLORS_KEY != instance_key: - raise PopulateCancelledError( - "A newer 'ProcessArtistColors' instance is running. Stopping this one." - ) + for artist in tqdm(all_artists, desc="Processing missing artist colors"): + artisthash = artist.artisthash + if PROCESS_ARTIST_COLORS_KEY != instance_key: + raise PopulateCancelledError( + "A newer 'ProcessArtistColors' instance is running. Stopping this one." + ) - # exists = adb.exists(artist.artisthash, cur=cur) - artist = ArtistData.find_one(artist.artisthash) - if artist and artist.color is not None: - continue + record = LibDataTable.find_one(artisthash, "artist") - colors = process_color(artist.artisthash, is_album=False) + if (record is not None) and (record.color is not None): + continue - if colors is None: - continue + colors = process_color(artisthash, is_album=False) - artist.set_colors(colors) - adb.insert_one_artist(cur, artist.artisthash, colors) - finally: - cur.close() + if colors is None: + continue + + artist = ArtistStore.artistmap.get(artisthash) + + if artist: + artist.set_color(colors[0]) + + # INFO: Write to the database. + if record is None: + LibDataTable.insert_one( + {"itemhash": artisthash, "color": colors[0], "itemtype": "artist"} + ) + else: + LibDataTable.update_one(artisthash, {"color": colors[0]}) diff --git a/app/lib/home/recentlyadded.py b/app/lib/home/recentlyadded.py index a3216830..4c2c98ab 100644 --- a/app/lib/home/recentlyadded.py +++ b/app/lib/home/recentlyadded.py @@ -96,10 +96,6 @@ def check_folder_type(group_: dict): key: str = group_["folder"] tracks: list[Track] = group_["tracks"] time: float = group_["time"] - - print(f"Checking folder: {key}") - print(f"Tracks: {len(tracks)}") - existing_artist_hashes: set[str] = set(ArtistStore.artistmap.keys()) existing_album_hashes: set[str] = set(AlbumStore.albummap.keys()) diff --git a/app/lib/index.py b/app/lib/index.py index 4749ba7e..604158f0 100644 --- a/app/lib/index.py +++ b/app/lib/index.py @@ -1,4 +1,4 @@ -from app.lib.mapstuff import map_favorites, map_scrobble_data +from app.lib.mapstuff import map_album_colors, map_artist_colors, map_favorites, map_scrobble_data from app.lib.populate import CordinateMedia from app.lib.tagger import IndexTracks from app.store.folder import FolderStore @@ -16,7 +16,9 @@ class IndexEverything: FolderStore.load_filepaths() map_scrobble_data() map_favorites() - # CordinateMedia(instance_key=str(time())) + map_artist_colors() + map_album_colors() + CordinateMedia(instance_key=str(time())) gc.collect() diff --git a/app/lib/mapstuff.py b/app/lib/mapstuff.py index 658e3b8a..7f356fe0 100644 --- a/app/lib/mapstuff.py +++ b/app/lib/mapstuff.py @@ -1,4 +1,4 @@ -from app.db.userdata import FavoritesTable, ScrobbleTable +from app.db.userdata import LibDataTable, FavoritesTable, ScrobbleTable from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore @@ -66,3 +66,23 @@ def map_favorites(): track = TrackStore.trackhashmap.get(entry.hash) if track: track.toggle_favorite_user(entry.userid) + + +def map_artist_colors(): + colors = LibDataTable.get_all_colors(type="artist") + + for color in colors: + artist = ArtistStore.artistmap.get(color["itemhash"]) + + if artist: + artist.set_color(color["color"]) + + +def map_album_colors(): + colors = LibDataTable.get_all_colors(type="album") + + for color in colors: + album = AlbumStore.albummap.get(color["itemhash"]) + + if album: + album.set_color(color["color"]) diff --git a/app/lib/populate.py b/app/lib/populate.py index 5506741e..0b12565c 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -8,7 +8,7 @@ from requests import ReadTimeout from app import settings from app.db.sqlite.tracks import SQLiteTrackMethods from app.lib.artistlib import CheckArtistImages -from app.lib.colorlib import ProcessArtistColors +from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors from app.lib.errors import PopulateCancelledError from app.lib.taglib import extract_thumb from app.logger import log @@ -40,6 +40,7 @@ class CordinateMedia: try: ProcessTrackThumbnails(instance_key) + ProcessAlbumColors(instance_key) ProcessArtistColors(instance_key) except PopulateCancelledError as e: log.warn(e) diff --git a/app/lib/tagger.py b/app/lib/tagger.py index b7613125..fab4f718 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -3,7 +3,6 @@ from app import settings from app.config import UserConfig from app.db.libdata import TrackTable -# from app.lib.populate import CordinateMedia from app.lib.taglib import extract_thumb, get_tags from app.models.album import Album from app.models.artist import Artist @@ -29,7 +28,6 @@ class IndexTracks: global POPULATE_KEY POPULATE_KEY = instance_key - # dirs_to_scan = sdb.get_root_dirs() dirs_to_scan = UserConfig().rootDirs if len(dirs_to_scan) == 0: @@ -159,12 +157,12 @@ def create_albums(): "playcount": track.playcount, "playduration": track.playduration, "title": track.album, - "trackcount": 1, + "tracks": {track.trackhash}, "extra": {}, } else: album = albums[track.albumhash] - album["trackcount"] += 1 + album["tracks"].add(track.trackhash) album["playcount"] += track.playcount album["playduration"] += track.playduration album["lastplayed"] = max(album["lastplayed"], track.lastplayed) @@ -186,8 +184,12 @@ def create_albums(): album["base_title"], _ = get_base_album_title(album["og_title"]) del genres + trackhashes = album.pop("tracks") + album["trackcount"] = len(trackhashes) - return [Album(**album) for album in albums.values()] + albums[album["albumhash"]] = (Album(**album), trackhashes) + + return list(albums.values()) # class IndexArtists: @@ -225,7 +227,7 @@ def create_artists(): "extra": {}, } else: - artist = artists[thisartist["artisthash"]] + artist: dict = artists[thisartist["artisthash"]] artist["duration"] += track.duration artist["playcount"] += track.playcount artist["playduration"] += track.playduration @@ -235,6 +237,8 @@ def create_artists(): artist["created_date"] = min(artist["created_date"], track.last_mod) artist["names"].add(thisartist["name"]) + artist.setdefault("albums", set()) + if thisartist.get("in_track", True): artist["tracks"].add(track.trackhash) @@ -257,12 +261,13 @@ def create_artists(): # INFO: Delete temporary keys del artist["names"] - del artist["tracks"] - del artist["albums"] + + tracks = artist.pop("tracks") + albums = artist.pop("albums") # INFO: Delete local variables del genres - return [Artist(**artist) for artist in artists.values()] - + artists[artist["artisthash"]] = (Artist(**artist), tracks, albums) + return list(artists.values()) diff --git a/app/models/artist.py b/app/models/artist.py index 540098a7..59ae76f4 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -55,6 +55,7 @@ class Artist: id: int = -1 image: str = "" + color: str = "" fav_userids: list[int] = dataclasses.field(default_factory=list) @property diff --git a/app/serializers/album.py b/app/serializers/album.py index 4bec0b4e..35405b4b 100644 --- a/app/serializers/album.py +++ b/app/serializers/album.py @@ -34,6 +34,7 @@ def serialize_for_card(album: Album): "type", "playduration", "genrehashes", + "fav_userids", "extra", "id", "lastplayed", diff --git a/app/store/albums.py b/app/store/albums.py index 14cc0086..7cbe8b8f 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -4,7 +4,6 @@ from pprint import pprint import random from typing import Iterable -from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb from app.lib.tagger import create_albums from app.models import Album, Track from app.store.artists import ArtistStore @@ -21,9 +20,9 @@ ALBUM_LOAD_KEY = "" class AlbumMapEntry: - def __init__(self, album: Album) -> None: + def __init__(self, album: Album, trackhashes: set[str]) -> None: self.album = album - self.trackhashes: set[str] = set() + self.trackhashes = trackhashes @property def basetitle(self): @@ -40,6 +39,9 @@ class AlbumMapEntry: self.album.toggle_favorite_user(userid) + def set_color(self, color: str): + self.album.color = color + class AlbumStore: albums: list[Album] = CustomList() @@ -67,26 +69,9 @@ class AlbumStore: print("Loading albums... ", end="") cls.albummap = { - album.albumhash: AlbumMapEntry(album=album) for album in create_albums() + album.albumhash: AlbumMapEntry(album=album, trackhashes=trackhashes) + for album, trackhashes in create_albums() } - tracks = remove_duplicates(TrackStore.get_flat_list()) - tracks = sorted(tracks, key=lambda t: t.albumhash) - grouped = groupby(tracks, lambda t: t.albumhash) - - for albumhash, tracks in grouped: - cls.albummap[albumhash].trackhashes = {t.trackhash for t in tracks} - - # db_albums: list[tuple] = aldb.get_all_albums() - - # for album in db_albums: - # albumhash = album[1] - # colors = json.loads(album[2]) - - # for _al in cls.albums: - # if _al.albumhash == albumhash: - # _al.set_colors(colors) - # break - print("Done!") @classmethod diff --git a/app/store/artists.py b/app/store/artists.py index b4d71ae2..f88dcb83 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -1,27 +1,22 @@ import json from typing import Iterable -from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb from app.lib.tagger import create_artists from app.models import Artist -from app.utils import flatten from app.utils.auth import get_current_userid -from app.utils.bisection import use_bisection from app.utils.customlist import CustomList -from app.utils.progressbar import tqdm -from .tracks import TrackStore - -# from .albums import AlbumStore from .tracks import TrackStore ARTIST_LOAD_KEY = "" class ArtistMapEntry: - def __init__(self, artist: Artist) -> None: + def __init__( + self, artist: Artist, albumhashes: set[str], trackhashes: set[str] + ) -> None: self.artist = artist - self.albumhashes: set[str] = set() - self.trackhashes: set[str] = set() + self.albumhashes: set[str] = albumhashes + self.trackhashes: set[str] = trackhashes def increment_playcount(self, duration: int, timestamp: int): self.artist.lastplayed = timestamp @@ -34,6 +29,9 @@ class ArtistMapEntry: self.artist.toggle_favorite_user(userid) + def set_color(self, color: str): + self.artist.color = color + class ArtistStore: artists: list[Artist] = CustomList() @@ -51,17 +49,19 @@ class ArtistStore: cls.artistmap.clear() cls.artistmap = { - artist.artisthash: ArtistMapEntry(artist=artist) - for artist in create_artists() + artist.artisthash: ArtistMapEntry( + artist=artist, albumhashes=albumhashes, trackhashes=trackhashes + ) + for artist, trackhashes, albumhashes in create_artists() } - for track in TrackStore.get_flat_list(): - if instance_key != ARTIST_LOAD_KEY: - return + # for track in TrackStore.get_flat_list(): + # if instance_key != ARTIST_LOAD_KEY: + # return - for hash in track.artisthashes: - cls.artistmap[hash].trackhashes.add(track.trackhash) - cls.artistmap[hash].albumhashes.add(track.albumhash) + # for hash in track.artisthashes: + # cls.artistmap[hash].trackhashes.add(track.trackhash) + # cls.artistmap[hash].albumhashes.add(track.albumhash) print("Done!") # for artist in ardb.get_all_artists(): diff --git a/manage.py b/manage.py index f2f11639..80faac61 100644 --- a/manage.py +++ b/manage.py @@ -21,9 +21,7 @@ import setproctitle from app.api import create_api from app.arg_handler import ProcessArgs -from app.lib.mapstuff import map_favorites, map_scrobble_data from app.lib.index import IndexEverything -from app.lib.watchdogg import Watcher as WatchDog from app.plugins.register import register_plugins from app.settings import FLASKVARS, TCOLOR, Info from app.setup import load_into_mem, run_setup From 04946831ceb367004133e8a2521ca201c60ebe24 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 2 Aug 2024 19:59:20 +0300 Subject: [PATCH 33/44] fix: stores not being populated on track indexing --- TODO.md | 3 ++- app/arg_handler.py | 10 ++++++++-- app/lib/index.py | 39 ++++++++++++++++++++++++++++++--------- app/setup/__init__.py | 27 ++++++++++++++++----------- manage.py | 4 ---- 5 files changed, 56 insertions(+), 27 deletions(-) diff --git a/TODO.md b/TODO.md index ee8a7879..1f0409bc 100644 --- a/TODO.md +++ b/TODO.md @@ -54,4 +54,5 @@ # Bug fixes -- Duplicates on search \ No newline at end of file +- Duplicates on search +- Audio stops on ending \ No newline at end of file diff --git a/app/arg_handler.py b/app/arg_handler.py index 249ef062..f6d6be1b 100644 --- a/app/arg_handler.py +++ b/app/arg_handler.py @@ -10,6 +10,7 @@ import PyInstaller.__main__ as bundler from app import settings from app.config import UserConfig +from app.db.userdata import UserTable from app.logger import log from app.print_help import HELP_MESSAGE from app.utils.auth import hash_password @@ -220,7 +221,8 @@ class ProcessArgs: sys.exit(0) username = username.strip() - user = authdb.get_user_by_username(username) + # user = authdb.get_user_by_username(username) + user = UserTable.get_by_username(username) if not user: print(f"User {username} not found") @@ -234,6 +236,10 @@ class ProcessArgs: sys.exit(0) password = hash_password(password) - user = authdb.update_user({"id": user.id, "password": password}) + # user = authdb.update_user({"id": user.id, "password": password}) + UserTable.update_one({ + "id": user.id, + "password": password + }) sys.exit(0) diff --git a/app/lib/index.py b/app/lib/index.py index 604158f0..4118663e 100644 --- a/app/lib/index.py +++ b/app/lib/index.py @@ -1,23 +1,44 @@ -from app.lib.mapstuff import map_album_colors, map_artist_colors, map_favorites, map_scrobble_data -from app.lib.populate import CordinateMedia -from app.lib.tagger import IndexTracks -from app.store.folder import FolderStore - - import gc from time import time - +from app.lib.mapstuff import ( + map_album_colors, + map_artist_colors, + map_favorites, + map_scrobble_data, +) +from app.lib.populate import CordinateMedia +from app.lib.tagger import IndexTracks +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.folder import FolderStore +from app.store.tracks import TrackStore from app.utils.threading import background +def load_and_map(): + key = str(time()) + FolderStore.load_filepaths() + AlbumStore.load_albums(key) + ArtistStore.load_artists(key) + + map_scrobble_data() + map_favorites() + map_artist_colors() + map_album_colors() + class IndexEverything: def __init__(self) -> None: IndexTracks(instance_key=time()) + + key = str(time()) + TrackStore.load_all_tracks(key) + AlbumStore.load_albums(key) + ArtistStore.load_artists(key) FolderStore.load_filepaths() + map_scrobble_data() map_favorites() - map_artist_colors() - map_album_colors() + CordinateMedia(instance_key=str(time())) gc.collect() diff --git a/app/setup/__init__.py b/app/setup/__init__.py index 7aa5e766..a28c44b0 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -2,7 +2,14 @@ Prepares the server for use. """ +from time import time import uuid +from app.lib.mapstuff import ( + map_album_colors, + map_artist_colors, + map_favorites, + map_scrobble_data, +) from app.setup.files import create_config_dir from app.setup.sqlite import run_migrations, setup_sqlite from app.store.albums import AlbumStore @@ -29,21 +36,19 @@ def run_setup(): setup_sqlite() run_migrations() - # try: - # load_settings() - # except IndexError: - # # settings table is empty - # pass - def load_into_mem(): """ Load all tracks, albums, and artists into memory. """ - # instance_key = get_random_str() - # INFO: Load all tracks, albums, and artists data into memory + key = str(time()) TrackStore.load_all_tracks(get_random_str()) - AlbumStore.load_albums('a') - ArtistStore.load_artists('a') - FolderStore.load_filepaths() \ No newline at end of file + AlbumStore.load_albums(key) + ArtistStore.load_artists(key) + FolderStore.load_filepaths() + + map_scrobble_data() + map_favorites() + map_artist_colors() + map_album_colors() diff --git a/manage.py b/manage.py index 80faac61..20d70e69 100644 --- a/manage.py +++ b/manage.py @@ -61,11 +61,7 @@ mimetypes.add_type("application/manifest+json", ".webmanifest") # Background tasks @background def bg_run_setup(): - pass - # run_periodic_scans() IndexEverything() - # map_scrobble_data() - # map_favorites() # @background From c77d0927c755b998e5f045213efacf3cf1cf6026 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 4 Aug 2024 10:19:11 +0300 Subject: [PATCH 34/44] remove deprecated db mappings + fix: cli password reset + delete old migrations --- TODO.md | 3 +- app/api/album.py | 2 - app/api/plugins/__init__.py | 15 +- app/api/settings.py | 15 +- app/arg_handler.py | 11 +- app/config.py | 3 +- app/db/sqlite/__init__.py | 18 -- app/db/sqlite/albumcolors.py | 66 ---- app/db/sqlite/artistcolors.py | 64 ---- app/db/sqlite/auth.py | 144 --------- app/db/sqlite/favorite.py | 121 ------- app/db/sqlite/lastfm/__init__.py | 0 app/db/sqlite/lastfm/similar_artists.py | 62 ---- app/db/sqlite/logger/tracks.py | 59 ---- app/db/sqlite/playlists.py | 228 ------------- app/db/sqlite/plugins/__init__.py | 83 ----- app/db/sqlite/queries.py | 123 ------- app/db/sqlite/settings.py | 151 --------- app/db/sqlite/tracks.py | 135 -------- app/db/userdata.py | 18 ++ app/lib/populate.py | 4 - app/lib/trackslib.py | 4 - app/lib/watchdogg.py | 4 - app/migrations/__init__.py | 5 +- app/migrations/v1_3_0/__init__.py | 306 ------------------ app/migrations/v1_4_9/__init__.py | 405 ------------------------ app/plugins/lyrics.py | 8 +- app/utils/auth.py | 1 - tests/__init__.py | 0 tests/sqlite/test_sqlite_actions.py | 59 ---- tests/test_utils.py | 34 -- 31 files changed, 40 insertions(+), 2111 deletions(-) delete mode 100644 app/db/sqlite/albumcolors.py delete mode 100644 app/db/sqlite/artistcolors.py delete mode 100644 app/db/sqlite/auth.py delete mode 100644 app/db/sqlite/favorite.py delete mode 100644 app/db/sqlite/lastfm/__init__.py delete mode 100644 app/db/sqlite/lastfm/similar_artists.py delete mode 100644 app/db/sqlite/logger/tracks.py delete mode 100644 app/db/sqlite/playlists.py delete mode 100644 app/db/sqlite/plugins/__init__.py delete mode 100644 app/db/sqlite/queries.py delete mode 100644 app/db/sqlite/settings.py delete mode 100644 app/db/sqlite/tracks.py delete mode 100644 app/migrations/v1_3_0/__init__.py delete mode 100644 app/migrations/v1_4_9/__init__.py delete mode 100644 tests/__init__.py delete mode 100644 tests/sqlite/test_sqlite_actions.py delete mode 100644 tests/test_utils.py diff --git a/TODO.md b/TODO.md index 1f0409bc..1dbe7a85 100644 --- a/TODO.md +++ b/TODO.md @@ -55,4 +55,5 @@ # Bug fixes - Duplicates on search -- Audio stops on ending \ No newline at end of file +- Audio stops on ending +- Port account settings to config on the frontend \ No newline at end of file diff --git a/app/api/album.py b/app/api/album.py index 1eb3660b..7a0cf17e 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -21,9 +21,7 @@ from app.utils.hashing import create_hash from app.lib.albumslib import sort_by_track_no from app.serializers.album import serialize_for_card_many from app.serializers.track import serialize_tracks -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -check_is_fav = favdb.check_is_favorite bp_tag = Tag(name="Album", description="Single album") api = APIBlueprint("album", __name__, url_prefix="/album", abp_tags=[bp_tag]) diff --git a/app/api/plugins/__init__.py b/app/api/plugins/__init__.py index 930bf603..9e86e18c 100644 --- a/app/api/plugins/__init__.py +++ b/app/api/plugins/__init__.py @@ -1,10 +1,8 @@ -from flask import Blueprint, request - from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from app.api.auth import admin_required -from app.db.sqlite.plugins import PluginsMethods +from app.db.userdata import PluginTable bp_tag = Tag(name="Plugins", description="Manage plugins") api = APIBlueprint("plugins", __name__, url_prefix="/plugins", abp_tags=[bp_tag]) @@ -15,8 +13,7 @@ def get_all_plugins(): """ List all plugins """ - plugins = PluginsMethods.get_all_plugins() - + plugins = PluginTable.get_all() return {"plugins": plugins} @@ -37,9 +34,7 @@ def activate_deactivate_plugin(body: PluginActivateBody): Activate/Deactivate plugin """ name = body.plugin - active = 1 if body.active else 0 - - PluginsMethods.plugin_set_active(name, active) + PluginTable.activate(name, body.active) return {"message": "OK"}, 200 @@ -62,7 +57,7 @@ def update_plugin_settings(body: PluginSettingsBody): if not plugin or not settings: return {"error": "Missing plugin or settings"}, 400 - PluginsMethods.update_plugin_settings(plugin_name=plugin, settings=settings) - plugin = PluginsMethods.get_plugin_by_name(plugin) + PluginTable.update_settings(plugin, settings) + plugin = PluginTable.get_by_name(plugin) return {"status": "success", "settings": plugin.settings} diff --git a/app/api/settings.py b/app/api/settings.py index 1e65f3d8..989ca298 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -1,18 +1,14 @@ from dataclasses import asdict from typing import Any -from flask import request from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from app.api.auth import admin_required -from app.db.sqlite.plugins import PluginsMethods as pdb -from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb from app.db.userdata import PluginTable from app.lib.index import index_everything -from app.lib.watchdogg import Watcher as WatchDog from app.logger import log -from app.settings import Info, Paths, SessionVarKeys +from app.settings import Info, SessionVarKeys from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore @@ -49,6 +45,7 @@ def reload_everything(instance_key: str): except Exception as e: log.error(e) + # CHECKPOINT: TEST SETTINGS API ENDPOINTS # @background @@ -217,10 +214,6 @@ def set_setting(body: SetSettingBody): if key not in mapp: return {"msg": "Invalid key!"}, 400 - sdb.set_setting(key, value) - - flag = mapp[key] - if key == "artist_separators": value = str(value).split(",") value = set(value) @@ -269,7 +262,11 @@ def update_config(body: UpdateConfigBody): Update the config file """ config = UserConfig() + if body.key == "artistSeparators": + body.value = body.value.split(",") + setattr(config, body.key, body.value) + print(getattr(config, body.key)) return { "msg": "Config updated!", diff --git a/app/arg_handler.py b/app/arg_handler.py index f6d6be1b..3424a3b0 100644 --- a/app/arg_handler.py +++ b/app/arg_handler.py @@ -13,11 +13,11 @@ from app.config import UserConfig from app.db.userdata import UserTable from app.logger import log from app.print_help import HELP_MESSAGE +from app.setup.sqlite import setup_sqlite from app.utils.auth import hash_password from app.utils.paths import getFlaskOpenApiPath from app.utils.xdg_utils import get_xdg_config_dir from app.utils.wintools import is_windows -from app.db.sqlite.auth import SQLiteAuthMethods as authdb ALLARGS = settings.ALLARGS ARGS = sys.argv[1:] @@ -209,6 +209,7 @@ class ProcessArgs: if ALLARGS.pswd in ARGS: print("SWING MUSIC v2.0.0 ") print("PASSWORD RECOVERY \n") + setup_sqlite() username: str = "" password: str = "" @@ -221,7 +222,6 @@ class ProcessArgs: sys.exit(0) username = username.strip() - # user = authdb.get_user_by_username(username) user = UserTable.get_by_username(username) if not user: @@ -235,11 +235,6 @@ class ProcessArgs: print("\nOperation cancelled! Exiting ...") sys.exit(0) - password = hash_password(password) - # user = authdb.update_user({"id": user.id, "password": password}) - UserTable.update_one({ - "id": user.id, - "password": password - }) + UserTable.update_one({"id": user.id, "password": hash_password(password)}) sys.exit(0) diff --git a/app/config.py b/app/config.py index bca9c69e..d49be130 100644 --- a/app/config.py +++ b/app/config.py @@ -36,7 +36,8 @@ class UserConfig: # misc enablePeriodicScans: bool = False - scanInterval: int = 60 * 10 # 10 minutes + scanInterval: int = 10 + enableWatchdog: bool = False # plugins enablePlugins: bool = True diff --git a/app/db/sqlite/__init__.py b/app/db/sqlite/__init__.py index 43bd79cd..f5ecf73b 100644 --- a/app/db/sqlite/__init__.py +++ b/app/db/sqlite/__init__.py @@ -1,21 +1,3 @@ """ This module contains the functions to interact with the SQLite database. """ - -import sqlite3 -from sqlite3 import Connection as SqlConn - - -def create_connection(db_file: str) -> SqlConn: - """ - Creates a connection to the specified database. - """ - conn = sqlite3.connect(db_file) - return conn - - -def create_tables(conn: SqlConn, sql_query: str): - """ - Executes the specifiend SQL file to create database tables. - """ - conn.executescript(sql_query) diff --git a/app/db/sqlite/albumcolors.py b/app/db/sqlite/albumcolors.py deleted file mode 100644 index 2e804924..00000000 --- a/app/db/sqlite/albumcolors.py +++ /dev/null @@ -1,66 +0,0 @@ -from sqlite3 import Cursor - -from .utils import SQLiteManager, tuples_to_albums - - -class SQLiteAlbumMethods: - @classmethod - def insert_one_album(cls, cur: Cursor, albumhash: str, colors: str): - """ - Inserts one album into the database - """ - - sql = """INSERT OR REPLACE INTO albums( - albumhash, - colors - ) VALUES(?,?) - """ - - cur.execute(sql, (albumhash, colors)) - lastrowid = cur.lastrowid - - return lastrowid - - @classmethod - def get_all_albums(cls): - with SQLiteManager() as cur: - cur.execute("SELECT * FROM albums") - albums = cur.fetchall() - cur.close() - - if albums is not None: - return albums - - return [] - - @staticmethod - def get_albums_by_albumartist(albumartist: str): - with SQLiteManager() as cur: - cur.execute("SELECT * FROM albums WHERE albumartist=?", (albumartist,)) - albums = cur.fetchall() - cur.close() - - if albums is not None: - return tuples_to_albums(albums) - - return [] - - @staticmethod - def exists(albumhash: str, cur: Cursor = None): - """ - Checks if an album exists in the database. - """ - - sql = "SELECT COUNT(1) FROM albums WHERE albumhash = ?" - - def _exists(cur: Cursor): - cur.execute(sql, (albumhash,)) - count = cur.fetchone()[0] - - return count != 0 - - if cur: - return _exists(cur) - - with SQLiteManager() as cur: - return _exists(cur) diff --git a/app/db/sqlite/artistcolors.py b/app/db/sqlite/artistcolors.py deleted file mode 100644 index 8268ea3f..00000000 --- a/app/db/sqlite/artistcolors.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Contains methods for reading and writing to the sqlite artists database. -""" - -import json -from sqlite3 import Cursor - -from .utils import SQLiteManager - - -class SQLiteArtistMethods: - @staticmethod - def insert_one_artist(cur: Cursor, artisthash: str, colors: list[str]): - """ - Inserts a single artist into the database. - """ - sql = """INSERT OR REPLACE INTO artists( - artisthash, - colors - ) VALUES(?,?) - """ - colors = json.dumps(colors) - cur.execute(sql, (artisthash, colors)) - - @staticmethod - def get_all_artists(cur_: Cursor = None): - """ - Get all artists from the database and return a generator of Artist objects - """ - sql = """SELECT * FROM artists""" - - if not cur_: - with SQLiteManager() as cur: - cur.execute(sql) - - for artist in cur.fetchall(): - yield artist - - cur.close() - - else: - cur_.execute(sql) - - for artist in cur_.fetchall(): - yield artist - - @staticmethod - def exists(artisthash: str, cur: Cursor = None): - """ - Checks if an artist exists in the database. - """ - sql = "SELECT COUNT(1) FROM artists WHERE artisthash = ?" - - def _exists(cur: Cursor): - cur.execute(sql, (artisthash,)) - count = cur.fetchone()[0] - - return count != 0 - - if cur: - return _exists(cur) - - with SQLiteManager() as cur: - return _exists(cur) diff --git a/app/db/sqlite/auth.py b/app/db/sqlite/auth.py deleted file mode 100644 index dede16b0..00000000 --- a/app/db/sqlite/auth.py +++ /dev/null @@ -1,144 +0,0 @@ -import json -from app.models.user import User -from app.utils.auth import hash_password -from app.db.sqlite.utils import SQLiteManager - - -class SQLiteAuthMethods: - """ - Methods for authenticating users. - """ - - @staticmethod - def insert_user(user: dict[str, str]): - """ - Insert a user into the database. - - :param user: A dict with the username, password and roles. - """ - sql = """INSERT INTO users( - username, - password, - roles - ) VALUES(:username, :password, :roles) - """ - - user_tuple = tuple(user.values()) - - with SQLiteManager(userdata_db=True) as cur: - cur = cur.execute(sql, user_tuple) - userid = cur.lastrowid - return userid - # if userid: - # # sleep - # user = SQLiteAuthMethods.get_user_by_id(userid).todict_simplified() - # cur.close() - # return user - - raise Exception(f"Failed to insert user: {user}") - - @staticmethod - def insert_default_user(): - """ - Inserts the default admin user. - """ - user = { - "username": "admin", - "password": hash_password("admin"), - "roles": json.dumps(["admin"]), - } - return SQLiteAuthMethods.insert_user(user) - - @staticmethod - def insert_guest_user(): - """ - Inserts the default guest user. - """ - user = { - "username": "guest", - "password": hash_password("guest"), - "roles": json.dumps(["guest"]), - } - - return SQLiteAuthMethods.insert_user(user) - - @staticmethod - def update_user(user: dict[str, str]): - """ - Update a user in the database. - - :param user: A dict with the user id and the fields to update. Ommited fields will not be updated. - """ - # get all user dict keys - keys = list(user.keys()) - sql = f"""UPDATE users SET - {', '.join([f"{key} = :{key}" for key in keys if key != 'id'])} - WHERE id = :id - """ - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, user) - cur.close() - - return SQLiteAuthMethods.get_user_by_id(user["id"]).todict() - - @staticmethod - def get_all_users(): - """ - Check if there are any users in the database. - """ - sql = "SELECT * FROM users" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - - data = cur.fetchall() - cur.close() - - return [User(*user) for user in data] - - @staticmethod - def get_user_by_username(username: str): - """ - Get a user by username. - """ - sql = "SELECT * FROM users WHERE username = ?" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (username,)) - - data = cur.fetchone() - cur.close() - - if data is not None: - return User(*data) - - return None - - @staticmethod - def get_user_by_id(userid: int): - """ - Get a user by id. - """ - sql = "SELECT * FROM users WHERE id = ?" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (userid,)) - - data = cur.fetchone() - cur.close() - - if data is not None: - return User(*data) - - return None - - @staticmethod - def delete_user_by_username(username: str): - """ - Delete a user by username. - """ - sql = "DELETE FROM users WHERE id = ?" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (3,)) - cur.close() diff --git a/app/db/sqlite/favorite.py b/app/db/sqlite/favorite.py deleted file mode 100644 index dfff2872..00000000 --- a/app/db/sqlite/favorite.py +++ /dev/null @@ -1,121 +0,0 @@ -from datetime import datetime - -from flask_jwt_extended import current_user -from app.models import FavType -from .utils import SQLiteManager - - -class SQLiteFavoriteMethods: - """THis class contains methods for interacting with the favorites table.""" - - @classmethod - def check_is_favorite(cls, itemhash: str, fav_type: str): - """ - Checks if an item is favorited. - """ - userid = current_user["id"] - - sql = """SELECT * FROM favorites WHERE hash = ? AND type = ? AND userid = ?""" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (itemhash, fav_type, userid)) - item = cur.fetchone() - cur.close() - return item is not None - - @classmethod - def insert_one_favorite(cls, fav_type: str, fav_hash: str): - """ - Inserts a single favorite into the database. - """ - # try to find the favorite in the database, if it exists, don't insert it - if cls.check_is_favorite(fav_hash, fav_type): - return - - sql = """INSERT INTO favorites(type, hash, timestamp, userid) VALUES(?,?,?,?)""" - current_timestamp = int(datetime.now().timestamp()) - with SQLiteManager(userdata_db=True) as cur: - userid = current_user["id"] - cur.execute(sql, (fav_type, fav_hash, current_timestamp, userid)) - cur.close() - - @classmethod - def get_all(cls) -> list[tuple]: - """ - Returns a list of all favorites. - """ - sql = """SELECT * FROM favorites WHERE userid = ?""" - with SQLiteManager(userdata_db=True) as cur: - userid = current_user["id"] - cur.execute(sql, (userid,)) - favs = cur.fetchall() - cur.close() - return [fav for fav in favs if fav[1] != ""] - - @classmethod - def get_favorites(cls, fav_type: str, userid: int = None) -> list[tuple]: - """ - Returns a list of favorite tracks. - - If userid is None, all favorites are returned. - """ - sql = """SELECT * FROM favorites WHERE type = ?""" - params = (fav_type,) - - if not userid: - sql += " AND userid = ?" - params = (fav_type, userid) - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, params) - all_favs = cur.fetchall() - cur.close() - return all_favs - - @classmethod - def get_fav_tracks(cls, userid: int = None) -> list[tuple]: - """ - Returns a list of favorite tracks. - """ - return cls.get_favorites(FavType.track, userid) - - @classmethod - def get_fav_albums(cls) -> list[tuple]: - """ - Returns a list of favorite albums. - """ - userid = current_user["id"] - return cls.get_favorites(FavType.album, userid) - - @classmethod - def get_fav_artists(cls) -> list[tuple]: - """ - Returns a list of favorite artists. - """ - userid = current_user["id"] - return cls.get_favorites(FavType.artist, userid) - - @classmethod - def delete_favorite(cls, fav_type: str, fav_hash: str): - """ - Deletes a favorite from the database. - """ - sql = """DELETE FROM favorites WHERE hash = ? AND type = ? AND userid = ?""" - - with SQLiteManager(userdata_db=True) as cur: - userid = current_user["id"] - cur.execute(sql, (fav_hash, fav_type, userid)) - cur.close() - - @classmethod - def get_track_count(cls) -> int: - """ - Returns the number of favorite tracks. - """ - sql = """SELECT COUNT(*) FROM favorites WHERE type = ? AND userid = ?""" - - with SQLiteManager(userdata_db=True) as cur: - userid = current_user["id"] - cur.execute(sql, (FavType.track, userid)) - count = cur.fetchone()[0] - cur.close() - return count diff --git a/app/db/sqlite/lastfm/__init__.py b/app/db/sqlite/lastfm/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/db/sqlite/lastfm/similar_artists.py b/app/db/sqlite/lastfm/similar_artists.py deleted file mode 100644 index 53a45b70..00000000 --- a/app/db/sqlite/lastfm/similar_artists.py +++ /dev/null @@ -1,62 +0,0 @@ -from app.models.lastfm import SimilarArtist - -from ..utils import SQLiteManager - - -class SQLiteLastFMSimilarArtists: - """ - This class contains methods for interacting with the lastfm_similar_artists table. - """ - - @classmethod - def insert_one(cls, artist: SimilarArtist): - """ - Inserts a single artist into the database. - """ - sql = """INSERT OR REPLACE INTO lastfm_similar_artists(artisthash, similar_artists) VALUES(?,?)""" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (artist.artisthash, artist.similar_artists)) - cur.close() - - @classmethod - def get_similar_artists_for(cls, artisthash: str): - """ - Returns a list of similar artists. - """ - sql = """SELECT * FROM lastfm_similar_artists WHERE artisthash = ?""" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (artisthash,)) - similar_artists = cur.fetchone() - cur.close() - - if similar_artists is None: - return None - - return SimilarArtist(artisthash, similar_artists[2]) - - @classmethod - def get_all(cls): - """ - Returns a list of all similar artists. - """ - sql = """SELECT * FROM lastfm_similar_artists""" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - similar_artists = cur.fetchall() - cur.close() - - for a in similar_artists: - yield SimilarArtist(a[1], a[2]) - - @classmethod - def exists(cls, artisthash: str): - """ - Checks if an artist exists in the database by counting the number of rows - """ - sql = """SELECT COUNT(*) FROM lastfm_similar_artists WHERE artisthash = ?""" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (artisthash,)) - count = cur.fetchone()[0] - cur.close() - return count > 0 diff --git a/app/db/sqlite/logger/tracks.py b/app/db/sqlite/logger/tracks.py deleted file mode 100644 index c95c123b..00000000 --- a/app/db/sqlite/logger/tracks.py +++ /dev/null @@ -1,59 +0,0 @@ -from flask_jwt_extended import current_user -from app.db.sqlite.utils import SQLiteManager -from app.models.logger import TrackLog as TrackLog -from app.utils.auth import get_current_userid - - -class SQLiteTrackLogger: - @classmethod - def insert_track(cls, trackhash: str, duration: int, source: str, timestamp: int): - """ - Inserts a track play record into the database - """ - - userid = get_current_userid() - with SQLiteManager(userdata_db=True) as cur: - sql = """INSERT OR REPLACE INTO track_logger( - trackhash, - duration, - timestamp, - source, - userid - ) VALUES(?,?,?,?,?) - """ - - cur.execute( - sql, (trackhash, duration, timestamp, source, userid) - ) - lastrowid = cur.lastrowid - - return lastrowid - - @classmethod - def get_all(cls): - """ - Returns all track play records from the database - """ - - with SQLiteManager(userdata_db=True) as cur: - userid = get_current_userid() - sql = f"""SELECT * FROM track_logger WHERE userid = {userid} ORDER BY timestamp DESC""" - - cur.execute(sql) - rows = cur.fetchall() - - return rows - - @classmethod - def get_recently_played(cls, start: int = 0, limit: int = 100): - """ - Returns a list of recently played tracks - """ - - with SQLiteManager(userdata_db=True) as cur: - sql = f"""SELECT * FROM track_logger WHERE userid = {current_user['id']} ORDER BY timestamp DESC LIMIT ?,?""" - - cur.execute(sql, (start, limit)) - rows = cur.fetchall() - - return [TrackLog(*row) for row in rows] diff --git a/app/db/sqlite/playlists.py b/app/db/sqlite/playlists.py deleted file mode 100644 index b4cca33e..00000000 --- a/app/db/sqlite/playlists.py +++ /dev/null @@ -1,228 +0,0 @@ -import json -from collections import OrderedDict - -from flask_jwt_extended import current_user - -from app.db.sqlite.utils import SQLiteManager, tuple_to_playlist, tuples_to_playlists -from app.utils.dates import create_new_date - - -class SQLitePlaylistMethods: - """ - This class contains methods for interacting with the playlists table. - """ - - @staticmethod - def update_last_updated(playlist_id: int): - """Updates the last updated date of a playlist.""" - sql = """UPDATE playlists SET last_updated = ? WHERE id = ?""" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (create_new_date(), playlist_id)) - - @staticmethod - def insert_one_playlist(playlist: dict): - # banner_pos, - # has_gif, - sql = """INSERT INTO playlists( - image, - last_updated, - name, - settings, - trackhashes, - userid - ) VALUES(:image, :last_updated, :name, :settings, :trackhashes, :userid) - """ - - playlist["userid"] = current_user["id"] - playlist = OrderedDict(sorted(playlist.items())) - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, playlist) - pid = cur.lastrowid - cur.close() - - p_tuple = (pid, *playlist.values()) - return tuple_to_playlist(p_tuple) - - @staticmethod - def count_playlist_by_name(name: str): - sql = f"SELECT COUNT(*) FROM playlists WHERE name = ? and userid = {current_user['id']}" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (name,)) - - data = cur.fetchone() - cur.close() - - return int(data[0]) - - @staticmethod - def get_all_playlists(): - with SQLiteManager(userdata_db=True) as cur: - userid = 1 - - try: - userid = current_user["id"] - except RuntimeError: - # Catch this error raised during migration execution - pass - - cur.execute(f"SELECT * FROM playlists WHERE userid = {userid}") - playlists = cur.fetchall() - cur.close() - - if playlists is not None: - return tuples_to_playlists(playlists) - - return [] - - @staticmethod - def get_playlist_by_id(playlist_id: int): - sql = f"SELECT * FROM playlists WHERE id = ? and userid = {current_user['id']}" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (playlist_id,)) - - data = cur.fetchone() - cur.close() - - if data is not None: - return tuple_to_playlist(data) - - return None - - # FIXME: Extract the "add_track_to_playlist" method to use it for both the artisthash and trackhash lists. - - @classmethod - def add_item_to_json_list(cls, playlist_id: int, field: str, items: set[str]): - """ - Adds a string item to a json dumped list using a playlist id and field name. - Takes the playlist ID, a field name, an item to add to the field. - """ - userid = 1 - - try: - userid = current_user["id"] - except RuntimeError: - # Catch this error raised during migration execution - pass - - sql = f"SELECT {field} FROM playlists WHERE id = ? and userid = {userid}" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (playlist_id,)) - data = cur.fetchone() - - if data is not None: - db_items: list[str] = json.loads(data[0]) - - # Remove duplicates, without changing the order. - for item in items: - if item in db_items: - items.remove(item) - - db_items.extend(items) - - sql = f"UPDATE playlists SET {field} = ? WHERE id = ?" - cur.execute(sql, (json.dumps(db_items), playlist_id)) - return len(items) - - cls.update_last_updated(playlist_id) - - @classmethod - def add_tracks_to_playlist(cls, playlist_id: int, trackhashes: list[str]): - """ - Adds trackhashes to a playlist - """ - return cls.add_item_to_json_list(playlist_id, "trackhashes", trackhashes) - - @classmethod - def update_playlist(cls, playlist_id: int, playlist: dict): - sql = f"""UPDATE playlists SET - image = ?, - last_updated = ?, - name = ?, - settings = ? - WHERE id = ? and userid = {current_user['id']} - """ - - del playlist["id"] - del playlist["trackhashes"] - playlist["settings"] = json.dumps(playlist["settings"]) - - playlist = OrderedDict(sorted(playlist.items())) - params = (*playlist.values(), playlist_id) - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, params) - - cls.update_last_updated(playlist_id) - - @classmethod - def update_settings(cls, playlist_id: int, settings: dict): - sql = f"""UPDATE playlists SET settings = ? WHERE id = ? and userid = {current_user['id']}""" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (json.dumps(settings), playlist_id)) - - cls.update_last_updated(playlist_id) - - @staticmethod - def delete_playlist(pid: str): - sql = f"DELETE FROM playlists WHERE id = ? and userid = {current_user['id']}" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (pid,)) - - @staticmethod - def remove_banner(playlistid: int): - sql = f"""UPDATE playlists SET image = NULL WHERE id = ? and userid = {current_user['id']}""" - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (playlistid,)) - - @classmethod - def remove_tracks_from_playlist(cls, playlistid: int, tracks: list[dict[str, int]]): - """ - Removes tracks from a playlist by trackhash and position. - """ - - sql = """UPDATE playlists SET trackhashes = ? WHERE id = ?""" - userid = 1 - - try: - userid = current_user["id"] - except RuntimeError: - # Catch this error raised during migration execution - pass - - with SQLiteManager(userdata_db=True) as cur: - cur.execute( - f"SELECT trackhashes FROM playlists WHERE id = ? and userid = {userid}", - (playlistid,), - ) - data = cur.fetchone() - - if data is None: - return - - trackhashes: list[str] = json.loads(data[0]) - to_remove = [] - - for track in tracks: - # { - # trackhash: str; - # index: int; - # } - index = trackhashes.index(track["trackhash"]) - - if index == track["index"]: - to_remove.append(track["trackhash"]) - - for trackhash in to_remove: - trackhashes.remove(trackhash) - - cur.execute(sql, (json.dumps(trackhashes), playlistid)) - - cls.update_last_updated(playlistid) diff --git a/app/db/sqlite/plugins/__init__.py b/app/db/sqlite/plugins/__init__.py deleted file mode 100644 index 0afc5d43..00000000 --- a/app/db/sqlite/plugins/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -import json - -from app.models.plugins import Plugin - -from ..utils import SQLiteManager - - -def plugin_tuple_to_obj(plugin_tuple: tuple) -> Plugin: - return Plugin( - name=plugin_tuple[1], - active=bool(plugin_tuple[3]), - settings=json.loads(plugin_tuple[4]), - ) - - -class PluginsMethods: - @classmethod - def insert_plugin(cls, plugin: Plugin): - """ - Inserts one plugin into the database - """ - - sql = """INSERT OR IGNORE INTO plugins( - name, - description, - active, - settings - ) VALUES(?,?,?,?) - """ - - with SQLiteManager(userdata_db=True) as cur: - cur.execute( - sql, - ( - plugin.name, - plugin.description, - int(plugin.active), - json.dumps(plugin.settings), - ), - ) - lastrowid = cur.lastrowid - - return lastrowid - - - @classmethod - def get_all_plugins(cls): - with SQLiteManager(userdata_db=True) as cur: - cur.execute("SELECT * FROM plugins") - plugins = cur.fetchall() - cur.close() - - if plugins is not None: - return [plugin_tuple_to_obj(plugin) for plugin in plugins] - - return [] - - @classmethod - def plugin_set_active(cls, name: str, active: int): - with SQLiteManager(userdata_db=True) as cur: - cur.execute("UPDATE plugins SET active=? WHERE name=?", (active, name)) - cur.close() - - @classmethod - def update_plugin_settings(cls, plugin_name: str, settings: dict): - with SQLiteManager(userdata_db=True) as cur: - cur.execute( - "UPDATE plugins SET settings=? WHERE name=?", - (json.dumps(settings), plugin_name), - ) - cur.close() - - @classmethod - def get_plugin_by_name(cls, name: str): - with SQLiteManager(userdata_db=True) as cur: - cur.execute("SELECT * FROM plugins WHERE name=?", (name,)) - plugin = cur.fetchone() - cur.close() - - if plugin is not None: - return plugin_tuple_to_obj(plugin) - - return None diff --git a/app/db/sqlite/queries.py b/app/db/sqlite/queries.py deleted file mode 100644 index 547b4ffe..00000000 --- a/app/db/sqlite/queries.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -This file contains the SQL queries to create the database tables. -""" - -CREATE_USERDATA_TABLES = """ -CREATE TABLE IF NOT EXISTS playlists ( - id integer PRIMARY KEY, - image text, - last_updated text not null, - name text not null, - settings text, - trackhashes text, - userid integer not null, - constraint fk_users foreign key (userid) references users(id) on delete cascade -); - -CREATE TABLE IF NOT EXISTS settings ( - id integer PRIMARY KEY, - root_dirs text NOT NULL, - exclude_dirs text, - artist_separators text NOT NULL default '/,;', - extract_feat integer NOT NULL DEFAULT 1, - remove_prod integer NOT NULL DEFAULT 1, - clean_album_title integer NOT NULL DEFAULT 1, - remove_remaster integer NOT NULL DEFAULT 1, - merge_albums integer NOT NULL DEFAULT 0, - show_albums_as_singles integer NOT NULL DEFAULT 0 -); - -CREATE TABLE IF NOT EXISTS lastfm_similar_artists ( - id integer PRIMARY KEY, - artisthash text NOT NULL, - similar_artists text NOT NULL, - UNIQUE (artisthash) -); - -CREATE TABLE IF NOT EXISTS plugins ( - id integer PRIMARY KEY, - name text NOT NULL UNIQUE, - description text NOT NULL, - active integer NOT NULL DEFAULT 0, - settings text -); - -CREATE TABLE IF NOT EXISTS track_logger ( - id integer PRIMARY KEY, - trackhash text NOT NULL, - duration integer NOT NULL, - timestamp integer NOT NULL, - source text, - userid integer NOT NULL DEFAULT 1, - constraint fk_users foreign key (userid) references users(id) on delete cascade -); - -CREATE TABLE IF NOT EXISTS users ( - id integer PRIMARY KEY, - username text NOT NULL UNIQUE, - firstname text, - lastname text, - password text NOT NULL, - email text, - image text, - roles text NOT NULL DEFAULT '["user"]' -) -""" - -CREATE_APPDB_TABLES = """ -CREATE TABLE IF NOT EXISTS tracks ( - id integer PRIMARY KEY, - album text NOT NULL, - albumartist text NOT NULL, - albumhash text NOT NULL, - artist text NOT NULL, - bitrate integer NOT NULL, - copyright text, - date integer NOT NULL, - disc integer NOT NULL, - duration integer NOT NULL, - filepath text NOT NULL, - folder text NOT NULL, - genre text, - title text NOT NULL, - track integer NOT NULL, - trackhash text NOT NULL, - last_mod float NOT NULL, - UNIQUE (filepath) -); - -CREATE TABLE IF NOT EXISTS albums ( - id integer PRIMARY KEY, - albumhash text NOT NULL, - colors text NOT NULL, - UNIQUE (albumhash) -); - -CREATE TABLE IF NOT EXISTS artists ( - id integer PRIMARY KEY, - artisthash text NOT NULL, - colors text, - bio text, - UNIQUE (artisthash) -); - -CREATE TABLE IF NOT EXISTS folders ( - id integer PRIMARY KEY, - path text NOT NULL, - trackcount integer NOT NULL -); -""" - -# changed from migrations to dbmigrations in v1.3.0 -# to avoid conflicts with the previous migrations. - -CREATE_MIGRATIONS_TABLE = """ -CREATE TABLE IF NOT EXISTS dbmigrations ( - id integer PRIMARY KEY, - version integer NOT NULL DEFAULT 0 -); - -INSERT INTO dbmigrations (version) -SELECT -1 -WHERE NOT EXISTS (SELECT 1 FROM dbmigrations); -""" diff --git a/app/db/sqlite/settings.py b/app/db/sqlite/settings.py deleted file mode 100644 index 0121fa9f..00000000 --- a/app/db/sqlite/settings.py +++ /dev/null @@ -1,151 +0,0 @@ -from pprint import pprint -from typing import Any - -from app.config import UserConfig -from app.db.sqlite.utils import SQLiteManager -from app.utils.wintools import win_replace_slash - - -# class SettingsSQLMethods: -# """ -# Methods for interacting with the settings table. -# """ - -# @staticmethod -# def get_all_settings(): -# """ -# Gets all settings from the database. -# """ - -# sql = "SELECT * FROM settings WHERE id = 1" - -# with SQLiteManager(userdata_db=True) as cur: -# cur.execute(sql) -# settings = cur.fetchone() -# cur.close() - -# # if root_dirs not set -# if settings is None: -# return [] - -# # omit id, root_dirs, and exclude_dirs -# return settings[3:] - -# @staticmethod -# def get_root_dirs() -> list[str]: -# """ -# Gets custom root directories from the database. -# """ - -# sql = "SELECT root_dirs FROM settings" - -# with SQLiteManager(userdata_db=True) as cur: -# cur.execute(sql) -# dirs = cur.fetchall() -# cur.close() - -# dirs = [_dir[0] for _dir in dirs] -# return [win_replace_slash(d) for d in dirs] - -# @staticmethod -# def add_root_dirs(dirs: list[str]): -# """ -# Add custom root directories to the database. -# """ - -# sql = "INSERT INTO settings (root_dirs) VALUES (?)" -# existing_dirs = SettingsSQLMethods.get_root_dirs() - -# dirs = [_dir for _dir in dirs if _dir not in existing_dirs] - -# if len(dirs) == 0: -# return - -# with SQLiteManager(userdata_db=True) as cur: -# for _dir in dirs: -# cur.execute(sql, (_dir,)) - -# @staticmethod -# def remove_root_dirs(dirs: list[str]): -# """ -# Remove custom root directories from the database. -# """ - -# sql = "DELETE FROM settings WHERE root_dirs = ?" - -# with SQLiteManager(userdata_db=True) as cur: -# for _dir in dirs: -# cur.execute(sql, (_dir,)) - -# # Not currently used anywhere, to be used later -# @staticmethod -# def add_excluded_dirs(dirs: list[str]): -# """ -# Add custom exclude directories to the database. -# """ - -# sql = "INSERT INTO settings (exclude_dirs) VALUES (?)" - -# with SQLiteManager(userdata_db=True) as cur: -# cur.executemany(sql, dirs) - -# @staticmethod -# def remove_excluded_dirs(dirs: list[str]): -# """ -# Remove custom exclude directories from the database. -# """ - -# sql = "DELETE FROM settings WHERE exclude_dirs = ?" - -# with SQLiteManager(userdata_db=True) as cur: -# cur.executemany(sql, dirs) - -# @staticmethod -# def get_excluded_dirs() -> list[str]: -# """ -# Gets custom exclude directories from the database. -# """ - -# sql = "SELECT exclude_dirs FROM settings" - -# with SQLiteManager(userdata_db=True) as cur: -# cur.execute(sql) -# dirs = cur.fetchall() -# return [_dir[0] for _dir in dirs] - -# @staticmethod -# def get_settings() -> dict[str, Any]: -# pass - -# @staticmethod -# def set_setting(key: str, value: Any): -# sql = f"UPDATE settings SET {key} = :value WHERE id = 1" - -# if type(value) == bool: -# value = str(int(value)) - -# with SQLiteManager(userdata_db=True) as cur: -# cur.execute(sql, {"value": value}) - - -# def load_settings(): -# # s = SettingsSQLMethods.get_all_settings() -# config = UserConfig() - - # try: - # db_separators: str = s[0] - # db_separators = db_separators.replace(" ", "") - # separators = db_separators.split(",") - # separators = set(separators) - # except IndexError: - # separators = {";", "/"} - - # SessionVars.ARTIST_SEPARATORS = config.artistSeparators - - # # boolean settings - # SessionVars.EXTRACT_FEAT = bool(s[1]) - # SessionVars.REMOVE_PROD = bool(s[2]) - # SessionVars.CLEAN_ALBUM_TITLE = bool(s[3]) - # SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4]) - # SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5]) - # SessionVars.SHOW_ALBUMS_AS_SINGLES = bool(s[6]) diff --git a/app/db/sqlite/tracks.py b/app/db/sqlite/tracks.py deleted file mode 100644 index 0324641b..00000000 --- a/app/db/sqlite/tracks.py +++ /dev/null @@ -1,135 +0,0 @@ -""" -Contains the SQLiteTrackMethods class which contains methods for -interacting with the tracks table. -""" - -from collections import OrderedDict -from sqlite3 import Cursor - -from app.db.sqlite.utils import tuple_to_track, tuples_to_tracks - -from .utils import SQLiteManager -from app.utils.unicode import handle_unicode - - -class SQLiteTrackMethods: - """ - This class contains all methods for interacting with the tracks table. - """ - - @classmethod - def insert_one_track(cls, track: dict, cur: Cursor): - """ - Inserts a single track into the database. - """ - sql = """INSERT OR REPLACE INTO tracks( - album, - albumartist, - albumhash, - artist, - bitrate, - copyright, - date, - disc, - duration, - filepath, - folder, - genre, - last_mod, - title, - track, - trackhash - ) VALUES(:album, :albumartist, :albumhash, :artist, :bitrate, :copyright, - :date, :disc, :duration, :filepath, :folder, :genre, :last_mod, :title, :track, :trackhash) - """ - - track = OrderedDict(sorted(track.items())) - - track["artist"] = track["artists"] - track["albumartist"] = track["albumartists"] - - del track["artists"] - del track["albumartists"] - - try: - cur.execute(sql, track) - except UnicodeEncodeError: - # for each of the values in the track, call handle_unicode on it - for key, value in track.items(): - track[key] = handle_unicode(value) - - cur.execute(sql, track) - - @classmethod - def insert_many_tracks(cls, tracks: list[dict]): - """ - Inserts a list of tracks into the database. - """ - - with SQLiteManager() as cur: - for track in tracks: - cls.insert_one_track(track, cur) - - @staticmethod - def get_all_tracks(): - """ - Get all tracks from the database and return a generator of Track objects - or an empty list. - """ - with SQLiteManager() as cur: - cur.execute("SELECT * FROM tracks") - rows = cur.fetchall() - - if rows is not None: - return tuples_to_tracks(rows) - - return [] - - @staticmethod - def get_track_by_trackhash(trackhash: str): - """ - Gets a track using its trackhash. Returns a Track object or None. - """ - with SQLiteManager() as cur: - cur.execute("SELECT * FROM tracks WHERE trackhash=?", (trackhash,)) - row = cur.fetchone() - - if row is not None: - return tuple_to_track(row) - - return None - - @staticmethod - def get_track_by_albumhash(albumhash: str): - """ - Gets a track using its albumhash. Returns a Track object or None. - """ - with SQLiteManager() as cur: - cur.execute("SELECT * FROM tracks WHERE albumhash=?", (albumhash,)) - row = cur.fetchone() - - if row is not None: - return tuple_to_track(row) - - return None - - @staticmethod - def remove_tracks_by_filepaths(filepaths: str | set[str]): - """ - Removes a track or tracks from the database using their filepaths. - """ - if isinstance(filepaths, str): - filepaths = {filepaths} - - with SQLiteManager() as cur: - for filepath in filepaths: - cur.execute("DELETE FROM tracks WHERE filepath=?", (filepath,)) - - @staticmethod - def remove_tracks_not_in_folders(folders: set[str]): - sql = "DELETE FROM tracks WHERE folder NOT IN ({})".format( - ",".join("?" * len(folders)) - ) - - with SQLiteManager() as cur: - cur.execute(sql, tuple(folders)) diff --git a/app/db/userdata.py b/app/db/userdata.py index 9f3bea2f..0c5f13bc 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -20,6 +20,7 @@ from app.db.utils import ( favorites_to_dataclass, playlist_to_dataclass, playlists_to_dataclasses, + plugin_to_dataclass, plugin_to_dataclasses, similar_artist_to_dataclass, similar_artists_to_dataclass, @@ -110,6 +111,23 @@ class PluginTable(Base): def get_all(cls): return plugin_to_dataclasses(cls.all()) + @classmethod + def activate(cls, name: str, value: bool): + return cls.execute( + update(cls).where(cls.name == name).values(active=value), commit=True + ) + + @classmethod + def get_by_name(cls, name: str): + result = cls.execute(select(cls).where(cls.name == name)) + return plugin_to_dataclass(result.fetchone()) + + @classmethod + def update_settings(cls, name: str, settings: dict[str, Any]): + return cls.execute( + update(cls).where(cls.name == name).values(settings=settings), commit=True + ) + class SimilarArtistTable(Base): __tablename__ = "notlastfm_similar_artists" diff --git a/app/lib/populate.py b/app/lib/populate.py index 0b12565c..703b9a76 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -6,7 +6,6 @@ from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout from app import settings -from app.db.sqlite.tracks import SQLiteTrackMethods from app.lib.artistlib import CheckArtistImages from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors from app.lib.errors import PopulateCancelledError @@ -22,9 +21,6 @@ from app.utils.progressbar import tqdm from app.db.userdata import SimilarArtistTable -get_all_tracks = SQLiteTrackMethods.get_all_tracks -insert_many_tracks = SQLiteTrackMethods.insert_many_tracks -remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths POPULATE_KEY = "" diff --git a/app/lib/trackslib.py b/app/lib/trackslib.py index c8a7a74e..438116a6 100644 --- a/app/lib/trackslib.py +++ b/app/lib/trackslib.py @@ -6,10 +6,6 @@ import os from app.lib.pydub.pydub import AudioSegment from app.lib.pydub.pydub.silence import detect_leading_silence, detect_silence - -from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb -from app.store.tracks import TrackStore -from app.utils.progressbar import tqdm from app.utils.threading import ThreadWithReturnValue diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py index 8d7b9955..0b577596 100644 --- a/app/lib/watchdogg.py +++ b/app/lib/watchdogg.py @@ -12,11 +12,7 @@ from watchdog.observers import Observer from app import settings from app.config import UserConfig -from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb - -# from app.db.sqlite.settings import SettingsSQLMethods as sdb from app.db.sqlite.tracks import SQLiteManager -from app.db.sqlite.tracks import SQLiteTrackMethods as db from app.lib.colorlib import process_color from app.lib.taglib import extract_thumb, get_tags from app.logger import log diff --git a/app/migrations/__init__.py b/app/migrations/__init__.py index 54b51bf5..caa338f0 100644 --- a/app/migrations/__init__.py +++ b/app/migrations/__init__.py @@ -5,13 +5,10 @@ Reads and applies the latest database migrations. """ import inspect -import sys from types import ModuleType # from app.db.sqlite.migrations import MigrationManager from app.db.metadata import MigrationTable -from app.logger import log -from app.migrations import v1_3_0, v1_4_9 from app.migrations.base import Migration @@ -41,7 +38,7 @@ def apply_migrations(): migrations past that index are applied and the new length is stored as the new migration index. """ - modules = [v1_3_0, v1_4_9] + modules = [] migrations = [get_all_migrations(m) for m in modules] # index = MigrationManager.get_index() diff --git a/app/migrations/v1_3_0/__init__.py b/app/migrations/v1_3_0/__init__.py deleted file mode 100644 index 9b20b6b0..00000000 --- a/app/migrations/v1_3_0/__init__.py +++ /dev/null @@ -1,306 +0,0 @@ -import json -import os -import shutil -import time -from collections import OrderedDict -from sqlite3 import OperationalError -from typing import Generator - -from app.db.sqlite.utils import SQLiteManager -from app.migrations.base import Migration -from app.settings import Paths -from app.utils.decorators import coroutine -from app.utils.hashing import create_hash - -# playlists table -# --------------- -# 0: id -# 1: banner_pos -# 2: has_gif -# 3: image -# 4: last_updated -# 5: name -# 6: trackhashes - - -class m1_RemoveSmallThumbnailFolder(Migration): - """ - Removes the small thumbnail folder. - - Because we are added a new folder "original" in the same directory, and the small thumbs folder is used to check if an album's thumbnail is already extracted. - - So we need to remove it, to force the app to extract thumbnails for all albums. - """ - - @staticmethod - def migrate(): - thumbs_sm_path = Paths.get_sm_thumb_path() - thumbs_lg_path = Paths.get_lg_thumb_path() - - for path in [thumbs_sm_path, thumbs_lg_path]: - if os.path.exists(path): - shutil.rmtree(path) - - for path in [thumbs_sm_path, thumbs_lg_path]: - os.makedirs(path, exist_ok=True) - - -class m2_RemovePlaylistArtistHashes(Migration): - """ - removes the artisthashes column from the playlists table. - """ - - @staticmethod - def migrate(): - # remove artisthashes column - sql = "ALTER TABLE playlists DROP COLUMN artisthashes" - - with SQLiteManager(userdata_db=True) as cur: - try: - cur.execute(sql) - except OperationalError: - pass - - cur.close() - - -class m3_AddSettingsToPlaylistTable(Migration): - """ - adds the settings column and removes the banner_pos and has_gif columns - to the playlists table. - """ - - @staticmethod - def migrate(): - select_playlists_sql = "SELECT * FROM playlists" - - with SQLiteManager(userdata_db=True) as cur: - create_playlist_table_sql = """CREATE TABLE IF NOT EXISTS playlists ( - id integer PRIMARY KEY, - image text, - last_updated text not null, - name text not null, - settings text, - trackhashes text - );""" - - insert_playlist_sql = """INSERT INTO playlists( - image, - last_updated, - name, - settings, - trackhashes - ) VALUES(:image, :last_updated, :name, :settings, :trackhashes) - """ - - cur.execute(select_playlists_sql) - - # load all playlists - playlists = cur.fetchall() - - # drop old playlists table - cur.execute("DROP TABLE playlists") - - # create new playlists table - cur.execute(create_playlist_table_sql) - - def transform_playlists(pipeline: Generator, playlists: tuple): - for playlist in playlists: - # create dict that matches the new schema - p = { - "id": playlist[0], - "name": playlist[5], - "image": playlist[3], - "trackhashes": playlist[6], - "last_updated": playlist[4], - "settings": json.dumps( - { - "has_gif": False, - "banner_pos": playlist[1], - "square_img": False, - "pinned": False, - } - ), - } - - pipeline.send(p) - - @coroutine - def insert_playlist(): - while True: - playlist = yield - p = OrderedDict(sorted(playlist.items())) - cur.execute(insert_playlist_sql, p) - - # insert playlists using a coroutine - # (my first coroutine) - pipeline = insert_playlist() - transform_playlists(pipeline, playlists) - pipeline.close() - - cur.close() - - -class m4_AddLastUpdatedToTrackTable(Migration): - """ - adds the last modified column to the tracks table. - """ - - @staticmethod - def migrate(): - # add last_mod column and default to current timestamp - timestamp = time.time() - sql = f"ALTER TABLE tracks ADD COLUMN last_mod text not null DEFAULT '{timestamp}'" - - with SQLiteManager() as cur: - try: - cur.execute(sql) - except OperationalError: - pass - - cur.close() - - -class m5_MovePlaylistsAndFavoritesTo10BitHashes(Migration): - """ - moves the playlists and favorites to 10 bit hashes. - """ - - @staticmethod - def migrate(): - def get_track_data_by_hash(trackhash: str, tracks: list[tuple]) -> tuple: - for track in tracks: - # trackhash is the 15th bit hash - if track[15] == trackhash: - # return artist, album, title - return track[4], track[1], track[13] - - def get_track_by_albumhash(albumhash: str, tracks: list[tuple]) -> tuple: - for track in tracks: - # albumhash is the 3rd bit hash - if track[3] == albumhash: - # return album, albumartist - return track[1], track[2] - - _base = "SELECT * FROM" - fetch_playlists_sql = f"{_base} playlists" - fetch_tracks_sql = f"{_base} tracks" - - update_playlist_hashes_sql = ( - "UPDATE playlists SET trackhashes = :trackhashes WHERE id = :id" - ) - fetch_favorites_sql = f"{_base} favorites" - update_fav_sql = "UPDATE favorites SET hash = :hash WHERE id = :id" - remove_fav_sql = "DELETE FROM favorites WHERE id = :id" - - db_tracks = [] - - # read tracks from db - with SQLiteManager() as cur: - cur.execute(fetch_tracks_sql) - db_tracks.extend(cur.fetchall()) - cur.close() - - # update playlists - with SQLiteManager(userdata_db=True) as cur: - cur.execute(fetch_playlists_sql) - playlists = cur.fetchall() - - # for each playlist - for p in playlists: - pid = p[0] - - # load trackhashes - trackhashes: list[str] = json.loads(p[5]) - - for index, t in enumerate(trackhashes): - (artist, album, title) = get_track_data_by_hash(t, db_tracks) - - # create new hash - new_hash = create_hash(artist, album, title, decode=True, limit=10) - trackhashes[index] = new_hash - - # convert to string - trackhashes = json.dumps(trackhashes) - - # save to db - cur.execute( - update_playlist_hashes_sql, {"trackhashes": trackhashes, "id": pid} - ) - - cur.close() - - # update favorites - with SQLiteManager(userdata_db=True) as cur: - cur.execute(fetch_favorites_sql) - favorites = cur.fetchall() - - # for each favorite - for f in favorites: - fid = f[0] - - fhash: str = f[1] - ftype: str = f[2] # "track" || "album" - - if ftype == "album": - (album, albumartist) = get_track_by_albumhash(fhash, db_tracks) - - # create new hash - new_hash = create_hash(album, albumartist, decode=True, limit=10) - - # save to db - cur.execute(update_fav_sql, {"hash": new_hash, "id": fid}) - continue - - if ftype == "track": - (artist, album, title) = get_track_data_by_hash(fhash, db_tracks) - - # create new hash - new_hash = create_hash(artist, album, title, decode=True, limit=10) - - # save to db - cur.execute(update_fav_sql, {"hash": new_hash, "id": fid}) - continue - - # remove favorites that are not track or album. ie. artists - cur.execute(remove_fav_sql, {"id": fid}) - - cur.close() - - -class m6_RemoveAllTracks(Migration): - """ - removes all tracks from the tracks table. - """ - - @staticmethod - def migrate(): - sql = "DELETE FROM tracks" - - with SQLiteManager() as cur: - cur.execute(sql) - cur.close() - - -class m7_UpdateAppSettingsTable(Migration): - @staticmethod - def migrate(): - drop_table_sql = "DROP TABLE settings" - create_table_sql = """ - CREATE TABLE IF NOT EXISTS settings ( - id integer PRIMARY KEY, - root_dirs text NOT NULL, - exclude_dirs text, - artist_separators text NOT NULL default '/,;', - extract_feat integer NOT NULL DEFAULT 1, - remove_prod integer NOT NULL DEFAULT 1, - clean_album_title integer NOT NULL DEFAULT 1, - remove_remaster integer NOT NULL DEFAULT 1, - merge_albums integer NOT NULL DEFAULT 0, - show_albums_as_singles integer NOT NULL DEFAULT 0 - ); - """ - - with SQLiteManager(userdata_db=True) as cur: - cur.execute(drop_table_sql) - cur.execute(create_table_sql) diff --git a/app/migrations/v1_4_9/__init__.py b/app/migrations/v1_4_9/__init__.py deleted file mode 100644 index be10c97b..00000000 --- a/app/migrations/v1_4_9/__init__.py +++ /dev/null @@ -1,405 +0,0 @@ -import os -import shutil -import sqlite3 -from time import time -from app.db.sqlite.utils import SQLiteManager -from app.migrations.base import Migration -from app.settings import Paths - -import hashlib -from unidecode import unidecode - -from app.db.sqlite.tracks import SQLiteTrackMethods as tdb -from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb -from app.db.sqlite.logger.tracks import SQLiteTrackLogger as ldb -from app.utils.hashing import create_hash - - -def create_sha256_hash(*args: str, decode=False, limit=10) -> str: - """ - This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments. - - Example use case: - - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID. - - :param args: The arguments to hash. - :param decode: Whether to decode the arguments before hashing. - :param limit: The number of characters to return. - - :return: The hash. - """ - - def remove_non_alnum(token: str) -> str: - token = token.lower().strip().replace(" ", "") - t = "".join(t for t in token if t.isalnum()) - - if t == "": - return token - - return t - - str_ = "".join(remove_non_alnum(t) for t in args) - - if decode: - str_ = unidecode(str_) - - str_ = str_.encode("utf-8") - str_ = hashlib.sha256(str_).hexdigest() - return str_[-limit:] - - -def create_sha1_hash(*args: str, decode=False, limit=10) -> str: - """ - This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments. - - Example use case: - - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID. - - :param args: The arguments to hash. - :param decode: Whether to decode the arguments before hashing. - :param limit: The number of characters to return. - - :return: The hash. - """ - - def remove_non_alnum(token: str) -> str: - token = token.lower().strip().replace(" ", "") - t = "".join(t for t in token if t.isalnum()) - - if t == "": - return token - - return t - - str_ = "".join(remove_non_alnum(t) for t in args) - - if decode: - str_ = unidecode(str_) - - str_ = str_.encode("utf-8") - str_ = hashlib.sha1(str_).hexdigest() - - return ( - str_[: limit // 2] + str_[-limit // 2 :] - if limit % 2 == 0 - else str_[: limit // 2] + str_[-limit // 2 - 1 :] - ) - - -class _1AddTimestampToFavoritesTable(Migration): - """ - Adds a timestamp column to the favorites table. - """ - - @staticmethod - def migrate(): - # INFO: add timestamp column with automatic current timestamp - sql = f"ALTER TABLE favorites ADD COLUMN timestamp INTEGER NOT NULL DEFAULT 0" - - # INFO: execute the sql - with SQLiteManager(userdata_db=True) as cur: - table_exists = cur.execute( - "select count(*) from pragma_table_info('favorites') where name = 'timestamp'" - ) - - table_exists = table_exists.fetchone() - - if table_exists[0] == 1: - return - - # INFO: Add the timestamp column to the favorites table - timestamp = int(time()) - cur.execute(sql) - cur.execute(f"UPDATE favorites SET timestamp = {timestamp}") - - -class _2DeleteOriginalThumbnails(Migration): - """ - Original thumbnails are too large and are not needed. - """ - - # TODO: Implement this migration - - @staticmethod - def migrate(): - imgpath = Paths.get_thumbs_path() - og_imgpath = os.path.join(imgpath, "original") - - if os.path.exists(og_imgpath): - shutil.rmtree(og_imgpath) - - -class _3MoveScrobbleToUserId1(Migration): - """ - Updates all track logs from user id = 0 to user id = 1 - """ - - @staticmethod - def migrate(): - sql = """ - UPDATE track_logger SET userid = 1 WHERE userid = 0; - ALTER TABLE track_logger RENAME TO _track_logger; - CREATE TABLE IF NOT EXISTS track_logger ( - id integer PRIMARY KEY, - trackhash text NOT NULL, - duration integer NOT NULL, - timestamp integer NOT NULL, - source text, - userid integer NOT NULL DEFAULT 1, - constraint fk_users foreign key (userid) references users(id) on delete cascade - ); - - INSERT INTO track_logger SELECT * FROM _track_logger; - DROP TABLE _track_logger; - """ - # INFO: Move the scrobble table to the user id 1 - with SQLiteManager(userdata_db=True) as cur: - cur.executescript(sql) - cur.close() - - -class _4AddUserIdToFavoritesTable(Migration): - """ - Adds a userid column to the favorites table. - """ - - @staticmethod - def migrate(): - # check if userid column exists - exists_sql = ( - "select count(*) from pragma_table_info('favorites') where name = 'userid'" - ) - sql = """ - ALTER TABLE favorites ADD userid INTEGER NOT NULL DEFAULT 1; - ALTER TABLE favorites RENAME TO _favorites; - - CREATE TABLE IF NOT EXISTS favorites ( - id integer PRIMARY KEY, - hash text not null, - type text not null, - timestamp integer not null default 0, - userid integer not null, - constraint fk_users foreign key (userid) references users(id) on delete cascade - ); - - INSERT INTO favorites SELECT * FROM _favorites; - DROP TABLE _favorites; - """ - - with SQLiteManager(userdata_db=True) as cur: - data = cur.execute(exists_sql) - data = data.fetchone() - - if data[0] == 1: - return # INFO: column already exists - - cur.executescript(sql) - - -class _5AddUserIdToPlaylistsTable(Migration): - """ - Adds a userid column to the playlists table. - """ - - @staticmethod - def migrate(): - # check if userid column exists - exists_sql = ( - "select count(*) from pragma_table_info('playlists') where name = 'userid'" - ) - - # Add the userid column to the playlists table - # Rename the old table to _playlists - # Create a new playlists table with the userid column - # Then, copy the data from the old table to the new table - # Finally, drop the old table - sql = """ - ALTER TABLE playlists ADD userid INTEGER NOT NULL DEFAULT 1; - ALTER TABLE playlists RENAME TO _playlists; - CREATE TABLE IF NOT EXISTS playlists ( - id integer PRIMARY KEY, - image text, - last_updated text not null, - name text not null, - settings text, - trackhashes text, - userid integer not null, - constraint fk_users foreign key (userid) references users(id) on delete cascade - ); - - INSERT INTO playlists SELECT * FROM _playlists; - DROP TABLE _playlists; - """ - - with SQLiteManager(userdata_db=True) as cur: - # INFO: Check if the column already exists - data = cur.execute(exists_sql) - data = data.fetchone() - - # INFO: If the column already exists, return - if data[0] == 1: - return # INFO: column already exists - - # INFO: Execute the sql - cur.executescript(sql) - - -class _6MoveHashesToSha1(Migration): - """ - Moves the 10 bit item hashes from sha256 to sha1 which is - faster and more lenient on less powerful devices. - - Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution. - """ - - # enabled: bool = False - - # pass - - # INFO: Apparentlly, every single table is affected by this migration. - # NOTE: Use generators to avoid memory issues. - - @classmethod - def port_track(cls, trackhash: str): - # get the track with the track hash - track = tdb.get_track_by_trackhash(trackhash) - - if track is None: - return - - title = track.og_title - if track.trackhash != trackhash: - # raise ValueError("Track hash mismatch") - print("Track hash mismatch") - title = track.title - else: - print("Porting track: ", track.title) - - # return the new hash - finalhash = create_sha1_hash( - ", ".join(a.name for a in track.artists), - track.og_album, - title, - ) - - if finalhash != create_hash( - ", ".join(a.name for a in track.artists), track.og_album, title - ): - raise ValueError("Hash mismatch") - - @classmethod - def port_album(cls, albumhash: str): - # get the first track with the album hash - track = tdb.get_track_by_albumhash(albumhash) - - if track is None: - return - - # return the new hash - return create_sha1_hash( - track.og_album, - ", ".join(a.name for a in track.albumartists), - ) - - @classmethod - def port_artist(cls, artisthash: str): - # find all tracks with the artist hash - tracks = [t for t in cls.tracks if artisthash in t.artist_hashes] - - if len(tracks) == 0: - return - - # find the artist name - artist = [ - a.name - for a in tracks[0].artists - if create_sha256_hash(a.name, decode=True) == artisthash - ][0] - - # return the new hash - return create_sha1_hash(artist, decode=True) - - @classmethod - def migrate_favorites(cls): - with SQLiteManager(userdata_db=True) as cur: - # read all favorites - data = cur.execute("SELECT * FROM favorites") - data = data.fetchall() - - for track in cls.tracks: - track.artist_hashes = "-".join( - [create_sha256_hash(a.name, decode=True) for a in track.artists] - ) - - for entry in data: - # hash is the 2nd column in the table - hash = entry[1] - - # entry type is the 3rd column in the table - if entry[2] == "track": - newhash = cls.port_track(hash) - - if newhash: - cur.execute( - f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'track'" - ) - - elif entry[2] == "album": - newhash = cls.port_album(hash) - - if newhash: - cur.execute( - f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'album'" - ) - - elif entry[2] == "artist": - newhash = cls.port_artist(hash) - - if newhash: - cur.execute( - f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'artist'" - ) - - @classmethod - def migrate_playlists(cls): - playlists = pdb.get_all_playlists() - - for playlist in playlists: - # remove previous hashes - to_remove = [ - {"trackhash": trackhash, "index": index} - for index, trackhash in enumerate(playlist.trackhashes) - ] - pdb.remove_tracks_from_playlist(playlist.id, to_remove) - - # add new hashes - newhashes = [ - cls.port_track(trackhash) for trackhash in playlist.trackhashes - ] - newhashes = [h for h in newhashes if h is not None] - pdb.add_tracks_to_playlist(playlist.id, newhashes) - - print("Ported playlist: ", playlist.name) - print("Total tracks: ", len(newhashes)) - - @classmethod - def migrate_scrobble(cls): - # read all logs - logs = ldb.get_all() - - with SQLiteManager(userdata_db=True) as cur: - # for each log, port the hash - for log in logs: - newhash = cls.port_track(log[1]) - - if newhash: - cur.execute( - f"UPDATE track_logger SET trackhash = '{newhash}' WHERE trackhash = '{log[1]}'" - ) - - @classmethod - def migrate(cls): - cls.tracks = list(tdb.get_all_tracks()) - cls.migrate_favorites() - # cls.migrate_playlists() - # cls.migrate_scrobble() diff --git a/app/plugins/lyrics.py b/app/plugins/lyrics.py index 413ad766..05c16005 100644 --- a/app/plugins/lyrics.py +++ b/app/plugins/lyrics.py @@ -7,7 +7,7 @@ from typing import List, Optional import requests from unidecode import unidecode -from app.db.sqlite.plugins import PluginsMethods +from app.db.userdata import PluginTable from app.plugins import Plugin, plugin_method from app.settings import Paths @@ -190,15 +190,13 @@ class LyricsProvider(LRCProvider): class Lyrics(Plugin): def __init__(self) -> None: - plugin = PluginsMethods.get_plugin_by_name("lyrics_finder") + plugin = PluginTable.get_by_name("lyrics_finder") if not plugin: return name = plugin.name - description = plugin.description - - super().__init__(name, description) + super().__init__(name, "Musixmatch lyrics finder") self.provider = LyricsProvider() diff --git a/app/utils/auth.py b/app/utils/auth.py index 983cd83f..2aaaf060 100644 --- a/app/utils/auth.py +++ b/app/utils/auth.py @@ -14,7 +14,6 @@ def hash_password(password: str) -> str: :return: The hashed password. """ - return hashlib.pbkdf2_hmac( "sha256", password.encode("utf-8"), UserConfig().serverId.encode("utf-8"), 100000 ).hex() diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/sqlite/test_sqlite_actions.py b/tests/sqlite/test_sqlite_actions.py deleted file mode 100644 index 141d5a0d..00000000 --- a/tests/sqlite/test_sqlite_actions.py +++ /dev/null @@ -1,59 +0,0 @@ -import json -import sqlite3 -import os -from app.db.sqlite.artistcolors import SQLiteArtistMethods -from app.db.sqlite.queries import CREATE_APPDB_TABLES - -from app.db.sqlite.utils import SQLiteManager - -db_path = "test.db" - - -def test_sqlite_manager(): - with SQLiteManager(test_db_path=db_path) as cur: - for query in CREATE_APPDB_TABLES.split(";"): - cur.execute(query) - - cur.execute( - "INSERT INTO tracks (album, albumartist, albumhash, artist, bitrate, copyright, date, disc, duration, filepath, folder, genre, last_mod, title, track, trackhash) VALUES ('Dummy Album', 'Dummy Album Artist', 'dummyalbumhash', 'Dummy Artist', 320, 'Dummy Copyright', 1630454400, 1, 180, '/path/to/dummy/file.mp3', '/path/to/dummy/folder', 'Dummy Genre', 1630454400.5, 'Dummy Title', 1, 'dummytrackhash');" - ) - - cur.execute("SELECT * FROM tracks") - result = cur.fetchone() - assert result[7] == 1630454400 - - # Test using a connection - with SQLiteManager(conn=sqlite3.connect(db_path)) as cur: - cur.execute("SELECT * FROM tracks") - result = cur.fetchone() - assert result[7] == 1630454400 - - -def test_insert_one_artist(): - color1 = "rgb(0, 0, 0)" - color2 = "rgb(255, 255, 255)" - - with SQLiteManager(test_db_path=db_path) as cur: - SQLiteArtistMethods.insert_one_artist(cur, "artisthash1", [color1, color2]) - cur.execute("SELECT * FROM artists WHERE artisthash=?", ("artisthash1",)) - - result = cur.fetchone() - assert result[1:] == ("artisthash1", json.dumps([color1, color2]), None) - - -def test_get_all_artists(): - with SQLiteManager(test_db_path=db_path) as cur: - artists = SQLiteArtistMethods.get_all_artists(cur) - - # assert that that the generator is not empty and that for each tuple has 4 elements - - try: - while True: - artist = next(artists) - assert len(artist) == 4 - except StopIteration: - pass - - -def test_remove_test_db(): - os.remove(db_path) diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index 01c72d13..00000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,34 +0,0 @@ -# from hypothesis import given -from app.utils.parsers import parse_feat_from_title - - -def test_extract_featured_artists_from_title(): - test_titles = [ - "Own it (Featuring Ed Sheeran & Stormzy)", - "Own it (Featuring Ed Sheeran and Stormzy)", - "Autograph (On my line)(Feat. Lil Peep)(Deluxe)", - "Why so sad? (with Juice Wrld, Lil Peep)", - "Why so sad? (with Juice Wrld/Lil Peep)", - "Simmer (with Burna Boy)", - "Simmer (without Burna Boy)", - ] - - results = [ - ["Ed Sheeran", "Stormzy"], - ["Ed Sheeran", "Stormzy"], - ["Lil Peep"], - ["Juice Wrld", "Lil Peep"], - ["Juice Wrld", "Lil Peep"], - ["Burna Boy"], - [], - ] - - for title, expected in zip(test_titles, results): - assert parse_feat_from_title(title)[0] == expected - - -# === HYPOTHESIS GHOSTWRITER TESTS === - -# @given(__dir=st.text(), full=st.booleans()) -# def test_fuzz_run_fast_scandir(__dir: str, full) -> None: -# app.utils.run_fast_scandir(_dir=__dir, full=full) From cc2d017816c0dfcd676ed60f4c483e5d50e0d117 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 4 Aug 2024 11:34:33 +0300 Subject: [PATCH 35/44] feat: implement folder and folder track sorting --- app/api/folder.py | 51 ++++++++++++++++++++++++++++++++++++++----- app/lib/folderslib.py | 32 ++++++++++++++++++++------- app/lib/index.py | 15 ++++--------- app/lib/sortlib.py | 44 +++++++++++++++++++++++++++++++++++++ app/models/folder.py | 1 - 5 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 app/lib/sortlib.py diff --git a/app/api/folder.py b/app/api/folder.py index 2972c3e8..ff612b75 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -23,11 +23,46 @@ api = APIBlueprint("folder", __name__, url_prefix="/folder", abp_tags=[tag]) class FolderTree(BaseModel): - folder: str = Field( - "$home", example="$home", description="The folder to things from" + folder: str = Field("$home", description="The folder to things from") + tracksortby: str = Field( + "default", + description="""The field to sort tracks by. Options: [ + "default", + "album", + "albumartists", + "artists", + "bitrate", + "date", + "disc", + "duration", + "lastmod", + "lastplayed", + "playduration", + "playcount", + "title", + ]""", + ) + tracksort_reverse: bool = Field( + False, + description="Whether to reverse the sort order of the tracks", + ) + foldersortby: str = Field( + "lastmod", + description="""The field to sort folders by. + Options: [ + "default", + "name", + "lastmod", + "trackcount", + ] + """, + ) + foldersort_reverse: bool = Field( + True, + description="Whether to reverse the sort order of the folders", ) start: int = Field(0, description="The start index") - end: int = Field(50, description="The end index") + limit: int = Field(50, description="The max number of items to return") tracks_only: bool = Field(False, description="Whether to only get tracks") @@ -69,9 +104,15 @@ def get_folder_tree(body: FolderTree): req_dir = "/" + req_dir if not req_dir.startswith("/") else req_dir res = get_files_and_dirs( - req_dir, start=body.start, end=body.end, tracks_only=tracks_only + req_dir, + start=body.start, + limit=body.limit, + tracks_only=tracks_only, + tracksortby=body.tracksortby, + foldersortby=body.foldersortby, + tracksort_reverse=body.tracksort_reverse, + foldersort_reverse=body.foldersort_reverse, ) - res["folders"] = sorted(res["folders"], key=lambda i: i.name) return res diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index ef7b7510..b04a2933 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -1,6 +1,7 @@ import os from pathlib import Path +from app.lib.sortlib import sort_folders, sort_tracks from app.logger import log from app.models import Folder from app.serializers.track import serialize_tracks @@ -11,7 +12,7 @@ from app.utils.wintools import win_replace_slash # from app.db.libdata import TrackTable as TrackDB -def create_folder(path: str, trackcount=0, foldercount=0) -> Folder: +def create_folder(path: str, trackcount=0) -> Folder: """ Creates a folder object from a path. """ @@ -22,7 +23,6 @@ def create_folder(path: str, trackcount=0, foldercount=0) -> Folder: path=win_replace_slash(str(folder)) + "/", is_sym=folder.is_symlink(), trackcount=trackcount, - foldercount=foldercount, ) @@ -46,14 +46,22 @@ def get_folders(paths: list[str]): """ folders = FolderStore.count_tracks_containing_paths(paths) return [ - create_folder(f["path"], f["trackcount"], foldercount=0) + create_folder(f["path"], f["trackcount"]) for f in folders if f["trackcount"] > 0 ] def get_files_and_dirs( - path: str, start: int, end: int, tracks_only: bool = False, skip_empty_folders=True + path: str, + start: int, + limit: int, + tracksortby: str, + foldersortby: str, + tracksort_reverse: bool, + foldersort_reverse: bool, + tracks_only: bool = False, + skip_empty_folders=True, ): """ Given a path, returns a list of tracks and folders in that immediate path. @@ -102,14 +110,17 @@ def get_files_and_dirs( tracks = [] if files: - if end == -1: - end = len(files) + if limit == -1: + limit = len(files) - tracks = list(FolderStore.get_tracks_by_filepaths(files[start:end])) + tracks = list(FolderStore.get_tracks_by_filepaths(files)) + tracks = sort_tracks(tracks, tracksortby, tracksort_reverse) + tracks = tracks[start : start + limit] folders = [] if not tracks_only: folders = get_folders(dirs) + folders = sort_folders(folders, foldersortby, foldersort_reverse) if skip_empty_folders and len(folders) == 1 and len(tracks) == 0: # INFO: When we only have one folder and no tracks, @@ -118,7 +129,11 @@ def get_files_and_dirs( return get_files_and_dirs( folders[0].path, start=start, - end=end, + limit=limit, + tracksortby=tracksortby, + foldersortby=foldersortby, + tracksort_reverse=tracksort_reverse, + foldersort_reverse=foldersort_reverse, tracks_only=tracks_only, skip_empty_folders=True, ) @@ -127,4 +142,5 @@ def get_files_and_dirs( "path": path, "tracks": serialize_tracks(tracks), "folders": folders, + "total": len(files), } diff --git a/app/lib/index.py b/app/lib/index.py index 4118663e..dff48dc1 100644 --- a/app/lib/index.py +++ b/app/lib/index.py @@ -14,17 +14,6 @@ from app.store.folder import FolderStore from app.store.tracks import TrackStore from app.utils.threading import background -def load_and_map(): - key = str(time()) - FolderStore.load_filepaths() - AlbumStore.load_albums(key) - ArtistStore.load_artists(key) - - map_scrobble_data() - map_favorites() - map_artist_colors() - map_album_colors() - class IndexEverything: def __init__(self) -> None: @@ -36,6 +25,10 @@ class IndexEverything: ArtistStore.load_artists(key) FolderStore.load_filepaths() + # map colors + map_album_colors() + map_artist_colors() + map_scrobble_data() map_favorites() diff --git a/app/lib/sortlib.py b/app/lib/sortlib.py new file mode 100644 index 00000000..3198f855 --- /dev/null +++ b/app/lib/sortlib.py @@ -0,0 +1,44 @@ +from itertools import groupby +import os +from pprint import pprint +from app.lib.albumslib import sort_by_track_no +from app.models.folder import Folder +from app.models.track import Track +from app.utils import flatten + + +def sort_tracks(tracks: list[Track], key: str, reverse: bool = False): + """ + Sorts a list of tracks by a key. + """ + if key == "default": + return tracks + + sortfunc = lambda x: getattr(x, key) + + if key == "artists" or key == "albumartists": + sortfunc = lambda x: getattr(x, key)[0]["name"] + + if key == "disc": + # INFO: Group tracks into albums, then sort them by disc number. + tracks = sorted(tracks, key=lambda x: x.album) + groups = groupby(tracks, lambda x: x.albumhash) + + return flatten([sort_by_track_no(list(g)) for k, g in groups]) + + return sorted(tracks, key=sortfunc, reverse=reverse) + + +def sort_folders(folders: list[Folder], key: str, reverse: bool = False): + """ + Sorts a list of folders by a key. + """ + if key == "default": + return folders + + sortfunc = lambda x: getattr(x, key) + + if key == "lastmod": + sortfunc = lambda x: os.path.getmtime(x.path) + + return sorted(folders, key=sortfunc, reverse=reverse) diff --git a/app/models/folder.py b/app/models/folder.py index cd92b0bc..f86dd324 100644 --- a/app/models/folder.py +++ b/app/models/folder.py @@ -7,4 +7,3 @@ class Folder: path: str is_sym: bool = False trackcount: int = 0 - foldercount: int = 0 From 8c1c46fe1eb01cabeb51fd9afbc44c43f39dbc37 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 4 Aug 2024 13:15:38 +0300 Subject: [PATCH 36/44] rename sort keys --- app/api/folder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/api/folder.py b/app/api/folder.py index ff612b75..c9fcd9bd 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -24,7 +24,7 @@ api = APIBlueprint("folder", __name__, url_prefix="/folder", abp_tags=[tag]) class FolderTree(BaseModel): folder: str = Field("$home", description="The folder to things from") - tracksortby: str = Field( + sorttracksby: str = Field( "default", description="""The field to sort tracks by. Options: [ "default", @@ -46,7 +46,7 @@ class FolderTree(BaseModel): False, description="Whether to reverse the sort order of the tracks", ) - foldersortby: str = Field( + sortfoldersby: str = Field( "lastmod", description="""The field to sort folders by. Options: [ @@ -58,7 +58,7 @@ class FolderTree(BaseModel): """, ) foldersort_reverse: bool = Field( - True, + False, description="Whether to reverse the sort order of the folders", ) start: int = Field(0, description="The start index") @@ -108,8 +108,8 @@ def get_folder_tree(body: FolderTree): start=body.start, limit=body.limit, tracks_only=tracks_only, - tracksortby=body.tracksortby, - foldersortby=body.foldersortby, + tracksortby=body.sorttracksby, + foldersortby=body.sortfoldersby, tracksort_reverse=body.tracksort_reverse, foldersort_reverse=body.foldersort_reverse, ) From e562fa691dd8685ca984d973df072165fbf6dd3d Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 4 Aug 2024 18:51:02 +0300 Subject: [PATCH 37/44] connect quick scan --- app/api/settings.py | 57 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/app/api/settings.py b/app/api/settings.py index 989ca298..09a5727d 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, Field from app.api.auth import admin_required from app.db.userdata import PluginTable -from app.lib.index import index_everything +from app.lib.index import IndexEverything, index_everything from app.logger import log from app.settings import Info, SessionVarKeys from app.store.albums import AlbumStore @@ -194,43 +194,43 @@ class SetSettingBody(BaseModel): ) -@api.post("/set") -@admin_required() -def set_setting(body: SetSettingBody): - """ - Set a setting. - """ - key = body.key - value = body.value +# @api.post("/set") +# @admin_required() +# def set_setting(body: SetSettingBody): +# """ +# Set a setting. +# """ +# key = body.key +# value = body.value - if key is None or value is None or key == "root_dirs": - return {"msg": "Invalid arguments!"}, 400 +# if key is None or value is None or key == "root_dirs": +# return {"msg": "Invalid arguments!"}, 400 - root_dir = sdb.get_root_dirs() +# root_dir = sdb.get_root_dirs() - if not root_dir: - return {"msg": "No root directories set!"}, 400 +# if not root_dir: +# return {"msg": "No root directories set!"}, 400 - if key not in mapp: - return {"msg": "Invalid key!"}, 400 +# if key not in mapp: +# return {"msg": "Invalid key!"}, 400 - if key == "artist_separators": - value = str(value).split(",") - value = set(value) +# if key == "artist_separators": +# value = str(value).split(",") +# value = set(value) - reload_all_for_set_setting() +# reload_all_for_set_setting() - # if value is a set, convert it to a string - # (artist_separators) - if type(value) == set: - value = ",".join(value) +# # if value is a set, convert it to a string +# # (artist_separators) +# if type(value) == set: +# value = ",".join(value) - return {"result": value} +# return {"result": value} @background -def run_populate(): - # populate.Populate(instance_key=get_random_str()) +def index_stuff(): + IndexEverything() pass @@ -239,8 +239,7 @@ def trigger_scan(): """ Triggers scan for new music """ - run_populate() - + index_stuff() return {"msg": "Scan triggered!"} From 6d2aac084d912a4244f0bbb87bd7e7b6de8e5891 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 4 Aug 2024 19:31:22 +0300 Subject: [PATCH 38/44] limit custom playlist items to 100 --- TODO.md | 1 - app/api/auth.py | 3 --- app/api/playlist.py | 5 +++++ app/api/settings.py | 39 +++++++++++++++------------------------ app/settings.py | 15 --------------- 5 files changed, 20 insertions(+), 43 deletions(-) diff --git a/TODO.md b/TODO.md index 1dbe7a85..9014d561 100644 --- a/TODO.md +++ b/TODO.md @@ -56,4 +56,3 @@ - Duplicates on search - Audio stops on ending -- Port account settings to config on the frontend \ No newline at end of file diff --git a/app/api/auth.py b/app/api/auth.py index 0cf41bf1..de36e78a 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -14,7 +14,6 @@ from pydantic import BaseModel, Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint -# from app.db.sqlite.auth import SQLiteAuthMethods as authdb from app.db.userdata import UserTable from app.utils.auth import check_password, hash_password from app.config import UserConfig @@ -307,8 +306,6 @@ def get_all_users(query: GetAllUsersQuery): Get all users (if you're an admin, you will also receive accounts settings) """ config = UserConfig() - # config.enableGuest = True - # config.usersOnLogin = True settings = { "enableGuest": False, "usersOnLogin": config.usersOnLogin, diff --git a/app/api/playlist.py b/app/api/playlist.py index a437bf8f..8ba74cdb 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -200,6 +200,11 @@ def get_playlist(path: PlaylistIDPath, query: GetPlaylistQuery): is_custom = playlistid in {p["name"] for p in custom_playlists} if is_custom: + if query.start != 0: + return { + "tracks": [], + } + handler = next( p["handler"] for p in custom_playlists if p["name"] == playlistid ) diff --git a/app/api/settings.py b/app/api/settings.py index 09a5727d..561c7395 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -6,14 +6,12 @@ from pydantic import BaseModel, Field from app.api.auth import admin_required from app.db.userdata import PluginTable -from app.lib.index import IndexEverything, index_everything +from app.lib.index import index_everything from app.logger import log -from app.settings import Info, SessionVarKeys +from app.settings import Info from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore -from app.utils.generators import get_random_str -from app.utils.threading import background from app.config import UserConfig bp_tag = Tag(name="Settings", description="Customize stuff") @@ -154,15 +152,15 @@ def get_root_dirs(): # maps settings to their parser flags -mapp = { - "artist_separators": SessionVarKeys.ARTIST_SEPARATORS, - "extract_feat": SessionVarKeys.EXTRACT_FEAT, - "remove_prod": SessionVarKeys.REMOVE_PROD, - "clean_album_title": SessionVarKeys.CLEAN_ALBUM_TITLE, - "remove_remaster": SessionVarKeys.REMOVE_REMASTER_FROM_TRACK, - "merge_albums": SessionVarKeys.MERGE_ALBUM_VERSIONS, - "show_albums_as_singles": SessionVarKeys.SHOW_ALBUMS_AS_SINGLES, -} +# mapp = { +# "artist_separators": SessionVarKeys.ARTIST_SEPARATORS, +# "extract_feat": SessionVarKeys.EXTRACT_FEAT, +# "remove_prod": SessionVarKeys.REMOVE_PROD, +# "clean_album_title": SessionVarKeys.CLEAN_ALBUM_TITLE, +# "remove_remaster": SessionVarKeys.REMOVE_REMASTER_FROM_TRACK, +# "merge_albums": SessionVarKeys.MERGE_ALBUM_VERSIONS, +# "show_albums_as_singles": SessionVarKeys.SHOW_ALBUMS_AS_SINGLES, +# } @api.get("") @@ -178,9 +176,9 @@ def get_all_settings(): return config -@background -def reload_all_for_set_setting(): - reload_everything(get_random_str()) +# @background +# def reload_all_for_set_setting(): +# reload_everything(get_random_str()) class SetSettingBody(BaseModel): @@ -227,19 +225,12 @@ class SetSettingBody(BaseModel): # return {"result": value} - -@background -def index_stuff(): - IndexEverything() - pass - - @api.get("/trigger-scan") def trigger_scan(): """ Triggers scan for new music """ - index_stuff() + index_everything() return {"msg": "Scan triggered!"} diff --git a/app/settings.py b/app/settings.py index 4534a1c9..1320454e 100644 --- a/app/settings.py +++ b/app/settings.py @@ -5,7 +5,6 @@ Contains default configs import os import subprocess import sys -from typing import Any from app import configs @@ -231,20 +230,6 @@ class SessionVars: SHOW_ALBUMS_AS_SINGLES = False -# TODO: Find a way to eliminate this class without breaking typings -class SessionVarKeys: - EXTRACT_FEAT = "EXTRACT_FEAT" - REMOVE_PROD = "REMOVE_PROD" - CLEAN_ALBUM_TITLE = "CLEAN_ALBUM_TITLE" - REMOVE_REMASTER_FROM_TRACK = "REMOVE_REMASTER_FROM_TRACK" - DO_PERIODIC_SCANS = "DO_PERIODIC_SCANS" - PERIODIC_SCAN_INTERVAL = "PERIODIC_SCAN_INTERVAL" - MERGE_ALBUM_VERSIONS = "MERGE_ALBUM_VERSIONS" - ARTIST_SEPARATORS = "ARTIST_SEPARATORS" - SHOW_ALBUMS_AS_SINGLES = "SHOW_ALBUMS_AS_SINGLES" - - - class TCOLOR: """ Terminal colors From cd992419c523abc2de11a4d0889938552d7c0f81 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 10 Aug 2024 08:42:13 +0300 Subject: [PATCH 39/44] implement artist split ingore list + move post processing of tags to the track model + rebuild stores on settings update via API + check files from the store instead of the db when streaming + remove deprecetated table columns +misc --- TODO.md | 10 +- app/api/settings.py | 128 +++----------------------- app/api/stream.py | 46 ++++++++-- app/config.py | 7 ++ app/db/libdata.py | 27 ++++-- app/db/utils.py | 7 +- app/lib/tagger.py | 2 +- app/lib/taglib.py | 177 ++++++++++++++++++------------------ app/models/track.py | 130 +++++++++++++++++++++++++- app/store/tracks.py | 2 - app/utils/parsers.py | 53 +++++++++-- tests/test_split_artists.py | 137 ++++++++++++++++++++++++++++ 12 files changed, 481 insertions(+), 245 deletions(-) create mode 100644 tests/test_split_artists.py diff --git a/TODO.md b/TODO.md index 9014d561..ab4ebd22 100644 --- a/TODO.md +++ b/TODO.md @@ -33,17 +33,9 @@ # THE BIG ONE -- Updating settings -- Cleaning out commented code - Watchdog - Periodic scans -- Remove legacy db methods -- Review: We don't need server side image colors -- Clean up main db and userdata modules -- Move plugins to a config file - What about our migrations? -- Add userid to queries -- Remove duplicates on artist page (test with Hanson) - Test foreign keys on delete - Normalize playlists table: - New table to hold playlist entries @@ -56,3 +48,5 @@ - Duplicates on search - Audio stops on ending +- Show users on account settings when logged in as admin and show users on login is disabled. +- \ No newline at end of file diff --git a/app/api/settings.py b/app/api/settings.py index 561c7395..c09e6b6e 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -7,11 +7,7 @@ from app.api.auth import admin_required from app.db.userdata import PluginTable from app.lib.index import index_everything -from app.logger import log from app.settings import Info -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore -from app.store.tracks import TrackStore from app.config import UserConfig bp_tag = Tag(name="Settings", description="Customize stuff") @@ -24,65 +20,6 @@ def get_child_dirs(parent: str, children: list[str]): return [_dir for _dir in children if _dir.startswith(parent) and _dir != parent] -def reload_everything(instance_key: str): - """ - Reloads all stores using the current database items - """ - try: - TrackStore.load_all_tracks(instance_key) - except Exception as e: - log.error(e) - - try: - AlbumStore.load_albums(instance_key=instance_key) - except Exception as e: - log.error(e) - - try: - ArtistStore.load_artists(instance_key) - except Exception as e: - log.error(e) - - -# CHECKPOINT: TEST SETTINGS API ENDPOINTS - -# @background -# def rebuild_store(db_dirs: list[str]): -# """ -# Restarts watchdog and rebuilds the music library. -# """ -# instance_key = get_random_str() - -# log.info("Rebuilding library...") -# trackdb.remove_tracks_not_in_folders(db_dirs) -# reload_everything(instance_key) - -# try: -# populate.Populate(instance_key=instance_key) -# except populate.PopulateCancelledError as e: -# print(e) -# reload_everything(instance_key) -# return - -# WatchDog().restart() - -# log.info("Rebuilding library... ✅") - - -# # I freaking don't know what this function does anymore -# def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]): -# """ -# Params: -# new_: will be added to the database -# removed_: will be removed from the database -# db_dirs_: will be used to remove tracks that -# are outside these directories from the database and store. -# """ -# sdb.remove_root_dirs(removed_) -# sdb.add_root_dirs(new_) -# rebuild_store(db_dirs_) - - class AddRootDirsBody(BaseModel): new_dirs: list[str] = Field( description="The new directories to add", @@ -151,18 +88,6 @@ def get_root_dirs(): return {"dirs": UserConfig().rootDirs} -# maps settings to their parser flags -# mapp = { -# "artist_separators": SessionVarKeys.ARTIST_SEPARATORS, -# "extract_feat": SessionVarKeys.EXTRACT_FEAT, -# "remove_prod": SessionVarKeys.REMOVE_PROD, -# "clean_album_title": SessionVarKeys.CLEAN_ALBUM_TITLE, -# "remove_remaster": SessionVarKeys.REMOVE_REMASTER_FROM_TRACK, -# "merge_albums": SessionVarKeys.MERGE_ALBUM_VERSIONS, -# "show_albums_as_singles": SessionVarKeys.SHOW_ALBUMS_AS_SINGLES, -# } - - @api.get("") def get_all_settings(): """ @@ -176,11 +101,6 @@ def get_all_settings(): return config -# @background -# def reload_all_for_set_setting(): -# reload_everything(get_random_str()) - - class SetSettingBody(BaseModel): key: str = Field( description="The setting key", @@ -192,39 +112,6 @@ class SetSettingBody(BaseModel): ) -# @api.post("/set") -# @admin_required() -# def set_setting(body: SetSettingBody): -# """ -# Set a setting. -# """ -# key = body.key -# value = body.value - -# if key is None or value is None or key == "root_dirs": -# return {"msg": "Invalid arguments!"}, 400 - -# root_dir = sdb.get_root_dirs() - -# if not root_dir: -# return {"msg": "No root directories set!"}, 400 - -# if key not in mapp: -# return {"msg": "Invalid key!"}, 400 - -# if key == "artist_separators": -# value = str(value).split(",") -# value = set(value) - -# reload_all_for_set_setting() - -# # if value is a set, convert it to a string -# # (artist_separators) -# if type(value) == set: -# value = ",".join(value) - -# return {"result": value} - @api.get("/trigger-scan") def trigger_scan(): """ @@ -256,7 +143,20 @@ def update_config(body: UpdateConfigBody): body.value = body.value.split(",") setattr(config, body.key, body.value) - print(getattr(config, body.key)) + + # INFO: Rebuild stores when these settings are updated + reset_stores_lists = { + "artistSeparators", + "artistSplitIgnoreList", + "removeProdBy", + "removeRemasterInfo", + "mergeAlbums", + "cleanAlbumTitle", + "showAlbumsAsSingles", + } + + if body.key in reset_stores_lists: + index_everything() return { "msg": "Config updated!", diff --git a/app/api/stream.py b/app/api/stream.py index db6aada2..b876d5d3 100644 --- a/app/api/stream.py +++ b/app/api/stream.py @@ -10,8 +10,7 @@ from pydantic import BaseModel, Field from app.api.apischemas import TrackHashSchema from app.lib.trackslib import get_silence_paddings -# from app.store.tracks import TrackStore -from app.db.libdata import TrackTable +from app.store.tracks import TrackStore from app.utils.files import guess_mime_type bp_tag = Tag(name="File", description="Audio files") @@ -35,10 +34,26 @@ def send_track_file_legacy(path: TrackHashSchema, query: SendTrackFileQuery): filepath = query.filepath msg = {"msg": "File Not Found"} - track = TrackTable.get_track_by_trackhash(trackhash, filepath) - track_exists = track is not None and os.path.exists(track.filepath) + track = None + tracks = TrackStore.get_tracks_by_filepaths([filepath]) - if track_exists: + + if len(tracks) > 0 and os.path.exists(filepath): + track = tracks[0] + else: + res = TrackStore.trackhashmap.get(trackhash) + + # When finding by trackhash, sort by bitrate + # and get the first track that exists + if res is not None: + tracks = sorted(res.tracks, key=lambda x: x.bitrate, reverse=True) + + for t in tracks: + if os.path.exists(t.filepath): + track = t + break + + if track is not None: audio_type = guess_mime_type(filepath) return send_file(filepath, mimetype=audio_type, conditional=True) @@ -57,10 +72,25 @@ def send_track_file(path: TrackHashSchema, query: SendTrackFileQuery): msg = {"msg": "File Not Found"} # If filepath is provided, try to send that - track = TrackTable.get_track_by_trackhash(trackhash, filepath) - track_exists = track is not None and os.path.exists(track.filepath) + track = None + tracks = TrackStore.get_tracks_by_filepaths([filepath]) - if track_exists: + if len(tracks) > 0 and os.path.exists(filepath): + track = tracks[0] + else: + res = TrackStore.trackhashmap.get(trackhash) + + # When finding by trackhash, sort by bitrate + # and get the first track that exists + if res is not None: + tracks = sorted(res.tracks, key=lambda x: x.bitrate, reverse=True) + + for t in tracks: + if os.path.exists(t.filepath): + track = t + break + + if track is not None: audio_type = guess_mime_type(filepath) return send_file_as_chunks(track.filepath, audio_type) diff --git a/app/config.py b/app/config.py index d49be130..b365527c 100644 --- a/app/config.py +++ b/app/config.py @@ -22,6 +22,13 @@ class UserConfig: rootDirs: list[str] = field(default_factory=list) excludeDirs: list[str] = field(default_factory=list) artistSeparators: set[str] = field(default_factory=lambda: {";", "/"}) + artistSplitIgnoreList: set[str] = field( + default_factory=lambda: { + "AC/DC", + "Bob marley & the wailers", + "Crosby, Stills, Nash & Young", + } + ) genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) # tracks diff --git a/app/db/libdata.py b/app/db/libdata.py index c6db3f15..c5f76a59 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -109,10 +109,10 @@ class TrackTable(Base): id: Mapped[int] = mapped_column(init=False, primary_key=True) album: Mapped[str] = mapped_column(String()) - albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) + albumartists: Mapped[str] = mapped_column(String()) albumhash: Mapped[str] = mapped_column(String(), index=True) - artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) - artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + # artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + artists: Mapped[str] = mapped_column(String()) bitrate: Mapped[int] = mapped_column(Integer()) copyright: Mapped[Optional[str]] = mapped_column(String()) date: Mapped[int] = mapped_column(Integer(), nullable=True) @@ -120,11 +120,11 @@ class TrackTable(Base): duration: Mapped[int] = mapped_column(Integer()) filepath: Mapped[str] = mapped_column(String(), index=True, unique=True) folder: Mapped[str] = mapped_column(String(), index=True) - genrehashes: Mapped[list[str]] = mapped_column(JSON(), index=True) - genres: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) + # genrehashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + genres: Mapped[Optional[str]] = mapped_column(String()) last_mod: Mapped[float] = mapped_column(Integer()) - og_album: Mapped[str] = mapped_column(String()) - og_title: Mapped[str] = mapped_column(String()) + # og_album: Mapped[str] = mapped_column(String()) + # og_title: Mapped[str] = mapped_column(String()) title: Mapped[str] = mapped_column(String()) track: Mapped[int] = mapped_column(Integer()) trackhash: Mapped[str] = mapped_column(String(), index=True) @@ -250,6 +250,19 @@ class TrackTable(Base): TrackTable, TrackTable.trackhash, trackhash, duration, timestamp ) + # @classmethod + # def update_artist_separators(cls, separators: set[str]): + # tracks = cls.get_all() + + # with DbEngine.manager(commit=True) as conn: + # for track in tracks: + # track.split_artists(separators) + # conn.execute( + # update(cls) + # .where(cls.trackhash == track.trackhash) + # .values(artists=track.artists, artisthashes=track.artisthashes) + # ) + class AlbumTable(Base): __tablename__ = "album" diff --git a/app/db/utils.py b/app/db/utils.py index 550a7537..09cf49eb 100644 --- a/app/db/utils.py +++ b/app/db/utils.py @@ -1,5 +1,6 @@ from typing import Any +from app.config import UserConfig from app.models import Album as AlbumModel, Artist as ArtistModel, Track as TrackModel from app.models.favorite import Favorite from app.models.lastfm import SimilarArtist @@ -9,12 +10,12 @@ from app.models.plugins import Plugin from app.models.user import User -def track_to_dataclass(track: Any): - return TrackModel(**track._asdict()) +def track_to_dataclass(track: Any, config: UserConfig): + return TrackModel(**track._asdict(), config=config) def tracks_to_dataclasses(tracks: Any): - return [track_to_dataclass(track) for track in tracks] + return [track_to_dataclass(track, UserConfig()) for track in tracks] def album_to_dataclass(album: Any): diff --git a/app/lib/tagger.py b/app/lib/tagger.py index fab4f718..92efbf81 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -124,7 +124,7 @@ class IndexTracks: log.warning("'Populate.tag_untagged': Populate key changed") return - tags = get_tags(file, artist_separators=config.artistSeparators) + tags = get_tags(file, config=config) if tags is not None: TrackTable.insert_one(tags) diff --git a/app/lib/taglib.py b/app/lib/taglib.py index e0a00ae1..96fce4f3 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -14,13 +14,7 @@ from tinytag import TinyTag from app.config import UserConfig from app.settings import Defaults, Paths from app.utils.hashing import create_hash -from app.utils.parsers import ( - clean_title, - get_base_title_and_versions, - parse_feat_from_title, - remove_prod, - split_artists, -) +from app.utils.parsers import split_artists from app.utils.wintools import win_replace_slash @@ -109,13 +103,13 @@ def clean_filename(filename: str): class ParseData: artist: str title: str - artist_separators: set[str] + config: UserConfig def __post_init__(self): - self.artist = split_artists(self.artist, self.artist_separators) + self.artist = split_artists(self.artist, self.config) -def extract_artist_title(filename: str, artist_separators: set[str]): +def extract_artist_title(filename: str, config: UserConfig): path = Path(filename).with_suffix("") path = clean_filename(str(path)) @@ -123,24 +117,30 @@ def extract_artist_title(filename: str, artist_separators: set[str]): split_result = [x.strip() for x in split_result] if len(split_result) == 1: - return ParseData("", split_result[0], artist_separators) + return ParseData( + "", + split_result[0], + config, + ) if len(split_result) > 2: try: int(split_result[0]) return ParseData( - split_result[1], " - ".join(split_result[2:]), artist_separators + split_result[1], + " - ".join(split_result[2:]), + config, ) except ValueError: pass artist = split_result[0] title = split_result[1] - return ParseData(artist, title, artist_separators) + return ParseData(artist, title, config) -def get_tags(filepath: str, artist_separators: set[str]): +def get_tags(filepath: str, config: UserConfig): """ Returns the tags for a given audio file. """ @@ -173,17 +173,20 @@ def get_tags(filepath: str, artist_separators: set[str]): for tag in to_filename: p = getattr(tags, tag) if p == "" or p is None: - parse_data = extract_artist_title(filename, artist_separators) - title = parse_data.title + parse_data = extract_artist_title(filename, config) + title = parse_data.title.replace("_", " ") setattr(tags, tag, title) + # tags.title = tags.title.replace("_", " ") + # tags.album = tags.album.replace("_", " ") + parse = ["artist", "albumartist"] for tag in parse: p = getattr(tags, tag) if p == "" or p is None: if not parse_data: - parse_data = extract_artist_title(filename, artist_separators) + parse_data = extract_artist_title(filename, config) artist = parse_data.artist @@ -229,112 +232,110 @@ def get_tags(filepath: str, artist_separators: set[str]): tags.artists = tags.artist tags.albumartists = tags.albumartist - split_artist = split_artists(tags.artist, separators=artist_separators) - split_albumartists = split_artists(tags.albumartist, separators=artist_separators) - new_title = tags.title + # split_artist = split_artists(tags.artist, separators=config.artistSeparators) + # split_albumartists = split_artists(tags.albumartist, separators=config.artistSeparators) + # new_title = tags.title # TODO: Figure out which is the best spot to create these hashes # create albumhash using og_album tags.albumhash = create_hash(tags.album or "", tags.albumartist) - config = UserConfig() - # extract featured artists - if config.extractFeaturedArtists: - feat, new_title = parse_feat_from_title( - tags.title, separators=artist_separators - ) - original_lower = "-".join([create_hash(a) for a in split_artist]) - split_artist.extend(a for a in feat if create_hash(a) not in original_lower) + # if config.extractFeaturedArtists: + # feat, new_title = parse_feat_from_title( + # tags.title, separators=config.artistSeparators + # ) + # original_lower = "-".join([create_hash(a) for a in split_artist]) + # split_artist.extend(a for a in feat if create_hash(a) not in original_lower) # if no albumartist, assign to the first artist if not tags.albumartist: - tags.albumartist = split_artist[:1] + tags.albumartist = split_artists(tags.artist, config)[:1] # create json objects for artists and albumartists - tags.artists = [ - { - "artisthash": create_hash(a, decode=True), - "name": a, - } - for a in split_artist - ] + # tags.artists = [ + # { + # "artisthash": create_hash(a, decode=True), + # "name": a, + # } + # for a in split_artist + # ] - tags.albumartists = [ - { - "artisthash": create_hash(a, decode=True), - "name": a, - } - for a in split_albumartists - ] + # tags.albumartists = [ + # { + # "artisthash": create_hash(a, decode=True), + # "name": a, + # } + # for a in split_albumartists + # ] - tags.artisthashes = list( - {a["artisthash"] for a in tags.artists} - ) + # tags.artisthashes = list( + # {a["artisthash"] for a in tags.artists} + # ) # remove prod by - if config.removeProdBy: - new_title = remove_prod(new_title) + # if config.removeProdBy: + # new_title = remove_prod(new_title) # if track is a single, ie. # if og_title == album, rename album to new_title - if tags.title == tags.album: - tags.album = new_title + # if tags.title == tags.album: + # tags.album = new_title # remove remaster from track title - if config.removeRemasterInfo: - new_title = clean_title(new_title) + # if config.removeRemasterInfo: + # new_title = clean_title(new_title) # save final title - tags.og_title = tags.title - tags.title = new_title - tags.og_album = tags.album + # tags.og_title = tags.title + # tags.title = new_title + # tags.og_album = tags.album # clean album title - if config.cleanAlbumTitle: - tags.album, _ = get_base_title_and_versions(tags.album, get_versions=False) + # if config.cleanAlbumTitle: + # tags.album, _ = get_base_title_and_versions(tags.album, get_versions=False) # merge album versions - if config.mergeAlbums: - tags.albumhash = create_hash( - tags.album, *(a["name"] for a in tags.albumartists) - ) + # if config.mergeAlbums: + # tags.albumhash = create_hash( + # tags.album, *(a["name"] for a in tags.albumartists) + # ) # process genres - if tags.genre: - src_genres: str = tags.genre - src_genres = src_genres.lower() - # separators = {"/", ";", "&"} - separators = set(config.genreSeparators) + # if tags.genre: + # src_genres: str = tags.genre + # src_genres = src_genres.lower() + # # separators = {"/", ";", "&"} + # separators = set(config.genreSeparators) - contains_rnb = "r&b" in src_genres - contains_rock = "rock & roll" in src_genres + # contains_rnb = "r&b" in src_genres + # contains_rock = "rock & roll" in src_genres - if contains_rnb: - src_genres = src_genres.replace("r&b", "RnB") + # if contains_rnb: + # src_genres = src_genres.replace("r&b", "RnB") - if contains_rock: - src_genres = src_genres.replace("rock & roll", "rock") + # if contains_rock: + # src_genres = src_genres.replace("rock & roll", "rock") - for s in separators: - src_genres = src_genres.replace(s, ",") + # for s in separators: + # src_genres = src_genres.replace(s, ",") - genres_list: list[str] = src_genres.split(",") - tags.genres = [ - {"name": g.strip(), "genrehash": create_hash(g.strip())} - for g in genres_list - ] - tags.genrehashes = [g["genrehash"] for g in tags.genres] - else: - tags.genres = [] - tags.genrehashes = [] + # genres_list: list[str] = src_genres.split(",") + # tags.genres = [ + # {"name": g.strip(), "genrehash": create_hash(g.strip())} + # for g in genres_list + # ] + # tags.genrehashes = [g["genrehash"] for g in tags.genres] + # else: + # tags.genres = [] + # tags.genrehashes = [] + + tags.genres = tags.genre # sub underscore with space - tags.title = tags.title.replace("_", " ") - tags.album = tags.album.replace("_", " ") - tags.trackhash = create_hash( - *[a["name"] for a in tags.artists], tags.album, tags.title - ) + # tags.title = tags.title.replace("_", " ") + # tags.album = tags.album.replace("_", " ") + tags.trackhash = create_hash(tags.artists, tags.album, tags.title) more_extra = { "audio_offset": tags.audio_offset, diff --git a/app/models/track.py b/app/models/track.py index 37436105..e9f7710f 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -1,6 +1,15 @@ from dataclasses import dataclass, field +from app.config import UserConfig from app.utils.auth import get_current_userid +from app.utils.hashing import create_hash +from app.utils.parsers import ( + clean_title, + get_base_title_and_versions, + parse_feat_from_title, + remove_prod, + split_artists, +) @dataclass(slots=True) @@ -13,7 +22,6 @@ class Track: album: str albumartists: list[dict[str, str]] albumhash: str - artisthashes: list[str] artists: list[dict[str, str]] bitrate: int copyright: str @@ -22,11 +30,8 @@ class Track: duration: int filepath: str folder: str - genres: list[dict[str, str]] - genrehashes: list[str] + genres: str | list[dict[str, str]] last_mod: int - og_album: str - og_title: str title: str track: int trackhash: str @@ -35,6 +40,12 @@ class Track: playcount: int playduration: int + config: UserConfig + og_album: str = "" + og_title: str = "" + artisthashes: list[str] = field(default_factory=list) + genrehashes: list[str] = field(default_factory=list) + _pos: int = 0 _ati: str = "" image: str = "" @@ -55,9 +66,118 @@ class Track: self.fav_userids.append(userid) def __post_init__(self): + self.og_title = self.title + self.og_album = self.album + self.image = self.albumhash + ".webp" self.extra = { "disc_total": self.extra.get("disc_total", 0), "track_total": self.extra.get("track_total", 0), "samplerate": self.extra.get("samplerate", -1), } + + self.split_artists() + self.map_with_config() + self.process_genres() + + # Remove duplicates from artists and albumartists + seen_artists = set() + self.artists = [ + d + for d in self.artists + if tuple(d.items()) not in seen_artists + and not seen_artists.add(tuple(d.items())) + ] + + seen_albumartists = set() + self.albumartists = [ + d + for d in self.albumartists + if tuple(d.items()) not in seen_albumartists + and not seen_albumartists.add(tuple(d.items())) + ] + + self.config = None + + def split_artists(self): + """ + Splits the artists and albumartists based on the given separators, and updates the artisthashes. + """ + + def split(artists: str): + return [ + {"name": a, "artisthash": create_hash(a, decode=True)} + for a in split_artists(artists, config=self.config) + ] + + self.artists = split(self.artists) + self.albumartists = split(self.albumartists) + self.artisthashes = [a["artisthash"] for a in self.artists] + + def map_with_config(self): + new_title = self.title + + # Extract featured artists + if self.config.extractFeaturedArtists: + feat, new_title = parse_feat_from_title(self.title, self.config) + feat = [ + {"name": f, "artisthash": create_hash(f, decode=True)} for f in feat + ] + feat = [f for f in feat if f["artisthash"] not in self.artisthashes] + self.artists.extend(feat) + self.artisthashes.extend([f["artisthash"] for f in feat]) + + # Update album title for singles + # ie. album: "Title (feat. Artist)" + # title: "Title (feat. Artist)" + # becomes: album: "Title", title: "Title" + if self.og_album == self.og_title: + self.album = new_title + + # Clean track title + if self.config.removeProdBy: + new_title = remove_prod(new_title) + + # if self.title == new_title: + # self.album = new_title + + if self.config.removeRemasterInfo: + new_title = clean_title(new_title) + + self.title = new_title + + # Clean album title + if self.config.cleanAlbumTitle: + self.album, _ = get_base_title_and_versions(self.album, get_versions=False) + + if self.config.mergeAlbums: + self.albumhash = create_hash( + self.album, *(a["name"] for a in self.albumartists) + ) + + def process_genres(self): + if self.genres: + src_genres: str = self.genres + + src_genres = src_genres.lower() + # separators = {"/", ";", "&"} + separators = set(self.config.genreSeparators) + + contains_rnb = "r&b" in src_genres + contains_rock = "rock & roll" in src_genres + + if contains_rnb: + src_genres = src_genres.replace("r&b", "RnB") + + if contains_rock: + src_genres = src_genres.replace("rock & roll", "rock") + + for s in separators: + src_genres = src_genres.replace(s, ",") + + genres_list: list[str] = src_genres.split(",") + self.genres = [ + {"name": g.strip(), "genrehash": create_hash(g.strip())} + for g in genres_list + ] + self.genrehashes = [g["genrehash"] for g in self.genres] diff --git a/app/store/tracks.py b/app/store/tracks.py index 21e56759..86f8c1b0 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -215,8 +215,6 @@ class TrackStore: def get_tracks_by_filepaths(cls, paths: list[str]) -> list[Track]: """ Returns all tracks matching the given paths. - - ⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔⛔ """ # tracks = sorted(cls.trackhashmap, key=lambda x: x.filepath) # tracks = use_bisection(tracks, "filepath", paths) diff --git a/app/utils/parsers.py b/app/utils/parsers.py index f45c871f..5959d50c 100644 --- a/app/utils/parsers.py +++ b/app/utils/parsers.py @@ -1,19 +1,54 @@ import re +from app.config import UserConfig from app.enums.album_versions import AlbumVersionEnum, get_all_keywords -def split_artists(src: str, separators: set[str]): +def split_artists(src: str, config: UserConfig): """ - Splits a string of artists into a list of artists. + Splits a string of artists into a list of artists, preserving those in ignoreList. + Case-insensitive matching is used for the ignoreList. """ - for sep in separators: - src = src.replace(sep, ",") + result = [] + current = "" + i = 0 - artists = src.split(",") - artists = [a.strip() for a in artists] + while i < len(src): + # Check if any ignored artist starts at this position (case-insensitive) + ignored_match = next( + ( + src[i : i + len(ignored)] + for ignored in config.artistSplitIgnoreList + if src.lower().startswith(ignored.lower(), i) + ), + None, + ) - return [a for a in artists if a] + if ignored_match: + # If we have accumulated any current string, add it to result + if current.strip(): + result.extend([a.strip() for a in current.split(",") if a.strip()]) + current = "" + # Add the ignored artist to the result (preserving original case) + result.append(ignored_match) + # Move past the ignored artist + i += len(ignored_match) + elif src[i] in config.artistSeparators: + # If we encounter a separator, process the current string + if current.strip(): + result.extend([a.strip() for a in current.split(",") if a.strip()]) + current = "" + i += 1 + else: + # If it's not an ignored artist or a separator, add to current + current += src[i] + i += 1 + + # Process any remaining current string + if current.strip(): + result.extend([a.strip() for a in current.split(",") if a.strip()]) + + return result def remove_prod(title: str) -> str: @@ -36,7 +71,7 @@ def remove_prod(title: str) -> str: return title.strip() -def parse_feat_from_title(title: str, separators: set[str]) -> tuple[list[str], str]: +def parse_feat_from_title(title: str, config: UserConfig) -> tuple[list[str], str]: """ Extracts featured artists from a song title using regex. """ @@ -54,7 +89,7 @@ def parse_feat_from_title(title: str, separators: set[str]) -> tuple[list[str], return [], title artists = match.group(1) - artists = split_artists(artists, separators) + artists = split_artists(artists, config) # remove "feat" group from title new_title = re.sub(regex, "", title, flags=re.IGNORECASE) diff --git a/tests/test_split_artists.py b/tests/test_split_artists.py new file mode 100644 index 00000000..a0e55411 --- /dev/null +++ b/tests/test_split_artists.py @@ -0,0 +1,137 @@ +import unittest + +def split_artists(src: str, separators: set[str], ignoreList: set[str] = set()): + """ + Splits a string of artists into a list of artists, preserving those in ignoreList. + Case-insensitive matching is used for the ignoreList. + """ + result = [] + current = "" + i = 0 + + # Convert ignoreList to lowercase for case-insensitive matching + ignore_lower = {artist.lower() for artist in ignoreList} + + while i < len(src): + # Check if any ignored artist starts at this position (case-insensitive) + ignored_match = next( + ( + src[i:i+len(ignored)] + for ignored in ignoreList + if src.lower().startswith(ignored.lower(), i) + ), + None + ) + + if ignored_match: + # If we have accumulated any current string, add it to result + if current.strip(): + result.extend([a.strip() for a in current.split(',') if a.strip()]) + current = "" + # Add the ignored artist to the result (preserving original case) + result.append(ignored_match) + # Move past the ignored artist + i += len(ignored_match) + elif src[i] in separators: + # If we encounter a separator, process the current string + if current.strip(): + result.extend([a.strip() for a in current.split(',') if a.strip()]) + current = "" + i += 1 + else: + # If it's not an ignored artist or a separator, add to current + current += src[i] + i += 1 + + # Process any remaining current string + if current.strip(): + result.extend([a.strip() for a in current.split(',') if a.strip()]) + + return result + + +class TestSplitArtists(unittest.TestCase): + + def test_basic_splitting(self): + self.assertEqual( + split_artists("Beatles, Queen; Rolling Stones", {";"}), + ["Beatles", "Queen", "Rolling Stones"], + ) + + def test_multiple_separators(self): + self.assertEqual( + split_artists("Beatles; Queen & Rolling Stones | ABBA", {";", "&", "|"}), + ["Beatles", "Queen", "Rolling Stones", "ABBA"], + ) + + def test_ignore_list(self): + self.assertEqual( + split_artists( + "Beatles; Earth, Wind & Fire; Queen", {";", "&"}, {"Earth, Wind & Fire"} + ), + ["Beatles", "Earth, Wind & Fire", "Queen"], + ) + + def test_empty_string(self): + self.assertEqual(split_artists("", {";"}), []) + + def test_only_separators(self): + self.assertEqual(split_artists(";;;", {";"}), []) + + def test_extra_spaces(self): + self.assertEqual( + split_artists(" Beatles ; Queen ", {";"}), ["Beatles", "Queen"] + ) + + def test_comma_splitting(self): + self.assertEqual( + split_artists("Beatles, Queen; Rolling Stones, ABBA", {";"}), + ["Beatles", "Queen", "Rolling Stones", "ABBA"], + ) + + def test_ignore_list_with_comma(self): + self.assertEqual( + split_artists( + "Beatles; Earth, Wind & Fire, Queen", {";"}, {"Earth, Wind & Fire"} + ), + ["Beatles", "Earth, Wind & Fire", "Queen"], + ) + + def test_ignore_list_with_separator(self): + self.assertEqual( + split_artists("Beatles; AC/DC", {"/", ";"}, {"AC/DC"}), ["Beatles", "AC/DC"] + ) + + def test_ignore_list_at_start(self): + self.assertEqual( + split_artists("AC/DC; Beatles", {"/", ";"}, {"AC/DC"}), ["AC/DC", "Beatles"] + ) + + def test_ignore_list_at_end(self): + self.assertEqual( + split_artists("Beatles; AC/DC", {"/", ";"}, {"AC/DC"}), ["Beatles", "AC/DC"] + ) + + def test_multiple_ignored_artists(self): + self.assertEqual( + split_artists( + "Beatles; AC/DC; Guns N' Roses; Queen", + {"/", ";", "'"}, + {"AC/DC", "Guns N' Roses"}, + ), + ["Beatles", "AC/DC", "Guns N' Roses", "Queen"], + ) + + def test_bob_marley(self): + self.assertEqual( + split_artists( + "Bob marley & The wailers; Beatles", + {";", "&"}, + {"Bob marley & the wailers"}, + ), + ["Bob marley & The wailers", "Beatles"], + ) + + +if __name__ == "__main__": + unittest.main() From ca31054f488d0b156edafff77850f746b59c2612 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 15 Aug 2024 17:07:34 +0300 Subject: [PATCH 40/44] fix: folder endpoint returning same track for different files of the same trackhash + fix: chunked streaming. return instead of yield chunks --- TODO.md | 2 +- app/api/auth.py | 6 +- app/api/stream.py | 186 ++++++++++++++++++++++++++++++++++++++---- app/db/userdata.py | 4 +- app/lib/tagger.py | 50 ++++++++++-- app/lib/taglib.py | 3 +- app/lib/transcoder.py | 78 ++++++++++++++++++ app/lib/watchdogg.py | 95 +++++++++++++-------- app/models/track.py | 30 ++++++- app/store/albums.py | 40 +++++---- app/store/artists.py | 103 ++++++++++++----------- app/store/folder.py | 10 ++- app/utils/__init__.py | 5 +- docs/README.md | 0 docs/streaming.md | 3 + docs/watchdog.md | 0 poetry.lock | 30 ++++++- pyproject.toml | 1 + 18 files changed, 508 insertions(+), 138 deletions(-) create mode 100644 app/lib/transcoder.py create mode 100644 docs/README.md create mode 100644 docs/streaming.md create mode 100644 docs/watchdog.md diff --git a/TODO.md b/TODO.md index ab4ebd22..6edebdd1 100644 --- a/TODO.md +++ b/TODO.md @@ -49,4 +49,4 @@ - Duplicates on search - Audio stops on ending - Show users on account settings when logged in as admin and show users on login is disabled. -- \ No newline at end of file +- Save both filepath and trackhash in favorites and playlists \ No newline at end of file diff --git a/app/api/auth.py b/app/api/auth.py index de36e78a..ff593452 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -108,14 +108,14 @@ class PairDeviceQuery(BaseModel): code: str = Field("", description="The code") -@api.post("/pair") +@api.get("/pair") @jwt_required(optional=True) -def pair_with_code(body: PairDeviceQuery): +def pair_with_code(query: PairDeviceQuery): """ Get an access token by sending a pair code. NOTE: A code can only be used once! """ global pair_token - token = pair_token.get(body.code, None) + token = pair_token.get(query.code, None) if token: pair_token = {} diff --git a/app/api/stream.py b/app/api/stream.py index b876d5d3..69bb6240 100644 --- a/app/api/stream.py +++ b/app/api/stream.py @@ -3,12 +3,16 @@ Contains all the track routes. """ import os +import tempfile +import time +from typing import Literal from flask import send_file, request, Response from flask_openapi3 import APIBlueprint, Tag from pydantic import BaseModel, Field from app.api.apischemas import TrackHashSchema from app.lib.trackslib import get_silence_paddings +from app.lib.transcoder import start_transcoding from app.store.tracks import TrackStore from app.utils.files import guess_mime_type @@ -17,10 +21,36 @@ bp_tag = Tag(name="File", description="Audio files") api = APIBlueprint("track", __name__, url_prefix="/file", abp_tags=[bp_tag]) +class TransCodeStore: + map: dict[str, str] = {} + + @classmethod + def add_file(cls, trackhash: str, filepath: str): + cls.map[trackhash] = filepath + + @classmethod + def remove_file(cls, trackhash: str): + del cls.map[trackhash] + + @classmethod + def find(cls, trackhash: str): + return cls.map.get(trackhash) + + class SendTrackFileQuery(BaseModel): filepath: str = Field( description="The filepath to play (if available)", default=None ) + quality: Literal["original", "1411", "800", "600", "320", "256", "128", "96"] = ( + Field( + "320", + description="The quality of the audio file. Options: original, 1411, 1024, 512, 320, 256, 128, 96", + ) + ) + container: Literal["mp3", "aac", "flac", "webm", "ogg"] = Field( + "flac", + description="The container format of the audio file. Options: mp3, aac, flac, webm, ogg", + ) @api.get("//legacy") @@ -29,6 +59,8 @@ def send_track_file_legacy(path: TrackHashSchema, query: SendTrackFileQuery): Get a playable audio file without Range support Returns a playable audio file that corresponds to the given filepath. Falls back to track hash if filepath is not found. + + NOTE: Does not support range requests or transcoding. """ trackhash = path.trackhash filepath = query.filepath @@ -37,7 +69,6 @@ def send_track_file_legacy(path: TrackHashSchema, query: SendTrackFileQuery): track = None tracks = TrackStore.get_tracks_by_filepaths([filepath]) - if len(tracks) > 0 and os.path.exists(filepath): track = tracks[0] else: @@ -66,10 +97,17 @@ def send_track_file(path: TrackHashSchema, query: SendTrackFileQuery): Get a playable audio file with Range headers support Returns a playable audio file that corresponds to the given filepath. Falls back to track hash if filepath is not found. + + Transcoding can be done by sending the quality and container query parameters. + + **NOTES:** + - Transcoded streams report incorrect duration during playback (idk why! FFMPEG gurus we need your help here). + - The quality parameter is the desired bitrate in kbps. + - The mp3 container is the best container for upto 320kbps (and has better duration reporting). The flac container allows for higher bitrates but it produces dramatically larger files (when transcoding from lossy formats). + - You can get the transcoded bitrate by checking the X-Transcoded-Bitrate header on the first request's response. """ trackhash = path.trackhash filepath = query.filepath - msg = {"msg": "File Not Found"} # If filepath is provided, try to send that track = None @@ -91,13 +129,87 @@ def send_track_file(path: TrackHashSchema, query: SendTrackFileQuery): break if track is not None: - audio_type = guess_mime_type(filepath) - return send_file_as_chunks(track.filepath, audio_type) + if query.quality == "original": + return send_file_as_chunks(track.filepath) - return msg, 404 + # prevent requesting over transcoding + max_bitrate = track.bitrate + requested_bitrate = int(query.quality) + + if query.container != "flac": + # drop to 320 for non-flac containers + requested_bitrate = min(320, requested_bitrate) + + quality = f"{min(max_bitrate, requested_bitrate)}k" + return transcode_and_stream(trackhash, track.filepath, quality, query.container) + + return {"msg": "File Not Found"}, 404 -def send_file_as_chunks(filepath: str, audio_type: str) -> Response: +def transcode_and_stream(trackhash: str, filepath: str, bitrate: str, container: str): + """ + Initiates transcoding and returns the first chunk of the transcoded file. + + The other chunks are streamed on subsequent requests and are rerouted to `send_file_as_chunks`. + """ + temp_file = TransCodeStore.find(trackhash) + if temp_file is not None: + return send_file_as_chunks(temp_file) + + format_params = { + "mp3": ["-c:a", "libmp3lame"], + "aac": ["-c:a", "aac"], + "webm": ["-c:a", "libopus"], + "ogg": ["-c:a", "libvorbis"], + "flac": ["-c:a", "flac"], + "wav": ["-c:a", "pcm_s16le"], + } + + # Create a temporary file + format = f".{container}" if container in format_params.keys() else ".flac" + container_args = ( + format_params[container] + if container in format_params.keys() + else format_params["flac"] + ) + temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=format) + temp_filename = temp_file.name + temp_file.close() + + TransCodeStore.add_file(trackhash, temp_filename) + start_transcoding(filepath, temp_filename, bitrate, container_args) + + chunk_size = 1024 * 512 # 0.5MB + file_size = os.path.getsize(filepath) + + def generate(): + # Poll for the output file + while ( + not os.path.exists(temp_filename) + or os.path.getsize(temp_filename) < chunk_size + ): + print(f"Waiting for transcoding to complete... filename: {temp_filename}") + time.sleep(0.1) # Wait for 100ms before checking again + + with open(temp_filename, "rb") as file: + file.seek(0) + return file.read(chunk_size) + + audio_type = guess_mime_type(temp_filename) + response = Response( + generate(), + 206, + mimetype=audio_type, + content_type=audio_type, + direct_passthrough=True, + ) + response.headers.add("Content-Range", f"bytes {0}-{chunk_size}/{file_size}") + response.headers.add("Accept-Ranges", "bytes") + response.headers.add("X-Transcoded-Bitrate", bitrate) + return response + + +def send_file_as_chunks(filepath: str) -> Response: """ Returns a Response object that streams the file in chunks. """ @@ -129,25 +241,69 @@ def send_file_as_chunks(filepath: str, audio_type: str) -> Response: file.seek(start) remaining_bytes = end - start + 1 - while remaining_bytes > 0: + retry_count = 0 + max_retries = 10 # 5 * 100ms = 500ms total wait time + + while remaining_bytes > 0 or retry_count < max_retries: + if retry_count == max_retries: + print("💚 sending final chunk! ...") + return ( + file.read(os.path.getsize(filepath) - file.tell()), + file.tell(), + True, + ) + print("\n\n") + print(f"file: {filepath}") + print(f"start: {start}") + print(f"end: {end}") + print(f"filesize: {os.path.getsize(filepath)}") + print(f"⭐ (O) Remaining bytes: {remaining_bytes}") + print(f"⭐ Remaining bytes: {remaining_bytes}") + print(f"⭐ Cursor position: {file.tell()}") # Read the chunk size or all the remaining bytes + + print(f"💚 remaining_bytes: {remaining_bytes}") + print(f"💚 retry_count: {retry_count}") + + if remaining_bytes < chunk_size: + time.sleep(0.25) + retry_count += 1 + remaining_bytes = os.path.getsize(filepath) - file.tell() + continue + chunk = file.read(min(chunk_size, remaining_bytes)) - yield chunk + if chunk: + remaining_bytes -= len(chunk) + return chunk, file.tell(), False + else: + # If no data is read, wait for 100ms before retrying + time.sleep(0.25) + retry_count += 1 - # Update the remaining bytes - remaining_bytes -= len(chunk) + # update remaining bytes + remaining_bytes = os.path.getsize(filepath) - file.tell() + print(f"▶ Remaining bytes: {remaining_bytes}") + return None, 0, True + + data, position, is_final = generate_chunks() + + audio_type = guess_mime_type(filepath) response = Response( - generate_chunks(), - 206, # Partial Content status code + response=data, + status=206, # Partial Content status code mimetype=audio_type, content_type=audio_type, direct_passthrough=True, ) - response.headers.add("Content-Range", f"bytes {start}-{end}/{file_size}") - response.headers.add("Accept-Ranges", "bytes") - response.headers.add("Content-Length", str(end - start + 1)) + bytes_to_add = chunk_size if not is_final else 0 + response.headers.add( + "Content-Range", + f"bytes {start}-{position}/{os.path.getsize(filepath) + bytes_to_add}", + ) + response.headers.add("Accept-Ranges", "bytes") + response.headers.add("Content-Length", str(len(data or []))) return response diff --git a/app/db/userdata.py b/app/db/userdata.py index 0c5f13bc..2d33ff1e 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -1,5 +1,5 @@ import datetime -from typing import Any +from typing import Any, Literal from sqlalchemy import ( JSON, Boolean, @@ -409,7 +409,7 @@ class LibDataTable(Base): ) @classmethod - def find_one(cls, hash: str, type: str): + def find_one(cls, hash: str, type: Literal["album", "artist"]): result = cls.execute( select(cls).where((cls.itemhash == hash) & (cls.itemtype == type)) ) diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 92efbf81..4c12cce3 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -8,11 +8,14 @@ from app.models.album import Album from app.models.artist import Artist from app.models.track import Track from app.store.folder import FolderStore +from app.store.tracks import TrackStore +from app.utils import flatten from app.utils.filesystem import run_fast_scandir from app.utils.parsers import get_base_album_title from app.utils.progressbar import tqdm from app.logger import log +from app.utils.remove_duplicates import remove_duplicates POPULATE_KEY: float = 0 @@ -136,9 +139,27 @@ class IndexTracks: print("Done") -def create_albums(): +def create_albums(_trackhashes: list[str] = []) -> list[tuple[Album, set[str]]]: + """ + Creates album objects using the indexed tracks. Takes in an optional + list of trackhashes to create the albums from. If no list is provided, + all tracks are used. + + The trackhashes are passed when creating albums from the watchdogg module. + + Returns a list of tuples containing the album and the trackhashes in the album. + ie: + + >>> list[tuple[Album, set[str]]] + """ albums = dict() - all_tracks: list[Track] = TrackTable.get_all() + + if _trackhashes: + all_tracks: list[Track] = TrackStore.get_tracks_by_trackhashes(_trackhashes) + else: + all_tracks: list[Track] = TrackStore.get_flat_list() + + all_tracks = remove_duplicates(all_tracks) for track in all_tracks: if track.albumhash not in albums: @@ -192,9 +213,28 @@ def create_albums(): return list(albums.values()) -# class IndexArtists: -def create_artists(): - all_tracks: list[Track] = TrackTable.get_all() +def create_artists( + artisthashes: list[str] = [], +) -> list[tuple[Artist, set[str], set[str]]]: + """ + Creates artist objects using the indexed tracks. Takes in an optional + list of artisthashes to create the artists from. If no list is provided, + all tracks are used. + + Returns a list of tuples containing the artist, the trackhashes for the artist + and the albumhashes for the artist. + ie: + + >>> list[tuple[Artist, set[str], set[str]]] + """ + if artisthashes: + all_tracks: list[Track] = flatten( + [TrackStore.get_tracks_by_artisthash(hash) for hash in artisthashes] + ) + else: + all_tracks: list[Track] = TrackStore.get_flat_list() + + all_tracks = remove_duplicates(all_tracks) artists = dict() for track in all_tracks: diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 96fce4f3..820715f7 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +import math import os from io import BytesIO from pathlib import Path @@ -206,7 +207,7 @@ def get_tags(filepath: str, config: UserConfig): to_round = ["bitrate", "duration"] for prop in to_round: try: - setattr(tags, prop, round(getattr(tags, prop))) + setattr(tags, prop, math.floor(getattr(tags, prop))) except TypeError: setattr(tags, prop, 0) diff --git a/app/lib/transcoder.py b/app/lib/transcoder.py new file mode 100644 index 00000000..befb978c --- /dev/null +++ b/app/lib/transcoder.py @@ -0,0 +1,78 @@ +from app.utils.threading import background + + +import subprocess + + +@background +def start_transcoding( + input_path: str, output_path: str, bitrate: str, container_args: list[str], compression_level: int = 12 +): + """ + Starts a background transcoding process for an audio file. + + This function uses FFmpeg to transcode an audio file from one format to another, + with specified bitrate and container format. It runs as a background task. + + Args: + input_path (str): The path to the input audio file. + output_path (str): The path where the transcoded file will be saved. + bitrate (str): The desired bitrate for the output file (e.g., "128k"). + container_args (list[str]): FFmpeg arguments specific to the output container format. + compression_level (int): Compression level (0-9, default: 6). + + Returns: + None + + Note: + This function is decorated with @background, which means it runs asynchronously. + The actual transcoding process is handled by FFmpeg in a subprocess. + The function will print status messages about the transcoding process. + """ + # Base command + command = [ + "ffmpeg", + "-i", + input_path, + "-map_metadata", "0", # Add this line to copy metadata + "-b:a", + bitrate, + "-vn", + "-compression_level", + str(compression_level), + # REVIEW: Idk what any flag below this point does! + "-movflags", + "faststart+frag_keyframe+empty_moov", + "-write_xing", + "0", + "-fflags", + "+bitexact", + ] + + # Add format-specific parameters + command.extend(container_args) + + # Add output path and overwrite flag + command.extend([output_path, "-y"]) + + process = subprocess.Popen( + command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + print(f"Started transcoding process with PID: {process.pid}") + + try: + # Wait for the process to complete + process.wait() + print(f"Transcoding process (PID: {process.pid}) completed") + except KeyboardInterrupt: + print(f"Transcoding interrupted. Terminating process (PID: {process.pid})") + finally: + # Ensure the process is terminated + try: + process.terminate() + process.wait(timeout=5) # Wait up to 5 seconds for graceful termination + except subprocess.TimeoutExpired: + print( + f"Process (PID: {process.pid}) did not terminate gracefully. Killing..." + ) + process.kill() \ No newline at end of file diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py index 0b577596..b22bbbd1 100644 --- a/app/lib/watchdogg.py +++ b/app/lib/watchdogg.py @@ -8,17 +8,20 @@ import sqlite3 import time from watchdog.events import PatternMatchingEventHandler +from watchdog.observers.api import BaseObserverSubclassCallable from watchdog.observers import Observer from app import settings from app.config import UserConfig -from app.db.sqlite.tracks import SQLiteManager +from app.db.libdata import TrackTable +from app.db.userdata import LibDataTable from app.lib.colorlib import process_color +from app.lib.tagger import create_albums, create_artists from app.lib.taglib import extract_thumb, get_tags from app.logger import log from app.models import Artist, Track from app.store.albums import AlbumStore -from app.store.artists import ArtistStore +from app.store.artists import ArtistMapEntry, ArtistStore from app.store.tracks import TrackStore @@ -27,7 +30,7 @@ class Watcher: Contains the methods for initializing and starting watchdog. """ - observers: list[Observer] = [] + observers: list[BaseObserverSubclassCallable] = [] def __init__(self): self.observer = Observer() @@ -126,10 +129,10 @@ class Watcher: self.run() -def handle_colors(cur: sqlite3.Cursor, albumhash: str): - exists = aldb.exists(albumhash, cur) +def handle_color(albumhash: str): + entry = LibDataTable.find_one(albumhash, "album") - if exists: + if entry and entry.color: return colors = process_color(albumhash, is_album=True) @@ -137,7 +140,12 @@ def handle_colors(cur: sqlite3.Cursor, albumhash: str): if colors is None: return - aldb.insert_one_album(cur=cur, albumhash=albumhash, colors=json.dumps(colors)) + if entry is None: + LibDataTable.insert_one( + {"itemhash": albumhash, "color": colors[0], "itemtype": "album"} + ) + else: + LibDataTable.update_one(albumhash, {"color": colors[0]}) return colors @@ -152,38 +160,42 @@ def add_track(filepath: str) -> None: TrackStore.remove_track_by_filepath(filepath) config = UserConfig() - tags = get_tags(filepath, artist_separators=config.artistSeparators) + tags = get_tags(filepath, config) # if the track is somehow invalid, return if tags is None or tags["bitrate"] == 0 or tags["duration"] == 0: return - colors = None - - with SQLiteManager() as cur: - db.insert_one_track(tags, cur) - extracted = extract_thumb(filepath, tags["albumhash"] + ".webp") - - if not extracted: - return - - colors = handle_colors(cur, tags["albumhash"]) + TrackTable.insert_one(tags) + extract_thumb(filepath, tags["albumhash"] + ".webp", overwrite=True) + colors = handle_color(tags["albumhash"]) track = Track(**tags) TrackStore.add_track(track) - if not AlbumStore.album_exists(track.albumhash): - album = AlbumStore.create_album(track) - album.set_colors(colors) - AlbumStore.add_album(album) + # SECTION: Index album + albumentry = AlbumStore.albummap.get(track.albumhash) - artists: list[Artist] = track.artists + track.albumartists # type: ignore + if albumentry is None: + album, trackhashes = create_albums([track.trackhash])[0] + AlbumStore.index_new_album(album, trackhashes) + else: + trackhash_exists = track.trackhash in albumentry.trackhashes + + if not trackhash_exists: + albumentry.trackhashes.add(track.trackhash) + albumentry.album.trackcount += 1 + albumentry.set_color(colors[0]) if colors else None + + # SECTION: Index artist + artists = create_artists(track.artisthashes) for artist in artists: - if not ArtistStore.artist_exists(artist.artisthash): - ArtistStore.add_artist(Artist(artist.name)) - - extract_thumb(filepath, track.image, overwrite=True) + ArtistStore.artistmap[artist[0].artisthash] = ArtistMapEntry( + artist=artist[0], + albumhashes=artist[1], + trackhashes=artist[2], + ) def remove_track(filepath: str) -> None: @@ -287,16 +299,33 @@ class Handler(PatternMatchingEventHandler): NOT FIRED IN WINDOWS """ try: - self.files_to_process.remove(event.src_path) - if os.path.getsize(event.src_path) > 0: + # Get initial file size + initial_size = os.path.getsize(event.src_path) + + # Wait for 10 seconds + time.sleep(10) + + # Check if file size has changed + current_size = os.path.getsize(event.src_path) + + if current_size > 0 and current_size == initial_size: path = self.get_abs_path(event.src_path) add_track(path) + # Remove from processing list only after successful processing + self.files_to_process.remove(event.src_path) + else: + # File is still being modified or has been deleted + log.info( + f"File {event.src_path} is still being modified. Skipping processing for now." + ) except FileNotFoundError: # file was closed and deleted. - pass + log.info(f"File {event.src_path} was closed and deleted before processing.") except ValueError: - # file was removed from the list by another event handler. - pass + # file was already removed from the list by another event handler. + log.info( + f"File {event.src_path} was already removed from the processing list." + ) def on_modified(self, event): # this event handler is triggered twice on windows @@ -317,7 +346,7 @@ class Handler(PatternMatchingEventHandler): if current_size == previous_size: # Wait for a short duration to ensure the file write operation is complete - time.sleep(5) + time.sleep(10) # Check the file size again try: diff --git a/app/models/track.py b/app/models/track.py index e9f7710f..4f493f8c 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -57,8 +57,10 @@ class Track: def toggle_favorite_user(self, userid: int): """ - Adds or removes the given user from the list of users - who have favorited the track. + Toggles the favorite status of the track for a given user. + + Args: + userid (int): The ID of the user toggling the favorite status. """ if userid in self.fav_userids: self.fav_userids.remove(userid) @@ -66,6 +68,11 @@ class Track: self.fav_userids.append(userid) def __post_init__(self): + """ + Performs post-initialization processing on the track object. + This includes setting original values, processing artists and genres, + and removing duplicate artists. + """ self.og_title = self.title self.og_album = self.album @@ -97,11 +104,13 @@ class Track: and not seen_albumartists.add(tuple(d.items())) ] + self.recreate_trackhash() self.config = None def split_artists(self): """ - Splits the artists and albumartists based on the given separators, and updates the artisthashes. + Splits the artists and albumartists based on the given separators, + and updates the artisthashes. """ def split(artists: str): @@ -115,6 +124,10 @@ class Track: self.artisthashes = [a["artisthash"] for a in self.artists] def map_with_config(self): + """ + Applies various transformations to the track's title and album + based on the user's configuration settings. + """ new_title = self.title # Extract featured artists @@ -156,6 +169,9 @@ class Track: ) def process_genres(self): + """ + Processes and standardizes the genre information for the track. + """ if self.genres: src_genres: str = self.genres @@ -181,3 +197,11 @@ class Track: for g in genres_list ] self.genrehashes = [g["genrehash"] for g in self.genres] + + def recreate_trackhash(self): + """ + Recreates the trackhash based on the current title, album, and artist information. + """ + self.trackhash = create_hash( + self.title, self.album, *(artist["name"] for artist in self.artists) + ) diff --git a/app/store/albums.py b/app/store/albums.py index 7cbe8b8f..36c60921 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -44,19 +44,19 @@ class AlbumMapEntry: class AlbumStore: - albums: list[Album] = CustomList() + # albums: list[Album] = CustomList() albummap: dict[str, AlbumMapEntry] = {} - @staticmethod - def create_album(track: Track): - """ - Creates album object from a track - """ - return Album( - albumhash=track.albumhash, - albumartists=track.albumartists, # type: ignore - title=track.og_album, - ) + # @staticmethod + # def create_album(track: Track): + # """ + # Creates album object from a track + # """ + # return Album( + # albumhash=track.albumhash, + # albumartists=track.albumartists, # type: ignore + # title=track.og_album, + # ) @classmethod def load_albums(cls, instance_key: str): @@ -74,6 +74,12 @@ class AlbumStore: } print("Done!") + @classmethod + def index_new_album(cls, album: Album, trackhashes: set[str]): + cls.albummap[album.albumhash] = AlbumMapEntry( + album=album, trackhashes=trackhashes + ) + @classmethod def get_flat_list(cls): """ @@ -141,12 +147,12 @@ class AlbumStore: master_string = "-".join(a.albumartists_hashes for a in cls.albums) return master_string.count(artisthash) - @classmethod - def album_exists(cls, albumhash: str) -> bool: - """ - Checks if an album exists. - """ - return albumhash in "-".join([a.albumhash for a in cls.albums]) + # @classmethod + # def album_exists(cls, albumhash: str) -> bool: + # """ + # Checks if an album exists. + # """ + # return albumhash in "-".join([a.albumhash for a in cls.albums]) @classmethod def remove_album(cls, album: Album): diff --git a/app/store/artists.py b/app/store/artists.py index f88dcb83..079be398 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -34,11 +34,10 @@ class ArtistMapEntry: class ArtistStore: - artists: list[Artist] = CustomList() artistmap: dict[str, ArtistMapEntry] = {} @classmethod - def load_artists(cls, instance_key: str): + def load_artists(cls, instance_key: str, _trackhashes: list[str] = []): """ Loads all artists from the database into the store. """ @@ -52,7 +51,7 @@ class ArtistStore: artist.artisthash: ArtistMapEntry( artist=artist, albumhashes=albumhashes, trackhashes=trackhashes ) - for artist, trackhashes, albumhashes in create_artists() + for artist, trackhashes, albumhashes in create_artists(_trackhashes) } # for track in TrackStore.get_flat_list(): @@ -77,35 +76,35 @@ class ArtistStore: """ return [a.artist for a in cls.artistmap.values()] - @classmethod - def map_artist_color(cls, artist_tuple: tuple): - """ - Maps a color to the corresponding artist. - """ + # @classmethod + # def map_artist_color(cls, artist_tuple: tuple): + # """ + # Maps a color to the corresponding artist. + # """ - artisthash = artist_tuple[1] - color = json.loads(artist_tuple[2]) + # artisthash = artist_tuple[1] + # color = json.loads(artist_tuple[2]) - for artist in cls.artists: - if artist.artisthash == artisthash: - artist.set_colors(color) - break + # for artist in cls.artists: + # if artist.artisthash == artisthash: + # artist.set_colors(color) + # break - @classmethod - def add_artist(cls, artist: Artist): - """ - Adds an artist to the store. - """ - cls.artists.append(artist) + # @classmethod + # def add_artist(cls, artist: Artist): + # """ + # Adds an artist to the store. + # """ + # cls.artists.append(artist) - @classmethod - def add_artists(cls, artists: list[Artist]): - """ - Adds multiple artists to the store. - """ - for artist in artists: - if artist not in cls.artists: - cls.artists.append(artist) + # @classmethod + # def add_artists(cls, artists: list[Artist]): + # """ + # Adds multiple artists to the store. + # """ + # for artist in artists: + # if artist not in cls.artists: + # cls.artists.append(artist) @classmethod def get_artist_by_hash(cls, artisthash: str): @@ -124,34 +123,34 @@ class ArtistStore: artists = [cls.get_artist_by_hash(hash) for hash in artisthashes] return [a for a in artists if a is not None] - @classmethod - def artist_exists(cls, artisthash: str) -> bool: - """ - Checks if an artist exists. - """ - return artisthash in "-".join([a.artisthash for a in cls.artists]) + # @classmethod + # def artist_exists(cls, artisthash: str) -> bool: + # """ + # Checks if an artist exists. + # """ + # return artisthash in "-".join([a.artisthash for a in cls.artists]) - @classmethod - def artist_has_tracks(cls, artisthash: str) -> bool: - """ - Checks if an artist has tracks. - """ - artists: set[str] = set() + # @classmethod + # def artist_has_tracks(cls, artisthash: str) -> bool: + # """ + # Checks if an artist has tracks. + # """ + # artists: set[str] = set() - for track in TrackStore.tracks: - artists.update(track.artist_hashes) - album_artists: list[str] = [a.artisthash for a in track.albumartists] - artists.update(album_artists) + # for track in TrackStore.tracks: + # artists.update(track.artist_hashes) + # album_artists: list[str] = [a.artisthash for a in track.albumartists] + # artists.update(album_artists) - master_hash = "-".join(artists) - return artisthash in master_hash + # master_hash = "-".join(artists) + # return artisthash in master_hash - @classmethod - def remove_artist_by_hash(cls, artisthash: str): - """ - Removes an artist from the store. - """ - cls.artists = CustomList(a for a in cls.artists if a.artisthash != artisthash) + # @classmethod + # def remove_artist_by_hash(cls, artisthash: str): + # """ + # Removes an artist from the store. + # """ + # cls.artists = CustomList(a for a in cls.artists if a.artisthash != artisthash) @classmethod def get_artist_tracks(cls, artisthash: str): diff --git a/app/store/folder.py b/app/store/folder.py index 34555ccb..b262fe42 100644 --- a/app/store/folder.py +++ b/app/store/folder.py @@ -39,10 +39,14 @@ class FolderStore: trackhash = cls.map.get(filepath) if trackhash: - track = TrackStore.trackhashmap.get(trackhash) + trackgroup = TrackStore.trackhashmap.get(trackhash) - if track: - yield track.tracks[0] + if trackgroup is None: + continue + + for track in trackgroup.tracks: + if track.filepath == filepath: + yield track @classmethod def count_tracks_containing_paths(cls, paths: list[str]): diff --git a/app/utils/__init__.py b/app/utils/__init__.py index c432160c..ca253dec 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -14,7 +14,8 @@ def format_number(number: float) -> str: return locale.format_string("%d", number, grouping=True) - - def flatten(list_: Iterable[list[T]]) -> list[T]: + """ + Flattens a list of lists into a single list. + """ return [item for sublist in list_ for item in sublist] diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..e69de29b diff --git a/docs/streaming.md b/docs/streaming.md new file mode 100644 index 00000000..87b11ac5 --- /dev/null +++ b/docs/streaming.md @@ -0,0 +1,3 @@ +## Streaming + +## Transcoding diff --git a/docs/watchdog.md b/docs/watchdog.md new file mode 100644 index 00000000..e69de29b diff --git a/poetry.lock b/poetry.lock index 06d77703..0af17fe4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -491,6 +491,23 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "ffmpeg-python" +version = "0.2.0" +description = "Python bindings for FFmpeg - with complex filtering support" +optional = false +python-versions = "*" +files = [ + {file = "ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127"}, + {file = "ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5"}, +] + +[package.dependencies] +future = "*" + +[package.extras] +dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"] + [[package]] name = "flask" version = "2.3.3" @@ -597,6 +614,17 @@ dotenv = ["python-dotenv"] email = ["email-validator"] yaml = ["pyyaml"] +[[package]] +name = "future" +version = "1.0.0" +description = "Clean single-source support for Python 3 and 2" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216"}, + {file = "future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05"}, +] + [[package]] name = "gevent" version = "23.9.1" @@ -2733,4 +2761,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "184c1c56051131473212b74de5719e2629c630555b13cd4cd2de371b3a2fb195" +content-hash = "43972b6ffadd14e5047f067a0258f2428ebe351df8bd032dc0bf05df379678a6" diff --git a/pyproject.toml b/pyproject.toml index 8275feb6..87d5b809 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ sqlalchemy = "^2.0.31" memory-profiler = "^0.61.0" sortedcontainers = "^2.4.0" xxhash = "^3.4.1" +ffmpeg-python = "^0.2.0" [tool.poetry.dev-dependencies] pylint = "^2.15.5" From 7852be5e3f2f3d369ea66d36944f4d52a2276e95 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 17 Aug 2024 12:19:24 +0300 Subject: [PATCH 41/44] implement backup and restore draft 1 + add extra fields for backup in favorites and scrobble data - not yet for the playlist tracks --- TODO.md | 3 +- app/api/__init__.py | 2 + app/api/backup_and_restore.py | 78 +++++++++++++++++++++++++++++++++++ app/api/favorites.py | 6 ++- app/api/scrobble/__init__.py | 5 ++- app/api/stream.py | 21 ++-------- app/db/userdata.py | 2 +- app/lib/extras.py | 37 +++++++++++++++++ 8 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 app/api/backup_and_restore.py create mode 100644 app/lib/extras.py diff --git a/TODO.md b/TODO.md index 6edebdd1..11178148 100644 --- a/TODO.md +++ b/TODO.md @@ -43,10 +43,11 @@ - New table to hold similar artist entries - Create 2 way relationships, such that if an artist A is similar to another B with a certain weight, then artist B is similar to A with the same weight, unless overwritten. +- Clean up tempfiles after transcoding # Bug fixes - Duplicates on search - Audio stops on ending - Show users on account settings when logged in as admin and show users on login is disabled. -- Save both filepath and trackhash in favorites and playlists \ No newline at end of file +- Save both filepath and trackhash in favorites and playlists diff --git a/app/api/__init__.py b/app/api/__init__.py index b883ca7a..f0d474f0 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -31,6 +31,7 @@ from app.api import ( getall, auth, stream, + backup_and_restore ) # TODO: Move this description to a separate file @@ -107,6 +108,7 @@ def create_api(): app.register_api(settings.api) app.register_api(colors.api) app.register_api(lyrics.api) + app.register_api(backup_and_restore.api) # Plugins app.register_api(plugins.api) diff --git a/app/api/backup_and_restore.py b/app/api/backup_and_restore.py new file mode 100644 index 00000000..69762c64 --- /dev/null +++ b/app/api/backup_and_restore.py @@ -0,0 +1,78 @@ +from dataclasses import asdict +import json +from pathlib import Path +import shutil +from time import time +from flask_openapi3 import Tag +from flask_openapi3 import APIBlueprint +from app.api.auth import admin_required + +from app.db.userdata import FavoritesTable, PlaylistTable, ScrobbleTable +from app.settings import Paths + +bp_tag = Tag(name="Backup and Restore", description="Backup and Restore") +api = APIBlueprint("backup_and_restore", __name__, url_prefix="/", abp_tags=[bp_tag]) + + +@api.post("/backup") +@admin_required() +def backup(): + """ + Create a backup file of your favorites, playlists and scrobble data. + """ + backup_dir = Path(Paths.get_app_dir()) / "backup" + backup_dir.mkdir(parents=True, exist_ok=True) + backup_name = f"backup.{int(time())}" + backup_file = backup_dir / f"{backup_name}.json" + + # INFO: Image folder for playlist images + img_folder = backup_dir / "images" / backup_name + img_folder.mkdir(parents=True, exist_ok=True) + + favorites = FavoritesTable.get_all() + favorites = [asdict(entry) for entry in favorites] + + scrobbles = ScrobbleTable.get_all(start=0) + scrobbles = [asdict(entry) for entry in scrobbles] + + # SECTION: Playlists + playlists = PlaylistTable.get_all() + playlist_dicts = [] + + for entry in playlists: + playlist = asdict(entry) + for key in ["_last_updated", "has_image", "images", "duration", "count"]: + del playlist[key] + + playlist_dicts.append(playlist) + + # copy images + if playlist["thumb"]: + img_path = Path(Paths.get_playlist_img_path()) / playlist["thumb"] + shutil.copy(img_path, img_folder / playlist["thumb"]) + + # !SECTION + + data = { + "favorites": favorites, + "scrobbles": scrobbles, + "playlists": playlist_dicts, + } + + with open(backup_file, "w") as f: + json.dump(data, f, indent=4) + + return { + "msg": "Backup created", + "data_path": str(backup_file), + "images_path": str(img_folder), + }, 200 + + +@api.post("/restore") +@admin_required() +def restore(): + """ + Restore your favorites, playlists and scrobble data from a backup file. + """ + return {"msg": "Restore"} diff --git a/app/api/favorites.py b/app/api/favorites.py index 7e8aaaae..c23594a5 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, Field from app.api.apischemas import GenericLimitSchema from app.db.libdata import TrackTable from app.db.userdata import FavoritesTable +from app.lib.extras import get_extra_info from app.models import FavType from app.settings import Defaults @@ -71,9 +72,12 @@ def toggle_favorite(body: FavoritesAddBody): """ Adds a favorite to the database. """ + extra = get_extra_info(body.hash, body.type) try: - FavoritesTable.insert_item({"hash": body.hash, "type": body.type}) + FavoritesTable.insert_item( + {"hash": body.hash, "type": body.type, "extra": extra} + ) except: return {"msg": "Failed! An error occured"}, 500 diff --git a/app/api/scrobble/__init__.py b/app/api/scrobble/__init__.py index 36400c2b..46ac1338 100644 --- a/app/api/scrobble/__init__.py +++ b/app/api/scrobble/__init__.py @@ -4,6 +4,7 @@ from pydantic import Field from app.api.apischemas import TrackHashSchema from app.db.userdata import ScrobbleTable +from app.lib.extras import get_extra_info from app.settings import Defaults from app.store.albums import AlbumStore from app.store.artists import ArtistStore @@ -39,7 +40,9 @@ def log_track(body: LogTrackBody): if trackentry is None: return {"msg": "Track not found."}, 404 - ScrobbleTable.add(dict(body)) + scrobble_data = dict(body) + scrobble_data["extra"] = get_extra_info(body.trackhash, "track") + ScrobbleTable.add(scrobble_data) # Update play data on the in-memory stores track = trackentry.tracks[0] diff --git a/app/api/stream.py b/app/api/stream.py index 69bb6240..a49a1ef8 100644 --- a/app/api/stream.py +++ b/app/api/stream.py @@ -247,23 +247,11 @@ def send_file_as_chunks(filepath: str) -> Response: while remaining_bytes > 0 or retry_count < max_retries: if retry_count == max_retries: print("💚 sending final chunk! ...") - return ( - file.read(os.path.getsize(filepath) - file.tell()), - file.tell(), - True, - ) - print("\n\n") - print(f"file: {filepath}") - print(f"start: {start}") - print(f"end: {end}") - print(f"filesize: {os.path.getsize(filepath)}") - print(f"⭐ (O) Remaining bytes: {remaining_bytes}") - print(f"⭐ Remaining bytes: {remaining_bytes}") - print(f"⭐ Cursor position: {file.tell()}") - # Read the chunk size or all the remaining bytes - print(f"💚 remaining_bytes: {remaining_bytes}") - print(f"💚 retry_count: {retry_count}") + pos = file.tell() + chunk = file.read(os.path.getsize(filepath) - pos) + + return chunk, pos, True if remaining_bytes < chunk_size: time.sleep(0.25) @@ -303,7 +291,6 @@ def send_file_as_chunks(filepath: str) -> Response: f"bytes {start}-{position}/{os.path.getsize(filepath) + bytes_to_add}", ) response.headers.add("Accept-Ranges", "bytes") - response.headers.add("Content-Length", str(len(data or []))) return response diff --git a/app/db/userdata.py b/app/db/userdata.py index 2d33ff1e..c1debdd8 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -266,7 +266,7 @@ class ScrobbleTable(Base): return cls.insert_one(item) @classmethod - def get_all(cls, start: int, limit: int | None): + def get_all(cls, start: int, limit: int | None = None): result = cls.execute( select(cls) .where(cls.userid == get_current_userid()) diff --git a/app/lib/extras.py b/app/lib/extras.py new file mode 100644 index 00000000..70ac58af --- /dev/null +++ b/app/lib/extras.py @@ -0,0 +1,37 @@ +from typing import Any +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore + + +def get_extra_info(hash: str, type: str): + """ + Generates extra info for a track, album or artist, which will be stored + in the database (in favorites, playlists and scrobble data) for backup and restore. + + The extra info contains all the fields needed to reconstruct the itemhash. The track contains an additional filepath field which can be used to locate the file when restoring. + """ + extra: dict[str, Any] = {} + + if type == "track": + trackentry = TrackStore.trackhashmap.get(hash) + if trackentry is not None: + track = trackentry.get_best() + + extra["filepath"] = track.filepath + extra["title"] = track.title + extra["artists"] = [a["name"] for a in track.artists] + extra["album"] = track.albumhash + + elif type == "album": + album = AlbumStore.get_album_by_hash(hash) + if album is not None: + extra["albumartists"] = [a["name"] for a in album.albumartists] + extra["title"] = album.title + + elif type == "artist": + artist = ArtistStore.get_artist_by_hash(hash) + if artist is not None: + extra["name"] = artist.name + + return extra From b4bc9cb3cd5436020cf9699864d3943ebab152b6 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 18 Aug 2024 06:56:51 +0300 Subject: [PATCH 42/44] fix: get all favorite tracks endpoint --- .github/changelog.md | 2 ++ TODO.md | 1 + app/api/artist.py | 6 +++++- app/api/favorites.py | 3 +-- app/api/playlist.py | 8 ++++---- app/db/userdata.py | 4 ++++ 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/changelog.md b/.github/changelog.md index 5132e9bf..09a6b020 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -5,6 +5,8 @@ - Auth - New artists/albums Sort by: last played, no. of streams, total stream duration - Option to show now playing track info on tab title. Go to Settings > Appearance to enable +- You can select which disc to play in an album +- Internal Backup and restore ## Improvements diff --git a/TODO.md b/TODO.md index 11178148..0b2c782e 100644 --- a/TODO.md +++ b/TODO.md @@ -44,6 +44,7 @@ - Create 2 way relationships, such that if an artist A is similar to another B with a certain weight, then artist B is similar to A with the same weight, unless overwritten. - Clean up tempfiles after transcoding +- Double sort artist tracks for consistency (alphabetically then by other field. eg. playcount) # Bug fixes diff --git a/app/api/artist.py b/app/api/artist.py index 150796cd..18fee334 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -2,7 +2,6 @@ Contains all the artist(s) routes. """ -from dataclasses import asdict import math import random from datetime import datetime @@ -48,6 +47,9 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): return {"error": "Artist not found"}, 404 tracks = TrackStore.get_tracks_by_trackhashes(entry.trackhashes) + tracks = sorted(tracks, key=lambda t: t.title) + tracks = sorted(tracks, key= lambda t: t.playcount, reverse=True) + print([{t.title, t.playcount, t.trackhash} for t in tracks]) tcount = len(tracks) artist = entry.artist @@ -162,6 +164,8 @@ def get_all_artist_tracks(path: ArtistHashSchema): Returns all artists by a given artist. """ tracks = ArtistStore.get_artist_tracks(path.artisthash) + tracks = sorted(tracks, key=lambda t: t.title) + tracks = sorted(tracks, key= lambda t: t.playcount, reverse=True) return serialize_tracks(tracks) diff --git a/app/api/favorites.py b/app/api/favorites.py index c23594a5..c0b1a3cb 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -131,9 +131,8 @@ def get_favorite_tracks(query: GetAllOfTypeQuery): Get favorite tracks """ tracks, total = FavoritesTable.get_fav_tracks(query.start, query.limit) - tracks.reverse() - tracks = TrackTable.get_tracks_by_trackhashes([t.hash for t in tracks]) + tracks = TrackStore.get_tracks_by_trackhashes([t.hash for t in tracks]) return {"tracks": serialize_tracks(tracks), "total": total} diff --git a/app/api/playlist.py b/app/api/playlist.py index 8ba74cdb..b05cde8c 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -13,7 +13,6 @@ from flask_openapi3 import APIBlueprint, FileStorage from app import models from app.api.apischemas import GenericLimitSchema -from app.db.libdata import TrackTable from app.db.userdata import PlaylistTable from app.lib import playlistlib from app.lib.albumslib import sort_by_track_no @@ -57,7 +56,7 @@ def get_path_trackhashes(path: str): """ Returns a list of trackhashes in a folder. """ - tracks = TrackTable.get_tracks_in_path(path) + tracks = TrackStore.get_tracks_in_path(path) return [t.trackhash for t in tracks] @@ -65,7 +64,7 @@ def get_album_trackhashes(albumhash: str): """ Returns a list of trackhashes in an album. """ - tracks = TrackTable.get_tracks_by_albumhash(albumhash) + tracks = TrackStore.get_tracks_by_albumhash(albumhash) tracks = sort_by_track_no(tracks) return [t.trackhash for t in tracks] @@ -75,7 +74,8 @@ def get_artist_trackhashes(artisthash: str): """ Returns a list of trackhashes for an artist. """ - tracks = TrackTable.get_tracks_by_artisthash(artisthash) + tracks = TrackStore.get_tracks_by_artisthash(artisthash) + tracks = sorted(tracks, key= lambda t: t.playcount, reverse=True) return [t.trackhash for t in tracks] diff --git a/app/db/userdata.py b/app/db/userdata.py index c1debdd8..277a2614 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -225,6 +225,10 @@ class FavoritesTable(Base): res = result.fetchall() if start == 0: + # if limit == -1, return all + if limit == -1: + limit = len(res) + return res[:limit], len(res) return res, -1 From 312f81b324d70b7275995e9c50c5de3c46affe90 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 22 Aug 2024 20:09:00 +0300 Subject: [PATCH 43/44] fix: sort order on artists --- app/api/artist.py | 8 +++----- app/api/playlist.py | 3 ++- app/lib/sortlib.py | 6 ++++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/api/artist.py b/app/api/artist.py index 18fee334..c1f95796 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -18,6 +18,7 @@ from app.api.apischemas import ( from app.config import UserConfig from app.db.userdata import SimilarArtistTable +from app.lib.sortlib import sort_tracks from app.serializers.album import serialize_for_card_many from app.serializers.artist import serialize_for_cards, serialize_for_card @@ -47,9 +48,7 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): return {"error": "Artist not found"}, 404 tracks = TrackStore.get_tracks_by_trackhashes(entry.trackhashes) - tracks = sorted(tracks, key=lambda t: t.title) - tracks = sorted(tracks, key= lambda t: t.playcount, reverse=True) - print([{t.title, t.playcount, t.trackhash} for t in tracks]) + tracks = sort_tracks(tracks, key="playcount", reverse=True) tcount = len(tracks) artist = entry.artist @@ -164,8 +163,7 @@ def get_all_artist_tracks(path: ArtistHashSchema): Returns all artists by a given artist. """ tracks = ArtistStore.get_artist_tracks(path.artisthash) - tracks = sorted(tracks, key=lambda t: t.title) - tracks = sorted(tracks, key= lambda t: t.playcount, reverse=True) + tracks = sort_tracks(tracks, key="playcount", reverse=True) return serialize_tracks(tracks) diff --git a/app/api/playlist.py b/app/api/playlist.py index b05cde8c..4ab9fa1e 100644 --- a/app/api/playlist.py +++ b/app/api/playlist.py @@ -18,6 +18,7 @@ from app.lib import playlistlib from app.lib.albumslib import sort_by_track_no from app.lib.home.recentlyadded import get_recently_added_playlist from app.lib.home.recentlyplayed import get_recently_played_playlist +from app.lib.sortlib import sort_tracks from app.models.playlist import Playlist from app.serializers.playlist import serialize_for_card from app.serializers.track import serialize_tracks @@ -75,7 +76,7 @@ def get_artist_trackhashes(artisthash: str): Returns a list of trackhashes for an artist. """ tracks = TrackStore.get_tracks_by_artisthash(artisthash) - tracks = sorted(tracks, key= lambda t: t.playcount, reverse=True) + tracks = sort_tracks(tracks, key="playcount", reverse=True) return [t.trackhash for t in tracks] diff --git a/app/lib/sortlib.py b/app/lib/sortlib.py index 3198f855..a1f49944 100644 --- a/app/lib/sortlib.py +++ b/app/lib/sortlib.py @@ -26,6 +26,12 @@ def sort_tracks(tracks: list[Track], key: str, reverse: bool = False): return flatten([sort_by_track_no(list(g)) for k, g in groups]) + # INFO: sort tracks by title for a fallback value + tracks = sorted(tracks, key=lambda t: t.title) + + if key == "title": + return tracks + return sorted(tracks, key=sortfunc, reverse=reverse) From f081da8442e1517d826ba9ea17b436167bbb8cfa Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 31 Aug 2024 12:06:53 +0300 Subject: [PATCH 44/44] fix: folder sort tracks by title --- app/lib/sortlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/sortlib.py b/app/lib/sortlib.py index a1f49944..8ca9a1f9 100644 --- a/app/lib/sortlib.py +++ b/app/lib/sortlib.py @@ -29,7 +29,7 @@ def sort_tracks(tracks: list[Track], key: str, reverse: bool = False): # INFO: sort tracks by title for a fallback value tracks = sorted(tracks, key=lambda t: t.title) - if key == "title": + if key == "title" and not reverse: return tracks return sorted(tracks, key=sortfunc, reverse=reverse)