From 4a9f804e706ad046599eba5311945e9098a886db Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 30 Jun 2024 15:06:33 +0300 Subject: [PATCH] combine userdata and swing db into one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit + port populate to new db interface + add genrehashes and hash info to tracks + properly structure new db table files + move helpers to dedicated utils file + move settings from db to config file + move artists, albums, auth and favorites endpoint to new db interface + use folder store to index filepaths + paginate favorite pages + 56 moretiny changes 😅 --- .github/changelog.md | 5 + TODO.md | 35 ++- app/api/__init__.py | 8 +- app/api/album.py | 67 ++--- app/api/artist.py | 103 +------ app/api/auth.py | 96 +++--- app/api/favorites.py | 170 +++++------ app/api/folder.py | 16 +- app/api/getall/__init__.py | 9 +- app/api/settings.py | 122 ++++---- app/api/stream.py | 2 +- app/arg_handler.py | 7 +- app/config.py | 10 +- app/db/__init__.py | 381 ++---------------------- app/db/libdata.py | 312 +++++++++++++++++++ app/db/metadata.py | 33 ++ app/db/sqlite/lastfm/similar_artists.py | 2 +- app/db/sqlite/plugins/__init__.py | 10 - app/db/sqlite/queries.py | 7 - app/db/sqlite/settings.py | 217 +++++++------- app/db/userdata.py | 240 +++++++++++++++ app/db/utils.py | 75 +++++ app/lib/albumslib.py | 54 +--- app/lib/artistlib.py | 119 +------- app/lib/colorlib.py | 66 ++-- app/lib/folderslib.py | 6 +- app/lib/populate.py | 276 ++++++++--------- app/lib/tagger.py | 196 ++++++++++-- app/lib/taglib.py | 66 ++-- app/lib/trackslib.py | 10 - app/lib/watchdogg.py | 10 +- app/migrations/__init__.py | 35 ++- app/models/album.py | 11 +- app/models/artist.py | 7 + app/models/favorite.py | 11 + app/models/lastfm.py | 19 +- app/models/plugins.py | 2 +- app/models/track.py | 31 +- app/models/user.py | 14 +- app/periodic_scan.py | 12 +- app/plugins/register.py | 17 +- app/requests/artists.py | 17 +- app/settings.py | 11 - app/setup/__init__.py | 21 +- app/setup/sqlite.py | 22 +- app/store/artists.py | 1 - app/store/folder.py | 95 ++++++ app/utils/dates.py | 2 +- app/utils/hashing.py | 1 + app/utils/parsers.py | 8 +- manage.py | 2 +- poetry.lock | 2 +- pyproject.toml | 1 + 53 files changed, 1719 insertions(+), 1353 deletions(-) create mode 100644 app/db/libdata.py create mode 100644 app/db/metadata.py create mode 100644 app/db/userdata.py create mode 100644 app/db/utils.py create mode 100644 app/models/favorite.py create mode 100644 app/store/folder.py diff --git a/.github/changelog.md b/.github/changelog.md index ec423361..d2993bf1 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -25,3 +25,8 @@ interface Genre { } ``` +- Pairing via QR Code has been split into 2 endpoint: + 1. `/getpaircode` + 2. `/pair` + +- \ No newline at end of file diff --git a/TODO.md b/TODO.md index 76547381..e1ed9eee 100644 --- a/TODO.md +++ b/TODO.md @@ -1,11 +1,13 @@ # TODO + - Migrations: - 1. Move userdata to new hashing algorithm - - favorites ✅ - - playlists - - scrobble - - images - - remove image colors + + 1. Move userdata to new hashing algorithm + - favorites ✅ + - playlists + - scrobble + - images + - remove image colors - Package jsoni and publish on PyPi - Rewrite stores to use dictionaries instead of list pools @@ -16,15 +18,30 @@ - Recreate album hash if featured artists are discover - Implement checking if is clean install and skip migrations! - # DONE + - Support auth headers - Add recently played playlist - Move user track logs to user zero - Move future logs to appropriate user id - Store (and read) from the correct user account: - 1. Playlists - 2. Favorites \ No newline at end of file + 1. Playlists + 2. Favorites + +# THE BIG ONE + +- Updating settings +- Cleaning out commented code +- Watchdog +- Periodic scans +- Remove legacy db methods +- Remove all stores +- Review: We don't need server side image colors +- Clean up main db and userdata modules +- Move plugins to a config file +- What about our migrations? +- Add userid to queries +- Remove duplicates on artist page (test with Hanson) \ No newline at end of file diff --git a/app/api/__init__.py b/app/api/__init__.py index f6eab130..b9eacf49 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -11,9 +11,9 @@ from flask_openapi3 import OpenAPI from flask_jwt_extended import JWTManager from app.config import UserConfig +from app.db.userdata import UserTable from app.settings import Info as AppInfo from .plugins import lyrics as lyrics_plugin -from app.db.sqlite.auth import SQLiteAuthMethods as authdb from app.api import ( album, artist, @@ -92,8 +92,10 @@ def create_api(): def user_lookup_callback(_jwt_header, jwt_data): identity = jwt_data["sub"] userid = identity["id"] - user = authdb.get_user_by_id(userid) - return user.todict() + user = UserTable.get_by_id(userid) + + if user: + return user.todict() # Register all the API blueprints with app.app_context(): diff --git a/app/api/album.py b/app/api/album.py index b915612c..6deb7f7c 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -12,15 +12,14 @@ from flask_openapi3 import APIBlueprint from app.api.apischemas import AlbumHashSchema, AlbumLimitSchema, ArtistHashSchema from app.config import UserConfig -from app.db import AlbumTable as AlbumDb, TrackTable as TrackDb +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable as AlbumDb, TrackTable as TrackDb +from app.db.userdata import SimilarArtistTable from app.settings import Defaults -from app.models import FavType, Track -from app.store.albums import AlbumStore -from app.store.tracks import TrackStore from app.utils.hashing import create_hash from app.lib.albumslib import sort_by_track_no from app.serializers.album import serialize_for_card, serialize_for_card_many -from app.serializers.track import serialize_track +from app.serializers.track import serialize_track, serialize_tracks from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb @@ -52,9 +51,22 @@ def get_album_tracks_and_info(body: AlbumHashSchema): album.type = album.check_type( tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles ) - album.populate_versions() - return {"info": album, "tracks": tracks} + track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks}) + + return { + "info": album, + "extra": { + # INFO: track_total is the sum of a set of track_total values from each track + # ASSUMPTIONS + # 1. All the tracks have the correct track totals + # 2. Tracks with the same track total are from the same disc + "track_total": track_total, + "avg_bitrate": sum(t.bitrate for t in tracks) // len(tracks), + }, + "copyright": tracks[0].copyright, + "tracks": serialize_tracks(tracks, remove_disc=False), + } @api.get("//tracks") @@ -68,7 +80,7 @@ def get_album_tracks(path: AlbumHashSchema): tracks = TrackDb.get_tracks_by_albumhash(path.albumhash) tracks = sort_by_track_no(tracks) - return tracks + return serialize_tracks(tracks) class GetMoreFromArtistsBody(AlbumLimitSchema): @@ -138,30 +150,14 @@ def get_album_versions(body: GetAlbumVersionsBody): artisthash = body.artisthash albums = AlbumDb.get_albums_by_base_title(base_title) - print(albums) albums = [ a for a in albums if a.og_title != og_album_title and artisthash in {a["artisthash"] for a in a.albumartists} ] - print(albums) - - # albums = AlbumStore.get_albums_by_artisthash(artisthash) - - # albums = [ - # a - # for a in albums - # if create_hash(a.base_title) == create_hash(base_title) - # and create_hash(og_album_title) != create_hash(a.og_title) - # ] - - # for a in albums: - # tracks = TrackStore.get_tracks_by_albumhash(a.albumhash) - # a.get_date_from_tracks(tracks) - - return albums + return serialize_for_card_many(albums) class GetSimilarAlbumsQuery(ArtistHashSchema, AlbumLimitSchema): @@ -178,24 +174,15 @@ def get_similar_albums(query: GetSimilarAlbumsQuery): artisthash = query.artisthash limit = query.limit - similar_artists = lastfmdb.get_similar_artists_for(artisthash) + similar_artists = SimilarArtistTable.get_by_hash(artisthash) if similar_artists is None: - return {"albums": []} + return [] artisthashes = similar_artists.get_artist_hash_set() + artists = ArtistTable.get_artists_by_artisthashes(artisthashes) - if len(artisthashes) == 0: - return {"albums": []} + albums = AlbumDb.get_albums_by_artisthashes([a.artisthash for a in artists]) + sample = random.sample(albums, min(len(albums), limit)) - albums = [AlbumStore.get_albums_by_artisthash(a) for a in artisthashes] - - albums = [a for sublist in albums for a in sublist] - albums = list({a.albumhash: a for a in albums}.values()) - - try: - albums = random.sample(albums, min(len(albums), limit)) - except ValueError: - pass - - return [serialize_for_card(a) for a in albums[:limit]] + return serialize_for_card_many(sample[:limit]) diff --git a/app/api/artist.py b/app/api/artist.py index 03a01874..65bc6798 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -2,12 +2,11 @@ Contains all the artist(s) routes. """ -from itertools import groupby import math import random from datetime import datetime +from itertools import groupby -from flask_jwt_extended import current_user from flask_openapi3 import APIBlueprint, Tag from pydantic import Field from app.api.apischemas import ( @@ -18,18 +17,13 @@ from app.api.apischemas import ( ) from app.config import UserConfig -from app.db import AlbumTable, ArtistTable, TrackTable -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as fmdb -from app.models import Album, FavType +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable +from app.db.userdata import SimilarArtistTable + from app.serializers.album import serialize_for_card_many from app.serializers.track import serialize_tracks -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore -from app.store.tracks import TrackStore - - bp_tag = Tag(name="Artist", description="Single artist") api = APIBlueprint("artist", __name__, url_prefix="/artist", abp_tags=[bp_tag]) @@ -45,8 +39,6 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): limit = query.limit artist = ArtistTable.get_artist_by_hash(artisthash) - print(artist) - if artist is None: return {"error": "Artist not found"}, 404 @@ -56,8 +48,6 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): if artist.albumcount == 0 and tcount < 10: limit = tcount - # artist.is_favorite = favdb.check_is_favorite(artisthash, FavType.artist) - try: year = datetime.fromtimestamp(artist.date).year except ValueError: @@ -106,7 +96,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): t.albumhash for t in tracks if t.albumhash not in {a.albumhash for a in albums} } - albums.extend(AlbumTable.get_albums_by_hash(missing_albumhashes)) + albums.extend(AlbumTable.get_albums_by_albumhashes(missing_albumhashes)) albumdict = {a.albumhash: a for a in albums} config = UserConfig() @@ -117,43 +107,6 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): if album: album.check_type(list(tracks), config.showAlbumsAsSingles) - # all_albums = AlbumStore.get_albums_by_artisthash(artisthash) - # start: check for missing albums. ie. compilations and features - # all_tracks = TrackStore.get_tracks_by_artisthash(artisthash) - - # track_albums = set(t.albumhash for t in all_tracks) - # missing_album_hashes = track_albums.difference(set(a.albumhash for a in all_albums)) - - # if len(missing_album_hashes) > 0: - # missing_albums = AlbumStore.get_albums_by_hashes(list(missing_album_hashes)) - # all_albums.extend(missing_albums) - - # end check - - # def get_album_tracks(albumhash: str): - # tracks = [t for t in all_tracks if t.albumhash == albumhash] - - # if len(tracks) > 0: - # return tracks - - # return TrackStore.get_tracks_by_albumhash(albumhash) - - # for a in all_albums: - # a.check_type() - - # album_tracks = get_album_tracks(a.albumhash) - - # if len(album_tracks) == 0: - # continue - - # a.get_date_from_tracks(album_tracks) - - # if a.date == 0: - # AlbumStore.remove_album_by_hash(a.albumhash) - # continue - - # a.is_single(album_tracks) - albums = [a for a in albumdict.values()] all_albums = sorted(albums, key=lambda a: str(a.date), reverse=True) @@ -174,29 +127,6 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): else: res["albums"].append(album) - # def remove_EPs_and_singles(albums_: list[Album]): - # albums_ = [a for a in albums_ if not a.type == "single"] - # albums_ = [a for a in albums_ if not a.type == "ep"] - # return albums_ - - # albums = filter(lambda a: artisthash in missing_albumhashes, all_albums) - # albums = list(albums) - # albums = remove_EPs_and_singles(albums) - - # compilations = [a for a in albums if a.is_compilation] - # for c in compilations: - # albums.remove(c) - - # appearances = filter(lambda a: artisthash not in a.albumartists_hashes, all_albums) - # appearances = list(appearances) - - # appearances = remove_EPs_and_singles(appearances) - - # artist = ArtistStore.get_artist_by_hash(artisthash) - - # if artist is None: - # return {"error": "Artist not found"}, 404 - if return_all: limit = len(all_albums) @@ -215,8 +145,8 @@ def get_all_artist_tracks(path: ArtistHashSchema): Returns all artists by a given artist. """ - tracks = TrackStore.get_tracks_by_artisthash(path.artisthash) - + # tracks = TrackStore.get_tracks_by_artisthash(path.artisthash) + tracks = TrackTable.get_tracks_by_artisthash(path.artisthash) return serialize_tracks(tracks) @@ -226,23 +156,14 @@ def get_similar_artists(path: ArtistHashSchema, query: ArtistLimitSchema): Get similar artists. """ limit = query.limit - - artist = ArtistStore.get_artist_by_hash(path.artisthash) - - if artist is None: - return {"error": "Artist not found"}, 404 - - result = fmdb.get_similar_artists_for(artist.artisthash) + result = SimilarArtistTable.get_by_hash(path.artisthash) if result is None: - return {"artists": []} + return [] - similar = ArtistStore.get_artists_by_hashes(result.get_artist_hash_set()) + similar = ArtistTable.get_artists_by_artisthashes(result.get_artist_hash_set()) if len(similar) > limit: - similar = random.sample(similar, limit) + similar = random.sample(similar, min(limit, len(similar))) return similar[:limit] - - -# TODO: Rewrite this file using generators where possible diff --git a/app/api/auth.py b/app/api/auth.py index 4f36250d..0cf41bf1 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -14,7 +14,8 @@ from pydantic import BaseModel, Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint -from app.db.sqlite.auth import SQLiteAuthMethods as authdb +# from app.db.sqlite.auth import SQLiteAuthMethods as authdb +from app.db.userdata import UserTable from app.utils.auth import check_password, hash_password from app.config import UserConfig @@ -65,7 +66,7 @@ def login(body: LoginBody): Authenticate using username and password """ - user = authdb.get_user_by_username(body.username) + user = UserTable.get_by_username(body.username) if user is None: return {"msg": "User not found"}, 404 @@ -87,42 +88,41 @@ def login(body: LoginBody): pair_token = dict() +@api.get("/getpaircode") +def get_pair(): + """ + Get a new pair code to log in to thee Swing Music mobile app + """ + # INFO: if user is already logged in, create a new pair code + token = create_new_token(get_jwt_identity()) + key = token["accesstoken"][-6:] + + global pair_token + pair_token = { + key: token, + } + + return {"code": key} + + class PairDeviceQuery(BaseModel): code: str = Field("", description="The code") -@api.get("/pair") +@api.post("/pair") @jwt_required(optional=True) -def pair_device(query: PairDeviceQuery): +def pair_with_code(body: PairDeviceQuery): """ - Pair the Swing Music mobile app with this server - - Send a code to get an access token. Send an authenticated request without the code to generate a new token. + Get an access token by sending a pair code. NOTE: A code can only be used once! """ - # INFO: if user is already logged in, create a new pair code - if current_user: - token = create_new_token(get_jwt_identity()) - key = token["accesstoken"][-6:] + global pair_token + token = pair_token.get(body.code, None) - global pair_token - pair_token = { - key: token, - } + if token: + pair_token = {} + return token - return {"code": key} - - # INFO: if there's a pair code, return the token - if query.code: - token = pair_token.get(query.code, None) - - if token: - # INFO: reset pair_token - pair_token = {} - return token - - return {"msg": "Invalid code"}, 400 - - return {"msg": "No code provided"}, 400 + return {"msg": "Invalid code"}, 400 @api.post("/refresh") @@ -133,6 +133,8 @@ def refresh(): >>> Headers: >>> Authorization: Bearer + + Won't work with cookies!!! """ user = get_jwt_identity() return create_new_token(user) @@ -153,7 +155,6 @@ def update_profile(body: UpdateProfileBody): """ user = { "id": body.id, - "email": body.email, "username": body.username, "password": body.password, "roles": body.roles, @@ -172,7 +173,8 @@ def update_profile(body: UpdateProfileBody): if "admin" not in current_user["roles"]: return {"msg": "Only admins can update roles"}, 403 - all_users = authdb.get_all_users() + # all_users = authdb.get_all_users() + all_users = UserTable.get_all() if "admin" not in body.roles: # check if we're removing the last admin admins = [user for user in all_users if "admin" in user.roles] @@ -195,7 +197,9 @@ def update_profile(body: UpdateProfileBody): clean_user = {k: v for k, v in user.items() if v} try: - return authdb.update_user(clean_user) + # return authdb.update_user(clean_user) + UserTable.update_one(clean_user) + return UserTable.get_by_id(user["id"]).todict() except sqlite3.IntegrityError: return {"msg": "Username already exists"}, 400 @@ -216,11 +220,18 @@ def create_user(body: UpdateProfileBody): } # check if user already exists - if authdb.get_user_by_username(user["username"]): + if UserTable.get_by_username(user["username"]): return {"msg": "Username already exists"}, 400 - userid = authdb.insert_user(user) - return authdb.get_user_by_id(userid).todict() + UserTable.insert_one(user) + user = UserTable.get_by_username(user["username"]) + + if user: + return user.todict() + + return { + "msg": "Failed to create user", + }, 500 @api.post("/profile/guest/create") @@ -230,14 +241,14 @@ def create_guest_user(): Create a guest user """ # check if guest user already exists - guest_user = authdb.get_user_by_username("guest") + guest_user = UserTable.get_by_username("guest") if guest_user: return { "msg": "Guest user already exists", }, 400 - userid = authdb.insert_guest_user() + userid = UserTable.insert_guest_user() if userid: return { @@ -264,12 +275,12 @@ def delete_user(body: DeleteUseBody): return {"msg": "Sorry! you cannot delete yourselfu"}, 400 # prevent deleting the only admin - users = authdb.get_all_users() + users = UserTable.get_all() admins = [user for user in users if "admin" in user.roles] if len(admins) == 1 and admins[0].username == body.username: return {"msg": "Cannot delete the only admin"}, 400 - authdb.delete_user_by_username(body.username) + UserTable.remove_by_username(body.username) return {"msg": f"User {body.username} deleted"} @@ -308,8 +319,7 @@ def get_all_users(query: GetAllUsersQuery): "users": [], } - users = authdb.get_all_users() - + users = UserTable.get_all() is_admin = current_user and "admin" in current_user["roles"] settings["enableGuest"] = [ user for user in users if user.username == "guest" @@ -355,8 +365,8 @@ def get_all_users(query: GetAllUsersQuery): if query.simplified: res["users"] = [user.todict_simplified() for user in users] - - res["users"] = [user.todict() for user in users] + else: + res["users"] = [user.todict() for user in users] return res diff --git a/app/api/favorites.py b/app/api/favorites.py index 94fa57b7..c6127a5c 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -6,18 +6,19 @@ from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from app.api.apischemas import GenericLimitSchema +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable +from app.db.userdata import FavoritesTable from app.models import FavType from app.settings import Defaults from app.utils.bisection import use_bisection -from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.serializers.track import serialize_track, serialize_tracks -from app.serializers.artist import serialize_for_card as serialize_artist, serialize_for_cards -from app.serializers.album import serialize_for_card, serialize_for_card_many - -from app.store.albums import AlbumStore -from app.store.tracks import TrackStore -from app.store.artists import ArtistStore +from app.serializers.artist import ( + serialize_for_card as serialize_artist, + serialize_for_cards, +) from app.utils.dates import timestamp_to_time_passed +from app.serializers.album import serialize_for_card, serialize_for_card_many bp_tag = Tag(name="Favorites", description="Your favorite items") api = APIBlueprint("favorites", __name__, url_prefix="/favorites", abp_tags=[bp_tag]) @@ -41,17 +42,18 @@ class FavoritesAddBody(BaseModel): @api.post("/add") -def add_favorite(body: FavoritesAddBody): +def toggle_favorite(body: FavoritesAddBody): """ Adds a favorite to the database. """ - itemhash = body.hash - itemtype = body.type + FavoritesTable.insert_item({"hash": body.hash, "type": body.type}) - favdb.insert_one_favorite(itemtype, itemhash) - - if itemtype == FavType.track: - TrackStore.make_track_fav(itemhash) + if body.type == FavType.track: + TrackTable.set_is_favorite(body.hash, True) + elif body.type == FavType.album: + AlbumTable.set_is_favorite(body.hash, True) + elif body.type == FavType.artist: + ArtistTable.set_is_favorite(body.hash, True) return {"msg": "Added to favorites"} @@ -61,80 +63,62 @@ def remove_favorite(body: FavoritesAddBody): """ Removes a favorite from the database. """ - itemhash = body.hash - itemtype = body.type + FavoritesTable.remove_item({"hash": body.hash, "type": body.type}) - favdb.delete_favorite(itemtype, itemhash) - - if itemtype == FavType.track: - TrackStore.remove_track_from_fav(itemhash) + if body.type == FavType.track: + TrackTable.set_is_favorite(body.hash, False) + elif body.type == FavType.album: + AlbumTable.set_is_favorite(body.hash, False) + elif body.type == FavType.artist: + ArtistTable.set_is_favorite(body.hash, False) return {"msg": "Removed from favorites"} +class GetAllOfTypeQuery(GenericLimitSchema): + """ + Extending this class will give you a model with the `limit` field + """ + + start: int = Field( + description="Where to start from", + example=Defaults.API_CARD_LIMIT, + default=Defaults.API_CARD_LIMIT, + ) + + @api.get("/albums") -def get_favorite_albums(query: GenericLimitSchema): +def get_favorite_albums(query: GetAllOfTypeQuery): """ Get favorite albums """ - limit = query.limit - albums = favdb.get_fav_albums() - albumhashes = [a[1] for a in albums] - albumhashes.reverse() + fav_albums, total = FavoritesTable.get_fav_albums(query.start, query.limit) + fav_albums.reverse() - src_albums = sorted(AlbumStore.albums, key=lambda x: x.albumhash) - - fav_albums = use_bisection(src_albums, "albumhash", albumhashes) - fav_albums = remove_none(fav_albums) - - if limit == 0: - limit = len(albums) - - return {"albums": serialize_for_card_many(fav_albums[:limit])} + return {"albums": serialize_for_card_many(fav_albums), "total": total} @api.get("/tracks") -def get_favorite_tracks(query: GenericLimitSchema): +def get_favorite_tracks(query: GetAllOfTypeQuery): """ Get favorite tracks """ - limit = query.limit - userid = current_user['id'] - - tracks = favdb.get_fav_tracks(userid) - trackhashes = [t[1] for t in tracks] - trackhashes.reverse() - src_tracks = sorted(TrackStore.tracks, key=lambda x: x.trackhash) - - tracks = use_bisection(src_tracks, "trackhash", trackhashes) - tracks = remove_none(tracks) - - if limit == 0: - limit = len(tracks) - - return {"tracks": serialize_tracks(tracks[:limit])} + tracks, total = FavoritesTable.get_fav_tracks(query.start, query.limit) + return {"tracks": serialize_tracks(tracks), "total": total} @api.get("/artists") -def get_favorite_artists(query: GenericLimitSchema): +def get_favorite_artists(query: GetAllOfTypeQuery): """ Get favorite artists """ - limit = query.limit + artists, total = FavoritesTable.get_fav_artists( + start=query.start, + limit=query.limit, + ) + artists.reverse() - artists = favdb.get_fav_artists() - artisthashes = [a[1] for a in artists] - artisthashes.reverse() - - src_artists = sorted(ArtistStore.artists, key=lambda x: x.artisthash) - - artists = use_bisection(src_artists, "artisthash", artisthashes) - artists = remove_none(artists) - - if limit == 0: - limit = len(artists) - - return {"artists": artists[:limit]} + return {"artists": [serialize_artist(a) for a in artists], "total": total} class GetAllFavoritesQuery(BaseModel): @@ -173,27 +157,29 @@ def get_all_favorites(query: GetAllFavoritesQuery): # largest is x2 to accound for broken hashes if any largest = max(track_limit, album_limit, artist_limit) - favs = favdb.get_all() + favs = FavoritesTable.get_all() favs.reverse() tracks = [] albums = [] artists = [] - track_master_hash = set(t.trackhash for t in TrackStore.tracks) - album_master_hash = set(a.albumhash for a in AlbumStore.albums) - artist_master_hash = set(a.artisthash for a in ArtistStore.artists) + track_master_hash = TrackTable.get_all_hashes() + album_master_hash = AlbumTable.get_all_hashes() + artist_master_hash = ArtistTable.get_all_hashes() + # INFO: Filter out invalid hashes (file not found or tags edited) for fav in favs: - # INFO: hash is [1], type is [2], timestamp is [3] - hash = fav[1] - if fav[2] == FavType.track: + hash = fav.hash + type = fav.type + + if type == FavType.track: tracks.append(hash) if hash in track_master_hash else None - if fav[2] == FavType.artist: + if type == FavType.artist: artists.append(hash) if hash in artist_master_hash else None - if fav[2] == FavType.album: + if type == FavType.album: albums.append(hash) if hash in album_master_hash else None count = { @@ -202,35 +188,26 @@ def get_all_favorites(query: GetAllFavoritesQuery): "artists": len(artists), } - src_tracks = sorted(TrackStore.tracks, key=lambda x: x.trackhash) - src_albums = sorted(AlbumStore.albums, key=lambda x: x.albumhash) - src_artists = sorted(ArtistStore.artists, key=lambda x: x.artisthash) - - tracks = use_bisection(src_tracks, "trackhash", tracks, limit=track_limit) - albums = use_bisection(src_albums, "albumhash", albums, limit=album_limit) - artists = use_bisection(src_artists, "artisthash", artists, limit=artist_limit) - - tracks = remove_none(tracks) - albums = remove_none(albums) - artists = remove_none(artists) + tracks = TrackTable.get_tracks_by_trackhashes(tracks, limit=track_limit) + albums = AlbumTable.get_albums_by_albumhashes(albums, limit=album_limit) + artists = ArtistTable.get_artists_by_artisthashes(artists, limit=artist_limit) recents = [] # first_n = favs for fav in favs: - # INFO: hash is [1], type is [2], timestamp is [3] if len(recents) >= largest: break - if fav[2] == FavType.album: - album = next((a for a in albums if a.albumhash == fav[1]), None) + if fav.type == FavType.album: + album = next((a for a in albums if a.albumhash == fav.hash), None) if album is None: continue album = serialize_for_card(album) album["help_text"] = "album" - album["time"] = timestamp_to_time_passed(fav[3]) + album["time"] = timestamp_to_time_passed(fav.timestamp) recents.append( { @@ -239,15 +216,15 @@ def get_all_favorites(query: GetAllFavoritesQuery): } ) - if fav[2] == FavType.artist: - artist = next((a for a in artists if a.artisthash == fav[1]), None) + if fav.type == FavType.artist: + artist = next((a for a in artists if a.artisthash == fav.hash), None) if artist is None: continue artist = serialize_artist(artist) artist["help_text"] = "artist" - artist["time"] = timestamp_to_time_passed(fav[3]) + artist["time"] = timestamp_to_time_passed(fav.timestamp) recents.append( { @@ -256,15 +233,15 @@ def get_all_favorites(query: GetAllFavoritesQuery): } ) - if fav[2] == FavType.track: - track = next((t for t in tracks if t.trackhash == fav[1]), None) + if fav.type == FavType.track: + track = next((t for t in tracks if t.trackhash == fav.hash), None) if track is None: continue track = serialize_track(track) track["help_text"] = "track" - track["time"] = timestamp_to_time_passed(fav[3]) + track["time"] = timestamp_to_time_passed(fav.timestamp) recents.append({"type": "track", "item": track}) @@ -284,6 +261,5 @@ def check_favorite(query: FavoritesAddBody): """ itemhash = query.hash itemtype = query.type - exists = favdb.check_is_favorite(itemhash, itemtype) - return {"is_favorite": exists} + return {"is_favorite": FavoritesTable.check_exists(itemhash, itemtype)} diff --git a/app/api/folder.py b/app/api/folder.py index c2d1cc85..098742af 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -10,11 +10,10 @@ from pydantic import BaseModel, Field from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from showinfm import show_in_file_manager -from memory_profiler import profile from app import settings -from app.db import TrackTable -from app.db.sqlite.settings import SettingsSQLMethods as db +from app.config import UserConfig +from app.db.libdata import TrackTable from app.lib.folderslib import GetFilesAndDirs, get_folders from app.serializers.track import serialize_track from app.utils.wintools import is_windows, win_replace_slash @@ -40,8 +39,9 @@ def get_folder_tree(body: FolderTree): req_dir = body.folder tracks_only = body.tracks_only - root_dirs = db.get_root_dirs() - root_dirs.sort() + + config = UserConfig() + root_dirs = config.rootDirs try: if req_dir == "$home" and root_dirs[0] == "$home": @@ -72,12 +72,6 @@ def get_folder_tree(body: FolderTree): return res - # return { - # "path": req_dir, - # "tracks": tracks, - # "folders": sorted(folders, key=lambda i: i.name), - # } - def get_all_drives(is_win: bool = False): """ diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index 31c72b27..f1fe2a35 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -6,7 +6,8 @@ from pydantic import BaseModel, Field from datetime import datetime from app.api.apischemas import GenericLimitSchema -from app.db import AlbumTable, ArtistTable +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable from app.store.albums import AlbumStore from app.store.artists import ArtistStore @@ -61,11 +62,11 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): is_artists = path.itemtype == "artists" if is_albums: - items, total = AlbumTable.get_all(query.start, query.limit) + items = AlbumTable.get_all() elif is_artists: - items, total = ArtistTable.get_all(query.start, query.limit) + items = ArtistTable.get_all() - # print(items) + total = len(items) start = query.start limit = query.limit diff --git a/app/api/settings.py b/app/api/settings.py index 4931bc52..3f1a3047 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -1,3 +1,4 @@ +from dataclasses import asdict from typing import Any from flask import request from flask_openapi3 import Tag @@ -6,12 +7,13 @@ from pydantic import BaseModel, Field from app.api.auth import admin_required from app.db.sqlite.plugins import PluginsMethods as pdb -from app.db.sqlite.settings import SettingsSQLMethods as sdb from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb +from app.db.userdata import PluginTable from app.lib import populate +from app.lib.tagger import index_everything from app.lib.watchdogg import Watcher as WatchDog from app.logger import log -from app.settings import Info, Paths, SessionVarKeys, set_flag +from app.settings import Info, Paths, SessionVarKeys from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore @@ -48,42 +50,43 @@ def reload_everything(instance_key: str): except Exception as e: log.error(e) +# CHECKPOINT: TEST SETTINGS API ENDPOINTS -@background -def rebuild_store(db_dirs: list[str]): - """ - Restarts watchdog and rebuilds the music library. - """ - instance_key = get_random_str() +# @background +# def rebuild_store(db_dirs: list[str]): +# """ +# Restarts watchdog and rebuilds the music library. +# """ +# instance_key = get_random_str() - log.info("Rebuilding library...") - trackdb.remove_tracks_not_in_folders(db_dirs) - reload_everything(instance_key) +# log.info("Rebuilding library...") +# trackdb.remove_tracks_not_in_folders(db_dirs) +# reload_everything(instance_key) - try: - populate.Populate(instance_key=instance_key) - except populate.PopulateCancelledError as e: - print(e) - reload_everything(instance_key) - return +# try: +# populate.Populate(instance_key=instance_key) +# except populate.PopulateCancelledError as e: +# print(e) +# reload_everything(instance_key) +# return - WatchDog().restart() +# WatchDog().restart() - log.info("Rebuilding library... ✅") +# log.info("Rebuilding library... ✅") -# I freaking don't know what this function does anymore -def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]): - """ - Params: - new_: will be added to the database - removed_: will be removed from the database - db_dirs_: will be used to remove tracks that - are outside these directories from the database and store. - """ - sdb.remove_root_dirs(removed_) - sdb.add_root_dirs(new_) - rebuild_store(db_dirs_) +# # I freaking don't know what this function does anymore +# def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]): +# """ +# Params: +# new_: will be added to the database +# removed_: will be removed from the database +# db_dirs_: will be used to remove tracks that +# are outside these directories from the database and store. +# """ +# sdb.remove_root_dirs(removed_) +# sdb.add_root_dirs(new_) +# rebuild_store(db_dirs_) class AddRootDirsBody(BaseModel): @@ -106,7 +109,8 @@ def add_root_dirs(body: AddRootDirsBody): new_dirs = body.new_dirs removed_dirs = body.removed - db_dirs = sdb.get_root_dirs() + config = UserConfig() + db_dirs = config.rootDirs home = "$home" db_home = any([d == home for d in db_dirs]) # if $home is in db @@ -114,13 +118,16 @@ def add_root_dirs(body: AddRootDirsBody): # handle $home case if db_home and incoming_home: - return {"msg": "Not changed!"} + return {"msg": "Not changed!"}, 304 + # if $home is the current root dir or the incoming root dir + # is $home, remove all root dirs if db_home or incoming_home: - sdb.remove_root_dirs(db_dirs) + config.rootDirs = [] if incoming_home: - finalize([home], [], [Paths.USER_HOME_DIR]) + config.rootDirs = [home] + index_everything() return {"root_dirs": [home]} # --- @@ -136,11 +143,10 @@ def add_root_dirs(body: AddRootDirsBody): pass db_dirs.extend(new_dirs) - db_dirs = [dir_ for dir_ in db_dirs if dir_ != home] + config.rootDirs = [dir_ for dir_ in db_dirs if dir_ != home] - finalize(new_dirs, removed_dirs, db_dirs) - - return {"root_dirs": db_dirs} + index_everything() + return {"root_dirs": config.rootDirs} @api.get("/get-root-dirs") @@ -148,9 +154,7 @@ def get_root_dirs(): """ Get root directories """ - dirs = sdb.get_root_dirs() - - return {"dirs": dirs} + return {"dirs": UserConfig().rootDirs} # maps settings to their parser flags @@ -170,35 +174,12 @@ def get_all_settings(): """ Get all settings """ + config = asdict(UserConfig()) + plugins = PluginTable.get_all() + config["plugins"] = plugins + config["version"] = Info.SWINGMUSIC_APP_VERSION - settings = sdb.get_all_settings() - plugins = pdb.get_all_plugins() - - key_list = list(mapp.keys()) - s = {} - - for key in key_list: - val_index = key_list.index(key) - - try: - s[key] = settings[val_index] - - if type(s[key]) == int: - s[key] = bool(s[key]) - if type(s[key]) == str: - s[key] = str(s[key]).split(",") - - except IndexError: - s[key] = None - - root_dirs = sdb.get_root_dirs() - s["root_dirs"] = root_dirs - s["plugins"] = plugins - s["version"] = Info.SWINGMUSIC_APP_VERSION - - return { - "settings": s, - } + return config @background @@ -245,7 +226,6 @@ def set_setting(body: SetSettingBody): value = str(value).split(",") value = set(value) - set_flag(flag, value) reload_all_for_set_setting() # if value is a set, convert it to a string diff --git a/app/api/stream.py b/app/api/stream.py index 3eaee2a6..4993539a 100644 --- a/app/api/stream.py +++ b/app/api/stream.py @@ -11,7 +11,7 @@ from app.api.apischemas import TrackHashSchema from app.lib.trackslib import get_silence_paddings # from app.store.tracks import TrackStore -from app.db import TrackTable +from app.db.libdata import TrackTable from app.utils.files import guess_mime_type bp_tag = Tag(name="File", description="Audio files") diff --git a/app/arg_handler.py b/app/arg_handler.py index b74acfe4..249ef062 100644 --- a/app/arg_handler.py +++ b/app/arg_handler.py @@ -9,6 +9,7 @@ import sys import PyInstaller.__main__ as bundler from app import settings +from app.config import UserConfig from app.logger import log from app.print_help import HELP_MESSAGE from app.utils.auth import hash_password @@ -160,7 +161,7 @@ class ProcessArgs: @staticmethod def handle_periodic_scan(): if any((a in ARGS for a in ALLARGS.no_periodic_scan)): - settings.SessionVars.DO_PERIODIC_SCANS = False + UserConfig().enablePeriodicScans = False @staticmethod def handle_periodic_scan_interval(): @@ -182,10 +183,10 @@ class ProcessArgs: sys.exit(0) if psi < 0: - print("WADAFUCK ARE YOU TRYING?") + print("WHAT ARE YOU TRYING?") sys.exit(0) - settings.SessionVars.PERIODIC_SCAN_INTERVAL = psi + UserConfig().scanInterval = psi @staticmethod def handle_help(): diff --git a/app/config.py b/app/config.py index b751f129..bca9c69e 100644 --- a/app/config.py +++ b/app/config.py @@ -6,6 +6,7 @@ from .settings import Paths # TODO: Publish this on PyPi + @dataclass class UserConfig: _config_path: str = "" @@ -20,7 +21,7 @@ class UserConfig: # lists rootDirs: list[str] = field(default_factory=list) excludeDirs: list[str] = field(default_factory=list) - artistSeparators: set[str] = field(default_factory=set) + artistSeparators: set[str] = field(default_factory=lambda: {";", "/"}) genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) # tracks @@ -33,6 +34,13 @@ class UserConfig: cleanAlbumTitle: bool = True showAlbumsAsSingles: bool = False + # misc + enablePeriodicScans: bool = False + scanInterval: int = 60 * 10 # 10 minutes + + # plugins + enablePlugins: bool = True + def __post_init__(self): """ Loads the config file and sets the values to this instance diff --git a/app/db/__init__.py b/app/db/__init__.py index c7321c80..ebc6d776 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -1,36 +1,21 @@ -from concurrent.futures import ThreadPoolExecutor -import json -import os -from pathlib import Path -from pprint import pprint -from typing import Any, Optional +from typing import Any -from memory_profiler import profile from sqlalchemy import ( - JSON, - Boolean, - Integer, - Row, - String, - Tuple, - and_, create_engine, + delete, insert, select, ) + +from sqlalchemy.engine import Engine +from sqlalchemy import event from sqlalchemy.orm import ( - Mapped, - mapped_column, DeclarativeBase, MappedAsDataclass, - sessionmaker, ) -from app.models import Track as TrackModel -from app.models import Album as AlbumModel -from app.models import Artist as ArtistModel -from app.utils.remove_duplicates import remove_duplicates - +# ============================================================ +# TODO: Make sure the database is created before we run this. fullpath = "/home/cwilvx/temp/swingmusic/swing.db" engine = create_engine( f"sqlite+pysqlite:///{fullpath}", @@ -39,85 +24,46 @@ engine = create_engine( pool_size=5, ) -if not os.path.exists(fullpath): - os.makedirs(Path(fullpath).parent) - -connection = engine.connect() -all_filepaths = list() +# connection = engine.connect() -def getIndexOfFirstMatch(strings: list[str], prefix: str): - """ - Find the index of the first path that starts with the given path. - - Uses a binary search algorithm to find the index. - """ - - left = 0 - right = len(strings) - 1 - - while left <= right: - mid = (left + right) // 2 - - if strings[mid].startswith(prefix): - if mid == 0 or not strings[mid - 1].startswith(prefix): - return mid - right = mid - 1 - elif strings[mid] < prefix: - left = mid + 1 - else: - right = mid - 1 - - return -1 - - -def countFilepathsInDir(dirpath: str): - """ - Return all the filepaths in a directory. - """ - global all_filepaths - index = getIndexOfFirstMatch(all_filepaths, dirpath) - - if index == -1: - return 0 - - paths: list[str] = [] - - for path in all_filepaths[index:]: - if path.startswith(dirpath): - paths.append(path) - else: - break - - return len(paths) +@event.listens_for(Engine, "connect") +def set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() class DbManager: def __init__(self, commit: bool = False): self.commit = commit - # self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) - # self.conn = self.engine.connect() - # pass + self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) + self.conn = self.engine.connect() def __enter__(self): - # return self.conn.execution_options(preserve_rowcount=True) - return connection + return self.conn.execution_options(preserve_rowcount=True) + # return connection def __exit__(self, exc_type, exc_val, exc_tb): if self.commit: - connection.commit() + self.conn.commit() - # self.conn.close() + self.conn.close() class Base(MappedAsDataclass, DeclarativeBase): + @classmethod + def execute(cls, stmt: Any, commit: bool = False): + with DbManager(commit=commit) as conn: + return conn.execute(stmt) + @classmethod def insert_many(cls, items: list[dict[str, Any]]): """ Inserts multiple items into the database. """ with DbManager(commit=True) as conn: - conn.execute(insert(cls).values(items)) + return conn.execute(insert(cls).values(items)) @classmethod def insert_one(cls, item: dict[str, Any]): @@ -127,277 +73,14 @@ class Base(MappedAsDataclass, DeclarativeBase): return cls.insert_many([item]) @classmethod - def get_all(cls): - """ - Returns all the items from the database. - """ - with DbManager() as conn: - result = conn.execute(select(cls)) - return result.fetchall() - - -class ArtistTable(Base): - __tablename__ = "artist" - - id: Mapped[int] = mapped_column(primary_key=True) - albumcount: Mapped[int] = mapped_column(Integer()) - artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True) - created_date: Mapped[int] = mapped_column(Integer()) - date: Mapped[int] = mapped_column(Integer()) - duration: Mapped[int] = mapped_column(Integer()) - genres: Mapped[str] = mapped_column(JSON()) - name: Mapped[str] = mapped_column(String(), index=True) - trackcount: Mapped[int] = mapped_column(Integer()) - is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + def remove_all(cls): + with DbManager(commit=True) as conn: + conn.execute(delete(cls)) @classmethod - def get_all(cls, start: int, limit: int): - with DbManager() as conn: - if start == 0: - result = conn.execute(select(cls)) - else: - result = conn.execute(select(cls).offset(start).limit(limit)) - - all = result.fetchall() - return artists_to_dataclasses(all), len(all) - - @classmethod - def get_artist_by_hash(cls, artisthash: str): - with DbManager() as conn: - result = conn.execute( - select(ArtistTable).where(ArtistTable.artisthash == artisthash) - ) - return artist_to_dataclass(result.fetchone()) + def all(cls): + return cls.execute(select(cls)) -class AlbumTable(Base): - __tablename__ = "album" - - id: Mapped[int] = mapped_column(primary_key=True) - albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) - artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) - albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) - base_title: Mapped[str] = mapped_column(String()) - color: Mapped[Optional[str]] = mapped_column(String()) - created_date: Mapped[int] = mapped_column(Integer()) - date: Mapped[int] = mapped_column(Integer()) - duration: Mapped[int] = mapped_column(Integer()) - genres: Mapped[str] = mapped_column(JSON()) - og_title: Mapped[str] = mapped_column(String()) - title: Mapped[str] = mapped_column(String()) - trackcount: Mapped[int] = mapped_column(Integer()) - - @classmethod - def get_album_by_albumhash(cls, hash: str): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.albumhash == hash) - ) - album = result.fetchone() - - if album: - return album_to_dataclass(album) - - @classmethod - def get_albums_by_hash(cls, hashes: set[str]): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)) - ) - return albums_to_dataclasses(result.fetchall()) - - @classmethod - def get_all(cls, start: int, limit: int): - with DbManager() as conn: - if start == 0: - result = conn.execute(select(AlbumTable)) - else: - result = conn.execute(select(AlbumTable).offset(start).limit(limit)) - - all = result.fetchall() - - return albums_to_dataclasses(all)[:limit], len(all) - - @classmethod - def get_albums_by_artisthashes(cls, artisthashes: list[dict[str, str]]): - with DbManager() as conn: - albums: list[AlbumModel] = [] - - for artist in artisthashes: - result = conn.execute( - # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! - select(AlbumTable).where(AlbumTable.albumartists.contains(artist)) - ) - albums.extend(albums_to_dataclasses(result.fetchall())) - - return albums - - @classmethod - def get_albums_by_base_title(cls, base_title: str): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.base_title == base_title) - ) - return albums_to_dataclasses(result.fetchall()) - - @classmethod - def get_albums_by_artisthash(cls, artisthash: str): - with DbManager() as conn: - result = conn.execute( - select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash)) - ) - return albums_to_dataclasses(result.all()) - - -class TrackTable(Base): - __tablename__ = "track" - - id: Mapped[int] = mapped_column(init=False, primary_key=True) - album: Mapped[str] = mapped_column(String()) - albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) - albumhash: Mapped[str] = mapped_column(String(), index=True) - artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) - artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) - bitrate: Mapped[int] = mapped_column(Integer()) - copyright: Mapped[Optional[str]] = mapped_column(String()) - date: Mapped[int] = mapped_column(Integer()) - disc: Mapped[int] = mapped_column(Integer()) - duration: Mapped[int] = mapped_column(Integer()) - filepath: Mapped[str] = mapped_column(String(), index=True, unique=True) - folder: Mapped[str] = mapped_column(String(), index=True) - genre: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) - last_mod: Mapped[float] = mapped_column(Integer()) - og_album: Mapped[str] = mapped_column(String()) - og_title: Mapped[str] = mapped_column(String()) - title: Mapped[str] = mapped_column(String()) - track: Mapped[int] = mapped_column(Integer()) - trackhash: Mapped[str] = mapped_column(String(), index=True) - extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) - - @classmethod - def get_tracks_by_filepaths(cls, filepaths: list[str]): - with DbManager() as conn: - result = conn.execute( - select(TrackTable).where(TrackTable.filepath.in_(filepaths)) - ) - return tracks_to_dataclasses(result.fetchall()) - - @classmethod - def count_tracks_containing_paths(cls, paths: list[str]): - results: list[dict[str, int | str]] = [] - - with ThreadPoolExecutor() as executor: - res = executor.map(countFilepathsInDir, paths) - results = [ - {"path": path, "trackcount": count} for path, count in zip(paths, res) - ] - - return results - - @classmethod - def get_tracks_by_albumhash(cls, albumhash: str): - with DbManager() as conn: - result = conn.execute( - select(TrackTable).where(TrackTable.albumhash == albumhash) - ) - tracks = tracks_to_dataclasses(result.fetchall()) - return remove_duplicates(tracks, is_album_tracks=True) - - @classmethod - def get_track_by_trackhash(cls, hash: str, filepath: str = ""): - with DbManager() as conn: - if filepath: - result = conn.execute( - select(TrackTable) - .where( - and_( - TrackTable.trackhash == hash, - TrackTable.filepath == filepath, - ) - ) - .order_by(TrackTable.bitrate.desc()) - ) - else: - result = conn.execute( - select(TrackTable).where(TrackTable.trackhash == hash) - ) - - track = result.fetchone() - - if track: - return track_to_dataclass(track) - - @classmethod - def get_tracks_by_artisthash(cls, artisthash: str): - with DbManager() as conn: - result = conn.execute( - select(TrackTable).where(TrackTable.artists.contains(artisthash)) - ) - return tracks_to_dataclasses(result.fetchall()) - - @classmethod - def get_tracks_in_path(cls, path: str): - with DbManager() as conn: - result = conn.execute( - select(TrackTable) - .where(TrackTable.filepath.contains(path)) - .order_by(TrackTable.last_mod) - ) - return tracks_to_dataclasses(result.fetchall()) - - -all_tracks = TrackTable.get_all() - -for track in all_tracks: - all_filepaths.append(track.filepath) - -all_filepaths.sort() - -# print("files in path: ",getFilepathsInDir("/home/cwilvx/Music/").__len__()) - - -# SECTION: Userdata database -class UserTable(Base): - __tablename__ = "user" - - id: Mapped[int] = mapped_column(primary_key=True) - username: Mapped[str] = mapped_column(String(), unique=True) - firstname: Mapped[Optional[str]] = mapped_column(String()) - lastname: Mapped[Optional[str]] = mapped_column(String()) - password: Mapped[str] = mapped_column(String()) - email: Mapped[Optional[str]] = mapped_column(String()) - image: Mapped[Optional[str]] = mapped_column(String()) - roles: Mapped[list[str]] = mapped_column(JSON(), default_factory=lambda: ["user"]) - extra: Mapped[Optional[dict[str, Any]]] = mapped_column( - JSON(), default_factory=dict - ) - - -# SECTION: HELPER FUNCTIONS - - -def artist_to_dataclass(artist: Any): - return ArtistModel(**artist._asdict()) - - -def artists_to_dataclasses(artists: Any): - return [artist_to_dataclass(artist) for artist in artists] - - -def album_to_dataclass(album: Any): - return AlbumModel(**album._asdict()) - - -def albums_to_dataclasses(albums: Any): - return [album_to_dataclass(album) for album in albums] - - -def track_to_dataclass(track: Any): - return TrackModel(**track._asdict()) - - -def tracks_to_dataclasses(tracks: Any): - return [track_to_dataclass(track) for track in tracks] - - -Base().metadata.create_all(engine) +def create_all(): + Base().metadata.create_all(engine) diff --git a/app/db/libdata.py b/app/db/libdata.py new file mode 100644 index 00000000..9d3ebbd6 --- /dev/null +++ b/app/db/libdata.py @@ -0,0 +1,312 @@ +from app.db import ( + Base as MasterBase, + DbManager, +) +from app.db.utils import ( + album_to_dataclass, + albums_to_dataclasses, + artist_to_dataclass, + artists_to_dataclasses, + track_to_dataclass, + tracks_to_dataclasses, +) +from app.models import Album as AlbumModel +from app.utils.remove_duplicates import remove_duplicates +from app.db import engine + +from sqlalchemy import JSON, Boolean, Integer, String, and_, delete, select, update +from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase + + +from typing import Any, Iterable, Optional + + +def create_all(): + """ + Create all the tables defined in this file. + """ + Base.metadata.create_all(engine) + + +class Base(MasterBase, DeclarativeBase): + @classmethod + def get_all_hashes(cls): + with DbManager() as conn: + if cls.__tablename__ == "track": + stmt = select(TrackTable.trackhash) + elif cls.__tablename__ == "album": + stmt = select(AlbumTable.albumhash) + elif cls.__tablename__ == "artist": + stmt = select(ArtistTable.artisthash) + + result = conn.execute(stmt) + return {row[0] for row in result.fetchall()} + + @classmethod + def set_is_favorite(cls, hash: str, is_favorite: bool): + """ + Set the 'is_favorite' flag for a specific hash. + + Args: + hash (str): The hash value. + is_favorite (bool): The value of the 'is_favorite' flag. + """ + with DbManager(commit=True) as conn: + if cls.__tablename__ == "track": + stmt = ( + update(cls) + .where(TrackTable.trackhash == hash) + .values(is_favorite=is_favorite) + ) + elif cls.__tablename__ == "album": + stmt = ( + update(cls) + .where(AlbumTable.albumhash == hash) + .values(is_favorite=is_favorite) + ) + elif cls.__tablename__ == "artist": + stmt = ( + update(cls) + .where(ArtistTable.artisthash == hash) + .values(is_favorite=is_favorite) + ) + + conn.execute(stmt) + + +class TrackTable(Base): + __tablename__ = "track" + + id: Mapped[int] = mapped_column(init=False, primary_key=True) + album: Mapped[str] = mapped_column(String()) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) + albumhash: Mapped[str] = mapped_column(String(), index=True) + artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + bitrate: Mapped[int] = mapped_column(Integer()) + copyright: Mapped[Optional[str]] = mapped_column(String()) + date: Mapped[int] = mapped_column(Integer(), nullable=True) + disc: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + filepath: Mapped[str] = mapped_column(String(), unique=True) + folder: Mapped[str] = mapped_column(String(), index=True) + genrehashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + genres: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) + last_mod: Mapped[float] = mapped_column(Integer()) + og_album: Mapped[str] = mapped_column(String()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + track: Mapped[int] = mapped_column(Integer()) + trackhash: Mapped[str] = mapped_column(String(), index=True) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + playcount: Mapped[int] = mapped_column(Integer()) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_by_filepaths(cls, filepaths: list[str]): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.filepath.in_(filepaths)) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_by_albumhash(cls, albumhash: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.albumhash == albumhash) + ) + tracks = tracks_to_dataclasses(result.fetchall()) + return remove_duplicates(tracks, is_album_tracks=True) + + @classmethod + def get_track_by_trackhash(cls, hash: str, filepath: str = ""): + with DbManager() as conn: + if filepath: + result = conn.execute( + select(TrackTable) + .where( + and_( + TrackTable.trackhash == hash, + TrackTable.filepath == filepath, + ) + ) + .order_by(TrackTable.bitrate.desc()) + ) + else: + result = conn.execute( + select(TrackTable).where(TrackTable.trackhash == hash) + ) + + track = result.fetchone() + + if track: + return track_to_dataclass(track) + + @classmethod + def get_tracks_by_artisthash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.artists.contains(artisthash)) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_in_path(cls, path: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable) + .where(TrackTable.filepath.contains(path)) + .order_by(TrackTable.last_mod) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def get_tracks_by_trackhashes(cls, hashes: Iterable[str], limit: int | None = None): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.trackhash.in_(hashes)).limit(limit) + ) + return tracks_to_dataclasses(result.fetchall()) + + @classmethod + def remove_tracks_by_filepaths(cls, filepaths: set[str]): + with DbManager(commit=True) as conn: + conn.execute(delete(TrackTable).where(TrackTable.filepath.in_(filepaths))) + + +class AlbumTable(Base): + __tablename__ = "album" + + id: Mapped[int] = mapped_column(primary_key=True) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) + albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) + base_title: Mapped[str] = mapped_column(String()) + color: Mapped[Optional[str]] = mapped_column(String()) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True) + genres: Mapped[str] = mapped_column(JSON()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + trackcount: Mapped[int] = mapped_column(Integer()) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(AlbumTable)) + all = result.fetchall() + return albums_to_dataclasses(all) + + @classmethod + def get_album_by_albumhash(cls, hash: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.albumhash == hash) + ) + album = result.fetchone() + + if album: + return album_to_dataclass(album) + + @classmethod + def get_albums_by_albumhashes(cls, hashes: Iterable[str], limit: int | None = None): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)).limit(limit) + ) + return albums_to_dataclasses(result.fetchall()) + + @classmethod + def get_albums_by_artisthashes(cls, artisthashes: list[str]): + with DbManager() as conn: + albums: list[AlbumModel] = [] + + for artist in artisthashes: + result = conn.execute( + # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! + select(AlbumTable).where(AlbumTable.albumartists.contains(artist)) + ) + albums.extend(albums_to_dataclasses(result.fetchall())) + + return albums + + @classmethod + def get_albums_by_base_title(cls, base_title: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.base_title == base_title) + ) + return albums_to_dataclasses(result.fetchall()) + + @classmethod + def get_albums_by_artisthash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash)) + ) + return albums_to_dataclasses(result.all()) + + +class ArtistTable(Base): + __tablename__ = "artist" + + id: Mapped[int] = mapped_column(primary_key=True) + albumcount: Mapped[int] = mapped_column(Integer()) + artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True) + genres: Mapped[str] = mapped_column(JSON()) + name: Mapped[str] = mapped_column(String(), index=True) + trackcount: Mapped[int] = mapped_column(Integer()) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + all = result.fetchall() + return artists_to_dataclasses(all) + + @classmethod + def get_artist_by_hash(cls, artisthash: str): + with DbManager() as conn: + result = conn.execute( + select(ArtistTable).where(ArtistTable.artisthash == artisthash) + ) + return artist_to_dataclass(result.fetchone()) + + @classmethod + def get_artisthashes_not_in(cls, artisthashes: list[str]): + with DbManager() as conn: + result = conn.execute( + select(ArtistTable.artisthash, ArtistTable.name).where( + ~ArtistTable.artisthash.in_(artisthashes) + ) + ) + return [{"artisthash": row[0], "name": row[1]} for row in result.fetchall()] + + @classmethod + def get_artists_by_artisthashes( + cls, hashes: Iterable[str], limit: int | None = None + ): + with DbManager() as conn: + result = conn.execute( + select(ArtistTable) + .where(ArtistTable.artisthash.in_(hashes)) + .limit(limit) + ) + return artists_to_dataclasses(result.fetchall()) diff --git a/app/db/metadata.py b/app/db/metadata.py new file mode 100644 index 00000000..16976b45 --- /dev/null +++ b/app/db/metadata.py @@ -0,0 +1,33 @@ +from app.db import Base, DbManager + + +from sqlalchemy import Integer, insert, select, update +from sqlalchemy.orm import Mapped, mapped_column + + +class MigrationTable(Base): + __tablename__ = "dbmigration" + + id: Mapped[int] = mapped_column(primary_key=True) + version: Mapped[int] = mapped_column(Integer()) + + @classmethod + def set_version(cls, version: int): + with DbManager(commit=True) as conn: + result = conn.execute( + update(cls).where(cls.id == 1).values(version=version) + ) + + if result.rowcount == 0: + conn.execute(insert(cls).values(id=1, version=version)) + + @classmethod + def get_version(cls): + with DbManager() as conn: + result = conn.execute(select(cls.version).where(cls.id == 1)) + result = result.fetchone() + + if result: + return result[0] + + return -1 \ No newline at end of file diff --git a/app/db/sqlite/lastfm/similar_artists.py b/app/db/sqlite/lastfm/similar_artists.py index 22a5825c..53a45b70 100644 --- a/app/db/sqlite/lastfm/similar_artists.py +++ b/app/db/sqlite/lastfm/similar_artists.py @@ -16,7 +16,7 @@ class SQLiteLastFMSimilarArtists: sql = """INSERT OR REPLACE INTO lastfm_similar_artists(artisthash, similar_artists) VALUES(?,?)""" with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, (artist.artisthash, artist.similar_artist_hashes)) + cur.execute(sql, (artist.artisthash, artist.similar_artists)) cur.close() @classmethod diff --git a/app/db/sqlite/plugins/__init__.py b/app/db/sqlite/plugins/__init__.py index 2cfd2345..0afc5d43 100644 --- a/app/db/sqlite/plugins/__init__.py +++ b/app/db/sqlite/plugins/__init__.py @@ -8,7 +8,6 @@ from ..utils import SQLiteManager def plugin_tuple_to_obj(plugin_tuple: tuple) -> Plugin: return Plugin( name=plugin_tuple[1], - description=plugin_tuple[2], active=bool(plugin_tuple[3]), settings=json.loads(plugin_tuple[4]), ) @@ -43,15 +42,6 @@ class PluginsMethods: return lastrowid - @classmethod - def insert_lyrics_plugin(cls): - plugin = Plugin( - name="lyrics_finder", - description="Find lyrics from the internet", - active=False, - settings={"auto_download": False}, - ) - cls.insert_plugin(plugin) @classmethod def get_all_plugins(cls): diff --git a/app/db/sqlite/queries.py b/app/db/sqlite/queries.py index 1e52b272..547b4ffe 100644 --- a/app/db/sqlite/queries.py +++ b/app/db/sqlite/queries.py @@ -14,13 +14,6 @@ CREATE TABLE IF NOT EXISTS playlists ( constraint fk_users foreign key (userid) references users(id) on delete cascade ); -CREATE TABLE IF NOT EXISTS favorites ( - id integer PRIMARY KEY, - hash text not null, - type text not null, - timestamp integer not null default 0 -); - CREATE TABLE IF NOT EXISTS settings ( id integer PRIMARY KEY, root_dirs text NOT NULL, diff --git a/app/db/sqlite/settings.py b/app/db/sqlite/settings.py index a597d1b2..0121fa9f 100644 --- a/app/db/sqlite/settings.py +++ b/app/db/sqlite/settings.py @@ -1,150 +1,151 @@ from pprint import pprint from typing import Any +from app.config import UserConfig from app.db.sqlite.utils import SQLiteManager -from app.settings import SessionVars from app.utils.wintools import win_replace_slash -class SettingsSQLMethods: - """ - Methods for interacting with the settings table. - """ +# class SettingsSQLMethods: +# """ +# Methods for interacting with the settings table. +# """ - @staticmethod - def get_all_settings(): - """ - Gets all settings from the database. - """ +# @staticmethod +# def get_all_settings(): +# """ +# Gets all settings from the database. +# """ - sql = "SELECT * FROM settings WHERE id = 1" +# sql = "SELECT * FROM settings WHERE id = 1" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - settings = cur.fetchone() - cur.close() +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql) +# settings = cur.fetchone() +# cur.close() - # if root_dirs not set - if settings is None: - return [] +# # if root_dirs not set +# if settings is None: +# return [] - # omit id, root_dirs, and exclude_dirs - return settings[3:] +# # omit id, root_dirs, and exclude_dirs +# return settings[3:] - @staticmethod - def get_root_dirs() -> list[str]: - """ - Gets custom root directories from the database. - """ +# @staticmethod +# def get_root_dirs() -> list[str]: +# """ +# Gets custom root directories from the database. +# """ - sql = "SELECT root_dirs FROM settings" +# sql = "SELECT root_dirs FROM settings" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - dirs = cur.fetchall() - cur.close() +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql) +# dirs = cur.fetchall() +# cur.close() - dirs = [_dir[0] for _dir in dirs] - return [win_replace_slash(d) for d in dirs] +# dirs = [_dir[0] for _dir in dirs] +# return [win_replace_slash(d) for d in dirs] - @staticmethod - def add_root_dirs(dirs: list[str]): - """ - Add custom root directories to the database. - """ +# @staticmethod +# def add_root_dirs(dirs: list[str]): +# """ +# Add custom root directories to the database. +# """ - sql = "INSERT INTO settings (root_dirs) VALUES (?)" - existing_dirs = SettingsSQLMethods.get_root_dirs() +# sql = "INSERT INTO settings (root_dirs) VALUES (?)" +# existing_dirs = SettingsSQLMethods.get_root_dirs() - dirs = [_dir for _dir in dirs if _dir not in existing_dirs] +# dirs = [_dir for _dir in dirs if _dir not in existing_dirs] - if len(dirs) == 0: - return +# if len(dirs) == 0: +# return - with SQLiteManager(userdata_db=True) as cur: - for _dir in dirs: - cur.execute(sql, (_dir,)) +# with SQLiteManager(userdata_db=True) as cur: +# for _dir in dirs: +# cur.execute(sql, (_dir,)) - @staticmethod - def remove_root_dirs(dirs: list[str]): - """ - Remove custom root directories from the database. - """ +# @staticmethod +# def remove_root_dirs(dirs: list[str]): +# """ +# Remove custom root directories from the database. +# """ - sql = "DELETE FROM settings WHERE root_dirs = ?" +# sql = "DELETE FROM settings WHERE root_dirs = ?" - with SQLiteManager(userdata_db=True) as cur: - for _dir in dirs: - cur.execute(sql, (_dir,)) +# with SQLiteManager(userdata_db=True) as cur: +# for _dir in dirs: +# cur.execute(sql, (_dir,)) - # Not currently used anywhere, to be used later - @staticmethod - def add_excluded_dirs(dirs: list[str]): - """ - Add custom exclude directories to the database. - """ +# # Not currently used anywhere, to be used later +# @staticmethod +# def add_excluded_dirs(dirs: list[str]): +# """ +# Add custom exclude directories to the database. +# """ - sql = "INSERT INTO settings (exclude_dirs) VALUES (?)" +# sql = "INSERT INTO settings (exclude_dirs) VALUES (?)" - with SQLiteManager(userdata_db=True) as cur: - cur.executemany(sql, dirs) +# with SQLiteManager(userdata_db=True) as cur: +# cur.executemany(sql, dirs) - @staticmethod - def remove_excluded_dirs(dirs: list[str]): - """ - Remove custom exclude directories from the database. - """ +# @staticmethod +# def remove_excluded_dirs(dirs: list[str]): +# """ +# Remove custom exclude directories from the database. +# """ - sql = "DELETE FROM settings WHERE exclude_dirs = ?" +# sql = "DELETE FROM settings WHERE exclude_dirs = ?" - with SQLiteManager(userdata_db=True) as cur: - cur.executemany(sql, dirs) +# with SQLiteManager(userdata_db=True) as cur: +# cur.executemany(sql, dirs) - @staticmethod - def get_excluded_dirs() -> list[str]: - """ - Gets custom exclude directories from the database. - """ +# @staticmethod +# def get_excluded_dirs() -> list[str]: +# """ +# Gets custom exclude directories from the database. +# """ - sql = "SELECT exclude_dirs FROM settings" +# sql = "SELECT exclude_dirs FROM settings" - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql) - dirs = cur.fetchall() - return [_dir[0] for _dir in dirs] +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql) +# dirs = cur.fetchall() +# return [_dir[0] for _dir in dirs] - @staticmethod - def get_settings() -> dict[str, Any]: - pass +# @staticmethod +# def get_settings() -> dict[str, Any]: +# pass - @staticmethod - def set_setting(key: str, value: Any): - sql = f"UPDATE settings SET {key} = :value WHERE id = 1" +# @staticmethod +# def set_setting(key: str, value: Any): +# sql = f"UPDATE settings SET {key} = :value WHERE id = 1" - if type(value) == bool: - value = str(int(value)) +# if type(value) == bool: +# value = str(int(value)) - with SQLiteManager(userdata_db=True) as cur: - cur.execute(sql, {"value": value}) +# with SQLiteManager(userdata_db=True) as cur: +# cur.execute(sql, {"value": value}) -def load_settings(): - s = SettingsSQLMethods.get_all_settings() +# def load_settings(): +# # s = SettingsSQLMethods.get_all_settings() +# config = UserConfig() - try: - db_separators: str = s[0] - db_separators = db_separators.replace(" ", "") - separators = db_separators.split(",") - separators = set(separators) - except IndexError: - separators = {";", "/"} + # try: + # db_separators: str = s[0] + # db_separators = db_separators.replace(" ", "") + # separators = db_separators.split(",") + # separators = set(separators) + # except IndexError: + # separators = {";", "/"} - SessionVars.ARTIST_SEPARATORS = separators + # SessionVars.ARTIST_SEPARATORS = config.artistSeparators - # boolean settings - SessionVars.EXTRACT_FEAT = bool(s[1]) - SessionVars.REMOVE_PROD = bool(s[2]) - SessionVars.CLEAN_ALBUM_TITLE = bool(s[3]) - SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4]) - SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5]) - SessionVars.SHOW_ALBUMS_AS_SINGLES = bool(s[6]) + # # boolean settings + # SessionVars.EXTRACT_FEAT = bool(s[1]) + # SessionVars.REMOVE_PROD = bool(s[2]) + # SessionVars.CLEAN_ALBUM_TITLE = bool(s[3]) + # SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4]) + # SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5]) + # SessionVars.SHOW_ALBUMS_AS_SINGLES = bool(s[6]) diff --git a/app/db/userdata.py b/app/db/userdata.py new file mode 100644 index 00000000..ce2b6708 --- /dev/null +++ b/app/db/userdata.py @@ -0,0 +1,240 @@ +import datetime +from shlex import join +from typing import Any +from flask_jwt_extended import current_user +from sqlalchemy import ( + JSON, + Boolean, + ForeignKey, + Integer, + String, + and_, + delete, + insert, + select, + update, + join, +) + +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.utils import ( + albums_to_dataclasses, + artists_to_dataclasses, + favorites_to_dataclass, + plugin_to_dataclasses, + similar_artist_to_dataclass, + similar_artists_to_dataclass, + tracks_to_dataclasses, + user_to_dataclass, + user_to_dataclasses, +) + +from app.db import Base, DbManager +from app.utils.auth import hash_password + + +class UserTable(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column(primary_key=True) + image: Mapped[str] = mapped_column(String(), nullable=True) + password: Mapped[str] = mapped_column(String()) + username: Mapped[str] = mapped_column(String(), index=True) + roles: Mapped[list[str]] = mapped_column(JSON(), default_factory=lambda: ["user"]) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) + + @classmethod + def get_all(cls): + result = cls.execute(select(cls)) + return user_to_dataclasses(result.fetchall()) + + @classmethod + def insert_default_user(cls): + user = { + "username": "admin", + "password": hash_password("admin"), + "roles": ["admin"], + } + + return cls.insert_one(user) + + @classmethod + def insert_guest_user(cls): + user = { + "username": "guest", + "password": hash_password("guest"), + "roles": ["guest"], + } + + return cls.insert_one(user) + + @classmethod + def get_by_id(cls, id: int): + with DbManager() as conn: + result = conn.execute(select(cls).where(cls.id == id)) + res = result.fetchone() + + if res: + return user_to_dataclass(res) + + @classmethod + def get_by_username(cls, username: str): + with DbManager() as conn: + result = conn.execute(select(cls).where(cls.username == username)) + res = result.fetchone() + + if res: + return user_to_dataclass(res) + + @classmethod + def update_one(cls, user: dict[str, Any]): + with DbManager(commit=True) as conn: + conn.execute(update(cls).where(cls.id == user["id"]).values(user)) + + @classmethod + def remove_by_username(cls, username: str): + return cls.execute(delete(cls).where(cls.username == username), commit=True) + + +class PluginTable(Base): + __tablename__ = "plugin" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(), unique=True) + active: Mapped[bool] = mapped_column(Boolean()) + settings: Mapped[dict[str, Any]] = mapped_column(JSON()) + extra: Mapped[dict[str, Any]] = mapped_column(JSON(), nullable=True) + + @classmethod + def get_all(cls): + return plugin_to_dataclasses(cls.all()) + + +class SimilarArtistTable(Base): + __tablename__ = "notlastfm_similar_artists" + + id: Mapped[int] = mapped_column(Integer(), primary_key=True) + artisthash: Mapped[str] = mapped_column(String(), index=True) + similar_artists: Mapped[dict[str, str]] = mapped_column(JSON()) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + return similar_artists_to_dataclass(result.fetchall()) + + @classmethod + def exists(cls, artisthash: str): + """ + Check whether an artisthash exists in the database. + """ + + with DbManager() as conn: + result = conn.execute( + select(cls.artisthash).where(cls.artisthash == artisthash) + ) + return result.fetchone() is not None + + @classmethod + def get_by_hash(cls, artisthash: str): + """ + Get a single artist by hash. + """ + + with DbManager() as conn: + result = conn.execute(select(cls).where(cls.artisthash == artisthash)) + result = result.fetchone() + + if result: + return similar_artist_to_dataclass(result) + + +class FavoritesTable(Base): + __tablename__ = "favorite" + + id: Mapped[int] = mapped_column(primary_key=True) + hash: Mapped[str] = mapped_column(String()) + type: Mapped[str] = mapped_column(String(), index=True) + timestamp: Mapped[int] = mapped_column(Integer(), index=True) + userid: Mapped[int] = mapped_column( + Integer(), ForeignKey("user.id"), default=1, index=True + ) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) + + @classmethod + def get_all(cls): + with DbManager() as conn: + result = conn.execute(select(cls)) + return favorites_to_dataclass(result.fetchall()) + + @classmethod + def insert_item(cls, item: dict[str, Any]): + item["timestamp"] = int(datetime.datetime.now().timestamp()) + item["userid"] = current_user["id"] + + with DbManager(commit=True) as conn: + conn.execute(insert(cls).values(item)) + + @classmethod + def remove_item(cls, item: dict[str, Any]): + with DbManager(commit=True) as conn: + conn.execute( + delete(cls).where( + (cls.hash == item["hash"]) & (cls.type == item["type"]) + ) + ) + + @classmethod + def check_exists(cls, hash: str, type: str): + result = cls.execute(select(cls).where((cls.hash == hash) & (cls.type == type))) + return result.fetchone() is not None + + @classmethod + def get_all_of_type(cls, table: Any, field: Any, type: str, start: int, limit: int): + result = cls.execute( + select(table) + .select_from(join(table, cls, field == cls.hash)) + .where(and_(cls.type == type, cls.userid == current_user["id"])) + .offset(start) + # INFO: If start is 0, fetch all so we can get the total count + .limit(limit if start != 0 else None) + ) + + res = result.fetchall() + + if start == 0: + return res[:limit], len(res) + + return res, -1 + + @classmethod + def get_fav_tracks(cls, start: int, limit: int): + from .libdata import TrackTable + + result, total = cls.get_all_of_type( + TrackTable, TrackTable.trackhash, "track", start, limit + ) + return tracks_to_dataclasses(result), total + + @classmethod + def get_fav_albums(cls, start: int, limit: int): + from .libdata import AlbumTable + + result, total = cls.get_all_of_type( + AlbumTable, AlbumTable.albumhash, "album", start, limit + ) + return albums_to_dataclasses(result), total + + @classmethod + def get_fav_artists(cls, start: int, limit: int): + from .libdata import ArtistTable + + result, total = cls.get_all_of_type( + ArtistTable, ArtistTable.artisthash, "artist", start, limit + ) + return artists_to_dataclasses(result), total diff --git a/app/db/utils.py b/app/db/utils.py new file mode 100644 index 00000000..0df36aab --- /dev/null +++ b/app/db/utils.py @@ -0,0 +1,75 @@ +from typing import Any + +from app.models import Album as AlbumModel, Artist as ArtistModel, Track as TrackModel +from app.models.favorite import Favorite +from app.models.lastfm import SimilarArtist +from app.models.plugins import Plugin +from app.models.user import User + + +def track_to_dataclass(track: Any): + return TrackModel(**track._asdict()) + + +def tracks_to_dataclasses(tracks: Any): + return [track_to_dataclass(track) for track in tracks] + + +def album_to_dataclass(album: Any): + return AlbumModel(**album._asdict()) + + +def albums_to_dataclasses(albums: Any): + return [album_to_dataclass(album) for album in albums] + + +def artist_to_dataclass(artist: Any): + return ArtistModel(**artist._asdict()) + + +def artists_to_dataclasses(artists: Any): + return [artist_to_dataclass(artist) for artist in artists] + + +# SECTION: User data helpers + + +def similar_artist_to_dataclass(entry: Any): + entry_dict = entry._asdict() + del entry_dict["id"] + + return SimilarArtist(**entry_dict) + + +def similar_artists_to_dataclass(entries: Any): + return [similar_artist_to_dataclass(entry) for entry in entries] + + +def favorite_to_dataclass(entry: Any): + entry_dict = entry._asdict() + del entry_dict["id"] + + return Favorite(**entry_dict) + + +def favorites_to_dataclass(entries: Any): + return [favorite_to_dataclass(entry) for entry in entries] + + +def user_to_dataclass(entry: Any): + entry_dict = entry._asdict() + return User(**entry_dict) + + +def user_to_dataclasses(entries: Any): + return [user_to_dataclass(entry) for entry in entries] + + +def plugin_to_dataclass(entry: Any): + entry_dict = entry._asdict() + del entry_dict["id"] + return Plugin(**entry_dict) + + +def plugin_to_dataclasses(entries: Any): + return [plugin_to_dataclass(entry) for entry in entries] diff --git a/app/lib/albumslib.py b/app/lib/albumslib.py index c8f9802e..36034f3a 100644 --- a/app/lib/albumslib.py +++ b/app/lib/albumslib.py @@ -12,63 +12,21 @@ from app.store.albums import AlbumStore from app.store.tracks import TrackStore -def create_albums(): - """ - Creates albums from the tracks in the store. - """ - - # group all tracks by albumhash - tracks = TrackStore.tracks - tracks = sorted(tracks, key=lambda t: t.albumhash) - grouped = groupby(tracks, lambda t: t.albumhash) - - # create albums from the groups - albums: list[Track] = [] - for albumhash, tracks in grouped: - count = len(list(tracks)) - duration = sum(t.duration for t in tracks) - created_date = min(t.created_date for t in tracks) - - album = AlbumStore.create_album(list(tracks)[0]) - album.set_count(count) - album.set_duration(duration) - album.set_created_date(created_date) - - albums.append(album) - - return albums - - -def validate_albums(): - """ - Removes albums that have no tracks. - - Probably albums that were added from incompletely written files. - """ - - album_hashes = {t.albumhash for t in TrackStore.tracks} - albums = AlbumStore.albums - - for album in albums: - if album.albumhash not in album_hashes: - AlbumStore.remove_album(album) - - def remove_duplicate_on_merge_versions(tracks: list[Track]) -> list[Track]: """ Removes duplicate tracks when merging versions of the same album. """ - + # TODO! pass -def sort_by_track_no(tracks: list[Track]) -> list[dict[str, Any]]: - tracks = [asdict(t) for t in tracks] +def sort_by_track_no(tracks: list[Track]): + # tracks = [asdict(t) for t in tracks] for t in tracks: - track = str(t["track"]).zfill(3) - t["_pos"] = int(f"{t['disc']}{track}") + track = str(t.track).zfill(3) + t._pos = int(f"{t.disc}{track}") - tracks = sorted(tracks, key=lambda t: t["_pos"]) + tracks = sorted(tracks, key=lambda t: t._pos) return tracks diff --git a/app/lib/artistlib.py b/app/lib/artistlib.py index b40cb56a..c101b515 100644 --- a/app/lib/artistlib.py +++ b/app/lib/artistlib.py @@ -1,5 +1,3 @@ -from collections import namedtuple -from itertools import groupby import os import urllib from concurrent.futures import ThreadPoolExecutor @@ -12,9 +10,10 @@ from requests.exceptions import ConnectionError as RequestConnectionError from requests.exceptions import ReadTimeout from app import settings -from app.models import Album, Artist, Track -from app.store import artists as artist_store -from app.store.tracks import TrackStore +from app.db.libdata import ArtistTable + +# from app.store import artists as artist_store +# from app.store.tracks import TrackStore from app.utils.hashing import create_hash from app.utils.progressbar import tqdm @@ -107,22 +106,15 @@ class CheckArtistImages: # read all files in the artist image folder path = settings.Paths.get_sm_artist_img_path() - processed = "".join(os.listdir(path)).replace("webp", "") - - # filter out artists that already have an image - artists = filter( - lambda a: a.artisthash not in processed, artist_store.ArtistStore.artists - ) - artists = list(artists) - - # process the rest - key_artist_map = ((instance_key, artist) for artist in artists) + processed = [path.replace(".webp", "") for path in os.listdir(path)] + unprocessed = ArtistTable.get_artisthashes_not_in(processed) + key_artist_map = ((instance_key, artist) for artist in unprocessed) with ThreadPoolExecutor(max_workers=14) as executor: res = list( tqdm( executor.map(self.download_image, key_artist_map), - total=len(artists), + total=len(unprocessed), desc="Downloading missing artist images", ) ) @@ -130,7 +122,7 @@ class CheckArtistImages: list(res) @staticmethod - def download_image(_map: tuple[str, Artist]): + def download_image(_map: tuple[str, dict[str, str]]): """ Checks if an artist image exists and downloads it if not. @@ -142,16 +134,17 @@ class CheckArtistImages: return img_path = ( - Path(settings.Paths.get_sm_artist_img_path()) / f"{artist.artisthash}.webp" + Path(settings.Paths.get_sm_artist_img_path()) + / f"{artist['artisthash']}.webp" ) if img_path.exists(): return - url = get_artist_image_link(artist.name) + url = get_artist_image_link(artist["name"]) if url is not None: - return DownloadImage(url, name=f"{artist.artisthash}.webp") + return DownloadImage(url, name=f"{artist['artisthash']}.webp") # def fetch_album_bio(title: str, albumartist: str) -> str | None: """ Returns the album bio for a given album. """ @@ -183,89 +176,3 @@ class CheckArtistImages: # def __call__(self): # return fetch_album_bio(self.title, self.albumartist) - - -def get_artists_from_tracks(tracks: list[Track]) -> set[str]: - """ - Extracts all artists from a list of tracks. Returns a list of Artists. - """ - artists = set() - - master_artist_list = [[x.name for x in t.artists] for t in tracks] - artists = artists.union(*master_artist_list) - - return artists - - -def get_albumartists(albums: list[Album]) -> set[str]: - artists = set() - - for album in albums: - albumartists = [a.name for a in album.albumartists] - - artists.update(albumartists) - - return artists - - -def get_all_artists(tracks: list[Track], albums: list[Album]) -> list[Artist]: - TrackInfo = namedtuple( - "TrackInfo", - [ - "artisthash", - "albumhash", - "trackhash", - "duration", - "artistname", - "created_date", - ], - ) - src_tracks = TrackStore.tracks - all_tracks: set[TrackInfo] = set() - - for track in src_tracks: - artist_hashes = {(a.name, a.artisthash) for a in track.artists}.union( - (a.name, a.artisthash) for a in track.albumartists - ) - - for artist in artist_hashes: - track_info = TrackInfo( - artistname=artist[0], - artisthash=artist[1], - albumhash=track.albumhash, - trackhash=track.trackhash, - duration=track.duration, - created_date=track.created_date, - # work on created date - ) - - all_tracks.add(track_info) - - all_tracks = sorted(all_tracks, key=lambda x: x.artisthash) - all_tracks = groupby(all_tracks, key=lambda x: x.artisthash) - - artists = [] - - for artisthash, tracks in all_tracks: - tracks: list[TrackInfo] = list(tracks) - - artistname = ( - sorted({t.artistname for t in tracks})[0] - if len(tracks) > 1 - else tracks[0].artistname - ) - - albumcount = len({t.albumhash for t in tracks}) - duration = sum(t.duration for t in tracks) - created_date = min(t.created_date for t in tracks) - - artist = Artist(name=artistname) - - artist.set_trackcount(len(tracks)) - artist.set_albumcount(albumcount) - artist.set_duration(duration) - artist.set_created_date(created_date) - - artists.append(artist) - - return artists diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index 086fcb2b..e71dbb1f 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -52,47 +52,47 @@ def process_color(item_hash: str, is_album=True): return get_image_colors(str(path)) -class ProcessAlbumColors: - """ - Extracts the most dominant color from the album art and saves it to the database. - """ +# class ProcessAlbumColors: +# """ +# Extracts the most dominant color from the album art and saves it to the database. +# """ - def __init__(self, instance_key: str) -> None: - global PROCESS_ALBUM_COLORS_KEY - PROCESS_ALBUM_COLORS_KEY = instance_key +# def __init__(self, instance_key: str) -> None: +# global PROCESS_ALBUM_COLORS_KEY +# PROCESS_ALBUM_COLORS_KEY = instance_key - albums = [ - a - for a in AlbumStore.albums - if a is not None and a.colors is not None and len(a.colors) == 0 - ] +# albums = [ +# a +# for a in AlbumStore.albums +# if a is not None and a.colors is not None and len(a.colors) == 0 +# ] - with SQLiteManager() as cur: - try: - for album in tqdm(albums, desc="Processing missing album colors"): - if PROCESS_ALBUM_COLORS_KEY != instance_key: - raise PopulateCancelledError( - "A newer 'ProcessAlbumColors' instance is running. Stopping this one." - ) +# with SQLiteManager() as cur: +# try: +# for album in tqdm(albums, desc="Processing missing album colors"): +# if PROCESS_ALBUM_COLORS_KEY != instance_key: +# raise PopulateCancelledError( +# "A newer 'ProcessAlbumColors' instance is running. Stopping this one." +# ) - # TODO: Stop hitting the database for every album. - # Instead, fetch all the data from the database and - # check from memory. +# # TODO: Stop hitting the database for every album. +# # Instead, fetch all the data from the database and +# # check from memory. - exists = aldb.exists(album.albumhash, cur=cur) - if exists: - continue +# exists = aldb.exists(album.albumhash, cur=cur) +# if exists: +# continue - colors = process_color(album.albumhash) +# colors = process_color(album.albumhash) - if colors is None: - continue +# if colors is None: +# continue - album.set_colors(colors) - color_str = json.dumps(colors) - aldb.insert_one_album(cur, album.albumhash, color_str) - finally: - cur.close() +# album.set_colors(colors) +# color_str = json.dumps(colors) +# aldb.insert_one_album(cur, album.albumhash, color_str) +# finally: +# cur.close() class ProcessArtistColors: diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index 0f3340c2..85542e11 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -5,9 +5,10 @@ from app.logger import log from app.models import Folder from app.serializers.track import serialize_tracks from app.settings import SUPPORTED_FILES +from app.store.folder import FolderStore from app.utils.wintools import win_replace_slash -from app.db import TrackTable as TrackDB +from app.db.libdata import TrackTable as TrackDB def create_folder(path: str, trackcount=0, foldercount=0) -> Folder: @@ -43,8 +44,7 @@ def get_folders(paths: list[str]): Filters out folders that don't have any tracks and returns a list of folder objects. """ - folders = TrackDB.count_tracks_containing_paths(paths) - + folders = FolderStore.count_tracks_containing_paths(paths) return [ create_folder(f["path"], f["trackcount"], foldercount=0) for f in folders diff --git a/app/lib/populate.py b/app/lib/populate.py index 593e0282..ea1e1afe 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -1,3 +1,4 @@ +from dataclasses import asdict import os from collections import deque from concurrent.futures import ThreadPoolExecutor @@ -7,28 +8,26 @@ from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout from app import settings -from app.db import TrackTable +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb -from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb -from app.db.sqlite.settings import SettingsSQLMethods as sdb + +# from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb from app.db.sqlite.tracks import SQLiteTrackMethods -from app.lib.albumslib import validate_albums from app.lib.artistlib import CheckArtistImages -from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors +from app.lib.colorlib import ProcessArtistColors from app.lib.errors import PopulateCancelledError -from app.lib.taglib import extract_thumb, get_tags -from app.lib.trackslib import validate_tracks +from app.lib.taglib import extract_thumb from app.logger import log from app.models import Album, Artist, Track from app.models.lastfm import SimilarArtist from app.requests.artists import fetch_similar_artists -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore -from app.store.tracks import TrackStore from app.utils.filesystem import run_fast_scandir from app.utils.network import has_connection from app.utils.progressbar import tqdm +from app.db.userdata import SimilarArtistTable + get_all_tracks = SQLiteTrackMethods.get_all_tracks insert_many_tracks = SQLiteTrackMethods.insert_many_tracks remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths @@ -44,50 +43,49 @@ class Populate: also checks if the album art exists in the image path, if not tries to extract it. """ - def __init__(self, instance_key: str) -> None: - return + # def __init__(self, instance_key: str) -> None: + # return + # if len(dirs_to_scan) == 0: + # log.warning( + # ( + # "The root directory is not configured. " + # + "Open the app in your webbrowser to configure." + # ) + # ) + # return + + # try: + # if dirs_to_scan[0] == "$home": + # dirs_to_scan = [settings.Paths.USER_HOME_DIR] + # except IndexError: + # pass + + # files = set() + + # for _dir in dirs_to_scan: + # files = files.union(run_fast_scandir(_dir, full=True)[1]) + + # unmodified, modified_tracks = self.remove_modified(tracks) + # untagged = files - unmodified + + # if len(untagged) != 0: + # self.tag_untagged(untagged, instance_key) + + # self.extract_thumb_with_overwrite(modified_tracks) + + +class CordinateMedia: + """ + Cordinates the extracting of thumbnails + """ + + def __init__(self, instance_key: str): global POPULATE_KEY POPULATE_KEY = instance_key - validate_tracks() - validate_albums() - - tracks = get_all_tracks() - - dirs_to_scan = sdb.get_root_dirs() - - if len(dirs_to_scan) == 0: - log.warning( - ( - "The root directory is not configured. " - + "Open the app in your webbrowser to configure." - ) - ) - return - - try: - if dirs_to_scan[0] == "$home": - dirs_to_scan = [settings.Paths.USER_HOME_DIR] - except IndexError: - pass - - files = set() - - for _dir in dirs_to_scan: - files = files.union(run_fast_scandir(_dir, full=True)[1]) - - unmodified, modified_tracks = self.remove_modified(tracks) - untagged = files - unmodified - - if len(untagged) != 0: - self.tag_untagged(untagged, instance_key) - - self.extract_thumb_with_overwrite(modified_tracks) - try: ProcessTrackThumbnails(instance_key) - ProcessAlbumColors(instance_key) ProcessArtistColors(instance_key) except PopulateCancelledError as e: log.warn(e) @@ -95,10 +93,6 @@ class Populate: tried_to_download_new_images = False - ArtistStore.load_artists(instance_key) - AlbumStore.load_albums(instance_key) - TrackStore.load_all_tracks(instance_key) - if has_connection(): tried_to_download_new_images = True try: @@ -123,101 +117,101 @@ class Populate: log.warn(e) return - @staticmethod - def remove_modified(tracks: Generator[TrackTable, None, None]): - """ - Removes tracks from the database that have been modified - since they were added to the database. - """ + # @staticmethod + # def remove_modified(tracks: Generator[TrackTable, None, None]): + # """ + # Removes tracks from the database that have been modified + # since they were added to the database. + # """ - unmodified_paths = set() - modified_tracks: list[TrackTable] = [] - modified_paths = set() + # unmodified_paths = set() + # modified_tracks: list[TrackTable] = [] + # modified_paths = set() - for track in tracks: - try: - if track.last_mod == round(os.path.getmtime(track.filepath)): - unmodified_paths.add(track.filepath) - continue - except (FileNotFoundError, OSError) as e: - log.warning(e) # REVIEW More informations = good - TrackStore.remove_track_obj(track) - remove_tracks_by_filepaths(track.filepath) + # for track in tracks: + # try: + # if track.last_mod == round(os.path.getmtime(track.filepath)): + # unmodified_paths.add(track.filepath) + # continue + # except (FileNotFoundError, OSError) as e: + # log.warning(e) # REVIEW More informations = good + # TrackStore.remove_track_obj(track) + # remove_tracks_by_filepaths(track.filepath) - modified_paths.add(track.filepath) - modified_tracks.append(track) + # modified_paths.add(track.filepath) + # modified_tracks.append(track) - TrackStore.remove_tracks_by_filepaths(modified_paths) - remove_tracks_by_filepaths(modified_paths) + # TrackStore.remove_tracks_by_filepaths(modified_paths) + # remove_tracks_by_filepaths(modified_paths) - return unmodified_paths, modified_tracks + # return unmodified_paths, modified_tracks - @staticmethod - def tag_untagged(untagged: set[str], key: str): - pass - # for file in tqdm(untagged, desc="Reading files"): - # if POPULATE_KEY != key: - # log.warning("'Populate.tag_untagged': Populate key changed") - # return + # @staticmethod + # def tag_untagged(untagged: set[str], key: str): + # pass + # for file in tqdm(untagged, desc="Reading files"): + # if POPULATE_KEY != key: + # log.warning("'Populate.tag_untagged': Populate key changed") + # return - # tags = get_tags(file) + # tags = get_tags(file) - # if tags is not None: - # TrackTable.insert_one(tags) + # if tags is not None: + # TrackTable.insert_one(tags) - # ============================================= + # ============================================= - # log.info("Found %s new tracks", len(untagged)) - # # tagged_tracks: deque[dict] = deque() - # # tagged_count = 0 + # log.info("Found %s new tracks", len(untagged)) + # # tagged_tracks: deque[dict] = deque() + # # tagged_count = 0 - # favs = favdb.get_fav_tracks() - # records = dict() + # favs = favdb.get_fav_tracks() + # records = dict() - # for fav in favs: - # r = records.setdefault(fav[1], set()) - # r.add(fav[4]) + # for fav in favs: + # r = records.setdefault(fav[1], set()) + # r.add(fav[4]) - # tagged_tracks.append(tags) - # track = Track(**tags) + # tagged_tracks.append(tags) + # track = Track(**tags) - # track.fav_userids = list(records.get(track.trackhash, set())) + # track.fav_userids = list(records.get(track.trackhash, set())) - # TrackStore.add_track(track) + # TrackStore.add_track(track) - # if not AlbumStore.album_exists(track.albumhash): - # AlbumStore.add_album(AlbumStore.create_album(track)) + # if not AlbumStore.album_exists(track.albumhash): + # AlbumStore.add_album(AlbumStore.create_album(track)) - # for artist in track.artists: - # if not ArtistStore.artist_exists(artist.artisthash): - # ArtistStore.add_artist(Artist(artist.name)) + # for artist in track.artists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) - # for artist in track.albumartists: - # if not ArtistStore.artist_exists(artist.artisthash): - # ArtistStore.add_artist(Artist(artist.name)) + # for artist in track.albumartists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) - # tagged_count += 1 - # else: - # log.warning("Could not read file: %s", file) + # tagged_count += 1 + # else: + # log.warning("Could not read file: %s", file) - # if len(tagged_tracks) > 0: - # log.info("Adding %s tracks to database", len(tagged_tracks)) - # insert_many_tracks(tagged_tracks) + # if len(tagged_tracks) > 0: + # log.info("Adding %s tracks to database", len(tagged_tracks)) + # insert_many_tracks(tagged_tracks) - # log.info("Added %s/%s tracks", tagged_count, len(untagged)) + # log.info("Added %s/%s tracks", tagged_count, len(untagged)) - @staticmethod - def extract_thumb_with_overwrite(tracks: list[TrackTable]): - """ - Extracts the thumbnail from a list of filepaths, - overwriting the existing thumbnail if it exists, - for modified files. - """ - for track in tracks: - try: - extract_thumb(track.filepath, track.image, overwrite=True) - except FileNotFoundError: - continue + # @staticmethod + # def extract_thumb_with_overwrite(tracks: list[TrackTable]): + # """ + # Extracts the thumbnail from a list of filepaths, + # overwriting the existing thumbnail if it exists, + # for modified files. + # """ + # for track in tracks: + # try: + # extract_thumb(track.filepath, track.image, overwrite=True) + # except FileNotFoundError: + # continue def get_image(_map: tuple[str, Album]): @@ -235,7 +229,8 @@ def get_image(_map: tuple[str, Album]): raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed") matching_tracks = filter( - lambda t: t.albumhash == album.albumhash, TrackStore.tracks + lambda t: t.albumhash == album.albumhash, + TrackTable.get_tracks_by_albumhash(album.albumhash), ) try: @@ -254,8 +249,12 @@ def get_image(_map: tuple[str, Album]): pass -_cpu_count = os.cpu_count() -CPU_COUNT = _cpu_count // 2 if _cpu_count > 2 else _cpu_count +def get_cpu_count(): + """ + Returns the number of CPUs on the machine. + """ + cpu_count = os.cpu_count() or 0 + return cpu_count // 2 if cpu_count > 2 else cpu_count class ProcessTrackThumbnails: @@ -275,14 +274,14 @@ class ProcessTrackThumbnails: # filter out albums that already have thumbnails albums = filter( - lambda album: album.albumhash not in processed, AlbumStore.albums + lambda album: album.albumhash not in processed, AlbumTable.get_all() ) albums = list(albums) # process the rest key_album_map = ((instance_key, album) for album in albums) - with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: + with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor: results = list( tqdm( executor.map(get_image, key_album_map), @@ -307,16 +306,17 @@ def save_similar_artists(_map: tuple[str, Artist]): "'FetchSimilarArtistsLastFM': Populate key changed" ) - if lastfmdb.exists(artist.artisthash): + if SimilarArtistTable.exists(artist.artisthash): return - artist_hashes = fetch_similar_artists(artist.name) - artist_ = SimilarArtist(artist.artisthash, "~".join(artist_hashes)) + artists = fetch_similar_artists(artist.name) - if len(artist_.similar_artist_hashes) == 0: + # INFO: Nones mean there was a connection error + if artists is None: return - lastfmdb.insert_one(artist_) + artist_ = SimilarArtist(artist.artisthash, artists) + SimilarArtistTable.insert_one(asdict(artist_)) class FetchSimilarArtistsLastFM: @@ -326,17 +326,17 @@ class FetchSimilarArtistsLastFM: def __init__(self, instance_key: str) -> None: # read all artists from db - processed = lastfmdb.get_all() + processed = SimilarArtistTable.get_all() processed = ".".join(a.artisthash for a in processed) # filter out artists that already have similar artists - artists = filter(lambda a: a.artisthash not in processed, ArtistStore.artists) + artists = filter(lambda a: a.artisthash not in processed, ArtistTable.get_all()) artists = list(artists) # process the rest key_artist_map = ((instance_key, artist) for artist in artists) - with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: + with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor: try: print("Processing similar artists") results = list( diff --git a/app/lib/tagger.py b/app/lib/tagger.py index b9ee9d04..d8a679a6 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -1,55 +1,165 @@ +import os from pprint import pprint -from app.db import AlbumTable, ArtistTable, TrackTable -from app.lib.taglib import get_tags +from time import time +from typing import Generator +from app import settings +from app.config import UserConfig +from app.db.libdata import ArtistTable +from app.db.libdata import AlbumTable, TrackTable +from app.lib.populate import CordinateMedia +from app.lib.taglib import extract_thumb, get_tags +from app.models.track import Track +from app.store.folder import FolderStore from app.utils.filesystem import run_fast_scandir from app.utils.parsers import get_base_album_title from app.utils.progressbar import tqdm +from app.logger import log +from app.utils.threading import background + +POPULATE_KEY: float = 0 + class IndexTracks: - def __init__(self) -> None: - dirs_to_scan = ["/home/cwilvx/Music"] + def __init__(self, instance_key: float) -> None: + """ + Indexes all tracks in the database. + + An instance key is used to prevent multiple instances of the + same class from running at the same time. + """ + global POPULATE_KEY + POPULATE_KEY = instance_key + + # dirs_to_scan = sdb.get_root_dirs() + dirs_to_scan = UserConfig().rootDirs + + if len(dirs_to_scan) == 0: + log.warning( + ( + "The root directory is not configured. " + + "Open the app in your webbrowser to configure." + ) + ) + return + + try: + if dirs_to_scan[0] == "$home": + dirs_to_scan = [settings.Paths.USER_HOME_DIR] + except IndexError: + pass files = set() for _dir in dirs_to_scan: files = files.union(run_fast_scandir(_dir, full=True)[1]) - self.tag_untagged(files) - # unmodified, modified_tracks = self.remove_modified(tracks) - # untagged = files - unmodified + unmodified, modified_tracks = self.filter_modded() + untagged = files - unmodified - def tag_untagged(self, files: set[str]): + self.tag_untagged(untagged, instance_key) + self.extract_thumb_with_overwrite(modified_tracks) + + @staticmethod + def extract_thumb_with_overwrite(tracks: list[dict[str, str]]): + """ + Extracts the thumbnail from a list of filepaths, + overwriting the existing thumbnail if it exists, + for modified files. + """ + for track in tracks: + try: + extract_thumb( + track["filepath"], track["trackhash"] + ".webp", overwrite=True + ) + except FileNotFoundError: + continue + + @staticmethod + def filter_modded(): + """ + Removes tracks from the database that have been modified + since they were indexed. + + Returns a tuple of unmodified paths and modified tracks. + Unmodified paths are indexed and the modified tracks are + + """ + + unmodified_paths = set() + modified_tracks: list[dict[str, str]] = [] + + to_remove = set() + + for track in TrackTable.get_all(): + try: + if track.last_mod == round(os.path.getmtime(track.filepath)): + unmodified_paths.add(track.filepath) + continue + except (FileNotFoundError, OSError) as e: + log.warning(e) # REVIEW More informations = good + to_remove.add(track.filepath) + + modified_tracks.append( + { + "filepath": track.filepath, + "trackhash": track.trackhash, + } + ) + + to_remove = to_remove.union(set(t["filepath"] for t in modified_tracks)) + TrackTable.remove_tracks_by_filepaths(to_remove) + + # REVIEW: Remove after testing! + track = TrackTable.get_tracks_by_filepaths(list(to_remove)[:1]) + if track: + raise Exception("Track not removed") + # ============================================================= + + return unmodified_paths, modified_tracks + + def get_untagged(self): + tracks = TrackTable.get_all() + + def tag_untagged(self, files: set[str], key: float): + config = UserConfig() for file in tqdm(files, desc="Reading files"): - # if POPULATE_KEY != key: - # log.warning("'Populate.tag_untagged': Populate key changed") - # return + if POPULATE_KEY != key: + log.warning("'Populate.tag_untagged': Populate key changed") + return - tags = get_tags(file) + tags = get_tags(file, artist_separators=config.artistSeparators) if tags is not None: TrackTable.insert_one(tags) + FolderStore.filepaths.add(tags["filepath"]) del tags + print(f"{len(files)} new files indexed") + print("Done") + + class IndexAlbums: def __init__(self) -> None: albums = dict() + all_tracks: list[Track] = TrackTable.get_all() - all_tracks: list[TrackTable] = TrackTable.get_all() + if len(all_tracks) == 0: + return for track in all_tracks: if track.albumhash not in albums: albums[track.albumhash] = { "albumartists": track.albumartists, - "artisthashes": [a['artisthash'] for a in track.albumartists], + "artisthashes": [a["artisthash"] for a in track.albumartists], "albumhash": track.albumhash, "base_title": None, "color": None, "created_date": None, "date": None, "duration": track.duration, - "genres": [*track.genre] if track.genre else [], + "genres": [*track.genres] if track.genres else [], "og_title": track.og_album, "title": track.album, "trackcount": 1, @@ -63,8 +173,8 @@ class IndexAlbums: album["dates"].append(track.date) album["created_dates"].append(track.last_mod) - if track.genre: - album["genres"].extend(track.genre) + if track.genres: + album["genres"].extend(track.genres) for album in albums.values(): album["date"] = min(album["dates"]) @@ -79,20 +189,23 @@ class IndexAlbums: album["genres"] = genres album["base_title"], _ = get_base_album_title(album["og_title"]) + del genres del album["dates"] del album["created_dates"] - pprint(albums) - + AlbumTable.remove_all() AlbumTable.insert_many(list(albums.values())) del albums class IndexArtists: def __init__(self) -> None: - all_tracks: list[TrackTable] = TrackTable.get_all() + all_tracks: list[Track] = TrackTable.get_all() artists = dict() + if len(all_tracks) == 0: + return + for track in all_tracks: this_artists = track.artists @@ -100,32 +213,33 @@ class IndexArtists: if a not in this_artists: this_artists.append(a) - for artist in this_artists: - if artist["artisthash"] not in artists: - artists[artist["artisthash"]] = { + for thisartist in this_artists: + if thisartist["artisthash"] not in artists: + artists[thisartist["artisthash"]] = { "albumcount": None, "albums": {track.albumhash}, - "artisthash": artist["artisthash"], + "artisthash": thisartist["artisthash"], "created_dates": [track.last_mod], "dates": [track.date], "date": None, "duration": track.duration, - "genres": track.genre if track.genre else [], - "name": artist["name"], + "genres": track.genres if track.genres else [], + "name": None, + "names": {thisartist["name"]}, "trackcount": None, "tracks": {track.trackhash}, } else: - artist = artists[artist["artisthash"]] + artist = artists[thisartist["artisthash"]] artist["duration"] += track.duration artist["albums"].add(track.albumhash) artist["tracks"].add(track.trackhash) artist["dates"].append(track.date) artist["created_dates"].append(track.last_mod) + artist["names"].add(thisartist["name"]) - if track.genre: - artist["genres"].extend(track.genre) - + if track.genres: + artist["genres"].extend(track.genres) for artist in artists.values(): artist["albumcount"] = len(artist["albums"]) @@ -140,19 +254,35 @@ class IndexArtists: genres.append(genre) artist["genres"] = genres + artist["name"] = sorted(artist["names"])[0] + # INFO: Delete temporary keys + del artist["names"] del artist["tracks"] del artist["albums"] del artist["dates"] del artist["created_dates"] - pprint(artists) + # INFO: Delete local variables + del genres + + ArtistTable.remove_all() ArtistTable.insert_many(list(artists.values())) del artists + class IndexEverything: def __init__(self) -> None: - IndexTracks() + IndexTracks(instance_key=time()) IndexAlbums() IndexArtists() - pass + FolderStore.load_filepaths() + + # pass + + CordinateMedia(instance_key=str(time())) + + +@background +def index_everything(): + return IndexEverything() diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 03cafc1c..a987cc17 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -5,6 +5,7 @@ from pathlib import Path from pprint import pprint import re import sys +from typing import Any import pendulum from PIL import Image, UnidentifiedImageError @@ -86,7 +87,7 @@ def extract_thumb(filepath: str, webp_path: str, overwrite=False) -> bool: return False -def parse_date(date_str: str | None) -> int | None: +def parse_date(date_str: str) -> int | None: """ Extracts the date from a string and returns a timestamp. """ @@ -108,12 +109,13 @@ def clean_filename(filename: str): class ParseData: artist: str title: str + artist_separators: set[str] def __post_init__(self): - self.artist = split_artists(self.artist) + self.artist = split_artists(self.artist, self.artist_separators) -def extract_artist_title(filename: str): +def extract_artist_title(filename: str, artist_separators: set[str]): path = Path(filename).with_suffix("") path = clean_filename(str(path)) @@ -121,22 +123,24 @@ def extract_artist_title(filename: str): split_result = [x.strip() for x in split_result] if len(split_result) == 1: - return ParseData("", split_result[0]) + return ParseData("", split_result[0], artist_separators) if len(split_result) > 2: try: int(split_result[0]) - return ParseData(split_result[1], " - ".join(split_result[2:])) + return ParseData( + split_result[1], " - ".join(split_result[2:]), artist_separators + ) except ValueError: pass artist = split_result[0] title = split_result[1] - return ParseData(artist, title) + return ParseData(artist, title, artist_separators) -def get_tags(filepath: str): +def get_tags(filepath: str, artist_separators: set[str]): """ Returns the tags for a given audio file. """ @@ -150,7 +154,7 @@ def get_tags(filepath: str): return None try: - tags = TinyTag.get(filepath) + tags: Any = TinyTag.get(filepath) except: # noqa: E722 return None @@ -169,7 +173,7 @@ def get_tags(filepath: str): for tag in to_filename: p = getattr(tags, tag) if p == "" or p is None: - parse_data = extract_artist_title(filename) + parse_data = extract_artist_title(filename, artist_separators) title = parse_data.title setattr(tags, tag, title) @@ -179,7 +183,7 @@ def get_tags(filepath: str): if p == "" or p is None: if not parse_data: - parse_data = extract_artist_title(filename) + parse_data = extract_artist_title(filename, artist_separators) artist = parse_data.artist @@ -225,8 +229,8 @@ def get_tags(filepath: str): tags.artists = tags.artist tags.albumartists = tags.albumartist - split_artist = split_artists(tags.artist) - split_albumartists = split_artists(tags.albumartist) + split_artist = split_artists(tags.artist, separators=artist_separators) + split_albumartists = split_artists(tags.albumartist, separators=artist_separators) new_title = tags.title # TODO: Figure out which is the best spot to create these hashes @@ -237,7 +241,9 @@ def get_tags(filepath: str): # extract featured artists if config.extractFeaturedArtists: - feat, new_title = parse_feat_from_title(tags.title) + feat, new_title = parse_feat_from_title( + tags.title, separators=artist_separators + ) original_lower = "-".join([create_hash(a) for a in split_artist]) split_artist.extend(a for a in feat if create_hash(a) not in original_lower) @@ -262,8 +268,9 @@ def get_tags(filepath: str): for a in split_albumartists ] - tags.artisthashes = list({a["artisthash"] for a in tags.artists + tags.albumartists}) - + tags.artisthashes = list( + {a["artisthash"] for a in tags.artists + tags.albumartists} + ) # remove prod by if config.removeProdBy: @@ -295,26 +302,32 @@ def get_tags(filepath: str): # process genres if tags.genre: - tags.genre = tags.genre.lower() + src_genres: str = tags.genre + src_genres = src_genres.lower() # separators = {"/", ";", "&"} separators = set(config.genreSeparators) - contains_rnb = "r&b" in tags.genre - contains_rock = "rock & roll" in tags.genre + contains_rnb = "r&b" in src_genres + contains_rock = "rock & roll" in src_genres if contains_rnb: - tags.genre = tags.genre.replace("r&b", "RnB") + src_genres = src_genres.replace("r&b", "RnB") if contains_rock: - tags.genre = tags.genre.replace("rock & roll", "rock") + src_genres = src_genres.replace("rock & roll", "rock") for s in separators: - tags.genre = tags.genre.replace(s, ",") + src_genres = src_genres.replace(s, ",") - tags.genre = tags.genre.split(",") - tags.genre = [ - {"name": g.strip(), "genrehash": create_hash(g.strip())} for g in tags.genre + genres_list: list[str] = src_genres.split(",") + tags.genres = [ + {"name": g.strip(), "genrehash": create_hash(g.strip())} + for g in genres_list ] + tags.genrehashes = [g["genrehash"] for g in tags.genres] + else: + tags.genres = [] + tags.genrehashes = [] # sub underscore with space tags.title = tags.title.replace("_", " ") @@ -333,6 +346,10 @@ def get_tags(filepath: str): "filesize": tags.filesize, "samplerate": tags.samplerate, "track_total": tags.track_total, + "hashinfo": { + "algo": "sha1", + "format": "[:5]+[-5:]", # first 5 + last 5 chars + }, } tags.extra = {**tags.extra, **more_extra} @@ -357,6 +374,7 @@ def get_tags(filepath: str): "bitdepth", "artist", "albumartist", + "genre", ] for tag in to_delete: diff --git a/app/lib/trackslib.py b/app/lib/trackslib.py index 085dda0c..c8a7a74e 100644 --- a/app/lib/trackslib.py +++ b/app/lib/trackslib.py @@ -13,16 +13,6 @@ from app.utils.progressbar import tqdm from app.utils.threading import ThreadWithReturnValue -def validate_tracks() -> None: - """ - Removes track records whose files no longer exist. - """ - for track in tqdm(TrackStore.tracks, desc="Validating tracks"): - if not os.path.exists(track.filepath): - TrackStore.remove_track_obj(track) - trackdb.remove_tracks_by_filepaths(track.filepath) - - def get_leading_silence_end(filepath: str): """ Returns the leading silence of a track. diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py index 0ba670c2..8d7b9955 100644 --- a/app/lib/watchdogg.py +++ b/app/lib/watchdogg.py @@ -11,8 +11,10 @@ from watchdog.events import PatternMatchingEventHandler from watchdog.observers import Observer from app import settings +from app.config import UserConfig from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb -from app.db.sqlite.settings import SettingsSQLMethods as sdb + +# from app.db.sqlite.settings import SettingsSQLMethods as sdb from app.db.sqlite.tracks import SQLiteManager from app.db.sqlite.tracks import SQLiteTrackMethods as db from app.lib.colorlib import process_color @@ -43,7 +45,8 @@ class Watcher: while trials < 10: try: - dirs = sdb.get_root_dirs() + # dirs = sdb.get_root_dirs() + dirs = UserConfig().rootDirs dirs = [rf"{d}" for d in dirs] dir_map = [ @@ -152,7 +155,8 @@ def add_track(filepath: str) -> None: TrackStore.remove_track_by_filepath(filepath) - tags = get_tags(filepath) + config = UserConfig() + tags = get_tags(filepath, artist_separators=config.artistSeparators) # if the track is somehow invalid, return if tags is None or tags["bitrate"] == 0 or tags["duration"] == 0: diff --git a/app/migrations/__init__.py b/app/migrations/__init__.py index 570cc115..54b51bf5 100644 --- a/app/migrations/__init__.py +++ b/app/migrations/__init__.py @@ -7,7 +7,9 @@ Reads and applies the latest database migrations. import inspect import sys from types import ModuleType -from app.db.sqlite.migrations import MigrationManager + +# from app.db.sqlite.migrations import MigrationManager +from app.db.metadata import MigrationTable from app.logger import log from app.migrations import v1_3_0, v1_4_9 from app.migrations.base import Migration @@ -42,26 +44,29 @@ def apply_migrations(): modules = [v1_3_0, v1_4_9] migrations = [get_all_migrations(m) for m in modules] - index = MigrationManager.get_index() + # index = MigrationManager.get_index() + index = MigrationTable.get_version() all_migrations = [migration for sublist in migrations for migration in sublist] to_apply: list[Migration] = [] # if index is from old release, # get migrations from the "migrations" list - if index < 3: - _migrations = migrations[index:] - to_apply = [migration for sublist in _migrations for migration in sublist] - else: - to_apply = all_migrations[index:] + + # if index < 3: + # _migrations = migrations[index:] + # to_apply = [migration for sublist in _migrations for migration in sublist] + # else: + # to_apply = all_migrations[index:] - for migration in to_apply: - # try: - migration.migrate() - log.info("Applied migration: %s", migration.__name__) - # except Exception as e: - # log.error("Failed to run migration: %s", migration.__name__) - # log.error(e) + # for migration in to_apply: + # # try: + # migration.migrate() + # log.info("Applied migration: %s", migration.__name__) + # except Exception as e: + # log.error("Failed to run migration: %s", migration.__name__) + # log.error(e) # sys.exit(0) - MigrationManager.set_index(len(all_migrations)) + # MigrationManager.set_index(len(all_migrations)) + MigrationTable.set_version(len(all_migrations)) diff --git a/app/models/album.py b/app/models/album.py index 337f24cf..aa843e43 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -2,9 +2,6 @@ import dataclasses import datetime from dataclasses import dataclass -from app.config import UserConfig -from app.settings import SessionVarKeys, get_flag - from ..utils.hashing import create_hash from ..utils.parsers import get_base_title_and_versions, parse_feat_from_title from .artist import Artist @@ -27,15 +24,21 @@ class Album: date: int duration: int genres: list[dict[str, str]] + genrehashes: list[str] og_title: str title: str trackcount: int + is_favorite: bool + extra: dict type: str = "album" + image: str = "" versions: list[str] = dataclasses.field(default_factory=list) def __post_init__(self): - self.date = datetime.datetime.fromtimestamp(self.date).year + # self.date = datetime.datetime.fromtimestamp(self.date).year + self.image = self.albumhash + ".webp" + self.populate_versions() # albumhash: str # title: str diff --git a/app/models/artist.py b/app/models/artist.py index 0c50a60d..1ef9aa26 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -44,6 +44,13 @@ class Artist: date: int duration: int genres: list[dict[str, str]] + genrehashes: list[str] name: str trackcount: int is_favorite: bool + extra: dict + + image: str = "" + + def __post_init__(self): + self.image = self.artisthash + ".webp" \ No newline at end of file diff --git a/app/models/favorite.py b/app/models/favorite.py new file mode 100644 index 00000000..ab02d593 --- /dev/null +++ b/app/models/favorite.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import Any, Literal + + +@dataclass +class Favorite: + hash: str + type: Literal["album", "track", "artist"] + timestamp: int + userid: int + extra: dict[str, Any] \ No newline at end of file diff --git a/app/models/lastfm.py b/app/models/lastfm.py index ebd2333c..573ab4fd 100644 --- a/app/models/lastfm.py +++ b/app/models/lastfm.py @@ -1,13 +1,28 @@ from dataclasses import dataclass +from typing import Any + + +@dataclass +class SimilarArtistEntry: + artisthash: str + name: str + weight: float + scrobbles: int + listeners: int @dataclass class SimilarArtist: artisthash: str - similar_artist_hashes: str + similar_artists: list[SimilarArtistEntry] + def get_artist_hash_set(self) -> set[str]: """ Returns a set of similar artists. """ - return set(self.similar_artist_hashes.split("~")) + if not self.similar_artists: + return set() + + # INFO: + return set(a['artisthash'] for a in self.similar_artists) diff --git a/app/models/plugins.py b/app/models/plugins.py index d3b55f88..fac7f6ec 100644 --- a/app/models/plugins.py +++ b/app/models/plugins.py @@ -4,7 +4,7 @@ from dataclasses import dataclass @dataclass class Plugin: name: str - description: str active: bool settings: dict + extra: dict diff --git a/app/models/track.py b/app/models/track.py index 0a853b3e..140df3cd 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -1,20 +1,4 @@ -from dataclasses import dataclass, field -import os -from pathlib import Path - -from flask_jwt_extended import current_user - -from app.settings import SessionVarKeys, get_flag -from app.utils.hashing import create_hash -from app.utils.parsers import ( - clean_title, - get_base_title_and_versions, - parse_feat_from_title, - remove_prod, - split_artists, -) - -from .artist import ArtistMinimal +from dataclasses import dataclass @dataclass(slots=True) @@ -28,7 +12,7 @@ class Track: albumartists: list[dict[str, str]] albumhash: str artisthashes: list[str] - artists: str + artists: list[dict[str, str]] bitrate: int copyright: str date: int @@ -36,7 +20,8 @@ class Track: duration: int filepath: str folder: str - genre: list[dict[str, str]] + genres: list[dict[str, str]] + genrehashes: list[str] last_mod: int og_album: str og_title: str @@ -48,7 +33,15 @@ class Track: is_favorite: bool = False _pos: int = 0 _ati: str = "" + image: str = "" + def __post_init__(self): + self.image = self.albumhash + ".webp" + self.extra = { + "disc_total": self.extra.get("disc_total", 0), + "track_total": self.extra.get("track_total", 0), + "samplerate": self.extra.get("samplerate", -1), + } # album: str # albumartists: str | list[ArtistMinimal] diff --git a/app/models/user.py b/app/models/user.py index b8d05d01..3aca9b95 100644 --- a/app/models/user.py +++ b/app/models/user.py @@ -5,19 +5,15 @@ import json @dataclass(slots=True) class User: id: int - username: str - firstname: str - lastname: str - password: str - email: str image: str + password: str + username: str + roles: list[str] + extra: dict[str, str] = field(default_factory=dict) # NOTE: roles: ['admin', 'user', 'curator'] roles: list[str] = field(default_factory=lambda: ["user"]) - def __post_init__(self): - self.roles = json.loads(self.roles) - def todict(self): this_dict = asdict(self) del this_dict["password"] @@ -28,5 +24,5 @@ class User: return { "id": self.id, "username": self.username, - "firstname": self.firstname, + "firstname": self.extra["firstname"] if self.extra else "", } diff --git a/app/periodic_scan.py b/app/periodic_scan.py index 7a0af7e3..f8b19133 100644 --- a/app/periodic_scan.py +++ b/app/periodic_scan.py @@ -1,14 +1,16 @@ """ This module contains functions for the server """ + import time +from app.config import UserConfig from app.lib.populate import Populate, PopulateCancelledError -from app.settings import SessionVarKeys, get_flag, get_scan_sleep_time from app.utils.generators import get_random_str from app.utils.threading import background from app.logger import log + @background def run_periodic_scans(): """ @@ -23,10 +25,7 @@ def run_periodic_scans(): # ValidateAlbumThumbs() # ValidatePlaylistThumbs() - run_periodic_scan = True - - while run_periodic_scan: - run_periodic_scan = get_flag(SessionVarKeys.DO_PERIODIC_SCANS) + while UserConfig().enablePeriodicScans: try: Populate(instance_key=get_random_str()) @@ -34,5 +33,4 @@ def run_periodic_scans(): log.error("'run_periodic_scans': Periodic scan cancelled.") pass - sleep_time = get_scan_sleep_time() - time.sleep(sleep_time) + time.sleep(UserConfig().scanInterval) diff --git a/app/plugins/register.py b/app/plugins/register.py index 7ace2260..b809722c 100644 --- a/app/plugins/register.py +++ b/app/plugins/register.py @@ -1,5 +1,18 @@ -from app.db.sqlite.plugins import PluginsMethods +from app.db.userdata import PluginTable +from sqlalchemy.exc import IntegrityError def register_plugins(): - PluginsMethods.insert_lyrics_plugin() + try: + PluginTable.insert_one( + { + "name": "lyrics_finder", + "active": False, + "settings": {"auto_download": False}, + "extra": { + "description": "Find lyrics from the internet", + }, + } + ) + except IntegrityError: + pass diff --git a/app/requests/artists.py b/app/requests/artists.py index 05e26336..784c9167 100644 --- a/app/requests/artists.py +++ b/app/requests/artists.py @@ -7,6 +7,7 @@ import urllib.parse import requests from requests import ConnectionError, HTTPError, ReadTimeout +from app.models.lastfm import SimilarArtistEntry from app.utils.hashing import create_hash @@ -20,7 +21,7 @@ def fetch_similar_artists(name: str): response = requests.get(url, timeout=10) response.raise_for_status() except (ConnectionError, ReadTimeout, HTTPError): - return [] + return None data = response.json() @@ -29,5 +30,15 @@ def fetch_similar_artists(name: str): except KeyError: return [] - for artist in artists: - yield create_hash(artist["name"]) + return [ + SimilarArtistEntry( + **{ + "artisthash": create_hash(artist["name"]), + "name": artist["name"], + "weight": artist["weight"], + "listeners": int(artist["listeners"]), + "scrobbles": int(artist["scrobbles"]), + } + ) + for artist in artists + ] diff --git a/app/settings.py b/app/settings.py index c3c308fd..d8a7036f 100644 --- a/app/settings.py +++ b/app/settings.py @@ -244,17 +244,6 @@ class SessionVarKeys: SHOW_ALBUMS_AS_SINGLES = "SHOW_ALBUMS_AS_SINGLES" -def get_flag(key: SessionVarKeys) -> bool: - return getattr(SessionVars, key) - - -def set_flag(key: SessionVarKeys, value: Any): - setattr(SessionVars, key, value) - - -def get_scan_sleep_time() -> int: - return SessionVars.PERIODIC_SCAN_INTERVAL - class TCOLOR: """ diff --git a/app/setup/__init__.py b/app/setup/__init__.py index c3d37f61..60738d68 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -3,11 +3,11 @@ Prepares the server for use. """ import uuid -from app.db.sqlite.settings import load_settings from app.setup.files import create_config_dir from app.setup.sqlite import run_migrations, setup_sqlite from app.store.albums import AlbumStore from app.store.artists import ArtistStore +from app.store.folder import FolderStore from app.store.tracks import TrackStore from app.utils.generators import get_random_str from app.config import UserConfig @@ -29,20 +29,21 @@ def run_setup(): setup_sqlite() run_migrations() - try: - load_settings() - except IndexError: - # settings table is empty - pass + # try: + # load_settings() + # except IndexError: + # # settings table is empty + # pass def load_into_mem(): """ Load all tracks, albums, and artists into memory. """ - instance_key = get_random_str() + # instance_key = get_random_str() # INFO: Load all tracks, albums, and artists into memory - TrackStore.load_all_tracks(instance_key) - AlbumStore.load_albums(instance_key) - ArtistStore.load_artists(instance_key) + # TrackStore.load_all_tracks(instance_key) + # AlbumStore.load_albums(instance_key) + # ArtistStore.load_artists(instance_key) + FolderStore.load_filepaths() \ No newline at end of file diff --git a/app/setup/sqlite.py b/app/setup/sqlite.py index 667d2e4b..8ce8dbe5 100644 --- a/app/setup/sqlite.py +++ b/app/setup/sqlite.py @@ -3,11 +3,15 @@ Module to setup Sqlite databases and tables. Applies migrations. """ +from app.db.userdata import UserTable from app.db.sqlite import create_connection, create_tables, queries from app.db.sqlite.auth import SQLiteAuthMethods as authdb from app.migrations import apply_migrations from app.settings import Db +from app.db import create_all +from app.db.libdata import create_all as create_all_libdata + def run_migrations(): """ @@ -20,18 +24,8 @@ def setup_sqlite(): """ Create Sqlite databases and tables. """ - # if os.path.exists(DB_PATH): - # os.remove(DB_PATH) + create_all() + create_all_libdata() - app_db_conn = create_connection(Db.get_app_db_path()) - user_db_conn = create_connection(Db.get_userdata_db_path()) - - create_tables(app_db_conn, queries.CREATE_APPDB_TABLES) - create_tables(user_db_conn, queries.CREATE_USERDATA_TABLES) - create_tables(app_db_conn, queries.CREATE_MIGRATIONS_TABLE) - - if not authdb.get_all_users(): - authdb.insert_default_user() - - app_db_conn.close() - user_db_conn.close() + if not UserTable.get_all(): + UserTable.insert_default_user() diff --git a/app/store/artists.py b/app/store/artists.py index 84bcb242..82cbf976 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -1,7 +1,6 @@ import json from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb -from app.lib.artistlib import get_all_artists from app.models import Artist from app.utils.bisection import use_bisection from app.utils.customlist import CustomList diff --git a/app/store/folder.py b/app/store/folder.py new file mode 100644 index 00000000..0bfe3ae3 --- /dev/null +++ b/app/store/folder.py @@ -0,0 +1,95 @@ +from sortedcontainers import SortedSet +from concurrent.futures import ThreadPoolExecutor + +from app.db.libdata import TrackTable + + +class FolderStore: + """ + The Folder store is used to hold all the indexed tracks filepaths in memory + for fast count operations when browsing the folder page. + + Counting from the database is super slow, even with a small number of folders to get the count for. Up to 700ms for 10 folders. By using this store, we are able to reduce that to less than 10ms. + """ + + filepaths: SortedSet = SortedSet() + + @classmethod + def load_filepaths(cls): + """ + Load all the filepaths from the database into memory. + + This is needed to speed up the process of counting the number of tracks in the folder page. + """ + cls.filepaths.clear() + + tracks = TrackTable.get_all() + for track in tracks: + cls.filepaths.add(track.filepath) + + + @classmethod + def count_tracks_containing_paths(cls, paths: list[str]): + """ + Count the number of tracks in each directory. + + Uses a ThreadPoolExecutor to count the number of tracks + in each directory for fast execution time. + """ + results: list[dict[str, int | str]] = [] + + with ThreadPoolExecutor() as executor: + res = executor.map(countFilepathsInDir, paths) + results = [ + {"path": path, "trackcount": count} for path, count in zip(paths, res) + ] + + return results + + +def getIndexOfFirstMatch(strings: list[str], prefix: str): + """ + Find the index of the first path that starts with the given path. + + Uses a binary search algorithm to find the index. + """ + + left = 0 + right = len(strings) - 1 + + while left <= right: + mid = (left + right) // 2 + + if strings[mid].startswith(prefix): + if mid == 0 or not strings[mid - 1].startswith(prefix): + return mid + right = mid - 1 + elif strings[mid] < prefix: + left = mid + 1 + else: + right = mid - 1 + + return -1 + + +def countFilepathsInDir(dirpath: str): + """ + Counts the number of filepaths that start with the given directory path. + + Gets the index of the first path that starts with the given directory path, + then checks each path after that to see if it starts with the given directory path. + """ + index = getIndexOfFirstMatch(FolderStore.filepaths, dirpath) + + if index == -1: + return 0 + + paths: list[str] = [] + + for path in FolderStore.filepaths[index:]: + if path.startswith(dirpath): + paths.append(path) + else: + break + + return len(paths) diff --git a/app/utils/dates.py b/app/utils/dates.py index f5495a2d..ebca97f1 100644 --- a/app/utils/dates.py +++ b/app/utils/dates.py @@ -26,7 +26,7 @@ def create_new_date(date: datetime = None) -> str: return date.strftime(_format) -def timestamp_to_time_passed(timestamp: str): +def timestamp_to_time_passed(timestamp: str | int): """ Converts a timestamp to time passed. e.g. 2 minutes ago, 1 hour ago, yesterday, 2 days ago, 2 weeks ago, etc. """ diff --git a/app/utils/hashing.py b/app/utils/hashing.py index 7e4f1b27..98a883ea 100644 --- a/app/utils/hashing.py +++ b/app/utils/hashing.py @@ -34,6 +34,7 @@ def create_hash(*args: str, decode=False, limit=10) -> str: str_ = str_.encode("utf-8") str_ = hashlib.sha1(str_).hexdigest() + # INFO: Return first 5 + last 5 characters return ( str_[: limit // 2] + str_[-limit // 2 :] if limit % 2 == 0 diff --git a/app/utils/parsers.py b/app/utils/parsers.py index 129b7835..f45c871f 100644 --- a/app/utils/parsers.py +++ b/app/utils/parsers.py @@ -1,14 +1,12 @@ import re from app.enums.album_versions import AlbumVersionEnum, get_all_keywords -from app.settings import SessionVarKeys, get_flag -def split_artists(src: str): +def split_artists(src: str, separators: set[str]): """ Splits a string of artists into a list of artists. """ - separators: set = get_flag(SessionVarKeys.ARTIST_SEPARATORS) for sep in separators: src = src.replace(sep, ",") @@ -38,7 +36,7 @@ def remove_prod(title: str) -> str: return title.strip() -def parse_feat_from_title(title: str) -> tuple[list[str], str]: +def parse_feat_from_title(title: str, separators: set[str]) -> tuple[list[str], str]: """ Extracts featured artists from a song title using regex. """ @@ -56,7 +54,7 @@ def parse_feat_from_title(title: str) -> tuple[list[str], str]: return [], title artists = match.group(1) - artists = split_artists(artists) + artists = split_artists(artists, separators) # remove "feat" group from title new_title = re.sub(regex, "", title, flags=re.IGNORECASE) diff --git a/manage.py b/manage.py index 20eda458..e0baabf2 100644 --- a/manage.py +++ b/manage.py @@ -45,7 +45,7 @@ mimetypes.add_type("image/gif", ".gif") mimetypes.add_type("font/woff", ".woff") mimetypes.add_type("application/manifest+json", ".webmanifest") -logging.disable(logging.CRITICAL) +# logging.disable(logging.CRITICAL) # werkzeug = logging.getLogger("werkzeug") # werkzeug.setLevel(logging.ERROR) diff --git a/poetry.lock b/poetry.lock index 914b502c..10c11ade 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2616,4 +2616,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "9c7ba20671a6a3b59dbb120e3e56ded7e4dfcbf2de14418bdef41059233cdcb1" +content-hash = "80cb2755efc6cec2cb20d50cb8927dee554991741283e70e7a2665e6253b895d" diff --git a/pyproject.toml b/pyproject.toml index de3dece2..65574f5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ flask-openapi3 = "^3.0.2" flask-jwt-extended = "^4.6.0" sqlalchemy = "^2.0.31" memory-profiler = "^0.61.0" +sortedcontainers = "^2.4.0" [tool.poetry.dev-dependencies] pylint = "^2.15.5"