TDvorak/swingmusic-extended
1
0
Fork 0
mirror of https://github.com/Dvorinka/swingmusic-extended.git synced 2026-06-03 20:13:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

combine userdata and swing db into one

Browse Source 
+ port populate to new db interface
+ add genrehashes and hash info to tracks
+ properly structure new db table files
+ move helpers to dedicated utils file
+ move settings from db to config file
+ move artists, albums, auth and favorites endpoint to new db interface
+ use folder store to index filepaths
+ paginate favorite pages
+ 56 moretiny changes 😅
This commit is contained in:
cwilvx
2024-06-30 15:06:33 +03:00
parent 1a66194c6c
commit 4a9f804e70
53 changed files with 1719 additions and 1353 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/changelog.md
+5
View File
@@ -25,3 +25,8 @@ interface Genre {
}
```
- Pairing via QR Code has been split into 2 endpoint:
1. `/getpaircode`
2. `/pair`
-
TODO.md
+18 -1
View File
@@ -1,5 +1,7 @@
# TODO
- Migrations:
1. Move userdata to new hashing algorithm
- favorites ✅
- playlists
@@ -16,11 +18,11 @@
- Recreate album hash if featured artists are discover
- Implement checking if is clean install and skip migrations!
<!-- CHECKPOINT -->
<!-- ALBUM PAGE! -->
# DONE
- Support auth headers
- Add recently played playlist
- Move user track logs to user zero
@@ -28,3 +30,18 @@
- Store (and read) from the correct user account:
1. Playlists
2. Favorites
# THE BIG ONE
- Updating settings
- Cleaning out commented code
- Watchdog
- Periodic scans
- Remove legacy db methods
- Remove all stores
- Review: We don't need server side image colors
- Clean up main db and userdata modules
- Move plugins to a config file
- What about our migrations?
- Add userid to queries
- Remove duplicates on artist page (test with Hanson)
app/api/__init__.py
+4 -2
View File
@@ -11,9 +11,9 @@ from flask_openapi3 import OpenAPI
from flask_jwt_extended import JWTManager
from app.config import UserConfig
from app.db.userdata import UserTable
from app.settings import Info as AppInfo
from .plugins import lyrics as lyrics_plugin
from app.db.sqlite.auth import SQLiteAuthMethods as authdb
from app.api import (
album,
artist,
@@ -92,7 +92,9 @@ def create_api():
def user_lookup_callback(_jwt_header, jwt_data):
identity = jwt_data["sub"]
userid = identity["id"]
user = authdb.get_user_by_id(userid)
user = UserTable.get_by_id(userid)
if user:
return user.todict()
# Register all the API blueprints
app/api/album.py
+27 -40
View File
@@ -12,15 +12,14 @@ from flask_openapi3 import APIBlueprint
from app.api.apischemas import AlbumHashSchema, AlbumLimitSchema, ArtistHashSchema
from app.config import UserConfig
from app.db import AlbumTable as AlbumDb, TrackTable as TrackDb
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable as AlbumDb, TrackTable as TrackDb
from app.db.userdata import SimilarArtistTable
from app.settings import Defaults
from app.models import FavType, Track
from app.store.albums import AlbumStore
from app.store.tracks import TrackStore
from app.utils.hashing import create_hash
from app.lib.albumslib import sort_by_track_no
from app.serializers.album import serialize_for_card, serialize_for_card_many
from app.serializers.track import serialize_track
from app.serializers.track import serialize_track, serialize_tracks
from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb
from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
@@ -52,9 +51,22 @@ def get_album_tracks_and_info(body: AlbumHashSchema):
album.type = album.check_type(
tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles
)
album.populate_versions()
return {"info": album, "tracks": tracks}
track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks})
return {
"info": album,
"extra": {
# INFO: track_total is the sum of a set of track_total values from each track
# ASSUMPTIONS
# 1. All the tracks have the correct track totals
# 2. Tracks with the same track total are from the same disc
"track_total": track_total,
"avg_bitrate": sum(t.bitrate for t in tracks) // len(tracks),
},
"copyright": tracks[0].copyright,
"tracks": serialize_tracks(tracks, remove_disc=False),
}
@api.get("/<albumhash>/tracks")
@@ -68,7 +80,7 @@ def get_album_tracks(path: AlbumHashSchema):
tracks = TrackDb.get_tracks_by_albumhash(path.albumhash)
tracks = sort_by_track_no(tracks)
return tracks
return serialize_tracks(tracks)
class GetMoreFromArtistsBody(AlbumLimitSchema):
@@ -138,30 +150,14 @@ def get_album_versions(body: GetAlbumVersionsBody):
artisthash = body.artisthash
albums = AlbumDb.get_albums_by_base_title(base_title)
print(albums)
albums = [
a
for a in albums
if a.og_title != og_album_title
and artisthash in {a["artisthash"] for a in a.albumartists}
]
print(albums)
# albums = AlbumStore.get_albums_by_artisthash(artisthash)
# albums = [
# a
# for a in albums
# if create_hash(a.base_title) == create_hash(base_title)
# and create_hash(og_album_title) != create_hash(a.og_title)
# ]
# for a in albums:
# tracks = TrackStore.get_tracks_by_albumhash(a.albumhash)
# a.get_date_from_tracks(tracks)
return albums
return serialize_for_card_many(albums)
class GetSimilarAlbumsQuery(ArtistHashSchema, AlbumLimitSchema):
@@ -178,24 +174,15 @@ def get_similar_albums(query: GetSimilarAlbumsQuery):
artisthash = query.artisthash
limit = query.limit
similar_artists = lastfmdb.get_similar_artists_for(artisthash)
similar_artists = SimilarArtistTable.get_by_hash(artisthash)
if similar_artists is None:
return {"albums": []}
return []
artisthashes = similar_artists.get_artist_hash_set()
artists = ArtistTable.get_artists_by_artisthashes(artisthashes)
if len(artisthashes) == 0:
return {"albums": []}
albums = AlbumDb.get_albums_by_artisthashes([a.artisthash for a in artists])
sample = random.sample(albums, min(len(albums), limit))
albums = [AlbumStore.get_albums_by_artisthash(a) for a in artisthashes]
albums = [a for sublist in albums for a in sublist]
albums = list({a.albumhash: a for a in albums}.values())
try:
albums = random.sample(albums, min(len(albums), limit))
except ValueError:
pass
return [serialize_for_card(a) for a in albums[:limit]]
return serialize_for_card_many(sample[:limit])
app/api/artist.py
+12 -91
View File
@@ -2,12 +2,11 @@
Contains all the artist(s) routes.
"""
from itertools import groupby
import math
import random
from datetime import datetime
from itertools import groupby
from flask_jwt_extended import current_user
from flask_openapi3 import APIBlueprint, Tag
from pydantic import Field
from app.api.apischemas import (
@@ -18,18 +17,13 @@ from app.api.apischemas import (
)
from app.config import UserConfig
from app.db import AlbumTable, ArtistTable, TrackTable
from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as fmdb
from app.models import Album, FavType
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable, TrackTable
from app.db.userdata import SimilarArtistTable
from app.serializers.album import serialize_for_card_many
from app.serializers.track import serialize_tracks
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
bp_tag = Tag(name="Artist", description="Single artist")
api = APIBlueprint("artist", __name__, url_prefix="/artist", abp_tags=[bp_tag])
@@ -45,8 +39,6 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema):
limit = query.limit
artist = ArtistTable.get_artist_by_hash(artisthash)
print(artist)
if artist is None:
return {"error": "Artist not found"}, 404
@@ -56,8 +48,6 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema):
if artist.albumcount == 0 and tcount < 10:
limit = tcount
# artist.is_favorite = favdb.check_is_favorite(artisthash, FavType.artist)
try:
year = datetime.fromtimestamp(artist.date).year
except ValueError:
@@ -106,7 +96,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery):
t.albumhash for t in tracks if t.albumhash not in {a.albumhash for a in albums}
}
albums.extend(AlbumTable.get_albums_by_hash(missing_albumhashes))
albums.extend(AlbumTable.get_albums_by_albumhashes(missing_albumhashes))
albumdict = {a.albumhash: a for a in albums}
config = UserConfig()
@@ -117,43 +107,6 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery):
if album:
album.check_type(list(tracks), config.showAlbumsAsSingles)
# all_albums = AlbumStore.get_albums_by_artisthash(artisthash)
# start: check for missing albums. ie. compilations and features
# all_tracks = TrackStore.get_tracks_by_artisthash(artisthash)
# track_albums = set(t.albumhash for t in all_tracks)
# missing_album_hashes = track_albums.difference(set(a.albumhash for a in all_albums))
# if len(missing_album_hashes) > 0:
# missing_albums = AlbumStore.get_albums_by_hashes(list(missing_album_hashes))
# all_albums.extend(missing_albums)
# end check
# def get_album_tracks(albumhash: str):
# tracks = [t for t in all_tracks if t.albumhash == albumhash]
# if len(tracks) > 0:
# return tracks
# return TrackStore.get_tracks_by_albumhash(albumhash)
# for a in all_albums:
# a.check_type()
# album_tracks = get_album_tracks(a.albumhash)
# if len(album_tracks) == 0:
# continue
# a.get_date_from_tracks(album_tracks)
# if a.date == 0:
# AlbumStore.remove_album_by_hash(a.albumhash)
# continue
# a.is_single(album_tracks)
albums = [a for a in albumdict.values()]
all_albums = sorted(albums, key=lambda a: str(a.date), reverse=True)
@@ -174,29 +127,6 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery):
else:
res["albums"].append(album)
# def remove_EPs_and_singles(albums_: list[Album]):
# albums_ = [a for a in albums_ if not a.type == "single"]
# albums_ = [a for a in albums_ if not a.type == "ep"]
# return albums_
# albums = filter(lambda a: artisthash in missing_albumhashes, all_albums)
# albums = list(albums)
# albums = remove_EPs_and_singles(albums)
# compilations = [a for a in albums if a.is_compilation]
# for c in compilations:
# albums.remove(c)
# appearances = filter(lambda a: artisthash not in a.albumartists_hashes, all_albums)
# appearances = list(appearances)
# appearances = remove_EPs_and_singles(appearances)
# artist = ArtistStore.get_artist_by_hash(artisthash)
# if artist is None:
# return {"error": "Artist not found"}, 404
if return_all:
limit = len(all_albums)
@@ -215,8 +145,8 @@ def get_all_artist_tracks(path: ArtistHashSchema):
Returns all artists by a given artist.
"""
tracks = TrackStore.get_tracks_by_artisthash(path.artisthash)
# tracks = TrackStore.get_tracks_by_artisthash(path.artisthash)
tracks = TrackTable.get_tracks_by_artisthash(path.artisthash)
return serialize_tracks(tracks)
@@ -226,23 +156,14 @@ def get_similar_artists(path: ArtistHashSchema, query: ArtistLimitSchema):
Get similar artists.
"""
limit = query.limit
artist = ArtistStore.get_artist_by_hash(path.artisthash)
if artist is None:
return {"error": "Artist not found"}, 404
result = fmdb.get_similar_artists_for(artist.artisthash)
result = SimilarArtistTable.get_by_hash(path.artisthash)
if result is None:
return {"artists": []}
return []
similar = ArtistStore.get_artists_by_hashes(result.get_artist_hash_set())
similar = ArtistTable.get_artists_by_artisthashes(result.get_artist_hash_set())
if len(similar) > limit:
similar = random.sample(similar, limit)
similar = random.sample(similar, min(limit, len(similar)))
return similar[:limit]
# TODO: Rewrite this file using generators where possible
app/api/auth.py
+42 -32
View File
@@ -14,7 +14,8 @@ from pydantic import BaseModel, Field
from flask_openapi3 import Tag
from flask_openapi3 import APIBlueprint
from app.db.sqlite.auth import SQLiteAuthMethods as authdb
# from app.db.sqlite.auth import SQLiteAuthMethods as authdb
from app.db.userdata import UserTable
from app.utils.auth import check_password, hash_password
from app.config import UserConfig
@@ -65,7 +66,7 @@ def login(body: LoginBody):
Authenticate using username and password
"""
user = authdb.get_user_by_username(body.username)
user = UserTable.get_by_username(body.username)
if user is None:
return {"msg": "User not found"}, 404
@@ -87,20 +88,12 @@ def login(body: LoginBody):
pair_token = dict()
class PairDeviceQuery(BaseModel):
code: str = Field("", description="The code")
@api.get("/pair")
@jwt_required(optional=True)
def pair_device(query: PairDeviceQuery):
@api.get("/getpaircode")
def get_pair():
"""
Pair the Swing Music mobile app with this server
Send a code to get an access token. Send an authenticated request without the code to generate a new token.
Get a new pair code to log in to thee Swing Music mobile app
"""
# INFO: if user is already logged in, create a new pair code
if current_user:
token = create_new_token(get_jwt_identity())
key = token["accesstoken"][-6:]
@@ -111,19 +104,26 @@ def pair_device(query: PairDeviceQuery):
return {"code": key}
# INFO: if there's a pair code, return the token
if query.code:
token = pair_token.get(query.code, None)
class PairDeviceQuery(BaseModel):
code: str = Field("", description="The code")
@api.post("/pair")
@jwt_required(optional=True)
def pair_with_code(body: PairDeviceQuery):
"""
Get an access token by sending a pair code. NOTE: A code can only be used once!
"""
global pair_token
token = pair_token.get(body.code, None)
if token:
# INFO: reset pair_token
pair_token = {}
return token
return {"msg": "Invalid code"}, 400
return {"msg": "No code provided"}, 400
@api.post("/refresh")
@jwt_required(refresh=True)
@@ -133,6 +133,8 @@ def refresh():
>>> Headers:
>>> Authorization: Bearer <refresh_token>
Won't work with cookies!!!
"""
user = get_jwt_identity()
return create_new_token(user)
@@ -153,7 +155,6 @@ def update_profile(body: UpdateProfileBody):
"""
user = {
"id": body.id,
"email": body.email,
"username": body.username,
"password": body.password,
"roles": body.roles,
@@ -172,7 +173,8 @@ def update_profile(body: UpdateProfileBody):
if "admin" not in current_user["roles"]:
return {"msg": "Only admins can update roles"}, 403
all_users = authdb.get_all_users()
# all_users = authdb.get_all_users()
all_users = UserTable.get_all()
if "admin" not in body.roles:
# check if we're removing the last admin
admins = [user for user in all_users if "admin" in user.roles]
@@ -195,7 +197,9 @@ def update_profile(body: UpdateProfileBody):
clean_user = {k: v for k, v in user.items() if v}
try:
return authdb.update_user(clean_user)
# return authdb.update_user(clean_user)
UserTable.update_one(clean_user)
return UserTable.get_by_id(user["id"]).todict()
except sqlite3.IntegrityError:
return {"msg": "Username already exists"}, 400
@@ -216,11 +220,18 @@ def create_user(body: UpdateProfileBody):
}
# check if user already exists
if authdb.get_user_by_username(user["username"]):
if UserTable.get_by_username(user["username"]):
return {"msg": "Username already exists"}, 400
userid = authdb.insert_user(user)
return authdb.get_user_by_id(userid).todict()
UserTable.insert_one(user)
user = UserTable.get_by_username(user["username"])
if user:
return user.todict()
return {
"msg": "Failed to create user",
}, 500
@api.post("/profile/guest/create")
@@ -230,14 +241,14 @@ def create_guest_user():
Create a guest user
"""
# check if guest user already exists
guest_user = authdb.get_user_by_username("guest")
guest_user = UserTable.get_by_username("guest")
if guest_user:
return {
"msg": "Guest user already exists",
}, 400
userid = authdb.insert_guest_user()
userid = UserTable.insert_guest_user()
if userid:
return {
@@ -264,12 +275,12 @@ def delete_user(body: DeleteUseBody):
return {"msg": "Sorry! you cannot delete yourselfu"}, 400
# prevent deleting the only admin
users = authdb.get_all_users()
users = UserTable.get_all()
admins = [user for user in users if "admin" in user.roles]
if len(admins) == 1 and admins[0].username == body.username:
return {"msg": "Cannot delete the only admin"}, 400
authdb.delete_user_by_username(body.username)
UserTable.remove_by_username(body.username)
return {"msg": f"User {body.username} deleted"}
@@ -308,8 +319,7 @@ def get_all_users(query: GetAllUsersQuery):
"users": [],
}
users = authdb.get_all_users()
users = UserTable.get_all()
is_admin = current_user and "admin" in current_user["roles"]
settings["enableGuest"] = [
user for user in users if user.username == "guest"
@@ -355,7 +365,7 @@ def get_all_users(query: GetAllUsersQuery):
if query.simplified:
res["users"] = [user.todict_simplified() for user in users]
else:
res["users"] = [user.todict() for user in users]
return res
app/api/favorites.py
+73 -97
View File
@@ -6,18 +6,19 @@ from flask_openapi3 import APIBlueprint
from pydantic import BaseModel, Field
from app.api.apischemas import GenericLimitSchema
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable, TrackTable
from app.db.userdata import FavoritesTable
from app.models import FavType
from app.settings import Defaults
from app.utils.bisection import use_bisection
from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
from app.serializers.track import serialize_track, serialize_tracks
from app.serializers.artist import serialize_for_card as serialize_artist, serialize_for_cards
from app.serializers.album import serialize_for_card, serialize_for_card_many
from app.store.albums import AlbumStore
from app.store.tracks import TrackStore
from app.store.artists import ArtistStore
from app.serializers.artist import (
serialize_for_card as serialize_artist,
serialize_for_cards,
)
from app.utils.dates import timestamp_to_time_passed
from app.serializers.album import serialize_for_card, serialize_for_card_many
bp_tag = Tag(name="Favorites", description="Your favorite items")
api = APIBlueprint("favorites", __name__, url_prefix="/favorites", abp_tags=[bp_tag])
@@ -41,17 +42,18 @@ class FavoritesAddBody(BaseModel):
@api.post("/add")
def add_favorite(body: FavoritesAddBody):
def toggle_favorite(body: FavoritesAddBody):
"""
Adds a favorite to the database.
"""
itemhash = body.hash
itemtype = body.type
FavoritesTable.insert_item({"hash": body.hash, "type": body.type})
favdb.insert_one_favorite(itemtype, itemhash)
if itemtype == FavType.track:
TrackStore.make_track_fav(itemhash)
if body.type == FavType.track:
TrackTable.set_is_favorite(body.hash, True)
elif body.type == FavType.album:
AlbumTable.set_is_favorite(body.hash, True)
elif body.type == FavType.artist:
ArtistTable.set_is_favorite(body.hash, True)
return {"msg": "Added to favorites"}
@@ -61,80 +63,62 @@ def remove_favorite(body: FavoritesAddBody):
"""
Removes a favorite from the database.
"""
itemhash = body.hash
itemtype = body.type
FavoritesTable.remove_item({"hash": body.hash, "type": body.type})
favdb.delete_favorite(itemtype, itemhash)
if itemtype == FavType.track:
TrackStore.remove_track_from_fav(itemhash)
if body.type == FavType.track:
TrackTable.set_is_favorite(body.hash, False)
elif body.type == FavType.album:
AlbumTable.set_is_favorite(body.hash, False)
elif body.type == FavType.artist:
ArtistTable.set_is_favorite(body.hash, False)
return {"msg": "Removed from favorites"}
class GetAllOfTypeQuery(GenericLimitSchema):
"""
Extending this class will give you a model with the `limit` field
"""
start: int = Field(
description="Where to start from",
example=Defaults.API_CARD_LIMIT,
default=Defaults.API_CARD_LIMIT,
)
@api.get("/albums")
def get_favorite_albums(query: GenericLimitSchema):
def get_favorite_albums(query: GetAllOfTypeQuery):
"""
Get favorite albums
"""
limit = query.limit
albums = favdb.get_fav_albums()
albumhashes = [a[1] for a in albums]
albumhashes.reverse()
fav_albums, total = FavoritesTable.get_fav_albums(query.start, query.limit)
fav_albums.reverse()
src_albums = sorted(AlbumStore.albums, key=lambda x: x.albumhash)
fav_albums = use_bisection(src_albums, "albumhash", albumhashes)
fav_albums = remove_none(fav_albums)
if limit == 0:
limit = len(albums)
return {"albums": serialize_for_card_many(fav_albums[:limit])}
return {"albums": serialize_for_card_many(fav_albums), "total": total}
@api.get("/tracks")
def get_favorite_tracks(query: GenericLimitSchema):
def get_favorite_tracks(query: GetAllOfTypeQuery):
"""
Get favorite tracks
"""
limit = query.limit
userid = current_user['id']
tracks = favdb.get_fav_tracks(userid)
trackhashes = [t[1] for t in tracks]
trackhashes.reverse()
src_tracks = sorted(TrackStore.tracks, key=lambda x: x.trackhash)
tracks = use_bisection(src_tracks, "trackhash", trackhashes)
tracks = remove_none(tracks)
if limit == 0:
limit = len(tracks)
return {"tracks": serialize_tracks(tracks[:limit])}
tracks, total = FavoritesTable.get_fav_tracks(query.start, query.limit)
return {"tracks": serialize_tracks(tracks), "total": total}
@api.get("/artists")
def get_favorite_artists(query: GenericLimitSchema):
def get_favorite_artists(query: GetAllOfTypeQuery):
"""
Get favorite artists
"""
limit = query.limit
artists, total = FavoritesTable.get_fav_artists(
start=query.start,
limit=query.limit,
)
artists.reverse()
artists = favdb.get_fav_artists()
artisthashes = [a[1] for a in artists]
artisthashes.reverse()
src_artists = sorted(ArtistStore.artists, key=lambda x: x.artisthash)
artists = use_bisection(src_artists, "artisthash", artisthashes)
artists = remove_none(artists)
if limit == 0:
limit = len(artists)
return {"artists": artists[:limit]}
return {"artists": [serialize_artist(a) for a in artists], "total": total}
class GetAllFavoritesQuery(BaseModel):
@@ -173,27 +157,29 @@ def get_all_favorites(query: GetAllFavoritesQuery):
# largest is x2 to accound for broken hashes if any
largest = max(track_limit, album_limit, artist_limit)
favs = favdb.get_all()
favs = FavoritesTable.get_all()
favs.reverse()
tracks = []
albums = []
artists = []
track_master_hash = set(t.trackhash for t in TrackStore.tracks)
album_master_hash = set(a.albumhash for a in AlbumStore.albums)
artist_master_hash = set(a.artisthash for a in ArtistStore.artists)
track_master_hash = TrackTable.get_all_hashes()
album_master_hash = AlbumTable.get_all_hashes()
artist_master_hash = ArtistTable.get_all_hashes()
# INFO: Filter out invalid hashes (file not found or tags edited)
for fav in favs:
# INFO: hash is [1], type is [2], timestamp is [3]
hash = fav[1]
if fav[2] == FavType.track:
hash = fav.hash
type = fav.type
if type == FavType.track:
tracks.append(hash) if hash in track_master_hash else None
if fav[2] == FavType.artist:
if type == FavType.artist:
artists.append(hash) if hash in artist_master_hash else None
if fav[2] == FavType.album:
if type == FavType.album:
albums.append(hash) if hash in album_master_hash else None
count = {
@@ -202,35 +188,26 @@ def get_all_favorites(query: GetAllFavoritesQuery):
"artists": len(artists),
}
src_tracks = sorted(TrackStore.tracks, key=lambda x: x.trackhash)
src_albums = sorted(AlbumStore.albums, key=lambda x: x.albumhash)
src_artists = sorted(ArtistStore.artists, key=lambda x: x.artisthash)
tracks = use_bisection(src_tracks, "trackhash", tracks, limit=track_limit)
albums = use_bisection(src_albums, "albumhash", albums, limit=album_limit)
artists = use_bisection(src_artists, "artisthash", artists, limit=artist_limit)
tracks = remove_none(tracks)
albums = remove_none(albums)
artists = remove_none(artists)
tracks = TrackTable.get_tracks_by_trackhashes(tracks, limit=track_limit)
albums = AlbumTable.get_albums_by_albumhashes(albums, limit=album_limit)
artists = ArtistTable.get_artists_by_artisthashes(artists, limit=artist_limit)
recents = []
# first_n = favs
for fav in favs:
# INFO: hash is [1], type is [2], timestamp is [3]
if len(recents) >= largest:
break
if fav[2] == FavType.album:
album = next((a for a in albums if a.albumhash == fav[1]), None)
if fav.type == FavType.album:
album = next((a for a in albums if a.albumhash == fav.hash), None)
if album is None:
continue
album = serialize_for_card(album)
album["help_text"] = "album"
album["time"] = timestamp_to_time_passed(fav[3])
album["time"] = timestamp_to_time_passed(fav.timestamp)
recents.append(
{
@@ -239,15 +216,15 @@ def get_all_favorites(query: GetAllFavoritesQuery):
}
)
if fav[2] == FavType.artist:
artist = next((a for a in artists if a.artisthash == fav[1]), None)
if fav.type == FavType.artist:
artist = next((a for a in artists if a.artisthash == fav.hash), None)
if artist is None:
continue
artist = serialize_artist(artist)
artist["help_text"] = "artist"
artist["time"] = timestamp_to_time_passed(fav[3])
artist["time"] = timestamp_to_time_passed(fav.timestamp)
recents.append(
{
@@ -256,15 +233,15 @@ def get_all_favorites(query: GetAllFavoritesQuery):
}
)
if fav[2] == FavType.track:
track = next((t for t in tracks if t.trackhash == fav[1]), None)
if fav.type == FavType.track:
track = next((t for t in tracks if t.trackhash == fav.hash), None)
if track is None:
continue
track = serialize_track(track)
track["help_text"] = "track"
track["time"] = timestamp_to_time_passed(fav[3])
track["time"] = timestamp_to_time_passed(fav.timestamp)
recents.append({"type": "track", "item": track})
@@ -284,6 +261,5 @@ def check_favorite(query: FavoritesAddBody):
"""
itemhash = query.hash
itemtype = query.type
exists = favdb.check_is_favorite(itemhash, itemtype)
return {"is_favorite": exists}
return {"is_favorite": FavoritesTable.check_exists(itemhash, itemtype)}
app/api/folder.py
+5 -11
View File
@@ -10,11 +10,10 @@ from pydantic import BaseModel, Field
from flask_openapi3 import Tag
from flask_openapi3 import APIBlueprint
from showinfm import show_in_file_manager
from memory_profiler import profile
from app import settings
from app.db import TrackTable
from app.db.sqlite.settings import SettingsSQLMethods as db
from app.config import UserConfig
from app.db.libdata import TrackTable
from app.lib.folderslib import GetFilesAndDirs, get_folders
from app.serializers.track import serialize_track
from app.utils.wintools import is_windows, win_replace_slash
@@ -40,8 +39,9 @@ def get_folder_tree(body: FolderTree):
req_dir = body.folder
tracks_only = body.tracks_only
root_dirs = db.get_root_dirs()
root_dirs.sort()
config = UserConfig()
root_dirs = config.rootDirs
try:
if req_dir == "$home" and root_dirs[0] == "$home":
@@ -72,12 +72,6 @@ def get_folder_tree(body: FolderTree):
return res
# return {
# "path": req_dir,
# "tracks": tracks,
# "folders": sorted(folders, key=lambda i: i.name),
# }
def get_all_drives(is_win: bool = False):
"""
app/api/getall/__init__.py
+5 -4
View File
@@ -6,7 +6,8 @@ from pydantic import BaseModel, Field
from datetime import datetime
from app.api.apischemas import GenericLimitSchema
from app.db import AlbumTable, ArtistTable
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
@@ -61,11 +62,11 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery):
is_artists = path.itemtype == "artists"
if is_albums:
items, total = AlbumTable.get_all(query.start, query.limit)
items = AlbumTable.get_all()
elif is_artists:
items, total = ArtistTable.get_all(query.start, query.limit)
items = ArtistTable.get_all()
# print(items)
total = len(items)
start = query.start
limit = query.limit
app/api/settings.py
+51 -71
View File
@@ -1,3 +1,4 @@
from dataclasses import asdict
from typing import Any
from flask import request
from flask_openapi3 import Tag
@@ -6,12 +7,13 @@ from pydantic import BaseModel, Field
from app.api.auth import admin_required
from app.db.sqlite.plugins import PluginsMethods as pdb
from app.db.sqlite.settings import SettingsSQLMethods as sdb
from app.db.sqlite.tracks import SQLiteTrackMethods as trackdb
from app.db.userdata import PluginTable
from app.lib import populate
from app.lib.tagger import index_everything
from app.lib.watchdogg import Watcher as WatchDog
from app.logger import log
from app.settings import Info, Paths, SessionVarKeys, set_flag
from app.settings import Info, Paths, SessionVarKeys
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
@@ -48,42 +50,43 @@ def reload_everything(instance_key: str):
except Exception as e:
log.error(e)
# CHECKPOINT: TEST SETTINGS API ENDPOINTS
@background
def rebuild_store(db_dirs: list[str]):
"""
Restarts watchdog and rebuilds the music library.
"""
instance_key = get_random_str()
# @background
# def rebuild_store(db_dirs: list[str]):
# """
# Restarts watchdog and rebuilds the music library.
# """
# instance_key = get_random_str()
log.info("Rebuilding library...")
trackdb.remove_tracks_not_in_folders(db_dirs)
reload_everything(instance_key)
# log.info("Rebuilding library...")
# trackdb.remove_tracks_not_in_folders(db_dirs)
# reload_everything(instance_key)
try:
populate.Populate(instance_key=instance_key)
except populate.PopulateCancelledError as e:
print(e)
reload_everything(instance_key)
return
# try:
# populate.Populate(instance_key=instance_key)
# except populate.PopulateCancelledError as e:
# print(e)
# reload_everything(instance_key)
# return
WatchDog().restart()
# WatchDog().restart()
log.info("Rebuilding library... ✅")
# log.info("Rebuilding library... ✅")
# I freaking don't know what this function does anymore
def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]):
"""
Params:
new_: will be added to the database
removed_: will be removed from the database
db_dirs_: will be used to remove tracks that
are outside these directories from the database and store.
"""
sdb.remove_root_dirs(removed_)
sdb.add_root_dirs(new_)
rebuild_store(db_dirs_)
# # I freaking don't know what this function does anymore
# def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]):
# """
# Params:
# new_: will be added to the database
# removed_: will be removed from the database
# db_dirs_: will be used to remove tracks that
# are outside these directories from the database and store.
# """
# sdb.remove_root_dirs(removed_)
# sdb.add_root_dirs(new_)
# rebuild_store(db_dirs_)
class AddRootDirsBody(BaseModel):
@@ -106,7 +109,8 @@ def add_root_dirs(body: AddRootDirsBody):
new_dirs = body.new_dirs
removed_dirs = body.removed
db_dirs = sdb.get_root_dirs()
config = UserConfig()
db_dirs = config.rootDirs
home = "$home"
db_home = any([d == home for d in db_dirs]) # if $home is in db
@@ -114,13 +118,16 @@ def add_root_dirs(body: AddRootDirsBody):
# handle $home case
if db_home and incoming_home:
return {"msg": "Not changed!"}
return {"msg": "Not changed!"}, 304
# if $home is the current root dir or the incoming root dir
# is $home, remove all root dirs
if db_home or incoming_home:
sdb.remove_root_dirs(db_dirs)
config.rootDirs = []
if incoming_home:
finalize([home], [], [Paths.USER_HOME_DIR])
config.rootDirs = [home]
index_everything()
return {"root_dirs": [home]}
# ---
@@ -136,11 +143,10 @@ def add_root_dirs(body: AddRootDirsBody):
pass
db_dirs.extend(new_dirs)
db_dirs = [dir_ for dir_ in db_dirs if dir_ != home]
config.rootDirs = [dir_ for dir_ in db_dirs if dir_ != home]
finalize(new_dirs, removed_dirs, db_dirs)
return {"root_dirs": db_dirs}
index_everything()
return {"root_dirs": config.rootDirs}
@api.get("/get-root-dirs")
@@ -148,9 +154,7 @@ def get_root_dirs():
"""
Get root directories
"""
dirs = sdb.get_root_dirs()
return {"dirs": dirs}
return {"dirs": UserConfig().rootDirs}
# maps settings to their parser flags
@@ -170,35 +174,12 @@ def get_all_settings():
"""
Get all settings
"""
config = asdict(UserConfig())
plugins = PluginTable.get_all()
config["plugins"] = plugins
config["version"] = Info.SWINGMUSIC_APP_VERSION
settings = sdb.get_all_settings()
plugins = pdb.get_all_plugins()
key_list = list(mapp.keys())
s = {}
for key in key_list:
val_index = key_list.index(key)
try:
s[key] = settings[val_index]
if type(s[key]) == int:
s[key] = bool(s[key])
if type(s[key]) == str:
s[key] = str(s[key]).split(",")
except IndexError:
s[key] = None
root_dirs = sdb.get_root_dirs()
s["root_dirs"] = root_dirs
s["plugins"] = plugins
s["version"] = Info.SWINGMUSIC_APP_VERSION
return {
"settings": s,
}
return config
@background
@@ -245,7 +226,6 @@ def set_setting(body: SetSettingBody):
value = str(value).split(",")
value = set(value)
set_flag(flag, value)
reload_all_for_set_setting()
# if value is a set, convert it to a string
app/api/stream.py
+1 -1
View File
@@ -11,7 +11,7 @@ from app.api.apischemas import TrackHashSchema
from app.lib.trackslib import get_silence_paddings
# from app.store.tracks import TrackStore
from app.db import TrackTable
from app.db.libdata import TrackTable
from app.utils.files import guess_mime_type
bp_tag = Tag(name="File", description="Audio files")
app/arg_handler.py
+4 -3
View File
@@ -9,6 +9,7 @@ import sys
import PyInstaller.__main__ as bundler
from app import settings
from app.config import UserConfig
from app.logger import log
from app.print_help import HELP_MESSAGE
from app.utils.auth import hash_password
@@ -160,7 +161,7 @@ class ProcessArgs:
@staticmethod
def handle_periodic_scan():
if any((a in ARGS for a in ALLARGS.no_periodic_scan)):
settings.SessionVars.DO_PERIODIC_SCANS = False
UserConfig().enablePeriodicScans = False
@staticmethod
def handle_periodic_scan_interval():
@@ -182,10 +183,10 @@ class ProcessArgs:
sys.exit(0)
if psi < 0:
print("WADAFUCK ARE YOU TRYING?")
print("WHAT ARE YOU TRYING?")
sys.exit(0)
settings.SessionVars.PERIODIC_SCAN_INTERVAL = psi
UserConfig().scanInterval = psi
@staticmethod
def handle_help():
app/config.py
+9 -1
View File
@@ -6,6 +6,7 @@ from .settings import Paths
# TODO: Publish this on PyPi
@dataclass
class UserConfig:
_config_path: str = ""
@@ -20,7 +21,7 @@ class UserConfig:
# lists
rootDirs: list[str] = field(default_factory=list)
excludeDirs: list[str] = field(default_factory=list)
artistSeparators: set[str] = field(default_factory=set)
artistSeparators: set[str] = field(default_factory=lambda: {";", "/"})
genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"})
# tracks
@@ -33,6 +34,13 @@ class UserConfig:
cleanAlbumTitle: bool = True
showAlbumsAsSingles: bool = False
# misc
enablePeriodicScans: bool = False
scanInterval: int = 60 * 10 # 10 minutes
# plugins
enablePlugins: bool = True
def __post_init__(self):
"""
Loads the config file and sets the values to this instance
app/db/__init__.py
+31 -348
View File
@@ -1,36 +1,21 @@
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
from pprint import pprint
from typing import Any, Optional
from typing import Any
from memory_profiler import profile
from sqlalchemy import (
JSON,
Boolean,
Integer,
Row,
String,
Tuple,
and_,
create_engine,
delete,
insert,
select,
)
from sqlalchemy.engine import Engine
from sqlalchemy import event
from sqlalchemy.orm import (
Mapped,
mapped_column,
DeclarativeBase,
MappedAsDataclass,
sessionmaker,
)
from app.models import Track as TrackModel
from app.models import Album as AlbumModel
from app.models import Artist as ArtistModel
from app.utils.remove_duplicates import remove_duplicates
# ============================================================
# TODO: Make sure the database is created before we run this.
fullpath = "/home/cwilvx/temp/swingmusic/swing.db"
engine = create_engine(
f"sqlite+pysqlite:///{fullpath}",
@@ -39,85 +24,46 @@ engine = create_engine(
pool_size=5,
)
if not os.path.exists(fullpath):
os.makedirs(Path(fullpath).parent)
connection = engine.connect()
all_filepaths = list()
# connection = engine.connect()
def getIndexOfFirstMatch(strings: list[str], prefix: str):
"""
Find the index of the first path that starts with the given path.
Uses a binary search algorithm to find the index.
"""
left = 0
right = len(strings) - 1
while left <= right:
mid = (left + right) // 2
if strings[mid].startswith(prefix):
if mid == 0 or not strings[mid - 1].startswith(prefix):
return mid
right = mid - 1
elif strings[mid] < prefix:
left = mid + 1
else:
right = mid - 1
return -1
def countFilepathsInDir(dirpath: str):
"""
Return all the filepaths in a directory.
"""
global all_filepaths
index = getIndexOfFirstMatch(all_filepaths, dirpath)
if index == -1:
return 0
paths: list[str] = []
for path in all_filepaths[index:]:
if path.startswith(dirpath):
paths.append(path)
else:
break
return len(paths)
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
class DbManager:
def __init__(self, commit: bool = False):
self.commit = commit
# self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True)
# self.conn = self.engine.connect()
# pass
self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True)
self.conn = self.engine.connect()
def __enter__(self):
# return self.conn.execution_options(preserve_rowcount=True)
return connection
return self.conn.execution_options(preserve_rowcount=True)
# return connection
def __exit__(self, exc_type, exc_val, exc_tb):
if self.commit:
connection.commit()
self.conn.commit()
# self.conn.close()
self.conn.close()
class Base(MappedAsDataclass, DeclarativeBase):
@classmethod
def execute(cls, stmt: Any, commit: bool = False):
with DbManager(commit=commit) as conn:
return conn.execute(stmt)
@classmethod
def insert_many(cls, items: list[dict[str, Any]]):
"""
Inserts multiple items into the database.
"""
with DbManager(commit=True) as conn:
conn.execute(insert(cls).values(items))
return conn.execute(insert(cls).values(items))
@classmethod
def insert_one(cls, item: dict[str, Any]):
@@ -127,277 +73,14 @@ class Base(MappedAsDataclass, DeclarativeBase):
return cls.insert_many([item])
@classmethod
def get_all(cls):
"""
Returns all the items from the database.
"""
with DbManager() as conn:
result = conn.execute(select(cls))
return result.fetchall()
class ArtistTable(Base):
__tablename__ = "artist"
id: Mapped[int] = mapped_column(primary_key=True)
albumcount: Mapped[int] = mapped_column(Integer())
artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True)
created_date: Mapped[int] = mapped_column(Integer())
date: Mapped[int] = mapped_column(Integer())
duration: Mapped[int] = mapped_column(Integer())
genres: Mapped[str] = mapped_column(JSON())
name: Mapped[str] = mapped_column(String(), index=True)
trackcount: Mapped[int] = mapped_column(Integer())
is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean())
def remove_all(cls):
with DbManager(commit=True) as conn:
conn.execute(delete(cls))
@classmethod
def get_all(cls, start: int, limit: int):
with DbManager() as conn:
if start == 0:
result = conn.execute(select(cls))
else:
result = conn.execute(select(cls).offset(start).limit(limit))
all = result.fetchall()
return artists_to_dataclasses(all), len(all)
@classmethod
def get_artist_by_hash(cls, artisthash: str):
with DbManager() as conn:
result = conn.execute(
select(ArtistTable).where(ArtistTable.artisthash == artisthash)
)
return artist_to_dataclass(result.fetchone())
class AlbumTable(Base):
__tablename__ = "album"
id: Mapped[int] = mapped_column(primary_key=True)
albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True)
artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True)
albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True)
base_title: Mapped[str] = mapped_column(String())
color: Mapped[Optional[str]] = mapped_column(String())
created_date: Mapped[int] = mapped_column(Integer())
date: Mapped[int] = mapped_column(Integer())
duration: Mapped[int] = mapped_column(Integer())
genres: Mapped[str] = mapped_column(JSON())
og_title: Mapped[str] = mapped_column(String())
title: Mapped[str] = mapped_column(String())
trackcount: Mapped[int] = mapped_column(Integer())
@classmethod
def get_album_by_albumhash(cls, hash: str):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.albumhash == hash)
)
album = result.fetchone()
if album:
return album_to_dataclass(album)
@classmethod
def get_albums_by_hash(cls, hashes: set[str]):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.albumhash.in_(hashes))
)
return albums_to_dataclasses(result.fetchall())
@classmethod
def get_all(cls, start: int, limit: int):
with DbManager() as conn:
if start == 0:
result = conn.execute(select(AlbumTable))
else:
result = conn.execute(select(AlbumTable).offset(start).limit(limit))
all = result.fetchall()
return albums_to_dataclasses(all)[:limit], len(all)
@classmethod
def get_albums_by_artisthashes(cls, artisthashes: list[dict[str, str]]):
with DbManager() as conn:
albums: list[AlbumModel] = []
for artist in artisthashes:
result = conn.execute(
# NOTE: The artist dict keys need to in the same order they appear in the db for this to work!
select(AlbumTable).where(AlbumTable.albumartists.contains(artist))
)
albums.extend(albums_to_dataclasses(result.fetchall()))
return albums
@classmethod
def get_albums_by_base_title(cls, base_title: str):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.base_title == base_title)
)
return albums_to_dataclasses(result.fetchall())
@classmethod
def get_albums_by_artisthash(cls, artisthash: str):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash))
)
return albums_to_dataclasses(result.all())
class TrackTable(Base):
__tablename__ = "track"
id: Mapped[int] = mapped_column(init=False, primary_key=True)
album: Mapped[str] = mapped_column(String())
albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON())
albumhash: Mapped[str] = mapped_column(String(), index=True)
artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True)
artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True)
bitrate: Mapped[int] = mapped_column(Integer())
copyright: Mapped[Optional[str]] = mapped_column(String())
date: Mapped[int] = mapped_column(Integer())
disc: Mapped[int] = mapped_column(Integer())
duration: Mapped[int] = mapped_column(Integer())
filepath: Mapped[str] = mapped_column(String(), index=True, unique=True)
folder: Mapped[str] = mapped_column(String(), index=True)
genre: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON())
last_mod: Mapped[float] = mapped_column(Integer())
og_album: Mapped[str] = mapped_column(String())
og_title: Mapped[str] = mapped_column(String())
title: Mapped[str] = mapped_column(String())
track: Mapped[int] = mapped_column(Integer())
trackhash: Mapped[str] = mapped_column(String(), index=True)
extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON())
@classmethod
def get_tracks_by_filepaths(cls, filepaths: list[str]):
with DbManager() as conn:
result = conn.execute(
select(TrackTable).where(TrackTable.filepath.in_(filepaths))
)
return tracks_to_dataclasses(result.fetchall())
@classmethod
def count_tracks_containing_paths(cls, paths: list[str]):
results: list[dict[str, int | str]] = []
with ThreadPoolExecutor() as executor:
res = executor.map(countFilepathsInDir, paths)
results = [
{"path": path, "trackcount": count} for path, count in zip(paths, res)
]
return results
@classmethod
def get_tracks_by_albumhash(cls, albumhash: str):
with DbManager() as conn:
result = conn.execute(
select(TrackTable).where(TrackTable.albumhash == albumhash)
)
tracks = tracks_to_dataclasses(result.fetchall())
return remove_duplicates(tracks, is_album_tracks=True)
@classmethod
def get_track_by_trackhash(cls, hash: str, filepath: str = ""):
with DbManager() as conn:
if filepath:
result = conn.execute(
select(TrackTable)
.where(
and_(
TrackTable.trackhash == hash,
TrackTable.filepath == filepath,
)
)
.order_by(TrackTable.bitrate.desc())
)
else:
result = conn.execute(
select(TrackTable).where(TrackTable.trackhash == hash)
)
track = result.fetchone()
if track:
return track_to_dataclass(track)
@classmethod
def get_tracks_by_artisthash(cls, artisthash: str):
with DbManager() as conn:
result = conn.execute(
select(TrackTable).where(TrackTable.artists.contains(artisthash))
)
return tracks_to_dataclasses(result.fetchall())
@classmethod
def get_tracks_in_path(cls, path: str):
with DbManager() as conn:
result = conn.execute(
select(TrackTable)
.where(TrackTable.filepath.contains(path))
.order_by(TrackTable.last_mod)
)
return tracks_to_dataclasses(result.fetchall())
all_tracks = TrackTable.get_all()
for track in all_tracks:
all_filepaths.append(track.filepath)
all_filepaths.sort()
# print("files in path: ",getFilepathsInDir("/home/cwilvx/Music/").__len__())
# SECTION: Userdata database
class UserTable(Base):
__tablename__ = "user"
id: Mapped[int] = mapped_column(primary_key=True)
username: Mapped[str] = mapped_column(String(), unique=True)
firstname: Mapped[Optional[str]] = mapped_column(String())
lastname: Mapped[Optional[str]] = mapped_column(String())
password: Mapped[str] = mapped_column(String())
email: Mapped[Optional[str]] = mapped_column(String())
image: Mapped[Optional[str]] = mapped_column(String())
roles: Mapped[list[str]] = mapped_column(JSON(), default_factory=lambda: ["user"])
extra: Mapped[Optional[dict[str, Any]]] = mapped_column(
JSON(), default_factory=dict
)
# SECTION: HELPER FUNCTIONS
def artist_to_dataclass(artist: Any):
return ArtistModel(**artist._asdict())
def artists_to_dataclasses(artists: Any):
return [artist_to_dataclass(artist) for artist in artists]
def album_to_dataclass(album: Any):
return AlbumModel(**album._asdict())
def albums_to_dataclasses(albums: Any):
return [album_to_dataclass(album) for album in albums]
def track_to_dataclass(track: Any):
return TrackModel(**track._asdict())
def tracks_to_dataclasses(tracks: Any):
return [track_to_dataclass(track) for track in tracks]
def all(cls):
return cls.execute(select(cls))
def create_all():
Base().metadata.create_all(engine)
app/db/libdata.py
+312
View File
@@ -0,0 +1,312 @@
from app.db import (
Base as MasterBase,
DbManager,
)
from app.db.utils import (
album_to_dataclass,
albums_to_dataclasses,
artist_to_dataclass,
artists_to_dataclasses,
track_to_dataclass,
tracks_to_dataclasses,
)
from app.models import Album as AlbumModel
from app.utils.remove_duplicates import remove_duplicates
from app.db import engine
from sqlalchemy import JSON, Boolean, Integer, String, and_, delete, select, update
from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase
from typing import Any, Iterable, Optional
def create_all():
"""
Create all the tables defined in this file.
"""
Base.metadata.create_all(engine)
class Base(MasterBase, DeclarativeBase):
@classmethod
def get_all_hashes(cls):
with DbManager() as conn:
if cls.__tablename__ == "track":
stmt = select(TrackTable.trackhash)
elif cls.__tablename__ == "album":
stmt = select(AlbumTable.albumhash)
elif cls.__tablename__ == "artist":
stmt = select(ArtistTable.artisthash)
result = conn.execute(stmt)
return {row[0] for row in result.fetchall()}
@classmethod
def set_is_favorite(cls, hash: str, is_favorite: bool):
"""
Set the 'is_favorite' flag for a specific hash.
Args:
hash (str): The hash value.
is_favorite (bool): The value of the 'is_favorite' flag.
"""
with DbManager(commit=True) as conn:
if cls.__tablename__ == "track":
stmt = (
update(cls)
.where(TrackTable.trackhash == hash)
.values(is_favorite=is_favorite)
)
elif cls.__tablename__ == "album":
stmt = (
update(cls)
.where(AlbumTable.albumhash == hash)
.values(is_favorite=is_favorite)
)
elif cls.__tablename__ == "artist":
stmt = (
update(cls)
.where(ArtistTable.artisthash == hash)
.values(is_favorite=is_favorite)
)
conn.execute(stmt)
class TrackTable(Base):
__tablename__ = "track"
id: Mapped[int] = mapped_column(init=False, primary_key=True)
album: Mapped[str] = mapped_column(String())
albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON())
albumhash: Mapped[str] = mapped_column(String(), index=True)
artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True)
artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True)
bitrate: Mapped[int] = mapped_column(Integer())
copyright: Mapped[Optional[str]] = mapped_column(String())
date: Mapped[int] = mapped_column(Integer(), nullable=True)
disc: Mapped[int] = mapped_column(Integer())
duration: Mapped[int] = mapped_column(Integer())
filepath: Mapped[str] = mapped_column(String(), unique=True)
folder: Mapped[str] = mapped_column(String(), index=True)
genrehashes: Mapped[list[str]] = mapped_column(JSON(), index=True)
genres: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON())
last_mod: Mapped[float] = mapped_column(Integer())
og_album: Mapped[str] = mapped_column(String())
og_title: Mapped[str] = mapped_column(String())
title: Mapped[str] = mapped_column(String())
track: Mapped[int] = mapped_column(Integer())
trackhash: Mapped[str] = mapped_column(String(), index=True)
is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean())
playcount: Mapped[int] = mapped_column(Integer())
extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON())
@classmethod
def get_all(cls):
with DbManager() as conn:
result = conn.execute(select(cls))
return tracks_to_dataclasses(result.fetchall())
@classmethod
def get_tracks_by_filepaths(cls, filepaths: list[str]):
with DbManager() as conn:
result = conn.execute(
select(TrackTable).where(TrackTable.filepath.in_(filepaths))
)
return tracks_to_dataclasses(result.fetchall())
@classmethod
def get_tracks_by_albumhash(cls, albumhash: str):
with DbManager() as conn:
result = conn.execute(
select(TrackTable).where(TrackTable.albumhash == albumhash)
)
tracks = tracks_to_dataclasses(result.fetchall())
return remove_duplicates(tracks, is_album_tracks=True)
@classmethod
def get_track_by_trackhash(cls, hash: str, filepath: str = ""):
with DbManager() as conn:
if filepath:
result = conn.execute(
select(TrackTable)
.where(
and_(
TrackTable.trackhash == hash,
TrackTable.filepath == filepath,
)
)
.order_by(TrackTable.bitrate.desc())
)
else:
result = conn.execute(
select(TrackTable).where(TrackTable.trackhash == hash)
)
track = result.fetchone()
if track:
return track_to_dataclass(track)
@classmethod
def get_tracks_by_artisthash(cls, artisthash: str):
with DbManager() as conn:
result = conn.execute(
select(TrackTable).where(TrackTable.artists.contains(artisthash))
)
return tracks_to_dataclasses(result.fetchall())
@classmethod
def get_tracks_in_path(cls, path: str):
with DbManager() as conn:
result = conn.execute(
select(TrackTable)
.where(TrackTable.filepath.contains(path))
.order_by(TrackTable.last_mod)
)
return tracks_to_dataclasses(result.fetchall())
@classmethod
def get_tracks_by_trackhashes(cls, hashes: Iterable[str], limit: int | None = None):
with DbManager() as conn:
result = conn.execute(
select(TrackTable).where(TrackTable.trackhash.in_(hashes)).limit(limit)
)
return tracks_to_dataclasses(result.fetchall())
@classmethod
def remove_tracks_by_filepaths(cls, filepaths: set[str]):
with DbManager(commit=True) as conn:
conn.execute(delete(TrackTable).where(TrackTable.filepath.in_(filepaths)))
class AlbumTable(Base):
__tablename__ = "album"
id: Mapped[int] = mapped_column(primary_key=True)
albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True)
artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True)
albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True)
base_title: Mapped[str] = mapped_column(String())
color: Mapped[Optional[str]] = mapped_column(String())
created_date: Mapped[int] = mapped_column(Integer())
date: Mapped[int] = mapped_column(Integer())
duration: Mapped[int] = mapped_column(Integer())
genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True)
genres: Mapped[str] = mapped_column(JSON())
og_title: Mapped[str] = mapped_column(String())
title: Mapped[str] = mapped_column(String())
trackcount: Mapped[int] = mapped_column(Integer())
is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean())
extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON())
@classmethod
def get_all(cls):
with DbManager() as conn:
result = conn.execute(select(AlbumTable))
all = result.fetchall()
return albums_to_dataclasses(all)
@classmethod
def get_album_by_albumhash(cls, hash: str):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.albumhash == hash)
)
album = result.fetchone()
if album:
return album_to_dataclass(album)
@classmethod
def get_albums_by_albumhashes(cls, hashes: Iterable[str], limit: int | None = None):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)).limit(limit)
)
return albums_to_dataclasses(result.fetchall())
@classmethod
def get_albums_by_artisthashes(cls, artisthashes: list[str]):
with DbManager() as conn:
albums: list[AlbumModel] = []
for artist in artisthashes:
result = conn.execute(
# NOTE: The artist dict keys need to in the same order they appear in the db for this to work!
select(AlbumTable).where(AlbumTable.albumartists.contains(artist))
)
albums.extend(albums_to_dataclasses(result.fetchall()))
return albums
@classmethod
def get_albums_by_base_title(cls, base_title: str):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.base_title == base_title)
)
return albums_to_dataclasses(result.fetchall())
@classmethod
def get_albums_by_artisthash(cls, artisthash: str):
with DbManager() as conn:
result = conn.execute(
select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash))
)
return albums_to_dataclasses(result.all())
class ArtistTable(Base):
__tablename__ = "artist"
id: Mapped[int] = mapped_column(primary_key=True)
albumcount: Mapped[int] = mapped_column(Integer())
artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True)
created_date: Mapped[int] = mapped_column(Integer())
date: Mapped[int] = mapped_column(Integer())
duration: Mapped[int] = mapped_column(Integer())
genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True)
genres: Mapped[str] = mapped_column(JSON())
name: Mapped[str] = mapped_column(String(), index=True)
trackcount: Mapped[int] = mapped_column(Integer())
is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean())
extra: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON())
@classmethod
def get_all(cls):
with DbManager() as conn:
result = conn.execute(select(cls))
all = result.fetchall()
return artists_to_dataclasses(all)
@classmethod
def get_artist_by_hash(cls, artisthash: str):
with DbManager() as conn:
result = conn.execute(
select(ArtistTable).where(ArtistTable.artisthash == artisthash)
)
return artist_to_dataclass(result.fetchone())
@classmethod
def get_artisthashes_not_in(cls, artisthashes: list[str]):
with DbManager() as conn:
result = conn.execute(
select(ArtistTable.artisthash, ArtistTable.name).where(
~ArtistTable.artisthash.in_(artisthashes)
)
)
return [{"artisthash": row[0], "name": row[1]} for row in result.fetchall()]
@classmethod
def get_artists_by_artisthashes(
cls, hashes: Iterable[str], limit: int | None = None
):
with DbManager() as conn:
result = conn.execute(
select(ArtistTable)
.where(ArtistTable.artisthash.in_(hashes))
.limit(limit)
)
return artists_to_dataclasses(result.fetchall())
app/db/metadata.py
+33
View File
@@ -0,0 +1,33 @@
from app.db import Base, DbManager
from sqlalchemy import Integer, insert, select, update
from sqlalchemy.orm import Mapped, mapped_column
class MigrationTable(Base):
__tablename__ = "dbmigration"
id: Mapped[int] = mapped_column(primary_key=True)
version: Mapped[int] = mapped_column(Integer())
@classmethod
def set_version(cls, version: int):
with DbManager(commit=True) as conn:
result = conn.execute(
update(cls).where(cls.id == 1).values(version=version)
)
if result.rowcount == 0:
conn.execute(insert(cls).values(id=1, version=version))
@classmethod
def get_version(cls):
with DbManager() as conn:
result = conn.execute(select(cls.version).where(cls.id == 1))
result = result.fetchone()
if result:
return result[0]
return -1
app/db/sqlite/lastfm/similar_artists.py
+1 -1
View File
@@ -16,7 +16,7 @@ class SQLiteLastFMSimilarArtists:
sql = """INSERT OR REPLACE INTO lastfm_similar_artists(artisthash, similar_artists) VALUES(?,?)"""
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql, (artist.artisthash, artist.similar_artist_hashes))
cur.execute(sql, (artist.artisthash, artist.similar_artists))
cur.close()
@classmethod
app/db/sqlite/plugins/__init__.py
-10
View File
@@ -8,7 +8,6 @@ from ..utils import SQLiteManager
def plugin_tuple_to_obj(plugin_tuple: tuple) -> Plugin:
return Plugin(
name=plugin_tuple[1],
description=plugin_tuple[2],
active=bool(plugin_tuple[3]),
settings=json.loads(plugin_tuple[4]),
)
@@ -43,15 +42,6 @@ class PluginsMethods:
return lastrowid
@classmethod
def insert_lyrics_plugin(cls):
plugin = Plugin(
name="lyrics_finder",
description="Find lyrics from the internet",
active=False,
settings={"auto_download": False},
)
cls.insert_plugin(plugin)
@classmethod
def get_all_plugins(cls):
app/db/sqlite/queries.py
-7
View File
@@ -14,13 +14,6 @@ CREATE TABLE IF NOT EXISTS playlists (
constraint fk_users foreign key (userid) references users(id) on delete cascade
);
CREATE TABLE IF NOT EXISTS favorites (
id integer PRIMARY KEY,
hash text not null,
type text not null,
timestamp integer not null default 0
);
CREATE TABLE IF NOT EXISTS settings (
id integer PRIMARY KEY,
root_dirs text NOT NULL,
app/db/sqlite/settings.py
+109 -108
View File
@@ -1,150 +1,151 @@
from pprint import pprint
from typing import Any
from app.config import UserConfig
from app.db.sqlite.utils import SQLiteManager
from app.settings import SessionVars
from app.utils.wintools import win_replace_slash
class SettingsSQLMethods:
"""
Methods for interacting with the settings table.
"""
# class SettingsSQLMethods:
# """
# Methods for interacting with the settings table.
# """
@staticmethod
def get_all_settings():
"""
Gets all settings from the database.
"""
# @staticmethod
# def get_all_settings():
# """
# Gets all settings from the database.
# """
sql = "SELECT * FROM settings WHERE id = 1"
# sql = "SELECT * FROM settings WHERE id = 1"
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql)
settings = cur.fetchone()
cur.close()
# with SQLiteManager(userdata_db=True) as cur:
# cur.execute(sql)
# settings = cur.fetchone()
# cur.close()
# if root_dirs not set
if settings is None:
return []
# # if root_dirs not set
# if settings is None:
# return []
# omit id, root_dirs, and exclude_dirs
return settings[3:]
# # omit id, root_dirs, and exclude_dirs
# return settings[3:]
@staticmethod
def get_root_dirs() -> list[str]:
"""
Gets custom root directories from the database.
"""
# @staticmethod
# def get_root_dirs() -> list[str]:
# """
# Gets custom root directories from the database.
# """
sql = "SELECT root_dirs FROM settings"
# sql = "SELECT root_dirs FROM settings"
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql)
dirs = cur.fetchall()
cur.close()
# with SQLiteManager(userdata_db=True) as cur:
# cur.execute(sql)
# dirs = cur.fetchall()
# cur.close()
dirs = [_dir[0] for _dir in dirs]
return [win_replace_slash(d) for d in dirs]
# dirs = [_dir[0] for _dir in dirs]
# return [win_replace_slash(d) for d in dirs]
@staticmethod
def add_root_dirs(dirs: list[str]):
"""
Add custom root directories to the database.
"""
# @staticmethod
# def add_root_dirs(dirs: list[str]):
# """
# Add custom root directories to the database.
# """
sql = "INSERT INTO settings (root_dirs) VALUES (?)"
existing_dirs = SettingsSQLMethods.get_root_dirs()
# sql = "INSERT INTO settings (root_dirs) VALUES (?)"
# existing_dirs = SettingsSQLMethods.get_root_dirs()
dirs = [_dir for _dir in dirs if _dir not in existing_dirs]
# dirs = [_dir for _dir in dirs if _dir not in existing_dirs]
if len(dirs) == 0:
return
# if len(dirs) == 0:
# return
with SQLiteManager(userdata_db=True) as cur:
for _dir in dirs:
cur.execute(sql, (_dir,))
# with SQLiteManager(userdata_db=True) as cur:
# for _dir in dirs:
# cur.execute(sql, (_dir,))
@staticmethod
def remove_root_dirs(dirs: list[str]):
"""
Remove custom root directories from the database.
"""
# @staticmethod
# def remove_root_dirs(dirs: list[str]):
# """
# Remove custom root directories from the database.
# """
sql = "DELETE FROM settings WHERE root_dirs = ?"
# sql = "DELETE FROM settings WHERE root_dirs = ?"
with SQLiteManager(userdata_db=True) as cur:
for _dir in dirs:
cur.execute(sql, (_dir,))
# with SQLiteManager(userdata_db=True) as cur:
# for _dir in dirs:
# cur.execute(sql, (_dir,))
# Not currently used anywhere, to be used later
@staticmethod
def add_excluded_dirs(dirs: list[str]):
"""
Add custom exclude directories to the database.
"""
# # Not currently used anywhere, to be used later
# @staticmethod
# def add_excluded_dirs(dirs: list[str]):
# """
# Add custom exclude directories to the database.
# """
sql = "INSERT INTO settings (exclude_dirs) VALUES (?)"
# sql = "INSERT INTO settings (exclude_dirs) VALUES (?)"
with SQLiteManager(userdata_db=True) as cur:
cur.executemany(sql, dirs)
# with SQLiteManager(userdata_db=True) as cur:
# cur.executemany(sql, dirs)
@staticmethod
def remove_excluded_dirs(dirs: list[str]):
"""
Remove custom exclude directories from the database.
"""
# @staticmethod
# def remove_excluded_dirs(dirs: list[str]):
# """
# Remove custom exclude directories from the database.
# """
sql = "DELETE FROM settings WHERE exclude_dirs = ?"
# sql = "DELETE FROM settings WHERE exclude_dirs = ?"
with SQLiteManager(userdata_db=True) as cur:
cur.executemany(sql, dirs)
# with SQLiteManager(userdata_db=True) as cur:
# cur.executemany(sql, dirs)
@staticmethod
def get_excluded_dirs() -> list[str]:
"""
Gets custom exclude directories from the database.
"""
# @staticmethod
# def get_excluded_dirs() -> list[str]:
# """
# Gets custom exclude directories from the database.
# """
sql = "SELECT exclude_dirs FROM settings"
# sql = "SELECT exclude_dirs FROM settings"
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql)
dirs = cur.fetchall()
return [_dir[0] for _dir in dirs]
# with SQLiteManager(userdata_db=True) as cur:
# cur.execute(sql)
# dirs = cur.fetchall()
# return [_dir[0] for _dir in dirs]
@staticmethod
def get_settings() -> dict[str, Any]:
pass
# @staticmethod
# def get_settings() -> dict[str, Any]:
# pass
@staticmethod
def set_setting(key: str, value: Any):
sql = f"UPDATE settings SET {key} = :value WHERE id = 1"
# @staticmethod
# def set_setting(key: str, value: Any):
# sql = f"UPDATE settings SET {key} = :value WHERE id = 1"
if type(value) == bool:
value = str(int(value))
# if type(value) == bool:
# value = str(int(value))
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql, {"value": value})
# with SQLiteManager(userdata_db=True) as cur:
# cur.execute(sql, {"value": value})
def load_settings():
s = SettingsSQLMethods.get_all_settings()
# def load_settings():
# # s = SettingsSQLMethods.get_all_settings()
# config = UserConfig()
try:
db_separators: str = s[0]
db_separators = db_separators.replace(" ", "")
separators = db_separators.split(",")
separators = set(separators)
except IndexError:
separators = {";", "/"}
# try:
# db_separators: str = s[0]
# db_separators = db_separators.replace(" ", "")
# separators = db_separators.split(",")
# separators = set(separators)
# except IndexError:
# separators = {";", "/"}
SessionVars.ARTIST_SEPARATORS = separators
# SessionVars.ARTIST_SEPARATORS = config.artistSeparators
# boolean settings
SessionVars.EXTRACT_FEAT = bool(s[1])
SessionVars.REMOVE_PROD = bool(s[2])
SessionVars.CLEAN_ALBUM_TITLE = bool(s[3])
SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4])
SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5])
SessionVars.SHOW_ALBUMS_AS_SINGLES = bool(s[6])
# # boolean settings
# SessionVars.EXTRACT_FEAT = bool(s[1])
# SessionVars.REMOVE_PROD = bool(s[2])
# SessionVars.CLEAN_ALBUM_TITLE = bool(s[3])
# SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4])
# SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5])
# SessionVars.SHOW_ALBUMS_AS_SINGLES = bool(s[6])
app/db/userdata.py
+240
View File
@@ -0,0 +1,240 @@
import datetime
from shlex import join
from typing import Any
from flask_jwt_extended import current_user
from sqlalchemy import (
JSON,
Boolean,
ForeignKey,
Integer,
String,
and_,
delete,
insert,
select,
update,
join,
)
from sqlalchemy.orm import Mapped, mapped_column
from app.db.utils import (
albums_to_dataclasses,
artists_to_dataclasses,
favorites_to_dataclass,
plugin_to_dataclasses,
similar_artist_to_dataclass,
similar_artists_to_dataclass,
tracks_to_dataclasses,
user_to_dataclass,
user_to_dataclasses,
)
from app.db import Base, DbManager
from app.utils.auth import hash_password
class UserTable(Base):
__tablename__ = "user"
id: Mapped[int] = mapped_column(primary_key=True)
image: Mapped[str] = mapped_column(String(), nullable=True)
password: Mapped[str] = mapped_column(String())
username: Mapped[str] = mapped_column(String(), index=True)
roles: Mapped[list[str]] = mapped_column(JSON(), default_factory=lambda: ["user"])
extra: Mapped[dict[str, Any]] = mapped_column(
JSON(), nullable=True, default_factory=dict
)
@classmethod
def get_all(cls):
result = cls.execute(select(cls))
return user_to_dataclasses(result.fetchall())
@classmethod
def insert_default_user(cls):
user = {
"username": "admin",
"password": hash_password("admin"),
"roles": ["admin"],
}
return cls.insert_one(user)
@classmethod
def insert_guest_user(cls):
user = {
"username": "guest",
"password": hash_password("guest"),
"roles": ["guest"],
}
return cls.insert_one(user)
@classmethod
def get_by_id(cls, id: int):
with DbManager() as conn:
result = conn.execute(select(cls).where(cls.id == id))
res = result.fetchone()
if res:
return user_to_dataclass(res)
@classmethod
def get_by_username(cls, username: str):
with DbManager() as conn:
result = conn.execute(select(cls).where(cls.username == username))
res = result.fetchone()
if res:
return user_to_dataclass(res)
@classmethod
def update_one(cls, user: dict[str, Any]):
with DbManager(commit=True) as conn:
conn.execute(update(cls).where(cls.id == user["id"]).values(user))
@classmethod
def remove_by_username(cls, username: str):
return cls.execute(delete(cls).where(cls.username == username), commit=True)
class PluginTable(Base):
__tablename__ = "plugin"
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(), unique=True)
active: Mapped[bool] = mapped_column(Boolean())
settings: Mapped[dict[str, Any]] = mapped_column(JSON())
extra: Mapped[dict[str, Any]] = mapped_column(JSON(), nullable=True)
@classmethod
def get_all(cls):
return plugin_to_dataclasses(cls.all())
class SimilarArtistTable(Base):
__tablename__ = "notlastfm_similar_artists"
id: Mapped[int] = mapped_column(Integer(), primary_key=True)
artisthash: Mapped[str] = mapped_column(String(), index=True)
similar_artists: Mapped[dict[str, str]] = mapped_column(JSON())
@classmethod
def get_all(cls):
with DbManager() as conn:
result = conn.execute(select(cls))
return similar_artists_to_dataclass(result.fetchall())
@classmethod
def exists(cls, artisthash: str):
"""
Check whether an artisthash exists in the database.
"""
with DbManager() as conn:
result = conn.execute(
select(cls.artisthash).where(cls.artisthash == artisthash)
)
return result.fetchone() is not None
@classmethod
def get_by_hash(cls, artisthash: str):
"""
Get a single artist by hash.
"""
with DbManager() as conn:
result = conn.execute(select(cls).where(cls.artisthash == artisthash))
result = result.fetchone()
if result:
return similar_artist_to_dataclass(result)
class FavoritesTable(Base):
__tablename__ = "favorite"
id: Mapped[int] = mapped_column(primary_key=True)
hash: Mapped[str] = mapped_column(String())
type: Mapped[str] = mapped_column(String(), index=True)
timestamp: Mapped[int] = mapped_column(Integer(), index=True)
userid: Mapped[int] = mapped_column(
Integer(), ForeignKey("user.id"), default=1, index=True
)
extra: Mapped[dict[str, Any]] = mapped_column(
JSON(), nullable=True, default_factory=dict
)
@classmethod
def get_all(cls):
with DbManager() as conn:
result = conn.execute(select(cls))
return favorites_to_dataclass(result.fetchall())
@classmethod
def insert_item(cls, item: dict[str, Any]):
item["timestamp"] = int(datetime.datetime.now().timestamp())
item["userid"] = current_user["id"]
with DbManager(commit=True) as conn:
conn.execute(insert(cls).values(item))
@classmethod
def remove_item(cls, item: dict[str, Any]):
with DbManager(commit=True) as conn:
conn.execute(
delete(cls).where(
(cls.hash == item["hash"]) & (cls.type == item["type"])
)
)
@classmethod
def check_exists(cls, hash: str, type: str):
result = cls.execute(select(cls).where((cls.hash == hash) & (cls.type == type)))
return result.fetchone() is not None
@classmethod
def get_all_of_type(cls, table: Any, field: Any, type: str, start: int, limit: int):
result = cls.execute(
select(table)
.select_from(join(table, cls, field == cls.hash))
.where(and_(cls.type == type, cls.userid == current_user["id"]))
.offset(start)
# INFO: If start is 0, fetch all so we can get the total count
.limit(limit if start != 0 else None)
)
res = result.fetchall()
if start == 0:
return res[:limit], len(res)
return res, -1
@classmethod
def get_fav_tracks(cls, start: int, limit: int):
from .libdata import TrackTable
result, total = cls.get_all_of_type(
TrackTable, TrackTable.trackhash, "track", start, limit
)
return tracks_to_dataclasses(result), total
@classmethod
def get_fav_albums(cls, start: int, limit: int):
from .libdata import AlbumTable
result, total = cls.get_all_of_type(
AlbumTable, AlbumTable.albumhash, "album", start, limit
)
return albums_to_dataclasses(result), total
@classmethod
def get_fav_artists(cls, start: int, limit: int):
from .libdata import ArtistTable
result, total = cls.get_all_of_type(
ArtistTable, ArtistTable.artisthash, "artist", start, limit
)
return artists_to_dataclasses(result), total
app/db/utils.py
+75
View File
@@ -0,0 +1,75 @@
from typing import Any
from app.models import Album as AlbumModel, Artist as ArtistModel, Track as TrackModel
from app.models.favorite import Favorite
from app.models.lastfm import SimilarArtist
from app.models.plugins import Plugin
from app.models.user import User
def track_to_dataclass(track: Any):
return TrackModel(**track._asdict())
def tracks_to_dataclasses(tracks: Any):
return [track_to_dataclass(track) for track in tracks]
def album_to_dataclass(album: Any):
return AlbumModel(**album._asdict())
def albums_to_dataclasses(albums: Any):
return [album_to_dataclass(album) for album in albums]
def artist_to_dataclass(artist: Any):
return ArtistModel(**artist._asdict())
def artists_to_dataclasses(artists: Any):
return [artist_to_dataclass(artist) for artist in artists]
# SECTION: User data helpers
def similar_artist_to_dataclass(entry: Any):
entry_dict = entry._asdict()
del entry_dict["id"]
return SimilarArtist(**entry_dict)
def similar_artists_to_dataclass(entries: Any):
return [similar_artist_to_dataclass(entry) for entry in entries]
def favorite_to_dataclass(entry: Any):
entry_dict = entry._asdict()
del entry_dict["id"]
return Favorite(**entry_dict)
def favorites_to_dataclass(entries: Any):
return [favorite_to_dataclass(entry) for entry in entries]
def user_to_dataclass(entry: Any):
entry_dict = entry._asdict()
return User(**entry_dict)
def user_to_dataclasses(entries: Any):
return [user_to_dataclass(entry) for entry in entries]
def plugin_to_dataclass(entry: Any):
entry_dict = entry._asdict()
del entry_dict["id"]
return Plugin(**entry_dict)
def plugin_to_dataclasses(entries: Any):
return [plugin_to_dataclass(entry) for entry in entries]
app/lib/albumslib.py
+6 -48
View File
@@ -12,63 +12,21 @@ from app.store.albums import AlbumStore
from app.store.tracks import TrackStore
def create_albums():
"""
Creates albums from the tracks in the store.
"""
# group all tracks by albumhash
tracks = TrackStore.tracks
tracks = sorted(tracks, key=lambda t: t.albumhash)
grouped = groupby(tracks, lambda t: t.albumhash)
# create albums from the groups
albums: list[Track] = []
for albumhash, tracks in grouped:
count = len(list(tracks))
duration = sum(t.duration for t in tracks)
created_date = min(t.created_date for t in tracks)
album = AlbumStore.create_album(list(tracks)[0])
album.set_count(count)
album.set_duration(duration)
album.set_created_date(created_date)
albums.append(album)
return albums
def validate_albums():
"""
Removes albums that have no tracks.
Probably albums that were added from incompletely written files.
"""
album_hashes = {t.albumhash for t in TrackStore.tracks}
albums = AlbumStore.albums
for album in albums:
if album.albumhash not in album_hashes:
AlbumStore.remove_album(album)
def remove_duplicate_on_merge_versions(tracks: list[Track]) -> list[Track]:
"""
Removes duplicate tracks when merging versions of the same album.
"""
# TODO!
pass
def sort_by_track_no(tracks: list[Track]) -> list[dict[str, Any]]:
tracks = [asdict(t) for t in tracks]
def sort_by_track_no(tracks: list[Track]):
# tracks = [asdict(t) for t in tracks]
for t in tracks:
track = str(t["track"]).zfill(3)
t["_pos"] = int(f"{t['disc']}{track}")
track = str(t.track).zfill(3)
t._pos = int(f"{t.disc}{track}")
tracks = sorted(tracks, key=lambda t: t["_pos"])
tracks = sorted(tracks, key=lambda t: t._pos)
return tracks
app/lib/artistlib.py
+13 -106
View File
@@ -1,5 +1,3 @@
from collections import namedtuple
from itertools import groupby
import os
import urllib
from concurrent.futures import ThreadPoolExecutor
@@ -12,9 +10,10 @@ from requests.exceptions import ConnectionError as RequestConnectionError
from requests.exceptions import ReadTimeout
from app import settings
from app.models import Album, Artist, Track
from app.store import artists as artist_store
from app.store.tracks import TrackStore
from app.db.libdata import ArtistTable
# from app.store import artists as artist_store
# from app.store.tracks import TrackStore
from app.utils.hashing import create_hash
from app.utils.progressbar import tqdm
@@ -107,22 +106,15 @@ class CheckArtistImages:
# read all files in the artist image folder
path = settings.Paths.get_sm_artist_img_path()
processed = "".join(os.listdir(path)).replace("webp", "")
# filter out artists that already have an image
artists = filter(
lambda a: a.artisthash not in processed, artist_store.ArtistStore.artists
)
artists = list(artists)
# process the rest
key_artist_map = ((instance_key, artist) for artist in artists)
processed = [path.replace(".webp", "") for path in os.listdir(path)]
unprocessed = ArtistTable.get_artisthashes_not_in(processed)
key_artist_map = ((instance_key, artist) for artist in unprocessed)
with ThreadPoolExecutor(max_workers=14) as executor:
res = list(
tqdm(
executor.map(self.download_image, key_artist_map),
total=len(artists),
total=len(unprocessed),
desc="Downloading missing artist images",
)
)
@@ -130,7 +122,7 @@ class CheckArtistImages:
list(res)
@staticmethod
def download_image(_map: tuple[str, Artist]):
def download_image(_map: tuple[str, dict[str, str]]):
"""
Checks if an artist image exists and downloads it if not.
@@ -142,16 +134,17 @@ class CheckArtistImages:
return
img_path = (
Path(settings.Paths.get_sm_artist_img_path()) / f"{artist.artisthash}.webp"
Path(settings.Paths.get_sm_artist_img_path())
/ f"{artist['artisthash']}.webp"
)
if img_path.exists():
return
url = get_artist_image_link(artist.name)
url = get_artist_image_link(artist["name"])
if url is not None:
return DownloadImage(url, name=f"{artist.artisthash}.webp")
return DownloadImage(url, name=f"{artist['artisthash']}.webp")
# def fetch_album_bio(title: str, albumartist: str) -> str | None: """ Returns the album bio for a given album. """
@@ -183,89 +176,3 @@ class CheckArtistImages:
# def __call__(self):
# return fetch_album_bio(self.title, self.albumartist)
def get_artists_from_tracks(tracks: list[Track]) -> set[str]:
"""
Extracts all artists from a list of tracks. Returns a list of Artists.
"""
artists = set()
master_artist_list = [[x.name for x in t.artists] for t in tracks]
artists = artists.union(*master_artist_list)
return artists
def get_albumartists(albums: list[Album]) -> set[str]:
artists = set()
for album in albums:
albumartists = [a.name for a in album.albumartists]
artists.update(albumartists)
return artists
def get_all_artists(tracks: list[Track], albums: list[Album]) -> list[Artist]:
TrackInfo = namedtuple(
"TrackInfo",
[
"artisthash",
"albumhash",
"trackhash",
"duration",
"artistname",
"created_date",
],
)
src_tracks = TrackStore.tracks
all_tracks: set[TrackInfo] = set()
for track in src_tracks:
artist_hashes = {(a.name, a.artisthash) for a in track.artists}.union(
(a.name, a.artisthash) for a in track.albumartists
)
for artist in artist_hashes:
track_info = TrackInfo(
artistname=artist[0],
artisthash=artist[1],
albumhash=track.albumhash,
trackhash=track.trackhash,
duration=track.duration,
created_date=track.created_date,
# work on created date
)
all_tracks.add(track_info)
all_tracks = sorted(all_tracks, key=lambda x: x.artisthash)
all_tracks = groupby(all_tracks, key=lambda x: x.artisthash)
artists = []
for artisthash, tracks in all_tracks:
tracks: list[TrackInfo] = list(tracks)
artistname = (
sorted({t.artistname for t in tracks})[0]
if len(tracks) > 1
else tracks[0].artistname
)
albumcount = len({t.albumhash for t in tracks})
duration = sum(t.duration for t in tracks)
created_date = min(t.created_date for t in tracks)
artist = Artist(name=artistname)
artist.set_trackcount(len(tracks))
artist.set_albumcount(albumcount)
artist.set_duration(duration)
artist.set_created_date(created_date)
artists.append(artist)
return artists
app/lib/colorlib.py
+33 -33
View File
@@ -52,47 +52,47 @@ def process_color(item_hash: str, is_album=True):
return get_image_colors(str(path))
class ProcessAlbumColors:
"""
Extracts the most dominant color from the album art and saves it to the database.
"""
# class ProcessAlbumColors:
# """
# Extracts the most dominant color from the album art and saves it to the database.
# """
def __init__(self, instance_key: str) -> None:
global PROCESS_ALBUM_COLORS_KEY
PROCESS_ALBUM_COLORS_KEY = instance_key
# def __init__(self, instance_key: str) -> None:
# global PROCESS_ALBUM_COLORS_KEY
# PROCESS_ALBUM_COLORS_KEY = instance_key
albums = [
a
for a in AlbumStore.albums
if a is not None and a.colors is not None and len(a.colors) == 0
]
# albums = [
# a
# for a in AlbumStore.albums
# if a is not None and a.colors is not None and len(a.colors) == 0
# ]
with SQLiteManager() as cur:
try:
for album in tqdm(albums, desc="Processing missing album colors"):
if PROCESS_ALBUM_COLORS_KEY != instance_key:
raise PopulateCancelledError(
"A newer 'ProcessAlbumColors' instance is running. Stopping this one."
)
# with SQLiteManager() as cur:
# try:
# for album in tqdm(albums, desc="Processing missing album colors"):
# if PROCESS_ALBUM_COLORS_KEY != instance_key:
# raise PopulateCancelledError(
# "A newer 'ProcessAlbumColors' instance is running. Stopping this one."
# )
# TODO: Stop hitting the database for every album.
# Instead, fetch all the data from the database and
# check from memory.
# # TODO: Stop hitting the database for every album.
# # Instead, fetch all the data from the database and
# # check from memory.
exists = aldb.exists(album.albumhash, cur=cur)
if exists:
continue
# exists = aldb.exists(album.albumhash, cur=cur)
# if exists:
# continue
colors = process_color(album.albumhash)
# colors = process_color(album.albumhash)
if colors is None:
continue
# if colors is None:
# continue
album.set_colors(colors)
color_str = json.dumps(colors)
aldb.insert_one_album(cur, album.albumhash, color_str)
finally:
cur.close()
# album.set_colors(colors)
# color_str = json.dumps(colors)
# aldb.insert_one_album(cur, album.albumhash, color_str)
# finally:
# cur.close()
class ProcessArtistColors:
app/lib/folderslib.py
+3 -3
View File
@@ -5,9 +5,10 @@ from app.logger import log
from app.models import Folder
from app.serializers.track import serialize_tracks
from app.settings import SUPPORTED_FILES
from app.store.folder import FolderStore
from app.utils.wintools import win_replace_slash
from app.db import TrackTable as TrackDB
from app.db.libdata import TrackTable as TrackDB
def create_folder(path: str, trackcount=0, foldercount=0) -> Folder:
@@ -43,8 +44,7 @@ def get_folders(paths: list[str]):
Filters out folders that don't have any tracks and
returns a list of folder objects.
"""
folders = TrackDB.count_tracks_containing_paths(paths)
folders = FolderStore.count_tracks_containing_paths(paths)
return [
create_folder(f["path"], f["trackcount"], foldercount=0)
for f in folders
app/lib/populate.py
+103 -103
View File
@@ -1,3 +1,4 @@
from dataclasses import asdict
import os
from collections import deque
from concurrent.futures import ThreadPoolExecutor
@@ -7,28 +8,26 @@ from requests import ConnectionError as RequestConnectionError
from requests import ReadTimeout
from app import settings
from app.db import TrackTable
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable, TrackTable
from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
from app.db.sqlite.settings import SettingsSQLMethods as sdb
# from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
from app.db.sqlite.tracks import SQLiteTrackMethods
from app.lib.albumslib import validate_albums
from app.lib.artistlib import CheckArtistImages
from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors
from app.lib.colorlib import ProcessArtistColors
from app.lib.errors import PopulateCancelledError
from app.lib.taglib import extract_thumb, get_tags
from app.lib.trackslib import validate_tracks
from app.lib.taglib import extract_thumb
from app.logger import log
from app.models import Album, Artist, Track
from app.models.lastfm import SimilarArtist
from app.requests.artists import fetch_similar_artists
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
from app.utils.filesystem import run_fast_scandir
from app.utils.network import has_connection
from app.utils.progressbar import tqdm
from app.db.userdata import SimilarArtistTable
get_all_tracks = SQLiteTrackMethods.get_all_tracks
insert_many_tracks = SQLiteTrackMethods.insert_many_tracks
remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths
@@ -44,50 +43,49 @@ class Populate:
also checks if the album art exists in the image path, if not tries to extract it.
"""
def __init__(self, instance_key: str) -> None:
return
# def __init__(self, instance_key: str) -> None:
# return
# if len(dirs_to_scan) == 0:
# log.warning(
# (
# "The root directory is not configured. "
# + "Open the app in your webbrowser to configure."
# )
# )
# return
# try:
# if dirs_to_scan[0] == "$home":
# dirs_to_scan = [settings.Paths.USER_HOME_DIR]
# except IndexError:
# pass
# files = set()
# for _dir in dirs_to_scan:
# files = files.union(run_fast_scandir(_dir, full=True)[1])
# unmodified, modified_tracks = self.remove_modified(tracks)
# untagged = files - unmodified
# if len(untagged) != 0:
# self.tag_untagged(untagged, instance_key)
# self.extract_thumb_with_overwrite(modified_tracks)
class CordinateMedia:
"""
Cordinates the extracting of thumbnails
"""
def __init__(self, instance_key: str):
global POPULATE_KEY
POPULATE_KEY = instance_key
validate_tracks()
validate_albums()
tracks = get_all_tracks()
dirs_to_scan = sdb.get_root_dirs()
if len(dirs_to_scan) == 0:
log.warning(
(
"The root directory is not configured. "
+ "Open the app in your webbrowser to configure."
)
)
return
try:
if dirs_to_scan[0] == "$home":
dirs_to_scan = [settings.Paths.USER_HOME_DIR]
except IndexError:
pass
files = set()
for _dir in dirs_to_scan:
files = files.union(run_fast_scandir(_dir, full=True)[1])
unmodified, modified_tracks = self.remove_modified(tracks)
untagged = files - unmodified
if len(untagged) != 0:
self.tag_untagged(untagged, instance_key)
self.extract_thumb_with_overwrite(modified_tracks)
try:
ProcessTrackThumbnails(instance_key)
ProcessAlbumColors(instance_key)
ProcessArtistColors(instance_key)
except PopulateCancelledError as e:
log.warn(e)
@@ -95,10 +93,6 @@ class Populate:
tried_to_download_new_images = False
ArtistStore.load_artists(instance_key)
AlbumStore.load_albums(instance_key)
TrackStore.load_all_tracks(instance_key)
if has_connection():
tried_to_download_new_images = True
try:
@@ -123,38 +117,38 @@ class Populate:
log.warn(e)
return
@staticmethod
def remove_modified(tracks: Generator[TrackTable, None, None]):
"""
Removes tracks from the database that have been modified
since they were added to the database.
"""
# @staticmethod
# def remove_modified(tracks: Generator[TrackTable, None, None]):
# """
# Removes tracks from the database that have been modified
# since they were added to the database.
# """
unmodified_paths = set()
modified_tracks: list[TrackTable] = []
modified_paths = set()
# unmodified_paths = set()
# modified_tracks: list[TrackTable] = []
# modified_paths = set()
for track in tracks:
try:
if track.last_mod == round(os.path.getmtime(track.filepath)):
unmodified_paths.add(track.filepath)
continue
except (FileNotFoundError, OSError) as e:
log.warning(e) # REVIEW More informations = good
TrackStore.remove_track_obj(track)
remove_tracks_by_filepaths(track.filepath)
# for track in tracks:
# try:
# if track.last_mod == round(os.path.getmtime(track.filepath)):
# unmodified_paths.add(track.filepath)
# continue
# except (FileNotFoundError, OSError) as e:
# log.warning(e) # REVIEW More informations = good
# TrackStore.remove_track_obj(track)
# remove_tracks_by_filepaths(track.filepath)
modified_paths.add(track.filepath)
modified_tracks.append(track)
# modified_paths.add(track.filepath)
# modified_tracks.append(track)
TrackStore.remove_tracks_by_filepaths(modified_paths)
remove_tracks_by_filepaths(modified_paths)
# TrackStore.remove_tracks_by_filepaths(modified_paths)
# remove_tracks_by_filepaths(modified_paths)
return unmodified_paths, modified_tracks
# return unmodified_paths, modified_tracks
@staticmethod
def tag_untagged(untagged: set[str], key: str):
pass
# @staticmethod
# def tag_untagged(untagged: set[str], key: str):
# pass
# for file in tqdm(untagged, desc="Reading files"):
# if POPULATE_KEY != key:
# log.warning("'Populate.tag_untagged': Populate key changed")
@@ -206,18 +200,18 @@ class Populate:
# log.info("Added %s/%s tracks", tagged_count, len(untagged))
@staticmethod
def extract_thumb_with_overwrite(tracks: list[TrackTable]):
"""
Extracts the thumbnail from a list of filepaths,
overwriting the existing thumbnail if it exists,
for modified files.
"""
for track in tracks:
try:
extract_thumb(track.filepath, track.image, overwrite=True)
except FileNotFoundError:
continue
# @staticmethod
# def extract_thumb_with_overwrite(tracks: list[TrackTable]):
# """
# Extracts the thumbnail from a list of filepaths,
# overwriting the existing thumbnail if it exists,
# for modified files.
# """
# for track in tracks:
# try:
# extract_thumb(track.filepath, track.image, overwrite=True)
# except FileNotFoundError:
# continue
def get_image(_map: tuple[str, Album]):
@@ -235,7 +229,8 @@ def get_image(_map: tuple[str, Album]):
raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed")
matching_tracks = filter(
lambda t: t.albumhash == album.albumhash, TrackStore.tracks
lambda t: t.albumhash == album.albumhash,
TrackTable.get_tracks_by_albumhash(album.albumhash),
)
try:
@@ -254,8 +249,12 @@ def get_image(_map: tuple[str, Album]):
pass
_cpu_count = os.cpu_count()
CPU_COUNT = _cpu_count // 2 if _cpu_count > 2 else _cpu_count
def get_cpu_count():
"""
Returns the number of CPUs on the machine.
"""
cpu_count = os.cpu_count() or 0
return cpu_count // 2 if cpu_count > 2 else cpu_count
class ProcessTrackThumbnails:
@@ -275,14 +274,14 @@ class ProcessTrackThumbnails:
# filter out albums that already have thumbnails
albums = filter(
lambda album: album.albumhash not in processed, AlbumStore.albums
lambda album: album.albumhash not in processed, AlbumTable.get_all()
)
albums = list(albums)
# process the rest
key_album_map = ((instance_key, album) for album in albums)
with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor:
with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor:
results = list(
tqdm(
executor.map(get_image, key_album_map),
@@ -307,16 +306,17 @@ def save_similar_artists(_map: tuple[str, Artist]):
"'FetchSimilarArtistsLastFM': Populate key changed"
)
if lastfmdb.exists(artist.artisthash):
if SimilarArtistTable.exists(artist.artisthash):
return
artist_hashes = fetch_similar_artists(artist.name)
artist_ = SimilarArtist(artist.artisthash, "~".join(artist_hashes))
artists = fetch_similar_artists(artist.name)
if len(artist_.similar_artist_hashes) == 0:
# INFO: Nones mean there was a connection error
if artists is None:
return
lastfmdb.insert_one(artist_)
artist_ = SimilarArtist(artist.artisthash, artists)
SimilarArtistTable.insert_one(asdict(artist_))
class FetchSimilarArtistsLastFM:
@@ -326,17 +326,17 @@ class FetchSimilarArtistsLastFM:
def __init__(self, instance_key: str) -> None:
# read all artists from db
processed = lastfmdb.get_all()
processed = SimilarArtistTable.get_all()
processed = ".".join(a.artisthash for a in processed)
# filter out artists that already have similar artists
artists = filter(lambda a: a.artisthash not in processed, ArtistStore.artists)
artists = filter(lambda a: a.artisthash not in processed, ArtistTable.get_all())
artists = list(artists)
# process the rest
key_artist_map = ((instance_key, artist) for artist in artists)
with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor:
with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor:
try:
print("Processing similar artists")
results = list(
app/lib/tagger.py
+163 -33
View File
@@ -1,55 +1,165 @@
import os
from pprint import pprint
from app.db import AlbumTable, ArtistTable, TrackTable
from app.lib.taglib import get_tags
from time import time
from typing import Generator
from app import settings
from app.config import UserConfig
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable, TrackTable
from app.lib.populate import CordinateMedia
from app.lib.taglib import extract_thumb, get_tags
from app.models.track import Track
from app.store.folder import FolderStore
from app.utils.filesystem import run_fast_scandir
from app.utils.parsers import get_base_album_title
from app.utils.progressbar import tqdm
from app.logger import log
from app.utils.threading import background
POPULATE_KEY: float = 0
class IndexTracks:
def __init__(self) -> None:
dirs_to_scan = ["/home/cwilvx/Music"]
def __init__(self, instance_key: float) -> None:
"""
Indexes all tracks in the database.
An instance key is used to prevent multiple instances of the
same class from running at the same time.
"""
global POPULATE_KEY
POPULATE_KEY = instance_key
# dirs_to_scan = sdb.get_root_dirs()
dirs_to_scan = UserConfig().rootDirs
if len(dirs_to_scan) == 0:
log.warning(
(
"The root directory is not configured. "
+ "Open the app in your webbrowser to configure."
)
)
return
try:
if dirs_to_scan[0] == "$home":
dirs_to_scan = [settings.Paths.USER_HOME_DIR]
except IndexError:
pass
files = set()
for _dir in dirs_to_scan:
files = files.union(run_fast_scandir(_dir, full=True)[1])
self.tag_untagged(files)
# unmodified, modified_tracks = self.remove_modified(tracks)
# untagged = files - unmodified
unmodified, modified_tracks = self.filter_modded()
untagged = files - unmodified
def tag_untagged(self, files: set[str]):
self.tag_untagged(untagged, instance_key)
self.extract_thumb_with_overwrite(modified_tracks)
@staticmethod
def extract_thumb_with_overwrite(tracks: list[dict[str, str]]):
"""
Extracts the thumbnail from a list of filepaths,
overwriting the existing thumbnail if it exists,
for modified files.
"""
for track in tracks:
try:
extract_thumb(
track["filepath"], track["trackhash"] + ".webp", overwrite=True
)
except FileNotFoundError:
continue
@staticmethod
def filter_modded():
"""
Removes tracks from the database that have been modified
since they were indexed.
Returns a tuple of unmodified paths and modified tracks.
Unmodified paths are indexed and the modified tracks are
"""
unmodified_paths = set()
modified_tracks: list[dict[str, str]] = []
to_remove = set()
for track in TrackTable.get_all():
try:
if track.last_mod == round(os.path.getmtime(track.filepath)):
unmodified_paths.add(track.filepath)
continue
except (FileNotFoundError, OSError) as e:
log.warning(e) # REVIEW More informations = good
to_remove.add(track.filepath)
modified_tracks.append(
{
"filepath": track.filepath,
"trackhash": track.trackhash,
}
)
to_remove = to_remove.union(set(t["filepath"] for t in modified_tracks))
TrackTable.remove_tracks_by_filepaths(to_remove)
# REVIEW: Remove after testing!
track = TrackTable.get_tracks_by_filepaths(list(to_remove)[:1])
if track:
raise Exception("Track not removed")
# =============================================================
return unmodified_paths, modified_tracks
def get_untagged(self):
tracks = TrackTable.get_all()
def tag_untagged(self, files: set[str], key: float):
config = UserConfig()
for file in tqdm(files, desc="Reading files"):
# if POPULATE_KEY != key:
# log.warning("'Populate.tag_untagged': Populate key changed")
# return
if POPULATE_KEY != key:
log.warning("'Populate.tag_untagged': Populate key changed")
return
tags = get_tags(file)
tags = get_tags(file, artist_separators=config.artistSeparators)
if tags is not None:
TrackTable.insert_one(tags)
FolderStore.filepaths.add(tags["filepath"])
del tags
print(f"{len(files)} new files indexed")
print("Done")
class IndexAlbums:
def __init__(self) -> None:
albums = dict()
all_tracks: list[Track] = TrackTable.get_all()
all_tracks: list[TrackTable] = TrackTable.get_all()
if len(all_tracks) == 0:
return
for track in all_tracks:
if track.albumhash not in albums:
albums[track.albumhash] = {
"albumartists": track.albumartists,
"artisthashes": [a['artisthash'] for a in track.albumartists],
"artisthashes": [a["artisthash"] for a in track.albumartists],
"albumhash": track.albumhash,
"base_title": None,
"color": None,
"created_date": None,
"date": None,
"duration": track.duration,
"genres": [*track.genre] if track.genre else [],
"genres": [*track.genres] if track.genres else [],
"og_title": track.og_album,
"title": track.album,
"trackcount": 1,
@@ -63,8 +173,8 @@ class IndexAlbums:
album["dates"].append(track.date)
album["created_dates"].append(track.last_mod)
if track.genre:
album["genres"].extend(track.genre)
if track.genres:
album["genres"].extend(track.genres)
for album in albums.values():
album["date"] = min(album["dates"])
@@ -79,20 +189,23 @@ class IndexAlbums:
album["genres"] = genres
album["base_title"], _ = get_base_album_title(album["og_title"])
del genres
del album["dates"]
del album["created_dates"]
pprint(albums)
AlbumTable.remove_all()
AlbumTable.insert_many(list(albums.values()))
del albums
class IndexArtists:
def __init__(self) -> None:
all_tracks: list[TrackTable] = TrackTable.get_all()
all_tracks: list[Track] = TrackTable.get_all()
artists = dict()
if len(all_tracks) == 0:
return
for track in all_tracks:
this_artists = track.artists
@@ -100,32 +213,33 @@ class IndexArtists:
if a not in this_artists:
this_artists.append(a)
for artist in this_artists:
if artist["artisthash"] not in artists:
artists[artist["artisthash"]] = {
for thisartist in this_artists:
if thisartist["artisthash"] not in artists:
artists[thisartist["artisthash"]] = {
"albumcount": None,
"albums": {track.albumhash},
"artisthash": artist["artisthash"],
"artisthash": thisartist["artisthash"],
"created_dates": [track.last_mod],
"dates": [track.date],
"date": None,
"duration": track.duration,
"genres": track.genre if track.genre else [],
"name": artist["name"],
"genres": track.genres if track.genres else [],
"name": None,
"names": {thisartist["name"]},
"trackcount": None,
"tracks": {track.trackhash},
}
else:
artist = artists[artist["artisthash"]]
artist = artists[thisartist["artisthash"]]
artist["duration"] += track.duration
artist["albums"].add(track.albumhash)
artist["tracks"].add(track.trackhash)
artist["dates"].append(track.date)
artist["created_dates"].append(track.last_mod)
artist["names"].add(thisartist["name"])
if track.genre:
artist["genres"].extend(track.genre)
if track.genres:
artist["genres"].extend(track.genres)
for artist in artists.values():
artist["albumcount"] = len(artist["albums"])
@@ -140,19 +254,35 @@ class IndexArtists:
genres.append(genre)
artist["genres"] = genres
artist["name"] = sorted(artist["names"])[0]
# INFO: Delete temporary keys
del artist["names"]
del artist["tracks"]
del artist["albums"]
del artist["dates"]
del artist["created_dates"]
pprint(artists)
# INFO: Delete local variables
del genres
ArtistTable.remove_all()
ArtistTable.insert_many(list(artists.values()))
del artists
class IndexEverything:
def __init__(self) -> None:
IndexTracks()
IndexTracks(instance_key=time())
IndexAlbums()
IndexArtists()
pass
FolderStore.load_filepaths()
# pass
CordinateMedia(instance_key=str(time()))
@background
def index_everything():
return IndexEverything()
app/lib/taglib.py
+42 -24
View File
@@ -5,6 +5,7 @@ from pathlib import Path
from pprint import pprint
import re
import sys
from typing import Any
import pendulum
from PIL import Image, UnidentifiedImageError
@@ -86,7 +87,7 @@ def extract_thumb(filepath: str, webp_path: str, overwrite=False) -> bool:
return False
def parse_date(date_str: str | None) -> int | None:
def parse_date(date_str: str) -> int | None:
"""
Extracts the date from a string and returns a timestamp.
"""
@@ -108,12 +109,13 @@ def clean_filename(filename: str):
class ParseData:
artist: str
title: str
artist_separators: set[str]
def __post_init__(self):
self.artist = split_artists(self.artist)
self.artist = split_artists(self.artist, self.artist_separators)
def extract_artist_title(filename: str):
def extract_artist_title(filename: str, artist_separators: set[str]):
path = Path(filename).with_suffix("")
path = clean_filename(str(path))
@@ -121,22 +123,24 @@ def extract_artist_title(filename: str):
split_result = [x.strip() for x in split_result]
if len(split_result) == 1:
return ParseData("", split_result[0])
return ParseData("", split_result[0], artist_separators)
if len(split_result) > 2:
try:
int(split_result[0])
return ParseData(split_result[1], " - ".join(split_result[2:]))
return ParseData(
split_result[1], " - ".join(split_result[2:]), artist_separators
)
except ValueError:
pass
artist = split_result[0]
title = split_result[1]
return ParseData(artist, title)
return ParseData(artist, title, artist_separators)
def get_tags(filepath: str):
def get_tags(filepath: str, artist_separators: set[str]):
"""
Returns the tags for a given audio file.
"""
@@ -150,7 +154,7 @@ def get_tags(filepath: str):
return None
try:
tags = TinyTag.get(filepath)
tags: Any = TinyTag.get(filepath)
except: # noqa: E722
return None
@@ -169,7 +173,7 @@ def get_tags(filepath: str):
for tag in to_filename:
p = getattr(tags, tag)
if p == "" or p is None:
parse_data = extract_artist_title(filename)
parse_data = extract_artist_title(filename, artist_separators)
title = parse_data.title
setattr(tags, tag, title)
@@ -179,7 +183,7 @@ def get_tags(filepath: str):
if p == "" or p is None:
if not parse_data:
parse_data = extract_artist_title(filename)
parse_data = extract_artist_title(filename, artist_separators)
artist = parse_data.artist
@@ -225,8 +229,8 @@ def get_tags(filepath: str):
tags.artists = tags.artist
tags.albumartists = tags.albumartist
split_artist = split_artists(tags.artist)
split_albumartists = split_artists(tags.albumartist)
split_artist = split_artists(tags.artist, separators=artist_separators)
split_albumartists = split_artists(tags.albumartist, separators=artist_separators)
new_title = tags.title
# TODO: Figure out which is the best spot to create these hashes
@@ -237,7 +241,9 @@ def get_tags(filepath: str):
# extract featured artists
if config.extractFeaturedArtists:
feat, new_title = parse_feat_from_title(tags.title)
feat, new_title = parse_feat_from_title(
tags.title, separators=artist_separators
)
original_lower = "-".join([create_hash(a) for a in split_artist])
split_artist.extend(a for a in feat if create_hash(a) not in original_lower)
@@ -262,8 +268,9 @@ def get_tags(filepath: str):
for a in split_albumartists
]
tags.artisthashes = list({a["artisthash"] for a in tags.artists + tags.albumartists})
tags.artisthashes = list(
{a["artisthash"] for a in tags.artists + tags.albumartists}
)
# remove prod by
if config.removeProdBy:
@@ -295,26 +302,32 @@ def get_tags(filepath: str):
# process genres
if tags.genre:
tags.genre = tags.genre.lower()
src_genres: str = tags.genre
src_genres = src_genres.lower()
# separators = {"/", ";", "&"}
separators = set(config.genreSeparators)
contains_rnb = "r&b" in tags.genre
contains_rock = "rock & roll" in tags.genre
contains_rnb = "r&b" in src_genres
contains_rock = "rock & roll" in src_genres
if contains_rnb:
tags.genre = tags.genre.replace("r&b", "RnB")
src_genres = src_genres.replace("r&b", "RnB")
if contains_rock:
tags.genre = tags.genre.replace("rock & roll", "rock")
src_genres = src_genres.replace("rock & roll", "rock")
for s in separators:
tags.genre = tags.genre.replace(s, ",")
src_genres = src_genres.replace(s, ",")
tags.genre = tags.genre.split(",")
tags.genre = [
{"name": g.strip(), "genrehash": create_hash(g.strip())} for g in tags.genre
genres_list: list[str] = src_genres.split(",")
tags.genres = [
{"name": g.strip(), "genrehash": create_hash(g.strip())}
for g in genres_list
]
tags.genrehashes = [g["genrehash"] for g in tags.genres]
else:
tags.genres = []
tags.genrehashes = []
# sub underscore with space
tags.title = tags.title.replace("_", " ")
@@ -333,6 +346,10 @@ def get_tags(filepath: str):
"filesize": tags.filesize,
"samplerate": tags.samplerate,
"track_total": tags.track_total,
"hashinfo": {
"algo": "sha1",
"format": "[:5]+[-5:]", # first 5 + last 5 chars
},
}
tags.extra = {**tags.extra, **more_extra}
@@ -357,6 +374,7 @@ def get_tags(filepath: str):
"bitdepth",
"artist",
"albumartist",
"genre",
]
for tag in to_delete:
app/lib/trackslib.py
-10
View File
@@ -13,16 +13,6 @@ from app.utils.progressbar import tqdm
from app.utils.threading import ThreadWithReturnValue
def validate_tracks() -> None:
"""
Removes track records whose files no longer exist.
"""
for track in tqdm(TrackStore.tracks, desc="Validating tracks"):
if not os.path.exists(track.filepath):
TrackStore.remove_track_obj(track)
trackdb.remove_tracks_by_filepaths(track.filepath)
def get_leading_silence_end(filepath: str):
"""
Returns the leading silence of a track.
app/lib/watchdogg.py
+7 -3
View File
@@ -11,8 +11,10 @@ from watchdog.events import PatternMatchingEventHandler
from watchdog.observers import Observer
from app import settings
from app.config import UserConfig
from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb
from app.db.sqlite.settings import SettingsSQLMethods as sdb
# from app.db.sqlite.settings import SettingsSQLMethods as sdb
from app.db.sqlite.tracks import SQLiteManager
from app.db.sqlite.tracks import SQLiteTrackMethods as db
from app.lib.colorlib import process_color
@@ -43,7 +45,8 @@ class Watcher:
while trials < 10:
try:
dirs = sdb.get_root_dirs()
# dirs = sdb.get_root_dirs()
dirs = UserConfig().rootDirs
dirs = [rf"{d}" for d in dirs]
dir_map = [
@@ -152,7 +155,8 @@ def add_track(filepath: str) -> None:
TrackStore.remove_track_by_filepath(filepath)
tags = get_tags(filepath)
config = UserConfig()
tags = get_tags(filepath, artist_separators=config.artistSeparators)
# if the track is somehow invalid, return
if tags is None or tags["bitrate"] == 0 or tags["duration"] == 0:
app/migrations/__init__.py
+17 -12
View File
@@ -7,7 +7,9 @@ Reads and applies the latest database migrations.
import inspect
import sys
from types import ModuleType
from app.db.sqlite.migrations import MigrationManager
# from app.db.sqlite.migrations import MigrationManager
from app.db.metadata import MigrationTable
from app.logger import log
from app.migrations import v1_3_0, v1_4_9
from app.migrations.base import Migration
@@ -42,26 +44,29 @@ def apply_migrations():
modules = [v1_3_0, v1_4_9]
migrations = [get_all_migrations(m) for m in modules]
index = MigrationManager.get_index()
# index = MigrationManager.get_index()
index = MigrationTable.get_version()
all_migrations = [migration for sublist in migrations for migration in sublist]
to_apply: list[Migration] = []
# if index is from old release,
# get migrations from the "migrations" list
if index < 3:
_migrations = migrations[index:]
to_apply = [migration for sublist in _migrations for migration in sublist]
else:
to_apply = all_migrations[index:]
for migration in to_apply:
# try:
migration.migrate()
log.info("Applied migration: %s", migration.__name__)
# if index < 3:
# _migrations = migrations[index:]
# to_apply = [migration for sublist in _migrations for migration in sublist]
# else:
# to_apply = all_migrations[index:]
# for migration in to_apply:
# # try:
# migration.migrate()
# log.info("Applied migration: %s", migration.__name__)
# except Exception as e:
# log.error("Failed to run migration: %s", migration.__name__)
# log.error(e)
# sys.exit(0)
MigrationManager.set_index(len(all_migrations))
# MigrationManager.set_index(len(all_migrations))
MigrationTable.set_version(len(all_migrations))
app/models/album.py
+7 -4
View File
@@ -2,9 +2,6 @@ import dataclasses
import datetime
from dataclasses import dataclass
from app.config import UserConfig
from app.settings import SessionVarKeys, get_flag
from ..utils.hashing import create_hash
from ..utils.parsers import get_base_title_and_versions, parse_feat_from_title
from .artist import Artist
@@ -27,15 +24,21 @@ class Album:
date: int
duration: int
genres: list[dict[str, str]]
genrehashes: list[str]
og_title: str
title: str
trackcount: int
is_favorite: bool
extra: dict
type: str = "album"
image: str = ""
versions: list[str] = dataclasses.field(default_factory=list)
def __post_init__(self):
self.date = datetime.datetime.fromtimestamp(self.date).year
# self.date = datetime.datetime.fromtimestamp(self.date).year
self.image = self.albumhash + ".webp"
self.populate_versions()
# albumhash: str
# title: str
app/models/artist.py
+7
View File
@@ -44,6 +44,13 @@ class Artist:
date: int
duration: int
genres: list[dict[str, str]]
genrehashes: list[str]
name: str
trackcount: int
is_favorite: bool
extra: dict
image: str = ""
def __post_init__(self):
self.image = self.artisthash + ".webp"
app/models/favorite.py
+11
View File
@@ -0,0 +1,11 @@
from dataclasses import dataclass
from typing import Any, Literal
@dataclass
class Favorite:
hash: str
type: Literal["album", "track", "artist"]
timestamp: int
userid: int
extra: dict[str, Any]
app/models/lastfm.py
+17 -2
View File
@@ -1,13 +1,28 @@
from dataclasses import dataclass
from typing import Any
@dataclass
class SimilarArtistEntry:
artisthash: str
name: str
weight: float
scrobbles: int
listeners: int
@dataclass
class SimilarArtist:
artisthash: str
similar_artist_hashes: str
similar_artists: list[SimilarArtistEntry]
def get_artist_hash_set(self) -> set[str]:
"""
Returns a set of similar artists.
"""
return set(self.similar_artist_hashes.split("~"))
if not self.similar_artists:
return set()
# INFO:
return set(a['artisthash'] for a in self.similar_artists)
app/models/plugins.py
+1 -1
View File
@@ -4,7 +4,7 @@ from dataclasses import dataclass
@dataclass
class Plugin:
name: str
description: str
active: bool
settings: dict
extra: dict
app/models/track.py
+12 -19
View File
@@ -1,20 +1,4 @@
from dataclasses import dataclass, field
import os
from pathlib import Path
from flask_jwt_extended import current_user
from app.settings import SessionVarKeys, get_flag
from app.utils.hashing import create_hash
from app.utils.parsers import (
clean_title,
get_base_title_and_versions,
parse_feat_from_title,
remove_prod,
split_artists,
)
from .artist import ArtistMinimal
from dataclasses import dataclass
@dataclass(slots=True)
@@ -28,7 +12,7 @@ class Track:
albumartists: list[dict[str, str]]
albumhash: str
artisthashes: list[str]
artists: str
artists: list[dict[str, str]]
bitrate: int
copyright: str
date: int
@@ -36,7 +20,8 @@ class Track:
duration: int
filepath: str
folder: str
genre: list[dict[str, str]]
genres: list[dict[str, str]]
genrehashes: list[str]
last_mod: int
og_album: str
og_title: str
@@ -48,7 +33,15 @@ class Track:
is_favorite: bool = False
_pos: int = 0
_ati: str = ""
image: str = ""
def __post_init__(self):
self.image = self.albumhash + ".webp"
self.extra = {
"disc_total": self.extra.get("disc_total", 0),
"track_total": self.extra.get("track_total", 0),
"samplerate": self.extra.get("samplerate", -1),
}
# album: str
# albumartists: str | list[ArtistMinimal]
app/models/user.py
+5 -9
View File
@@ -5,19 +5,15 @@ import json
@dataclass(slots=True)
class User:
id: int
username: str
firstname: str
lastname: str
password: str
email: str
image: str
password: str
username: str
roles: list[str]
extra: dict[str, str] = field(default_factory=dict)
# NOTE: roles: ['admin', 'user', 'curator']
roles: list[str] = field(default_factory=lambda: ["user"])
def __post_init__(self):
self.roles = json.loads(self.roles)
def todict(self):
this_dict = asdict(self)
del this_dict["password"]
@@ -28,5 +24,5 @@ class User:
return {
"id": self.id,
"username": self.username,
"firstname": self.firstname,
"firstname": self.extra["firstname"] if self.extra else "",
}
app/periodic_scan.py
+5 -7
View File
@@ -1,14 +1,16 @@
"""
This module contains functions for the server
"""
import time
from app.config import UserConfig
from app.lib.populate import Populate, PopulateCancelledError
from app.settings import SessionVarKeys, get_flag, get_scan_sleep_time
from app.utils.generators import get_random_str
from app.utils.threading import background
from app.logger import log
@background
def run_periodic_scans():
"""
@@ -23,10 +25,7 @@ def run_periodic_scans():
# ValidateAlbumThumbs()
# ValidatePlaylistThumbs()
run_periodic_scan = True
while run_periodic_scan:
run_periodic_scan = get_flag(SessionVarKeys.DO_PERIODIC_SCANS)
while UserConfig().enablePeriodicScans:
try:
Populate(instance_key=get_random_str())
@@ -34,5 +33,4 @@ def run_periodic_scans():
log.error("'run_periodic_scans': Periodic scan cancelled.")
pass
sleep_time = get_scan_sleep_time()
time.sleep(sleep_time)
time.sleep(UserConfig().scanInterval)
app/plugins/register.py
+15 -2
View File
@@ -1,5 +1,18 @@
from app.db.sqlite.plugins import PluginsMethods
from app.db.userdata import PluginTable
from sqlalchemy.exc import IntegrityError
def register_plugins():
PluginsMethods.insert_lyrics_plugin()
try:
PluginTable.insert_one(
{
"name": "lyrics_finder",
"active": False,
"settings": {"auto_download": False},
"extra": {
"description": "Find lyrics from the internet",
},
}
)
except IntegrityError:
pass
app/requests/artists.py
+14 -3
View File
@@ -7,6 +7,7 @@ import urllib.parse
import requests
from requests import ConnectionError, HTTPError, ReadTimeout
from app.models.lastfm import SimilarArtistEntry
from app.utils.hashing import create_hash
@@ -20,7 +21,7 @@ def fetch_similar_artists(name: str):
response = requests.get(url, timeout=10)
response.raise_for_status()
except (ConnectionError, ReadTimeout, HTTPError):
return []
return None
data = response.json()
@@ -29,5 +30,15 @@ def fetch_similar_artists(name: str):
except KeyError:
return []
for artist in artists:
yield create_hash(artist["name"])
return [
SimilarArtistEntry(
**{
"artisthash": create_hash(artist["name"]),
"name": artist["name"],
"weight": artist["weight"],
"listeners": int(artist["listeners"]),
"scrobbles": int(artist["scrobbles"]),
}
)
for artist in artists
]
app/settings.py
-11
View File
@@ -244,17 +244,6 @@ class SessionVarKeys:
SHOW_ALBUMS_AS_SINGLES = "SHOW_ALBUMS_AS_SINGLES"
def get_flag(key: SessionVarKeys) -> bool:
return getattr(SessionVars, key)
def set_flag(key: SessionVarKeys, value: Any):
setattr(SessionVars, key, value)
def get_scan_sleep_time() -> int:
return SessionVars.PERIODIC_SCAN_INTERVAL
class TCOLOR:
"""
app/setup/__init__.py
+11 -10
View File
@@ -3,11 +3,11 @@ Prepares the server for use.
"""
import uuid
from app.db.sqlite.settings import load_settings
from app.setup.files import create_config_dir
from app.setup.sqlite import run_migrations, setup_sqlite
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.folder import FolderStore
from app.store.tracks import TrackStore
from app.utils.generators import get_random_str
from app.config import UserConfig
@@ -29,20 +29,21 @@ def run_setup():
setup_sqlite()
run_migrations()
try:
load_settings()
except IndexError:
# settings table is empty
pass
# try:
# load_settings()
# except IndexError:
# # settings table is empty
# pass
def load_into_mem():
"""
Load all tracks, albums, and artists into memory.
"""
instance_key = get_random_str()
# instance_key = get_random_str()
# INFO: Load all tracks, albums, and artists into memory
TrackStore.load_all_tracks(instance_key)
AlbumStore.load_albums(instance_key)
ArtistStore.load_artists(instance_key)
# TrackStore.load_all_tracks(instance_key)
# AlbumStore.load_albums(instance_key)
# ArtistStore.load_artists(instance_key)
FolderStore.load_filepaths()
app/setup/sqlite.py
+8 -14
View File
@@ -3,11 +3,15 @@ Module to setup Sqlite databases and tables.
Applies migrations.
"""
from app.db.userdata import UserTable
from app.db.sqlite import create_connection, create_tables, queries
from app.db.sqlite.auth import SQLiteAuthMethods as authdb
from app.migrations import apply_migrations
from app.settings import Db
from app.db import create_all
from app.db.libdata import create_all as create_all_libdata
def run_migrations():
"""
@@ -20,18 +24,8 @@ def setup_sqlite():
"""
Create Sqlite databases and tables.
"""
# if os.path.exists(DB_PATH):
# os.remove(DB_PATH)
create_all()
create_all_libdata()
app_db_conn = create_connection(Db.get_app_db_path())
user_db_conn = create_connection(Db.get_userdata_db_path())
create_tables(app_db_conn, queries.CREATE_APPDB_TABLES)
create_tables(user_db_conn, queries.CREATE_USERDATA_TABLES)
create_tables(app_db_conn, queries.CREATE_MIGRATIONS_TABLE)
if not authdb.get_all_users():
authdb.insert_default_user()
app_db_conn.close()
user_db_conn.close()
if not UserTable.get_all():
UserTable.insert_default_user()
app/store/artists.py
-1
View File
@@ -1,7 +1,6 @@
import json
from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb
from app.lib.artistlib import get_all_artists
from app.models import Artist
from app.utils.bisection import use_bisection
from app.utils.customlist import CustomList
app/store/folder.py
+95
View File
@@ -0,0 +1,95 @@
from sortedcontainers import SortedSet
from concurrent.futures import ThreadPoolExecutor
from app.db.libdata import TrackTable
class FolderStore:
"""
The Folder store is used to hold all the indexed tracks filepaths in memory
for fast count operations when browsing the folder page.
Counting from the database is super slow, even with a small number of folders to get the count for. Up to 700ms for 10 folders. By using this store, we are able to reduce that to less than 10ms.
"""
filepaths: SortedSet = SortedSet()
@classmethod
def load_filepaths(cls):
"""
Load all the filepaths from the database into memory.
This is needed to speed up the process of counting the number of tracks in the folder page.
"""
cls.filepaths.clear()
tracks = TrackTable.get_all()
for track in tracks:
cls.filepaths.add(track.filepath)
@classmethod
def count_tracks_containing_paths(cls, paths: list[str]):
"""
Count the number of tracks in each directory.
Uses a ThreadPoolExecutor to count the number of tracks
in each directory for fast execution time.
"""
results: list[dict[str, int | str]] = []
with ThreadPoolExecutor() as executor:
res = executor.map(countFilepathsInDir, paths)
results = [
{"path": path, "trackcount": count} for path, count in zip(paths, res)
]
return results
def getIndexOfFirstMatch(strings: list[str], prefix: str):
"""
Find the index of the first path that starts with the given path.
Uses a binary search algorithm to find the index.
"""
left = 0
right = len(strings) - 1
while left <= right:
mid = (left + right) // 2
if strings[mid].startswith(prefix):
if mid == 0 or not strings[mid - 1].startswith(prefix):
return mid
right = mid - 1
elif strings[mid] < prefix:
left = mid + 1
else:
right = mid - 1
return -1
def countFilepathsInDir(dirpath: str):
"""
Counts the number of filepaths that start with the given directory path.
Gets the index of the first path that starts with the given directory path,
then checks each path after that to see if it starts with the given directory path.
"""
index = getIndexOfFirstMatch(FolderStore.filepaths, dirpath)
if index == -1:
return 0
paths: list[str] = []
for path in FolderStore.filepaths[index:]:
if path.startswith(dirpath):
paths.append(path)
else:
break
return len(paths)
app/utils/dates.py
+1 -1
View File
@@ -26,7 +26,7 @@ def create_new_date(date: datetime = None) -> str:
return date.strftime(_format)
def timestamp_to_time_passed(timestamp: str):
def timestamp_to_time_passed(timestamp: str | int):
"""
Converts a timestamp to time passed. e.g. 2 minutes ago, 1 hour ago, yesterday, 2 days ago, 2 weeks ago, etc.
"""
app/utils/hashing.py
+1
View File
@@ -34,6 +34,7 @@ def create_hash(*args: str, decode=False, limit=10) -> str:
str_ = str_.encode("utf-8")
str_ = hashlib.sha1(str_).hexdigest()
# INFO: Return first 5 + last 5 characters
return (
str_[: limit // 2] + str_[-limit // 2 :]
if limit % 2 == 0
app/utils/parsers.py
+3 -5
View File
@@ -1,14 +1,12 @@
import re
from app.enums.album_versions import AlbumVersionEnum, get_all_keywords
from app.settings import SessionVarKeys, get_flag
def split_artists(src: str):
def split_artists(src: str, separators: set[str]):
"""
Splits a string of artists into a list of artists.
"""
separators: set = get_flag(SessionVarKeys.ARTIST_SEPARATORS)
for sep in separators:
src = src.replace(sep, ",")
@@ -38,7 +36,7 @@ def remove_prod(title: str) -> str:
return title.strip()
def parse_feat_from_title(title: str) -> tuple[list[str], str]:
def parse_feat_from_title(title: str, separators: set[str]) -> tuple[list[str], str]:
"""
Extracts featured artists from a song title using regex.
"""
@@ -56,7 +54,7 @@ def parse_feat_from_title(title: str) -> tuple[list[str], str]:
return [], title
artists = match.group(1)
artists = split_artists(artists)
artists = split_artists(artists, separators)
# remove "feat" group from title
new_title = re.sub(regex, "", title, flags=re.IGNORECASE)
manage.py
+1 -1
View File
@@ -45,7 +45,7 @@ mimetypes.add_type("image/gif", ".gif")
mimetypes.add_type("font/woff", ".woff")
mimetypes.add_type("application/manifest+json", ".webmanifest")
logging.disable(logging.CRITICAL)
# logging.disable(logging.CRITICAL)
# werkzeug = logging.getLogger("werkzeug")
# werkzeug.setLevel(logging.ERROR)
poetry.lock
Generated
+1 -1
View File
@@ -2616,4 +2616,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.12"
content-hash = "9c7ba20671a6a3b59dbb120e3e56ded7e4dfcbf2de14418bdef41059233cdcb1"
content-hash = "80cb2755efc6cec2cb20d50cb8927dee554991741283e70e7a2665e6253b895d"
pyproject.toml
+1
View File
@@ -28,6 +28,7 @@ flask-openapi3 = "^3.0.2"
flask-jwt-extended = "^4.6.0"
sqlalchemy = "^2.0.31"
memory-profiler = "^0.61.0"
sortedcontainers = "^2.4.0"
[tool.poetry.dev-dependencies]
pylint = "^2.15.5"