From c42ec4dcded20171bcca3905275c46088cd2b4fe Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 24 Jun 2024 00:26:47 +0300 Subject: [PATCH] start: rewrite the database layer using a freaking ORM + start ditching in-mem stores + move main db table to a new name + experiments! --- .github/changelog.md | 13 ++ TODO.md | 7 + app/api/album.py | 117 ++++++------- app/api/artist.py | 2 +- app/api/folder.py | 2 + app/api/getall/__init__.py | 17 +- app/config.py | 1 + app/db/__init__.py | 232 +++++++++++++++++++++++++ app/db/sqlite/auth.py | 2 - app/db/sqlite/logger/tracks.py | 7 +- app/db/sqlite/playlists.py | 36 +++- app/db/sqlite/tracks.py | 14 ++ app/lib/folderslib.py | 75 ++++---- app/lib/populate.py | 70 ++++---- app/lib/searchlib.py | 2 +- app/lib/tagger.py | 154 +++++++++++++++++ app/lib/taglib.py | 104 ++++++++++- app/migrations/__init__.py | 10 +- app/migrations/v1_4_9/__init__.py | 278 +++++++++++++++++++++++++++--- app/models/album.py | 208 ++++++++++++---------- app/models/artist.py | 6 + app/models/track.py | 252 +++++++++++++++------------ app/utils/auth.py | 13 ++ db.py | 74 ++++++++ manage.py | 10 +- poetry.lock | 89 +++++++++- pyproject.toml | 1 + 27 files changed, 1399 insertions(+), 397 deletions(-) create mode 100644 app/lib/tagger.py create mode 100644 db.py diff --git a/.github/changelog.md b/.github/changelog.md index 0a151692..ec423361 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -12,3 +12,16 @@ - ## Development + + +## THE BIG ONE API CHANGES + +- genre is no longer a string, but a struct: + +```ts +interface Genre { + name: str; + genrehash: str; +} +``` + diff --git a/TODO.md b/TODO.md index 0812b840..7f5cc591 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,11 @@ # TODO - Migrations: 1. Move userdata to new hashing algorithm + - favorites ✅ + - playlists + - scrobble + - images + - remove image colors - Package jsoni and publish on PyPi - Rewrite stores to use dictionaries instead of list pools @@ -8,6 +13,8 @@ - Disable the watchdog by default, and mark it as experimental - rename userid to server id in config file - Look into seeding jwts using user password + server id +- Recreate album hash if featured artists are discover +- Implement checking if is clean install and skip migrations! # DONE - Support auth headers diff --git a/app/api/album.py b/app/api/album.py index 139e1621..b915612c 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -2,6 +2,7 @@ Contains all the album routes. """ +from itertools import groupby import random from flask_jwt_extended import current_user @@ -10,13 +11,15 @@ from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from app.api.apischemas import AlbumHashSchema, AlbumLimitSchema, ArtistHashSchema +from app.config import UserConfig +from app.db import AlbumTable as AlbumDb, TrackTable as TrackDb from app.settings import Defaults from app.models import FavType, Track from app.store.albums import AlbumStore from app.store.tracks import TrackStore from app.utils.hashing import create_hash from app.lib.albumslib import sort_by_track_no -from app.serializers.album import serialize_for_card +from app.serializers.album import serialize_for_card, serialize_for_card_many from app.serializers.track import serialize_track from app.db.sqlite.albumcolors import SQLiteAlbumMethods as adb from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb @@ -38,47 +41,20 @@ def get_album_tracks_and_info(body: AlbumHashSchema): Returns album info and tracks for the given albumhash. """ albumhash = body.albumhash - - error_msg = {"error": "Album not created yet."} - album = AlbumStore.get_album_by_hash(albumhash) + album = AlbumDb.get_album_by_albumhash(albumhash) if album is None: - return error_msg, 404 + return {"error": "Album not found"}, 404 - tracks = TrackStore.get_tracks_by_albumhash(albumhash) - - if tracks is None: - return error_msg, 404 - - if len(tracks) == 0: - return error_msg, 404 - - def get_album_genres(tracks: list[Track]): - genres = set() - - for track in tracks: - if track.genre is not None: - genres.update(track.genre) - - return list(genres) - - album.genres = get_album_genres(tracks) - album.count = len(tracks) - - album.get_date_from_tracks(tracks) + tracks = TrackDb.get_tracks_by_albumhash(albumhash) + album.trackcount = len(tracks) album.duration = sum(t.duration for t in tracks) + album.type = album.check_type( + tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles + ) + album.populate_versions() - album.check_is_single(tracks) - - if not album.is_single: - album.check_type() - - album.is_favorite = check_is_fav(albumhash, FavType.album) - - return { - "tracks": [serialize_track(t, remove_disc=False) for t in tracks], - "info": album, - } + return {"info": album, "tracks": tracks} @api.get("//tracks") @@ -89,16 +65,16 @@ def get_album_tracks(path: AlbumHashSchema): Returns all the tracks in the given album, sorted by disc and track number. NOTE: No album info is returned. """ - tracks = TrackStore.get_tracks_by_albumhash(path.albumhash) + tracks = TrackDb.get_tracks_by_albumhash(path.albumhash) tracks = sort_by_track_no(tracks) return tracks class GetMoreFromArtistsBody(AlbumLimitSchema): - albumartists: str = Field( + albumartists: list = Field( description="The artist hashes to get more albums from", - example=Defaults.API_ARTISTHASH, + example='[{"name": "Khalid", "artisthash": "94ca2dba1c"}]', ) base_title: str = Field( @@ -119,29 +95,25 @@ def get_more_from_artist(body: GetMoreFromArtistsBody): limit = body.limit base_title = body.base_title - albumartists: list[str] = albumartists.split(",") + all_albums = AlbumDb.get_albums_by_artisthashes(albumartists) - albums = [ - { - "artisthash": a, - "albums": AlbumStore.get_albums_by_albumartist( - a, limit, exclude=base_title - ), - } - for a in albumartists + # filter out albums with the same base title + all_albums = filter( + lambda a: create_hash(a.base_title) != create_hash(base_title), all_albums + ) + all_albums = list(all_albums) + + if not len(all_albums): + return [] + + # group by first albumartist's artisthash + groups = groupby(all_albums, lambda a: a.albumartists[0]["artisthash"]) + + return [ + {"artisthash": g[0], "albums": serialize_for_card_many(list(g[1])[:limit])} + for g in groups ] - albums = [ - { - "artisthash": a["artisthash"], - "albums": [serialize_for_card(a_) for a_ in (a["albums"])], - } - for a in albums - if len(a["albums"]) > 0 - ] - - return albums - class GetAlbumVersionsBody(ArtistHashSchema): og_album_title: str = Field( @@ -165,18 +137,29 @@ def get_album_versions(body: GetAlbumVersionsBody): base_title = body.base_title artisthash = body.artisthash - albums = AlbumStore.get_albums_by_artisthash(artisthash) - + albums = AlbumDb.get_albums_by_base_title(base_title) + print(albums) albums = [ a for a in albums - if create_hash(a.base_title) == create_hash(base_title) - and create_hash(og_album_title) != create_hash(a.og_title) + if a.og_title != og_album_title + and artisthash in {a["artisthash"] for a in a.albumartists} ] - for a in albums: - tracks = TrackStore.get_tracks_by_albumhash(a.albumhash) - a.get_date_from_tracks(tracks) + print(albums) + + # albums = AlbumStore.get_albums_by_artisthash(artisthash) + + # albums = [ + # a + # for a in albums + # if create_hash(a.base_title) == create_hash(base_title) + # and create_hash(og_album_title) != create_hash(a.og_title) + # ] + + # for a in albums: + # tracks = TrackStore.get_tracks_by_albumhash(a.albumhash) + # a.get_date_from_tracks(tracks) return albums diff --git a/app/api/artist.py b/app/api/artist.py index 94c8f6ef..2d69e542 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -133,7 +133,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): AlbumStore.remove_album_by_hash(a.albumhash) continue - a.check_is_single(album_tracks) + a.is_single(album_tracks) all_albums = sorted(all_albums, key=lambda a: str(a.date), reverse=True) diff --git a/app/api/folder.py b/app/api/folder.py index bec377d0..71e8ac5b 100644 --- a/app/api/folder.py +++ b/app/api/folder.py @@ -66,7 +66,9 @@ def get_folder_tree(body: FolderTree): else: req_dir = "/" + req_dir if not req_dir.startswith("/") else req_dir + print('stuff!') res = GetFilesAndDirs(req_dir, tracks_only=tracks_only)() + print(res['folders']) res["folders"] = sorted(res["folders"], key=lambda i: i.name) return res diff --git a/app/api/getall/__init__.py b/app/api/getall/__init__.py index b3199ca9..607fa08a 100644 --- a/app/api/getall/__init__.py +++ b/app/api/getall/__init__.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field from datetime import datetime from app.api.apischemas import GenericLimitSchema +from app.db import AlbumTable, ArtistTable from app.store.albums import AlbumStore from app.store.artists import ArtistStore @@ -59,17 +60,19 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): is_albums = path.itemtype == "albums" is_artists = path.itemtype == "artists" - items = AlbumStore.albums + if is_albums: + items = AlbumTable.get_all(query.start, query.limit) + elif is_artists: + items = ArtistTable.get_all(query.start, query.limit) - if is_artists: - items = ArtistStore.artists + print(items) start = query.start limit = query.limit sort = query.sortby reverse = query.reverse == "1" - sort_is_count = sort == "count" + sort_is_count = sort == "trackcount" sort_is_duration = sort == "duration" sort_is_create_date = sort == "created_date" @@ -81,7 +84,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): lambda_sort = lambda x: getattr(x, sort) if sort_is_artist: - lambda_sort = lambda x: getattr(x, sort)[0].name + lambda_sort = lambda x: getattr(x, sort)[0]["name"] sorted_items = sorted(items, key=lambda_sort, reverse=reverse) items = sorted_items[start : start + limit] @@ -101,7 +104,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): if sort_is_count: item_dict["help_text"] = ( - f"{format_number(item.count)} track{'' if item.count == 1 else 's'}" + f"{format_number(item.trackcount)} track{'' if item.trackcount == 1 else 's'}" ) if sort_is_duration: @@ -114,7 +117,7 @@ def get_all_items(path: GetAllItemsPath, query: GetAllItemsQuery): if sort_is_artist_albumcount: item_dict["help_text"] = ( - f"{format_number(item.albumcount)} album{'' if item.albumcount == 1 else 's'}" + f"{format_number(item['albumcount'])} album{'' if item['albumcount'] == 1 else 's'}" ) album_list.append(item_dict) diff --git a/app/config.py b/app/config.py index 00e6cde5..2d477ea0 100644 --- a/app/config.py +++ b/app/config.py @@ -21,6 +21,7 @@ class UserConfig: rootDirs: list[str] = field(default_factory=list) excludeDirs: list[str] = field(default_factory=list) artistSeparators: set[str] = field(default_factory=list) + genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) # tracks extractFeaturedArtists: bool = True diff --git a/app/db/__init__.py b/app/db/__init__.py index e69de29b..dbea0ad6 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -0,0 +1,232 @@ +import json +from pprint import pprint +from typing import Any, Optional + +from sqlalchemy import ( + JSON, + Boolean, + Integer, + Row, + String, + Tuple, + create_engine, + insert, + select, +) +from sqlalchemy.orm import ( + Mapped, + mapped_column, + DeclarativeBase, + MappedAsDataclass, + sessionmaker, +) + +from app.models import Track as TrackModel +from app.models import Album as AlbumModel +from app.utils.remove_duplicates import remove_duplicates + + +fullpath = "/home/cwilvx/temp/swingmusic/swing.db" +engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=False) + + +def todict(track: Any): + return track._asdict() + + +def todicts(tracks: list[Any]): + return [todict(track) for track in tracks] + + +class DbManager: + def __init__(self): + self.engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) + self.conn = self.engine.connect() + + def __enter__(self): + return self.conn.execution_options(preserve_rowcount=True) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.conn.commit() + self.conn.close() + + +class Base(MappedAsDataclass, DeclarativeBase): + @classmethod + def insert_many(cls, items: list[dict[str, Any]]): + """ + Inserts multiple items into the database. + """ + with DbManager() as conn: + conn.execute(insert(cls).values(items)) + + @classmethod + def insert_one(cls, item: dict[str, Any]): + """ + Inserts a single item into the database. + """ + return cls.insert_many([item]) + + @classmethod + def get_all(cls): + """ + Returns all the items from the database. + """ + with DbManager() as conn: + result = conn.execute(select(cls)) + return result.fetchall() + + +class ArtistTable(Base): + __tablename__ = "artist" + + id: Mapped[int] = mapped_column(primary_key=True) + albumcount: Mapped[int] = mapped_column(Integer()) + artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genres: Mapped[str] = mapped_column(JSON()) + name: Mapped[str] = mapped_column(String(), index=True) + trackcount: Mapped[int] = mapped_column(Integer()) + is_favorite: Mapped[Optional[bool]] = mapped_column(Boolean()) + + @classmethod + def get_all(cls, start: int, limit: int): + with DbManager() as conn: + result = conn.execute(select(cls).offset(start).limit(limit)) + return albums_to_dataclasses(result.fetchall()) + + +class AlbumTable(Base): + __tablename__ = "album" + + id: Mapped[int] = mapped_column(primary_key=True) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) + base_title: Mapped[str] = mapped_column(String()) + color: Mapped[Optional[str]] = mapped_column(String()) + created_date: Mapped[int] = mapped_column(Integer()) + date: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + genres: Mapped[str] = mapped_column(JSON()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + trackcount: Mapped[int] = mapped_column(Integer()) + + @classmethod + def get_album_by_albumhash(cls, hash: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.albumhash == hash) + ) + album = result.fetchone() + + if album: + return album_to_dataclass(album) + + @classmethod + def get_all(cls, start: int, limit: int): + with DbManager() as conn: + result = conn.execute(select(AlbumTable).offset(start).limit(limit)) + return albums_to_dataclasses(result.fetchall()) + + @classmethod + def get_albums_by_artisthashes(cls, artisthashes: list[dict[str, str]]): + with DbManager() as conn: + albums: list[AlbumModel] = [] + + for artist in artisthashes: + result = conn.execute( + # NOTE: The artist dict keys need to in the same order they appear in the db for this to work! + select(AlbumTable).where(AlbumTable.albumartists.contains(artist)) + ) + albums.extend(albums_to_dataclasses(result.fetchall())) + + print(albums) + return albums + + @classmethod + def get_albums_by_base_title(cls, base_title: str): + with DbManager() as conn: + result = conn.execute( + select(AlbumTable).where(AlbumTable.base_title == base_title) + ) + return albums_to_dataclasses(result.fetchall()) + + +class TrackTable(Base): + __tablename__ = "track" + + id: Mapped[int] = mapped_column(init=False, primary_key=True) + album: Mapped[str] = mapped_column(String()) + albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON()) + albumhash: Mapped[str] = mapped_column(String(), index=True) + artists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) + bitrate: Mapped[int] = mapped_column(Integer()) + copyright: Mapped[Optional[str]] = mapped_column(String()) + date: Mapped[int] = mapped_column(Integer()) + disc: Mapped[int] = mapped_column(Integer()) + duration: Mapped[int] = mapped_column(Integer()) + filepath: Mapped[str] = mapped_column(String(), unique=True) + folder: Mapped[str] = mapped_column(String(), index=True) + genre: Mapped[Optional[list[dict[str, str]]]] = mapped_column(JSON()) + last_mod: Mapped[float] = mapped_column(Integer()) + og_album: Mapped[str] = mapped_column(String()) + og_title: Mapped[str] = mapped_column(String()) + title: Mapped[str] = mapped_column(String()) + track: Mapped[int] = mapped_column(Integer()) + trackhash: Mapped[str] = mapped_column(String(), index=True) + + @classmethod + def get_tracks_by_filepaths(cls, filepaths: list[str]): + print(filepaths[0]) + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.filepath.in_(filepaths)) + ) + return [dict(r) for r in result.mappings().fetchall()] + + @classmethod + def count_tracks_containing_paths(cls, paths: list[str]): + results: list[dict[str, int | str]] = [] + + with DbManager() as conn: + for path in paths: + result = conn.execute( + select(TrackTable).where(TrackTable.filepath.contains(path)) + ) + results.append({"path": path, "trackcount": result.all().__len__()}) + + return results + + @classmethod + def get_tracks_by_albumhash(cls, albumhash: str): + with DbManager() as conn: + result = conn.execute( + select(TrackTable).where(TrackTable.albumhash == albumhash) + ) + tracks = tracks_to_dataclasses(result.fetchall()) + return remove_duplicates(tracks, is_album_tracks=True) + + +# SECTION: HELPER FUNCTIONS + + +def album_to_dataclass(album: Row[AlbumTable]): + return AlbumModel(**album._asdict()) + + +def albums_to_dataclasses(albums: list[Row[AlbumTable]]): + return [album_to_dataclass(album) for album in albums] + + +def track_to_dataclass(track: Row[TrackTable]): + return TrackModel(**track._asdict()) + + +def tracks_to_dataclasses(tracks: list[Row[TrackTable]]): + return [track_to_dataclass(track) for track in tracks] + + +Base().metadata.create_all(engine) diff --git a/app/db/sqlite/auth.py b/app/db/sqlite/auth.py index e79dfe78..dede16b0 100644 --- a/app/db/sqlite/auth.py +++ b/app/db/sqlite/auth.py @@ -75,7 +75,6 @@ class SQLiteAuthMethods: {', '.join([f"{key} = :{key}" for key in keys if key != 'id'])} WHERE id = :id """ - print(sql, user) with SQLiteManager(userdata_db=True) as cur: cur.execute(sql, user) @@ -140,7 +139,6 @@ class SQLiteAuthMethods: Delete a user by username. """ sql = "DELETE FROM users WHERE id = ?" - print("deleting user: ", username) with SQLiteManager(userdata_db=True) as cur: cur.execute(sql, (3,)) cur.close() diff --git a/app/db/sqlite/logger/tracks.py b/app/db/sqlite/logger/tracks.py index 7f43e8b2..c95c123b 100644 --- a/app/db/sqlite/logger/tracks.py +++ b/app/db/sqlite/logger/tracks.py @@ -1,6 +1,7 @@ from flask_jwt_extended import current_user from app.db.sqlite.utils import SQLiteManager from app.models.logger import TrackLog as TrackLog +from app.utils.auth import get_current_userid class SQLiteTrackLogger: @@ -10,6 +11,7 @@ class SQLiteTrackLogger: Inserts a track play record into the database """ + userid = get_current_userid() with SQLiteManager(userdata_db=True) as cur: sql = """INSERT OR REPLACE INTO track_logger( trackhash, @@ -21,7 +23,7 @@ class SQLiteTrackLogger: """ cur.execute( - sql, (trackhash, duration, timestamp, source, current_user["id"]) + sql, (trackhash, duration, timestamp, source, userid) ) lastrowid = cur.lastrowid @@ -34,7 +36,8 @@ class SQLiteTrackLogger: """ with SQLiteManager(userdata_db=True) as cur: - sql = f"""SELECT * FROM track_logger WHERE userid = {current_user['id']} ORDER BY timestamp DESC""" + userid = get_current_userid() + sql = f"""SELECT * FROM track_logger WHERE userid = {userid} ORDER BY timestamp DESC""" cur.execute(sql) rows = cur.fetchall() diff --git a/app/db/sqlite/playlists.py b/app/db/sqlite/playlists.py index bfed7185..b4cca33e 100644 --- a/app/db/sqlite/playlists.py +++ b/app/db/sqlite/playlists.py @@ -60,7 +60,15 @@ class SQLitePlaylistMethods: @staticmethod def get_all_playlists(): with SQLiteManager(userdata_db=True) as cur: - cur.execute(f"SELECT * FROM playlists WHERE userid = {current_user['id']}") + userid = 1 + + try: + userid = current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + pass + + cur.execute(f"SELECT * FROM playlists WHERE userid = {userid}") playlists = cur.fetchall() cur.close() @@ -92,7 +100,15 @@ class SQLitePlaylistMethods: Adds a string item to a json dumped list using a playlist id and field name. Takes the playlist ID, a field name, an item to add to the field. """ - sql = f"SELECT {field} FROM playlists WHERE id = ? and userid = {current_user['id']}" + userid = 1 + + try: + userid = current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + pass + + sql = f"SELECT {field} FROM playlists WHERE id = ? and userid = {userid}" with SQLiteManager(userdata_db=True) as cur: cur.execute(sql, (playlist_id,)) @@ -173,10 +189,17 @@ class SQLitePlaylistMethods: """ sql = """UPDATE playlists SET trackhashes = ? WHERE id = ?""" + userid = 1 + + try: + userid = current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + pass with SQLiteManager(userdata_db=True) as cur: cur.execute( - f"SELECT trackhashes FROM playlists WHERE id = ? and userid = {current_user['id']}", + f"SELECT trackhashes FROM playlists WHERE id = ? and userid = {userid}", (playlistid,), ) data = cur.fetchone() @@ -185,17 +208,20 @@ class SQLitePlaylistMethods: return trackhashes: list[str] = json.loads(data[0]) + to_remove = [] for track in tracks: # { # trackhash: str; # index: int; # } - index = trackhashes.index(track["trackhash"]) if index == track["index"]: - trackhashes.remove(track["trackhash"]) + to_remove.append(track["trackhash"]) + + for trackhash in to_remove: + trackhashes.remove(trackhash) cur.execute(sql, (json.dumps(trackhashes), playlistid)) diff --git a/app/db/sqlite/tracks.py b/app/db/sqlite/tracks.py index c8bed28c..0324641b 100644 --- a/app/db/sqlite/tracks.py +++ b/app/db/sqlite/tracks.py @@ -98,6 +98,20 @@ class SQLiteTrackMethods: return tuple_to_track(row) return None + + @staticmethod + def get_track_by_albumhash(albumhash: str): + """ + Gets a track using its albumhash. Returns a Track object or None. + """ + with SQLiteManager() as cur: + cur.execute("SELECT * FROM tracks WHERE albumhash=?", (albumhash,)) + row = cur.fetchone() + + if row is not None: + return tuple_to_track(row) + + return None @staticmethod def remove_tracks_by_filepaths(filepaths: str | set[str]): diff --git a/app/lib/folderslib.py b/app/lib/folderslib.py index 112a72d7..ccf1c499 100644 --- a/app/lib/folderslib.py +++ b/app/lib/folderslib.py @@ -8,6 +8,7 @@ from app.settings import SUPPORTED_FILES from app.utils.wintools import win_replace_slash from app.store.tracks import TrackStore +from app.db import TrackTable as TrackDB def create_folder(path: str, trackcount=0, foldercount=0) -> Folder: @@ -37,44 +38,52 @@ def get_first_child_from_path(root: str, maybe_child: str): return os.path.join(root, first) + def get_folders(paths: list[str]): """ Filters out folders that don't have any tracks and returns a list of folder objects. """ - count_dict = { - "tracks": {path: 0 for path in paths}, - # folders are immediate children of the root folder - "folders": {path: set() for path in paths}, - } - - for track in TrackStore.tracks: - for path in paths: - - # a child path should be longer than the root path - if len(track.folder) >= len(path) and track.folder.startswith(path): - count_dict["tracks"][path] += 1 - - # counting subfolders - p = get_first_child_from_path(path, track.folder) - - if p: - count_dict["folders"][path].add(p) - - folders = [ - { - "path": path, - "trackcount": count_dict["tracks"][path], - "foldercount": len(count_dict["folders"][path]), - } - for path in paths - ] + folders = TrackDB.count_tracks_containing_paths(paths) return [ - create_folder(f["path"], f["trackcount"], f["foldercount"]) + create_folder(f["path"], f["trackcount"], foldercount=0) for f in folders if f["trackcount"] > 0 ] + # count_dict = { + # "tracks": {path: 0 for path in paths}, + # # folders are immediate children of the root folder + # "folders": {path: set() for path in paths}, + # } + + # for track in TrackStore.tracks: + # for path in paths: + + # # a child path should be longer than the root path + # if len(track.folder) >= len(path) and track.folder.startswith(path): + # count_dict["tracks"][path] += 1 + + # # counting subfolders + # p = get_first_child_from_path(path, track.folder) + + # if p: + # count_dict["folders"][path].add(p) + + # folders = [ + # { + # "path": path, + # "trackcount": count_dict["tracks"][path], + # "foldercount": len(count_dict["folders"][path]), + # } + # for path in paths + # ] + + # return [ + # create_folder(f["path"], f["trackcount"], f["foldercount"]) + # for f in folders + # if f["trackcount"] > 0 + # ] class GetFilesAndDirs: @@ -131,7 +140,13 @@ class GetFilesAndDirs: files_.sort(key=lambda f: f["time"]) files = [f["path"] for f in files_] - tracks = TrackStore.get_tracks_by_filepaths(files) + tracks = [] + if files: + tracks = TrackDB.get_tracks_by_filepaths(files) + print("printing files") + print(tracks) + + # tracks = TrackStore.get_tracks_by_filepaths(files) folders = [] if not self.tracks_only: @@ -145,7 +160,7 @@ class GetFilesAndDirs: return { "path": path, - "tracks": serialize_tracks(tracks), + "tracks": tracks, "folders": folders, } diff --git a/app/lib/populate.py b/app/lib/populate.py index ceab17d8..4ed73060 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -7,6 +7,7 @@ from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout from app import settings +from app.db import TrackTable from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb from app.db.sqlite.settings import SettingsSQLMethods as sdb @@ -121,14 +122,14 @@ class Populate: return @staticmethod - def remove_modified(tracks: Generator[Track, None, None]): + def remove_modified(tracks: Generator[TrackTable, None, None]): """ Removes tracks from the database that have been modified since they were added to the database. """ unmodified_paths = set() - modified_tracks: list[Track] = [] + modified_tracks: list[TrackTable] = [] modified_paths = set() for track in tracks: @@ -151,18 +152,6 @@ class Populate: @staticmethod def tag_untagged(untagged: set[str], key: str): - log.info("Found %s new tracks", len(untagged)) - tagged_tracks: deque[dict] = deque() - tagged_count = 0 - - favs = favdb.get_fav_tracks() - records = dict() - - for fav in favs: - r = records.setdefault(fav[1], set()) - r.add(fav[4]) - - for file in tqdm(untagged, desc="Reading files"): if POPULATE_KEY != key: log.warning("'Populate.tag_untagged': Populate key changed") @@ -171,36 +160,49 @@ class Populate: tags = get_tags(file) if tags is not None: - tagged_tracks.append(tags) - track = Track(**tags) + TrackTable.insert_one(tags) - track.fav_userids = list(records.get(track.trackhash, set())) + # log.info("Found %s new tracks", len(untagged)) + # # tagged_tracks: deque[dict] = deque() + # # tagged_count = 0 - TrackStore.add_track(track) + # favs = favdb.get_fav_tracks() + # records = dict() - if not AlbumStore.album_exists(track.albumhash): - AlbumStore.add_album(AlbumStore.create_album(track)) + # for fav in favs: + # r = records.setdefault(fav[1], set()) + # r.add(fav[4]) - for artist in track.artists: - if not ArtistStore.artist_exists(artist.artisthash): - ArtistStore.add_artist(Artist(artist.name)) + # tagged_tracks.append(tags) + # track = Track(**tags) - for artist in track.albumartists: - if not ArtistStore.artist_exists(artist.artisthash): - ArtistStore.add_artist(Artist(artist.name)) + # track.fav_userids = list(records.get(track.trackhash, set())) - tagged_count += 1 - else: - log.warning("Could not read file: %s", file) + # TrackStore.add_track(track) - if len(tagged_tracks) > 0: - log.info("Adding %s tracks to database", len(tagged_tracks)) - insert_many_tracks(tagged_tracks) + # if not AlbumStore.album_exists(track.albumhash): + # AlbumStore.add_album(AlbumStore.create_album(track)) - log.info("Added %s/%s tracks", tagged_count, len(untagged)) + # for artist in track.artists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) + + # for artist in track.albumartists: + # if not ArtistStore.artist_exists(artist.artisthash): + # ArtistStore.add_artist(Artist(artist.name)) + + # tagged_count += 1 + # else: + # log.warning("Could not read file: %s", file) + + # if len(tagged_tracks) > 0: + # log.info("Adding %s tracks to database", len(tagged_tracks)) + # insert_many_tracks(tagged_tracks) + + # log.info("Added %s/%s tracks", tagged_count, len(untagged)) @staticmethod - def extract_thumb_with_overwrite(tracks: list[Track]): + def extract_thumb_with_overwrite(tracks: list[TrackTable]): """ Extracts the thumbnail from a list of filepaths, overwriting the existing thumbnail if it exists, diff --git a/app/lib/searchlib.py b/app/lib/searchlib.py index 8bca6420..a23d67fa 100644 --- a/app/lib/searchlib.py +++ b/app/lib/searchlib.py @@ -195,7 +195,7 @@ class TopResults: except AttributeError: item.duration = 0 - item.check_is_single(tracks) + item.is_single(tracks) if not item.is_single: item.check_type() diff --git a/app/lib/tagger.py b/app/lib/tagger.py new file mode 100644 index 00000000..38101f80 --- /dev/null +++ b/app/lib/tagger.py @@ -0,0 +1,154 @@ +from pprint import pprint +from app.db import AlbumTable, ArtistTable, TrackTable +from app.lib.taglib import get_tags +from app.utils.filesystem import run_fast_scandir +from app.utils.parsers import get_base_album_title +from app.utils.progressbar import tqdm + + +class IndexTracks: + def __init__(self) -> None: + dirs_to_scan = ["/home/cwilvx/Music"] + + files = set() + + for _dir in dirs_to_scan: + files = files.union(run_fast_scandir(_dir, full=True)[1]) + + self.tag_untagged(files) + # unmodified, modified_tracks = self.remove_modified(tracks) + # untagged = files - unmodified + + def tag_untagged(self, files: set[str]): + for file in tqdm(files, desc="Reading files"): + # if POPULATE_KEY != key: + # log.warning("'Populate.tag_untagged': Populate key changed") + # return + + tags = get_tags(file) + + if tags is not None: + TrackTable.insert_one(tags) + + +class IndexAlbums: + def __init__(self) -> None: + albums = dict() + + all_tracks: list[TrackTable] = TrackTable.get_all() + + for track in all_tracks: + if track.albumhash not in albums: + albums[track.albumhash] = { + "albumartists": track.albumartists, + "albumhash": track.albumhash, + "base_title": None, + "color": None, + "created_date": None, + "date": None, + "duration": track.duration, + "genres": [*track.genre] if track.genre else [], + "og_title": track.og_album, + "title": track.album, + "trackcount": 1, + "dates": [track.date], + "created_dates": [track.last_mod], + } + else: + album = albums[track.albumhash] + album["trackcount"] += 1 + album["duration"] += track.duration + album["dates"].append(track.date) + album["created_dates"].append(track.last_mod) + + if track.genre: + album["genres"].append(track.genre) + + for album in albums.values(): + album["date"] = min(album["dates"]) + album["created_date"] = min(album["created_dates"]) + + genres = [] + + for genre in album["genres"]: + if genre not in genres: + genres.append(genre) + + album["genres"] = genres + album["base_title"], _ = get_base_album_title(album["og_title"]) + + del album["dates"] + del album["created_dates"] + + pprint(albums) + + AlbumTable.insert_many(list(albums.values())) + + +class IndexArtists: + def __init__(self) -> None: + all_tracks: list[TrackTable] = TrackTable.get_all() + artists = dict() + + for track in all_tracks: + this_artists = track.artists + + for a in track.albumartists: + if a not in this_artists: + this_artists.append(a) + + for artist in this_artists: + if artist["artisthash"] not in artists: + artists[artist["artisthash"]] = { + "albumcount": None, + "albums": {track.albumhash}, + "artisthash": artist["artisthash"], + "created_dates": [track.last_mod], + "dates": [track.date], + "date": None, + "duration": track.duration, + "genres": [*track.genre] if track.genre else [], + "name": artist["name"], + "trackcount": None, + "tracks": {track.trackhash}, + } + else: + artist = artists[artist["artisthash"]] + artist["duration"] += track.duration + artist["albums"].add(track.albumhash) + artist["tracks"].add(track.trackhash) + artist["dates"].append(track.date) + artist["created_dates"].append(track.last_mod) + + if track.genre: + artist["genres"].append(track.genre) + + + for artist in artists.values(): + artist["albumcount"] = len(artist["albums"]) + artist["trackcount"] = len(artist["tracks"]) + artist["date"] = min(artist["dates"]) + artist["created_date"] = min(artist["created_dates"]) + + genres = [] + + for genre in artist["genres"]: + if genre not in genres: + genres.append(genre) + + artist["genres"] = genres + + del artist["tracks"] + del artist["albums"] + del artist["dates"] + del artist["created_dates"] + + pprint(artists) + ArtistTable.insert_many(list(artists.values())) + +class IndexEverything: + def __init__(self) -> None: + # IndexTracks() + # IndexAlbums() + # IndexArtists() + pass diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 3d7e9f6d..68cfd519 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -8,9 +8,16 @@ import pendulum from PIL import Image, UnidentifiedImageError from tinytag import TinyTag +from app.config import UserConfig from app.settings import Defaults, Paths from app.utils.hashing import create_hash -from app.utils.parsers import split_artists +from app.utils.parsers import ( + clean_title, + get_base_title_and_versions, + parse_feat_from_title, + remove_prod, + split_artists, +) from app.utils.wintools import win_replace_slash @@ -206,9 +213,7 @@ def get_tags(filepath: str): except KeyError: tags.copyright = None - tags.albumhash = create_hash(tags.album, tags.albumartist) - tags.trackhash = create_hash(tags.artist, tags.album, tags.title) - tags.image = f"{tags.albumhash}.webp" + # tags.image = f"{tags.albumhash}.webp" tags.folder = win_replace_slash(os.path.dirname(filepath)) tags.date = parse_date(tags.year) or int(last_mod) @@ -218,9 +223,100 @@ def get_tags(filepath: str): tags.artists = tags.artist tags.albumartists = tags.albumartist + split_artist = split_artists(tags.artist) + split_albumartists = split_artists(tags.albumartist) + new_title = tags.title + + # TODO: Figure out which is the best spot to create these hashes + # create albumhash using og_album + tags.albumhash = create_hash(tags.album or "", tags.albumartist) + + config = UserConfig() + + # extract featured artists + if config.extractFeaturedArtists: + feat, new_title = parse_feat_from_title(tags.title) + original_lower = "-".join([create_hash(a) for a in split_artist]) + split_artist.extend(a for a in feat if create_hash(a) not in original_lower) + + # if no albumartist, assign to the first artist + if not tags.albumartist: + tags.albumartist = split_artist[:1] + + # create json objects for artists and albumartists + tags.artists = [ + { + "artisthash": create_hash(a, decode=True), + "name": a, + } + for a in split_artist + ] + + tags.albumartists = [ + { + "artisthash": create_hash(a, decode=True), + "name": a, + } + for a in split_albumartists + ] + + # remove prod by + if config.removeProdBy: + new_title = remove_prod(new_title) + + # if track is a single, ie. + # if og_title == album, rename album to new_title + if tags.title == tags.album: + tags.album = new_title + + # remove remaster from track title + if config.removeRemasterInfo: + new_title = clean_title(new_title) + + # save final title + tags.og_title = tags.title + tags.title = new_title + tags.og_album = tags.album + + # clean album title + if config.cleanAlbumTitle: + tags.album, _ = get_base_title_and_versions(tags.album, get_versions=False) + + # merge album versions + if config.mergeAlbums: + tags.albumhash = create_hash( + tags.album, *(a["name"] for a in tags.albumartists) + ) + + # process genres + if tags.genre: + tags.genre = tags.genre.lower() + # separators = {"/", ";", "&"} + separators = set(config.genreSeparators) + + contains_rnb = "r&b" in tags.genre + contains_rock = "rock & roll" in tags.genre + + if contains_rnb: + tags.genre = tags.genre.replace("r&b", "RnB") + + if contains_rock: + tags.genre = tags.genre.replace("rock & roll", "rock") + + for s in separators: + tags.genre = tags.genre.replace(s, ",") + + tags.genre = tags.genre.split(",") + tags.genre = [ + {"name": g.strip(), "genrehash": create_hash(g.strip())} for g in tags.genre + ] + # sub underscore with space tags.title = tags.title.replace("_", " ") tags.album = tags.album.replace("_", " ") + tags.trackhash = create_hash( + *[a["name"] for a in tags.artists], tags.album, tags.title + ) tags = tags.__dict__ diff --git a/app/migrations/__init__.py b/app/migrations/__init__.py index 9e473e92..570cc115 100644 --- a/app/migrations/__init__.py +++ b/app/migrations/__init__.py @@ -5,6 +5,7 @@ Reads and applies the latest database migrations. """ import inspect +import sys from types import ModuleType from app.db.sqlite.migrations import MigrationManager from app.logger import log @@ -55,11 +56,12 @@ def apply_migrations(): to_apply = all_migrations[index:] for migration in to_apply: - try: + # try: migration.migrate() log.info("Applied migration: %s", migration.__name__) - except Exception as e: - log.error("Failed to run migration: %s", migration.__name__) - log.error(e) + # except Exception as e: + # log.error("Failed to run migration: %s", migration.__name__) + # log.error(e) + # sys.exit(0) MigrationManager.set_index(len(all_migrations)) diff --git a/app/migrations/v1_4_9/__init__.py b/app/migrations/v1_4_9/__init__.py index b5729c4f..be10c97b 100644 --- a/app/migrations/v1_4_9/__init__.py +++ b/app/migrations/v1_4_9/__init__.py @@ -1,9 +1,90 @@ import os import shutil +import sqlite3 +from time import time from app.db.sqlite.utils import SQLiteManager from app.migrations.base import Migration from app.settings import Paths +import hashlib +from unidecode import unidecode + +from app.db.sqlite.tracks import SQLiteTrackMethods as tdb +from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb +from app.db.sqlite.logger.tracks import SQLiteTrackLogger as ldb +from app.utils.hashing import create_hash + + +def create_sha256_hash(*args: str, decode=False, limit=10) -> str: + """ + This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments. + + Example use case: + - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID. + + :param args: The arguments to hash. + :param decode: Whether to decode the arguments before hashing. + :param limit: The number of characters to return. + + :return: The hash. + """ + + def remove_non_alnum(token: str) -> str: + token = token.lower().strip().replace(" ", "") + t = "".join(t for t in token if t.isalnum()) + + if t == "": + return token + + return t + + str_ = "".join(remove_non_alnum(t) for t in args) + + if decode: + str_ = unidecode(str_) + + str_ = str_.encode("utf-8") + str_ = hashlib.sha256(str_).hexdigest() + return str_[-limit:] + + +def create_sha1_hash(*args: str, decode=False, limit=10) -> str: + """ + This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments. + + Example use case: + - Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID. + + :param args: The arguments to hash. + :param decode: Whether to decode the arguments before hashing. + :param limit: The number of characters to return. + + :return: The hash. + """ + + def remove_non_alnum(token: str) -> str: + token = token.lower().strip().replace(" ", "") + t = "".join(t for t in token if t.isalnum()) + + if t == "": + return token + + return t + + str_ = "".join(remove_non_alnum(t) for t in args) + + if decode: + str_ = unidecode(str_) + + str_ = str_.encode("utf-8") + str_ = hashlib.sha1(str_).hexdigest() + + return ( + str_[: limit // 2] + str_[-limit // 2 :] + if limit % 2 == 0 + else str_[: limit // 2] + str_[-limit // 2 - 1 :] + ) + class _1AddTimestampToFavoritesTable(Migration): """ @@ -13,37 +94,23 @@ class _1AddTimestampToFavoritesTable(Migration): @staticmethod def migrate(): # INFO: add timestamp column with automatic current timestamp - sql = f"ALTER TABLE favorites ADD COLUMN IF NOT EXISTS timestamp INTEGER NOT NULL DEFAULT 0" + sql = f"ALTER TABLE favorites ADD COLUMN timestamp INTEGER NOT NULL DEFAULT 0" # INFO: execute the sql with SQLiteManager(userdata_db=True) as cur: - try: - # INFO: Add the timestamp column to the favorites table - cur.execute(sql) + table_exists = cur.execute( + "select count(*) from pragma_table_info('favorites') where name = 'timestamp'" + ) - # INFO: Set all the timestamps to the current time - cur.execute("UPDATE favorites SET timestamp = strftime('%s', 'now')") - except Exception as e: - # INFO: timestamp column already exists - pass - finally: - cur.close() + table_exists = table_exists.fetchone() + if table_exists[0] == 1: + return -class _4MoveHashesToSha1(Migration): - """ - Moves the 10 bit item hashes from sha256 to sha1 which is - faster and more lenient on less powerful devices. - - Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution. - """ - - enabled: bool = False - - pass - - # INFO: Apparentlly, every single table is affected by this migration. - # NOTE: Use generators to avoid memory issues. + # INFO: Add the timestamp column to the favorites table + timestamp = int(time()) + cur.execute(sql) + cur.execute(f"UPDATE favorites SET timestamp = {timestamp}") class _2DeleteOriginalThumbnails(Migration): @@ -175,3 +242,164 @@ class _5AddUserIdToPlaylistsTable(Migration): # INFO: Execute the sql cur.executescript(sql) + + +class _6MoveHashesToSha1(Migration): + """ + Moves the 10 bit item hashes from sha256 to sha1 which is + faster and more lenient on less powerful devices. + + Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution. + """ + + # enabled: bool = False + + # pass + + # INFO: Apparentlly, every single table is affected by this migration. + # NOTE: Use generators to avoid memory issues. + + @classmethod + def port_track(cls, trackhash: str): + # get the track with the track hash + track = tdb.get_track_by_trackhash(trackhash) + + if track is None: + return + + title = track.og_title + if track.trackhash != trackhash: + # raise ValueError("Track hash mismatch") + print("Track hash mismatch") + title = track.title + else: + print("Porting track: ", track.title) + + # return the new hash + finalhash = create_sha1_hash( + ", ".join(a.name for a in track.artists), + track.og_album, + title, + ) + + if finalhash != create_hash( + ", ".join(a.name for a in track.artists), track.og_album, title + ): + raise ValueError("Hash mismatch") + + @classmethod + def port_album(cls, albumhash: str): + # get the first track with the album hash + track = tdb.get_track_by_albumhash(albumhash) + + if track is None: + return + + # return the new hash + return create_sha1_hash( + track.og_album, + ", ".join(a.name for a in track.albumartists), + ) + + @classmethod + def port_artist(cls, artisthash: str): + # find all tracks with the artist hash + tracks = [t for t in cls.tracks if artisthash in t.artist_hashes] + + if len(tracks) == 0: + return + + # find the artist name + artist = [ + a.name + for a in tracks[0].artists + if create_sha256_hash(a.name, decode=True) == artisthash + ][0] + + # return the new hash + return create_sha1_hash(artist, decode=True) + + @classmethod + def migrate_favorites(cls): + with SQLiteManager(userdata_db=True) as cur: + # read all favorites + data = cur.execute("SELECT * FROM favorites") + data = data.fetchall() + + for track in cls.tracks: + track.artist_hashes = "-".join( + [create_sha256_hash(a.name, decode=True) for a in track.artists] + ) + + for entry in data: + # hash is the 2nd column in the table + hash = entry[1] + + # entry type is the 3rd column in the table + if entry[2] == "track": + newhash = cls.port_track(hash) + + if newhash: + cur.execute( + f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'track'" + ) + + elif entry[2] == "album": + newhash = cls.port_album(hash) + + if newhash: + cur.execute( + f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'album'" + ) + + elif entry[2] == "artist": + newhash = cls.port_artist(hash) + + if newhash: + cur.execute( + f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'artist'" + ) + + @classmethod + def migrate_playlists(cls): + playlists = pdb.get_all_playlists() + + for playlist in playlists: + # remove previous hashes + to_remove = [ + {"trackhash": trackhash, "index": index} + for index, trackhash in enumerate(playlist.trackhashes) + ] + pdb.remove_tracks_from_playlist(playlist.id, to_remove) + + # add new hashes + newhashes = [ + cls.port_track(trackhash) for trackhash in playlist.trackhashes + ] + newhashes = [h for h in newhashes if h is not None] + pdb.add_tracks_to_playlist(playlist.id, newhashes) + + print("Ported playlist: ", playlist.name) + print("Total tracks: ", len(newhashes)) + + @classmethod + def migrate_scrobble(cls): + # read all logs + logs = ldb.get_all() + + with SQLiteManager(userdata_db=True) as cur: + # for each log, port the hash + for log in logs: + newhash = cls.port_track(log[1]) + + if newhash: + cur.execute( + f"UPDATE track_logger SET trackhash = '{newhash}' WHERE trackhash = '{log[1]}'" + ) + + @classmethod + def migrate(cls): + cls.tracks = list(tdb.get_all_tracks()) + cls.migrate_favorites() + # cls.migrate_playlists() + # cls.migrate_scrobble() diff --git a/app/models/album.py b/app/models/album.py index 36d1629e..63949b17 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -2,6 +2,7 @@ import dataclasses import datetime from dataclasses import dataclass +from app.config import UserConfig from app.settings import SessionVarKeys, get_flag from ..utils.hashing import create_hash @@ -16,94 +17,111 @@ class Album: Creates an album object """ + id: int + albumartists: list[dict[str, str]] albumhash: str + base_title: str + color: str + created_date: int + date: int + duration: int + genres: list[dict[str, str]] + og_title: str title: str - albumartists: list[Artist] + trackcount: int - albumartists_hashes: str = "" - image: str = "" - count: int = 0 - duration: int = 0 - colors: list[str] = dataclasses.field(default_factory=list) - date: str = "" - - created_date: int = 0 - og_title: str = "" - base_title: str = "" - is_soundtrack: bool = False - is_compilation: bool = False - is_single: bool = False - is_EP: bool = False - is_favorite: bool = False - is_live: bool = False - - genres: list[str] = dataclasses.field(default_factory=list) + type: str = "album" versions: list[str] = dataclasses.field(default_factory=list) def __post_init__(self): - self.title = self.title.strip() - self.og_title = self.title - self.image = self.albumhash + ".webp" + self.date = datetime.datetime.fromtimestamp(self.date).year - # Fetch album artists from title - if get_flag(SessionVarKeys.EXTRACT_FEAT): - featured, self.title = parse_feat_from_title(self.title) + # albumhash: str + # title: str + # albumartists: list[Artist] - if len(featured) > 0: - original_lower = "-".join([a.name.lower() for a in self.albumartists]) - self.albumartists.extend( - [Artist(a) for a in featured if a.lower() not in original_lower] - ) + # albumartists_hashes: str = "" + # image: str = "" + # count: int = 0 + # duration: int = 0 + # colors: list[str] = dataclasses.field(default_factory=list) + # date: str = "" - from ..store.tracks import TrackStore + # created_date: int = 0 + # og_title: str = "" + # base_title: str = "" + # is_soundtrack: bool = False + # is_compilation: bool = False + # is_single: bool = False + # is_EP: bool = False + # is_favorite: bool = False + # is_live: bool = False - TrackStore.append_track_artists(self.albumhash, featured, self.title) + # genres: list[str] = dataclasses.field(default_factory=list) - # Handle album version data - if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): - get_versions = not get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS) + # def __post_init__(self): + # self.title = self.title.strip() + # self.og_title = self.title + # self.image = self.albumhash + ".webp" - self.title, self.versions = get_base_title_and_versions( - self.title, get_versions=get_versions - ) - self.base_title = self.title + # # Fetch album artists from title + # if get_flag(SessionVarKeys.EXTRACT_FEAT): + # featured, self.title = parse_feat_from_title(self.title) - if "super_deluxe" in self.versions: - self.versions.remove("deluxe") + # if len(featured) > 0: + # original_lower = "-".join([a.name.lower() for a in self.albumartists]) + # self.albumartists.extend( + # [Artist(a) for a in featured if a.lower() not in original_lower] + # ) - if "original" in self.versions and self.check_is_soundtrack(): - self.versions.remove("original") + # from ..store.tracks import TrackStore - self.versions = [v.replace("_", " ") for v in self.versions] - else: - self.base_title = get_base_title_and_versions( - self.title, get_versions=False - )[0] + # TrackStore.append_track_artists(self.albumhash, featured, self.title) - self.albumartists_hashes = "-".join(a.artisthash for a in self.albumartists) + # # Handle album version data + # else: + # self.base_title = get_base_title_and_versions( + # self.title, get_versions=False + # )[0] - def set_colors(self, colors: list[str]): - self.colors = colors + # self.albumartists_hashes = "-".join(a.artisthash for a in self.albumartists) - def check_type(self): + # # def set_colors(self, colors: list[str]): + # # self.colors = colors + def populate_versions(self): + _, self.versions = get_base_title_and_versions(self.og_title, get_versions=True) + + if "super_deluxe" in self.versions: + self.versions.remove("deluxe") + + # at this point, we should know the type of album + if "original" in self.versions and self.type == "soundtrack": + self.versions.remove("original") + + self.versions = [v.replace("_", " ") for v in self.versions] + + def check_type(self, tracks: list[Track], singleTrackAsSingle: bool): """ Runs all the checks to determine the type of album. """ - self.is_soundtrack = self.check_is_soundtrack() - if self.is_soundtrack: - return + if self.is_single(tracks, singleTrackAsSingle): + return "single" - self.is_live = self.check_is_live_album() - if self.is_live: - return + if self.is_soundtrack(): + return "soundtrack" - self.is_compilation = self.check_is_compilation() - if self.is_compilation: - return + if self.is_live_album(): + return "live album" - self.is_EP = self.check_is_ep() + if self.is_compilation(): + return "compilation" - def check_is_soundtrack(self) -> bool: + if self.is_ep(): + return "ep" + + return "album" + + def is_soundtrack(self) -> bool: """ Checks if the album is a soundtrack. """ @@ -114,11 +132,11 @@ class Album: return False - def check_is_compilation(self) -> bool: + def is_compilation(self) -> bool: """ Checks if the album is a compilation. """ - artists = [a.name for a in self.albumartists] + artists = [a["name"] for a in self.albumartists] artists = "".join(artists).lower() if "various artists" in artists: @@ -137,7 +155,7 @@ class Album: "biggest hits", "the hits", "the ultimate", - "compilation" + "compilation", } for substring in substrings: @@ -146,7 +164,7 @@ class Album: return False - def check_is_live_album(self): + def is_live_album(self): """ Checks if the album is a live album. """ @@ -157,7 +175,7 @@ class Album: return False - def check_is_ep(self) -> bool: + def is_ep(self) -> bool: """ Checks if the album is an EP. """ @@ -165,22 +183,22 @@ class Album: # TODO: check against number of tracks - def check_is_single(self, tracks: list[Track]): + def is_single(self, tracks: list[Track], singleTrackAsSingle: bool): """ Checks if the album is a single. """ keywords = ["single version", "- single"] - show_albums_as_singles = get_flag(SessionVarKeys.SHOW_ALBUMS_AS_SINGLES) + # show_albums_as_singles = get_flag(SessionVarKeys.SHOW_ALBUMS_AS_SINGLES) for keyword in keywords: if keyword in self.og_title.lower(): - self.is_single = True - return + return True - if show_albums_as_singles and len(tracks) == 1: - self.is_single = True - return + # REVIEW: Reading from the config file in a for loop will be slow + # TODO: Find a + if singleTrackAsSingle and len(tracks) == 1: + return True if ( len(tracks) == 1 @@ -192,29 +210,29 @@ class Album: # and tracks[0].disc == 1 # TODO: Review -> Are the above commented checks necessary? ): - self.is_single = True + return True - def get_date_from_tracks(self, tracks: list[Track]): - """ - Gets the date of the album its tracks. + # def get_date_from_tracks(self, tracks: list[Track]): + # """ + # Gets the date of the album its tracks. - Args: - tracks (list[Track]): The tracks of the album. - """ - if self.date: - return + # Args: + # tracks (list[Track]): The tracks of the album. + # """ + # if self.date: + # return - dates = (int(t.date) for t in tracks if t.date) - try: - self.date = datetime.datetime.fromtimestamp(min(dates)).year - except: - self.date = datetime.datetime.now().year + # dates = (int(t.date) for t in tracks if t.date) + # try: + # self.date = datetime.datetime.fromtimestamp(min(dates)).year + # except: + # self.date = datetime.datetime.now().year - def set_count(self, count: int): - self.count = count + # def set_count(self, count: int): + # self.count = count - def set_duration(self, duration: int): - self.duration = duration + # def set_duration(self, duration: int): + # self.duration = duration - def set_created_date(self, created_date: int): - self.created_date = created_date + # def set_created_date(self, created_date: int): + # self.created_date = created_date diff --git a/app/models/artist.py b/app/models/artist.py index c8b6eb10..1e0446a7 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -23,6 +23,12 @@ class ArtistMinimal: if self.artisthash == "5a37d5315e": self.name = "Juice WRLD" + def to_json(self): + return { + "name": self.name, + "artisthash": self.artisthash, + } + @dataclass(slots=True) class Artist(ArtistMinimal): diff --git a/app/models/track.py b/app/models/track.py index ba928cc5..d4c638bc 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -4,7 +4,6 @@ from pathlib import Path from flask_jwt_extended import current_user - from app.settings import SessionVarKeys, get_flag from app.utils.hashing import create_hash from app.utils.parsers import ( @@ -24,10 +23,11 @@ class Track: Track class """ + id: int album: str - albumartists: str | list[ArtistMinimal] + albumartists: list[dict[str, str]] albumhash: str - artists: str | list[ArtistMinimal] + artists: str bitrate: int copyright: str date: int @@ -35,152 +35,174 @@ class Track: duration: int filepath: str folder: str - genre: str | list[str] + genre: list[dict[str, str]] + last_mod: int + og_album: str + og_title: str title: str track: int trackhash: str - last_mod: str | int - image: str = "" - artist_hashes: str = "" + _pos: int = 0 + _ati: str = "" - fav_userids: list = field(default_factory=list) - """ - A string of user ids separated by commas. - """ - # is_favorite: bool = False + # album: str + # albumartists: str | list[ArtistMinimal] + # albumhash: str + # artists: str | list[ArtistMinimal] + # bitrate: int + # copyright: str + # date: int + # disc: int + # duration: int + # filepath: str + # folder: str + # genre: str | list[str] + # title: str + # track: int + # trackhash: str + # last_mod: str | int - @property - def is_favorite(self): - return current_user['id'] in self.fav_userids + # image: str = "" + # artist_hashes: str = "" - # temporary attributes - _pos: int = 0 # for sorting tracks by disc and track number - _ati: str = ( - "" # (album track identifier) for removing duplicates when merging album versions - ) + # fav_userids: list = field(default_factory=list) + # """ + # A string of user ids separated by commas. + # """ + # # is_favorite: bool = False - og_title: str = "" - og_album: str = "" - created_date: float = 0.0 + # @property + # def is_favorite(self): + # return current_user["id"] in self.fav_userids - def set_created_date(self): - try: - self.created_date = Path(self.filepath).stat().st_ctime - except FileNotFoundError: - pass + # # temporary attributes + # _pos: int = 0 # for sorting tracks by disc and track number + # _ati: str = ( + # "" # (album track identifier) for removing duplicates when merging album versions + # ) - def __post_init__(self): - self.og_title = self.title - self.og_album = self.album - self.last_mod = int(self.last_mod) - self.date = int(self.date) + # og_title: str = "" + # og_album: str = "" + # created_date: float = 0.0 - # add a trailing slash to the folder path - # to avoid matching a folder starting with the same name as the root path - # eg. .../Music and .../Music Videos - self.folder = os.path.join(self.folder, "") + # def set_created_date(self): + # try: + # self.created_date = Path(self.filepath).stat().st_ctime + # except FileNotFoundError: + # pass - if self.artists is not None: - artists = split_artists(self.artists) - new_title = self.title + # def __post_init__(self): + # self.og_title = self.title + # self.og_album = self.album + # self.last_mod = int(self.last_mod) + # self.date = int(self.date) - if get_flag(SessionVarKeys.EXTRACT_FEAT): - featured, new_title = parse_feat_from_title(self.title) - original_lower = "-".join([create_hash(a) for a in artists]) - artists.extend( - a for a in featured if create_hash(a) not in original_lower - ) + # # add a trailing slash to the folder path + # # to avoid matching a folder starting with the same name as the root path + # # eg. .../Music and .../Music Videos + # self.folder = os.path.join(self.folder, "") - self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists) - self.artists = [ArtistMinimal(a) for a in artists] + # if self.artists is not None: + # artists = split_artists(self.artists) + # new_title = self.title - albumartists = split_artists(self.albumartists) + # if get_flag(SessionVarKeys.EXTRACT_FEAT): + # featured, new_title = parse_feat_from_title(self.title) + # original_lower = "-".join([create_hash(a) for a in artists]) + # artists.extend( + # a for a in featured if create_hash(a) not in original_lower + # ) - if not albumartists: - self.albumartists = self.artists[:1] - else: - self.albumartists = [ArtistMinimal(a) for a in albumartists] + # self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists) + # self.artists = [ArtistMinimal(a) for a in artists] - if get_flag(SessionVarKeys.REMOVE_PROD): - new_title = remove_prod(new_title) + # albumartists = split_artists(self.albumartists) - # if track is a single - if self.og_title == self.album: - self.rename_album(new_title) + # if not albumartists: + # self.albumartists = self.artists[:1] + # else: + # self.albumartists = [ArtistMinimal(a) for a in albumartists] - if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK): - new_title = clean_title(new_title) + # if get_flag(SessionVarKeys.REMOVE_PROD): + # new_title = remove_prod(new_title) - self.title = new_title + # if track is a single + # if self.og_title == self.album: + # self.rename_album(new_title) - if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): - self.album, _ = get_base_title_and_versions( - self.album, get_versions=False - ) + # if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK): + # new_title = clean_title(new_title) - if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS): - self.recreate_albumhash() + # self.title = new_title - self.image = self.albumhash + ".webp" + # if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): + # self.album, _ = get_base_title_and_versions( + # self.album, get_versions=False + # ) - if self.genre is not None and self.genre != "": - self.genre = self.genre.lower() - separators = {"/", ";", "&"} + # if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS): + # self.recreate_albumhash() - contains_rnb = "r&b" in self.genre - contains_rock = "rock & roll" in self.genre + # self.image = self.albumhash + ".webp" - if contains_rnb: - self.genre = self.genre.replace("r&b", "RnB") + # if self.genre is not None and self.genre != "": + # self.genre = self.genre.lower() + # separators = {"/", ";", "&"} - if contains_rock: - self.genre = self.genre.replace("rock & roll", "rock") + # contains_rnb = "r&b" in self.genre + # contains_rock = "rock & roll" in self.genre - for s in separators: - self.genre: str = self.genre.replace(s, ",") + # if contains_rnb: + # self.genre = self.genre.replace("r&b", "RnB") - self.genre = self.genre.split(",") - self.genre = [g.strip() for g in self.genre] + # if contains_rock: + # self.genre = self.genre.replace("rock & roll", "rock") - self.recreate_hash() - self.set_created_date() + # for s in separators: + # self.genre: str = self.genre.replace(s, ",") - def recreate_hash(self): - """ - Recreates a track hash if the track title was altered - to prevent duplicate tracks having different hashes. - """ - if self.og_title == self.title and self.og_album == self.album: - return + # self.genre = self.genre.split(",") + # self.genre = [g.strip() for g in self.genre] - self.trackhash = create_hash( - ", ".join(a.name for a in self.artists), self.og_album, self.title - ) + # self.recreate_hash() + # self.set_created_date() - def recreate_artists_hash(self): - """ - Recreates a track's artist hashes if the artist list was altered - """ - self.artist_hashes = "-".join(a.artisthash for a in self.artists) + # def recreate_hash(self): + # """ + # Recreates a track hash if the track title was altered + # to prevent duplicate tracks having different hashes. + # """ + # if self.og_title == self.title and self.og_album == self.album: + # return - def recreate_albumhash(self): - """ - Recreates an albumhash of a track to merge all versions of an album. - """ - albumartists = (a.name for a in self.albumartists) - self.albumhash = create_hash(self.album, *albumartists) + # self.trackhash = create_hash( + # ", ".join(a.name for a in self.artists), self.og_album, self.title + # ) - def rename_album(self, new_album: str): - """ - Renames an album - """ - self.album = new_album + # def recreate_artists_hash(self): + # """ + # Recreates a track's artist hashes if the artist list was altered + # """ + # self.artist_hashes = "-".join(a.artisthash for a in self.artists) - def add_artists(self, artists: list[str], new_album_title: str): - for artist in artists: - if create_hash(artist, decode=True) not in self.artist_hashes: - self.artists.append(ArtistMinimal(artist)) + # def recreate_albumhash(self): + # """ + # Recreates an albumhash of a track to merge all versions of an album. + # """ + # albumartists = (a.name for a in self.albumartists) + # self.albumhash = create_hash(self.album, *albumartists) - self.recreate_artists_hash() - self.rename_album(new_album_title) + # def rename_album(self, new_album: str): + # """ + # Renames an album + # """ + # self.album = new_album + + # def add_artists(self, artists: list[str], new_album_title: str): + # for artist in artists: + # if create_hash(artist, decode=True) not in self.artist_hashes: + # self.artists.append(ArtistMinimal(artist)) + + # self.recreate_artists_hash() + # self.rename_album(new_album_title) diff --git a/app/utils/auth.py b/app/utils/auth.py index 87f20c75..a19656a3 100644 --- a/app/utils/auth.py +++ b/app/utils/auth.py @@ -1,6 +1,8 @@ import hmac import hashlib +from flask_jwt_extended import current_user + from app.config import UserConfig @@ -29,3 +31,14 @@ def check_password(password: str, hashed: str) -> bool: """ return hmac.compare_digest(hash_password(password), hashed) + + +def get_current_userid() -> int: + """ + Get the current session user. + """ + try: + return current_user["id"] + except RuntimeError: + # Catch this error raised during migration execution + return 1 diff --git a/db.py b/db.py new file mode 100644 index 00000000..5ab99a23 --- /dev/null +++ b/db.py @@ -0,0 +1,74 @@ +from sqlalchemy import create_engine, text, Table, Column, Integer, String, MetaData, select +from sqlalchemy.orm import DeclarativeBase + +from typing import List, Optional + +from sqlalchemy.orm import Mapped, mapped_column, relationship + +fullpath = "/home/cwilvx/temp/swingmusic/swing.db" +engine = create_engine(f"sqlite+pysqlite:///{fullpath}", echo=True) + +class Base(DeclarativeBase): + pass + +class Tracks(Base): + __tablename__ = "tracks" + + id: Mapped[int] = mapped_column(primary_key=True) + album: Mapped[str] = mapped_column(String()) + albumartist: Mapped[str] = mapped_column(String()) + copyright: Mapped[Optional[str]] + + def __repr__(self): + return f"" + +stmt = select(Tracks.album, Tracks.copyright).where(Tracks.album == "RAVAGE") +print(stmt) + +with engine.connect() as conn: + result = conn.execute(stmt) + for row in result: + print(row) + +# Base.metadata.create_all(engine) + +# metadata = MetaData() +# track_table = Table( +# "tracks", +# metadata, +# Column("id", Integer, primary_key=True, autoincrement=True), +# Column("album", String), +# Column("albumartist", String), +# Column("albumhash", String), +# Column("artist", String), +# Column("bitrate", Integer), +# Column("copyright", String), +# Column("date", Integer), +# Column("disc", Integer), +# Column("duration", Integer), +# Column("filepath", String), +# Column("folder", String), +# Column("genre", String), +# Column("title", String), +# Column("track", Integer), +# Column("trackhash", String), +# Column("last_mod", Integer), +# ) + +# metadata.create_all(engine) + + + + + + + +# with engine.connect() as conn: +# result = conn.execute( +# text("SELECT * FROM tracks where trackhash = :trackhash"), +# {"trackhash": "93acbea22b"}, +# ) +# # print(result.all()) + +# for r in result.mappings(): +# print(r["trackhash"]) diff --git a/manage.py b/manage.py index ddf13c76..62ddf2e1 100644 --- a/manage.py +++ b/manage.py @@ -21,6 +21,7 @@ import setproctitle from app.api import create_api from app.arg_handler import ProcessArgs +from app.lib.tagger import IndexEverything from app.lib.watchdogg import Watcher as WatchDog from app.periodic_scan import run_periodic_scans from app.plugins.register import register_plugins @@ -49,10 +50,11 @@ werkzeug.setLevel(logging.ERROR) # Background tasks -# @background -# def bg_run_setup(): -# pass +@background +def bg_run_setup(): + pass # run_periodic_scans() + IndexEverything() # @background @@ -63,7 +65,7 @@ werkzeug.setLevel(logging.ERROR) @background def run_swingmusic(): log_startup_info() - # bg_run_setup() + bg_run_setup() register_plugins() # start_watchdog() diff --git a/poetry.lock b/poetry.lock index f03244a1..2444e038 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2164,6 +2164,93 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "sqlalchemy" +version = "2.0.31" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, + {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, + {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + [[package]] name = "tabulate" version = "0.9.0" @@ -2515,4 +2602,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "5722346cbfc224340877e337d4cee2c8e8a3a7ea68f9a64d9c5806e0ebcf919a" +content-hash = "333baa055ac4a32ed914fb46025a48559575806dafba7db5aac97a3878ade23c" diff --git a/pyproject.toml b/pyproject.toml index 2cd8ede6..d521a0a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ watchdog = "^4.0.0" pendulum = "^3.0.0" flask-openapi3 = "^3.0.2" flask-jwt-extended = "^4.6.0" +sqlalchemy = "^2.0.31" [tool.poetry.dev-dependencies] pylint = "^2.15.5"