TDvorak/swingmusic-extended
1
0
Fork 0
mirror of https://github.com/Dvorinka/swingmusic-extended.git synced 2026-06-04 20:43:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

combine userdata and swing db into one

Browse Source 
+ port populate to new db interface
+ add genrehashes and hash info to tracks
+ properly structure new db table files
+ move helpers to dedicated utils file
+ move settings from db to config file
+ move artists, albums, auth and favorites endpoint to new db interface
+ use folder store to index filepaths
+ paginate favorite pages
+ 56 moretiny changes 😅
This commit is contained in:
cwilvx
2024-06-30 15:06:33 +03:00
parent 1a66194c6c
commit 4a9f804e70
53 changed files with 1719 additions and 1353 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/lib/albumslib.py
+6 -48
View File
@@ -12,63 +12,21 @@ from app.store.albums import AlbumStore
from app.store.tracks import TrackStore
def create_albums():
"""
Creates albums from the tracks in the store.
"""
# group all tracks by albumhash
tracks = TrackStore.tracks
tracks = sorted(tracks, key=lambda t: t.albumhash)
grouped = groupby(tracks, lambda t: t.albumhash)
# create albums from the groups
albums: list[Track] = []
for albumhash, tracks in grouped:
count = len(list(tracks))
duration = sum(t.duration for t in tracks)
created_date = min(t.created_date for t in tracks)
album = AlbumStore.create_album(list(tracks)[0])
album.set_count(count)
album.set_duration(duration)
album.set_created_date(created_date)
albums.append(album)
return albums
def validate_albums():
"""
Removes albums that have no tracks.
Probably albums that were added from incompletely written files.
"""
album_hashes = {t.albumhash for t in TrackStore.tracks}
albums = AlbumStore.albums
for album in albums:
if album.albumhash not in album_hashes:
AlbumStore.remove_album(album)
def remove_duplicate_on_merge_versions(tracks: list[Track]) -> list[Track]:
"""
Removes duplicate tracks when merging versions of the same album.
"""
# TODO!
pass
def sort_by_track_no(tracks: list[Track]) -> list[dict[str, Any]]:
tracks = [asdict(t) for t in tracks]
def sort_by_track_no(tracks: list[Track]):
# tracks = [asdict(t) for t in tracks]
for t in tracks:
track = str(t["track"]).zfill(3)
t["_pos"] = int(f"{t['disc']}{track}")
track = str(t.track).zfill(3)
t._pos = int(f"{t.disc}{track}")
tracks = sorted(tracks, key=lambda t: t["_pos"])
tracks = sorted(tracks, key=lambda t: t._pos)
return tracks
app/lib/artistlib.py
+13 -106
View File
@@ -1,5 +1,3 @@
from collections import namedtuple
from itertools import groupby
import os
import urllib
from concurrent.futures import ThreadPoolExecutor
@@ -12,9 +10,10 @@ from requests.exceptions import ConnectionError as RequestConnectionError
from requests.exceptions import ReadTimeout
from app import settings
from app.models import Album, Artist, Track
from app.store import artists as artist_store
from app.store.tracks import TrackStore
from app.db.libdata import ArtistTable
# from app.store import artists as artist_store
# from app.store.tracks import TrackStore
from app.utils.hashing import create_hash
from app.utils.progressbar import tqdm
@@ -107,22 +106,15 @@ class CheckArtistImages:
# read all files in the artist image folder
path = settings.Paths.get_sm_artist_img_path()
processed = "".join(os.listdir(path)).replace("webp", "")
# filter out artists that already have an image
artists = filter(
lambda a: a.artisthash not in processed, artist_store.ArtistStore.artists
)
artists = list(artists)
# process the rest
key_artist_map = ((instance_key, artist) for artist in artists)
processed = [path.replace(".webp", "") for path in os.listdir(path)]
unprocessed = ArtistTable.get_artisthashes_not_in(processed)
key_artist_map = ((instance_key, artist) for artist in unprocessed)
with ThreadPoolExecutor(max_workers=14) as executor:
res = list(
tqdm(
executor.map(self.download_image, key_artist_map),
total=len(artists),
total=len(unprocessed),
desc="Downloading missing artist images",
)
)
@@ -130,7 +122,7 @@ class CheckArtistImages:
list(res)
@staticmethod
def download_image(_map: tuple[str, Artist]):
def download_image(_map: tuple[str, dict[str, str]]):
"""
Checks if an artist image exists and downloads it if not.
@@ -142,16 +134,17 @@ class CheckArtistImages:
return
img_path = (
Path(settings.Paths.get_sm_artist_img_path()) / f"{artist.artisthash}.webp"
Path(settings.Paths.get_sm_artist_img_path())
/ f"{artist['artisthash']}.webp"
)
if img_path.exists():
return
url = get_artist_image_link(artist.name)
url = get_artist_image_link(artist["name"])
if url is not None:
return DownloadImage(url, name=f"{artist.artisthash}.webp")
return DownloadImage(url, name=f"{artist['artisthash']}.webp")
# def fetch_album_bio(title: str, albumartist: str) -> str | None: """ Returns the album bio for a given album. """
@@ -183,89 +176,3 @@ class CheckArtistImages:
# def __call__(self):
# return fetch_album_bio(self.title, self.albumartist)
def get_artists_from_tracks(tracks: list[Track]) -> set[str]:
"""
Extracts all artists from a list of tracks. Returns a list of Artists.
"""
artists = set()
master_artist_list = [[x.name for x in t.artists] for t in tracks]
artists = artists.union(*master_artist_list)
return artists
def get_albumartists(albums: list[Album]) -> set[str]:
artists = set()
for album in albums:
albumartists = [a.name for a in album.albumartists]
artists.update(albumartists)
return artists
def get_all_artists(tracks: list[Track], albums: list[Album]) -> list[Artist]:
TrackInfo = namedtuple(
"TrackInfo",
[
"artisthash",
"albumhash",
"trackhash",
"duration",
"artistname",
"created_date",
],
)
src_tracks = TrackStore.tracks
all_tracks: set[TrackInfo] = set()
for track in src_tracks:
artist_hashes = {(a.name, a.artisthash) for a in track.artists}.union(
(a.name, a.artisthash) for a in track.albumartists
)
for artist in artist_hashes:
track_info = TrackInfo(
artistname=artist[0],
artisthash=artist[1],
albumhash=track.albumhash,
trackhash=track.trackhash,
duration=track.duration,
created_date=track.created_date,
# work on created date
)
all_tracks.add(track_info)
all_tracks = sorted(all_tracks, key=lambda x: x.artisthash)
all_tracks = groupby(all_tracks, key=lambda x: x.artisthash)
artists = []
for artisthash, tracks in all_tracks:
tracks: list[TrackInfo] = list(tracks)
artistname = (
sorted({t.artistname for t in tracks})[0]
if len(tracks) > 1
else tracks[0].artistname
)
albumcount = len({t.albumhash for t in tracks})
duration = sum(t.duration for t in tracks)
created_date = min(t.created_date for t in tracks)
artist = Artist(name=artistname)
artist.set_trackcount(len(tracks))
artist.set_albumcount(albumcount)
artist.set_duration(duration)
artist.set_created_date(created_date)
artists.append(artist)
return artists
app/lib/colorlib.py
+33 -33
View File
@@ -52,47 +52,47 @@ def process_color(item_hash: str, is_album=True):
return get_image_colors(str(path))
class ProcessAlbumColors:
"""
Extracts the most dominant color from the album art and saves it to the database.
"""
# class ProcessAlbumColors:
# """
# Extracts the most dominant color from the album art and saves it to the database.
# """
def __init__(self, instance_key: str) -> None:
global PROCESS_ALBUM_COLORS_KEY
PROCESS_ALBUM_COLORS_KEY = instance_key
# def __init__(self, instance_key: str) -> None:
# global PROCESS_ALBUM_COLORS_KEY
# PROCESS_ALBUM_COLORS_KEY = instance_key
albums = [
a
for a in AlbumStore.albums
if a is not None and a.colors is not None and len(a.colors) == 0
]
# albums = [
# a
# for a in AlbumStore.albums
# if a is not None and a.colors is not None and len(a.colors) == 0
# ]
with SQLiteManager() as cur:
try:
for album in tqdm(albums, desc="Processing missing album colors"):
if PROCESS_ALBUM_COLORS_KEY != instance_key:
raise PopulateCancelledError(
"A newer 'ProcessAlbumColors' instance is running. Stopping this one."
)
# with SQLiteManager() as cur:
# try:
# for album in tqdm(albums, desc="Processing missing album colors"):
# if PROCESS_ALBUM_COLORS_KEY != instance_key:
# raise PopulateCancelledError(
# "A newer 'ProcessAlbumColors' instance is running. Stopping this one."
# )
# TODO: Stop hitting the database for every album.
# Instead, fetch all the data from the database and
# check from memory.
# # TODO: Stop hitting the database for every album.
# # Instead, fetch all the data from the database and
# # check from memory.
exists = aldb.exists(album.albumhash, cur=cur)
if exists:
continue
# exists = aldb.exists(album.albumhash, cur=cur)
# if exists:
# continue
colors = process_color(album.albumhash)
# colors = process_color(album.albumhash)
if colors is None:
continue
# if colors is None:
# continue
album.set_colors(colors)
color_str = json.dumps(colors)
aldb.insert_one_album(cur, album.albumhash, color_str)
finally:
cur.close()
# album.set_colors(colors)
# color_str = json.dumps(colors)
# aldb.insert_one_album(cur, album.albumhash, color_str)
# finally:
# cur.close()
class ProcessArtistColors:
app/lib/folderslib.py
+3 -3
View File
@@ -5,9 +5,10 @@ from app.logger import log
from app.models import Folder
from app.serializers.track import serialize_tracks
from app.settings import SUPPORTED_FILES
from app.store.folder import FolderStore
from app.utils.wintools import win_replace_slash
from app.db import TrackTable as TrackDB
from app.db.libdata import TrackTable as TrackDB
def create_folder(path: str, trackcount=0, foldercount=0) -> Folder:
@@ -43,8 +44,7 @@ def get_folders(paths: list[str]):
Filters out folders that don't have any tracks and
returns a list of folder objects.
"""
folders = TrackDB.count_tracks_containing_paths(paths)
folders = FolderStore.count_tracks_containing_paths(paths)
return [
create_folder(f["path"], f["trackcount"], foldercount=0)
for f in folders
app/lib/populate.py
+138 -138
View File
@@ -1,3 +1,4 @@
from dataclasses import asdict
import os
from collections import deque
from concurrent.futures import ThreadPoolExecutor
@@ -7,28 +8,26 @@ from requests import ConnectionError as RequestConnectionError
from requests import ReadTimeout
from app import settings
from app.db import TrackTable
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable, TrackTable
from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
from app.db.sqlite.settings import SettingsSQLMethods as sdb
# from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
from app.db.sqlite.tracks import SQLiteTrackMethods
from app.lib.albumslib import validate_albums
from app.lib.artistlib import CheckArtistImages
from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors
from app.lib.colorlib import ProcessArtistColors
from app.lib.errors import PopulateCancelledError
from app.lib.taglib import extract_thumb, get_tags
from app.lib.trackslib import validate_tracks
from app.lib.taglib import extract_thumb
from app.logger import log
from app.models import Album, Artist, Track
from app.models.lastfm import SimilarArtist
from app.requests.artists import fetch_similar_artists
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
from app.utils.filesystem import run_fast_scandir
from app.utils.network import has_connection
from app.utils.progressbar import tqdm
from app.db.userdata import SimilarArtistTable
get_all_tracks = SQLiteTrackMethods.get_all_tracks
insert_many_tracks = SQLiteTrackMethods.insert_many_tracks
remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths
@@ -44,50 +43,49 @@ class Populate:
also checks if the album art exists in the image path, if not tries to extract it.
"""
def __init__(self, instance_key: str) -> None:
return
# def __init__(self, instance_key: str) -> None:
# return
# if len(dirs_to_scan) == 0:
# log.warning(
# (
# "The root directory is not configured. "
# + "Open the app in your webbrowser to configure."
# )
# )
# return
# try:
# if dirs_to_scan[0] == "$home":
# dirs_to_scan = [settings.Paths.USER_HOME_DIR]
# except IndexError:
# pass
# files = set()
# for _dir in dirs_to_scan:
# files = files.union(run_fast_scandir(_dir, full=True)[1])
# unmodified, modified_tracks = self.remove_modified(tracks)
# untagged = files - unmodified
# if len(untagged) != 0:
# self.tag_untagged(untagged, instance_key)
# self.extract_thumb_with_overwrite(modified_tracks)
class CordinateMedia:
"""
Cordinates the extracting of thumbnails
"""
def __init__(self, instance_key: str):
global POPULATE_KEY
POPULATE_KEY = instance_key
validate_tracks()
validate_albums()
tracks = get_all_tracks()
dirs_to_scan = sdb.get_root_dirs()
if len(dirs_to_scan) == 0:
log.warning(
(
"The root directory is not configured. "
+ "Open the app in your webbrowser to configure."
)
)
return
try:
if dirs_to_scan[0] == "$home":
dirs_to_scan = [settings.Paths.USER_HOME_DIR]
except IndexError:
pass
files = set()
for _dir in dirs_to_scan:
files = files.union(run_fast_scandir(_dir, full=True)[1])
unmodified, modified_tracks = self.remove_modified(tracks)
untagged = files - unmodified
if len(untagged) != 0:
self.tag_untagged(untagged, instance_key)
self.extract_thumb_with_overwrite(modified_tracks)
try:
ProcessTrackThumbnails(instance_key)
ProcessAlbumColors(instance_key)
ProcessArtistColors(instance_key)
except PopulateCancelledError as e:
log.warn(e)
@@ -95,10 +93,6 @@ class Populate:
tried_to_download_new_images = False
ArtistStore.load_artists(instance_key)
AlbumStore.load_albums(instance_key)
TrackStore.load_all_tracks(instance_key)
if has_connection():
tried_to_download_new_images = True
try:
@@ -123,101 +117,101 @@ class Populate:
log.warn(e)
return
@staticmethod
def remove_modified(tracks: Generator[TrackTable, None, None]):
"""
Removes tracks from the database that have been modified
since they were added to the database.
"""
# @staticmethod
# def remove_modified(tracks: Generator[TrackTable, None, None]):
# """
# Removes tracks from the database that have been modified
# since they were added to the database.
# """
unmodified_paths = set()
modified_tracks: list[TrackTable] = []
modified_paths = set()
# unmodified_paths = set()
# modified_tracks: list[TrackTable] = []
# modified_paths = set()
for track in tracks:
try:
if track.last_mod == round(os.path.getmtime(track.filepath)):
unmodified_paths.add(track.filepath)
continue
except (FileNotFoundError, OSError) as e:
log.warning(e) # REVIEW More informations = good
TrackStore.remove_track_obj(track)
remove_tracks_by_filepaths(track.filepath)
# for track in tracks:
# try:
# if track.last_mod == round(os.path.getmtime(track.filepath)):
# unmodified_paths.add(track.filepath)
# continue
# except (FileNotFoundError, OSError) as e:
# log.warning(e) # REVIEW More informations = good
# TrackStore.remove_track_obj(track)
# remove_tracks_by_filepaths(track.filepath)
modified_paths.add(track.filepath)
modified_tracks.append(track)
# modified_paths.add(track.filepath)
# modified_tracks.append(track)
TrackStore.remove_tracks_by_filepaths(modified_paths)
remove_tracks_by_filepaths(modified_paths)
# TrackStore.remove_tracks_by_filepaths(modified_paths)
# remove_tracks_by_filepaths(modified_paths)
return unmodified_paths, modified_tracks
# return unmodified_paths, modified_tracks
@staticmethod
def tag_untagged(untagged: set[str], key: str):
pass
# for file in tqdm(untagged, desc="Reading files"):
# if POPULATE_KEY != key:
# log.warning("'Populate.tag_untagged': Populate key changed")
# return
# @staticmethod
# def tag_untagged(untagged: set[str], key: str):
# pass
# for file in tqdm(untagged, desc="Reading files"):
# if POPULATE_KEY != key:
# log.warning("'Populate.tag_untagged': Populate key changed")
# return
# tags = get_tags(file)
# tags = get_tags(file)
# if tags is not None:
# TrackTable.insert_one(tags)
# if tags is not None:
# TrackTable.insert_one(tags)
# =============================================
# =============================================
# log.info("Found %s new tracks", len(untagged))
# # tagged_tracks: deque[dict] = deque()
# # tagged_count = 0
# log.info("Found %s new tracks", len(untagged))
# # tagged_tracks: deque[dict] = deque()
# # tagged_count = 0
# favs = favdb.get_fav_tracks()
# records = dict()
# favs = favdb.get_fav_tracks()
# records = dict()
# for fav in favs:
# r = records.setdefault(fav[1], set())
# r.add(fav[4])
# for fav in favs:
# r = records.setdefault(fav[1], set())
# r.add(fav[4])
# tagged_tracks.append(tags)
# track = Track(**tags)
# tagged_tracks.append(tags)
# track = Track(**tags)
# track.fav_userids = list(records.get(track.trackhash, set()))
# track.fav_userids = list(records.get(track.trackhash, set()))
# TrackStore.add_track(track)
# TrackStore.add_track(track)
# if not AlbumStore.album_exists(track.albumhash):
# AlbumStore.add_album(AlbumStore.create_album(track))
# if not AlbumStore.album_exists(track.albumhash):
# AlbumStore.add_album(AlbumStore.create_album(track))
# for artist in track.artists:
# if not ArtistStore.artist_exists(artist.artisthash):
# ArtistStore.add_artist(Artist(artist.name))
# for artist in track.artists:
# if not ArtistStore.artist_exists(artist.artisthash):
# ArtistStore.add_artist(Artist(artist.name))
# for artist in track.albumartists:
# if not ArtistStore.artist_exists(artist.artisthash):
# ArtistStore.add_artist(Artist(artist.name))
# for artist in track.albumartists:
# if not ArtistStore.artist_exists(artist.artisthash):
# ArtistStore.add_artist(Artist(artist.name))
# tagged_count += 1
# else:
# log.warning("Could not read file: %s", file)
# tagged_count += 1
# else:
# log.warning("Could not read file: %s", file)
# if len(tagged_tracks) > 0:
# log.info("Adding %s tracks to database", len(tagged_tracks))
# insert_many_tracks(tagged_tracks)
# if len(tagged_tracks) > 0:
# log.info("Adding %s tracks to database", len(tagged_tracks))
# insert_many_tracks(tagged_tracks)
# log.info("Added %s/%s tracks", tagged_count, len(untagged))
# log.info("Added %s/%s tracks", tagged_count, len(untagged))
@staticmethod
def extract_thumb_with_overwrite(tracks: list[TrackTable]):
"""
Extracts the thumbnail from a list of filepaths,
overwriting the existing thumbnail if it exists,
for modified files.
"""
for track in tracks:
try:
extract_thumb(track.filepath, track.image, overwrite=True)
except FileNotFoundError:
continue
# @staticmethod
# def extract_thumb_with_overwrite(tracks: list[TrackTable]):
# """
# Extracts the thumbnail from a list of filepaths,
# overwriting the existing thumbnail if it exists,
# for modified files.
# """
# for track in tracks:
# try:
# extract_thumb(track.filepath, track.image, overwrite=True)
# except FileNotFoundError:
# continue
def get_image(_map: tuple[str, Album]):
@@ -235,7 +229,8 @@ def get_image(_map: tuple[str, Album]):
raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed")
matching_tracks = filter(
lambda t: t.albumhash == album.albumhash, TrackStore.tracks
lambda t: t.albumhash == album.albumhash,
TrackTable.get_tracks_by_albumhash(album.albumhash),
)
try:
@@ -254,8 +249,12 @@ def get_image(_map: tuple[str, Album]):
pass
_cpu_count = os.cpu_count()
CPU_COUNT = _cpu_count // 2 if _cpu_count > 2 else _cpu_count
def get_cpu_count():
"""
Returns the number of CPUs on the machine.
"""
cpu_count = os.cpu_count() or 0
return cpu_count // 2 if cpu_count > 2 else cpu_count
class ProcessTrackThumbnails:
@@ -275,14 +274,14 @@ class ProcessTrackThumbnails:
# filter out albums that already have thumbnails
albums = filter(
lambda album: album.albumhash not in processed, AlbumStore.albums
lambda album: album.albumhash not in processed, AlbumTable.get_all()
)
albums = list(albums)
# process the rest
key_album_map = ((instance_key, album) for album in albums)
with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor:
with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor:
results = list(
tqdm(
executor.map(get_image, key_album_map),
@@ -307,16 +306,17 @@ def save_similar_artists(_map: tuple[str, Artist]):
"'FetchSimilarArtistsLastFM': Populate key changed"
)
if lastfmdb.exists(artist.artisthash):
if SimilarArtistTable.exists(artist.artisthash):
return
artist_hashes = fetch_similar_artists(artist.name)
artist_ = SimilarArtist(artist.artisthash, "~".join(artist_hashes))
artists = fetch_similar_artists(artist.name)
if len(artist_.similar_artist_hashes) == 0:
# INFO: Nones mean there was a connection error
if artists is None:
return
lastfmdb.insert_one(artist_)
artist_ = SimilarArtist(artist.artisthash, artists)
SimilarArtistTable.insert_one(asdict(artist_))
class FetchSimilarArtistsLastFM:
@@ -326,17 +326,17 @@ class FetchSimilarArtistsLastFM:
def __init__(self, instance_key: str) -> None:
# read all artists from db
processed = lastfmdb.get_all()
processed = SimilarArtistTable.get_all()
processed = ".".join(a.artisthash for a in processed)
# filter out artists that already have similar artists
artists = filter(lambda a: a.artisthash not in processed, ArtistStore.artists)
artists = filter(lambda a: a.artisthash not in processed, ArtistTable.get_all())
artists = list(artists)
# process the rest
key_artist_map = ((instance_key, artist) for artist in artists)
with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor:
with ThreadPoolExecutor(max_workers=get_cpu_count()) as executor:
try:
print("Processing similar artists")
results = list(
app/lib/tagger.py
+163 -33
View File
@@ -1,55 +1,165 @@
import os
from pprint import pprint
from app.db import AlbumTable, ArtistTable, TrackTable
from app.lib.taglib import get_tags
from time import time
from typing import Generator
from app import settings
from app.config import UserConfig
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable, TrackTable
from app.lib.populate import CordinateMedia
from app.lib.taglib import extract_thumb, get_tags
from app.models.track import Track
from app.store.folder import FolderStore
from app.utils.filesystem import run_fast_scandir
from app.utils.parsers import get_base_album_title
from app.utils.progressbar import tqdm
from app.logger import log
from app.utils.threading import background
POPULATE_KEY: float = 0
class IndexTracks:
def __init__(self) -> None:
dirs_to_scan = ["/home/cwilvx/Music"]
def __init__(self, instance_key: float) -> None:
"""
Indexes all tracks in the database.
An instance key is used to prevent multiple instances of the
same class from running at the same time.
"""
global POPULATE_KEY
POPULATE_KEY = instance_key
# dirs_to_scan = sdb.get_root_dirs()
dirs_to_scan = UserConfig().rootDirs
if len(dirs_to_scan) == 0:
log.warning(
(
"The root directory is not configured. "
+ "Open the app in your webbrowser to configure."
)
)
return
try:
if dirs_to_scan[0] == "$home":
dirs_to_scan = [settings.Paths.USER_HOME_DIR]
except IndexError:
pass
files = set()
for _dir in dirs_to_scan:
files = files.union(run_fast_scandir(_dir, full=True)[1])
self.tag_untagged(files)
# unmodified, modified_tracks = self.remove_modified(tracks)
# untagged = files - unmodified
unmodified, modified_tracks = self.filter_modded()
untagged = files - unmodified
def tag_untagged(self, files: set[str]):
self.tag_untagged(untagged, instance_key)
self.extract_thumb_with_overwrite(modified_tracks)
@staticmethod
def extract_thumb_with_overwrite(tracks: list[dict[str, str]]):
"""
Extracts the thumbnail from a list of filepaths,
overwriting the existing thumbnail if it exists,
for modified files.
"""
for track in tracks:
try:
extract_thumb(
track["filepath"], track["trackhash"] + ".webp", overwrite=True
)
except FileNotFoundError:
continue
@staticmethod
def filter_modded():
"""
Removes tracks from the database that have been modified
since they were indexed.
Returns a tuple of unmodified paths and modified tracks.
Unmodified paths are indexed and the modified tracks are
"""
unmodified_paths = set()
modified_tracks: list[dict[str, str]] = []
to_remove = set()
for track in TrackTable.get_all():
try:
if track.last_mod == round(os.path.getmtime(track.filepath)):
unmodified_paths.add(track.filepath)
continue
except (FileNotFoundError, OSError) as e:
log.warning(e) # REVIEW More informations = good
to_remove.add(track.filepath)
modified_tracks.append(
{
"filepath": track.filepath,
"trackhash": track.trackhash,
}
)
to_remove = to_remove.union(set(t["filepath"] for t in modified_tracks))
TrackTable.remove_tracks_by_filepaths(to_remove)
# REVIEW: Remove after testing!
track = TrackTable.get_tracks_by_filepaths(list(to_remove)[:1])
if track:
raise Exception("Track not removed")
# =============================================================
return unmodified_paths, modified_tracks
def get_untagged(self):
tracks = TrackTable.get_all()
def tag_untagged(self, files: set[str], key: float):
config = UserConfig()
for file in tqdm(files, desc="Reading files"):
# if POPULATE_KEY != key:
# log.warning("'Populate.tag_untagged': Populate key changed")
# return
if POPULATE_KEY != key:
log.warning("'Populate.tag_untagged': Populate key changed")
return
tags = get_tags(file)
tags = get_tags(file, artist_separators=config.artistSeparators)
if tags is not None:
TrackTable.insert_one(tags)
FolderStore.filepaths.add(tags["filepath"])
del tags
print(f"{len(files)} new files indexed")
print("Done")
class IndexAlbums:
def __init__(self) -> None:
albums = dict()
all_tracks: list[Track] = TrackTable.get_all()
all_tracks: list[TrackTable] = TrackTable.get_all()
if len(all_tracks) == 0:
return
for track in all_tracks:
if track.albumhash not in albums:
albums[track.albumhash] = {
"albumartists": track.albumartists,
"artisthashes": [a['artisthash'] for a in track.albumartists],
"artisthashes": [a["artisthash"] for a in track.albumartists],
"albumhash": track.albumhash,
"base_title": None,
"color": None,
"created_date": None,
"date": None,
"duration": track.duration,
"genres": [*track.genre] if track.genre else [],
"genres": [*track.genres] if track.genres else [],
"og_title": track.og_album,
"title": track.album,
"trackcount": 1,
@@ -63,8 +173,8 @@ class IndexAlbums:
album["dates"].append(track.date)
album["created_dates"].append(track.last_mod)
if track.genre:
album["genres"].extend(track.genre)
if track.genres:
album["genres"].extend(track.genres)
for album in albums.values():
album["date"] = min(album["dates"])
@@ -79,20 +189,23 @@ class IndexAlbums:
album["genres"] = genres
album["base_title"], _ = get_base_album_title(album["og_title"])
del genres
del album["dates"]
del album["created_dates"]
pprint(albums)
AlbumTable.remove_all()
AlbumTable.insert_many(list(albums.values()))
del albums
class IndexArtists:
def __init__(self) -> None:
all_tracks: list[TrackTable] = TrackTable.get_all()
all_tracks: list[Track] = TrackTable.get_all()
artists = dict()
if len(all_tracks) == 0:
return
for track in all_tracks:
this_artists = track.artists
@@ -100,32 +213,33 @@ class IndexArtists:
if a not in this_artists:
this_artists.append(a)
for artist in this_artists:
if artist["artisthash"] not in artists:
artists[artist["artisthash"]] = {
for thisartist in this_artists:
if thisartist["artisthash"] not in artists:
artists[thisartist["artisthash"]] = {
"albumcount": None,
"albums": {track.albumhash},
"artisthash": artist["artisthash"],
"artisthash": thisartist["artisthash"],
"created_dates": [track.last_mod],
"dates": [track.date],
"date": None,
"duration": track.duration,
"genres": track.genre if track.genre else [],
"name": artist["name"],
"genres": track.genres if track.genres else [],
"name": None,
"names": {thisartist["name"]},
"trackcount": None,
"tracks": {track.trackhash},
}
else:
artist = artists[artist["artisthash"]]
artist = artists[thisartist["artisthash"]]
artist["duration"] += track.duration
artist["albums"].add(track.albumhash)
artist["tracks"].add(track.trackhash)
artist["dates"].append(track.date)
artist["created_dates"].append(track.last_mod)
artist["names"].add(thisartist["name"])
if track.genre:
artist["genres"].extend(track.genre)
if track.genres:
artist["genres"].extend(track.genres)
for artist in artists.values():
artist["albumcount"] = len(artist["albums"])
@@ -140,19 +254,35 @@ class IndexArtists:
genres.append(genre)
artist["genres"] = genres
artist["name"] = sorted(artist["names"])[0]
# INFO: Delete temporary keys
del artist["names"]
del artist["tracks"]
del artist["albums"]
del artist["dates"]
del artist["created_dates"]
pprint(artists)
# INFO: Delete local variables
del genres
ArtistTable.remove_all()
ArtistTable.insert_many(list(artists.values()))
del artists
class IndexEverything:
def __init__(self) -> None:
IndexTracks()
IndexTracks(instance_key=time())
IndexAlbums()
IndexArtists()
pass
FolderStore.load_filepaths()
# pass
CordinateMedia(instance_key=str(time()))
@background
def index_everything():
return IndexEverything()
app/lib/taglib.py
+42 -24
View File
@@ -5,6 +5,7 @@ from pathlib import Path
from pprint import pprint
import re
import sys
from typing import Any
import pendulum
from PIL import Image, UnidentifiedImageError
@@ -86,7 +87,7 @@ def extract_thumb(filepath: str, webp_path: str, overwrite=False) -> bool:
return False
def parse_date(date_str: str | None) -> int | None:
def parse_date(date_str: str) -> int | None:
"""
Extracts the date from a string and returns a timestamp.
"""
@@ -108,12 +109,13 @@ def clean_filename(filename: str):
class ParseData:
artist: str
title: str
artist_separators: set[str]
def __post_init__(self):
self.artist = split_artists(self.artist)
self.artist = split_artists(self.artist, self.artist_separators)
def extract_artist_title(filename: str):
def extract_artist_title(filename: str, artist_separators: set[str]):
path = Path(filename).with_suffix("")
path = clean_filename(str(path))
@@ -121,22 +123,24 @@ def extract_artist_title(filename: str):
split_result = [x.strip() for x in split_result]
if len(split_result) == 1:
return ParseData("", split_result[0])
return ParseData("", split_result[0], artist_separators)
if len(split_result) > 2:
try:
int(split_result[0])
return ParseData(split_result[1], " - ".join(split_result[2:]))
return ParseData(
split_result[1], " - ".join(split_result[2:]), artist_separators
)
except ValueError:
pass
artist = split_result[0]
title = split_result[1]
return ParseData(artist, title)
return ParseData(artist, title, artist_separators)
def get_tags(filepath: str):
def get_tags(filepath: str, artist_separators: set[str]):
"""
Returns the tags for a given audio file.
"""
@@ -150,7 +154,7 @@ def get_tags(filepath: str):
return None
try:
tags = TinyTag.get(filepath)
tags: Any = TinyTag.get(filepath)
except: # noqa: E722
return None
@@ -169,7 +173,7 @@ def get_tags(filepath: str):
for tag in to_filename:
p = getattr(tags, tag)
if p == "" or p is None:
parse_data = extract_artist_title(filename)
parse_data = extract_artist_title(filename, artist_separators)
title = parse_data.title
setattr(tags, tag, title)
@@ -179,7 +183,7 @@ def get_tags(filepath: str):
if p == "" or p is None:
if not parse_data:
parse_data = extract_artist_title(filename)
parse_data = extract_artist_title(filename, artist_separators)
artist = parse_data.artist
@@ -225,8 +229,8 @@ def get_tags(filepath: str):
tags.artists = tags.artist
tags.albumartists = tags.albumartist
split_artist = split_artists(tags.artist)
split_albumartists = split_artists(tags.albumartist)
split_artist = split_artists(tags.artist, separators=artist_separators)
split_albumartists = split_artists(tags.albumartist, separators=artist_separators)
new_title = tags.title
# TODO: Figure out which is the best spot to create these hashes
@@ -237,7 +241,9 @@ def get_tags(filepath: str):
# extract featured artists
if config.extractFeaturedArtists:
feat, new_title = parse_feat_from_title(tags.title)
feat, new_title = parse_feat_from_title(
tags.title, separators=artist_separators
)
original_lower = "-".join([create_hash(a) for a in split_artist])
split_artist.extend(a for a in feat if create_hash(a) not in original_lower)
@@ -262,8 +268,9 @@ def get_tags(filepath: str):
for a in split_albumartists
]
tags.artisthashes = list({a["artisthash"] for a in tags.artists + tags.albumartists})
tags.artisthashes = list(
{a["artisthash"] for a in tags.artists + tags.albumartists}
)
# remove prod by
if config.removeProdBy:
@@ -295,26 +302,32 @@ def get_tags(filepath: str):
# process genres
if tags.genre:
tags.genre = tags.genre.lower()
src_genres: str = tags.genre
src_genres = src_genres.lower()
# separators = {"/", ";", "&"}
separators = set(config.genreSeparators)
contains_rnb = "r&b" in tags.genre
contains_rock = "rock & roll" in tags.genre
contains_rnb = "r&b" in src_genres
contains_rock = "rock & roll" in src_genres
if contains_rnb:
tags.genre = tags.genre.replace("r&b", "RnB")
src_genres = src_genres.replace("r&b", "RnB")
if contains_rock:
tags.genre = tags.genre.replace("rock & roll", "rock")
src_genres = src_genres.replace("rock & roll", "rock")
for s in separators:
tags.genre = tags.genre.replace(s, ",")
src_genres = src_genres.replace(s, ",")
tags.genre = tags.genre.split(",")
tags.genre = [
{"name": g.strip(), "genrehash": create_hash(g.strip())} for g in tags.genre
genres_list: list[str] = src_genres.split(",")
tags.genres = [
{"name": g.strip(), "genrehash": create_hash(g.strip())}
for g in genres_list
]
tags.genrehashes = [g["genrehash"] for g in tags.genres]
else:
tags.genres = []
tags.genrehashes = []
# sub underscore with space
tags.title = tags.title.replace("_", " ")
@@ -333,6 +346,10 @@ def get_tags(filepath: str):
"filesize": tags.filesize,
"samplerate": tags.samplerate,
"track_total": tags.track_total,
"hashinfo": {
"algo": "sha1",
"format": "[:5]+[-5:]", # first 5 + last 5 chars
},
}
tags.extra = {**tags.extra, **more_extra}
@@ -357,6 +374,7 @@ def get_tags(filepath: str):
"bitdepth",
"artist",
"albumartist",
"genre",
]
for tag in to_delete:
app/lib/trackslib.py
-10
View File
@@ -13,16 +13,6 @@ from app.utils.progressbar import tqdm
from app.utils.threading import ThreadWithReturnValue
def validate_tracks() -> None:
"""
Removes track records whose files no longer exist.
"""
for track in tqdm(TrackStore.tracks, desc="Validating tracks"):
if not os.path.exists(track.filepath):
TrackStore.remove_track_obj(track)
trackdb.remove_tracks_by_filepaths(track.filepath)
def get_leading_silence_end(filepath: str):
"""
Returns the leading silence of a track.
app/lib/watchdogg.py
+7 -3
View File
@@ -11,8 +11,10 @@ from watchdog.events import PatternMatchingEventHandler
from watchdog.observers import Observer
from app import settings
from app.config import UserConfig
from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb
from app.db.sqlite.settings import SettingsSQLMethods as sdb
# from app.db.sqlite.settings import SettingsSQLMethods as sdb
from app.db.sqlite.tracks import SQLiteManager
from app.db.sqlite.tracks import SQLiteTrackMethods as db
from app.lib.colorlib import process_color
@@ -43,7 +45,8 @@ class Watcher:
while trials < 10:
try:
dirs = sdb.get_root_dirs()
# dirs = sdb.get_root_dirs()
dirs = UserConfig().rootDirs
dirs = [rf"{d}" for d in dirs]
dir_map = [
@@ -152,7 +155,8 @@ def add_track(filepath: str) -> None:
TrackStore.remove_track_by_filepath(filepath)
tags = get_tags(filepath)
config = UserConfig()
tags = get_tags(filepath, artist_separators=config.artistSeparators)
# if the track is somehow invalid, return
if tags is None or tags["bitrate"] == 0 or tags["duration"] == 0: