From 4c09350b411f582b6031c51b35cd588687348ffb Mon Sep 17 00:00:00 2001 From: geoffrey45 Date: Wed, 4 May 2022 01:36:27 +0300 Subject: [PATCH 1/2] move the populate function to separate file --- server/app/functions.py | 140 ++----------------------- server/app/imgserver/__init__.py | 14 ++- server/app/lib/albumslib.py | 6 +- server/app/lib/populate.py | 171 +++++++++++++++++++++++++++++++ server/app/lib/trackslib.py | 5 +- server/app/models.py | 1 + 6 files changed, 195 insertions(+), 142 deletions(-) create mode 100644 server/app/lib/populate.py diff --git a/server/app/functions.py b/server/app/functions.py index e9eaf41f..e94440f3 100644 --- a/server/app/functions.py +++ b/server/app/functions.py @@ -1,25 +1,18 @@ """ This module contains functions for the server """ -import datetime import os import time -from dataclasses import asdict from io import BytesIO import requests from app import api from app import helpers -from app import instances -from app import models from app import settings -from app.lib import albumslib -from app.lib import folderslib from app.lib import watchdoge -from app.lib.taglib import get_tags -from app.logger import Log from PIL import Image from progress.bar import Bar +from app.lib.populate import Populate @helpers.background @@ -27,6 +20,7 @@ def reindex_tracks(): """ Checks for new songs every 5 minutes. """ + is_underway = False while True: populate() @@ -44,129 +38,11 @@ def start_watchdog(): def populate(): - """ - Populate the database with all songs in the music directory - - checks if the song is in the database, if not, it adds it - also checks if the album art exists in the image path, if not tries to - extract it. - """ - start = time.time() - db_tracks = instances.tracks_instance.get_all_tracks() - tagged_tracks = [] - albums = [] - folders = set() - - files = helpers.run_fast_scandir(settings.HOME_DIR, [".flac", ".mp3"], full=True)[1] - - _bar = Bar("Checking files", max=len(files)) - for track in db_tracks: - if track["filepath"] in files: - files.remove(track["filepath"]) - _bar.next() - - _bar.finish() - - Log(f"Found {len(files)} untagged files") - - _bar = Bar("Tagging files", max=len(files)) - for file in files: - tags = get_tags(file) - foldername = os.path.dirname(file) - folders.add(foldername) - - if tags is not None: - tagged_tracks.append(tags) - api.DB_TRACKS.append(tags) - - _bar.next() - _bar.finish() - - Log(f"Tagged {len(tagged_tracks)} tracks") - - pre_albums = [] - - for t in tagged_tracks: - a = { - "title": t["album"], - "artist": t["albumartist"], - } - - if a not in pre_albums: - pre_albums.append(a) - - exist_count = 0 - _bar = Bar("Creating albums", max=len(pre_albums)) - for aa in pre_albums: - albumindex = albumslib.find_album(aa["title"], aa["artist"]) - - if albumindex is None: - track = [ - track - for track in tagged_tracks - if track["album"] == aa["title"] - and track["albumartist"] == aa["artist"] - ][0] - - album = albumslib.create_album(track) - api.ALBUMS.append(album) - albums.append(album) - - instances.album_instance.insert_album(asdict(album)) - - else: - exist_count += 1 - - _bar.next() - - _bar.finish() - - Log(f"{exist_count} of {len(albums)} were already in the database") - - _bar = Bar("Creating tracks", max=len(tagged_tracks)) - for track in tagged_tracks: - try: - album_index = albumslib.find_album(track["album"], track["albumartist"]) - album = api.ALBUMS[album_index] - - track["image"] = album.image - upsert_id = instances.tracks_instance.insert_song(track) - - track["_id"] = {"$oid": str(upsert_id)} - api.TRACKS.append(models.Track(track)) - except TypeError: - # Bug: some albums are not found although they exist in `api.ALBUMS`. It has something to do with the bisection method used or sorting. Not sure yet. - pass - - _bar.next() - - _bar.finish() - - Log(f"Added {len(tagged_tracks)} new tracks and {len(albums)} new albums") - - _bar = Bar("Creating folders", max=len(folders)) - for folder in folders: - if folder not in api.VALID_FOLDERS: - api.VALID_FOLDERS.add(folder) - fff = folderslib.create_folder(folder) - api.FOLDERS.append(fff) - - _bar.next() - - _bar.finish() - - Log(f"Created {len(api.FOLDERS)} folders") - - end = time.time() - - print( - str(datetime.timedelta(seconds=round(end - start))) - + " elapsed for " - + str(len(files)) - + " files" - ) + pop = Populate() + pop.run() +@helpers.background def fetch_image_path(artist: str) -> str or None: """ Returns a direct link to an artist image. @@ -185,6 +61,7 @@ def fetch_image_path(artist: str) -> str or None: return None +@helpers.background def fetch_artist_images(): """Downloads the artists images""" @@ -238,3 +115,8 @@ def fetch_album_bio(title: str, albumartist: str): bio = None return bio + + +# TODO +# - Move the populate function to a new file and probably into a new class +# - Start movement from functional programming to OOP to OOP diff --git a/server/app/imgserver/__init__.py b/server/app/imgserver/__init__.py index 7e751a3d..f747e325 100644 --- a/server/app/imgserver/__init__.py +++ b/server/app/imgserver/__init__.py @@ -23,12 +23,11 @@ def hello(): @app.route("/thumb/") def send_thumbnail(path: str): - name = path + ".webp" - path = join(THUMB_PATH, name) - exists = os.path.exists(path) + fpath = join(THUMB_PATH, path) + exists = os.path.exists(fpath) if exists: - return send_from_directory(THUMB_PATH, name) + return send_from_directory(THUMB_PATH, path) return {"msg": "Not found"}, 404 @@ -36,12 +35,11 @@ def send_thumbnail(path: str): @app.route("/artist/") def send_artist_image(path: str): print(ARTIST_PATH) - name = path + ".webp" - path = join(ARTIST_PATH, name) - exists = os.path.exists(path) + fpath = join(ARTIST_PATH, path) + exists = os.path.exists(fpath) if exists: - return send_from_directory(ARTIST_PATH, name) + return send_from_directory(ARTIST_PATH, path) return {"msg": "Not found"}, 404 diff --git a/server/app/lib/albumslib.py b/server/app/lib/albumslib.py index 3433bff0..5633eedf 100644 --- a/server/app/lib/albumslib.py +++ b/server/app/lib/albumslib.py @@ -114,17 +114,15 @@ def get_album_image(album: list) -> str: Gets the image of an album. """ - uri = settings.IMG_THUMB_URI - for track in album: img_p = gen_random_path() exists = taglib.extract_thumb(track["filepath"], webp_path=img_p) if exists: - return uri + img_p + return img_p - return uri + use_defaults() + return use_defaults() def get_album_tracks(album: str, artist: str) -> List: diff --git a/server/app/lib/populate.py b/server/app/lib/populate.py new file mode 100644 index 00000000..e068928c --- /dev/null +++ b/server/app/lib/populate.py @@ -0,0 +1,171 @@ +from dataclasses import asdict +from app.helpers import run_fast_scandir +from app import settings +from app.instances import tracks_instance, album_instance +from progress.bar import Bar +from app.logger import Log +from app.lib.taglib import get_tags +from os import path +from app.lib.albumslib import find_album, create_album +from app import api +from app.models import Track +from app.lib import folderslib + + +class Populate: + """ + Populate the database with all songs in the music directory + + checks if the song is in the database, if not, it adds it + also checks if the album art exists in the image path, if not tries to + extract it. + """ + + def __init__(self) -> None: + self.files = [] + self.db_tracks = [] + self.tagged_tracks = [] + self.folders = set() + self.pre_albums = [] + self.albums = [] + + self.files = run_fast_scandir(settings.HOME_DIR, [".flac", ".mp3"])[1] + self.db_tracks = tracks_instance.get_all_tracks() + + def run(self): + self.check_untagged() + + if len(self.files) == 0: + return + + self.tag_files() + self.create_pre_albums() + self.create_albums() + self.create_tracks() + self.create_folders() + + def check_untagged(self): + """ + Loops through all the tracks in db tracks removing each + from the list of tagged tracks if it exists. + We will now only have untagged tracks left in `files`. + """ + bar = Bar("Checking untagged", max=len(self.db_tracks)) + for track in self.db_tracks: + if track["filepath"] in self.files: + self.files.remove(track["filepath"]) + bar.next() + + bar.finish() + Log(f"Found {len(self.files)} untagged tracks") + + def tag_files(self): + """ + Loops through all the untagged files and tags them. + """ + bar = Bar("Tagging files", max=len(self.files)) + for file in self.files: + tags = get_tags(file) + folder = path.dirname(file) + self.folders.add(folder) + + if tags is not None: + self.tagged_tracks.append(tags) + api.DB_TRACKS.append(tags) + + bar.next() + bar.finish() + Log(f"Tagged {len(self.tagged_tracks)} files") + + def create_pre_albums(self): + """ + Creates pre-albums for the all tagged tracks. + """ + bar = Bar("Creating pre-albums", max=len(self.tagged_tracks)) + for track in self.tagged_tracks: + album = {"title": track["album"], "artist": track["albumartist"]} + + if album not in self.pre_albums: + self.pre_albums.append(album) + bar.next() + + bar.finish() + Log(f"Created {len(self.pre_albums)} pre-albums") + + def create_albums(self): + """ + Uses the pre-albums to create new albums and add them to the database. + """ + exist_count = 0 + + bar = Bar("Creating albums", max=len(self.pre_albums)) + for album in self.pre_albums: + index = find_album(album["title"], album["artist"]) + + if index is None: + try: + track = [ + track + for track in self.tagged_tracks + if track["album"] == album["title"] + and track["albumartist"] == album["artist"] + ][0] + + album = create_album(track) + api.ALBUMS.append(album) + self.albums.append(album) + + album_instance.insert_album(asdict(album)) + + except IndexError: + print("😠\n") + print(album) + + else: + exist_count += 1 + + bar.next() + bar.finish() + Log( + f"{exist_count} of {len(self.pre_albums)} albums were already in the database" + ) + + def create_tracks(self): + """ + Loops through all the tagged tracks creating complete track objects using the `models.Track` model. + """ + bar = Bar("Creating tracks", max=len(self.tagged_tracks)) + failed_count = 0 + for track in self.tagged_tracks: + try: + album_index = find_album(track["album"], track["albumartist"]) + album = api.ALBUMS[album_index] + track["image"] = album.image + upsert_id = tracks_instance.insert_song(track) + track["_id"] = {"$oid": str(upsert_id)} + api.TRACKS.append(Track(track)) + except: + # Bug: some albums are not found although they exist in `api.ALBUMS`. It has something to do with the bisection method used or sorting. Not sure yet. + failed_count += 1 + bar.next() + bar.finish() + + Log( + f"Added {len(self.tagged_tracks) - failed_count} of {len(self.tagged_tracks)} new tracks and {len(self.albums)} new albums" + ) + + def create_folders(self): + """ + Creates the folder objects for all the tracks. + """ + bar = Bar("Creating folders", max=len(self.folders)) + old_f_count = len(api.FOLDERS) + for folder in self.folders: + api.VALID_FOLDERS.add(folder) + fff = folderslib.create_folder(folder) + api.FOLDERS.append(fff) + bar.next() + + bar.finish() + + Log(f"Created {len(self.folders)} new folders") diff --git a/server/app/lib/trackslib.py b/server/app/lib/trackslib.py index 14751552..1aa25373 100644 --- a/server/app/lib/trackslib.py +++ b/server/app/lib/trackslib.py @@ -26,8 +26,11 @@ def create_all_tracks() -> List[models.Track]: except FileNotFoundError: instances.tracks_instance.remove_song_by_id(track["_id"]["$oid"]) api.DB_TRACKS.remove(track) + try: + tracks.append(models.Track(track)) + except KeyError: + print(track) - tracks.append(models.Track(track)) _bar.next() _bar.finish() diff --git a/server/app/models.py b/server/app/models.py index 862c6f7b..89686a2e 100644 --- a/server/app/models.py +++ b/server/app/models.py @@ -32,6 +32,7 @@ class Track: discnumber: int def __init__(self, tags): + self.trackid = tags["_id"]["$oid"] self.title = tags["title"] self.artists = tags["artists"].split(", ") From 559b36bd7b792cbdae16af91ec08ae1e5738f503 Mon Sep 17 00:00:00 2001 From: "restyled-io[bot]" <32688539+restyled-io[bot]@users.noreply.github.com> Date: Wed, 4 May 2022 01:42:26 +0300 Subject: [PATCH 2/2] Restyle Move populate to new file (#48) --- server/app/functions.py | 13 ++++++------- server/app/imgserver/__init__.py | 4 +++- server/app/lib/albumslib.py | 11 +++++------ server/app/lib/populate.py | 32 ++++++++++++++++---------------- server/app/models.py | 2 +- server/setup/nginx.setup.sh | 2 +- server/start.sh | 5 ++--- 7 files changed, 34 insertions(+), 35 deletions(-) diff --git a/server/app/functions.py b/server/app/functions.py index e94440f3..6ad8e2ab 100644 --- a/server/app/functions.py +++ b/server/app/functions.py @@ -10,9 +10,9 @@ from app import api from app import helpers from app import settings from app.lib import watchdoge +from app.lib.populate import Populate from PIL import Image from progress.bar import Bar -from app.lib.populate import Populate @helpers.background @@ -76,9 +76,8 @@ def fetch_artist_images(): _bar = Bar("Processing images", max=len(artists)) for artist in artists: - file_path = ( - helpers.app_dir + "/images/artists/" + artist.replace("/", "::") + ".webp" - ) + file_path = (helpers.app_dir + "/images/artists/" + + artist.replace("/", "::") + ".webp") if not os.path.exists(file_path): img_path = fetch_image_path(artist) @@ -100,8 +99,7 @@ def fetch_album_bio(title: str, albumartist: str): Returns the album bio for a given album. """ last_fm_url = "http://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key={}&artist={}&album={}&format=json".format( - settings.LAST_FM_API_KEY, albumartist, title - ) + settings.LAST_FM_API_KEY, albumartist, title) try: response = requests.get(last_fm_url) @@ -110,7 +108,8 @@ def fetch_album_bio(title: str, albumartist: str): return None try: - bio = data["album"]["wiki"]["summary"].split(' int or None: iter += 1 mid = (left + right) // 2 - if api.ALBUMS[mid].title == albumtitle and api.ALBUMS[mid].artist == artist: + if api.ALBUMS[mid].title == albumtitle and api.ALBUMS[ + mid].artist == artist: return mid if api.ALBUMS[mid].title < albumtitle: @@ -155,8 +155,7 @@ def create_album(track) -> models.Album: album["date"] = album_tracks[0]["date"] album["artistimage"] = urllib.parse.quote_plus( - album_tracks[0]["albumartist"] + ".webp" - ) + album_tracks[0]["albumartist"] + ".webp") album["image"] = get_album_image(album_tracks) diff --git a/server/app/lib/populate.py b/server/app/lib/populate.py index e068928c..4a929471 100644 --- a/server/app/lib/populate.py +++ b/server/app/lib/populate.py @@ -1,15 +1,18 @@ from dataclasses import asdict -from app.helpers import run_fast_scandir -from app import settings -from app.instances import tracks_instance, album_instance -from progress.bar import Bar -from app.logger import Log -from app.lib.taglib import get_tags from os import path -from app.lib.albumslib import find_album, create_album + from app import api -from app.models import Track +from app import settings +from app.helpers import run_fast_scandir +from app.instances import album_instance +from app.instances import tracks_instance from app.lib import folderslib +from app.lib.albumslib import create_album +from app.lib.albumslib import find_album +from app.lib.taglib import get_tags +from app.logger import Log +from app.models import Track +from progress.bar import Bar class Populate: @@ -105,8 +108,7 @@ class Populate: if index is None: try: track = [ - track - for track in self.tagged_tracks + track for track in self.tagged_tracks if track["album"] == album["title"] and track["albumartist"] == album["artist"] ][0] @@ -126,9 +128,8 @@ class Populate: bar.next() bar.finish() - Log( - f"{exist_count} of {len(self.pre_albums)} albums were already in the database" - ) + Log(f"{exist_count} of {len(self.pre_albums)} albums were already in the database" + ) def create_tracks(self): """ @@ -150,9 +151,8 @@ class Populate: bar.next() bar.finish() - Log( - f"Added {len(self.tagged_tracks) - failed_count} of {len(self.tagged_tracks)} new tracks and {len(self.albums)} new albums" - ) + Log(f"Added {len(self.tagged_tracks) - failed_count} of {len(self.tagged_tracks)} new tracks and {len(self.albums)} new albums" + ) def create_folders(self): """ diff --git a/server/app/models.py b/server/app/models.py index 89686a2e..27e8c45b 100644 --- a/server/app/models.py +++ b/server/app/models.py @@ -32,7 +32,7 @@ class Track: discnumber: int def __init__(self, tags): - + self.trackid = tags["_id"]["$oid"] self.title = tags["title"] self.artists = tags["artists"].split(", ") diff --git a/server/setup/nginx.setup.sh b/server/setup/nginx.setup.sh index 5c9dedd1..10026f6f 100755 --- a/server/setup/nginx.setup.sh +++ b/server/setup/nginx.setup.sh @@ -2,4 +2,4 @@ # 2. create symbolic links for each file in sites-available to sites-enable sudo cp ./nginx-sites/* /etc/nginx/sites-available -sudo ln -s /etc/nginx/sites-available/* /etc/nginx/sites-enabled -f \ No newline at end of file +sudo ln -s /etc/nginx/sites-available/* /etc/nginx/sites-enabled -f diff --git a/server/start.sh b/server/start.sh index 0b7bede2..b86f0ca5 100755 --- a/server/start.sh +++ b/server/start.sh @@ -6,8 +6,7 @@ gpath=$(poetry run which gunicorn) cd app -$gpath -b 0.0.0.0:9877 -w 4 --threads=2 "imgserver:app" & +"$gpath" -b 0.0.0.0:9877 -w 4 --threads=2 "imgserver:app" & echo "Booted image server" cd ../ -$gpath -b 0.0.0.0:9876 -w 1 --threads=4 "manage:create_app()" #--log-level=debug - +"$gpath" -b 0.0.0.0:9876 -w 1 --threads=4 "manage:create_app()" #--log-level=debug