fix: folder endpoint returning same track for different files of the same trackhash

+ fix: chunked streaming. return instead of yield chunks

app/lib/tagger.py

@@ -8,11 +8,14 @@ from app.models.album import Album
 from app.models.artist import Artist
 from app.models.track import Track
 from app.store.folder import FolderStore
+from app.store.tracks import TrackStore
+from app.utils import flatten
 from app.utils.filesystem import run_fast_scandir
 from app.utils.parsers import get_base_album_title
 from app.utils.progressbar import tqdm
 
 from app.logger import log
+from app.utils.remove_duplicates import remove_duplicates
 
 POPULATE_KEY: float = 0
 
@@ -136,9 +139,27 @@ class IndexTracks:
         print("Done")
 
 
-def create_albums():
+def create_albums(_trackhashes: list[str] = []) -> list[tuple[Album, set[str]]]:
+    """
+    Creates album objects using the indexed tracks. Takes in an optional
+    list of trackhashes to create the albums from. If no list is provided,
+    all tracks are used.
+
+    The trackhashes are passed when creating albums from the watchdogg module.
+
+    Returns a list of tuples containing the album and the trackhashes in the album.
+    ie:
+
+    >>> list[tuple[Album, set[str]]]
+    """
     albums = dict()
-    all_tracks: list[Track] = TrackTable.get_all()
+
+    if _trackhashes:
+        all_tracks: list[Track] = TrackStore.get_tracks_by_trackhashes(_trackhashes)
+    else:
+        all_tracks: list[Track] = TrackStore.get_flat_list()
+
+    all_tracks = remove_duplicates(all_tracks)
 
     for track in all_tracks:
         if track.albumhash not in albums:
@@ -192,9 +213,28 @@ def create_albums():
     return list(albums.values())
 
 
-# class IndexArtists:
-def create_artists():
-    all_tracks: list[Track] = TrackTable.get_all()
+def create_artists(
+    artisthashes: list[str] = [],
+) -> list[tuple[Artist, set[str], set[str]]]:
+    """
+    Creates artist objects using the indexed tracks. Takes in an optional
+    list of artisthashes to create the artists from. If no list is provided,
+    all tracks are used.
+
+    Returns a list of tuples containing the artist, the trackhashes for the artist
+    and the albumhashes for the artist.
+    ie:
+
+    >>> list[tuple[Artist, set[str], set[str]]]
+    """
+    if artisthashes:
+        all_tracks: list[Track] = flatten(
+            [TrackStore.get_tracks_by_artisthash(hash) for hash in artisthashes]
+        )
+    else:
+        all_tracks: list[Track] = TrackStore.get_flat_list()
+
+    all_tracks = remove_duplicates(all_tracks)
     artists = dict()
 
     for track in all_tracks:

app/lib/taglib.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+import math
 import os
 from io import BytesIO
 from pathlib import Path
@@ -206,7 +207,7 @@ def get_tags(filepath: str, config: UserConfig):
     to_round = ["bitrate", "duration"]
     for prop in to_round:
         try:
-            setattr(tags, prop, round(getattr(tags, prop)))
+            setattr(tags, prop, math.floor(getattr(tags, prop)))
         except TypeError:
             setattr(tags, prop, 0)
 

app/lib/transcoder.py

@@ -0,0 +1,78 @@
+from app.utils.threading import background
+
+
+import subprocess
+
+
+@background
+def start_transcoding(
+    input_path: str, output_path: str, bitrate: str, container_args: list[str], compression_level: int = 12
+):
+    """
+    Starts a background transcoding process for an audio file.
+
+    This function uses FFmpeg to transcode an audio file from one format to another,
+    with specified bitrate and container format. It runs as a background task.
+
+    Args:
+        input_path (str): The path to the input audio file.
+        output_path (str): The path where the transcoded file will be saved.
+        bitrate (str): The desired bitrate for the output file (e.g., "128k").
+        container_args (list[str]): FFmpeg arguments specific to the output container format.
+        compression_level (int): Compression level (0-9, default: 6).
+
+    Returns:
+        None
+
+    Note:
+        This function is decorated with @background, which means it runs asynchronously.
+        The actual transcoding process is handled by FFmpeg in a subprocess.
+        The function will print status messages about the transcoding process.
+    """
+    # Base command
+    command = [
+        "ffmpeg",
+        "-i",
+        input_path,
+        "-map_metadata", "0",  # Add this line to copy metadata
+        "-b:a",
+        bitrate,
+        "-vn",
+        "-compression_level",
+        str(compression_level),
+        # REVIEW: Idk what any flag below this point does!
+        "-movflags",
+        "faststart+frag_keyframe+empty_moov",
+        "-write_xing",
+        "0",
+        "-fflags",
+        "+bitexact",
+    ]
+
+    # Add format-specific parameters
+    command.extend(container_args)
+
+    # Add output path and overwrite flag
+    command.extend([output_path, "-y"])
+
+    process = subprocess.Popen(
+        command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    )
+    print(f"Started transcoding process with PID: {process.pid}")
+
+    try:
+        # Wait for the process to complete
+        process.wait()
+        print(f"Transcoding process (PID: {process.pid}) completed")
+    except KeyboardInterrupt:
+        print(f"Transcoding interrupted. Terminating process (PID: {process.pid})")
+    finally:
+        # Ensure the process is terminated
+        try:
+            process.terminate()
+            process.wait(timeout=5)  # Wait up to 5 seconds for graceful termination
+        except subprocess.TimeoutExpired:
+            print(
+                f"Process (PID: {process.pid}) did not terminate gracefully. Killing..."
+            )
+            process.kill()

app/lib/watchdogg.py

@@ -8,17 +8,20 @@ import sqlite3
 import time
 
 from watchdog.events import PatternMatchingEventHandler
+from watchdog.observers.api import BaseObserverSubclassCallable
 from watchdog.observers import Observer
 
 from app import settings
 from app.config import UserConfig
-from app.db.sqlite.tracks import SQLiteManager
+from app.db.libdata import TrackTable
+from app.db.userdata import LibDataTable
 from app.lib.colorlib import process_color
+from app.lib.tagger import create_albums, create_artists
 from app.lib.taglib import extract_thumb, get_tags
 from app.logger import log
 from app.models import Artist, Track
 from app.store.albums import AlbumStore
-from app.store.artists import ArtistStore
+from app.store.artists import ArtistMapEntry, ArtistStore
 from app.store.tracks import TrackStore
 
 
@@ -27,7 +30,7 @@ class Watcher:
     Contains the methods for initializing and starting watchdog.
     """
 
-    observers: list[Observer] = []
+    observers: list[BaseObserverSubclassCallable] = []
 
     def __init__(self):
         self.observer = Observer()
@@ -126,10 +129,10 @@ class Watcher:
         self.run()
 
 
-def handle_colors(cur: sqlite3.Cursor, albumhash: str):
-    exists = aldb.exists(albumhash, cur)
+def handle_color(albumhash: str):
+    entry = LibDataTable.find_one(albumhash, "album")
 
-    if exists:
+    if entry and entry.color:
         return
 
     colors = process_color(albumhash, is_album=True)
@@ -137,7 +140,12 @@ def handle_colors(cur: sqlite3.Cursor, albumhash: str):
     if colors is None:
         return
 
-    aldb.insert_one_album(cur=cur, albumhash=albumhash, colors=json.dumps(colors))
+    if entry is None:
+        LibDataTable.insert_one(
+            {"itemhash": albumhash, "color": colors[0], "itemtype": "album"}
+        )
+    else:
+        LibDataTable.update_one(albumhash, {"color": colors[0]})
 
     return colors
 
@@ -152,38 +160,42 @@ def add_track(filepath: str) -> None:
     TrackStore.remove_track_by_filepath(filepath)
 
     config = UserConfig()
-    tags = get_tags(filepath, artist_separators=config.artistSeparators)
+    tags = get_tags(filepath, config)
 
     # if the track is somehow invalid, return
     if tags is None or tags["bitrate"] == 0 or tags["duration"] == 0:
         return
 
-    colors = None
-
-    with SQLiteManager() as cur:
-        db.insert_one_track(tags, cur)
-        extracted = extract_thumb(filepath, tags["albumhash"] + ".webp")
-
-        if not extracted:
-            return
-
-        colors = handle_colors(cur, tags["albumhash"])
+    TrackTable.insert_one(tags)
+    extract_thumb(filepath, tags["albumhash"] + ".webp", overwrite=True)
 
+    colors = handle_color(tags["albumhash"])
     track = Track(**tags)
     TrackStore.add_track(track)
 
-    if not AlbumStore.album_exists(track.albumhash):
-        album = AlbumStore.create_album(track)
-        album.set_colors(colors)
-        AlbumStore.add_album(album)
+    # SECTION: Index album
+    albumentry = AlbumStore.albummap.get(track.albumhash)
 
-    artists: list[Artist] = track.artists + track.albumartists  # type: ignore
+    if albumentry is None:
+        album, trackhashes = create_albums([track.trackhash])[0]
+        AlbumStore.index_new_album(album, trackhashes)
+    else:
+        trackhash_exists = track.trackhash in albumentry.trackhashes
+
+        if not trackhash_exists:
+            albumentry.trackhashes.add(track.trackhash)
+            albumentry.album.trackcount += 1
+            albumentry.set_color(colors[0]) if colors else None
+
+    # SECTION: Index artist
+    artists = create_artists(track.artisthashes)
 
     for artist in artists:
-        if not ArtistStore.artist_exists(artist.artisthash):
-            ArtistStore.add_artist(Artist(artist.name))
-
-    extract_thumb(filepath, track.image, overwrite=True)
+        ArtistStore.artistmap[artist[0].artisthash] = ArtistMapEntry(
+            artist=artist[0],
+            albumhashes=artist[1],
+            trackhashes=artist[2],
+        )
 
 
 def remove_track(filepath: str) -> None:
@@ -287,16 +299,33 @@ class Handler(PatternMatchingEventHandler):
         NOT FIRED IN WINDOWS
         """
         try:
-            self.files_to_process.remove(event.src_path)
-            if os.path.getsize(event.src_path) > 0:
+            # Get initial file size
+            initial_size = os.path.getsize(event.src_path)
+
+            # Wait for 10 seconds
+            time.sleep(10)
+
+            # Check if file size has changed
+            current_size = os.path.getsize(event.src_path)
+
+            if current_size > 0 and current_size == initial_size:
                 path = self.get_abs_path(event.src_path)
                 add_track(path)
+                # Remove from processing list only after successful processing
+                self.files_to_process.remove(event.src_path)
+            else:
+                # File is still being modified or has been deleted
+                log.info(
+                    f"File {event.src_path} is still being modified. Skipping processing for now."
+                )
         except FileNotFoundError:
             # file was closed and deleted.
-            pass
+            log.info(f"File {event.src_path} was closed and deleted before processing.")
         except ValueError:
-            # file was removed from the list by another event handler.
-            pass
+            # file was already removed from the list by another event handler.
+            log.info(
+                f"File {event.src_path} was already removed from the processing list."
+            )
 
     def on_modified(self, event):
         # this event handler is triggered twice on windows
@@ -317,7 +346,7 @@ class Handler(PatternMatchingEventHandler):
 
         if current_size == previous_size:
             # Wait for a short duration to ensure the file write operation is complete
-            time.sleep(5)
+            time.sleep(10)
 
             # Check the file size again
             try: