diff --git a/.gitignore b/.gitignore index 7b09bc4d..7cbee9ee 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ logs.txt TODO.md testdata.py -test.py \ No newline at end of file +test.py +nohup.out diff --git a/.trunk/configs/.hadolint.yaml b/.trunk/configs/.hadolint.yaml new file mode 100644 index 00000000..98bf0cd2 --- /dev/null +++ b/.trunk/configs/.hadolint.yaml @@ -0,0 +1,4 @@ +# Following source doesn't work in most setups +ignored: + - SC1090 + - SC1091 diff --git a/.trunk/configs/.isort.cfg b/.trunk/configs/.isort.cfg new file mode 100644 index 00000000..b9fb3f3e --- /dev/null +++ b/.trunk/configs/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile=black diff --git a/.trunk/configs/.markdownlint.yaml b/.trunk/configs/.markdownlint.yaml new file mode 100644 index 00000000..fb940393 --- /dev/null +++ b/.trunk/configs/.markdownlint.yaml @@ -0,0 +1,10 @@ +# Autoformatter friendly markdownlint config (all formatting rules disabled) +default: true +blank_lines: false +bullet: false +html: false +indentation: false +line_length: false +spaces: false +url: false +whitespace: false diff --git a/.trunk/configs/.shellcheckrc b/.trunk/configs/.shellcheckrc new file mode 100644 index 00000000..8c7b1ada --- /dev/null +++ b/.trunk/configs/.shellcheckrc @@ -0,0 +1,7 @@ +enable=all +source-path=SCRIPTDIR +disable=SC2154 + +# If you're having issues with shellcheck following source, disable the errors via: +# disable=SC1090 +# disable=SC1091 diff --git a/.trunk/configs/.yamllint.yaml b/.trunk/configs/.yamllint.yaml new file mode 100644 index 00000000..4d444662 --- /dev/null +++ b/.trunk/configs/.yamllint.yaml @@ -0,0 +1,10 @@ +rules: + quoted-strings: + required: only-when-needed + extra-allowed: ["{|}"] + empty-values: + forbid-in-block-mappings: true + forbid-in-flow-mappings: true + key-duplicates: {} + octal-values: + forbid-implicit-octal: true diff --git a/.trunk/configs/ruff.toml b/.trunk/configs/ruff.toml new file mode 100644 index 00000000..f5a235cf --- /dev/null +++ b/.trunk/configs/ruff.toml @@ -0,0 +1,5 @@ +# Generic, formatter-friendly config. +select = ["B", "D3", "E", "F"] + +# Never enforce `E501` (line length violations). This should be handled by formatters. +ignore = ["E501"] diff --git a/.trunk/configs/svgo.config.js b/.trunk/configs/svgo.config.js new file mode 100644 index 00000000..b257d134 --- /dev/null +++ b/.trunk/configs/svgo.config.js @@ -0,0 +1,14 @@ +module.exports = { + plugins: [ + { + name: "preset-default", + params: { + overrides: { + removeViewBox: false, // https://github.com/svg/svgo/issues/1128 + sortAttrs: true, + removeOffCanvasPaths: true, + }, + }, + }, + ], +}; diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml new file mode 100644 index 00000000..f742b1d5 --- /dev/null +++ b/.trunk/trunk.yaml @@ -0,0 +1,46 @@ +# This file controls the behavior of Trunk: https://docs.trunk.io/cli +# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml +version: 0.1 +cli: + version: 1.19.0 +# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) +plugins: + sources: + - id: trunk + ref: v1.4.2 + uri: https://github.com/trunk-io/plugins +# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) +runtimes: + enabled: + - go@1.21.0 + - node@18.12.1 + - python@3.10.8 +# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) +lint: + enabled: + - actionlint@1.6.26 + - bandit@1.7.7 + - black@24.1.1 + - checkov@3.2.1 + - git-diff-check + - hadolint@2.12.0 + - isort@5.13.2 + - markdownlint@0.39.0 + - osv-scanner@1.6.1 + - prettier@3.2.4 + - ruff@0.1.15 + - shellcheck@0.9.0 + - shfmt@3.6.0 + - svgo@3.2.0 + - taplo@0.8.1 + - terrascan@1.18.11 + - trivy@0.48.3 + - trufflehog@3.66.1 + - yamllint@1.33.0 +actions: + disabled: + - trunk-announce + - trunk-check-pre-push + - trunk-fmt-pre-commit + enabled: + - trunk-upgrade-available diff --git a/app/api/settings.py b/app/api/settings.py index 9d387ac9..e5d3306b 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -59,7 +59,8 @@ def rebuild_store(db_dirs: list[str]): try: populate.Populate(instance_key=instance_key) - except populate.PopulateCancelledError: + except populate.PopulateCancelledError as e: + print(e) reload_everything(instance_key) return @@ -68,7 +69,7 @@ def rebuild_store(db_dirs: list[str]): log.info("Rebuilding library... ✅") -# I freaking don't know what this function does anymore +# I freaking don't know what this function does anymore def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]): """ Params: diff --git a/app/arg_handler.py b/app/arg_handler.py index 64367343..7cae6f7d 100644 --- a/app/arg_handler.py +++ b/app/arg_handler.py @@ -57,8 +57,8 @@ class HandleArgs: value = settings.Keys.get(key) if not value: - log.error(f"ERROR: {key} not set in environment") - sys.exit(0) + log.error(f"WARNING: {key} not set in environment") + #sys.exit(0) lines.append(f'{key} = "{value}"\n') diff --git a/app/lib/populate.py b/app/lib/populate.py index dcb5a692..48ede325 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -104,6 +104,7 @@ class Populate: log.error( "Internet connection lost. Downloading artist images suspended." ) + log.error(e) # REVIEW More informations = good else: log.warning(f"No internet connection. Downloading artist images suspended!") @@ -113,6 +114,7 @@ class Populate: if has_connection(): try: + print("Attempting to download similar artists...") FetchSimilarArtistsLastFM(instance_key) except PopulateCancelledError as e: log.warn(e) @@ -135,6 +137,7 @@ class Populate: unmodified_paths.add(track.filepath) continue except (FileNotFoundError, OSError) as e: + log.warning(e) # REVIEW More informations = good TrackStore.remove_track_obj(track) remove_tracks_by_filepaths(track.filepath) @@ -286,6 +289,7 @@ def save_similar_artists(_map: tuple[str, Artist]): instance_key, artist = _map if POPULATE_KEY != instance_key: + print("Warning: Populate key changed") raise PopulateCancelledError( "'FetchSimilarArtistsLastFM': Populate key changed" ) @@ -321,6 +325,7 @@ class FetchSimilarArtistsLastFM: with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: try: + print("Processing similar artists") results = list( tqdm( executor.map(save_similar_artists, key_artist_map), diff --git a/app/requests/artists.py b/app/requests/artists.py index abaa6c2d..cca86563 100644 --- a/app/requests/artists.py +++ b/app/requests/artists.py @@ -6,7 +6,7 @@ import urllib.parse import requests from requests import ConnectionError, HTTPError, ReadTimeout -from app import settings +#from app import settings from app.utils.hashing import create_hash @@ -16,6 +16,10 @@ def fetch_similar_artists(name: str): """ url = f"https://kerve.last.fm/kerve/similarartists?artist={urllib.parse.quote_plus(name, safe='')}&autocorrect=1&tracks=1&image_size=large&limit=250&format=json" + # REVIEW This is the old way of doing it. The new way is to use the Kerve API. + #url = f"https://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={urllib.parse.quote_plus(name, safe='')}&api_key={settings.Keys.LASTFM_API_KEY}&format=json&limit=250" + # TODO Cannot be tested due to PR message + url = f"https://kerve.last.fm/kerve/similarartists?artist={urllib.parse.quote_plus(name, safe='')}&autocorrect=1&tracks=1&image_size=large&limit=250&format=json" try: response = requests.get(url, timeout=10) response.raise_for_status() diff --git a/app/settings.py b/app/settings.py index 5cddc750..5d66643c 100644 --- a/app/settings.py +++ b/app/settings.py @@ -275,6 +275,9 @@ class Keys: @classmethod def load(cls): + # TODO Remove this. Just an handy flag to test the app without the API key + # IS_BUILD = True + if IS_BUILD: cls.SWINGMUSIC_APP_VERSION = configs.SWINGMUSIC_APP_VERSION cls.GIT_LATEST_COMMIT_HASH = configs.GIT_LATEST_COMMIT_HASH diff --git a/app/utils/hashing.py b/app/utils/hashing.py index 0dfebf3c..e9ed147d 100644 --- a/app/utils/hashing.py +++ b/app/utils/hashing.py @@ -32,5 +32,11 @@ def create_hash(*args: str, decode=False, limit=10) -> str: str_ = unidecode(str_) str_ = str_.encode("utf-8") - str_ = hashlib.sha256(str_).hexdigest() - return str_[-limit:] + str_ = hashlib.sha1(str_).hexdigest() + # REVIEW Switched to sha1 hashlib.sha256(str_).hexdigest() + + # REVIEW Take the first limit/2 and last limit/2 characters + # This is to avoid collisions + return str_[:limit // 2] + str_[-limit // 2:] if limit % 2 == 0 else str_[:limit // 2] + str_[-limit // 2 - 1:] + + # return str_[-limit:] diff --git a/app/utils/network.py b/app/utils/network.py index becb11bb..e4f1467b 100644 --- a/app/utils/network.py +++ b/app/utils/network.py @@ -3,10 +3,12 @@ import socket as Socket def has_connection(host="google.it", port=80, timeout=3): """ + # REVIEW Was: Host: 8.8.8.8 (google-public-dns-a.google.com) OpenPort: 53/tcp Service: domain (DNS/TCP) """ + try: Socket.setdefaulttimeout(timeout) Socket.socket(Socket.AF_INET, Socket.SOCK_STREAM).connect((host, port)) diff --git a/build.sh b/build.sh index fc397746..ba5c7edb 100755 --- a/build.sh +++ b/build.sh @@ -1,9 +1,19 @@ -#!/bin/zsh +#!/bin/bash +# REVIEW Above: bash is way more compatible than other shells # builds the latest version of the client and server -cd ../swingmusic-client -yarn build --outDir ../swingmusic/client +# REVIEW These are not useful if you dont have the source code +#cd ../swingmusic-client +#yarn build --outDir ../swingmusic/client +#../swingmusic -cd ../swingmusic +# REVIEW Cleaning up +rm -rf build dist + +# REVIEW Install poetry & requirements +poetry || pip install poetry +poetry install --no-root + +# Build the app poetry run python manage.py --build \ No newline at end of file diff --git a/dev_build.sh b/dev_build.sh new file mode 100644 index 00000000..edc8e958 --- /dev/null +++ b/dev_build.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# REVIEW Above: bash is way more compatible than other shells + +# builds the latest version of the client and server + +# NOTE Changes directory to the webclient directory and builds it +cd ../swingmusic-client || exit # REVIEW Failsafe exit +yarn build --outDir ../swingmusic/client +cd ../swingmusic || exit # REVIEW Failsafe exit + +# REVIEW Optional cleaning up +# rm -rf build dist + +# REVIEW Install poetry & requirements +# Build the app +poetry run python manage.py --build \ No newline at end of file diff --git a/run b/run new file mode 100755 index 00000000..4795296f --- /dev/null +++ b/run @@ -0,0 +1,3 @@ +#!/bin/bash + +poetry run python manage.py