diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a704fa5..038881a1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,12 +60,6 @@ jobs: - name: Install dependencies run: uv sync --frozen - # Import code reads/writes the Docker image's paths directly - - name: Create container paths used by the app - run: | - sudo mkdir -p /usr/src/app/temp /usr/src/app/import_presets - sudo chown -R "$(id -u):$(id -g)" /usr/src/app - - name: Check for missing migrations working-directory: app run: uv run python manage.py makemigrations --check --dry-run diff --git a/.gitignore b/.gitignore index 93ba54a4..7c4160ef 100644 --- a/.gitignore +++ b/.gitignore @@ -168,3 +168,4 @@ postgres_data/ # Private local uploads app/attachments/ +app/temp/ diff --git a/README.md b/README.md index 9154a734..ba019170 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ To create the first user, open the container's console using Unraid's UI, by cli | ENABLE_SOFT_DELETE | true\|false | false | Whether to enable transactions soft delete, if enabled, deleted transactions will remain in the database. Useful for imports and avoiding duplicate entries. | | KEEP_DELETED_TRANSACTIONS_FOR | int | 365 | Time in days to keep soft deleted transactions for. If 0, will keep all transactions indefinitely. Only works if ENABLE_SOFT_DELETE is true. | | TASK_WORKERS | int | 1 | How many workers to have for async tasks. One should be enough for most use cases | +| IMPORT_TEMP_ROOT | string | /usr/src/app/temp | Where uploaded import files wait to be processed. Defaults to the `temp` folder inside the app folder. Must be writable and shared between the web and task processes. | +| CACHE_ROOT | string | /var/tmp | The writable folder the file-based caches are stored in. | | DEMO | true\|false | false | If demo mode is enabled. | | ADMIN_EMAIL | string | None | Automatically creates an admin account with this email. Must have `ADMIN_PASSWORD` also set. | | ADMIN_PASSWORD | string | None | Automatically creates an admin account with this password. Must have `ADMIN_EMAIL` also set. | diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 24393480..2453e55e 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -314,6 +314,9 @@ LOCALE_PATHS = [BASE_DIR / "locale"] STATIC_URL = "static/" STATIC_ROOT = BASE_DIR / "static_files" ATTACHMENT_MEDIA_ROOT = BASE_DIR / "attachments" +IMPORT_PRESETS_ROOT = BASE_DIR / "import_presets" +# Web and task workers must see the same folder: uploads wait here to be processed +IMPORT_TEMP_ROOT = Path(os.getenv("IMPORT_TEMP_ROOT") or BASE_DIR / "temp").resolve() STATICFILES_DIRS = [ ROOT_DIR / "frontend" / "build", @@ -325,15 +328,17 @@ STATICFILES_FINDERS = [ "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] +CACHE_ROOT = Path(os.getenv("CACHE_ROOT") or "/var/tmp").resolve() + CACHES = { "default": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/var/tmp/django_cache", + "LOCATION": CACHE_ROOT / "django_cache", }, # Kept apart from "default": cachalot fills it and FileBasedCache culls random entries past MAX_ENTRIES "update_check": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/var/tmp/django_cache_update_check", + "LOCATION": CACHE_ROOT / "django_cache_update_check", }, } diff --git a/app/apps/api/views/imports.py b/app/apps/api/views/imports.py index f8e6edb7..9abb945d 100644 --- a/app/apps/api/views/imports.py +++ b/app/apps/api/views/imports.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.core.files.storage import FileSystemStorage from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view, inline_serializer @@ -130,7 +131,7 @@ class ImportViewSet(viewsets.ViewSet): uploaded_file = serializer.validated_data["file"] # Save file to temp location - fs = FileSystemStorage(location="/usr/src/app/temp") + fs = FileSystemStorage(location=settings.IMPORT_TEMP_ROOT) filename = fs.save(uploaded_file.name, uploaded_file) file_path = fs.path(filename) diff --git a/app/apps/common/oauth_views.py b/app/apps/common/oauth_views.py index 54733580..a73b3b84 100644 --- a/app/apps/common/oauth_views.py +++ b/app/apps/common/oauth_views.py @@ -88,7 +88,7 @@ def _dcr_initial_access_token_ok(request): scheme, _, value = header.partition(" ") if scheme.lower() != "bearer" or not value: return False - return hmac.compare_digest(value, expected) + return hmac.compare_digest(value.encode(), expected.encode()) @require_http_methods(["GET"]) diff --git a/app/apps/common/tests/test_oauth.py b/app/apps/common/tests/test_oauth.py index b096ada9..7453c923 100644 --- a/app/apps/common/tests/test_oauth.py +++ b/app/apps/common/tests/test_oauth.py @@ -224,6 +224,18 @@ class DynamicClientRegistrationInitialAccessTokenTests(TestCase): self.assertEqual(response.json()["error"], "invalid_token") self.assertEqual(Application.objects.count(), 0) + def test_rejects_registration_with_non_ascii_initial_access_token(self): + response = self.client.post( + reverse("oauth-dynamic-client-registration"), + data=json.dumps({"redirect_uris": ["http://127.0.0.1:8765/callback"]}), + content_type="application/json", + HTTP_AUTHORIZATION="Bearer s3cret-iat\u00e9", + ) + + self.assertEqual(response.status_code, 401) + self.assertEqual(response.json()["error"], "invalid_token") + self.assertEqual(Application.objects.count(), 0) + def test_allows_registration_with_initial_access_token(self): response = self.client.post( reverse("oauth-dynamic-client-registration"), diff --git a/app/apps/common/widgets/tom_select.py b/app/apps/common/widgets/tom_select.py index 5c9b0548..0f22d106 100644 --- a/app/apps/common/widgets/tom_select.py +++ b/app/apps/common/widgets/tom_select.py @@ -1,3 +1,5 @@ +import json + from django.forms import SelectMultiple, widgets from django.urls import reverse from django.utils.translation import gettext_lazy as _ @@ -142,6 +144,7 @@ class TransactionSelect(TomSelect): def build_attrs(self, base_attrs, extra_attrs=None): attrs = super().build_attrs(base_attrs, extra_attrs) + attrs["data-render"] = "transaction" if self.load_income and self.load_expense: attrs["data-load"] = reverse("transactions_search") @@ -155,3 +158,13 @@ class TransactionSelect(TomSelect): ) return attrs + + def create_option(self, name, value, *args, **kwargs): + option = super().create_option(name, value, *args, **kwargs) + instance = getattr(value, "instance", None) + if instance is not None: + from apps.transactions.utils.select_search import transaction_option + + # tom-select reads this to render the selected transaction like the search results + option["attrs"]["data-data"] = json.dumps(transaction_option(instance)) + return option diff --git a/app/apps/currencies/exchange_rates/base.py b/app/apps/currencies/exchange_rates/base.py index 636d49fd..098ce5bd 100644 --- a/app/apps/currencies/exchange_rates/base.py +++ b/app/apps/currencies/exchange_rates/base.py @@ -8,9 +8,12 @@ from apps.currencies.models import Currency class ExchangeRateProvider(ABC): rates_inverted = False + # Help shown under the service's Options field. None hides the field + options_help = None - def __init__(self, api_key: Optional[str] = None): + def __init__(self, api_key: Optional[str] = None, options: str = ""): self.api_key = api_key + self.options = options or "" @abstractmethod def get_rates( @@ -24,6 +27,11 @@ class ExchangeRateProvider(ABC): """Return True if the service requires an API key""" return True + @classmethod + def clean_options(cls, options: str) -> str: + """Validate the raw Options text on save. Raise ValidationError if invalid""" + return options + @staticmethod def invert_rate(rate: Decimal) -> Decimal: """Invert the given rate.""" diff --git a/app/apps/currencies/exchange_rates/fetcher.py b/app/apps/currencies/exchange_rates/fetcher.py index a84fb4e5..c8450f70 100644 --- a/app/apps/currencies/exchange_rates/fetcher.py +++ b/app/apps/currencies/exchange_rates/fetcher.py @@ -19,6 +19,7 @@ PROVIDER_MAPPING = { "twelvedatamarkets": providers.TwelveDataMarketsProvider, "yfinance": providers.YFinanceMarketsProvider, "fxmacrodata": providers.FXMacroDataProvider, + "allratestoday": providers.AllRatesTodayProvider, } diff --git a/app/apps/currencies/exchange_rates/providers.py b/app/apps/currencies/exchange_rates/providers.py index 4d79d08c..77c86e13 100644 --- a/app/apps/currencies/exchange_rates/providers.py +++ b/app/apps/currencies/exchange_rates/providers.py @@ -1,11 +1,14 @@ import logging +import re import time import requests from decimal import Decimal from typing import Tuple, List, Optional, Dict +from django.core.exceptions import ValidationError from django.db.models import QuerySet +from django.utils.translation import gettext_lazy as _ from apps.currencies.models import Currency, ExchangeRate from apps.currencies.exchange_rates.base import ExchangeRateProvider @@ -681,3 +684,166 @@ class FXMacroDataProvider(ExchangeRateProvider): ) return results + + +class AllRatesTodayProvider(ExchangeRateProvider): + """ + Implementation for the central bank tables served by allratestoday.com. + + Each service reads the latest published table of one central bank, picked by + the bank code in the Options field (e.g. "bcb" or "rbi"). An empty field + reads the ECB table. The endpoint needs no API key. + """ + + BASE_URL = "https://allratestoday.com/api/open/central-bank" + DEFAULT_BANK = "ecb" + BANK_CODE_RE = re.compile(r"[a-z0-9_-]{2,20}") + rates_inverted = False # A base/quote row says how many quote one base buys + options_help = _( + "Central bank code, e.g. 'bcb' or 'rbi'. Leave empty to use the ECB table." + ) + + # A pair can have several rows. Prefer a published mid, else the mean of a buy + # and sell quote. Period averages are left out, they are not a current rate + MID_TYPES = ( + "reference", + "middle", + "official", + "indicative", + "close", + "spot", + "informational", + ) + BUY_SELL_TYPES = (("buy", "sell"), ("tt_buy", "tt_sell")) + + def __init__(self, api_key: str = None, options: str = ""): + super().__init__(api_key, options) + self.session = requests.Session() + + @property + def bank(self) -> str: + return self.options.strip().lower() or self.DEFAULT_BANK + + @classmethod + def requires_api_key(cls) -> bool: + return False + + @classmethod + def clean_options(cls, options: str) -> str: + bank = options.strip().lower() + if bank and not cls.BANK_CODE_RE.fullmatch(bank): + raise ValidationError(_("Invalid bank code, it should look like 'bcb'.")) + return bank + + def _parse_table(self, rows: list) -> Dict[Tuple[str, str], Decimal]: + """Reduce the table to one rate per pair, in both directions""" + grouped = {} + for row in rows: + if not isinstance(row, dict): + continue + try: + value = Decimal(str(row.get("value"))) + except ArithmeticError: + continue + # A bad cell skips its own row, not the whole table + if not value.is_finite() or value <= 0: + continue + pair = (row.get("base"), row.get("quote")) + grouped.setdefault(pair, {})[row.get("type")] = value + + rates = {} + for (base, quote), values in grouped.items(): + rate = next((values[t] for t in self.MID_TYPES if t in values), None) + if rate is None: + rate = next( + ( + (values[buy] + values[sell]) / 2 + for buy, sell in self.BUY_SELL_TYPES + if buy in values and sell in values + ), + None, + ) + if rate is None: + logger.debug( + f"AllRatesToday {self.bank}: no usable rate type for " + f"{base}/{quote}, got {sorted(map(str, values))}" + ) + continue + rates[(base, quote)] = rate + rates.setdefault((quote, base), self.invert_rate(rate)) + + if grouped and not rates: + types = sorted({str(t) for values in grouped.values() for t in values}) + logger.warning( + f"AllRatesToday table for {self.bank} has no usable rate types, " + f"got {types}" + ) + + return rates + + @staticmethod + def _find_rate( + rates: Dict[Tuple[str, str], Decimal], base: str, quote: str + ) -> Optional[Decimal]: + """Look up a pair, crossing it through a shared currency if needed""" + if (base, quote) in rates: + return rates[(base, quote)] + + for (from_code, via), rate in rates.items(): + if from_code == base and (via, quote) in rates: + return rate * rates[(via, quote)] + + return None + + def get_rates( + self, target_currencies: QuerySet, exchange_currencies: set + ) -> List[Tuple[Currency, Currency, Decimal]]: + results = [] + + if not self.BANK_CODE_RE.fullmatch(self.bank): + logger.error("Invalid AllRatesToday bank code, it should look like 'bcb'") + return results + + try: + response = self.session.get(f"{self.BASE_URL}/{self.bank}", timeout=30) + response.raise_for_status() + data = response.json() + + if data.get("stale"): + logger.warning( + f"AllRatesToday table for {self.bank} is stale, " + f"last published on {data.get('rate_date')}" + ) + + rates = self._parse_table(data["rates"]) + except requests.RequestException as e: + logger.error(f"Error fetching AllRatesToday table for {self.bank}: {e}") + return results + except (KeyError, TypeError, ValueError, AttributeError, ArithmeticError) as e: + logger.error( + f"Unexpected response structure from AllRatesToday " + f"for {self.bank}: {e}" + ) + return results + + for target_currency in target_currencies: + if target_currency.exchange_currency not in exchange_currencies: + continue + + base_currency = target_currency.exchange_currency + + if base_currency.code == target_currency.code: + results.append((base_currency, target_currency, Decimal("1"))) + continue + + rate = self._find_rate(rates, base_currency.code, target_currency.code) + if rate is None: + logger.error( + f"No rate for {base_currency.code}/{target_currency.code} " + f"in the AllRatesToday table for {self.bank}" + ) + continue + + results.append((base_currency, target_currency, rate)) + + return results diff --git a/app/apps/currencies/forms.py b/app/apps/currencies/forms.py index ea2fef05..526368a2 100644 --- a/app/apps/currencies/forms.py +++ b/app/apps/currencies/forms.py @@ -1,13 +1,17 @@ +import json + from apps.common.widgets.crispy.daisyui import Switch from apps.common.widgets.crispy.submit import NoClassSubmit from apps.common.widgets.datepicker import AirDateTimePickerInput from apps.common.widgets.decimal import ArbitraryDecimalDisplayNumberInput from apps.common.widgets.tom_select import TomSelect +from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService from crispy_forms.bootstrap import FormActions from crispy_forms.helper import FormHelper from crispy_forms.layout import Column, Layout, Row from django import forms +from django.core.exceptions import ValidationError from django.forms import CharField from django.utils.translation import gettext_lazy as _ @@ -103,6 +107,7 @@ class ExchangeRateServiceForm(forms.ModelForm): "service_type", "is_active", "api_key", + "options", "interval_type", "fetch_interval", "target_currencies", @@ -110,10 +115,34 @@ class ExchangeRateServiceForm(forms.ModelForm): "one_per_day", "max_rates", ] + widgets = {"options": forms.Textarea(attrs={"rows": 2})} + + # Shows API Key and Options only for providers that use them + SERVICE_TYPE_SCRIPT = """ + init send change to me end + on change js(me) + const p = JSON.parse(me.dataset.providers)[me.value] || {}; + document.getElementById('div_id_api_key').classList.toggle('hidden', !p.api_key); + document.getElementById('div_id_options').classList.toggle('hidden', !p.options); + const help = document.getElementById('id_options_helptext'); + if (help) help.textContent = p.options || ''; + end + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + providers = { + key: { + "api_key": provider.requires_api_key(), + "options": str(provider.options_help or ""), + } + for key, provider in PROVIDER_MAPPING.items() + } + self.fields["service_type"].widget.attrs.update( + {"data-providers": json.dumps(providers), "_": self.SERVICE_TYPE_SCRIPT} + ) + self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" @@ -122,6 +151,7 @@ class ExchangeRateServiceForm(forms.ModelForm): "service_type", Switch("is_active"), "api_key", + "options", Row( Column("interval_type"), Column("fetch_interval"), @@ -144,3 +174,15 @@ class ExchangeRateServiceForm(forms.ModelForm): NoClassSubmit("submit", _("Add"), css_class="btn btn-primary"), ), ) + + def clean(self): + cleaned_data = super().clean() + provider = PROVIDER_MAPPING.get(cleaned_data.get("service_type")) + if provider: + try: + cleaned_data["options"] = provider.clean_options( + cleaned_data.get("options") or "" + ) + except ValidationError as e: + self.add_error("options", e) + return cleaned_data diff --git a/app/apps/currencies/migrations/0026_exchangerateservice_options.py b/app/apps/currencies/migrations/0026_exchangerateservice_options.py new file mode 100644 index 00000000..57681750 --- /dev/null +++ b/app/apps/currencies/migrations/0026_exchangerateservice_options.py @@ -0,0 +1,41 @@ +# Generated by Django 5.2.17 on 2026-11-08 05:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("currencies", "0025_alter_exchangerateservice_service_type"), + ] + + operations = [ + migrations.AddField( + model_name="exchangerateservice", + name="options", + field=models.TextField( + blank=True, + default="", + help_text="Provider specific settings", + verbose_name="Options", + ), + ), + migrations.AlterField( + model_name="exchangerateservice", + name="service_type", + field=models.CharField( + choices=[ + ("coingecko_free", "CoinGecko (Demo/Free)"), + ("coingecko_pro", "CoinGecko (Pro)"), + ("transitive", "Transitive (Calculated from Existing Rates)"), + ("frankfurter", "Frankfurter"), + ("twelvedata", "TwelveData"), + ("twelvedatamarkets", "TwelveData Markets"), + ("yfinance", "Yahoo Finance"), + ("fxmacrodata", "FXMacroData"), + ("allratestoday", "AllRatesToday Central Banks"), + ], + max_length=255, + verbose_name="Service Type", + ), + ), + ] diff --git a/app/apps/currencies/models.py b/app/apps/currencies/models.py index 791b02d1..6cf9c173 100644 --- a/app/apps/currencies/models.py +++ b/app/apps/currencies/models.py @@ -107,6 +107,7 @@ class ExchangeRateService(models.Model): TWELVEDATA_MARKETS = "twelvedatamarkets", "TwelveData Markets" YFINANCE = "yfinance", "Yahoo Finance" FXMACRODATA = "fxmacrodata", "FXMacroData" + ALLRATESTODAY = "allratestoday", "AllRatesToday Central Banks" class IntervalType(models.TextChoices): ON = "on", _("On") @@ -125,6 +126,12 @@ class ExchangeRateService(models.Model): verbose_name=_("API Key"), help_text=_("API key for the service (if required)"), ) + options = models.TextField( + blank=True, + default="", + verbose_name=_("Options"), + help_text=_("Provider specific settings"), + ) interval_type = models.CharField( max_length=255, choices=IntervalType.choices, @@ -189,7 +196,9 @@ class ExchangeRateService(models.Model): from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING provider_class = PROVIDER_MAPPING[self.service_type] - return provider_class(self.api_key) + provider = provider_class(self.api_key) + provider.options = self.options or "" + return provider @staticmethod def _parse_hour_ranges(interval_str: str) -> Set[int]: diff --git a/app/apps/currencies/tests/test_allratestoday_provider.py b/app/apps/currencies/tests/test_allratestoday_provider.py new file mode 100644 index 00000000..151ef94b --- /dev/null +++ b/app/apps/currencies/tests/test_allratestoday_provider.py @@ -0,0 +1,280 @@ +import json +from decimal import Decimal +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import requests +from django.core.exceptions import ValidationError +from django.test import TestCase as DjangoTestCase + +from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING +from apps.currencies.exchange_rates.providers import AllRatesTodayProvider +from apps.currencies.forms import ExchangeRateServiceForm +from apps.currencies.models import ExchangeRateService + + +class _FakeCurrency: + def __init__(self, code, exchange_currency=None): + self.code = code + self.exchange_currency = exchange_currency + + +def _row(base, quote, value, type_="reference"): + return {"base": base, "quote": quote, "type": type_, "value": value} + + +def _response(payload): + response = MagicMock() + response.status_code = 200 + response.json.return_value = payload + return response + + +class AllRatesTodayProviderTests(TestCase): + def setUp(self): + self.brl = _FakeCurrency("BRL") + self.usd = _FakeCurrency("USD") + self.eur = _FakeCurrency("EUR") + self.provider = AllRatesTodayProvider(options="bcb") + + def _get_rates(self, rows, target, base): + target.exchange_currency = base + with patch.object( + self.provider.session, "get", return_value=_response({"rates": rows}) + ): + return self.provider.get_rates([target], {base}) + + def test_requests_the_configured_bank(self): + self.usd.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5.4)]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ) as mock_get: + self.provider.get_rates([self.usd], {self.brl}) + + mock_get.assert_called_once_with( + "https://allratestoday.com/api/open/central-bank/bcb", timeout=30 + ) + + def test_fetches_one_table_for_all_pairs(self): + self.usd.exchange_currency = self.brl + self.eur.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5.4), _row("EUR", "BRL", 6.2)]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ) as mock_get: + rates = self.provider.get_rates([self.usd, self.eur], {self.brl}) + + mock_get.assert_called_once() + self.assertEqual(len(rates), 2) + + def test_empty_field_reads_the_ecb_table(self): + for options in (None, "", " "): + self.assertEqual(AllRatesTodayProvider(options=options).bank, "ecb") + + def test_normalizes_the_bank_code(self): + self.assertEqual(AllRatesTodayProvider(options=" BCB ").bank, "bcb") + + def test_ignores_the_api_key(self): + self.assertEqual(AllRatesTodayProvider("bcb").bank, "ecb") + + def test_clean_options_validates_the_bank_code(self): + self.assertEqual(AllRatesTodayProvider.clean_options(" BCB "), "bcb") + self.assertEqual(AllRatesTodayProvider.clean_options(""), "") + with self.assertRaises(ValidationError): + AllRatesTodayProvider.clean_options("../latest/USD") + + def test_rejects_malformed_bank_code_without_a_request(self): + provider = AllRatesTodayProvider(options="../latest/USD") + self.usd.exchange_currency = self.brl + + with patch.object(provider.session, "get") as mock_get: + rates = provider.get_rates([self.usd], {self.brl}) + + mock_get.assert_not_called() + self.assertEqual(rates, []) + + def test_uses_a_row_in_the_requested_direction(self): + rates = self._get_rates([_row("EUR", "USD", 1.16)], self.usd, self.eur) + + self.assertEqual(rates, [(self.eur, self.usd, Decimal("1.16"))]) + + def test_inverts_a_row_in_the_opposite_direction(self): + # bcb quotes foreign currencies in BRL, so BRL -> USD has to be inverted + rates = self._get_rates([_row("USD", "BRL", 5)], self.brl, self.usd) + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))]) + + rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.brl) + self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))]) + + def test_crosses_pairs_through_the_table_home_currency(self): + rows = [_row("USD", "BRL", 5), _row("EUR", "BRL", 6)] + + rates = self._get_rates(rows, self.eur, self.usd) + + self.assertEqual(len(rates), 1) + self.assertAlmostEqual(rates[0][2], Decimal("0.8333333"), places=6) + + def test_same_currency_is_one(self): + rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.usd) + + self.assertEqual(rates, [(self.usd, self.usd, Decimal("1"))]) + + def test_prefers_the_published_mid_over_buy_and_sell(self): + rows = [ + _row("USD", "BRL", 5.3, "buy"), + _row("USD", "BRL", 5.4, "middle"), + _row("USD", "BRL", 5.7, "sell"), + ] + + rates = self._get_rates(rows, self.brl, self.usd) + + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5.4"))]) + + def test_averages_buy_and_sell_without_a_mid(self): + rows = [_row("USD", "BRL", 4, "buy"), _row("USD", "BRL", 6, "sell")] + + rates = self._get_rates(rows, self.brl, self.usd) + + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))]) + + def test_averages_tt_buy_and_sell_without_a_mid(self): + rows = [ + _row("USD", "MUR", 44.0, "tt_buy"), + _row("USD", "MUR", 46.0, "tt_sell"), + ] + + result = self._get_rates(rows, self.usd, _FakeCurrency("MUR")) + + self.assertEqual(result[0][2], Decimal("1") / Decimal("45.0")) + + def test_ignores_rows_of_other_types(self): + rows = [_row("USD", "BRL", 5, "monthly_average")] + + self.assertEqual(self._get_rates(rows, self.brl, self.usd), []) + + def test_logs_a_table_without_usable_types(self): + self.usd.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5, "monthly_average")]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + with self.assertLogs( + "apps.currencies.exchange_rates.providers", level="WARNING" + ) as logs: + self.provider.get_rates([self.usd], {self.brl}) + + self.assertIn("monthly_average", logs.output[0]) + + def test_skips_pair_missing_from_the_table(self): + rows = [_row("USD", "BRL", 5)] + + self.assertEqual(self._get_rates(rows, self.eur, self.usd), []) + + def test_ignores_zero_and_malformed_rows(self): + rows = [ + "nope", + _row("USD", "BRL", 0), + _row("USD", "BRL", "5,4"), + _row("USD", "BRL", None), + _row("USD", "BRL", "Infinity"), + _row("USD", "BRL", "NaN"), + _row("EUR", "BRL", 6), + ] + + rates = self._get_rates(rows, self.brl, self.eur) + + self.assertEqual(rates, [(self.eur, self.brl, Decimal("6"))]) + + def test_logs_a_stale_table_and_still_uses_it(self): + self.usd.exchange_currency = self.brl + payload = { + "stale": True, + "rate_date": "2026-10-01", + "rates": [_row("BRL", "USD", 0.2)], + } + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + with self.assertLogs( + "apps.currencies.exchange_rates.providers", level="WARNING" + ) as logs: + rates = self.provider.get_rates([self.usd], {self.brl}) + + self.assertIn("2026-10-01", logs.output[0]) + self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))]) + + def test_returns_nothing_on_request_error(self): + self.usd.exchange_currency = self.brl + + with patch.object( + self.provider.session, + "get", + side_effect=requests.RequestException("404 Not Found"), + ): + rates = self.provider.get_rates([self.usd], {self.brl}) + + self.assertEqual(rates, []) + + def test_returns_nothing_on_malformed_body(self): + self.usd.exchange_currency = self.brl + + for payload in ({}, [], {"rates": None}): + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), []) + + def test_returns_nothing_on_non_json_body(self): + self.usd.exchange_currency = self.brl + response = _response({}) + response.json.side_effect = ValueError("not json") + + with patch.object(self.provider.session, "get", return_value=response): + self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), []) + + def test_is_registered_without_an_api_key(self): + self.assertFalse(AllRatesTodayProvider.requires_api_key()) + self.assertIs(PROVIDER_MAPPING["allratestoday"], AllRatesTodayProvider) + self.assertEqual(ExchangeRateService.ServiceType.ALLRATESTODAY, "allratestoday") + + def test_service_passes_its_options_to_the_provider(self): + service = ExchangeRateService(service_type="allratestoday", options="rbi") + self.assertEqual(service.get_provider().bank, "rbi") + + +class ExchangeRateServiceFormOptionsTests(DjangoTestCase): + def _form(self, **data): + return ExchangeRateServiceForm( + data={ + "name": "Rates", + "service_type": "allratestoday", + "interval_type": "every", + "fetch_interval": "24", + **data, + } + ) + + def test_saves_cleaned_options(self): + form = self._form(options=" BCB ") + + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.save().options, "bcb") + + def test_rejects_invalid_options_on_save(self): + form = self._form(options="../latest/USD") + + self.assertFalse(form.is_valid()) + self.assertIn("options", form.errors) + + def test_tells_the_page_which_fields_each_provider_uses(self): + attrs = self._form().fields["service_type"].widget.attrs + providers = json.loads(attrs["data-providers"]) + + self.assertEqual(providers["allratestoday"]["api_key"], False) + self.assertIn("bank code", providers["allratestoday"]["options"]) + self.assertEqual(providers["twelvedata"], {"api_key": True, "options": ""}) diff --git a/app/apps/dca/forms.py b/app/apps/dca/forms.py index a435766a..03a9f539 100644 --- a/app/apps/dca/forms.py +++ b/app/apps/dca/forms.py @@ -12,7 +12,7 @@ from apps.dca.models import DCAEntry, DCAStrategy from apps.transactions.models import Transaction, TransactionCategory, TransactionTag from crispy_forms.bootstrap import AccordionGroup, FormActions, Accordion from crispy_forms.helper import FormHelper -from crispy_forms.layout import HTML, Column, Layout, Row +from crispy_forms.layout import HTML, Column, Div, Layout, Row from django import forms from django.utils.translation import gettext_lazy as _ @@ -157,34 +157,21 @@ class DCAEntryForm(forms.ModelForm): AccordionGroup( _("Create transaction"), Switch("create_transaction"), - Row( - Column( - Row( - Column( - "from_account", - ), - ), - Row( - Column("from_category"), - Column("from_tags"), - ), + Div( + "from_account", + Row( + Column("from_category"), + Column("from_tags"), ), - css_class="p-1 mx-1 my-3 border rounded-3", + css_class="bg-base-100 rounded-box p-4 border-base-content/60 border my-3", ), - Row( - Column( - Row( - Column( - "to_account", - css_class="form-group", - ), - ), - Row( - Column("to_category"), - Column("to_tags"), - ), + Div( + "to_account", + Row( + Column("to_category"), + Column("to_tags"), ), - css_class="p-1 mx-1 my-3 border rounded-3", + css_class="bg-base-100 rounded-box p-4 border-base-content/60 border", ), active=False, ), diff --git a/app/apps/import_app/services/presets.py b/app/apps/import_app/services/presets.py index 824a2468..e4cef311 100644 --- a/app/apps/import_app/services/presets.py +++ b/app/apps/import_app/services/presets.py @@ -1,17 +1,17 @@ import json from pathlib import Path +from django.conf import settings + from apps.import_app.models import ImportProfile class PresetService: - PRESET_PATH = "/usr/src/app/import_presets" - @classmethod def get_all_presets(cls): presets = [] - for folder in Path(cls.PRESET_PATH).iterdir(): + for folder in settings.IMPORT_PRESETS_ROOT.iterdir(): if folder.is_dir(): manifest_path = folder / "manifest.json" config_path = folder / "config.yml" diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index bed860ac..2bc16981 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -7,12 +7,14 @@ import zipfile from django.db import transaction from datetime import datetime, date from decimal import Decimal, InvalidOperation +from pathlib import Path from typing import Dict, Any, Literal, Union import openpyxl import xlrd import yaml from cachalot.api import cachalot_disabled +from django.conf import settings from django.core.exceptions import FieldDoesNotExist from django.utils import timezone from openpyxl.utils.exceptions import InvalidFileException @@ -39,8 +41,6 @@ logger = logging.getLogger(__name__) class ImportService: - TEMP_DIR = "/usr/src/app/temp" - def __init__(self, import_run: ImportRun): self.import_run: ImportRun = import_run self.profile: ImportProfile = import_run.profile @@ -54,7 +54,7 @@ class ImportService: self.mapping: Dict[str, version_1.ColumnMapping] = self.config.mapping # Ensure temp directory exists - os.makedirs(self.TEMP_DIR, exist_ok=True) + os.makedirs(settings.IMPORT_TEMP_ROOT, exist_ok=True) def _load_config(self) -> version_1.ImportProfileSchema: yaml_data = yaml.safe_load(self.profile.yaml_config) @@ -1131,9 +1131,12 @@ class ImportService: Validates that the file path is within the allowed temporary directory. Returns the absolute path. """ - abs_path = os.path.abspath(file_path) - if not abs_path.startswith(self.TEMP_DIR): - raise ValueError(f"Invalid file path. File must be in {self.TEMP_DIR}") + # resolve() like IMPORT_TEMP_ROOT, so symlinks can't point outside it + abs_path = str(Path(file_path).resolve()) + if not Path(abs_path).is_relative_to(settings.IMPORT_TEMP_ROOT): + raise ValueError( + f"Invalid file path. File must be in {settings.IMPORT_TEMP_ROOT}" + ) return abs_path def process_file(self, file_path: str): diff --git a/app/apps/import_app/tests/test_paths.py b/app/apps/import_app/tests/test_paths.py new file mode 100644 index 00000000..a9eb7807 --- /dev/null +++ b/app/apps/import_app/tests/test_paths.py @@ -0,0 +1,161 @@ +"""Tests for the folders import code reads from and writes to.""" + +import json +import os +import runpy +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +from django.conf import settings +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import SimpleTestCase, override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APIClient + +from apps.import_app.models import ImportRun +from apps.import_app.services import PresetService +from apps.import_app.services.v1 import ImportService +from apps.import_app.tests.test_view_permissions import ( + HTMX, + ImportObjectPermissionTestCase, +) + + +def load_settings(**env): + """Evaluates the settings module from scratch under the given env vars.""" + with patch.dict(os.environ, env): + return runpy.run_path(str(Path(settings.BASE_DIR) / "WYGIWYH" / "settings.py")) + + +class PathSettingsTests(SimpleTestCase): + def test_defaults_match_the_docker_image_layout(self): + loaded = load_settings(IMPORT_TEMP_ROOT="", CACHE_ROOT="") + + base_dir = loaded["BASE_DIR"] + self.assertEqual(loaded["IMPORT_TEMP_ROOT"], base_dir / "temp") + self.assertEqual(loaded["IMPORT_PRESETS_ROOT"], base_dir / "import_presets") + self.assertEqual( + loaded["CACHES"]["default"]["LOCATION"], + Path("/var/tmp/django_cache").resolve(), + ) + self.assertEqual( + loaded["CACHES"]["update_check"]["LOCATION"], + Path("/var/tmp/django_cache_update_check").resolve(), + ) + + def test_env_vars_move_temp_and_cache_folders(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root).resolve() + loaded = load_settings( + IMPORT_TEMP_ROOT=str(root / "uploads"), CACHE_ROOT=str(root / "cache") + ) + + self.assertEqual(loaded["IMPORT_TEMP_ROOT"], root / "uploads") + self.assertEqual( + loaded["CACHES"]["default"]["LOCATION"], root / "cache" / "django_cache" + ) + self.assertEqual( + loaded["CACHES"]["update_check"]["LOCATION"], + root / "cache" / "django_cache_update_check", + ) + + +class PresetServiceTests(SimpleTestCase): + def test_lists_the_presets_shipped_with_the_app(self): + folder_names = { + preset["folder_name"] for preset in PresetService.get_all_presets() + } + + self.assertIn("qif_standard", folder_names) + + def test_reads_presets_from_configured_folder(self): + with tempfile.TemporaryDirectory() as root: + preset = Path(root) / "my_bank" + preset.mkdir() + (preset / "manifest.json").write_text(json.dumps({"name": "My Bank"})) + (preset / "config.yml").write_text("settings: {}") + + with override_settings(IMPORT_PRESETS_ROOT=Path(root)): + presets = PresetService.get_all_presets() + + self.assertEqual([p["name"] for p in presets], ["My Bank"]) + + +class ImportTempRootTests(ImportObjectPermissionTestCase): + def setUp(self): + super().setUp() + self.temp_root = Path(tempfile.mkdtemp()).resolve() + self.addCleanup(shutil.rmtree, self.temp_root, ignore_errors=True) + temp_root_override = override_settings(IMPORT_TEMP_ROOT=self.temp_root) + temp_root_override.enable() + self.addCleanup(temp_root_override.disable) + + @patch("apps.import_app.views.process_import.defer") + def test_web_upload_is_stored_in_temp_root(self, mock_defer): + self.login(self.owner) + response = self.client.post( + reverse("import_run_add", kwargs={"profile_id": self.private_profile.id}), + {"file": SimpleUploadedFile("web.csv", b"date,amount\n")}, + **HTMX, + ) + + self.assertEqual(response.status_code, 204) + self.assertEqual( + mock_defer.call_args.kwargs["file_path"], str(self.temp_root / "web.csv") + ) + self.assertTrue((self.temp_root / "web.csv").is_file()) + + @patch("apps.api.views.imports.process_import.defer") + def test_api_upload_is_stored_in_temp_root(self, mock_defer): + client = APIClient() + client.force_authenticate(user=self.owner) + response = client.post( + "/api/import/import/", + { + "profile_id": self.private_profile.id, + "file": SimpleUploadedFile("api.csv", b"date,amount\n"), + }, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual( + mock_defer.call_args.kwargs["file_path"], str(self.temp_root / "api.csv") + ) + self.assertTrue((self.temp_root / "api.csv").is_file()) + + def _service(self): + run = ImportRun.objects.create( + profile=self.private_profile, file_name="file.csv", owner=self.owner + ) + return ImportService(run) + + def test_service_accepts_files_inside_temp_root(self): + path = str(self.temp_root / "file.csv") + + self.assertEqual(self._service()._validate_file_path(path), path) + + def test_service_rejects_files_outside_temp_root(self): + service = self._service() + + for path in ( + "/etc/passwd", + self.temp_root / ".." / "file.csv", + # Shares the temp root as a string prefix, but is a different folder + f"{self.temp_root}-other/file.csv", + ): + with self.subTest(path=path), self.assertRaises(ValueError): + service._validate_file_path(str(path)) + + def test_service_rejects_symlinks_pointing_outside_temp_root(self): + link = self.temp_root / "link.csv" + try: + link.symlink_to("/etc/passwd") + except OSError: + self.skipTest("symlinks not available") + + with self.assertRaises(ValueError): + self._service()._validate_file_path(str(link)) diff --git a/app/apps/import_app/tests/test_qif_import.py b/app/apps/import_app/tests/test_qif_import.py index a79662b3..d2b7e9db 100644 --- a/app/apps/import_app/tests/test_qif_import.py +++ b/app/apps/import_app/tests/test_qif_import.py @@ -1,7 +1,7 @@ from decimal import Decimal import os import shutil -from django.test import TestCase +from django.test import TestCase, override_settings from django.contrib.auth import get_user_model from apps.accounts.models import Account, AccountGroup from apps.currencies.models import Currency @@ -15,10 +15,11 @@ from apps.transactions.models import ( class QIFImportTests(TestCase): def setUp(self): - # Patch TEMP_DIR for testing - self.original_temp_dir = ImportService.TEMP_DIR + # Point the import temp folder somewhere disposable for testing self.test_dir = os.path.abspath("temp_test_import") - ImportService.TEMP_DIR = self.test_dir + temp_root_override = override_settings(IMPORT_TEMP_ROOT=self.test_dir) + temp_root_override.enable() + self.addCleanup(temp_root_override.disable) os.makedirs(self.test_dir, exist_ok=True) # Create user and set context @@ -41,7 +42,6 @@ class QIFImportTests(TestCase): def tearDown(self): delete_current_user() - ImportService.TEMP_DIR = self.original_temp_dir if os.path.exists(self.test_dir): shutil.rmtree(self.test_dir) diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index adef7679..45dc61ab 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -1,5 +1,6 @@ import shutil +from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied @@ -204,7 +205,7 @@ def import_run_add(request, profile_id): if form.is_valid(): uploaded_file = request.FILES["file"] - fs = FileSystemStorage(location="/usr/src/app/temp") + fs = FileSystemStorage(location=settings.IMPORT_TEMP_ROOT) filename = fs.save(uploaded_file.name, uploaded_file) file_path = fs.path(filename) diff --git a/app/apps/transactions/tests/test_transaction_search.py b/app/apps/transactions/tests/test_transaction_search.py new file mode 100644 index 00000000..6b02e95b --- /dev/null +++ b/app/apps/transactions/tests/test_transaction_search.py @@ -0,0 +1,74 @@ +from datetime import timedelta +from decimal import Decimal + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone + +from apps.accounts.models import Account +from apps.currencies.models import Currency +from apps.transactions.models import ( + Transaction, + TransactionCategory, + TransactionEntity, + TransactionTag, +) + + +class TransactionSearchTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user( + email="search@test.com", password="pass" + ) + self.client.login(username="search@test.com", password="pass") + self.user.settings.date_format = "d/m/Y" + self.user.settings.number_format = "DC" + self.user.settings.save() + + currency = Currency.objects.create(code="BRL", name="Real", decimal_places=2) + self.account = Account.objects.create(name="Wallet", currency=currency) + self.today = timezone.localdate(timezone.now()) + + def make(self, days, description="", amount="10", **kwargs): + return Transaction.objects.create( + account=self.account, + type=Transaction.Type.EXPENSE, + date=self.today + timedelta(days=days), + reference_date=(self.today + timedelta(days=days)).replace(day=1), + amount=Decimal(amount), + description=description, + **kwargs, + ) + + def search(self, q=""): + response = self.client.get(reverse("transactions_search"), {"q": q}) + self.assertEqual(response.status_code, 200) + return [row["description"] for row in response.json()] + + def test_order_matches_monthly_view(self): + for days, name in [(-30, "old"), (-2, "recent"), (-1, "yesterday"), + (10, "far"), (2, "soon"), (1, "tomorrow"), (0, "today")]: + self.make(days, name) + self.assertEqual( + self.search(), + ["today", "tomorrow", "yesterday", "soon", "far", "recent", "old"], + ) + + def test_search_fields(self): + t = self.make(-5, "Groceries", amount="1234.56") + t.category = TransactionCategory.objects.create(name="Food") + t.save() + t.tags.add(TransactionTag.objects.create(name="home"), + TransactionTag.objects.create(name="weekly")) + t.entities.add(TransactionEntity.objects.create(name="Market")) + self.make(-6, "Other", amount="5") + + day = (self.today - timedelta(days=5)) + for q in ["food", "market", "1.234,56", "1234", + day.strftime("%d/%m/%Y"), day.strftime("%d/%m"), + "home weekly", "groceries food"]: + self.assertEqual(self.search(q), ["Groceries"], q) + + self.assertEqual(self.search("wallet"), ["Groceries", "Other"]) + self.assertEqual(self.search("groceries nomatch"), []) diff --git a/app/apps/transactions/utils/select_search.py b/app/apps/transactions/utils/select_search.py new file mode 100644 index 00000000..161383a5 --- /dev/null +++ b/app/apps/transactions/utils/select_search.py @@ -0,0 +1,95 @@ +import datetime +import re +from decimal import Decimal, InvalidOperation + +from apps.common.functions.format import get_format +from apps.common.utils.django import django_to_python_datetime +from apps.transactions.templatetags.currency_display import currency_display +from django.db.models import Q +from django.template.defaultfilters import date as date_filter + + +def transaction_option(t): + """Data used by the transaction picker (tom-select) to render a mini transaction.""" + currency = t.account.currency + return { + "value": str(t.id), + "text": str(t), + "description": t.description, + "date": date_filter(t.date, "SHORT_DATE_FORMAT"), + "type": t.type, + "is_paid": t.is_paid, + "amount": currency_display( + t.amount, + currency.prefix, + currency.suffix, + currency.decimal_places, + string=True, + ), + "account": ( + f"{t.account.group.name} • {t.account.name}" + if t.account.group + else t.account.name + ), + "category": t.category.name if t.category else "", + "tags": [tag.name for tag in t.tags.all()], + "entities": [entity.name for entity in t.entities.all()], + } + + +def _parse_date(term): + """Match a term against the user's date format; without a year it matches any year.""" + fmt = django_to_python_datetime(get_format("SHORT_DATE_FORMAT")) + try: + return Q(date=datetime.datetime.strptime(term, fmt).date()) + except ValueError: + pass + # Drop the year and its adjacent separator, e.g. "%d/%m/%Y" -> "%d/%m" + no_year = re.sub(r"[^%\w]?%[Yy][^%\w]?", "", fmt, count=1).strip() + if no_year == fmt or not no_year: + return None + try: + # 2000 is a leap year, so 29/02 parses + parsed = datetime.datetime.strptime(f"{term} 2000", f"{no_year} %Y") + except ValueError: + return None + return Q(date__day=parsed.day, date__month=parsed.month) + + +def _parse_amount(term): + if not re.fullmatch(r"[\d.,\s]*\d[\d.,\s]*", term): + return None + thousand = get_format("THOUSAND_SEPARATOR") + decimal = get_format("DECIMAL_SEPARATOR") + normalized = term.replace(thousand, "").replace(decimal, ".") + try: + value = Decimal(normalized) + except InvalidOperation: + return None + if decimal not in term: + # Whole number: "12" finds 12.00 to 12.99 + return Q(amount__gte=value, amount__lt=value + 1) + return Q(amount=value) + + +def search_transactions(queryset, search_term): + """Every word must match some field (description, category, entity, date, amount...).""" + matches = queryset.model.objects.all() + for term in search_term.split(): + term_q = ( + Q(description__icontains=term) + | Q(notes__icontains=term) + | Q(internal_note__icontains=term) + | Q(tags__name__icontains=term) + | Q(entities__name__icontains=term) + | Q(category__name__icontains=term) + | Q(account__name__icontains=term) + | Q(account__group__name__icontains=term) + ) + for extra in (_parse_date(term), _parse_amount(term)): + if extra is not None: + term_q |= extra + # One filter() per word so different words can match different tags/entities + matches = matches.filter(term_q) + # id__in avoids duplicate rows from the tag/entity joins + return queryset.filter(id__in=matches.values("id")) diff --git a/app/apps/transactions/views/transactions.py b/app/apps/transactions/views/transactions.py index 9ceab0a5..07b7f731 100644 --- a/app/apps/transactions/views/transactions.py +++ b/app/apps/transactions/views/transactions.py @@ -20,11 +20,15 @@ from apps.transactions.utils.calculations import ( calculate_percentage_distribution, ) from apps.transactions.utils.default_ordering import default_order +from apps.transactions.utils.select_search import ( + search_transactions, + transaction_option, +) from dateutil.relativedelta import relativedelta from django.contrib import messages from django.contrib.auth.decorators import login_required from django.core.paginator import Paginator -from django.db.models import Case, IntegerField, Q, Value, When +from django.db.models import Case, F, IntegerField, Q, Value, When from django.http import FileResponse, Http404, HttpResponse, JsonResponse, QueryDict from django.shortcuts import get_object_or_404, render from django.utils import timezone @@ -984,51 +988,41 @@ def transactions_trash_can_list(request): @login_required @require_http_methods(["GET"]) def get_recent_transactions(request, filter_type=None): - """Return the 100 most recent non-deleted transactions with optional search.""" - # Get search term from query params + """Return up to 100 non-deleted transactions, ordered like the monthly view, with optional search.""" search_term = request.GET.get("q", "").strip() today = timezone.localdate(timezone.now()) yesterday = today - timezone.timedelta(days=1) tomorrow = today + timezone.timedelta(days=1) - # Base queryset with selected fields + queryset = Transaction.objects.filter(deleted=False) + + if filter_type == "expenses": + queryset = queryset.filter(type=Transaction.Type.EXPENSE) + elif filter_type == "income": + queryset = queryset.filter(type=Transaction.Type.INCOME) + + if search_term: + queryset = search_transactions(queryset, search_term) + + # Today, tomorrow, yesterday, then upcoming (soonest first), then past (newest first) queryset = ( - Transaction.objects.filter(deleted=False) - .annotate( + queryset.annotate( date_order=Case( When(date=today, then=Value(0)), When(date=tomorrow, then=Value(1)), When(date=yesterday, then=Value(2)), When(date__gt=tomorrow, then=Value(3)), - When(date__lt=yesterday, then=Value(4)), - default=Value(5), + default=Value(4), output_field=IntegerField(), - ) + ), + upcoming_date=Case(When(date__gt=tomorrow, then=F("date"))), ) - .select_related("account", "category") - .order_by("date_order", "date", "id") + .select_related("account__currency", "account__group", "category") + .prefetch_related("tags", "entities") + .order_by( + "date_order", F("upcoming_date").asc(nulls_last=True), "-date", "-id" + )[:100] ) - if filter_type: - if filter_type == "expenses": - queryset = queryset.filter(type=Transaction.Type.EXPENSE) - elif filter_type == "income": - queryset = queryset.filter(type=Transaction.Type.INCOME) - - # Apply search if provided - if search_term: - queryset = queryset.filter( - Q(description__icontains=search_term) - | Q(notes__icontains=search_term) - | Q(internal_note__icontains=search_term) - | Q(tags__name__icontains=search_term) - | Q(category__name__icontains=search_term) - ) - - # Prepare data for JSON response - data = [] - for t in queryset: - data.append({"text": str(t), "value": str(t.id)}) - - return JsonResponse(data, safe=False) + return JsonResponse([transaction_option(t) for t in queryset], safe=False) diff --git a/app/locale/de/LC_MESSAGES/django.po b/app/locale/de/LC_MESSAGES/django.po index 782caca2..70fd32d0 100644 --- a/app/locale/de/LC_MESSAGES/django.po +++ b/app/locale/de/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-11-02 19:49+0000\n" +"POT-Creation-Date: 2026-11-08 13:19+0000\n" "PO-Revision-Date: 2026-08-10 12:57+0000\n" "Last-Translator: GitEbu \n" "Language-Team: German \n" "Language-Team: LANGUAGE \n" @@ -23,8 +23,8 @@ msgid "Group name" msgstr "" #: apps/accounts/forms.py:43 apps/accounts/forms.py:109 -#: apps/currencies/forms.py:53 apps/currencies/forms.py:87 -#: apps/currencies/forms.py:136 apps/dca/forms.py:46 apps/dca/forms.py:205 +#: apps/currencies/forms.py:57 apps/currencies/forms.py:91 +#: apps/currencies/forms.py:166 apps/dca/forms.py:46 apps/dca/forms.py:192 #: apps/import_app/forms.py:32 apps/rules/forms.py:60 apps/rules/forms.py:100 #: apps/rules/forms.py:385 apps/transactions/forms.py:219 #: apps/transactions/forms.py:452 apps/transactions/forms.py:571 @@ -38,9 +38,9 @@ msgid "Update" msgstr "" #: apps/accounts/forms.py:49 apps/accounts/forms.py:115 -#: apps/common/widgets/tom_select.py:13 apps/currencies/forms.py:59 -#: apps/currencies/forms.py:93 apps/currencies/forms.py:142 -#: apps/dca/forms.py:52 apps/dca/forms.py:211 apps/import_app/forms.py:38 +#: apps/common/widgets/tom_select.py:15 apps/currencies/forms.py:63 +#: apps/currencies/forms.py:97 apps/currencies/forms.py:172 +#: apps/dca/forms.py:52 apps/dca/forms.py:198 apps/import_app/forms.py:38 #: apps/rules/forms.py:66 apps/rules/forms.py:106 apps/rules/forms.py:391 #: apps/transactions/forms.py:206 apps/transactions/forms.py:226 #: apps/transactions/forms.py:459 apps/transactions/forms.py:946 @@ -109,7 +109,7 @@ msgstr "" msgid "Balance against" msgstr "" -#: apps/accounts/forms.py:189 apps/currencies/forms.py:66 +#: apps/accounts/forms.py:189 apps/currencies/forms.py:70 #: apps/dca/models.py:158 apps/rules/forms.py:176 apps/rules/forms.py:190 #: apps/rules/models.py:38 apps/rules/models.py:279 #: apps/transactions/forms.py:86 apps/transactions/forms.py:482 @@ -234,7 +234,7 @@ msgstr "" #: apps/accounts/views/account_groups.py:109 #: apps/accounts/views/accounts.py:137 apps/dca/views.py:106 -#: apps/import_app/views.py:147 apps/rules/views.py:191 +#: apps/import_app/views.py:148 apps/rules/views.py:191 #: apps/transactions/views/categories.py:163 #: apps/transactions/views/entities.py:135 apps/transactions/views/tags.py:131 msgid "Item no longer shared with you" @@ -242,7 +242,7 @@ msgstr "" #: apps/accounts/views/account_groups.py:132 #: apps/accounts/views/accounts.py:182 apps/dca/views.py:131 -#: apps/import_app/views.py:273 apps/rules/views.py:217 +#: apps/import_app/views.py:274 apps/rules/views.py:217 #: apps/transactions/views/categories.py:188 #: apps/transactions/views/entities.py:160 apps/transactions/views/tags.py:154 msgid "Ownership taken successfully" @@ -250,7 +250,7 @@ msgstr "" #: apps/accounts/views/account_groups.py:152 #: apps/accounts/views/accounts.py:106 apps/dca/views.py:151 -#: apps/import_app/views.py:294 apps/rules/views.py:238 +#: apps/import_app/views.py:295 apps/rules/views.py:238 #: apps/transactions/views/categories.py:130 #: apps/transactions/views/entities.py:180 apps/transactions/views/tags.py:174 msgid "Configuration saved successfully" @@ -334,7 +334,7 @@ msgid "Error creating new instance" msgstr "" #: apps/common/fields/forms/grouped_select.py:24 -#: apps/common/widgets/tom_select.py:92 apps/common/widgets/tom_select.py:95 +#: apps/common/widgets/tom_select.py:94 apps/common/widgets/tom_select.py:97 msgid "Ungrouped" msgstr "" @@ -480,11 +480,11 @@ msgstr "" msgid "Now" msgstr "" -#: apps/common/widgets/tom_select.py:11 +#: apps/common/widgets/tom_select.py:13 msgid "Remove" msgstr "" -#: apps/common/widgets/tom_select.py:15 templates/mini_tools/simulator.html:187 +#: apps/common/widgets/tom_select.py:17 templates/mini_tools/simulator.html:187 #: templates/mini_tools/unit_price_calculator.html:180 #: templates/monthly_overview/pages/overview.html:286 #: templates/transactions/fragments/filter_actions.html:48 @@ -492,15 +492,24 @@ msgstr "" msgid "Clear" msgstr "" -#: apps/common/widgets/tom_select.py:16 +#: apps/common/widgets/tom_select.py:18 msgid "No results..." msgstr "" -#: apps/currencies/forms.py:16 apps/currencies/models.py:22 +#: apps/currencies/exchange_rates/providers.py:703 +msgid "" +"Central bank code, e.g. 'bcb' or 'rbi'. Leave empty to use the ECB table." +msgstr "" + +#: apps/currencies/exchange_rates/providers.py:735 +msgid "Invalid bank code, it should look like 'bcb'." +msgstr "" + +#: apps/currencies/forms.py:20 apps/currencies/models.py:22 msgid "Prefix" msgstr "" -#: apps/currencies/forms.py:17 apps/currencies/models.py:23 +#: apps/currencies/forms.py:21 apps/currencies/models.py:23 msgid "Suffix" msgstr "" @@ -562,27 +571,27 @@ msgstr "" msgid "From and To currencies cannot be the same." msgstr "" -#: apps/currencies/models.py:112 +#: apps/currencies/models.py:113 msgid "On" msgstr "" -#: apps/currencies/models.py:113 +#: apps/currencies/models.py:114 msgid "Every X hours" msgstr "" -#: apps/currencies/models.py:114 +#: apps/currencies/models.py:115 msgid "Not on" msgstr "" -#: apps/currencies/models.py:116 +#: apps/currencies/models.py:117 msgid "Service Name" msgstr "" -#: apps/currencies/models.py:118 +#: apps/currencies/models.py:119 msgid "Service Type" msgstr "" -#: apps/currencies/models.py:120 apps/transactions/filters.py:27 +#: apps/currencies/models.py:121 apps/transactions/filters.py:27 #: apps/transactions/models.py:253 apps/transactions/models.py:277 #: apps/transactions/models.py:301 templates/categories/fragments/list.html:16 #: templates/entities/fragments/list.html:16 @@ -594,77 +603,85 @@ msgstr "" msgid "Active" msgstr "" -#: apps/currencies/models.py:125 +#: apps/currencies/models.py:126 msgid "API Key" msgstr "" -#: apps/currencies/models.py:126 +#: apps/currencies/models.py:127 msgid "API key for the service (if required)" msgstr "" -#: apps/currencies/models.py:131 -msgid "Interval Type" +#: apps/currencies/models.py:132 +msgid "Options" msgstr "" -#: apps/currencies/models.py:135 -msgid "Interval" +#: apps/currencies/models.py:133 +msgid "Provider specific settings" msgstr "" #: apps/currencies/models.py:138 -msgid "Last Successful Fetch" +msgid "Interval Type" +msgstr "" + +#: apps/currencies/models.py:142 +msgid "Interval" msgstr "" #: apps/currencies/models.py:145 +msgid "Last Successful Fetch" +msgstr "" + +#: apps/currencies/models.py:152 msgid "Target Currencies" msgstr "" -#: apps/currencies/models.py:147 +#: apps/currencies/models.py:154 msgid "" "Select currencies to fetch exchange rates for. Rates will be fetched for " "each currency against their set exchange currency." msgstr "" -#: apps/currencies/models.py:155 +#: apps/currencies/models.py:162 msgid "Target Accounts" msgstr "" -#: apps/currencies/models.py:157 +#: apps/currencies/models.py:164 msgid "" "Select accounts to fetch exchange rates for. Rates will be fetched for each " "account's currency against their set exchange currency." msgstr "" -#: apps/currencies/models.py:164 +#: apps/currencies/models.py:171 msgid "Single exchange rate" msgstr "" -#: apps/currencies/models.py:167 +#: apps/currencies/models.py:174 msgid "Create one exchange rate and keep updating it. Avoids database clutter." msgstr "" -#: apps/currencies/models.py:172 +#: apps/currencies/models.py:179 msgid "Exchange Rate Service" msgstr "" -#: apps/currencies/models.py:173 +#: apps/currencies/models.py:180 msgid "Exchange Rate Services" msgstr "" -#: apps/currencies/models.py:225 +#: apps/currencies/models.py:234 msgid "'Every X hours' interval type requires a positive integer." msgstr "" -#: apps/currencies/models.py:234 +#: apps/currencies/models.py:243 msgid "'Every X hours' interval must be between 1 and 24." msgstr "" -#: apps/currencies/models.py:248 +#: apps/currencies/models.py:257 msgid "" "Invalid hour format. Use comma-separated hours (0-23) and/or ranges (e.g., " "'1-5,8,10-12')." msgstr "" -#: apps/currencies/models.py:259 +#: apps/currencies/models.py:268 msgid "" "Invalid format. Please check the requirements for your selected interval " "type." @@ -734,21 +751,21 @@ msgstr "" msgid "Income Transaction" msgstr "" -#: apps/dca/forms.py:192 +#: apps/dca/forms.py:179 msgid "Link transaction" msgstr "" -#: apps/dca/forms.py:275 apps/dca/forms.py:276 apps/dca/forms.py:281 -#: apps/dca/forms.py:285 +#: apps/dca/forms.py:262 apps/dca/forms.py:263 apps/dca/forms.py:268 +#: apps/dca/forms.py:272 msgid "You must provide an account." msgstr "" -#: apps/dca/forms.py:290 apps/transactions/forms.py:752 +#: apps/dca/forms.py:277 apps/transactions/forms.py:752 #: apps/transactions/models.py:956 msgid "From and To accounts must be different." msgstr "" -#: apps/dca/forms.py:304 +#: apps/dca/forms.py:291 #, python-format msgid "DCA for %(strategy_name)s" msgstr "" @@ -1003,23 +1020,23 @@ msgstr "" msgid "File name" msgstr "" -#: apps/import_app/views.py:75 +#: apps/import_app/views.py:76 msgid "Import Profile added successfully" msgstr "" -#: apps/import_app/views.py:113 +#: apps/import_app/views.py:114 msgid "Import Profile update successfully" msgstr "" -#: apps/import_app/views.py:142 +#: apps/import_app/views.py:143 msgid "Import Profile deleted successfully" msgstr "" -#: apps/import_app/views.py:222 +#: apps/import_app/views.py:223 msgid "Import Run queued successfully" msgstr "" -#: apps/import_app/views.py:249 +#: apps/import_app/views.py:250 msgid "Run deleted successfully" msgstr "" @@ -1847,8 +1864,8 @@ msgid "Item deleted successfully" msgstr "" #: apps/transactions/views/quick_transactions.py:164 -#: apps/transactions/views/transactions.py:143 -#: apps/transactions/views/transactions.py:328 +#: apps/transactions/views/transactions.py:147 +#: apps/transactions/views/transactions.py:332 msgid "Transaction added successfully" msgstr "" @@ -1884,42 +1901,42 @@ msgstr "" msgid "Tag deleted successfully" msgstr "" -#: apps/transactions/views/transactions.py:61 +#: apps/transactions/views/transactions.py:65 msgid "Attachment uploaded successfully" msgstr "" -#: apps/transactions/views/transactions.py:112 +#: apps/transactions/views/transactions.py:116 msgid "Attachment deleted successfully" msgstr "" -#: apps/transactions/views/transactions.py:355 +#: apps/transactions/views/transactions.py:359 msgid "Transaction updated successfully" msgstr "" -#: apps/transactions/views/transactions.py:406 +#: apps/transactions/views/transactions.py:410 #, python-format msgid "%(count)s transaction updated successfully" msgid_plural "%(count)s transactions updated successfully" msgstr[0] "" msgstr[1] "" -#: apps/transactions/views/transactions.py:433 +#: apps/transactions/views/transactions.py:437 msgid "Transaction duplicated successfully" msgstr "" -#: apps/transactions/views/transactions.py:478 +#: apps/transactions/views/transactions.py:482 msgid "Transaction deleted successfully" msgstr "" -#: apps/transactions/views/transactions.py:508 +#: apps/transactions/views/transactions.py:512 msgid "Transaction restored successfully" msgstr "" -#: apps/transactions/views/transactions.py:534 +#: apps/transactions/views/transactions.py:538 msgid "Transfer added successfully" msgstr "" -#: apps/transactions/views/transactions.py:562 +#: apps/transactions/views/transactions.py:566 msgid "Transfer updated successfully" msgstr "" @@ -2496,7 +2513,7 @@ msgstr "" msgid "Pick a month" msgstr "" -#: templates/common/fragments/toasts.html:20 +#: templates/common/fragments/toasts.html:18 #: templates/cotton/components/fab.html:9 #: templates/cotton/components/sidebar_collapsible_panel.html:28 #: templates/extends/offcanvas.html:4 templates/includes/sidebar.html:44 diff --git a/app/locale/es/LC_MESSAGES/django.po b/app/locale/es/LC_MESSAGES/django.po index 88a6a9a9..10745cca 100644 --- a/app/locale/es/LC_MESSAGES/django.po +++ b/app/locale/es/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-11-02 19:49+0000\n" +"POT-Creation-Date: 2026-11-08 13:19+0000\n" "PO-Revision-Date: 2026-07-10 04:57+0000\n" "Last-Translator: Juan David Afanador \n" "Language-Team: Spanish \n" "Language-Team: French \n" "Language-Team: Hungarian \n" "Language-Team: Italian \n" "Language-Team: Dutch \n" "Language-Team: Polish \n" "Language-Team: Portuguese (Brazil) \n" "Language-Team: Russian \n" "Language-Team: Swedish \n" "Language-Team: Swahili \n" "Language-Team: Ukrainian \n" "Language-Team: Chinese (Traditional Han script) -
-
- -
- {{ message.tags | toast_title }} -
{{ message }}
-
-
- + +
+ {{ message.tags | toast_title }} +
{{ message }}
+ {# -mt-1 centers the 2rem button on the 1.5rem title line #} +
{% endfor %} {% endif %} diff --git a/docker/prod/django/Dockerfile b/docker/prod/django/Dockerfile index 9ad00cb2..fd83829f 100644 --- a/docker/prod/django/Dockerfile +++ b/docker/prod/django/Dockerfile @@ -69,9 +69,7 @@ RUN sed -i 's/\r$//g' /start && \ COPY --chown=app:app ./app . RUN chown -R app:app /usr/src/app && \ - chown -R app:app /usr/src/frontend && \ - mkdir /temp && \ - chown -R app:app /temp + chown -R app:app /usr/src/frontend USER app diff --git a/frontend/src/js/select.js b/frontend/src/js/select.js index f40494ab..dc573918 100644 --- a/frontend/src/js/select.js +++ b/frontend/src/js/select.js @@ -3,6 +3,31 @@ import * as Popper from "@popperjs/core"; import '../styles/_tom-select.scss' +// Mini version of cotton/transaction/item.html, used by TransactionSelect +function renderTransaction(data, escape, compact) { + if (!data.date) { + return `
${escape(data.text)}
`; + } + const kind = data.type === 'EX' ? 'expense' : 'income'; + const meta = [ + data.category && `${escape(data.category)}`, + data.tags && data.tags.length && `${escape(data.tags.join(', '))}`, + data.entities && data.entities.length && `${escape(data.entities.join(', '))}`, + ].filter(Boolean).join(''); + return `
+ +
+
${escape(data.date)}
+
${escape(data.description || '')}
+ ${meta && !compact ? `
${meta}
` : ''} +
+
+
${escape(data.amount)}
+ +
+
`; +} + window.TomSelect = function createDynamicTomSelect(element) { const schedulePopperUpdate = function (instance) { // Wait for TomSelect DOM updates before recalculating dropdown position. @@ -98,12 +123,25 @@ window.TomSelect = function createDynamicTomSelect(element) { }; } + if (element.dataset.render === 'transaction') { + config.dataAttr = 'data'; // dataset key of the data-data attribute + config.render.option = (data, escape) => renderTransaction(data, escape, false); + config.render.item = (data, escape) => renderTransaction(data, escape, true); + } + if (element.dataset.load) { + // The server searches and orders; don't let tom-select re-filter or re-sort the results + config.score = () => () => 1; + config.sortField = [{field: '$order'}]; + config.preload = 'focus'; + config.shouldLoad = () => true; config.load = function (query, callback) { let url = element.dataset.load + '?q=' + encodeURIComponent(query); fetch(url) .then(response => response.json()) .then(json => { + // Drop results from the previous query, keeping the selected item + this.clearOptions(); callback(json); }).catch(() => { callback(); diff --git a/frontend/src/styles/_tom-select.scss b/frontend/src/styles/_tom-select.scss index 73f5122a..5cd4637b 100644 --- a/frontend/src/styles/_tom-select.scss +++ b/frontend/src/styles/_tom-select.scss @@ -399,3 +399,98 @@ .tomselect-checkbox { margin-right: 0.5rem; } + +/* Mini transaction (TransactionSelect) */ +.ts-tx { + display: flex; + align-items: center; + gap: 0.75rem; + width: 100%; + min-width: 0; + padding: 0.375rem 0.5rem; + border-left: 4px solid var(--color-success); + border-radius: var(--radius-field, 0.25rem); + background-color: var(--color-base-100); + font-family: var(--font-mono, ui-monospace, monospace); + font-size: 0.8125rem; + line-height: 1.25rem; + text-align: left; +} + +.ts-tx-expense { + border-left-color: var(--color-error); +} + +.ts-tx-paid { + color: color-mix(in oklab, var(--color-base-content) 50%, transparent); + font-size: 1.125rem; +} + +.ts-tx-main, +.ts-tx-side { + min-width: 0; +} + +.ts-tx-main { + flex: 1 1 auto; +} + +.ts-tx-side { + flex: 0 1 auto; + max-width: 45%; + text-align: right; +} + +.ts-tx-date, +.ts-tx-meta, +.ts-tx-account { + color: color-mix(in oklab, var(--color-base-content) 70%, transparent); + font-size: 0.75rem; +} + +.ts-tx i.fa-fw { + margin-right: 0.25rem; + font-size: 0.625rem; +} + +.ts-tx-desc { + color: var(--color-base-content); + font-size: 0.875rem; +} + +.ts-tx-meta { + display: flex; + flex-wrap: wrap; + column-gap: 0.75rem; +} + +.ts-tx-amount { + font-weight: 500; + white-space: nowrap; +} + +.ts-tx-desc, +.ts-tx-account { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* Selected value inside the control: no card chrome, leave room for the clear button and arrow */ +.ts-control .ts-tx-compact { + padding: 0 1.5rem 0 0.5rem; + background: transparent; +} + +.ts-dropdown .option:has(.ts-tx) { + padding: 0.25rem 0.375rem; +} + +.ts-dropdown .active .ts-tx { + background-color: transparent; +} + +.ts-control > .item:has(.ts-tx) { + flex: 1 1 auto; + min-width: 0; +}