diff --git a/app/apps/currencies/exchange_rates/base.py b/app/apps/currencies/exchange_rates/base.py index 636d49fd..098ce5bd 100644 --- a/app/apps/currencies/exchange_rates/base.py +++ b/app/apps/currencies/exchange_rates/base.py @@ -8,9 +8,12 @@ from apps.currencies.models import Currency class ExchangeRateProvider(ABC): rates_inverted = False + # Help shown under the service's Options field. None hides the field + options_help = None - def __init__(self, api_key: Optional[str] = None): + def __init__(self, api_key: Optional[str] = None, options: str = ""): self.api_key = api_key + self.options = options or "" @abstractmethod def get_rates( @@ -24,6 +27,11 @@ class ExchangeRateProvider(ABC): """Return True if the service requires an API key""" return True + @classmethod + def clean_options(cls, options: str) -> str: + """Validate the raw Options text on save. Raise ValidationError if invalid""" + return options + @staticmethod def invert_rate(rate: Decimal) -> Decimal: """Invert the given rate.""" diff --git a/app/apps/currencies/exchange_rates/fetcher.py b/app/apps/currencies/exchange_rates/fetcher.py index d70f088b..6b486304 100644 --- a/app/apps/currencies/exchange_rates/fetcher.py +++ b/app/apps/currencies/exchange_rates/fetcher.py @@ -19,6 +19,7 @@ PROVIDER_MAPPING = { "twelvedatamarkets": providers.TwelveDataMarketsProvider, "yfinance": providers.YFinanceMarketsProvider, "fxmacrodata": providers.FXMacroDataProvider, + "allratestoday": providers.AllRatesTodayProvider, } diff --git a/app/apps/currencies/exchange_rates/providers.py b/app/apps/currencies/exchange_rates/providers.py index 4d79d08c..77c86e13 100644 --- a/app/apps/currencies/exchange_rates/providers.py +++ b/app/apps/currencies/exchange_rates/providers.py @@ -1,11 +1,14 @@ import logging +import re import time import requests from decimal import Decimal from typing import Tuple, List, Optional, Dict +from django.core.exceptions import ValidationError from django.db.models import QuerySet +from django.utils.translation import gettext_lazy as _ from apps.currencies.models import Currency, ExchangeRate from apps.currencies.exchange_rates.base import ExchangeRateProvider @@ -681,3 +684,166 @@ class FXMacroDataProvider(ExchangeRateProvider): ) return results + + +class AllRatesTodayProvider(ExchangeRateProvider): + """ + Implementation for the central bank tables served by allratestoday.com. + + Each service reads the latest published table of one central bank, picked by + the bank code in the Options field (e.g. "bcb" or "rbi"). An empty field + reads the ECB table. The endpoint needs no API key. + """ + + BASE_URL = "https://allratestoday.com/api/open/central-bank" + DEFAULT_BANK = "ecb" + BANK_CODE_RE = re.compile(r"[a-z0-9_-]{2,20}") + rates_inverted = False # A base/quote row says how many quote one base buys + options_help = _( + "Central bank code, e.g. 'bcb' or 'rbi'. Leave empty to use the ECB table." + ) + + # A pair can have several rows. Prefer a published mid, else the mean of a buy + # and sell quote. Period averages are left out, they are not a current rate + MID_TYPES = ( + "reference", + "middle", + "official", + "indicative", + "close", + "spot", + "informational", + ) + BUY_SELL_TYPES = (("buy", "sell"), ("tt_buy", "tt_sell")) + + def __init__(self, api_key: str = None, options: str = ""): + super().__init__(api_key, options) + self.session = requests.Session() + + @property + def bank(self) -> str: + return self.options.strip().lower() or self.DEFAULT_BANK + + @classmethod + def requires_api_key(cls) -> bool: + return False + + @classmethod + def clean_options(cls, options: str) -> str: + bank = options.strip().lower() + if bank and not cls.BANK_CODE_RE.fullmatch(bank): + raise ValidationError(_("Invalid bank code, it should look like 'bcb'.")) + return bank + + def _parse_table(self, rows: list) -> Dict[Tuple[str, str], Decimal]: + """Reduce the table to one rate per pair, in both directions""" + grouped = {} + for row in rows: + if not isinstance(row, dict): + continue + try: + value = Decimal(str(row.get("value"))) + except ArithmeticError: + continue + # A bad cell skips its own row, not the whole table + if not value.is_finite() or value <= 0: + continue + pair = (row.get("base"), row.get("quote")) + grouped.setdefault(pair, {})[row.get("type")] = value + + rates = {} + for (base, quote), values in grouped.items(): + rate = next((values[t] for t in self.MID_TYPES if t in values), None) + if rate is None: + rate = next( + ( + (values[buy] + values[sell]) / 2 + for buy, sell in self.BUY_SELL_TYPES + if buy in values and sell in values + ), + None, + ) + if rate is None: + logger.debug( + f"AllRatesToday {self.bank}: no usable rate type for " + f"{base}/{quote}, got {sorted(map(str, values))}" + ) + continue + rates[(base, quote)] = rate + rates.setdefault((quote, base), self.invert_rate(rate)) + + if grouped and not rates: + types = sorted({str(t) for values in grouped.values() for t in values}) + logger.warning( + f"AllRatesToday table for {self.bank} has no usable rate types, " + f"got {types}" + ) + + return rates + + @staticmethod + def _find_rate( + rates: Dict[Tuple[str, str], Decimal], base: str, quote: str + ) -> Optional[Decimal]: + """Look up a pair, crossing it through a shared currency if needed""" + if (base, quote) in rates: + return rates[(base, quote)] + + for (from_code, via), rate in rates.items(): + if from_code == base and (via, quote) in rates: + return rate * rates[(via, quote)] + + return None + + def get_rates( + self, target_currencies: QuerySet, exchange_currencies: set + ) -> List[Tuple[Currency, Currency, Decimal]]: + results = [] + + if not self.BANK_CODE_RE.fullmatch(self.bank): + logger.error("Invalid AllRatesToday bank code, it should look like 'bcb'") + return results + + try: + response = self.session.get(f"{self.BASE_URL}/{self.bank}", timeout=30) + response.raise_for_status() + data = response.json() + + if data.get("stale"): + logger.warning( + f"AllRatesToday table for {self.bank} is stale, " + f"last published on {data.get('rate_date')}" + ) + + rates = self._parse_table(data["rates"]) + except requests.RequestException as e: + logger.error(f"Error fetching AllRatesToday table for {self.bank}: {e}") + return results + except (KeyError, TypeError, ValueError, AttributeError, ArithmeticError) as e: + logger.error( + f"Unexpected response structure from AllRatesToday " + f"for {self.bank}: {e}" + ) + return results + + for target_currency in target_currencies: + if target_currency.exchange_currency not in exchange_currencies: + continue + + base_currency = target_currency.exchange_currency + + if base_currency.code == target_currency.code: + results.append((base_currency, target_currency, Decimal("1"))) + continue + + rate = self._find_rate(rates, base_currency.code, target_currency.code) + if rate is None: + logger.error( + f"No rate for {base_currency.code}/{target_currency.code} " + f"in the AllRatesToday table for {self.bank}" + ) + continue + + results.append((base_currency, target_currency, rate)) + + return results diff --git a/app/apps/currencies/forms.py b/app/apps/currencies/forms.py index 3c9ff64f..3090e8d5 100644 --- a/app/apps/currencies/forms.py +++ b/app/apps/currencies/forms.py @@ -1,13 +1,17 @@ +import json + from apps.common.widgets.crispy.daisyui import Switch from apps.common.widgets.crispy.submit import NoClassSubmit from apps.common.widgets.datepicker import AirDateTimePickerInput from apps.common.widgets.decimal import ArbitraryDecimalDisplayNumberInput from apps.common.widgets.tom_select import TomSelect +from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService from crispy_forms.bootstrap import FormActions from crispy_forms.helper import FormHelper from crispy_forms.layout import Column, Layout, Row from django import forms +from django.core.exceptions import ValidationError from django.forms import CharField from django.utils.translation import gettext_lazy as _ @@ -103,16 +107,41 @@ class ExchangeRateServiceForm(forms.ModelForm): "service_type", "is_active", "api_key", + "options", "interval_type", "fetch_interval", "target_currencies", "target_accounts", "singleton", ] + widgets = {"options": forms.Textarea(attrs={"rows": 2})} + + # Shows API Key and Options only for providers that use them + SERVICE_TYPE_SCRIPT = """ + init send change to me end + on change js(me) + const p = JSON.parse(me.dataset.providers)[me.value] || {}; + document.getElementById('div_id_api_key').classList.toggle('hidden', !p.api_key); + document.getElementById('div_id_options').classList.toggle('hidden', !p.options); + const help = document.getElementById('id_options_helptext'); + if (help) help.textContent = p.options || ''; + end + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + providers = { + key: { + "api_key": provider.requires_api_key(), + "options": str(provider.options_help or ""), + } + for key, provider in PROVIDER_MAPPING.items() + } + self.fields["service_type"].widget.attrs.update( + {"data-providers": json.dumps(providers), "_": self.SERVICE_TYPE_SCRIPT} + ) + self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" @@ -122,6 +151,7 @@ class ExchangeRateServiceForm(forms.ModelForm): Switch("is_active"), Switch("singleton"), "api_key", + "options", Row( Column("interval_type"), Column("fetch_interval"), @@ -142,3 +172,15 @@ class ExchangeRateServiceForm(forms.ModelForm): NoClassSubmit("submit", _("Add"), css_class="btn btn-primary"), ), ) + + def clean(self): + cleaned_data = super().clean() + provider = PROVIDER_MAPPING.get(cleaned_data.get("service_type")) + if provider: + try: + cleaned_data["options"] = provider.clean_options( + cleaned_data.get("options") or "" + ) + except ValidationError as e: + self.add_error("options", e) + return cleaned_data diff --git a/app/apps/currencies/migrations/0026_exchangerateservice_options.py b/app/apps/currencies/migrations/0026_exchangerateservice_options.py new file mode 100644 index 00000000..57681750 --- /dev/null +++ b/app/apps/currencies/migrations/0026_exchangerateservice_options.py @@ -0,0 +1,41 @@ +# Generated by Django 5.2.17 on 2026-11-08 05:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("currencies", "0025_alter_exchangerateservice_service_type"), + ] + + operations = [ + migrations.AddField( + model_name="exchangerateservice", + name="options", + field=models.TextField( + blank=True, + default="", + help_text="Provider specific settings", + verbose_name="Options", + ), + ), + migrations.AlterField( + model_name="exchangerateservice", + name="service_type", + field=models.CharField( + choices=[ + ("coingecko_free", "CoinGecko (Demo/Free)"), + ("coingecko_pro", "CoinGecko (Pro)"), + ("transitive", "Transitive (Calculated from Existing Rates)"), + ("frankfurter", "Frankfurter"), + ("twelvedata", "TwelveData"), + ("twelvedatamarkets", "TwelveData Markets"), + ("yfinance", "Yahoo Finance"), + ("fxmacrodata", "FXMacroData"), + ("allratestoday", "AllRatesToday Central Banks"), + ], + max_length=255, + verbose_name="Service Type", + ), + ), + ] diff --git a/app/apps/currencies/models.py b/app/apps/currencies/models.py index 89e4c583..cd01d36b 100644 --- a/app/apps/currencies/models.py +++ b/app/apps/currencies/models.py @@ -107,6 +107,7 @@ class ExchangeRateService(models.Model): TWELVEDATA_MARKETS = "twelvedatamarkets", "TwelveData Markets" YFINANCE = "yfinance", "Yahoo Finance" FXMACRODATA = "fxmacrodata", "FXMacroData" + ALLRATESTODAY = "allratestoday", "AllRatesToday Central Banks" class IntervalType(models.TextChoices): ON = "on", _("On") @@ -125,6 +126,12 @@ class ExchangeRateService(models.Model): verbose_name=_("API Key"), help_text=_("API key for the service (if required)"), ) + options = models.TextField( + blank=True, + default="", + verbose_name=_("Options"), + help_text=_("Provider specific settings"), + ) interval_type = models.CharField( max_length=255, choices=IntervalType.choices, @@ -180,7 +187,9 @@ class ExchangeRateService(models.Model): from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING provider_class = PROVIDER_MAPPING[self.service_type] - return provider_class(self.api_key) + provider = provider_class(self.api_key) + provider.options = self.options or "" + return provider @staticmethod def _parse_hour_ranges(interval_str: str) -> Set[int]: diff --git a/app/apps/currencies/tests/test_allratestoday_provider.py b/app/apps/currencies/tests/test_allratestoday_provider.py new file mode 100644 index 00000000..151ef94b --- /dev/null +++ b/app/apps/currencies/tests/test_allratestoday_provider.py @@ -0,0 +1,280 @@ +import json +from decimal import Decimal +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import requests +from django.core.exceptions import ValidationError +from django.test import TestCase as DjangoTestCase + +from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING +from apps.currencies.exchange_rates.providers import AllRatesTodayProvider +from apps.currencies.forms import ExchangeRateServiceForm +from apps.currencies.models import ExchangeRateService + + +class _FakeCurrency: + def __init__(self, code, exchange_currency=None): + self.code = code + self.exchange_currency = exchange_currency + + +def _row(base, quote, value, type_="reference"): + return {"base": base, "quote": quote, "type": type_, "value": value} + + +def _response(payload): + response = MagicMock() + response.status_code = 200 + response.json.return_value = payload + return response + + +class AllRatesTodayProviderTests(TestCase): + def setUp(self): + self.brl = _FakeCurrency("BRL") + self.usd = _FakeCurrency("USD") + self.eur = _FakeCurrency("EUR") + self.provider = AllRatesTodayProvider(options="bcb") + + def _get_rates(self, rows, target, base): + target.exchange_currency = base + with patch.object( + self.provider.session, "get", return_value=_response({"rates": rows}) + ): + return self.provider.get_rates([target], {base}) + + def test_requests_the_configured_bank(self): + self.usd.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5.4)]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ) as mock_get: + self.provider.get_rates([self.usd], {self.brl}) + + mock_get.assert_called_once_with( + "https://allratestoday.com/api/open/central-bank/bcb", timeout=30 + ) + + def test_fetches_one_table_for_all_pairs(self): + self.usd.exchange_currency = self.brl + self.eur.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5.4), _row("EUR", "BRL", 6.2)]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ) as mock_get: + rates = self.provider.get_rates([self.usd, self.eur], {self.brl}) + + mock_get.assert_called_once() + self.assertEqual(len(rates), 2) + + def test_empty_field_reads_the_ecb_table(self): + for options in (None, "", " "): + self.assertEqual(AllRatesTodayProvider(options=options).bank, "ecb") + + def test_normalizes_the_bank_code(self): + self.assertEqual(AllRatesTodayProvider(options=" BCB ").bank, "bcb") + + def test_ignores_the_api_key(self): + self.assertEqual(AllRatesTodayProvider("bcb").bank, "ecb") + + def test_clean_options_validates_the_bank_code(self): + self.assertEqual(AllRatesTodayProvider.clean_options(" BCB "), "bcb") + self.assertEqual(AllRatesTodayProvider.clean_options(""), "") + with self.assertRaises(ValidationError): + AllRatesTodayProvider.clean_options("../latest/USD") + + def test_rejects_malformed_bank_code_without_a_request(self): + provider = AllRatesTodayProvider(options="../latest/USD") + self.usd.exchange_currency = self.brl + + with patch.object(provider.session, "get") as mock_get: + rates = provider.get_rates([self.usd], {self.brl}) + + mock_get.assert_not_called() + self.assertEqual(rates, []) + + def test_uses_a_row_in_the_requested_direction(self): + rates = self._get_rates([_row("EUR", "USD", 1.16)], self.usd, self.eur) + + self.assertEqual(rates, [(self.eur, self.usd, Decimal("1.16"))]) + + def test_inverts_a_row_in_the_opposite_direction(self): + # bcb quotes foreign currencies in BRL, so BRL -> USD has to be inverted + rates = self._get_rates([_row("USD", "BRL", 5)], self.brl, self.usd) + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))]) + + rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.brl) + self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))]) + + def test_crosses_pairs_through_the_table_home_currency(self): + rows = [_row("USD", "BRL", 5), _row("EUR", "BRL", 6)] + + rates = self._get_rates(rows, self.eur, self.usd) + + self.assertEqual(len(rates), 1) + self.assertAlmostEqual(rates[0][2], Decimal("0.8333333"), places=6) + + def test_same_currency_is_one(self): + rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.usd) + + self.assertEqual(rates, [(self.usd, self.usd, Decimal("1"))]) + + def test_prefers_the_published_mid_over_buy_and_sell(self): + rows = [ + _row("USD", "BRL", 5.3, "buy"), + _row("USD", "BRL", 5.4, "middle"), + _row("USD", "BRL", 5.7, "sell"), + ] + + rates = self._get_rates(rows, self.brl, self.usd) + + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5.4"))]) + + def test_averages_buy_and_sell_without_a_mid(self): + rows = [_row("USD", "BRL", 4, "buy"), _row("USD", "BRL", 6, "sell")] + + rates = self._get_rates(rows, self.brl, self.usd) + + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))]) + + def test_averages_tt_buy_and_sell_without_a_mid(self): + rows = [ + _row("USD", "MUR", 44.0, "tt_buy"), + _row("USD", "MUR", 46.0, "tt_sell"), + ] + + result = self._get_rates(rows, self.usd, _FakeCurrency("MUR")) + + self.assertEqual(result[0][2], Decimal("1") / Decimal("45.0")) + + def test_ignores_rows_of_other_types(self): + rows = [_row("USD", "BRL", 5, "monthly_average")] + + self.assertEqual(self._get_rates(rows, self.brl, self.usd), []) + + def test_logs_a_table_without_usable_types(self): + self.usd.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5, "monthly_average")]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + with self.assertLogs( + "apps.currencies.exchange_rates.providers", level="WARNING" + ) as logs: + self.provider.get_rates([self.usd], {self.brl}) + + self.assertIn("monthly_average", logs.output[0]) + + def test_skips_pair_missing_from_the_table(self): + rows = [_row("USD", "BRL", 5)] + + self.assertEqual(self._get_rates(rows, self.eur, self.usd), []) + + def test_ignores_zero_and_malformed_rows(self): + rows = [ + "nope", + _row("USD", "BRL", 0), + _row("USD", "BRL", "5,4"), + _row("USD", "BRL", None), + _row("USD", "BRL", "Infinity"), + _row("USD", "BRL", "NaN"), + _row("EUR", "BRL", 6), + ] + + rates = self._get_rates(rows, self.brl, self.eur) + + self.assertEqual(rates, [(self.eur, self.brl, Decimal("6"))]) + + def test_logs_a_stale_table_and_still_uses_it(self): + self.usd.exchange_currency = self.brl + payload = { + "stale": True, + "rate_date": "2026-10-01", + "rates": [_row("BRL", "USD", 0.2)], + } + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + with self.assertLogs( + "apps.currencies.exchange_rates.providers", level="WARNING" + ) as logs: + rates = self.provider.get_rates([self.usd], {self.brl}) + + self.assertIn("2026-10-01", logs.output[0]) + self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))]) + + def test_returns_nothing_on_request_error(self): + self.usd.exchange_currency = self.brl + + with patch.object( + self.provider.session, + "get", + side_effect=requests.RequestException("404 Not Found"), + ): + rates = self.provider.get_rates([self.usd], {self.brl}) + + self.assertEqual(rates, []) + + def test_returns_nothing_on_malformed_body(self): + self.usd.exchange_currency = self.brl + + for payload in ({}, [], {"rates": None}): + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), []) + + def test_returns_nothing_on_non_json_body(self): + self.usd.exchange_currency = self.brl + response = _response({}) + response.json.side_effect = ValueError("not json") + + with patch.object(self.provider.session, "get", return_value=response): + self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), []) + + def test_is_registered_without_an_api_key(self): + self.assertFalse(AllRatesTodayProvider.requires_api_key()) + self.assertIs(PROVIDER_MAPPING["allratestoday"], AllRatesTodayProvider) + self.assertEqual(ExchangeRateService.ServiceType.ALLRATESTODAY, "allratestoday") + + def test_service_passes_its_options_to_the_provider(self): + service = ExchangeRateService(service_type="allratestoday", options="rbi") + self.assertEqual(service.get_provider().bank, "rbi") + + +class ExchangeRateServiceFormOptionsTests(DjangoTestCase): + def _form(self, **data): + return ExchangeRateServiceForm( + data={ + "name": "Rates", + "service_type": "allratestoday", + "interval_type": "every", + "fetch_interval": "24", + **data, + } + ) + + def test_saves_cleaned_options(self): + form = self._form(options=" BCB ") + + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.save().options, "bcb") + + def test_rejects_invalid_options_on_save(self): + form = self._form(options="../latest/USD") + + self.assertFalse(form.is_valid()) + self.assertIn("options", form.errors) + + def test_tells_the_page_which_fields_each_provider_uses(self): + attrs = self._form().fields["service_type"].widget.attrs + providers = json.loads(attrs["data-providers"]) + + self.assertEqual(providers["allratestoday"]["api_key"], False) + self.assertIn("bank code", providers["allratestoday"]["options"]) + self.assertEqual(providers["twelvedata"], {"api_key": True, "options": ""})