From c171e0419ad0b55936df5abb8871288c74e12870 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 16 Jan 2025 14:09:33 -0300 Subject: [PATCH 01/60] feat: add import app boilerplate --- app/WYGIWYH/settings.py | 1 + app/apps/import/__init__.py | 0 app/apps/import/admin.py | 3 +++ app/apps/import/apps.py | 6 ++++++ app/apps/import/migrations/__init__.py | 0 app/apps/import/models.py | 3 +++ app/apps/import/tests.py | 3 +++ app/apps/import/views.py | 3 +++ 8 files changed, 19 insertions(+) create mode 100644 app/apps/import/__init__.py create mode 100644 app/apps/import/admin.py create mode 100644 app/apps/import/apps.py create mode 100644 app/apps/import/migrations/__init__.py create mode 100644 app/apps/import/models.py create mode 100644 app/apps/import/tests.py create mode 100644 app/apps/import/views.py diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index d10dddd..e4e7c73 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -64,6 +64,7 @@ INSTALLED_APPS = [ "apps.accounts.apps.AccountsConfig", "apps.common.apps.CommonConfig", "apps.net_worth.apps.NetWorthConfig", + "apps.import.apps.ImportConfig", "apps.api.apps.ApiConfig", "cachalot", "rest_framework", diff --git a/app/apps/import/__init__.py b/app/apps/import/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import/admin.py b/app/apps/import/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/app/apps/import/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/app/apps/import/apps.py b/app/apps/import/apps.py new file mode 100644 index 0000000..fdfa08d --- /dev/null +++ b/app/apps/import/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ImportConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "apps.import" diff --git a/app/apps/import/migrations/__init__.py b/app/apps/import/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import/models.py b/app/apps/import/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/app/apps/import/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/app/apps/import/tests.py b/app/apps/import/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/app/apps/import/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/app/apps/import/views.py b/app/apps/import/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/app/apps/import/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From fbb26b8442c744438388fb6544a7477256fdd187 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Fri, 17 Jan 2025 17:40:51 -0300 Subject: [PATCH 02/60] feat: rename app, some work on schema --- app/WYGIWYH/settings.py | 2 +- app/WYGIWYH/urls.py | 1 + app/apps/import/admin.py | 3 - app/apps/import/models.py | 3 - app/apps/import/views.py | 3 - app/apps/{import => import_app}/__init__.py | 0 app/apps/import_app/admin.py | 6 + app/apps/{import => import_app}/apps.py | 2 +- .../migrations/__init__.py | 0 app/apps/import_app/models.py | 74 ++++++ app/apps/import_app/schemas.py | 0 app/apps/import_app/schemas/__init__.py | 8 + app/apps/import_app/schemas/v1.py | 104 ++++++++ app/apps/import_app/services.py | 0 app/apps/import_app/services/__init__.py | 1 + app/apps/import_app/services/v1.py | 237 ++++++++++++++++++ app/apps/import_app/tasks.py | 18 ++ app/apps/{import => import_app}/tests.py | 0 app/apps/import_app/urls.py | 6 + app/apps/import_app/views.py | 26 ++ app/apps/transactions/models.py | 1 + requirements.txt | 2 + 22 files changed, 486 insertions(+), 11 deletions(-) delete mode 100644 app/apps/import/admin.py delete mode 100644 app/apps/import/models.py delete mode 100644 app/apps/import/views.py rename app/apps/{import => import_app}/__init__.py (100%) create mode 100644 app/apps/import_app/admin.py rename app/apps/{import => import_app}/apps.py (81%) rename app/apps/{import => import_app}/migrations/__init__.py (100%) create mode 100644 app/apps/import_app/models.py create mode 100644 app/apps/import_app/schemas.py create mode 100644 app/apps/import_app/schemas/__init__.py create mode 100644 app/apps/import_app/schemas/v1.py create mode 100644 app/apps/import_app/services.py create mode 100644 app/apps/import_app/services/__init__.py create mode 100644 app/apps/import_app/services/v1.py create mode 100644 app/apps/import_app/tasks.py rename app/apps/{import => import_app}/tests.py (100%) create mode 100644 app/apps/import_app/urls.py create mode 100644 app/apps/import_app/views.py diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index e4e7c73..8243c91 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -64,7 +64,7 @@ INSTALLED_APPS = [ "apps.accounts.apps.AccountsConfig", "apps.common.apps.CommonConfig", "apps.net_worth.apps.NetWorthConfig", - "apps.import.apps.ImportConfig", + "apps.import_app.apps.ImportConfig", "apps.api.apps.ApiConfig", "cachalot", "rest_framework", diff --git a/app/WYGIWYH/urls.py b/app/WYGIWYH/urls.py index 5a465a5..eb4357d 100644 --- a/app/WYGIWYH/urls.py +++ b/app/WYGIWYH/urls.py @@ -47,4 +47,5 @@ urlpatterns = [ path("", include("apps.calendar_view.urls")), path("", include("apps.dca.urls")), path("", include("apps.mini_tools.urls")), + path("", include("apps.import_app.urls")), ] diff --git a/app/apps/import/admin.py b/app/apps/import/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/app/apps/import/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/app/apps/import/models.py b/app/apps/import/models.py deleted file mode 100644 index 71a8362..0000000 --- a/app/apps/import/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/app/apps/import/views.py b/app/apps/import/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/app/apps/import/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/app/apps/import/__init__.py b/app/apps/import_app/__init__.py similarity index 100% rename from app/apps/import/__init__.py rename to app/apps/import_app/__init__.py diff --git a/app/apps/import_app/admin.py b/app/apps/import_app/admin.py new file mode 100644 index 0000000..cbccf2b --- /dev/null +++ b/app/apps/import_app/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from apps.import_app import models + +# Register your models here. +admin.site.register(models.ImportRun) +admin.site.register(models.ImportProfile) diff --git a/app/apps/import/apps.py b/app/apps/import_app/apps.py similarity index 81% rename from app/apps/import/apps.py rename to app/apps/import_app/apps.py index fdfa08d..4dbe90c 100644 --- a/app/apps/import/apps.py +++ b/app/apps/import_app/apps.py @@ -3,4 +3,4 @@ from django.apps import AppConfig class ImportConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "apps.import" + name = "apps.import_app" diff --git a/app/apps/import/migrations/__init__.py b/app/apps/import_app/migrations/__init__.py similarity index 100% rename from app/apps/import/migrations/__init__.py rename to app/apps/import_app/migrations/__init__.py diff --git a/app/apps/import_app/models.py b/app/apps/import_app/models.py new file mode 100644 index 0000000..aca04e3 --- /dev/null +++ b/app/apps/import_app/models.py @@ -0,0 +1,74 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class ImportProfile(models.Model): + class Versions(models.IntegerChoices): + VERSION_1 = 1, _("Version 1") + + name = models.CharField(max_length=100) + yaml_config = models.TextField(help_text=_("YAML configuration")) + version = models.IntegerField( + choices=Versions, + default=Versions.VERSION_1, + verbose_name=_("Version"), + ) + + def __str__(self): + return self.name + + class Meta: + ordering = ["name"] + + +class ImportRun(models.Model): + class Status(models.TextChoices): + QUEUED = "QUEUED", _("Queued") + PROCESSING = "PROCESSING", _("Processing") + FAILED = "FAILED", _("Failed") + FINISHED = "FINISHED", _("Finished") + + status = models.CharField( + max_length=10, + choices=Status, + default=Status.QUEUED, + verbose_name=_("Status"), + ) + profile = models.ForeignKey( + ImportProfile, + on_delete=models.CASCADE, + ) + file_name = models.CharField( + max_length=10000, + help_text=_("File name"), + ) + transactions = models.ManyToManyField( + "transactions.Transaction", related_name="import_runs" + ) + tags = models.ManyToManyField( + "transactions.TransactionTag", related_name="import_runs" + ) + categories = models.ManyToManyField( + "transactions.TransactionCategory", related_name="import_runs" + ) + entities = models.ManyToManyField( + "transactions.TransactionEntity", related_name="import_runs" + ) + currencies = models.ManyToManyField( + "currencies.Currency", related_name="import_runs" + ) + + logs = models.TextField(blank=True) + processed_rows = models.IntegerField(default=0) + total_rows = models.IntegerField(default=0) + successful_rows = models.IntegerField(default=0) + skipped_rows = models.IntegerField(default=0) + failed_rows = models.IntegerField(default=0) + started_at = models.DateTimeField(null=True) + finished_at = models.DateTimeField(null=True) + + @property + def progress(self): + if self.total_rows == 0: + return 0 + return (self.processed_rows / self.total_rows) * 100 diff --git a/app/apps/import_app/schemas.py b/app/apps/import_app/schemas.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import_app/schemas/__init__.py b/app/apps/import_app/schemas/__init__.py new file mode 100644 index 0000000..f68ce79 --- /dev/null +++ b/app/apps/import_app/schemas/__init__.py @@ -0,0 +1,8 @@ +from apps.import_app.schemas.v1 import ( + ImportProfileSchema as SchemaV1, + ColumnMapping as ColumnMappingV1, + # TransformationRule as TransformationRuleV1, + ImportSettings as SettingsV1, + HashTransformationRule as HashTransformationRuleV1, + CompareDeduplicationRule as CompareDeduplicationRuleV1, +) diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py new file mode 100644 index 0000000..1cc7dc5 --- /dev/null +++ b/app/apps/import_app/schemas/v1.py @@ -0,0 +1,104 @@ +from typing import Dict, List, Optional, Literal +from pydantic import BaseModel, Field + + +class CompareDeduplicationRule(BaseModel): + type: Literal["compare"] + fields: Dict = Field( + ..., description="Match header and fields to compare for deduplication" + ) + match_type: Literal["lax", "strict"] + + +class ReplaceTransformationRule(BaseModel): + field: str + type: Literal["replace", "regex"] = Field( + ..., description="Type of transformation: replace or regex" + ) + pattern: str = Field(..., description="Pattern to match") + replacement: str = Field(..., description="Value to replace with") + + +class DateFormatTransformationRule(BaseModel): + field: str + type: Literal["date_format"] = Field( + ..., description="Type of transformation: replace or regex" + ) + original_format: str = Field(..., description="Original date format") + new_format: str = Field(..., description="New date format to use") + + +class HashTransformationRule(BaseModel): + fields: List[str] + type: Literal["hash"] + + +class MergeTransformationRule(BaseModel): + fields: List[str] + type: Literal["merge"] + separator: str = Field(default=" ", description="Separator to use when merging") + + +class SplitTransformationRule(BaseModel): + fields: List[str] + type: Literal["split"] + separator: str = Field(default=",", description="Separator to use when splitting") + index: int | None = Field( + default=0, description="Index to return as value. Empty to return all." + ) + + +class ImportSettings(BaseModel): + skip_errors: bool = Field( + default=False, + description="If True, errors during import will be logged and skipped", + ) + file_type: Literal["csv"] = "csv" + delimiter: str = Field(default=",", description="CSV delimiter character") + encoding: str = Field(default="utf-8", description="File encoding") + skip_rows: int = Field( + default=0, description="Number of rows to skip at the beginning of the file" + ) + importing: Literal[ + "transactions", "accounts", "currencies", "categories", "tags", "entities" + ] + + +class ColumnMapping(BaseModel): + source: Optional[str] = Field( + default=None, + description="CSV column header. If None, the field will be generated from transformations", + ) + target: Literal[ + "account", + "type", + "is_paid", + "date", + "reference_date", + "amount", + "notes", + "category", + "tags", + "entities", + "internal_note", + ] = Field(..., description="Transaction field to map to") + default_value: Optional[str] = None + required: bool = False + transformations: Optional[ + List[ + ReplaceTransformationRule + | DateFormatTransformationRule + | HashTransformationRule + | MergeTransformationRule + | SplitTransformationRule + ] + ] = Field(default_factory=list) + + +class ImportProfileSchema(BaseModel): + settings: ImportSettings + column_mapping: Dict[str, ColumnMapping] + deduplication: List[CompareDeduplicationRule] = Field( + default_factory=list, + description="Rules for deduplicating records during import", + ) diff --git a/app/apps/import_app/services.py b/app/apps/import_app/services.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import_app/services/__init__.py b/app/apps/import_app/services/__init__.py new file mode 100644 index 0000000..6001902 --- /dev/null +++ b/app/apps/import_app/services/__init__.py @@ -0,0 +1 @@ +from apps.import_app.services.v1 import ImportService as ImportServiceV1 diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py new file mode 100644 index 0000000..333eb6e --- /dev/null +++ b/app/apps/import_app/services/v1.py @@ -0,0 +1,237 @@ +import csv +import hashlib +import re +from datetime import datetime +from typing import Dict, Any, Literal + +import yaml + +from django.db import transaction +from django.core.files.storage import default_storage +from django.utils import timezone + +from apps.import_app.models import ImportRun, ImportProfile +from apps.import_app.schemas import ( + SchemaV1, + ColumnMappingV1, + SettingsV1, + HashTransformationRuleV1, + CompareDeduplicationRuleV1, +) +from apps.transactions.models import Transaction + + +class ImportService: + def __init__(self, import_run: ImportRun): + self.import_run: ImportRun = import_run + self.profile: ImportProfile = import_run.profile + self.config: SchemaV1 = self._load_config() + self.settings: SettingsV1 = self.config.settings + self.deduplication: list[CompareDeduplicationRuleV1] = self.config.deduplication + self.mapping: Dict[str, ColumnMappingV1] = self.config.column_mapping + + def _load_config(self) -> SchemaV1: + yaml_data = yaml.safe_load(self.profile.yaml_config) + + if self.profile.version == ImportProfile.Versions.VERSION_1: + return SchemaV1(**yaml_data) + + raise ValueError(f"Unsupported version: {self.profile.version}") + + def _log(self, level: str, message: str, **kwargs) -> None: + """Add a log entry to the import run logs""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # Format additional context if present + context = "" + if kwargs: + context = " - " + ", ".join(f"{k}={v}" for k, v in kwargs.items()) + + log_line = f"[{timestamp}] {level.upper()}: {message}{context}\n" + + # Append to existing logs + self.import_run.logs += log_line + self.import_run.save(update_fields=["logs"]) + + def _update_status( + self, new_status: Literal["PROCESSING", "FAILED", "FINISHED"] + ) -> None: + if new_status == "PROCESSING": + self.import_run.status = ImportRun.Status.PROCESSING + elif new_status == "FAILED": + self.import_run.status = ImportRun.Status.FAILED + elif new_status == "FINISHED": + self.import_run.status = ImportRun.Status.FINISHED + + self.import_run.save(update_fields=["status"]) + + @staticmethod + def _transform_value( + value: str, mapping: ColumnMappingV1, row: Dict[str, str] = None + ) -> Any: + transformed = value + + for transform in mapping.transformations: + if transform.type == "hash": + if not isinstance(transform, HashTransformationRuleV1): + continue + + # Collect all values to be hashed + values_to_hash = [] + for field in transform.fields: + if field in row: + values_to_hash.append(str(row[field])) + + # Create hash from concatenated values + if values_to_hash: + concatenated = "|".join(values_to_hash) + transformed = hashlib.sha256(concatenated.encode()).hexdigest() + + elif transform.type == "replace": + transformed = transformed.replace( + transform.pattern, transform.replacement + ) + elif transform.type == "regex": + transformed = re.sub( + transform.pattern, transform.replacement, transformed + ) + elif transform.type == "date_format": + transformed = datetime.strptime( + transformed, transform.pattern + ).strftime(transform.replacement) + + return transformed + + def _map_row_to_transaction(self, row: Dict[str, str]) -> Dict[str, Any]: + transaction_data = {} + + for field, mapping in self.mapping.items(): + # If source is None, use None as the initial value + value = row.get(mapping.source) if mapping.source else None + + # Use default_value if value is None + if value is None: + value = mapping.default_value + + if mapping.required and value is None and not mapping.transformations: + raise ValueError(f"Required field {field} is missing") + + # Apply transformations even if initial value is None + if mapping.transformations: + value = self._transform_value(value, mapping, row) + + if value is not None: + transaction_data[field] = value + + return transaction_data + + def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool: + for rule in self.deduplication: + if rule.type == "compare": + query = Transaction.objects.all() + + # Build query conditions for each field in the rule + for field, header in rule.fields.items(): + if field in transaction_data: + if rule.match_type == "strict": + query = query.filter(**{field: transaction_data[field]}) + else: # lax matching + query = query.filter( + **{f"{field}__iexact": transaction_data[field]} + ) + + # If we found any matching transaction, it's a duplicate + if query.exists(): + return True + + return False + + def _process_csv(self, file_path): + with open(file_path, "r", encoding=self.settings.encoding) as csv_file: + reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + + # Count total rows + self.import_run.total_rows = sum(1 for _ in reader) + csv_file.seek(0) + reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + + self._log("info", f"Starting import with {self.import_run.total_rows} rows") + + # Skip specified number of rows + for _ in range(self.settings.skip_rows): + next(reader) + + if self.settings.skip_rows: + self._log("info", f"Skipped {self.settings.skip_rows} initial rows") + + for row_number, row in enumerate(reader, start=1): + try: + transaction_data = self._map_row_to_transaction(row) + + if transaction_data: + if self.deduplication and self._check_duplicate_transaction( + transaction_data + ): + self.import_run.skipped_rows += 1 + self._log("info", f"Skipped duplicate row {row_number}") + continue + + self.import_run.transactions.add(transaction_data) + self.import_run.successful_rows += 1 + self._log("debug", f"Successfully processed row {row_number}") + + self.import_run.processed_rows += 1 + self.import_run.save( + update_fields=[ + "processed_rows", + "successful_rows", + "skipped_rows", + ] + ) + + except Exception as e: + if not self.settings.skip_errors: + self._log( + "error", + f"Fatal error processing row {row_number}: {str(e)}", + ) + self._update_status("FAILED") + raise + else: + self._log( + "warning", f"Error processing row {row_number}: {str(e)}" + ) + self.import_run.failed_rows += 1 + self.import_run.save(update_fields=["failed_rows"]) + + def process_file(self, file_path: str): + self._update_status("PROCESSING") + self.import_run.started_at = timezone.now() + self.import_run.save(update_fields=["started_at"]) + + self._log("info", "Starting import process") + + try: + if self.settings.file_type == "csv": + self._process_csv(file_path) + + if self.import_run.processed_rows == self.import_run.total_rows: + self._update_status("FINISHED") + self._log( + "info", + f"Import completed successfully. " + f"Successful: {self.import_run.successful_rows}, " + f"Failed: {self.import_run.failed_rows}, " + f"Skipped: {self.import_run.skipped_rows}", + ) + + except Exception as e: + self._update_status("FAILED") + self._log("error", f"Import failed: {str(e)}") + raise Exception("Import failed") + + finally: + self._log("info", "Cleaning up temporary files") + default_storage.delete(file_path) + self.import_run.finished_at = timezone.now() + self.import_run.save(update_fields=["finished_at"]) diff --git a/app/apps/import_app/tasks.py b/app/apps/import_app/tasks.py new file mode 100644 index 0000000..25efcbc --- /dev/null +++ b/app/apps/import_app/tasks.py @@ -0,0 +1,18 @@ +import logging + +from procrastinate.contrib.django import app + +from apps.import_app.models import ImportRun +from apps.import_app.services import ImportServiceV1 + +logger = logging.getLogger(__name__) + + +@app.task(queue="imports") +def process_import(import_run_id: int, file_path: str): + try: + import_run = ImportRun.objects.get(id=import_run_id) + import_service = ImportServiceV1(import_run) + import_service.process_file(file_path) + except ImportRun.DoesNotExist: + raise ValueError(f"ImportRun with id {import_run_id} not found") diff --git a/app/apps/import/tests.py b/app/apps/import_app/tests.py similarity index 100% rename from app/apps/import/tests.py rename to app/apps/import_app/tests.py diff --git a/app/apps/import_app/urls.py b/app/apps/import_app/urls.py new file mode 100644 index 0000000..aea8670 --- /dev/null +++ b/app/apps/import_app/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +import apps.import_app.views as views + +urlpatterns = [ + path("import/", views.ImportRunCreateView.as_view(), name="import"), +] diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py new file mode 100644 index 0000000..d5b1d94 --- /dev/null +++ b/app/apps/import_app/views.py @@ -0,0 +1,26 @@ +from django.views.generic import CreateView +from apps.import_app.models import ImportRun +from apps.import_app.services import ImportServiceV1 + + +class ImportRunCreateView(CreateView): + model = ImportRun + fields = ["profile"] + + def form_valid(self, form): + response = super().form_valid(form) + + import_run = form.instance + file = self.request.FILES["file"] + + # Save uploaded file temporarily + temp_file_path = f"/tmp/import_{import_run.id}.csv" + with open(temp_file_path, "wb+") as destination: + for chunk in file.chunks(): + destination.write(chunk) + + # Process the import + import_service = ImportServiceV1(import_run) + import_service.process_file(temp_file_path) + + return response diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index 70bbc94..f131518 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -141,6 +141,7 @@ class Transaction(models.Model): related_name="transactions", verbose_name=_("Recurring Transaction"), ) + internal_note = models.TextField(blank=True, verbose_name=_("Internal Note")) class Meta: verbose_name = _("Transaction") diff --git a/requirements.txt b/requirements.txt index b4e4f02..af9d39b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,5 @@ requests~=2.32.3 pytz~=2024.2 python-dateutil~=2.9.0.post0 simpleeval~=1.0.0 +pydantic~=2.10.5 +PyYAML~=6.0.2 From 86dac632c4bc9edd949aef294844563fb207fa46 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 11:27:14 -0300 Subject: [PATCH 03/60] feat(import): improve schema definition --- app/apps/import_app/schemas/__init__.py | 9 +- app/apps/import_app/schemas/v1.py | 330 +++++++++++++++- app/apps/import_app/services/v1.py | 506 +++++++++++++++++++----- 3 files changed, 717 insertions(+), 128 deletions(-) diff --git a/app/apps/import_app/schemas/__init__.py b/app/apps/import_app/schemas/__init__.py index f68ce79..530268d 100644 --- a/app/apps/import_app/schemas/__init__.py +++ b/app/apps/import_app/schemas/__init__.py @@ -1,8 +1 @@ -from apps.import_app.schemas.v1 import ( - ImportProfileSchema as SchemaV1, - ColumnMapping as ColumnMappingV1, - # TransformationRule as TransformationRuleV1, - ImportSettings as SettingsV1, - HashTransformationRule as HashTransformationRuleV1, - CompareDeduplicationRule as CompareDeduplicationRuleV1, -) +import apps.import_app.schemas.v1 as version_1 diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 1cc7dc5..043f2a9 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -1,5 +1,5 @@ from typing import Dict, List, Optional, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator, field_validator class CompareDeduplicationRule(BaseModel): @@ -9,6 +9,12 @@ class CompareDeduplicationRule(BaseModel): ) match_type: Literal["lax", "strict"] + @field_validator("fields", mode="before") + def coerce_fields_to_dict(cls, v): + if isinstance(v, list): + return {k: v for d in v for k, v in d.items()} + return v + class ReplaceTransformationRule(BaseModel): field: str @@ -17,6 +23,10 @@ class ReplaceTransformationRule(BaseModel): ) pattern: str = Field(..., description="Pattern to match") replacement: str = Field(..., description="Value to replace with") + exclusive: bool = Field( + default=False, + description="If it should match against the last transformation or the original value", + ) class DateFormatTransformationRule(BaseModel): @@ -48,7 +58,7 @@ class SplitTransformationRule(BaseModel): ) -class ImportSettings(BaseModel): +class CSVImportSettings(BaseModel): skip_errors: bool = Field( default=False, description="If True, errors during import will be logged and skipped", @@ -56,7 +66,7 @@ class ImportSettings(BaseModel): file_type: Literal["csv"] = "csv" delimiter: str = Field(default=",", description="CSV delimiter character") encoding: str = Field(default="utf-8", description="File encoding") - skip_rows: int = Field( + skip_lines: int = Field( default=0, description="Number of rows to skip at the beginning of the file" ) importing: Literal[ @@ -69,20 +79,7 @@ class ColumnMapping(BaseModel): default=None, description="CSV column header. If None, the field will be generated from transformations", ) - target: Literal[ - "account", - "type", - "is_paid", - "date", - "reference_date", - "amount", - "notes", - "category", - "tags", - "entities", - "internal_note", - ] = Field(..., description="Transaction field to map to") - default_value: Optional[str] = None + default: Optional[str] = None required: bool = False transformations: Optional[ List[ @@ -95,10 +92,305 @@ class ColumnMapping(BaseModel): ] = Field(default_factory=list) +class TransactionAccountMapping(ColumnMapping): + target: Literal["account"] = Field(..., description="Transaction field to map to") + type: Literal["id", "name"] = "name" + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class TransactionTypeMapping(ColumnMapping): + target: Literal["type"] = Field(..., description="Transaction field to map to") + detection_method: Literal["sign", "always_income", "always_expense"] = "sign" + coerce_to: Literal["transaction_type"] = Field("transaction_type", frozen=True) + + +class TransactionIsPaidMapping(ColumnMapping): + target: Literal["is_paid"] = Field(..., description="Transaction field to map to") + detection_method: Literal["sign", "boolean", "always_paid", "always_unpaid"] + coerce_to: Literal["is_paid"] = Field("is_paid", frozen=True) + + +class TransactionDateMapping(ColumnMapping): + target: Literal["date"] = Field(..., description="Transaction field to map to") + format: List[str] | str + coerce_to: Literal["date"] = Field("date", frozen=True) + + +class TransactionReferenceDateMapping(ColumnMapping): + target: Literal["reference_date"] = Field( + ..., description="Transaction field to map to" + ) + format: List[str] | str + coerce_to: Literal["date"] = Field("date", frozen=True) + + +class TransactionAmountMapping(ColumnMapping): + target: Literal["amount"] = Field(..., description="Transaction field to map to") + coerce_to: Literal["positive_decimal"] = Field("positive_decimal", frozen=True) + + +class TransactionDescriptionMapping(ColumnMapping): + target: Literal["description"] = Field( + ..., description="Transaction field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class TransactionNotesMapping(ColumnMapping): + target: Literal["notes"] = Field(..., description="Transaction field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class TransactionTagsMapping(ColumnMapping): + target: Literal["tags"] = Field(..., description="Transaction field to map to") + create: bool = Field( + default=True, description="Create new tags if they doesn't exist" + ) + coerce_to: Literal["list"] = Field("list", frozen=True) + + +class TransactionEntitiesMapping(ColumnMapping): + target: Literal["entities"] = Field(..., description="Transaction field to map to") + create: bool = Field( + default=True, description="Create new entities if they doesn't exist" + ) + coerce_to: Literal["list"] = Field("list", frozen=True) + + +class TransactionCategoryMapping(ColumnMapping): + target: Literal["category"] = Field(..., description="Transaction field to map to") + create: bool = Field( + default=True, description="Create category if it doesn't exist" + ) + type: Literal["id", "name"] = "name" + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class TransactionInternalMapping(ColumnMapping): + target: Literal["internal_note"] = Field( + ..., description="Transaction field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CategoryNameMapping(ColumnMapping): + target: Literal["category_name"] = Field( + ..., description="Category field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CategoryMuteMapping(ColumnMapping): + target: Literal["category_mute"] = Field( + ..., description="Category field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class CategoryActiveMapping(ColumnMapping): + target: Literal["category_active"] = Field( + ..., description="Category field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class TagNameMapping(ColumnMapping): + target: Literal["tag_name"] = Field(..., description="Tag field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class TagActiveMapping(ColumnMapping): + target: Literal["tag_active"] = Field(..., description="Tag field to map to") + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class EntityNameMapping(ColumnMapping): + target: Literal["entity_name"] = Field(..., description="Entity field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class EntityActiveMapping(ColumnMapping): + target: Literal["entitiy_active"] = Field(..., description="Entity field to map to") + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class AccountNameMapping(ColumnMapping): + target: Literal["account_name"] = Field(..., description="Account field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class AccountGroupMapping(ColumnMapping): + target: Literal["account_group"] = Field(..., description="Account field to map to") + type: Literal["id", "name"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class AccountCurrencyMapping(ColumnMapping): + target: Literal["account_currency"] = Field( + ..., description="Account field to map to" + ) + type: Literal["id", "name", "code"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class AccountExchangeCurrencyMapping(ColumnMapping): + target: Literal["account_exchange_currency"] = Field( + ..., description="Account field to map to" + ) + type: Literal["id", "name", "code"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class AccountIsAssetMapping(ColumnMapping): + target: Literal["account_is_asset"] = Field( + ..., description="Account field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class AccountIsArchivedMapping(ColumnMapping): + target: Literal["account_is_archived"] = Field( + ..., description="Account field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class CurrencyCodeMapping(ColumnMapping): + target: Literal["currency_code"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencyNameMapping(ColumnMapping): + target: Literal["currency_name"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencyDecimalPlacesMapping(ColumnMapping): + target: Literal["currency_decimal_places"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["int"] = Field("int", frozen=True) + + +class CurrencyPrefixMapping(ColumnMapping): + target: Literal["currency_prefix"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencySuffixMapping(ColumnMapping): + target: Literal["currency_suffix"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencyExchangeMapping(ColumnMapping): + target: Literal["currency_exchange"] = Field( + ..., description="Currency field to map to" + ) + type: Literal["id", "name", "code"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + class ImportProfileSchema(BaseModel): - settings: ImportSettings - column_mapping: Dict[str, ColumnMapping] + settings: CSVImportSettings + mapping: Dict[ + str, + TransactionAccountMapping + | TransactionTypeMapping + | TransactionIsPaidMapping + | TransactionDateMapping + | TransactionReferenceDateMapping + | TransactionAmountMapping + | TransactionDescriptionMapping + | TransactionNotesMapping + | TransactionTagsMapping + | TransactionEntitiesMapping + | TransactionCategoryMapping + | TransactionInternalMapping + | CategoryNameMapping + | CategoryMuteMapping + | CategoryActiveMapping + | TagNameMapping + | TagActiveMapping + | EntityNameMapping + | EntityActiveMapping + | AccountNameMapping + | AccountGroupMapping + | AccountCurrencyMapping + | AccountExchangeCurrencyMapping + | AccountIsAssetMapping + | AccountIsArchivedMapping + | CurrencyCodeMapping + | CurrencyNameMapping + | CurrencyDecimalPlacesMapping + | CurrencyPrefixMapping + | CurrencySuffixMapping + | CurrencyExchangeMapping, + ] deduplication: List[CompareDeduplicationRule] = Field( default_factory=list, description="Rules for deduplicating records during import", ) + + @model_validator(mode="after") + def validate_mappings(self) -> "ImportProfileSchema": + import_type = self.settings.importing + + # Define allowed mapping types for each import type + allowed_mappings = { + "transactions": ( + TransactionAccountMapping, + TransactionTypeMapping, + TransactionIsPaidMapping, + TransactionDateMapping, + TransactionReferenceDateMapping, + TransactionAmountMapping, + TransactionDescriptionMapping, + TransactionNotesMapping, + TransactionTagsMapping, + TransactionEntitiesMapping, + TransactionCategoryMapping, + TransactionInternalMapping, + ), + "accounts": ( + AccountNameMapping, + AccountGroupMapping, + AccountCurrencyMapping, + AccountExchangeCurrencyMapping, + AccountIsAssetMapping, + AccountIsArchivedMapping, + ), + "currencies": ( + CurrencyCodeMapping, + CurrencyNameMapping, + CurrencyDecimalPlacesMapping, + CurrencyPrefixMapping, + CurrencySuffixMapping, + CurrencyExchangeMapping, + ), + "categories": ( + CategoryNameMapping, + CategoryMuteMapping, + CategoryActiveMapping, + ), + "tags": (TagNameMapping, TagActiveMapping), + "entities": (EntityNameMapping, EntityActiveMapping), + } + + allowed_types = allowed_mappings[import_type] + + for field_name, mapping in self.mapping.items(): + if not isinstance(mapping, allowed_types): + raise ValueError( + f"Mapping type '{type(mapping).__name__}' is not allowed when importing {import_type}. " + f"Allowed types are: {', '.join(t.__name__ for t in allowed_types)}" + ) + + return self diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 333eb6e..069115b 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -1,42 +1,56 @@ import csv import hashlib +import logging +import os import re from datetime import datetime -from typing import Dict, Any, Literal +from decimal import Decimal +from typing import Dict, Any, Literal, Union import yaml - from django.db import transaction -from django.core.files.storage import default_storage from django.utils import timezone +from apps.accounts.models import Account, AccountGroup +from apps.currencies.models import Currency from apps.import_app.models import ImportRun, ImportProfile -from apps.import_app.schemas import ( - SchemaV1, - ColumnMappingV1, - SettingsV1, - HashTransformationRuleV1, - CompareDeduplicationRuleV1, +from apps.import_app.schemas import version_1 +from apps.transactions.models import ( + Transaction, + TransactionCategory, + TransactionTag, + TransactionEntity, ) -from apps.transactions.models import Transaction + +logger = logging.getLogger(__name__) class ImportService: + TEMP_DIR = "/usr/src/app/temp" + def __init__(self, import_run: ImportRun): self.import_run: ImportRun = import_run self.profile: ImportProfile = import_run.profile - self.config: SchemaV1 = self._load_config() - self.settings: SettingsV1 = self.config.settings - self.deduplication: list[CompareDeduplicationRuleV1] = self.config.deduplication - self.mapping: Dict[str, ColumnMappingV1] = self.config.column_mapping + self.config: version_1.ImportProfileSchema = self._load_config() + self.settings: version_1.CSVImportSettings = self.config.settings + self.deduplication: list[version_1.CompareDeduplicationRule] = ( + self.config.deduplication + ) + self.mapping: Dict[str, version_1.ColumnMapping] = self.config.mapping - def _load_config(self) -> SchemaV1: + # Ensure temp directory exists + os.makedirs(self.TEMP_DIR, exist_ok=True) + + def _load_config(self) -> version_1.ImportProfileSchema: yaml_data = yaml.safe_load(self.profile.yaml_config) - - if self.profile.version == ImportProfile.Versions.VERSION_1: - return SchemaV1(**yaml_data) - - raise ValueError(f"Unsupported version: {self.profile.version}") + try: + config = version_1.ImportProfileSchema(**yaml_data) + except Exception as e: + self._log("error", f"Fatal error processing YAML config: {str(e)}") + self._update_status("FAILED") + raise e + else: + return config def _log(self, level: str, message: str, **kwargs) -> None: """Add a log entry to the import run logs""" @@ -53,6 +67,48 @@ class ImportService: self.import_run.logs += log_line self.import_run.save(update_fields=["logs"]) + def _update_totals( + self, + field: Literal["total", "processed", "successful", "skipped", "failed"], + value: int, + ) -> None: + if field == "total": + self.import_run.total_rows = value + self.import_run.save(update_fields=["total_rows"]) + elif field == "processed": + self.import_run.processed_rows = value + self.import_run.save(update_fields=["processed_rows"]) + elif field == "successful": + self.import_run.successful_rows = value + self.import_run.save(update_fields=["successful_rows"]) + elif field == "skipped": + self.import_run.skipped_rows = value + self.import_run.save(update_fields=["skipped_rows"]) + elif field == "failed": + self.import_run.failed_rows = value + self.import_run.save(update_fields=["failed_rows"]) + + def _increment_totals( + self, + field: Literal["total", "processed", "successful", "skipped", "failed"], + value: int, + ) -> None: + if field == "total": + self.import_run.total_rows = self.import_run.total_rows + value + self.import_run.save(update_fields=["total_rows"]) + elif field == "processed": + self.import_run.processed_rows = self.import_run.processed_rows + value + self.import_run.save(update_fields=["processed_rows"]) + elif field == "successful": + self.import_run.successful_rows = self.import_run.successful_rows + value + self.import_run.save(update_fields=["successful_rows"]) + elif field == "skipped": + self.import_run.skipped_rows = self.import_run.skipped_rows + value + self.import_run.save(update_fields=["skipped_rows"]) + elif field == "failed": + self.import_run.failed_rows = self.import_run.failed_rows + value + self.import_run.save(update_fields=["failed_rows"]) + def _update_status( self, new_status: Literal["PROCESSING", "FAILED", "FINISHED"] ) -> None: @@ -67,15 +123,12 @@ class ImportService: @staticmethod def _transform_value( - value: str, mapping: ColumnMappingV1, row: Dict[str, str] = None + value: str, mapping: version_1.ColumnMapping, row: Dict[str, str] = None ) -> Any: transformed = value for transform in mapping.transformations: if transform.type == "hash": - if not isinstance(transform, HashTransformationRuleV1): - continue - # Collect all values to be hashed values_to_hash = [] for field in transform.fields: @@ -88,47 +141,143 @@ class ImportService: transformed = hashlib.sha256(concatenated.encode()).hexdigest() elif transform.type == "replace": - transformed = transformed.replace( - transform.pattern, transform.replacement - ) + if transform.exclusive: + transformed = value.replace( + transform.pattern, transform.replacement + ) + else: + transformed = transformed.replace( + transform.pattern, transform.replacement + ) elif transform.type == "regex": - transformed = re.sub( - transform.pattern, transform.replacement, transformed - ) + if transform.exclusive: + transformed = re.sub( + transform.pattern, transform.replacement, value + ) + else: + transformed = re.sub( + transform.pattern, transform.replacement, transformed + ) elif transform.type == "date_format": transformed = datetime.strptime( - transformed, transform.pattern - ).strftime(transform.replacement) + transformed, transform.original_format + ).strftime(transform.new_format) + elif transform.type == "merge": + values_to_merge = [] + for field in transform.fields: + if field in row: + values_to_merge.append(str(row[field])) + transformed = transform.separator.join(values_to_merge) + elif transform.type == "split": + parts = transformed.split(transform.separator) + if transform.index is not None: + transformed = parts[transform.index] if parts else "" + else: + transformed = parts return transformed - def _map_row_to_transaction(self, row: Dict[str, str]) -> Dict[str, Any]: - transaction_data = {} + def _create_transaction(self, data: Dict[str, Any]) -> Transaction: + tags = [] + entities = [] + # Handle related objects first + if "category" in data: + category_name = data.pop("category") + category, _ = TransactionCategory.objects.get_or_create(name=category_name) + data["category"] = category + self.import_run.categories.add(category) - for field, mapping in self.mapping.items(): - # If source is None, use None as the initial value - value = row.get(mapping.source) if mapping.source else None + if "account" in data: + account_id = data.pop("account") + account = None + if isinstance(account_id, str): + account = Account.objects.get(name=account_id) + elif isinstance(account_id, int): + account = Account.objects.get(id=account_id) + data["account"] = account + # self.import_run.acc.add(category) - # Use default_value if value is None - if value is None: - value = mapping.default_value + if "tags" in data: + tag_names = data.pop("tags").split(",") + for tag_name in tag_names: + tag, _ = TransactionTag.objects.get_or_create(name=tag_name.strip()) + tags.append(tag) + self.import_run.tags.add(tag) - if mapping.required and value is None and not mapping.transformations: - raise ValueError(f"Required field {field} is missing") + if "entities" in data: + entity_names = data.pop("entities").split(",") + for entity_name in entity_names: + entity, _ = TransactionEntity.objects.get_or_create( + name=entity_name.strip() + ) + entities.append(entity) + self.import_run.entities.add(entity) - # Apply transformations even if initial value is None - if mapping.transformations: - value = self._transform_value(value, mapping, row) + if "amount" in data: + amount = data.pop("amount") + data["amount"] = abs(Decimal(amount)) - if value is not None: - transaction_data[field] = value + # Create the transaction + new_transaction = Transaction.objects.create(**data) + self.import_run.transactions.add(new_transaction) - return transaction_data + # Add many-to-many relationships + if tags: + new_transaction.tags.set(tags) + if entities: + new_transaction.entities.set(entities) + + return new_transaction + + def _create_account(self, data: Dict[str, Any]) -> Account: + if "group" in data: + group_name = data.pop("group") + group, _ = AccountGroup.objects.get_or_create(name=group_name) + data["group"] = group + + # Handle currency references + if "currency" in data: + currency = Currency.objects.get(code=data["currency"]) + data["currency"] = currency + self.import_run.currencies.add(currency) + + if "exchange_currency" in data: + exchange_currency = Currency.objects.get(code=data["exchange_currency"]) + data["exchange_currency"] = exchange_currency + self.import_run.currencies.add(exchange_currency) + + return Account.objects.create(**data) + + def _create_currency(self, data: Dict[str, Any]) -> Currency: + # Handle exchange currency reference + if "exchange_currency" in data: + exchange_currency = Currency.objects.get(code=data["exchange_currency"]) + data["exchange_currency"] = exchange_currency + self.import_run.currencies.add(exchange_currency) + + currency = Currency.objects.create(**data) + self.import_run.currencies.add(currency) + return currency + + def _create_category(self, data: Dict[str, Any]) -> TransactionCategory: + category = TransactionCategory.objects.create(**data) + self.import_run.categories.add(category) + return category + + def _create_tag(self, data: Dict[str, Any]) -> TransactionTag: + tag = TransactionTag.objects.create(**data) + self.import_run.tags.add(tag) + return tag + + def _create_entity(self, data: Dict[str, Any]) -> TransactionEntity: + entity = TransactionEntity.objects.create(**data) + self.import_run.entities.add(entity) + return entity def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool: for rule in self.deduplication: if rule.type == "compare": - query = Transaction.objects.all() + query = Transaction.objects.all().values("id") # Build query conditions for each field in the rule for field, header in rule.fields.items(): @@ -146,65 +295,214 @@ class ImportService: return False - def _process_csv(self, file_path): - with open(file_path, "r", encoding=self.settings.encoding) as csv_file: - reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + def _coerce_type( + self, value: str, mapping: version_1.ColumnMapping + ) -> Union[str, int, bool, Decimal, datetime, list]: + if not value: + return None + + coerce_to = mapping.coerce_to + + if "|" in coerce_to: + types = coerce_to.split("|") + for t in types: + try: + return self._coerce_single_type(value, t, mapping) + except ValueError: + continue + raise ValueError( + f"Could not coerce '{value}' to any of the types: {coerce_to}" + ) + else: + return self._coerce_single_type(value, coerce_to, mapping) + + def _coerce_single_type( + self, value: str, coerce_to: str, mapping: version_1.ColumnMapping + ) -> Union[str, int, bool, Decimal, datetime.date, list]: + if coerce_to == "str": + return str(value) + elif coerce_to == "int": + if hasattr(mapping, "type") and mapping.type == "id": + return int(value) + elif hasattr(mapping, "type") and mapping.type in ["name", "code"]: + return str(value) + else: + return int(value) + elif coerce_to == "bool": + return value.lower() in ["true", "1", "yes", "y", "on"] + elif coerce_to == "positive_decimal": + return abs(Decimal(value)) + elif coerce_to == "date": + if isinstance( + mapping, + ( + version_1.TransactionDateMapping, + version_1.TransactionReferenceDateMapping, + ), + ): + formats = ( + mapping.format + if isinstance(mapping.format, list) + else [mapping.format] + ) + for fmt in formats: + try: + return datetime.strptime(value, fmt).date() + except ValueError: + continue + raise ValueError( + f"Could not parse date '{value}' with any of the provided formats" + ) + else: + raise ValueError( + "Date coercion is only supported for TransactionDateMapping and TransactionReferenceDateMapping" + ) + elif coerce_to == "list": + return ( + value + if isinstance(value, list) + else [item.strip() for item in value.split(",") if item.strip()] + ) + elif coerce_to == "transaction_type": + if isinstance(mapping, version_1.TransactionTypeMapping): + if mapping.detection_method == "sign": + return ( + Transaction.Type.EXPENSE + if value.startswith("-") + else Transaction.Type.INCOME + ) + elif mapping.detection_method == "always_income": + return Transaction.Type.INCOME + elif mapping.detection_method == "always_expense": + return Transaction.Type.EXPENSE + raise ValueError("Invalid transaction type detection method") + elif coerce_to == "is_paid": + if isinstance(mapping, version_1.TransactionIsPaidMapping): + if mapping.detection_method == "sign": + return not value.startswith("-") + elif mapping.detection_method == "boolean": + return value.lower() in ["true", "1", "yes", "y", "on"] + elif mapping.detection_method == "always_paid": + return True + elif mapping.detection_method == "always_unpaid": + return False + raise ValueError("Invalid is_paid detection method") + else: + raise ValueError(f"Unsupported coercion type: {coerce_to}") + + def _map_row(self, row: Dict[str, str]) -> Dict[str, Any]: + mapped_data = {} + + for field, mapping in self.mapping.items(): + # If source is None, use None as the initial value + value = row.get(mapping.source) if mapping.source else None + + # Use default_value if value is None + if value is None: + value = mapping.default + + if mapping.required and value is None and not mapping.transformations: + raise ValueError(f"Required field {field} is missing") + + # Apply transformations + if mapping.transformations: + value = self._transform_value(value, mapping, row) + + value = self._coerce_type(value, mapping) + + if value is not None: + # Remove the prefix from the target field + target = mapping.target + if self.settings.importing == "transactions": + mapped_data[target] = value + else: + # Remove the model prefix (e.g., "account_" from "account_name") + field_name = target.split("_", 1)[1] + mapped_data[field_name] = value + + return mapped_data + + def _process_row(self, row: Dict[str, str], row_number: int) -> None: + try: + mapped_data = self._map_row(row) + + if mapped_data: + # Handle different import types + if self.settings.importing == "transactions": + if self.deduplication and self._check_duplicate_transaction( + mapped_data + ): + self._increment_totals("skipped", 1) + self._log("info", f"Skipped duplicate row {row_number}") + return + self._create_transaction(mapped_data) + elif self.settings.importing == "accounts": + self._create_account(mapped_data) + elif self.settings.importing == "currencies": + self._create_currency(mapped_data) + elif self.settings.importing == "categories": + self._create_category(mapped_data) + elif self.settings.importing == "tags": + self._create_tag(mapped_data) + elif self.settings.importing == "entities": + self._create_entity(mapped_data) + + self._increment_totals("successful", value=1) + self._log("info", f"Successfully processed row {row_number}") + + self._increment_totals("processed", value=1) + + except Exception as e: + if not self.settings.skip_errors: + self._log("error", f"Fatal error processing row {row_number}: {str(e)}") + self._update_status("FAILED") + raise + else: + self._log("warning", f"Error processing row {row_number}: {str(e)}") + self._increment_totals("failed", value=1) + + logger.error(f"Fatal error processing row {row_number}", exc_info=e) + + def _process_csv(self, file_path): + # First pass: count rows + with open(file_path, "r", encoding=self.settings.encoding) as csv_file: + # Skip specified number of rows + for _ in range(self.settings.skip_lines): + next(csv_file) + + reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + self._update_totals("total", value=sum(1 for _ in reader)) + + with open(file_path, "r", encoding=self.settings.encoding) as csv_file: + # Skip specified number of rows + for _ in range(self.settings.skip_lines): + next(csv_file) + if self.settings.skip_lines: + self._log("info", f"Skipped {self.settings.skip_lines} initial lines") - # Count total rows - self.import_run.total_rows = sum(1 for _ in reader) - csv_file.seek(0) reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) self._log("info", f"Starting import with {self.import_run.total_rows} rows") - # Skip specified number of rows - for _ in range(self.settings.skip_rows): - next(reader) + with transaction.atomic(): + for row_number, row in enumerate(reader, start=1): + self._process_row(row, row_number) + self._increment_totals("processed", value=1) - if self.settings.skip_rows: - self._log("info", f"Skipped {self.settings.skip_rows} initial rows") - - for row_number, row in enumerate(reader, start=1): - try: - transaction_data = self._map_row_to_transaction(row) - - if transaction_data: - if self.deduplication and self._check_duplicate_transaction( - transaction_data - ): - self.import_run.skipped_rows += 1 - self._log("info", f"Skipped duplicate row {row_number}") - continue - - self.import_run.transactions.add(transaction_data) - self.import_run.successful_rows += 1 - self._log("debug", f"Successfully processed row {row_number}") - - self.import_run.processed_rows += 1 - self.import_run.save( - update_fields=[ - "processed_rows", - "successful_rows", - "skipped_rows", - ] - ) - - except Exception as e: - if not self.settings.skip_errors: - self._log( - "error", - f"Fatal error processing row {row_number}: {str(e)}", - ) - self._update_status("FAILED") - raise - else: - self._log( - "warning", f"Error processing row {row_number}: {str(e)}" - ) - self.import_run.failed_rows += 1 - self.import_run.save(update_fields=["failed_rows"]) + def _validate_file_path(self, file_path: str) -> str: + """ + Validates that the file path is within the allowed temporary directory. + Returns the absolute path. + """ + abs_path = os.path.abspath(file_path) + if not abs_path.startswith(self.TEMP_DIR): + raise ValueError(f"Invalid file path. File must be in {self.TEMP_DIR}") + return abs_path def process_file(self, file_path: str): + # Validate and get absolute path + file_path = self._validate_file_path(file_path) + self._update_status("PROCESSING") self.import_run.started_at = timezone.now() self.import_run.save(update_fields=["started_at"]) @@ -232,6 +530,12 @@ class ImportService: finally: self._log("info", "Cleaning up temporary files") - default_storage.delete(file_path) + try: + if os.path.exists(file_path): + os.remove(file_path) + self._log("info", f"Deleted temporary file: {file_path}") + except OSError as e: + self._log("warning", f"Failed to delete temporary file: {str(e)}") + self.import_run.finished_at = timezone.now() self.import_run.save(update_fields=["finished_at"]) From a94e0b4904fbd2d6f8cbc54a6939a816469783e5 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 11:45:06 -0300 Subject: [PATCH 04/60] docs(requirements): add django_ace --- app/WYGIWYH/settings.py | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 8243c91..e219d6f 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -70,6 +70,7 @@ INSTALLED_APPS = [ "rest_framework", "drf_spectacular", "django_cotton", + "django_ace", "apps.rules.apps.RulesConfig", "apps.calendar_view.apps.CalendarViewConfig", "apps.dca.apps.DcaConfig", diff --git a/requirements.txt b/requirements.txt index af9d39b..8c24038 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ django-filter==24.3 django-debug-toolbar==4.3.0 django-cachalot~=2.6.3 django-cotton~=1.2.1 +django_ace~=1.36.2 djangorestframework~=3.15.2 drf-spectacular~=0.27.2 From 238f205513344f50f5d7568a810597a91d4f6922 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 11:47:33 -0300 Subject: [PATCH 05/60] docker: add temp volume --- docker-compose.dev.yml | 3 ++- docker-compose.prod.yml | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index c06c0fd..133d522 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,6 +1,6 @@ volumes: wygiwyh_dev_postgres_data: {} - temp: + wygiwyh_temp: services: web: &django @@ -13,6 +13,7 @@ services: volumes: - ./app/:/usr/src/app/:z - ./frontend/:/usr/src/frontend:z + - wygiwyh_temp:/usr/src/app/temp/ ports: - "${OUTBOUND_PORT}:8000" env_file: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a12b4ed..b840e46 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -9,6 +9,8 @@ services: - .env depends_on: - db + volumes: + - wygiwyh_temp:/usr/src/app/temp/ restart: unless-stopped db: @@ -29,5 +31,10 @@ services: - db env_file: - .env + volumes: + - wygiwyh_temp:/usr/src/app/temp/ command: /start-procrastinate restart: unless-stopped + +volumes: + wygiwyh_temp: From 3ccb0e19eb3070a47b760e2a297a666d9558178f Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:55:17 -0300 Subject: [PATCH 06/60] feat(transactions): soft delete --- app/apps/transactions/admin.py | 18 +++++ .../0028_transaction_internal_note.py | 18 +++++ .../0029_alter_transaction_options.py | 17 ++++ ...nsaction_deleted_transaction_deleted_at.py | 23 ++++++ .../0031_alter_transaction_deleted.py | 18 +++++ ...ction_created_at_transaction_updated_at.py | 25 ++++++ app/apps/transactions/models.py | 77 +++++++++++++++++++ 7 files changed, 196 insertions(+) create mode 100644 app/apps/transactions/migrations/0028_transaction_internal_note.py create mode 100644 app/apps/transactions/migrations/0029_alter_transaction_options.py create mode 100644 app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py create mode 100644 app/apps/transactions/migrations/0031_alter_transaction_deleted.py create mode 100644 app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py diff --git a/app/apps/transactions/admin.py b/app/apps/transactions/admin.py index 5a4ef15..df4d1c8 100644 --- a/app/apps/transactions/admin.py +++ b/app/apps/transactions/admin.py @@ -12,7 +12,14 @@ from apps.transactions.models import ( @admin.register(Transaction) class TransactionModelAdmin(admin.ModelAdmin): + def get_queryset(self, request): + # Use the all_objects manager to show all transactions, including deleted ones + return self.model.all_objects.all() + + list_filter = ["deleted", "type", "is_paid", "date", "account"] + list_display = [ + "deleted", "description", "type", "account__name", @@ -22,6 +29,17 @@ class TransactionModelAdmin(admin.ModelAdmin): "reference_date", ] + actions = ["hard_delete_selected"] + + def hard_delete_selected(self, request, queryset): + for obj in queryset: + obj.hard_delete() + self.message_user( + request, f"Successfully hard deleted {queryset.count()} transactions." + ) + + hard_delete_selected.short_description = "Hard delete selected transactions" + class TransactionInline(admin.TabularInline): model = Transaction diff --git a/app/apps/transactions/migrations/0028_transaction_internal_note.py b/app/apps/transactions/migrations/0028_transaction_internal_note.py new file mode 100644 index 0000000..c88c11d --- /dev/null +++ b/app/apps/transactions/migrations/0028_transaction_internal_note.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.5 on 2025-01-19 00:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0027_alter_transaction_description'), + ] + + operations = [ + migrations.AddField( + model_name='transaction', + name='internal_note', + field=models.TextField(blank=True, verbose_name='Internal Note'), + ), + ] diff --git a/app/apps/transactions/migrations/0029_alter_transaction_options.py b/app/apps/transactions/migrations/0029_alter_transaction_options.py new file mode 100644 index 0000000..c06b7cd --- /dev/null +++ b/app/apps/transactions/migrations/0029_alter_transaction_options.py @@ -0,0 +1,17 @@ +# Generated by Django 5.1.5 on 2025-01-19 14:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0028_transaction_internal_note'), + ] + + operations = [ + migrations.AlterModelOptions( + name='transaction', + options={'default_manager_name': 'objects', 'verbose_name': 'Transaction', 'verbose_name_plural': 'Transactions'}, + ), + ] diff --git a/app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py b/app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py new file mode 100644 index 0000000..35f4c91 --- /dev/null +++ b/app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.5 on 2025-01-19 14:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0029_alter_transaction_options'), + ] + + operations = [ + migrations.AddField( + model_name='transaction', + name='deleted', + field=models.BooleanField(default=False, verbose_name='Deleted'), + ), + migrations.AddField( + model_name='transaction', + name='deleted_at', + field=models.DateTimeField(blank=True, null=True, verbose_name='Deleted At'), + ), + ] diff --git a/app/apps/transactions/migrations/0031_alter_transaction_deleted.py b/app/apps/transactions/migrations/0031_alter_transaction_deleted.py new file mode 100644 index 0000000..b5d2dc4 --- /dev/null +++ b/app/apps/transactions/migrations/0031_alter_transaction_deleted.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.5 on 2025-01-19 15:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0030_transaction_deleted_transaction_deleted_at'), + ] + + operations = [ + migrations.AlterField( + model_name='transaction', + name='deleted', + field=models.BooleanField(db_index=True, default=False, verbose_name='Deleted'), + ), + ] diff --git a/app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py b/app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py new file mode 100644 index 0000000..46e76ae --- /dev/null +++ b/app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.5 on 2025-01-19 16:48 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0031_alter_transaction_deleted'), + ] + + operations = [ + migrations.AddField( + model_name='transaction', + name='created_at', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='transaction', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index f131518..2bd2a68 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -6,6 +6,7 @@ from django.db import models, transaction from django.db.models import Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from django.conf import settings from apps.common.fields.month_year import MonthYearModelField from apps.common.functions.decimals import truncate_decimal @@ -15,6 +16,53 @@ from apps.transactions.validators import validate_decimal_places, validate_non_n logger = logging.getLogger() +class SoftDeleteQuerySet(models.QuerySet): + def delete(self): + if not settings.ENABLE_SOFT_DELETION: + # If soft deletion is disabled, perform a normal delete + return super().delete() + + # Separate the queryset into already deleted and not deleted objects + already_deleted = self.filter(deleted=True) + not_deleted = self.filter(deleted=False) + + # Use a transaction to ensure atomicity + with transaction.atomic(): + # Perform hard delete on already deleted objects + hard_deleted_count = already_deleted._raw_delete(already_deleted.db) + + # Perform soft delete on not deleted objects + soft_deleted_count = not_deleted.update( + deleted=True, deleted_at=timezone.now() + ) + + # Return a tuple of counts as expected by Django's delete method + return ( + hard_deleted_count + soft_deleted_count, + {"Transaction": hard_deleted_count + soft_deleted_count}, + ) + + def hard_delete(self): + return super().delete() + + +class SoftDeleteManager(models.Manager): + def get_queryset(self): + qs = SoftDeleteQuerySet(self.model, using=self._db) + return qs if not settings.ENABLE_SOFT_DELETION else qs.filter(deleted=False) + + +class AllObjectsManager(models.Manager): + def get_queryset(self): + return SoftDeleteQuerySet(self.model, using=self._db) + + +class DeletedObjectsManager(models.Manager): + def get_queryset(self): + qs = SoftDeleteQuerySet(self.model, using=self._db) + return qs if not settings.ENABLE_SOFT_DELETION else qs.filter(deleted=True) + + class TransactionCategory(models.Model): name = models.CharField(max_length=255, verbose_name=_("Name"), unique=True) mute = models.BooleanField(default=False, verbose_name=_("Mute")) @@ -143,10 +191,24 @@ class Transaction(models.Model): ) internal_note = models.TextField(blank=True, verbose_name=_("Internal Note")) + deleted = models.BooleanField( + default=False, verbose_name=_("Deleted"), db_index=True + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + deleted_at = models.DateTimeField( + null=True, blank=True, verbose_name=_("Deleted At") + ) + + objects = SoftDeleteManager.from_queryset(SoftDeleteQuerySet)() + all_objects = AllObjectsManager.from_queryset(SoftDeleteQuerySet)() + deleted_objects = DeletedObjectsManager.from_queryset(SoftDeleteQuerySet)() + class Meta: verbose_name = _("Transaction") verbose_name_plural = _("Transactions") db_table = "transactions" + default_manager_name = "objects" def save(self, *args, **kwargs): self.amount = truncate_decimal( @@ -161,6 +223,17 @@ class Transaction(models.Model): self.full_clean() super().save(*args, **kwargs) + def delete(self, *args, **kwargs): + if settings.ENABLE_SOFT_DELETION: + self.deleted = True + self.deleted_at = timezone.now() + self.save() + else: + super().delete(*args, **kwargs) + + def hard_delete(self, *args, **kwargs): + super().delete(*args, **kwargs) + def exchanged_amount(self): if self.account.exchange_currency: converted_amount, prefix, suffix, decimal_places = convert( @@ -179,6 +252,10 @@ class Transaction(models.Model): return None + def __str__(self): + type_display = self.get_type_display() + return f"{self.description} - {type_display} - {self.account} - {self.date}" + class InstallmentPlan(models.Model): class Recurrence(models.TextChoices): From f96d8d286298902791263d15973b5699c261596d Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:55:25 -0300 Subject: [PATCH 07/60] feat(transactions): soft delete --- app/WYGIWYH/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index e219d6f..155408d 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -336,3 +336,5 @@ else: } CACHALOT_UNCACHABLE_TABLES = ("django_migrations", "procrastinate_jobs") + +ENABLE_SOFT_DELETION = os.environ.get("ENABLE_SOFT_DELETION", "False").lower() == "true" From 2d8864773ce13d6cc18abd3a73e89a88c9fe2d03 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:56:13 -0300 Subject: [PATCH 08/60] feat(import): disable cache when running --- app/apps/import_app/services.py | 0 app/apps/import_app/services/v1.py | 84 ++++++++++++++++-------------- 2 files changed, 44 insertions(+), 40 deletions(-) delete mode 100644 app/apps/import_app/services.py diff --git a/app/apps/import_app/services.py b/app/apps/import_app/services.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 069115b..7735342 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -7,8 +7,9 @@ from datetime import datetime from decimal import Decimal from typing import Dict, Any, Literal, Union +import cachalot.api import yaml -from django.db import transaction +from cachalot.api import cachalot_disabled from django.utils import timezone from apps.accounts.models import Account, AccountGroup @@ -277,7 +278,7 @@ class ImportService: def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool: for rule in self.deduplication: if rule.type == "compare": - query = Transaction.objects.all().values("id") + query = Transaction.all_objects.all().values("id") # Build query conditions for each field in the rule for field, header in rule.fields.items(): @@ -484,10 +485,9 @@ class ImportService: self._log("info", f"Starting import with {self.import_run.total_rows} rows") - with transaction.atomic(): - for row_number, row in enumerate(reader, start=1): - self._process_row(row, row_number) - self._increment_totals("processed", value=1) + for row_number, row in enumerate(reader, start=1): + self._process_row(row, row_number) + self._increment_totals("processed", value=1) def _validate_file_path(self, file_path: str) -> str: """ @@ -500,42 +500,46 @@ class ImportService: return abs_path def process_file(self, file_path: str): - # Validate and get absolute path - file_path = self._validate_file_path(file_path) + with cachalot_disabled(): + # Validate and get absolute path + file_path = self._validate_file_path(file_path) - self._update_status("PROCESSING") - self.import_run.started_at = timezone.now() - self.import_run.save(update_fields=["started_at"]) + self._update_status("PROCESSING") + self.import_run.started_at = timezone.now() + self.import_run.save(update_fields=["started_at"]) - self._log("info", "Starting import process") + self._log("info", "Starting import process") - try: - if self.settings.file_type == "csv": - self._process_csv(file_path) - - if self.import_run.processed_rows == self.import_run.total_rows: - self._update_status("FINISHED") - self._log( - "info", - f"Import completed successfully. " - f"Successful: {self.import_run.successful_rows}, " - f"Failed: {self.import_run.failed_rows}, " - f"Skipped: {self.import_run.skipped_rows}", - ) - - except Exception as e: - self._update_status("FAILED") - self._log("error", f"Import failed: {str(e)}") - raise Exception("Import failed") - - finally: - self._log("info", "Cleaning up temporary files") try: - if os.path.exists(file_path): - os.remove(file_path) - self._log("info", f"Deleted temporary file: {file_path}") - except OSError as e: - self._log("warning", f"Failed to delete temporary file: {str(e)}") + if self.settings.file_type == "csv": + self._process_csv(file_path) - self.import_run.finished_at = timezone.now() - self.import_run.save(update_fields=["finished_at"]) + if self.import_run.processed_rows == self.import_run.total_rows: + self._update_status("FINISHED") + self._log( + "info", + f"Import completed successfully. " + f"Successful: {self.import_run.successful_rows}, " + f"Failed: {self.import_run.failed_rows}, " + f"Skipped: {self.import_run.skipped_rows}", + ) + + except Exception as e: + self._update_status("FAILED") + self._log("error", f"Import failed: {str(e)}") + raise Exception("Import failed") + + finally: + self._log("info", "Cleaning up temporary files") + try: + if os.path.exists(file_path): + os.remove(file_path) + self._log("info", f"Deleted temporary file: {file_path}") + except OSError as e: + self._log("warning", f"Failed to delete temporary file: {str(e)}") + + self.import_run.finished_at = timezone.now() + self.import_run.save(update_fields=["finished_at"]) + + if self.import_run.successful_rows >= 1: + cachalot.api.invalidate() From ba0c54767c405806f93f1a0eb7526948529435cd Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:56:29 -0300 Subject: [PATCH 09/60] feat(import): add migrations --- .../import_app/migrations/0001_initial.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 app/apps/import_app/migrations/0001_initial.py diff --git a/app/apps/import_app/migrations/0001_initial.py b/app/apps/import_app/migrations/0001_initial.py new file mode 100644 index 0000000..bcce0fe --- /dev/null +++ b/app/apps/import_app/migrations/0001_initial.py @@ -0,0 +1,51 @@ +# Generated by Django 5.1.5 on 2025-01-19 00:44 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('currencies', '0006_currency_exchange_currency'), + ('transactions', '0028_transaction_internal_note'), + ] + + operations = [ + migrations.CreateModel( + name='ImportProfile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('yaml_config', models.TextField(help_text='YAML configuration')), + ('version', models.IntegerField(choices=[(1, 'Version 1')], default=1, verbose_name='Version')), + ], + options={ + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='ImportRun', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('QUEUED', 'Queued'), ('PROCESSING', 'Processing'), ('FAILED', 'Failed'), ('FINISHED', 'Finished')], default='QUEUED', max_length=10, verbose_name='Status')), + ('file_name', models.CharField(help_text='File name', max_length=10000)), + ('logs', models.TextField(blank=True)), + ('processed_rows', models.IntegerField(default=0)), + ('total_rows', models.IntegerField(default=0)), + ('successful_rows', models.IntegerField(default=0)), + ('skipped_rows', models.IntegerField(default=0)), + ('failed_rows', models.IntegerField(default=0)), + ('started_at', models.DateTimeField(null=True)), + ('finished_at', models.DateTimeField(null=True)), + ('categories', models.ManyToManyField(related_name='import_runs', to='transactions.transactioncategory')), + ('currencies', models.ManyToManyField(related_name='import_runs', to='currencies.currency')), + ('entities', models.ManyToManyField(related_name='import_runs', to='transactions.transactionentity')), + ('profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='import_app.importprofile')), + ('tags', models.ManyToManyField(related_name='import_runs', to='transactions.transactiontag')), + ('transactions', models.ManyToManyField(related_name='import_runs', to='transactions.transaction')), + ], + ), + ] From 3ef6b0ac5ce1702394ad23eddde17cf63e0d7129 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 15:16:47 -0300 Subject: [PATCH 10/60] feat(settings): add KEEP_DELETED_TRANSACTIONS_FOR variable --- app/WYGIWYH/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 155408d..83950f2 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -337,4 +337,5 @@ else: CACHALOT_UNCACHABLE_TABLES = ("django_migrations", "procrastinate_jobs") -ENABLE_SOFT_DELETION = os.environ.get("ENABLE_SOFT_DELETION", "False").lower() == "true" +ENABLE_SOFT_DELETION = os.getenv("ENABLE_SOFT_DELETION", "True").lower() == "true" +KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365")) From ae91c5196795f9956685bf61819022146f71a2ca Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 15:17:18 -0300 Subject: [PATCH 11/60] feat(transactions:tasks): add old deleted transactions cleanup task --- app/apps/transactions/tasks.py | 39 ++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/app/apps/transactions/tasks.py b/app/apps/transactions/tasks.py index e0bfafc..5f1c42f 100644 --- a/app/apps/transactions/tasks.py +++ b/app/apps/transactions/tasks.py @@ -1,9 +1,13 @@ import logging +from datetime import timedelta + +from cachalot.api import cachalot_disabled, invalidate +from django.utils import timezone +from django.conf import settings from procrastinate.contrib.django import app -from apps.transactions.models import RecurringTransaction - +from apps.transactions.models import RecurringTransaction, Transaction logger = logging.getLogger(__name__) @@ -19,3 +23,34 @@ def generate_recurring_transactions(timestamp=None): exc_info=True, ) raise e + + +@app.periodic(cron="10 1 * * *") +@app.task +def cleanup_deleted_transactions(): + with cachalot_disabled(): + if ( + settings.ENABLE_SOFT_DELETION + and settings.KEEP_DELETED_TRANSACTIONS_FOR == 0 + ): + return "KEEP_DELETED_TRANSACTIONS_FOR is 0, no cleanup performed." + + if not settings.ENABLE_SOFT_DELETION: + # Hard delete all soft-deleted transactions + deleted_count, _ = Transaction.deleted_objects.all().hard_delete() + return ( + f"Hard deleted {deleted_count} transactions (soft deletion disabled)." + ) + + # Calculate the cutoff date + cutoff_date = timezone.now() - timedelta( + days=settings.KEEP_DELETED_TRANSACTIONS_FOR + ) + + invalidate("transactions.Transaction") + + # Hard delete soft-deleted transactions older than the cutoff date + old_transactions = Transaction.deleted_objects.filter(deleted_at__lt=cutoff_date) + deleted_count, _ = old_transactions.hard_delete() + + return f"Hard deleted {deleted_count} objects older than {settings.KEEP_DELETED_TRANSACTIONS_FOR} days." From e73e1dfc2592d09724061b2363e39576bd5335c3 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 15:20:25 -0300 Subject: [PATCH 12/60] feat(import:v1:schema): add option for triggering rules --- app/apps/import_app/schemas/v1.py | 1 + app/apps/import_app/services/v1.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 043f2a9..74e37a1 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -69,6 +69,7 @@ class CSVImportSettings(BaseModel): skip_lines: int = Field( default=0, description="Number of rows to skip at the beginning of the file" ) + trigger_transaction_rules: bool = True importing: Literal[ "transactions", "accounts", "currencies", "categories", "tags", "entities" ] diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 7735342..0416caf 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -22,6 +22,7 @@ from apps.transactions.models import ( TransactionTag, TransactionEntity, ) +from apps.rules.signals import transaction_created logger = logging.getLogger(__name__) @@ -228,6 +229,9 @@ class ImportService: if entities: new_transaction.entities.set(entities) + if self.settings.trigger_transaction_rules: + transaction_created.send(sender=new_transaction) + return new_transaction def _create_account(self, data: Dict[str, Any]) -> Account: From 8db13b082b17518c926f55efa2b69af36177055d Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:30:17 -0300 Subject: [PATCH 13/60] feat(import): some layouts --- app/apps/import_app/forms.py | 58 +++++++++++++++++ .../import_app/fragments/profiles/add.html | 11 ++++ .../import_app/fragments/profiles/edit.html | 11 ++++ .../import_app/fragments/profiles/list.html | 65 +++++++++++++++++++ .../import_app/fragments/runs/add.html | 11 ++++ .../import_app/fragments/runs/list.html | 9 +++ 6 files changed, 165 insertions(+) create mode 100644 app/apps/import_app/forms.py create mode 100644 app/templates/import_app/fragments/profiles/add.html create mode 100644 app/templates/import_app/fragments/profiles/edit.html create mode 100644 app/templates/import_app/fragments/profiles/list.html create mode 100644 app/templates/import_app/fragments/runs/add.html create mode 100644 app/templates/import_app/fragments/runs/list.html diff --git a/app/apps/import_app/forms.py b/app/apps/import_app/forms.py new file mode 100644 index 0000000..78ee3d7 --- /dev/null +++ b/app/apps/import_app/forms.py @@ -0,0 +1,58 @@ +from crispy_forms.bootstrap import FormActions +from crispy_forms.helper import FormHelper +from crispy_forms.layout import ( + Layout, +) +from django import forms +from django.utils.translation import gettext_lazy as _ +from django_ace import AceWidget + +from apps.import_app.models import ImportProfile +from apps.common.widgets.crispy.submit import NoClassSubmit + + +class ImportProfileForm(forms.ModelForm): + class Meta: + model = ImportProfile + fields = [ + "name", + "version", + "yaml_config", + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.helper = FormHelper() + self.helper.form_tag = False + self.helper.form_method = "post" + self.helper.layout = Layout("name", "version", "yaml_config") + + if self.instance and self.instance.pk: + self.helper.layout.append( + FormActions( + NoClassSubmit( + "submit", _("Update"), css_class="btn btn-outline-primary w-100" + ), + ), + ) + else: + self.helper.layout.append( + FormActions( + NoClassSubmit( + "submit", _("Add"), css_class="btn btn-outline-primary w-100" + ), + ), + ) + + +class ImportRunFileUploadForm(forms.Form): + file = forms.FileField(label=_("Select a file")) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.helper = FormHelper() + self.helper.form_tag = False + self.helper.form_method = "post" + self.helper.layout = Layout("file") diff --git a/app/templates/import_app/fragments/profiles/add.html b/app/templates/import_app/fragments/profiles/add.html new file mode 100644 index 0000000..beda873 --- /dev/null +++ b/app/templates/import_app/fragments/profiles/add.html @@ -0,0 +1,11 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Add new import profile' %}{% endblock %} + +{% block body %} +
+ {% crispy form %} +
+{% endblock %} diff --git a/app/templates/import_app/fragments/profiles/edit.html b/app/templates/import_app/fragments/profiles/edit.html new file mode 100644 index 0000000..fa94bef --- /dev/null +++ b/app/templates/import_app/fragments/profiles/edit.html @@ -0,0 +1,11 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Edit import profile' %}{% endblock %} + +{% block body %} +
+ {% crispy form %} +
+{% endblock %} diff --git a/app/templates/import_app/fragments/profiles/list.html b/app/templates/import_app/fragments/profiles/list.html new file mode 100644 index 0000000..f1f34d2 --- /dev/null +++ b/app/templates/import_app/fragments/profiles/list.html @@ -0,0 +1,65 @@ +{% load i18n %} +
+
+ {% spaceless %} +
{% translate 'Import Profiles' %} + + +
+ {% endspaceless %} +
+ +
+
+ {% if profiles %} + + + + + + + + + + + {% for profile in profiles %} + + + + + + {% endfor %} + +
{% translate 'Name' %}{% translate 'Version' %}
+
+ + +{# #} +{#
#} + 		{{ profile.name }}{{ profile.get_version_display }}
+ {% else %} + + {% endif %} +
+
+
diff --git a/app/templates/import_app/fragments/runs/add.html b/app/templates/import_app/fragments/runs/add.html new file mode 100644 index 0000000..d5a5b89 --- /dev/null +++ b/app/templates/import_app/fragments/runs/add.html @@ -0,0 +1,11 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Import file' %}{% endblock %} + +{% block body %} +
+ {% crispy form %} +
+{% endblock %} diff --git a/app/templates/import_app/fragments/runs/list.html b/app/templates/import_app/fragments/runs/list.html new file mode 100644 index 0000000..0697d26 --- /dev/null +++ b/app/templates/import_app/fragments/runs/list.html @@ -0,0 +1,9 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Runs for ' %}{{ profile.name }}{% endblock %} + +{% block body %} + +{% endblock %} From 4cc32e3f579a82ae1f11bbe4255b9a8289968729 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:30:40 -0300 Subject: [PATCH 14/60] feat(import): test yaml_config before saving --- app/apps/import_app/models.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/apps/import_app/models.py b/app/apps/import_app/models.py index aca04e3..b489c43 100644 --- a/app/apps/import_app/models.py +++ b/app/apps/import_app/models.py @@ -1,13 +1,18 @@ +import yaml + +from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ +from apps.import_app.schemas import version_1 + class ImportProfile(models.Model): class Versions(models.IntegerChoices): VERSION_1 = 1, _("Version 1") - name = models.CharField(max_length=100) - yaml_config = models.TextField(help_text=_("YAML configuration")) + name = models.CharField(max_length=100, verbose_name=_("Name")) + yaml_config = models.TextField(verbose_name=_("YAML Configuration")) version = models.IntegerField( choices=Versions, default=Versions.VERSION_1, @@ -20,6 +25,14 @@ class ImportProfile(models.Model): class Meta: ordering = ["name"] + def clean(self): + if self.version and self.version == self.Versions.VERSION_1: + try: + yaml_data = yaml.safe_load(self.yaml_config) + version_1.ImportProfileSchema(**yaml_data) + except Exception as e: + raise ValidationError({"yaml_config": _("Invalid YAML Configuration")}) + class ImportRun(models.Model): class Status(models.TextChoices): From b9810ce06296035e58eaeaee3cb51286e6c0addd Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:30:59 -0300 Subject: [PATCH 15/60] feat(import): some layouts --- app/templates/import_app/pages/profiles_index.html | 8 ++++++++ app/templates/import_app/pages/runs_index.html | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 app/templates/import_app/pages/profiles_index.html create mode 100644 app/templates/import_app/pages/runs_index.html diff --git a/app/templates/import_app/pages/profiles_index.html b/app/templates/import_app/pages/profiles_index.html new file mode 100644 index 0000000..a5c59ee --- /dev/null +++ b/app/templates/import_app/pages/profiles_index.html @@ -0,0 +1,8 @@ +{% extends "layouts/base.html" %} +{% load i18n %} + +{% block title %}{% translate 'Import Profiles' %}{% endblock %} + +{% block content %} +
+{% endblock %} diff --git a/app/templates/import_app/pages/runs_index.html b/app/templates/import_app/pages/runs_index.html new file mode 100644 index 0000000..38a48a6 --- /dev/null +++ b/app/templates/import_app/pages/runs_index.html @@ -0,0 +1,8 @@ +{% extends "layouts/base.html" %} +{% load i18n %} + +{% block title %}{% translate 'Import Runs' %}{% endblock %} + +{% block content %} +
+{% endblock %} From 0fccdbe573c057a917688457c3cc25a056a8e59f Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:31:12 -0300 Subject: [PATCH 16/60] feat(import): some views and urls --- app/apps/import_app/tasks.py | 2 +- app/apps/import_app/urls.py | 37 +++++++- app/apps/import_app/views.py | 168 +++++++++++++++++++++++++++++++---- 3 files changed, 186 insertions(+), 21 deletions(-) diff --git a/app/apps/import_app/tasks.py b/app/apps/import_app/tasks.py index 25efcbc..cf6f3a7 100644 --- a/app/apps/import_app/tasks.py +++ b/app/apps/import_app/tasks.py @@ -8,7 +8,7 @@ from apps.import_app.services import ImportServiceV1 logger = logging.getLogger(__name__) -@app.task(queue="imports") +@app.task def process_import(import_run_id: int, file_path: str): try: import_run = ImportRun.objects.get(id=import_run_id) diff --git a/app/apps/import_app/urls.py b/app/apps/import_app/urls.py index aea8670..c2608a3 100644 --- a/app/apps/import_app/urls.py +++ b/app/apps/import_app/urls.py @@ -2,5 +2,40 @@ from django.urls import path import apps.import_app.views as views urlpatterns = [ - path("import/", views.ImportRunCreateView.as_view(), name="import"), + path("import/", views.import_view, name="import"), + path( + "import/profiles/", + views.import_profile_index, + name="import_profiles_index", + ), + path( + "import/profiles/list/", + views.import_profile_list, + name="import_profiles_list", + ), + path( + "import/profiles/add/", + views.import_profile_add, + name="import_profiles_add", + ), + path( + "import/profiles//edit/", + views.import_profile_edit, + name="import_profile_edit", + ), + path( + "import/profiles//runs/", + views.import_run_add, + name="import_profile_runs_index", + ), + path( + "import/profiles//runs/list/", + views.import_run_add, + name="import_profile_runs_list", + ), + path( + "import/profiles//runs/add/", + views.import_run_add, + name="import_run_add", + ), ] diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index d5b1d94..ce65a2b 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -1,26 +1,156 @@ -from django.views.generic import CreateView -from apps.import_app.models import ImportRun -from apps.import_app.services import ImportServiceV1 +import shutil + +from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.files.storage import FileSystemStorage +from django.http import HttpResponse +from django.shortcuts import render, get_object_or_404 +from django.views.decorators.http import require_http_methods +from django.utils.translation import gettext_lazy as _ + +from apps.common.decorators.htmx import only_htmx +from apps.import_app.forms import ImportRunFileUploadForm, ImportProfileForm +from apps.import_app.models import ImportRun, ImportProfile +from apps.import_app.tasks import process_import -class ImportRunCreateView(CreateView): - model = ImportRun - fields = ["profile"] +def import_view(request): + import_profile = ImportProfile.objects.get(id=2) + shutil.copyfile( + "/usr/src/app/apps/import_app/teste2.csv", "/usr/src/app/temp/teste2.csv" + ) + ir = ImportRun.objects.create(profile=import_profile, file_name="teste.csv") + process_import.defer( + import_run_id=ir.id, + file_path="/usr/src/app/temp/teste2.csv", + ) + return HttpResponse("Hello, world. You're at the polls page.") - def form_valid(self, form): - response = super().form_valid(form) - import_run = form.instance - file = self.request.FILES["file"] +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_index(request): + return render( + request, + "import_app/pages/profiles_index.html", + ) - # Save uploaded file temporarily - temp_file_path = f"/tmp/import_{import_run.id}.csv" - with open(temp_file_path, "wb+") as destination: - for chunk in file.chunks(): - destination.write(chunk) - # Process the import - import_service = ImportServiceV1(import_run) - import_service.process_file(temp_file_path) +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_list(request): + profiles = ImportProfile.objects.all() - return response + return render( + request, + "import_app/fragments/profiles/list.html", + {"profiles": profiles}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_add(request): + if request.method == "POST": + form = ImportProfileForm(request.POST) + + if form.is_valid(): + form.save() + messages.success(request, _("Import Profile added successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated, hide_offcanvas", + }, + ) + else: + form = ImportProfileForm() + + return render( + request, + "import_app/fragments/profiles/add.html", + {"form": form}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_edit(request, profile_id): + profile = get_object_or_404(ImportProfile, id=profile_id) + + if request.method == "POST": + form = ImportProfileForm(request.POST, instance=profile) + + if form.is_valid(): + form.save() + messages.success(request, _("Import Profile update successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated, hide_offcanvas", + }, + ) + else: + form = ImportProfileForm(instance=profile) + + return render( + request, + "import_app/fragments/profiles/edit.html", + {"form": form, "profile": profile}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_run_list(request, profile_id): + profile = ImportProfile.objects.get(id=profile_id) + + runs = ImportRun.objects.filter(profile=profile).order_by("id") + + return render( + request, + "import_app/fragments/runs/list.html", + {"profile": profile, "runs": runs}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_run_add(request, profile_id): + profile = ImportProfile.objects.get(id=profile_id) + + if request.method == "POST": + form = ImportRunFileUploadForm(request.POST, request.FILES) + + if form.is_valid(): + uploaded_file = request.FILES["file"] + fs = FileSystemStorage(location="/usr/src/app/temp") + filename = fs.save(uploaded_file.name, uploaded_file) + file_path = fs.path(filename) + + import_run = ImportRun.objects.create(profile=profile, file_name=filename) + + # Defer the procrastinate task + process_import.defer(import_run_id=import_run.id, file_path=file_path) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated, hide_offcanvas", + }, + ) + else: + form = ImportRunFileUploadForm() + + return render( + request, + "import_app/fragments/runs/add.html", + {"form": form, "profile": profile}, + ) From 02adfd828a8f2f15f69b2f04651b1a1ddf8905a7 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 23:09:49 -0300 Subject: [PATCH 17/60] feat(transactions): add internal_id field to transactions --- app/apps/import_app/schemas/v1.py | 15 +++++++++++--- .../0033_transaction_internal_id.py | 20 +++++++++++++++++++ app/apps/transactions/models.py | 3 +++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 app/apps/transactions/migrations/0033_transaction_internal_id.py diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 74e37a1..22df7c2 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -167,13 +167,20 @@ class TransactionCategoryMapping(ColumnMapping): coerce_to: Literal["str|int"] = Field("str|int", frozen=True) -class TransactionInternalMapping(ColumnMapping): +class TransactionInternalNoteMapping(ColumnMapping): target: Literal["internal_note"] = Field( ..., description="Transaction field to map to" ) coerce_to: Literal["str"] = Field("str", frozen=True) +class TransactionInternalIDMapping(ColumnMapping): + target: Literal["internal_id"] = Field( + ..., description="Transaction field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + class CategoryNameMapping(ColumnMapping): target: Literal["category_name"] = Field( ..., description="Category field to map to" @@ -314,7 +321,8 @@ class ImportProfileSchema(BaseModel): | TransactionTagsMapping | TransactionEntitiesMapping | TransactionCategoryMapping - | TransactionInternalMapping + | TransactionInternalNoteMapping + | TransactionInternalIDMapping | CategoryNameMapping | CategoryMuteMapping | CategoryActiveMapping @@ -358,7 +366,8 @@ class ImportProfileSchema(BaseModel): TransactionTagsMapping, TransactionEntitiesMapping, TransactionCategoryMapping, - TransactionInternalMapping, + TransactionInternalNoteMapping, + TransactionInternalIDMapping, ), "accounts": ( AccountNameMapping, diff --git a/app/apps/transactions/migrations/0033_transaction_internal_id.py b/app/apps/transactions/migrations/0033_transaction_internal_id.py new file mode 100644 index 0000000..b7d578c --- /dev/null +++ b/app/apps/transactions/migrations/0033_transaction_internal_id.py @@ -0,0 +1,20 @@ +# Generated by Django 5.1.5 on 2025-01-21 01:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("transactions", "0032_transaction_created_at_transaction_updated_at"), + ] + + operations = [ + migrations.AddField( + model_name="transaction", + name="internal_id", + field=models.TextField( + blank=True, null=True, unique=True, verbose_name="Internal ID" + ), + ), + ] diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index 2bd2a68..85ff53a 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -190,6 +190,9 @@ class Transaction(models.Model): verbose_name=_("Recurring Transaction"), ) internal_note = models.TextField(blank=True, verbose_name=_("Internal Note")) + internal_id = models.TextField( + blank=True, null=True, unique=True, verbose_name=_("Internal ID") + ) deleted = models.BooleanField( default=False, verbose_name=_("Deleted"), db_index=True From 32b5864736a7a02b4eeacbc2c078cdac1138696f Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 23:10:11 -0300 Subject: [PATCH 18/60] feat(transactions): make deleted_at readonly on admin --- app/apps/transactions/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/apps/transactions/admin.py b/app/apps/transactions/admin.py index df4d1c8..8f37317 100644 --- a/app/apps/transactions/admin.py +++ b/app/apps/transactions/admin.py @@ -19,15 +19,16 @@ class TransactionModelAdmin(admin.ModelAdmin): list_filter = ["deleted", "type", "is_paid", "date", "account"] list_display = [ - "deleted", + "date", "description", "type", "account__name", "amount", "account__currency__code", - "date", "reference_date", + "deleted", ] + readonly_fields = ["deleted_at"] actions = ["hard_delete_selected"] From d96787cfebe4c8e50762612a9c441575c3331f15 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Wed, 22 Jan 2025 01:41:17 -0300 Subject: [PATCH 19/60] feat(import): more UI and endpoints --- app/apps/import_app/forms.py | 9 +- app/apps/import_app/services/v1.py | 18 ++- app/apps/import_app/urls.py | 22 +++- app/apps/import_app/views.py | 60 +++++++++- .../import_app/fragments/profiles/list.html | 38 ++++-- .../import_app/fragments/runs/add.html | 4 +- .../import_app/fragments/runs/list.html | 111 ++++++++++++++++++ 7 files changed, 228 insertions(+), 34 deletions(-) diff --git a/app/apps/import_app/forms.py b/app/apps/import_app/forms.py index 78ee3d7..f300721 100644 --- a/app/apps/import_app/forms.py +++ b/app/apps/import_app/forms.py @@ -55,4 +55,11 @@ class ImportRunFileUploadForm(forms.Form): self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" - self.helper.layout = Layout("file") + self.helper.layout = Layout( + "file", + FormActions( + NoClassSubmit( + "submit", _("Import"), css_class="btn btn-outline-primary w-100" + ), + ), + ) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 0416caf..abda751 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -491,7 +491,6 @@ class ImportService: for row_number, row in enumerate(reader, start=1): self._process_row(row, row_number) - self._increment_totals("processed", value=1) def _validate_file_path(self, file_path: str) -> str: """ @@ -518,15 +517,14 @@ class ImportService: if self.settings.file_type == "csv": self._process_csv(file_path) - if self.import_run.processed_rows == self.import_run.total_rows: - self._update_status("FINISHED") - self._log( - "info", - f"Import completed successfully. " - f"Successful: {self.import_run.successful_rows}, " - f"Failed: {self.import_run.failed_rows}, " - f"Skipped: {self.import_run.skipped_rows}", - ) + self._update_status("FINISHED") + self._log( + "info", + f"Import completed successfully. " + f"Successful: {self.import_run.successful_rows}, " + f"Failed: {self.import_run.failed_rows}, " + f"Skipped: {self.import_run.skipped_rows}", + ) except Exception as e: self._update_status("FAILED") diff --git a/app/apps/import_app/urls.py b/app/apps/import_app/urls.py index c2608a3..beb65ba 100644 --- a/app/apps/import_app/urls.py +++ b/app/apps/import_app/urls.py @@ -13,6 +13,11 @@ urlpatterns = [ views.import_profile_list, name="import_profiles_list", ), + path( + "import/profiles//delete/", + views.import_profile_delete, + name="import_profile_delete", + ), path( "import/profiles/add/", views.import_profile_add, @@ -24,14 +29,19 @@ urlpatterns = [ name="import_profile_edit", ), path( - "import/profiles//runs/", - views.import_run_add, - name="import_profile_runs_index", + "import/profiles//runs/list/", + views.import_runs_list, + name="import_profile_runs_list", ), path( - "import/profiles//runs/list/", - views.import_run_add, - name="import_profile_runs_list", + "import/profiles//runs//log/", + views.import_run_log, + name="import_run_log", + ), + path( + "import/profiles//runs//delete/", + views.import_run_delete, + name="import_run_delete", ), path( "import/profiles//runs/add/", diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index ce65a2b..6b869fd 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -5,6 +5,7 @@ from django.contrib.auth.decorators import login_required from django.core.files.storage import FileSystemStorage from django.http import HttpResponse from django.shortcuts import render, get_object_or_404 +from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods from django.utils.translation import gettext_lazy as _ @@ -107,11 +108,30 @@ def import_profile_edit(request, profile_id): @only_htmx @login_required -@require_http_methods(["GET", "POST"]) -def import_run_list(request, profile_id): +@csrf_exempt +@require_http_methods(["DELETE"]) +def import_profile_delete(request, profile_id): profile = ImportProfile.objects.get(id=profile_id) - runs = ImportRun.objects.filter(profile=profile).order_by("id") + profile.delete() + + messages.success(request, _("Import Profile deleted successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated", + }, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_runs_list(request, profile_id): + profile = ImportProfile.objects.get(id=profile_id) + + runs = ImportRun.objects.filter(profile=profile).order_by("-id") return render( request, @@ -120,6 +140,19 @@ def import_run_list(request, profile_id): ) +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_run_log(request, profile_id, run_id): + run = ImportRun.objects.get(profile__id=profile_id, id=run_id) + + return render( + request, + "import_app/fragments/runs/log.html", + {"run": run}, + ) + + @only_htmx @login_required @require_http_methods(["GET", "POST"]) @@ -140,6 +173,8 @@ def import_run_add(request, profile_id): # Defer the procrastinate task process_import.defer(import_run_id=import_run.id, file_path=file_path) + messages.success(request, _("Import Run queued successfully")) + return HttpResponse( status=204, headers={ @@ -154,3 +189,22 @@ def import_run_add(request, profile_id): "import_app/fragments/runs/add.html", {"form": form, "profile": profile}, ) + + +@only_htmx +@login_required +@csrf_exempt +@require_http_methods(["DELETE"]) +def import_run_delete(request, profile_id, run_id): + run = ImportRun.objects.get(profile__id=profile_id, id=run_id) + + run.delete() + + messages.success(request, _("Run deleted successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated", + }, + ) diff --git a/app/templates/import_app/fragments/profiles/list.html b/app/templates/import_app/fragments/profiles/list.html index f1f34d2..2872897 100644 --- a/app/templates/import_app/fragments/profiles/list.html +++ b/app/templates/import_app/fragments/profiles/list.html @@ -38,18 +38,32 @@ hx-get="{% url 'import_profile_edit' profile_id=profile.id %}" hx-target="#generic-offcanvas"> -{# #} -{# #} + + + + + + {{ profile.name }} {{ profile.get_version_display }} diff --git a/app/templates/import_app/fragments/runs/add.html b/app/templates/import_app/fragments/runs/add.html index d5a5b89..9997044 100644 --- a/app/templates/import_app/fragments/runs/add.html +++ b/app/templates/import_app/fragments/runs/add.html @@ -2,10 +2,10 @@ {% load i18n %} {% load crispy_forms_tags %} -{% block title %}{% translate 'Import file' %}{% endblock %} +{% block title %}{% translate 'Import file with profile' %} {{ profile.name }}{% endblock %} {% block body %} -
+ {% crispy form %}
{% endblock %} diff --git a/app/templates/import_app/fragments/runs/list.html b/app/templates/import_app/fragments/runs/list.html index 0697d26..f67054c 100644 --- a/app/templates/import_app/fragments/runs/list.html +++ b/app/templates/import_app/fragments/runs/list.html @@ -5,5 +5,116 @@ {% block title %}{% translate 'Runs for ' %}{{ profile.name }}{% endblock %} {% block body %} +
+ {% if runs %} +
+ {% for run in runs %} +
+
+
+ {{ run.get_status_display }} +
+
+
{{ run.id }}({{ run.file_name }})
+
+
+
+
+
+
+ {% trans 'Total Items' %} +
+
+ {{ run.total_rows }} +
+
+
+
+
+
+
+
+ {% trans 'Processed Items' %} +
+
+ {{ run.processed_rows }} +
+
+
+
+ +
+
+
+
+ {% trans 'Skipped Items' %} +
+
+ {{ run.skipped_rows }} +
+
+
+
+ +
+
+
+
+ {% trans 'Failed Items' %} +
+
+ {{ run.failed_rows }} +
+
+
+
+ +
+
+
+
+ {% trans 'Successful Items' %} +
+
+ {{ run.successful_rows }} +
+
+
+
+ +
+
+
+ + + +
+
+
+ {% endfor %} + {% else %} + + {% endif %} +
+
{% endblock %} From 8992cd98b55ebf65a54965ba078e67b5830711ca Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 16 Jan 2025 14:09:33 -0300 Subject: [PATCH 20/60] feat: add import app boilerplate --- app/WYGIWYH/settings.py | 1 + app/apps/import/__init__.py | 0 app/apps/import/admin.py | 3 +++ app/apps/import/apps.py | 6 ++++++ app/apps/import/migrations/__init__.py | 0 app/apps/import/models.py | 3 +++ app/apps/import/tests.py | 3 +++ app/apps/import/views.py | 3 +++ 8 files changed, 19 insertions(+) create mode 100644 app/apps/import/__init__.py create mode 100644 app/apps/import/admin.py create mode 100644 app/apps/import/apps.py create mode 100644 app/apps/import/migrations/__init__.py create mode 100644 app/apps/import/models.py create mode 100644 app/apps/import/tests.py create mode 100644 app/apps/import/views.py diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index b2eeba1..a2336cb 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -64,6 +64,7 @@ INSTALLED_APPS = [ "apps.accounts.apps.AccountsConfig", "apps.common.apps.CommonConfig", "apps.net_worth.apps.NetWorthConfig", + "apps.import.apps.ImportConfig", "apps.api.apps.ApiConfig", "cachalot", "rest_framework", diff --git a/app/apps/import/__init__.py b/app/apps/import/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import/admin.py b/app/apps/import/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/app/apps/import/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/app/apps/import/apps.py b/app/apps/import/apps.py new file mode 100644 index 0000000..fdfa08d --- /dev/null +++ b/app/apps/import/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ImportConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "apps.import" diff --git a/app/apps/import/migrations/__init__.py b/app/apps/import/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import/models.py b/app/apps/import/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/app/apps/import/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/app/apps/import/tests.py b/app/apps/import/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/app/apps/import/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/app/apps/import/views.py b/app/apps/import/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/app/apps/import/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 493bf268bb3607c875699211c709878e173806fc Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Fri, 17 Jan 2025 17:40:51 -0300 Subject: [PATCH 21/60] feat: rename app, some work on schema --- app/WYGIWYH/settings.py | 2 +- app/WYGIWYH/urls.py | 1 + app/apps/import/admin.py | 3 - app/apps/import/models.py | 3 - app/apps/import/views.py | 3 - app/apps/{import => import_app}/__init__.py | 0 app/apps/import_app/admin.py | 6 + app/apps/{import => import_app}/apps.py | 2 +- .../migrations/__init__.py | 0 app/apps/import_app/models.py | 74 ++++++ app/apps/import_app/schemas.py | 0 app/apps/import_app/schemas/__init__.py | 8 + app/apps/import_app/schemas/v1.py | 104 ++++++++ app/apps/import_app/services.py | 0 app/apps/import_app/services/__init__.py | 1 + app/apps/import_app/services/v1.py | 237 ++++++++++++++++++ app/apps/import_app/tasks.py | 18 ++ app/apps/{import => import_app}/tests.py | 0 app/apps/import_app/urls.py | 6 + app/apps/import_app/views.py | 26 ++ app/apps/transactions/models.py | 1 + requirements.txt | 2 + 22 files changed, 486 insertions(+), 11 deletions(-) delete mode 100644 app/apps/import/admin.py delete mode 100644 app/apps/import/models.py delete mode 100644 app/apps/import/views.py rename app/apps/{import => import_app}/__init__.py (100%) create mode 100644 app/apps/import_app/admin.py rename app/apps/{import => import_app}/apps.py (81%) rename app/apps/{import => import_app}/migrations/__init__.py (100%) create mode 100644 app/apps/import_app/models.py create mode 100644 app/apps/import_app/schemas.py create mode 100644 app/apps/import_app/schemas/__init__.py create mode 100644 app/apps/import_app/schemas/v1.py create mode 100644 app/apps/import_app/services.py create mode 100644 app/apps/import_app/services/__init__.py create mode 100644 app/apps/import_app/services/v1.py create mode 100644 app/apps/import_app/tasks.py rename app/apps/{import => import_app}/tests.py (100%) create mode 100644 app/apps/import_app/urls.py create mode 100644 app/apps/import_app/views.py diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index a2336cb..d597d5d 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -64,7 +64,7 @@ INSTALLED_APPS = [ "apps.accounts.apps.AccountsConfig", "apps.common.apps.CommonConfig", "apps.net_worth.apps.NetWorthConfig", - "apps.import.apps.ImportConfig", + "apps.import_app.apps.ImportConfig", "apps.api.apps.ApiConfig", "cachalot", "rest_framework", diff --git a/app/WYGIWYH/urls.py b/app/WYGIWYH/urls.py index 5a465a5..eb4357d 100644 --- a/app/WYGIWYH/urls.py +++ b/app/WYGIWYH/urls.py @@ -47,4 +47,5 @@ urlpatterns = [ path("", include("apps.calendar_view.urls")), path("", include("apps.dca.urls")), path("", include("apps.mini_tools.urls")), + path("", include("apps.import_app.urls")), ] diff --git a/app/apps/import/admin.py b/app/apps/import/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/app/apps/import/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/app/apps/import/models.py b/app/apps/import/models.py deleted file mode 100644 index 71a8362..0000000 --- a/app/apps/import/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/app/apps/import/views.py b/app/apps/import/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/app/apps/import/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/app/apps/import/__init__.py b/app/apps/import_app/__init__.py similarity index 100% rename from app/apps/import/__init__.py rename to app/apps/import_app/__init__.py diff --git a/app/apps/import_app/admin.py b/app/apps/import_app/admin.py new file mode 100644 index 0000000..cbccf2b --- /dev/null +++ b/app/apps/import_app/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from apps.import_app import models + +# Register your models here. +admin.site.register(models.ImportRun) +admin.site.register(models.ImportProfile) diff --git a/app/apps/import/apps.py b/app/apps/import_app/apps.py similarity index 81% rename from app/apps/import/apps.py rename to app/apps/import_app/apps.py index fdfa08d..4dbe90c 100644 --- a/app/apps/import/apps.py +++ b/app/apps/import_app/apps.py @@ -3,4 +3,4 @@ from django.apps import AppConfig class ImportConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "apps.import" + name = "apps.import_app" diff --git a/app/apps/import/migrations/__init__.py b/app/apps/import_app/migrations/__init__.py similarity index 100% rename from app/apps/import/migrations/__init__.py rename to app/apps/import_app/migrations/__init__.py diff --git a/app/apps/import_app/models.py b/app/apps/import_app/models.py new file mode 100644 index 0000000..aca04e3 --- /dev/null +++ b/app/apps/import_app/models.py @@ -0,0 +1,74 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class ImportProfile(models.Model): + class Versions(models.IntegerChoices): + VERSION_1 = 1, _("Version 1") + + name = models.CharField(max_length=100) + yaml_config = models.TextField(help_text=_("YAML configuration")) + version = models.IntegerField( + choices=Versions, + default=Versions.VERSION_1, + verbose_name=_("Version"), + ) + + def __str__(self): + return self.name + + class Meta: + ordering = ["name"] + + +class ImportRun(models.Model): + class Status(models.TextChoices): + QUEUED = "QUEUED", _("Queued") + PROCESSING = "PROCESSING", _("Processing") + FAILED = "FAILED", _("Failed") + FINISHED = "FINISHED", _("Finished") + + status = models.CharField( + max_length=10, + choices=Status, + default=Status.QUEUED, + verbose_name=_("Status"), + ) + profile = models.ForeignKey( + ImportProfile, + on_delete=models.CASCADE, + ) + file_name = models.CharField( + max_length=10000, + help_text=_("File name"), + ) + transactions = models.ManyToManyField( + "transactions.Transaction", related_name="import_runs" + ) + tags = models.ManyToManyField( + "transactions.TransactionTag", related_name="import_runs" + ) + categories = models.ManyToManyField( + "transactions.TransactionCategory", related_name="import_runs" + ) + entities = models.ManyToManyField( + "transactions.TransactionEntity", related_name="import_runs" + ) + currencies = models.ManyToManyField( + "currencies.Currency", related_name="import_runs" + ) + + logs = models.TextField(blank=True) + processed_rows = models.IntegerField(default=0) + total_rows = models.IntegerField(default=0) + successful_rows = models.IntegerField(default=0) + skipped_rows = models.IntegerField(default=0) + failed_rows = models.IntegerField(default=0) + started_at = models.DateTimeField(null=True) + finished_at = models.DateTimeField(null=True) + + @property + def progress(self): + if self.total_rows == 0: + return 0 + return (self.processed_rows / self.total_rows) * 100 diff --git a/app/apps/import_app/schemas.py b/app/apps/import_app/schemas.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import_app/schemas/__init__.py b/app/apps/import_app/schemas/__init__.py new file mode 100644 index 0000000..f68ce79 --- /dev/null +++ b/app/apps/import_app/schemas/__init__.py @@ -0,0 +1,8 @@ +from apps.import_app.schemas.v1 import ( + ImportProfileSchema as SchemaV1, + ColumnMapping as ColumnMappingV1, + # TransformationRule as TransformationRuleV1, + ImportSettings as SettingsV1, + HashTransformationRule as HashTransformationRuleV1, + CompareDeduplicationRule as CompareDeduplicationRuleV1, +) diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py new file mode 100644 index 0000000..1cc7dc5 --- /dev/null +++ b/app/apps/import_app/schemas/v1.py @@ -0,0 +1,104 @@ +from typing import Dict, List, Optional, Literal +from pydantic import BaseModel, Field + + +class CompareDeduplicationRule(BaseModel): + type: Literal["compare"] + fields: Dict = Field( + ..., description="Match header and fields to compare for deduplication" + ) + match_type: Literal["lax", "strict"] + + +class ReplaceTransformationRule(BaseModel): + field: str + type: Literal["replace", "regex"] = Field( + ..., description="Type of transformation: replace or regex" + ) + pattern: str = Field(..., description="Pattern to match") + replacement: str = Field(..., description="Value to replace with") + + +class DateFormatTransformationRule(BaseModel): + field: str + type: Literal["date_format"] = Field( + ..., description="Type of transformation: replace or regex" + ) + original_format: str = Field(..., description="Original date format") + new_format: str = Field(..., description="New date format to use") + + +class HashTransformationRule(BaseModel): + fields: List[str] + type: Literal["hash"] + + +class MergeTransformationRule(BaseModel): + fields: List[str] + type: Literal["merge"] + separator: str = Field(default=" ", description="Separator to use when merging") + + +class SplitTransformationRule(BaseModel): + fields: List[str] + type: Literal["split"] + separator: str = Field(default=",", description="Separator to use when splitting") + index: int | None = Field( + default=0, description="Index to return as value. Empty to return all." + ) + + +class ImportSettings(BaseModel): + skip_errors: bool = Field( + default=False, + description="If True, errors during import will be logged and skipped", + ) + file_type: Literal["csv"] = "csv" + delimiter: str = Field(default=",", description="CSV delimiter character") + encoding: str = Field(default="utf-8", description="File encoding") + skip_rows: int = Field( + default=0, description="Number of rows to skip at the beginning of the file" + ) + importing: Literal[ + "transactions", "accounts", "currencies", "categories", "tags", "entities" + ] + + +class ColumnMapping(BaseModel): + source: Optional[str] = Field( + default=None, + description="CSV column header. If None, the field will be generated from transformations", + ) + target: Literal[ + "account", + "type", + "is_paid", + "date", + "reference_date", + "amount", + "notes", + "category", + "tags", + "entities", + "internal_note", + ] = Field(..., description="Transaction field to map to") + default_value: Optional[str] = None + required: bool = False + transformations: Optional[ + List[ + ReplaceTransformationRule + | DateFormatTransformationRule + | HashTransformationRule + | MergeTransformationRule + | SplitTransformationRule + ] + ] = Field(default_factory=list) + + +class ImportProfileSchema(BaseModel): + settings: ImportSettings + column_mapping: Dict[str, ColumnMapping] + deduplication: List[CompareDeduplicationRule] = Field( + default_factory=list, + description="Rules for deduplicating records during import", + ) diff --git a/app/apps/import_app/services.py b/app/apps/import_app/services.py new file mode 100644 index 0000000..e69de29 diff --git a/app/apps/import_app/services/__init__.py b/app/apps/import_app/services/__init__.py new file mode 100644 index 0000000..6001902 --- /dev/null +++ b/app/apps/import_app/services/__init__.py @@ -0,0 +1 @@ +from apps.import_app.services.v1 import ImportService as ImportServiceV1 diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py new file mode 100644 index 0000000..333eb6e --- /dev/null +++ b/app/apps/import_app/services/v1.py @@ -0,0 +1,237 @@ +import csv +import hashlib +import re +from datetime import datetime +from typing import Dict, Any, Literal + +import yaml + +from django.db import transaction +from django.core.files.storage import default_storage +from django.utils import timezone + +from apps.import_app.models import ImportRun, ImportProfile +from apps.import_app.schemas import ( + SchemaV1, + ColumnMappingV1, + SettingsV1, + HashTransformationRuleV1, + CompareDeduplicationRuleV1, +) +from apps.transactions.models import Transaction + + +class ImportService: + def __init__(self, import_run: ImportRun): + self.import_run: ImportRun = import_run + self.profile: ImportProfile = import_run.profile + self.config: SchemaV1 = self._load_config() + self.settings: SettingsV1 = self.config.settings + self.deduplication: list[CompareDeduplicationRuleV1] = self.config.deduplication + self.mapping: Dict[str, ColumnMappingV1] = self.config.column_mapping + + def _load_config(self) -> SchemaV1: + yaml_data = yaml.safe_load(self.profile.yaml_config) + + if self.profile.version == ImportProfile.Versions.VERSION_1: + return SchemaV1(**yaml_data) + + raise ValueError(f"Unsupported version: {self.profile.version}") + + def _log(self, level: str, message: str, **kwargs) -> None: + """Add a log entry to the import run logs""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # Format additional context if present + context = "" + if kwargs: + context = " - " + ", ".join(f"{k}={v}" for k, v in kwargs.items()) + + log_line = f"[{timestamp}] {level.upper()}: {message}{context}\n" + + # Append to existing logs + self.import_run.logs += log_line + self.import_run.save(update_fields=["logs"]) + + def _update_status( + self, new_status: Literal["PROCESSING", "FAILED", "FINISHED"] + ) -> None: + if new_status == "PROCESSING": + self.import_run.status = ImportRun.Status.PROCESSING + elif new_status == "FAILED": + self.import_run.status = ImportRun.Status.FAILED + elif new_status == "FINISHED": + self.import_run.status = ImportRun.Status.FINISHED + + self.import_run.save(update_fields=["status"]) + + @staticmethod + def _transform_value( + value: str, mapping: ColumnMappingV1, row: Dict[str, str] = None + ) -> Any: + transformed = value + + for transform in mapping.transformations: + if transform.type == "hash": + if not isinstance(transform, HashTransformationRuleV1): + continue + + # Collect all values to be hashed + values_to_hash = [] + for field in transform.fields: + if field in row: + values_to_hash.append(str(row[field])) + + # Create hash from concatenated values + if values_to_hash: + concatenated = "|".join(values_to_hash) + transformed = hashlib.sha256(concatenated.encode()).hexdigest() + + elif transform.type == "replace": + transformed = transformed.replace( + transform.pattern, transform.replacement + ) + elif transform.type == "regex": + transformed = re.sub( + transform.pattern, transform.replacement, transformed + ) + elif transform.type == "date_format": + transformed = datetime.strptime( + transformed, transform.pattern + ).strftime(transform.replacement) + + return transformed + + def _map_row_to_transaction(self, row: Dict[str, str]) -> Dict[str, Any]: + transaction_data = {} + + for field, mapping in self.mapping.items(): + # If source is None, use None as the initial value + value = row.get(mapping.source) if mapping.source else None + + # Use default_value if value is None + if value is None: + value = mapping.default_value + + if mapping.required and value is None and not mapping.transformations: + raise ValueError(f"Required field {field} is missing") + + # Apply transformations even if initial value is None + if mapping.transformations: + value = self._transform_value(value, mapping, row) + + if value is not None: + transaction_data[field] = value + + return transaction_data + + def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool: + for rule in self.deduplication: + if rule.type == "compare": + query = Transaction.objects.all() + + # Build query conditions for each field in the rule + for field, header in rule.fields.items(): + if field in transaction_data: + if rule.match_type == "strict": + query = query.filter(**{field: transaction_data[field]}) + else: # lax matching + query = query.filter( + **{f"{field}__iexact": transaction_data[field]} + ) + + # If we found any matching transaction, it's a duplicate + if query.exists(): + return True + + return False + + def _process_csv(self, file_path): + with open(file_path, "r", encoding=self.settings.encoding) as csv_file: + reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + + # Count total rows + self.import_run.total_rows = sum(1 for _ in reader) + csv_file.seek(0) + reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + + self._log("info", f"Starting import with {self.import_run.total_rows} rows") + + # Skip specified number of rows + for _ in range(self.settings.skip_rows): + next(reader) + + if self.settings.skip_rows: + self._log("info", f"Skipped {self.settings.skip_rows} initial rows") + + for row_number, row in enumerate(reader, start=1): + try: + transaction_data = self._map_row_to_transaction(row) + + if transaction_data: + if self.deduplication and self._check_duplicate_transaction( + transaction_data + ): + self.import_run.skipped_rows += 1 + self._log("info", f"Skipped duplicate row {row_number}") + continue + + self.import_run.transactions.add(transaction_data) + self.import_run.successful_rows += 1 + self._log("debug", f"Successfully processed row {row_number}") + + self.import_run.processed_rows += 1 + self.import_run.save( + update_fields=[ + "processed_rows", + "successful_rows", + "skipped_rows", + ] + ) + + except Exception as e: + if not self.settings.skip_errors: + self._log( + "error", + f"Fatal error processing row {row_number}: {str(e)}", + ) + self._update_status("FAILED") + raise + else: + self._log( + "warning", f"Error processing row {row_number}: {str(e)}" + ) + self.import_run.failed_rows += 1 + self.import_run.save(update_fields=["failed_rows"]) + + def process_file(self, file_path: str): + self._update_status("PROCESSING") + self.import_run.started_at = timezone.now() + self.import_run.save(update_fields=["started_at"]) + + self._log("info", "Starting import process") + + try: + if self.settings.file_type == "csv": + self._process_csv(file_path) + + if self.import_run.processed_rows == self.import_run.total_rows: + self._update_status("FINISHED") + self._log( + "info", + f"Import completed successfully. " + f"Successful: {self.import_run.successful_rows}, " + f"Failed: {self.import_run.failed_rows}, " + f"Skipped: {self.import_run.skipped_rows}", + ) + + except Exception as e: + self._update_status("FAILED") + self._log("error", f"Import failed: {str(e)}") + raise Exception("Import failed") + + finally: + self._log("info", "Cleaning up temporary files") + default_storage.delete(file_path) + self.import_run.finished_at = timezone.now() + self.import_run.save(update_fields=["finished_at"]) diff --git a/app/apps/import_app/tasks.py b/app/apps/import_app/tasks.py new file mode 100644 index 0000000..25efcbc --- /dev/null +++ b/app/apps/import_app/tasks.py @@ -0,0 +1,18 @@ +import logging + +from procrastinate.contrib.django import app + +from apps.import_app.models import ImportRun +from apps.import_app.services import ImportServiceV1 + +logger = logging.getLogger(__name__) + + +@app.task(queue="imports") +def process_import(import_run_id: int, file_path: str): + try: + import_run = ImportRun.objects.get(id=import_run_id) + import_service = ImportServiceV1(import_run) + import_service.process_file(file_path) + except ImportRun.DoesNotExist: + raise ValueError(f"ImportRun with id {import_run_id} not found") diff --git a/app/apps/import/tests.py b/app/apps/import_app/tests.py similarity index 100% rename from app/apps/import/tests.py rename to app/apps/import_app/tests.py diff --git a/app/apps/import_app/urls.py b/app/apps/import_app/urls.py new file mode 100644 index 0000000..aea8670 --- /dev/null +++ b/app/apps/import_app/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +import apps.import_app.views as views + +urlpatterns = [ + path("import/", views.ImportRunCreateView.as_view(), name="import"), +] diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py new file mode 100644 index 0000000..d5b1d94 --- /dev/null +++ b/app/apps/import_app/views.py @@ -0,0 +1,26 @@ +from django.views.generic import CreateView +from apps.import_app.models import ImportRun +from apps.import_app.services import ImportServiceV1 + + +class ImportRunCreateView(CreateView): + model = ImportRun + fields = ["profile"] + + def form_valid(self, form): + response = super().form_valid(form) + + import_run = form.instance + file = self.request.FILES["file"] + + # Save uploaded file temporarily + temp_file_path = f"/tmp/import_{import_run.id}.csv" + with open(temp_file_path, "wb+") as destination: + for chunk in file.chunks(): + destination.write(chunk) + + # Process the import + import_service = ImportServiceV1(import_run) + import_service.process_file(temp_file_path) + + return response diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index 70bbc94..f131518 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -141,6 +141,7 @@ class Transaction(models.Model): related_name="transactions", verbose_name=_("Recurring Transaction"), ) + internal_note = models.TextField(blank=True, verbose_name=_("Internal Note")) class Meta: verbose_name = _("Transaction") diff --git a/requirements.txt b/requirements.txt index b4e4f02..af9d39b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,5 @@ requests~=2.32.3 pytz~=2024.2 python-dateutil~=2.9.0.post0 simpleeval~=1.0.0 +pydantic~=2.10.5 +PyYAML~=6.0.2 From 50efc51f878971010654b46a768163b918c2e881 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 11:27:14 -0300 Subject: [PATCH 22/60] feat(import): improve schema definition --- app/apps/import_app/schemas/__init__.py | 9 +- app/apps/import_app/schemas/v1.py | 330 +++++++++++++++- app/apps/import_app/services/v1.py | 506 +++++++++++++++++++----- 3 files changed, 717 insertions(+), 128 deletions(-) diff --git a/app/apps/import_app/schemas/__init__.py b/app/apps/import_app/schemas/__init__.py index f68ce79..530268d 100644 --- a/app/apps/import_app/schemas/__init__.py +++ b/app/apps/import_app/schemas/__init__.py @@ -1,8 +1 @@ -from apps.import_app.schemas.v1 import ( - ImportProfileSchema as SchemaV1, - ColumnMapping as ColumnMappingV1, - # TransformationRule as TransformationRuleV1, - ImportSettings as SettingsV1, - HashTransformationRule as HashTransformationRuleV1, - CompareDeduplicationRule as CompareDeduplicationRuleV1, -) +import apps.import_app.schemas.v1 as version_1 diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 1cc7dc5..043f2a9 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -1,5 +1,5 @@ from typing import Dict, List, Optional, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator, field_validator class CompareDeduplicationRule(BaseModel): @@ -9,6 +9,12 @@ class CompareDeduplicationRule(BaseModel): ) match_type: Literal["lax", "strict"] + @field_validator("fields", mode="before") + def coerce_fields_to_dict(cls, v): + if isinstance(v, list): + return {k: v for d in v for k, v in d.items()} + return v + class ReplaceTransformationRule(BaseModel): field: str @@ -17,6 +23,10 @@ class ReplaceTransformationRule(BaseModel): ) pattern: str = Field(..., description="Pattern to match") replacement: str = Field(..., description="Value to replace with") + exclusive: bool = Field( + default=False, + description="If it should match against the last transformation or the original value", + ) class DateFormatTransformationRule(BaseModel): @@ -48,7 +58,7 @@ class SplitTransformationRule(BaseModel): ) -class ImportSettings(BaseModel): +class CSVImportSettings(BaseModel): skip_errors: bool = Field( default=False, description="If True, errors during import will be logged and skipped", @@ -56,7 +66,7 @@ class ImportSettings(BaseModel): file_type: Literal["csv"] = "csv" delimiter: str = Field(default=",", description="CSV delimiter character") encoding: str = Field(default="utf-8", description="File encoding") - skip_rows: int = Field( + skip_lines: int = Field( default=0, description="Number of rows to skip at the beginning of the file" ) importing: Literal[ @@ -69,20 +79,7 @@ class ColumnMapping(BaseModel): default=None, description="CSV column header. If None, the field will be generated from transformations", ) - target: Literal[ - "account", - "type", - "is_paid", - "date", - "reference_date", - "amount", - "notes", - "category", - "tags", - "entities", - "internal_note", - ] = Field(..., description="Transaction field to map to") - default_value: Optional[str] = None + default: Optional[str] = None required: bool = False transformations: Optional[ List[ @@ -95,10 +92,305 @@ class ColumnMapping(BaseModel): ] = Field(default_factory=list) +class TransactionAccountMapping(ColumnMapping): + target: Literal["account"] = Field(..., description="Transaction field to map to") + type: Literal["id", "name"] = "name" + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class TransactionTypeMapping(ColumnMapping): + target: Literal["type"] = Field(..., description="Transaction field to map to") + detection_method: Literal["sign", "always_income", "always_expense"] = "sign" + coerce_to: Literal["transaction_type"] = Field("transaction_type", frozen=True) + + +class TransactionIsPaidMapping(ColumnMapping): + target: Literal["is_paid"] = Field(..., description="Transaction field to map to") + detection_method: Literal["sign", "boolean", "always_paid", "always_unpaid"] + coerce_to: Literal["is_paid"] = Field("is_paid", frozen=True) + + +class TransactionDateMapping(ColumnMapping): + target: Literal["date"] = Field(..., description="Transaction field to map to") + format: List[str] | str + coerce_to: Literal["date"] = Field("date", frozen=True) + + +class TransactionReferenceDateMapping(ColumnMapping): + target: Literal["reference_date"] = Field( + ..., description="Transaction field to map to" + ) + format: List[str] | str + coerce_to: Literal["date"] = Field("date", frozen=True) + + +class TransactionAmountMapping(ColumnMapping): + target: Literal["amount"] = Field(..., description="Transaction field to map to") + coerce_to: Literal["positive_decimal"] = Field("positive_decimal", frozen=True) + + +class TransactionDescriptionMapping(ColumnMapping): + target: Literal["description"] = Field( + ..., description="Transaction field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class TransactionNotesMapping(ColumnMapping): + target: Literal["notes"] = Field(..., description="Transaction field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class TransactionTagsMapping(ColumnMapping): + target: Literal["tags"] = Field(..., description="Transaction field to map to") + create: bool = Field( + default=True, description="Create new tags if they doesn't exist" + ) + coerce_to: Literal["list"] = Field("list", frozen=True) + + +class TransactionEntitiesMapping(ColumnMapping): + target: Literal["entities"] = Field(..., description="Transaction field to map to") + create: bool = Field( + default=True, description="Create new entities if they doesn't exist" + ) + coerce_to: Literal["list"] = Field("list", frozen=True) + + +class TransactionCategoryMapping(ColumnMapping): + target: Literal["category"] = Field(..., description="Transaction field to map to") + create: bool = Field( + default=True, description="Create category if it doesn't exist" + ) + type: Literal["id", "name"] = "name" + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class TransactionInternalMapping(ColumnMapping): + target: Literal["internal_note"] = Field( + ..., description="Transaction field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CategoryNameMapping(ColumnMapping): + target: Literal["category_name"] = Field( + ..., description="Category field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CategoryMuteMapping(ColumnMapping): + target: Literal["category_mute"] = Field( + ..., description="Category field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class CategoryActiveMapping(ColumnMapping): + target: Literal["category_active"] = Field( + ..., description="Category field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class TagNameMapping(ColumnMapping): + target: Literal["tag_name"] = Field(..., description="Tag field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class TagActiveMapping(ColumnMapping): + target: Literal["tag_active"] = Field(..., description="Tag field to map to") + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class EntityNameMapping(ColumnMapping): + target: Literal["entity_name"] = Field(..., description="Entity field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class EntityActiveMapping(ColumnMapping): + target: Literal["entitiy_active"] = Field(..., description="Entity field to map to") + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class AccountNameMapping(ColumnMapping): + target: Literal["account_name"] = Field(..., description="Account field to map to") + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class AccountGroupMapping(ColumnMapping): + target: Literal["account_group"] = Field(..., description="Account field to map to") + type: Literal["id", "name"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class AccountCurrencyMapping(ColumnMapping): + target: Literal["account_currency"] = Field( + ..., description="Account field to map to" + ) + type: Literal["id", "name", "code"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class AccountExchangeCurrencyMapping(ColumnMapping): + target: Literal["account_exchange_currency"] = Field( + ..., description="Account field to map to" + ) + type: Literal["id", "name", "code"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + +class AccountIsAssetMapping(ColumnMapping): + target: Literal["account_is_asset"] = Field( + ..., description="Account field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class AccountIsArchivedMapping(ColumnMapping): + target: Literal["account_is_archived"] = Field( + ..., description="Account field to map to" + ) + coerce_to: Literal["bool"] = Field("bool", frozen=True) + + +class CurrencyCodeMapping(ColumnMapping): + target: Literal["currency_code"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencyNameMapping(ColumnMapping): + target: Literal["currency_name"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencyDecimalPlacesMapping(ColumnMapping): + target: Literal["currency_decimal_places"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["int"] = Field("int", frozen=True) + + +class CurrencyPrefixMapping(ColumnMapping): + target: Literal["currency_prefix"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencySuffixMapping(ColumnMapping): + target: Literal["currency_suffix"] = Field( + ..., description="Currency field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + +class CurrencyExchangeMapping(ColumnMapping): + target: Literal["currency_exchange"] = Field( + ..., description="Currency field to map to" + ) + type: Literal["id", "name", "code"] + coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + + class ImportProfileSchema(BaseModel): - settings: ImportSettings - column_mapping: Dict[str, ColumnMapping] + settings: CSVImportSettings + mapping: Dict[ + str, + TransactionAccountMapping + | TransactionTypeMapping + | TransactionIsPaidMapping + | TransactionDateMapping + | TransactionReferenceDateMapping + | TransactionAmountMapping + | TransactionDescriptionMapping + | TransactionNotesMapping + | TransactionTagsMapping + | TransactionEntitiesMapping + | TransactionCategoryMapping + | TransactionInternalMapping + | CategoryNameMapping + | CategoryMuteMapping + | CategoryActiveMapping + | TagNameMapping + | TagActiveMapping + | EntityNameMapping + | EntityActiveMapping + | AccountNameMapping + | AccountGroupMapping + | AccountCurrencyMapping + | AccountExchangeCurrencyMapping + | AccountIsAssetMapping + | AccountIsArchivedMapping + | CurrencyCodeMapping + | CurrencyNameMapping + | CurrencyDecimalPlacesMapping + | CurrencyPrefixMapping + | CurrencySuffixMapping + | CurrencyExchangeMapping, + ] deduplication: List[CompareDeduplicationRule] = Field( default_factory=list, description="Rules for deduplicating records during import", ) + + @model_validator(mode="after") + def validate_mappings(self) -> "ImportProfileSchema": + import_type = self.settings.importing + + # Define allowed mapping types for each import type + allowed_mappings = { + "transactions": ( + TransactionAccountMapping, + TransactionTypeMapping, + TransactionIsPaidMapping, + TransactionDateMapping, + TransactionReferenceDateMapping, + TransactionAmountMapping, + TransactionDescriptionMapping, + TransactionNotesMapping, + TransactionTagsMapping, + TransactionEntitiesMapping, + TransactionCategoryMapping, + TransactionInternalMapping, + ), + "accounts": ( + AccountNameMapping, + AccountGroupMapping, + AccountCurrencyMapping, + AccountExchangeCurrencyMapping, + AccountIsAssetMapping, + AccountIsArchivedMapping, + ), + "currencies": ( + CurrencyCodeMapping, + CurrencyNameMapping, + CurrencyDecimalPlacesMapping, + CurrencyPrefixMapping, + CurrencySuffixMapping, + CurrencyExchangeMapping, + ), + "categories": ( + CategoryNameMapping, + CategoryMuteMapping, + CategoryActiveMapping, + ), + "tags": (TagNameMapping, TagActiveMapping), + "entities": (EntityNameMapping, EntityActiveMapping), + } + + allowed_types = allowed_mappings[import_type] + + for field_name, mapping in self.mapping.items(): + if not isinstance(mapping, allowed_types): + raise ValueError( + f"Mapping type '{type(mapping).__name__}' is not allowed when importing {import_type}. " + f"Allowed types are: {', '.join(t.__name__ for t in allowed_types)}" + ) + + return self diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 333eb6e..069115b 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -1,42 +1,56 @@ import csv import hashlib +import logging +import os import re from datetime import datetime -from typing import Dict, Any, Literal +from decimal import Decimal +from typing import Dict, Any, Literal, Union import yaml - from django.db import transaction -from django.core.files.storage import default_storage from django.utils import timezone +from apps.accounts.models import Account, AccountGroup +from apps.currencies.models import Currency from apps.import_app.models import ImportRun, ImportProfile -from apps.import_app.schemas import ( - SchemaV1, - ColumnMappingV1, - SettingsV1, - HashTransformationRuleV1, - CompareDeduplicationRuleV1, +from apps.import_app.schemas import version_1 +from apps.transactions.models import ( + Transaction, + TransactionCategory, + TransactionTag, + TransactionEntity, ) -from apps.transactions.models import Transaction + +logger = logging.getLogger(__name__) class ImportService: + TEMP_DIR = "/usr/src/app/temp" + def __init__(self, import_run: ImportRun): self.import_run: ImportRun = import_run self.profile: ImportProfile = import_run.profile - self.config: SchemaV1 = self._load_config() - self.settings: SettingsV1 = self.config.settings - self.deduplication: list[CompareDeduplicationRuleV1] = self.config.deduplication - self.mapping: Dict[str, ColumnMappingV1] = self.config.column_mapping + self.config: version_1.ImportProfileSchema = self._load_config() + self.settings: version_1.CSVImportSettings = self.config.settings + self.deduplication: list[version_1.CompareDeduplicationRule] = ( + self.config.deduplication + ) + self.mapping: Dict[str, version_1.ColumnMapping] = self.config.mapping - def _load_config(self) -> SchemaV1: + # Ensure temp directory exists + os.makedirs(self.TEMP_DIR, exist_ok=True) + + def _load_config(self) -> version_1.ImportProfileSchema: yaml_data = yaml.safe_load(self.profile.yaml_config) - - if self.profile.version == ImportProfile.Versions.VERSION_1: - return SchemaV1(**yaml_data) - - raise ValueError(f"Unsupported version: {self.profile.version}") + try: + config = version_1.ImportProfileSchema(**yaml_data) + except Exception as e: + self._log("error", f"Fatal error processing YAML config: {str(e)}") + self._update_status("FAILED") + raise e + else: + return config def _log(self, level: str, message: str, **kwargs) -> None: """Add a log entry to the import run logs""" @@ -53,6 +67,48 @@ class ImportService: self.import_run.logs += log_line self.import_run.save(update_fields=["logs"]) + def _update_totals( + self, + field: Literal["total", "processed", "successful", "skipped", "failed"], + value: int, + ) -> None: + if field == "total": + self.import_run.total_rows = value + self.import_run.save(update_fields=["total_rows"]) + elif field == "processed": + self.import_run.processed_rows = value + self.import_run.save(update_fields=["processed_rows"]) + elif field == "successful": + self.import_run.successful_rows = value + self.import_run.save(update_fields=["successful_rows"]) + elif field == "skipped": + self.import_run.skipped_rows = value + self.import_run.save(update_fields=["skipped_rows"]) + elif field == "failed": + self.import_run.failed_rows = value + self.import_run.save(update_fields=["failed_rows"]) + + def _increment_totals( + self, + field: Literal["total", "processed", "successful", "skipped", "failed"], + value: int, + ) -> None: + if field == "total": + self.import_run.total_rows = self.import_run.total_rows + value + self.import_run.save(update_fields=["total_rows"]) + elif field == "processed": + self.import_run.processed_rows = self.import_run.processed_rows + value + self.import_run.save(update_fields=["processed_rows"]) + elif field == "successful": + self.import_run.successful_rows = self.import_run.successful_rows + value + self.import_run.save(update_fields=["successful_rows"]) + elif field == "skipped": + self.import_run.skipped_rows = self.import_run.skipped_rows + value + self.import_run.save(update_fields=["skipped_rows"]) + elif field == "failed": + self.import_run.failed_rows = self.import_run.failed_rows + value + self.import_run.save(update_fields=["failed_rows"]) + def _update_status( self, new_status: Literal["PROCESSING", "FAILED", "FINISHED"] ) -> None: @@ -67,15 +123,12 @@ class ImportService: @staticmethod def _transform_value( - value: str, mapping: ColumnMappingV1, row: Dict[str, str] = None + value: str, mapping: version_1.ColumnMapping, row: Dict[str, str] = None ) -> Any: transformed = value for transform in mapping.transformations: if transform.type == "hash": - if not isinstance(transform, HashTransformationRuleV1): - continue - # Collect all values to be hashed values_to_hash = [] for field in transform.fields: @@ -88,47 +141,143 @@ class ImportService: transformed = hashlib.sha256(concatenated.encode()).hexdigest() elif transform.type == "replace": - transformed = transformed.replace( - transform.pattern, transform.replacement - ) + if transform.exclusive: + transformed = value.replace( + transform.pattern, transform.replacement + ) + else: + transformed = transformed.replace( + transform.pattern, transform.replacement + ) elif transform.type == "regex": - transformed = re.sub( - transform.pattern, transform.replacement, transformed - ) + if transform.exclusive: + transformed = re.sub( + transform.pattern, transform.replacement, value + ) + else: + transformed = re.sub( + transform.pattern, transform.replacement, transformed + ) elif transform.type == "date_format": transformed = datetime.strptime( - transformed, transform.pattern - ).strftime(transform.replacement) + transformed, transform.original_format + ).strftime(transform.new_format) + elif transform.type == "merge": + values_to_merge = [] + for field in transform.fields: + if field in row: + values_to_merge.append(str(row[field])) + transformed = transform.separator.join(values_to_merge) + elif transform.type == "split": + parts = transformed.split(transform.separator) + if transform.index is not None: + transformed = parts[transform.index] if parts else "" + else: + transformed = parts return transformed - def _map_row_to_transaction(self, row: Dict[str, str]) -> Dict[str, Any]: - transaction_data = {} + def _create_transaction(self, data: Dict[str, Any]) -> Transaction: + tags = [] + entities = [] + # Handle related objects first + if "category" in data: + category_name = data.pop("category") + category, _ = TransactionCategory.objects.get_or_create(name=category_name) + data["category"] = category + self.import_run.categories.add(category) - for field, mapping in self.mapping.items(): - # If source is None, use None as the initial value - value = row.get(mapping.source) if mapping.source else None + if "account" in data: + account_id = data.pop("account") + account = None + if isinstance(account_id, str): + account = Account.objects.get(name=account_id) + elif isinstance(account_id, int): + account = Account.objects.get(id=account_id) + data["account"] = account + # self.import_run.acc.add(category) - # Use default_value if value is None - if value is None: - value = mapping.default_value + if "tags" in data: + tag_names = data.pop("tags").split(",") + for tag_name in tag_names: + tag, _ = TransactionTag.objects.get_or_create(name=tag_name.strip()) + tags.append(tag) + self.import_run.tags.add(tag) - if mapping.required and value is None and not mapping.transformations: - raise ValueError(f"Required field {field} is missing") + if "entities" in data: + entity_names = data.pop("entities").split(",") + for entity_name in entity_names: + entity, _ = TransactionEntity.objects.get_or_create( + name=entity_name.strip() + ) + entities.append(entity) + self.import_run.entities.add(entity) - # Apply transformations even if initial value is None - if mapping.transformations: - value = self._transform_value(value, mapping, row) + if "amount" in data: + amount = data.pop("amount") + data["amount"] = abs(Decimal(amount)) - if value is not None: - transaction_data[field] = value + # Create the transaction + new_transaction = Transaction.objects.create(**data) + self.import_run.transactions.add(new_transaction) - return transaction_data + # Add many-to-many relationships + if tags: + new_transaction.tags.set(tags) + if entities: + new_transaction.entities.set(entities) + + return new_transaction + + def _create_account(self, data: Dict[str, Any]) -> Account: + if "group" in data: + group_name = data.pop("group") + group, _ = AccountGroup.objects.get_or_create(name=group_name) + data["group"] = group + + # Handle currency references + if "currency" in data: + currency = Currency.objects.get(code=data["currency"]) + data["currency"] = currency + self.import_run.currencies.add(currency) + + if "exchange_currency" in data: + exchange_currency = Currency.objects.get(code=data["exchange_currency"]) + data["exchange_currency"] = exchange_currency + self.import_run.currencies.add(exchange_currency) + + return Account.objects.create(**data) + + def _create_currency(self, data: Dict[str, Any]) -> Currency: + # Handle exchange currency reference + if "exchange_currency" in data: + exchange_currency = Currency.objects.get(code=data["exchange_currency"]) + data["exchange_currency"] = exchange_currency + self.import_run.currencies.add(exchange_currency) + + currency = Currency.objects.create(**data) + self.import_run.currencies.add(currency) + return currency + + def _create_category(self, data: Dict[str, Any]) -> TransactionCategory: + category = TransactionCategory.objects.create(**data) + self.import_run.categories.add(category) + return category + + def _create_tag(self, data: Dict[str, Any]) -> TransactionTag: + tag = TransactionTag.objects.create(**data) + self.import_run.tags.add(tag) + return tag + + def _create_entity(self, data: Dict[str, Any]) -> TransactionEntity: + entity = TransactionEntity.objects.create(**data) + self.import_run.entities.add(entity) + return entity def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool: for rule in self.deduplication: if rule.type == "compare": - query = Transaction.objects.all() + query = Transaction.objects.all().values("id") # Build query conditions for each field in the rule for field, header in rule.fields.items(): @@ -146,65 +295,214 @@ class ImportService: return False - def _process_csv(self, file_path): - with open(file_path, "r", encoding=self.settings.encoding) as csv_file: - reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + def _coerce_type( + self, value: str, mapping: version_1.ColumnMapping + ) -> Union[str, int, bool, Decimal, datetime, list]: + if not value: + return None + + coerce_to = mapping.coerce_to + + if "|" in coerce_to: + types = coerce_to.split("|") + for t in types: + try: + return self._coerce_single_type(value, t, mapping) + except ValueError: + continue + raise ValueError( + f"Could not coerce '{value}' to any of the types: {coerce_to}" + ) + else: + return self._coerce_single_type(value, coerce_to, mapping) + + def _coerce_single_type( + self, value: str, coerce_to: str, mapping: version_1.ColumnMapping + ) -> Union[str, int, bool, Decimal, datetime.date, list]: + if coerce_to == "str": + return str(value) + elif coerce_to == "int": + if hasattr(mapping, "type") and mapping.type == "id": + return int(value) + elif hasattr(mapping, "type") and mapping.type in ["name", "code"]: + return str(value) + else: + return int(value) + elif coerce_to == "bool": + return value.lower() in ["true", "1", "yes", "y", "on"] + elif coerce_to == "positive_decimal": + return abs(Decimal(value)) + elif coerce_to == "date": + if isinstance( + mapping, + ( + version_1.TransactionDateMapping, + version_1.TransactionReferenceDateMapping, + ), + ): + formats = ( + mapping.format + if isinstance(mapping.format, list) + else [mapping.format] + ) + for fmt in formats: + try: + return datetime.strptime(value, fmt).date() + except ValueError: + continue + raise ValueError( + f"Could not parse date '{value}' with any of the provided formats" + ) + else: + raise ValueError( + "Date coercion is only supported for TransactionDateMapping and TransactionReferenceDateMapping" + ) + elif coerce_to == "list": + return ( + value + if isinstance(value, list) + else [item.strip() for item in value.split(",") if item.strip()] + ) + elif coerce_to == "transaction_type": + if isinstance(mapping, version_1.TransactionTypeMapping): + if mapping.detection_method == "sign": + return ( + Transaction.Type.EXPENSE + if value.startswith("-") + else Transaction.Type.INCOME + ) + elif mapping.detection_method == "always_income": + return Transaction.Type.INCOME + elif mapping.detection_method == "always_expense": + return Transaction.Type.EXPENSE + raise ValueError("Invalid transaction type detection method") + elif coerce_to == "is_paid": + if isinstance(mapping, version_1.TransactionIsPaidMapping): + if mapping.detection_method == "sign": + return not value.startswith("-") + elif mapping.detection_method == "boolean": + return value.lower() in ["true", "1", "yes", "y", "on"] + elif mapping.detection_method == "always_paid": + return True + elif mapping.detection_method == "always_unpaid": + return False + raise ValueError("Invalid is_paid detection method") + else: + raise ValueError(f"Unsupported coercion type: {coerce_to}") + + def _map_row(self, row: Dict[str, str]) -> Dict[str, Any]: + mapped_data = {} + + for field, mapping in self.mapping.items(): + # If source is None, use None as the initial value + value = row.get(mapping.source) if mapping.source else None + + # Use default_value if value is None + if value is None: + value = mapping.default + + if mapping.required and value is None and not mapping.transformations: + raise ValueError(f"Required field {field} is missing") + + # Apply transformations + if mapping.transformations: + value = self._transform_value(value, mapping, row) + + value = self._coerce_type(value, mapping) + + if value is not None: + # Remove the prefix from the target field + target = mapping.target + if self.settings.importing == "transactions": + mapped_data[target] = value + else: + # Remove the model prefix (e.g., "account_" from "account_name") + field_name = target.split("_", 1)[1] + mapped_data[field_name] = value + + return mapped_data + + def _process_row(self, row: Dict[str, str], row_number: int) -> None: + try: + mapped_data = self._map_row(row) + + if mapped_data: + # Handle different import types + if self.settings.importing == "transactions": + if self.deduplication and self._check_duplicate_transaction( + mapped_data + ): + self._increment_totals("skipped", 1) + self._log("info", f"Skipped duplicate row {row_number}") + return + self._create_transaction(mapped_data) + elif self.settings.importing == "accounts": + self._create_account(mapped_data) + elif self.settings.importing == "currencies": + self._create_currency(mapped_data) + elif self.settings.importing == "categories": + self._create_category(mapped_data) + elif self.settings.importing == "tags": + self._create_tag(mapped_data) + elif self.settings.importing == "entities": + self._create_entity(mapped_data) + + self._increment_totals("successful", value=1) + self._log("info", f"Successfully processed row {row_number}") + + self._increment_totals("processed", value=1) + + except Exception as e: + if not self.settings.skip_errors: + self._log("error", f"Fatal error processing row {row_number}: {str(e)}") + self._update_status("FAILED") + raise + else: + self._log("warning", f"Error processing row {row_number}: {str(e)}") + self._increment_totals("failed", value=1) + + logger.error(f"Fatal error processing row {row_number}", exc_info=e) + + def _process_csv(self, file_path): + # First pass: count rows + with open(file_path, "r", encoding=self.settings.encoding) as csv_file: + # Skip specified number of rows + for _ in range(self.settings.skip_lines): + next(csv_file) + + reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) + self._update_totals("total", value=sum(1 for _ in reader)) + + with open(file_path, "r", encoding=self.settings.encoding) as csv_file: + # Skip specified number of rows + for _ in range(self.settings.skip_lines): + next(csv_file) + if self.settings.skip_lines: + self._log("info", f"Skipped {self.settings.skip_lines} initial lines") - # Count total rows - self.import_run.total_rows = sum(1 for _ in reader) - csv_file.seek(0) reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter) self._log("info", f"Starting import with {self.import_run.total_rows} rows") - # Skip specified number of rows - for _ in range(self.settings.skip_rows): - next(reader) + with transaction.atomic(): + for row_number, row in enumerate(reader, start=1): + self._process_row(row, row_number) + self._increment_totals("processed", value=1) - if self.settings.skip_rows: - self._log("info", f"Skipped {self.settings.skip_rows} initial rows") - - for row_number, row in enumerate(reader, start=1): - try: - transaction_data = self._map_row_to_transaction(row) - - if transaction_data: - if self.deduplication and self._check_duplicate_transaction( - transaction_data - ): - self.import_run.skipped_rows += 1 - self._log("info", f"Skipped duplicate row {row_number}") - continue - - self.import_run.transactions.add(transaction_data) - self.import_run.successful_rows += 1 - self._log("debug", f"Successfully processed row {row_number}") - - self.import_run.processed_rows += 1 - self.import_run.save( - update_fields=[ - "processed_rows", - "successful_rows", - "skipped_rows", - ] - ) - - except Exception as e: - if not self.settings.skip_errors: - self._log( - "error", - f"Fatal error processing row {row_number}: {str(e)}", - ) - self._update_status("FAILED") - raise - else: - self._log( - "warning", f"Error processing row {row_number}: {str(e)}" - ) - self.import_run.failed_rows += 1 - self.import_run.save(update_fields=["failed_rows"]) + def _validate_file_path(self, file_path: str) -> str: + """ + Validates that the file path is within the allowed temporary directory. + Returns the absolute path. + """ + abs_path = os.path.abspath(file_path) + if not abs_path.startswith(self.TEMP_DIR): + raise ValueError(f"Invalid file path. File must be in {self.TEMP_DIR}") + return abs_path def process_file(self, file_path: str): + # Validate and get absolute path + file_path = self._validate_file_path(file_path) + self._update_status("PROCESSING") self.import_run.started_at = timezone.now() self.import_run.save(update_fields=["started_at"]) @@ -232,6 +530,12 @@ class ImportService: finally: self._log("info", "Cleaning up temporary files") - default_storage.delete(file_path) + try: + if os.path.exists(file_path): + os.remove(file_path) + self._log("info", f"Deleted temporary file: {file_path}") + except OSError as e: + self._log("warning", f"Failed to delete temporary file: {str(e)}") + self.import_run.finished_at = timezone.now() self.import_run.save(update_fields=["finished_at"]) From 87345cf235bb1422f8ede4485182332b069fb72a Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 11:45:06 -0300 Subject: [PATCH 23/60] docs(requirements): add django_ace --- app/WYGIWYH/settings.py | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index d597d5d..36cea84 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -70,6 +70,7 @@ INSTALLED_APPS = [ "rest_framework", "drf_spectacular", "django_cotton", + "django_ace", "apps.rules.apps.RulesConfig", "apps.calendar_view.apps.CalendarViewConfig", "apps.dca.apps.DcaConfig", diff --git a/requirements.txt b/requirements.txt index af9d39b..8c24038 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ django-filter==24.3 django-debug-toolbar==4.3.0 django-cachalot~=2.6.3 django-cotton~=1.2.1 +django_ace~=1.36.2 djangorestframework~=3.15.2 drf-spectacular~=0.27.2 From 3440d4405e13250b319c21074f76441270e5b4dd Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 11:47:33 -0300 Subject: [PATCH 24/60] docker: add temp volume --- docker-compose.dev.yml | 3 ++- docker-compose.prod.yml | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index c06c0fd..133d522 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,6 +1,6 @@ volumes: wygiwyh_dev_postgres_data: {} - temp: + wygiwyh_temp: services: web: &django @@ -13,6 +13,7 @@ services: volumes: - ./app/:/usr/src/app/:z - ./frontend/:/usr/src/frontend:z + - wygiwyh_temp:/usr/src/app/temp/ ports: - "${OUTBOUND_PORT}:8000" env_file: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a12b4ed..b840e46 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -9,6 +9,8 @@ services: - .env depends_on: - db + volumes: + - wygiwyh_temp:/usr/src/app/temp/ restart: unless-stopped db: @@ -29,5 +31,10 @@ services: - db env_file: - .env + volumes: + - wygiwyh_temp:/usr/src/app/temp/ command: /start-procrastinate restart: unless-stopped + +volumes: + wygiwyh_temp: From a52f682c4fdcc50b3c7a1fbccafd0043cf8370c3 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:55:17 -0300 Subject: [PATCH 25/60] feat(transactions): soft delete --- app/apps/transactions/admin.py | 18 +++++ .../0028_transaction_internal_note.py | 18 +++++ .../0029_alter_transaction_options.py | 17 ++++ ...nsaction_deleted_transaction_deleted_at.py | 23 ++++++ .../0031_alter_transaction_deleted.py | 18 +++++ ...ction_created_at_transaction_updated_at.py | 25 ++++++ app/apps/transactions/models.py | 77 +++++++++++++++++++ 7 files changed, 196 insertions(+) create mode 100644 app/apps/transactions/migrations/0028_transaction_internal_note.py create mode 100644 app/apps/transactions/migrations/0029_alter_transaction_options.py create mode 100644 app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py create mode 100644 app/apps/transactions/migrations/0031_alter_transaction_deleted.py create mode 100644 app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py diff --git a/app/apps/transactions/admin.py b/app/apps/transactions/admin.py index 5a4ef15..df4d1c8 100644 --- a/app/apps/transactions/admin.py +++ b/app/apps/transactions/admin.py @@ -12,7 +12,14 @@ from apps.transactions.models import ( @admin.register(Transaction) class TransactionModelAdmin(admin.ModelAdmin): + def get_queryset(self, request): + # Use the all_objects manager to show all transactions, including deleted ones + return self.model.all_objects.all() + + list_filter = ["deleted", "type", "is_paid", "date", "account"] + list_display = [ + "deleted", "description", "type", "account__name", @@ -22,6 +29,17 @@ class TransactionModelAdmin(admin.ModelAdmin): "reference_date", ] + actions = ["hard_delete_selected"] + + def hard_delete_selected(self, request, queryset): + for obj in queryset: + obj.hard_delete() + self.message_user( + request, f"Successfully hard deleted {queryset.count()} transactions." + ) + + hard_delete_selected.short_description = "Hard delete selected transactions" + class TransactionInline(admin.TabularInline): model = Transaction diff --git a/app/apps/transactions/migrations/0028_transaction_internal_note.py b/app/apps/transactions/migrations/0028_transaction_internal_note.py new file mode 100644 index 0000000..c88c11d --- /dev/null +++ b/app/apps/transactions/migrations/0028_transaction_internal_note.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.5 on 2025-01-19 00:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0027_alter_transaction_description'), + ] + + operations = [ + migrations.AddField( + model_name='transaction', + name='internal_note', + field=models.TextField(blank=True, verbose_name='Internal Note'), + ), + ] diff --git a/app/apps/transactions/migrations/0029_alter_transaction_options.py b/app/apps/transactions/migrations/0029_alter_transaction_options.py new file mode 100644 index 0000000..c06b7cd --- /dev/null +++ b/app/apps/transactions/migrations/0029_alter_transaction_options.py @@ -0,0 +1,17 @@ +# Generated by Django 5.1.5 on 2025-01-19 14:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0028_transaction_internal_note'), + ] + + operations = [ + migrations.AlterModelOptions( + name='transaction', + options={'default_manager_name': 'objects', 'verbose_name': 'Transaction', 'verbose_name_plural': 'Transactions'}, + ), + ] diff --git a/app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py b/app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py new file mode 100644 index 0000000..35f4c91 --- /dev/null +++ b/app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.5 on 2025-01-19 14:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0029_alter_transaction_options'), + ] + + operations = [ + migrations.AddField( + model_name='transaction', + name='deleted', + field=models.BooleanField(default=False, verbose_name='Deleted'), + ), + migrations.AddField( + model_name='transaction', + name='deleted_at', + field=models.DateTimeField(blank=True, null=True, verbose_name='Deleted At'), + ), + ] diff --git a/app/apps/transactions/migrations/0031_alter_transaction_deleted.py b/app/apps/transactions/migrations/0031_alter_transaction_deleted.py new file mode 100644 index 0000000..b5d2dc4 --- /dev/null +++ b/app/apps/transactions/migrations/0031_alter_transaction_deleted.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.5 on 2025-01-19 15:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0030_transaction_deleted_transaction_deleted_at'), + ] + + operations = [ + migrations.AlterField( + model_name='transaction', + name='deleted', + field=models.BooleanField(db_index=True, default=False, verbose_name='Deleted'), + ), + ] diff --git a/app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py b/app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py new file mode 100644 index 0000000..46e76ae --- /dev/null +++ b/app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.5 on 2025-01-19 16:48 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0031_alter_transaction_deleted'), + ] + + operations = [ + migrations.AddField( + model_name='transaction', + name='created_at', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='transaction', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index f131518..2bd2a68 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -6,6 +6,7 @@ from django.db import models, transaction from django.db.models import Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from django.conf import settings from apps.common.fields.month_year import MonthYearModelField from apps.common.functions.decimals import truncate_decimal @@ -15,6 +16,53 @@ from apps.transactions.validators import validate_decimal_places, validate_non_n logger = logging.getLogger() +class SoftDeleteQuerySet(models.QuerySet): + def delete(self): + if not settings.ENABLE_SOFT_DELETION: + # If soft deletion is disabled, perform a normal delete + return super().delete() + + # Separate the queryset into already deleted and not deleted objects + already_deleted = self.filter(deleted=True) + not_deleted = self.filter(deleted=False) + + # Use a transaction to ensure atomicity + with transaction.atomic(): + # Perform hard delete on already deleted objects + hard_deleted_count = already_deleted._raw_delete(already_deleted.db) + + # Perform soft delete on not deleted objects + soft_deleted_count = not_deleted.update( + deleted=True, deleted_at=timezone.now() + ) + + # Return a tuple of counts as expected by Django's delete method + return ( + hard_deleted_count + soft_deleted_count, + {"Transaction": hard_deleted_count + soft_deleted_count}, + ) + + def hard_delete(self): + return super().delete() + + +class SoftDeleteManager(models.Manager): + def get_queryset(self): + qs = SoftDeleteQuerySet(self.model, using=self._db) + return qs if not settings.ENABLE_SOFT_DELETION else qs.filter(deleted=False) + + +class AllObjectsManager(models.Manager): + def get_queryset(self): + return SoftDeleteQuerySet(self.model, using=self._db) + + +class DeletedObjectsManager(models.Manager): + def get_queryset(self): + qs = SoftDeleteQuerySet(self.model, using=self._db) + return qs if not settings.ENABLE_SOFT_DELETION else qs.filter(deleted=True) + + class TransactionCategory(models.Model): name = models.CharField(max_length=255, verbose_name=_("Name"), unique=True) mute = models.BooleanField(default=False, verbose_name=_("Mute")) @@ -143,10 +191,24 @@ class Transaction(models.Model): ) internal_note = models.TextField(blank=True, verbose_name=_("Internal Note")) + deleted = models.BooleanField( + default=False, verbose_name=_("Deleted"), db_index=True + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + deleted_at = models.DateTimeField( + null=True, blank=True, verbose_name=_("Deleted At") + ) + + objects = SoftDeleteManager.from_queryset(SoftDeleteQuerySet)() + all_objects = AllObjectsManager.from_queryset(SoftDeleteQuerySet)() + deleted_objects = DeletedObjectsManager.from_queryset(SoftDeleteQuerySet)() + class Meta: verbose_name = _("Transaction") verbose_name_plural = _("Transactions") db_table = "transactions" + default_manager_name = "objects" def save(self, *args, **kwargs): self.amount = truncate_decimal( @@ -161,6 +223,17 @@ class Transaction(models.Model): self.full_clean() super().save(*args, **kwargs) + def delete(self, *args, **kwargs): + if settings.ENABLE_SOFT_DELETION: + self.deleted = True + self.deleted_at = timezone.now() + self.save() + else: + super().delete(*args, **kwargs) + + def hard_delete(self, *args, **kwargs): + super().delete(*args, **kwargs) + def exchanged_amount(self): if self.account.exchange_currency: converted_amount, prefix, suffix, decimal_places = convert( @@ -179,6 +252,10 @@ class Transaction(models.Model): return None + def __str__(self): + type_display = self.get_type_display() + return f"{self.description} - {type_display} - {self.account} - {self.date}" + class InstallmentPlan(models.Model): class Recurrence(models.TextChoices): From 8a127a9f4ff545591138126a80c1ebba0cc0c331 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:55:25 -0300 Subject: [PATCH 26/60] feat(transactions): soft delete --- app/WYGIWYH/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 36cea84..f663074 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -337,3 +337,5 @@ else: } CACHALOT_UNCACHABLE_TABLES = ("django_migrations", "procrastinate_jobs") + +ENABLE_SOFT_DELETION = os.environ.get("ENABLE_SOFT_DELETION", "False").lower() == "true" From 2ff33526aeb09cb2faebdfcaf489c995bab0c738 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:56:13 -0300 Subject: [PATCH 27/60] feat(import): disable cache when running --- app/apps/import_app/services.py | 0 app/apps/import_app/services/v1.py | 84 ++++++++++++++++-------------- 2 files changed, 44 insertions(+), 40 deletions(-) delete mode 100644 app/apps/import_app/services.py diff --git a/app/apps/import_app/services.py b/app/apps/import_app/services.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 069115b..7735342 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -7,8 +7,9 @@ from datetime import datetime from decimal import Decimal from typing import Dict, Any, Literal, Union +import cachalot.api import yaml -from django.db import transaction +from cachalot.api import cachalot_disabled from django.utils import timezone from apps.accounts.models import Account, AccountGroup @@ -277,7 +278,7 @@ class ImportService: def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool: for rule in self.deduplication: if rule.type == "compare": - query = Transaction.objects.all().values("id") + query = Transaction.all_objects.all().values("id") # Build query conditions for each field in the rule for field, header in rule.fields.items(): @@ -484,10 +485,9 @@ class ImportService: self._log("info", f"Starting import with {self.import_run.total_rows} rows") - with transaction.atomic(): - for row_number, row in enumerate(reader, start=1): - self._process_row(row, row_number) - self._increment_totals("processed", value=1) + for row_number, row in enumerate(reader, start=1): + self._process_row(row, row_number) + self._increment_totals("processed", value=1) def _validate_file_path(self, file_path: str) -> str: """ @@ -500,42 +500,46 @@ class ImportService: return abs_path def process_file(self, file_path: str): - # Validate and get absolute path - file_path = self._validate_file_path(file_path) + with cachalot_disabled(): + # Validate and get absolute path + file_path = self._validate_file_path(file_path) - self._update_status("PROCESSING") - self.import_run.started_at = timezone.now() - self.import_run.save(update_fields=["started_at"]) + self._update_status("PROCESSING") + self.import_run.started_at = timezone.now() + self.import_run.save(update_fields=["started_at"]) - self._log("info", "Starting import process") + self._log("info", "Starting import process") - try: - if self.settings.file_type == "csv": - self._process_csv(file_path) - - if self.import_run.processed_rows == self.import_run.total_rows: - self._update_status("FINISHED") - self._log( - "info", - f"Import completed successfully. " - f"Successful: {self.import_run.successful_rows}, " - f"Failed: {self.import_run.failed_rows}, " - f"Skipped: {self.import_run.skipped_rows}", - ) - - except Exception as e: - self._update_status("FAILED") - self._log("error", f"Import failed: {str(e)}") - raise Exception("Import failed") - - finally: - self._log("info", "Cleaning up temporary files") try: - if os.path.exists(file_path): - os.remove(file_path) - self._log("info", f"Deleted temporary file: {file_path}") - except OSError as e: - self._log("warning", f"Failed to delete temporary file: {str(e)}") + if self.settings.file_type == "csv": + self._process_csv(file_path) - self.import_run.finished_at = timezone.now() - self.import_run.save(update_fields=["finished_at"]) + if self.import_run.processed_rows == self.import_run.total_rows: + self._update_status("FINISHED") + self._log( + "info", + f"Import completed successfully. " + f"Successful: {self.import_run.successful_rows}, " + f"Failed: {self.import_run.failed_rows}, " + f"Skipped: {self.import_run.skipped_rows}", + ) + + except Exception as e: + self._update_status("FAILED") + self._log("error", f"Import failed: {str(e)}") + raise Exception("Import failed") + + finally: + self._log("info", "Cleaning up temporary files") + try: + if os.path.exists(file_path): + os.remove(file_path) + self._log("info", f"Deleted temporary file: {file_path}") + except OSError as e: + self._log("warning", f"Failed to delete temporary file: {str(e)}") + + self.import_run.finished_at = timezone.now() + self.import_run.save(update_fields=["finished_at"]) + + if self.import_run.successful_rows >= 1: + cachalot.api.invalidate() From 18d8e8ed1aad13ba8b33045f073dcf9f6b7d5868 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 13:56:29 -0300 Subject: [PATCH 28/60] feat(import): add migrations --- .../import_app/migrations/0001_initial.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 app/apps/import_app/migrations/0001_initial.py diff --git a/app/apps/import_app/migrations/0001_initial.py b/app/apps/import_app/migrations/0001_initial.py new file mode 100644 index 0000000..bcce0fe --- /dev/null +++ b/app/apps/import_app/migrations/0001_initial.py @@ -0,0 +1,51 @@ +# Generated by Django 5.1.5 on 2025-01-19 00:44 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('currencies', '0006_currency_exchange_currency'), + ('transactions', '0028_transaction_internal_note'), + ] + + operations = [ + migrations.CreateModel( + name='ImportProfile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('yaml_config', models.TextField(help_text='YAML configuration')), + ('version', models.IntegerField(choices=[(1, 'Version 1')], default=1, verbose_name='Version')), + ], + options={ + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='ImportRun', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('QUEUED', 'Queued'), ('PROCESSING', 'Processing'), ('FAILED', 'Failed'), ('FINISHED', 'Finished')], default='QUEUED', max_length=10, verbose_name='Status')), + ('file_name', models.CharField(help_text='File name', max_length=10000)), + ('logs', models.TextField(blank=True)), + ('processed_rows', models.IntegerField(default=0)), + ('total_rows', models.IntegerField(default=0)), + ('successful_rows', models.IntegerField(default=0)), + ('skipped_rows', models.IntegerField(default=0)), + ('failed_rows', models.IntegerField(default=0)), + ('started_at', models.DateTimeField(null=True)), + ('finished_at', models.DateTimeField(null=True)), + ('categories', models.ManyToManyField(related_name='import_runs', to='transactions.transactioncategory')), + ('currencies', models.ManyToManyField(related_name='import_runs', to='currencies.currency')), + ('entities', models.ManyToManyField(related_name='import_runs', to='transactions.transactionentity')), + ('profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='import_app.importprofile')), + ('tags', models.ManyToManyField(related_name='import_runs', to='transactions.transactiontag')), + ('transactions', models.ManyToManyField(related_name='import_runs', to='transactions.transaction')), + ], + ), + ] From f2cc0705053165d064a22a2a5a087c04e93ecbbb Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 15:16:47 -0300 Subject: [PATCH 29/60] feat(settings): add KEEP_DELETED_TRANSACTIONS_FOR variable --- app/WYGIWYH/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index f663074..960b0ec 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -338,4 +338,5 @@ else: CACHALOT_UNCACHABLE_TABLES = ("django_migrations", "procrastinate_jobs") -ENABLE_SOFT_DELETION = os.environ.get("ENABLE_SOFT_DELETION", "False").lower() == "true" +ENABLE_SOFT_DELETION = os.getenv("ENABLE_SOFT_DELETION", "True").lower() == "true" +KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365")) From 34e6914d41f01647174557667da2f60458bce8e6 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 15:17:18 -0300 Subject: [PATCH 30/60] feat(transactions:tasks): add old deleted transactions cleanup task --- app/apps/transactions/tasks.py | 39 ++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/app/apps/transactions/tasks.py b/app/apps/transactions/tasks.py index e0bfafc..5f1c42f 100644 --- a/app/apps/transactions/tasks.py +++ b/app/apps/transactions/tasks.py @@ -1,9 +1,13 @@ import logging +from datetime import timedelta + +from cachalot.api import cachalot_disabled, invalidate +from django.utils import timezone +from django.conf import settings from procrastinate.contrib.django import app -from apps.transactions.models import RecurringTransaction - +from apps.transactions.models import RecurringTransaction, Transaction logger = logging.getLogger(__name__) @@ -19,3 +23,34 @@ def generate_recurring_transactions(timestamp=None): exc_info=True, ) raise e + + +@app.periodic(cron="10 1 * * *") +@app.task +def cleanup_deleted_transactions(): + with cachalot_disabled(): + if ( + settings.ENABLE_SOFT_DELETION + and settings.KEEP_DELETED_TRANSACTIONS_FOR == 0 + ): + return "KEEP_DELETED_TRANSACTIONS_FOR is 0, no cleanup performed." + + if not settings.ENABLE_SOFT_DELETION: + # Hard delete all soft-deleted transactions + deleted_count, _ = Transaction.deleted_objects.all().hard_delete() + return ( + f"Hard deleted {deleted_count} transactions (soft deletion disabled)." + ) + + # Calculate the cutoff date + cutoff_date = timezone.now() - timedelta( + days=settings.KEEP_DELETED_TRANSACTIONS_FOR + ) + + invalidate("transactions.Transaction") + + # Hard delete soft-deleted transactions older than the cutoff date + old_transactions = Transaction.deleted_objects.filter(deleted_at__lt=cutoff_date) + deleted_count, _ = old_transactions.hard_delete() + + return f"Hard deleted {deleted_count} objects older than {settings.KEEP_DELETED_TRANSACTIONS_FOR} days." From 76df16e48999670cf04b36f59eba8e4f10c1a804 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 19 Jan 2025 15:20:25 -0300 Subject: [PATCH 31/60] feat(import:v1:schema): add option for triggering rules --- app/apps/import_app/schemas/v1.py | 1 + app/apps/import_app/services/v1.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 043f2a9..74e37a1 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -69,6 +69,7 @@ class CSVImportSettings(BaseModel): skip_lines: int = Field( default=0, description="Number of rows to skip at the beginning of the file" ) + trigger_transaction_rules: bool = True importing: Literal[ "transactions", "accounts", "currencies", "categories", "tags", "entities" ] diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 7735342..0416caf 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -22,6 +22,7 @@ from apps.transactions.models import ( TransactionTag, TransactionEntity, ) +from apps.rules.signals import transaction_created logger = logging.getLogger(__name__) @@ -228,6 +229,9 @@ class ImportService: if entities: new_transaction.entities.set(entities) + if self.settings.trigger_transaction_rules: + transaction_created.send(sender=new_transaction) + return new_transaction def _create_account(self, data: Dict[str, Any]) -> Account: From 61d5aba67ce785c9fafc0081c2dd2034fe43988e Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:30:17 -0300 Subject: [PATCH 32/60] feat(import): some layouts --- app/apps/import_app/forms.py | 58 +++++++++++++++++ .../import_app/fragments/profiles/add.html | 11 ++++ .../import_app/fragments/profiles/edit.html | 11 ++++ .../import_app/fragments/profiles/list.html | 65 +++++++++++++++++++ .../import_app/fragments/runs/add.html | 11 ++++ .../import_app/fragments/runs/list.html | 9 +++ 6 files changed, 165 insertions(+) create mode 100644 app/apps/import_app/forms.py create mode 100644 app/templates/import_app/fragments/profiles/add.html create mode 100644 app/templates/import_app/fragments/profiles/edit.html create mode 100644 app/templates/import_app/fragments/profiles/list.html create mode 100644 app/templates/import_app/fragments/runs/add.html create mode 100644 app/templates/import_app/fragments/runs/list.html diff --git a/app/apps/import_app/forms.py b/app/apps/import_app/forms.py new file mode 100644 index 0000000..78ee3d7 --- /dev/null +++ b/app/apps/import_app/forms.py @@ -0,0 +1,58 @@ +from crispy_forms.bootstrap import FormActions +from crispy_forms.helper import FormHelper +from crispy_forms.layout import ( + Layout, +) +from django import forms +from django.utils.translation import gettext_lazy as _ +from django_ace import AceWidget + +from apps.import_app.models import ImportProfile +from apps.common.widgets.crispy.submit import NoClassSubmit + + +class ImportProfileForm(forms.ModelForm): + class Meta: + model = ImportProfile + fields = [ + "name", + "version", + "yaml_config", + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.helper = FormHelper() + self.helper.form_tag = False + self.helper.form_method = "post" + self.helper.layout = Layout("name", "version", "yaml_config") + + if self.instance and self.instance.pk: + self.helper.layout.append( + FormActions( + NoClassSubmit( + "submit", _("Update"), css_class="btn btn-outline-primary w-100" + ), + ), + ) + else: + self.helper.layout.append( + FormActions( + NoClassSubmit( + "submit", _("Add"), css_class="btn btn-outline-primary w-100" + ), + ), + ) + + +class ImportRunFileUploadForm(forms.Form): + file = forms.FileField(label=_("Select a file")) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.helper = FormHelper() + self.helper.form_tag = False + self.helper.form_method = "post" + self.helper.layout = Layout("file") diff --git a/app/templates/import_app/fragments/profiles/add.html b/app/templates/import_app/fragments/profiles/add.html new file mode 100644 index 0000000..beda873 --- /dev/null +++ b/app/templates/import_app/fragments/profiles/add.html @@ -0,0 +1,11 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Add new import profile' %}{% endblock %} + +{% block body %} +
+ {% crispy form %} +
+{% endblock %} diff --git a/app/templates/import_app/fragments/profiles/edit.html b/app/templates/import_app/fragments/profiles/edit.html new file mode 100644 index 0000000..fa94bef --- /dev/null +++ b/app/templates/import_app/fragments/profiles/edit.html @@ -0,0 +1,11 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Edit import profile' %}{% endblock %} + +{% block body %} +
+ {% crispy form %} +
+{% endblock %} diff --git a/app/templates/import_app/fragments/profiles/list.html b/app/templates/import_app/fragments/profiles/list.html new file mode 100644 index 0000000..f1f34d2 --- /dev/null +++ b/app/templates/import_app/fragments/profiles/list.html @@ -0,0 +1,65 @@ +{% load i18n %} +
+
+ {% spaceless %} +
{% translate 'Import Profiles' %} + + +
+ {% endspaceless %} +
+ +
+
+ {% if profiles %} + + + + + + + + + + + {% for profile in profiles %} + + + + + + {% endfor %} + +
{% translate 'Name' %}{% translate 'Version' %}
+
+ + +{# #} +{#
#} + 		{{ profile.name }}{{ profile.get_version_display }}
+ {% else %} + + {% endif %} +
+
+
diff --git a/app/templates/import_app/fragments/runs/add.html b/app/templates/import_app/fragments/runs/add.html new file mode 100644 index 0000000..d5a5b89 --- /dev/null +++ b/app/templates/import_app/fragments/runs/add.html @@ -0,0 +1,11 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Import file' %}{% endblock %} + +{% block body %} +
+ {% crispy form %} +
+{% endblock %} diff --git a/app/templates/import_app/fragments/runs/list.html b/app/templates/import_app/fragments/runs/list.html new file mode 100644 index 0000000..0697d26 --- /dev/null +++ b/app/templates/import_app/fragments/runs/list.html @@ -0,0 +1,9 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Runs for ' %}{{ profile.name }}{% endblock %} + +{% block body %} + +{% endblock %} From 0f14fd0c6275fcdbb91231088b35fa0e7511a471 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:30:40 -0300 Subject: [PATCH 33/60] feat(import): test yaml_config before saving --- app/apps/import_app/models.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/apps/import_app/models.py b/app/apps/import_app/models.py index aca04e3..b489c43 100644 --- a/app/apps/import_app/models.py +++ b/app/apps/import_app/models.py @@ -1,13 +1,18 @@ +import yaml + +from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ +from apps.import_app.schemas import version_1 + class ImportProfile(models.Model): class Versions(models.IntegerChoices): VERSION_1 = 1, _("Version 1") - name = models.CharField(max_length=100) - yaml_config = models.TextField(help_text=_("YAML configuration")) + name = models.CharField(max_length=100, verbose_name=_("Name")) + yaml_config = models.TextField(verbose_name=_("YAML Configuration")) version = models.IntegerField( choices=Versions, default=Versions.VERSION_1, @@ -20,6 +25,14 @@ class ImportProfile(models.Model): class Meta: ordering = ["name"] + def clean(self): + if self.version and self.version == self.Versions.VERSION_1: + try: + yaml_data = yaml.safe_load(self.yaml_config) + version_1.ImportProfileSchema(**yaml_data) + except Exception as e: + raise ValidationError({"yaml_config": _("Invalid YAML Configuration")}) + class ImportRun(models.Model): class Status(models.TextChoices): From 07fcbe1f458bf08b13cd6529037085f45ccff87f Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:30:59 -0300 Subject: [PATCH 34/60] feat(import): some layouts --- app/templates/import_app/pages/profiles_index.html | 8 ++++++++ app/templates/import_app/pages/runs_index.html | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 app/templates/import_app/pages/profiles_index.html create mode 100644 app/templates/import_app/pages/runs_index.html diff --git a/app/templates/import_app/pages/profiles_index.html b/app/templates/import_app/pages/profiles_index.html new file mode 100644 index 0000000..a5c59ee --- /dev/null +++ b/app/templates/import_app/pages/profiles_index.html @@ -0,0 +1,8 @@ +{% extends "layouts/base.html" %} +{% load i18n %} + +{% block title %}{% translate 'Import Profiles' %}{% endblock %} + +{% block content %} +
+{% endblock %} diff --git a/app/templates/import_app/pages/runs_index.html b/app/templates/import_app/pages/runs_index.html new file mode 100644 index 0000000..38a48a6 --- /dev/null +++ b/app/templates/import_app/pages/runs_index.html @@ -0,0 +1,8 @@ +{% extends "layouts/base.html" %} +{% load i18n %} + +{% block title %}{% translate 'Import Runs' %}{% endblock %} + +{% block content %} +
+{% endblock %} From 6f096fd3ffc9b4ce8cea484fa65433fb6eef9a24 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 14:31:12 -0300 Subject: [PATCH 35/60] feat(import): some views and urls --- app/apps/import_app/tasks.py | 2 +- app/apps/import_app/urls.py | 37 +++++++- app/apps/import_app/views.py | 168 +++++++++++++++++++++++++++++++---- 3 files changed, 186 insertions(+), 21 deletions(-) diff --git a/app/apps/import_app/tasks.py b/app/apps/import_app/tasks.py index 25efcbc..cf6f3a7 100644 --- a/app/apps/import_app/tasks.py +++ b/app/apps/import_app/tasks.py @@ -8,7 +8,7 @@ from apps.import_app.services import ImportServiceV1 logger = logging.getLogger(__name__) -@app.task(queue="imports") +@app.task def process_import(import_run_id: int, file_path: str): try: import_run = ImportRun.objects.get(id=import_run_id) diff --git a/app/apps/import_app/urls.py b/app/apps/import_app/urls.py index aea8670..c2608a3 100644 --- a/app/apps/import_app/urls.py +++ b/app/apps/import_app/urls.py @@ -2,5 +2,40 @@ from django.urls import path import apps.import_app.views as views urlpatterns = [ - path("import/", views.ImportRunCreateView.as_view(), name="import"), + path("import/", views.import_view, name="import"), + path( + "import/profiles/", + views.import_profile_index, + name="import_profiles_index", + ), + path( + "import/profiles/list/", + views.import_profile_list, + name="import_profiles_list", + ), + path( + "import/profiles/add/", + views.import_profile_add, + name="import_profiles_add", + ), + path( + "import/profiles//edit/", + views.import_profile_edit, + name="import_profile_edit", + ), + path( + "import/profiles//runs/", + views.import_run_add, + name="import_profile_runs_index", + ), + path( + "import/profiles//runs/list/", + views.import_run_add, + name="import_profile_runs_list", + ), + path( + "import/profiles//runs/add/", + views.import_run_add, + name="import_run_add", + ), ] diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index d5b1d94..ce65a2b 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -1,26 +1,156 @@ -from django.views.generic import CreateView -from apps.import_app.models import ImportRun -from apps.import_app.services import ImportServiceV1 +import shutil + +from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.files.storage import FileSystemStorage +from django.http import HttpResponse +from django.shortcuts import render, get_object_or_404 +from django.views.decorators.http import require_http_methods +from django.utils.translation import gettext_lazy as _ + +from apps.common.decorators.htmx import only_htmx +from apps.import_app.forms import ImportRunFileUploadForm, ImportProfileForm +from apps.import_app.models import ImportRun, ImportProfile +from apps.import_app.tasks import process_import -class ImportRunCreateView(CreateView): - model = ImportRun - fields = ["profile"] +def import_view(request): + import_profile = ImportProfile.objects.get(id=2) + shutil.copyfile( + "/usr/src/app/apps/import_app/teste2.csv", "/usr/src/app/temp/teste2.csv" + ) + ir = ImportRun.objects.create(profile=import_profile, file_name="teste.csv") + process_import.defer( + import_run_id=ir.id, + file_path="/usr/src/app/temp/teste2.csv", + ) + return HttpResponse("Hello, world. You're at the polls page.") - def form_valid(self, form): - response = super().form_valid(form) - import_run = form.instance - file = self.request.FILES["file"] +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_index(request): + return render( + request, + "import_app/pages/profiles_index.html", + ) - # Save uploaded file temporarily - temp_file_path = f"/tmp/import_{import_run.id}.csv" - with open(temp_file_path, "wb+") as destination: - for chunk in file.chunks(): - destination.write(chunk) - # Process the import - import_service = ImportServiceV1(import_run) - import_service.process_file(temp_file_path) +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_list(request): + profiles = ImportProfile.objects.all() - return response + return render( + request, + "import_app/fragments/profiles/list.html", + {"profiles": profiles}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_add(request): + if request.method == "POST": + form = ImportProfileForm(request.POST) + + if form.is_valid(): + form.save() + messages.success(request, _("Import Profile added successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated, hide_offcanvas", + }, + ) + else: + form = ImportProfileForm() + + return render( + request, + "import_app/fragments/profiles/add.html", + {"form": form}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_profile_edit(request, profile_id): + profile = get_object_or_404(ImportProfile, id=profile_id) + + if request.method == "POST": + form = ImportProfileForm(request.POST, instance=profile) + + if form.is_valid(): + form.save() + messages.success(request, _("Import Profile update successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated, hide_offcanvas", + }, + ) + else: + form = ImportProfileForm(instance=profile) + + return render( + request, + "import_app/fragments/profiles/edit.html", + {"form": form, "profile": profile}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_run_list(request, profile_id): + profile = ImportProfile.objects.get(id=profile_id) + + runs = ImportRun.objects.filter(profile=profile).order_by("id") + + return render( + request, + "import_app/fragments/runs/list.html", + {"profile": profile, "runs": runs}, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_run_add(request, profile_id): + profile = ImportProfile.objects.get(id=profile_id) + + if request.method == "POST": + form = ImportRunFileUploadForm(request.POST, request.FILES) + + if form.is_valid(): + uploaded_file = request.FILES["file"] + fs = FileSystemStorage(location="/usr/src/app/temp") + filename = fs.save(uploaded_file.name, uploaded_file) + file_path = fs.path(filename) + + import_run = ImportRun.objects.create(profile=profile, file_name=filename) + + # Defer the procrastinate task + process_import.defer(import_run_id=import_run.id, file_path=file_path) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated, hide_offcanvas", + }, + ) + else: + form = ImportRunFileUploadForm() + + return render( + request, + "import_app/fragments/runs/add.html", + {"form": form, "profile": profile}, + ) From 00b8727664bfe3ee7a27c7167a100fa0618d1058 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 23:09:49 -0300 Subject: [PATCH 36/60] feat(transactions): add internal_id field to transactions --- app/apps/import_app/schemas/v1.py | 15 +++++++++++--- .../0033_transaction_internal_id.py | 20 +++++++++++++++++++ app/apps/transactions/models.py | 3 +++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 app/apps/transactions/migrations/0033_transaction_internal_id.py diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 74e37a1..22df7c2 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -167,13 +167,20 @@ class TransactionCategoryMapping(ColumnMapping): coerce_to: Literal["str|int"] = Field("str|int", frozen=True) -class TransactionInternalMapping(ColumnMapping): +class TransactionInternalNoteMapping(ColumnMapping): target: Literal["internal_note"] = Field( ..., description="Transaction field to map to" ) coerce_to: Literal["str"] = Field("str", frozen=True) +class TransactionInternalIDMapping(ColumnMapping): + target: Literal["internal_id"] = Field( + ..., description="Transaction field to map to" + ) + coerce_to: Literal["str"] = Field("str", frozen=True) + + class CategoryNameMapping(ColumnMapping): target: Literal["category_name"] = Field( ..., description="Category field to map to" @@ -314,7 +321,8 @@ class ImportProfileSchema(BaseModel): | TransactionTagsMapping | TransactionEntitiesMapping | TransactionCategoryMapping - | TransactionInternalMapping + | TransactionInternalNoteMapping + | TransactionInternalIDMapping | CategoryNameMapping | CategoryMuteMapping | CategoryActiveMapping @@ -358,7 +366,8 @@ class ImportProfileSchema(BaseModel): TransactionTagsMapping, TransactionEntitiesMapping, TransactionCategoryMapping, - TransactionInternalMapping, + TransactionInternalNoteMapping, + TransactionInternalIDMapping, ), "accounts": ( AccountNameMapping, diff --git a/app/apps/transactions/migrations/0033_transaction_internal_id.py b/app/apps/transactions/migrations/0033_transaction_internal_id.py new file mode 100644 index 0000000..b7d578c --- /dev/null +++ b/app/apps/transactions/migrations/0033_transaction_internal_id.py @@ -0,0 +1,20 @@ +# Generated by Django 5.1.5 on 2025-01-21 01:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("transactions", "0032_transaction_created_at_transaction_updated_at"), + ] + + operations = [ + migrations.AddField( + model_name="transaction", + name="internal_id", + field=models.TextField( + blank=True, null=True, unique=True, verbose_name="Internal ID" + ), + ), + ] diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index 2bd2a68..85ff53a 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -190,6 +190,9 @@ class Transaction(models.Model): verbose_name=_("Recurring Transaction"), ) internal_note = models.TextField(blank=True, verbose_name=_("Internal Note")) + internal_id = models.TextField( + blank=True, null=True, unique=True, verbose_name=_("Internal ID") + ) deleted = models.BooleanField( default=False, verbose_name=_("Deleted"), db_index=True From a415e285ee261264282a9d93c175e4172def92a2 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Mon, 20 Jan 2025 23:10:11 -0300 Subject: [PATCH 37/60] feat(transactions): make deleted_at readonly on admin --- app/apps/transactions/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/apps/transactions/admin.py b/app/apps/transactions/admin.py index df4d1c8..8f37317 100644 --- a/app/apps/transactions/admin.py +++ b/app/apps/transactions/admin.py @@ -19,15 +19,16 @@ class TransactionModelAdmin(admin.ModelAdmin): list_filter = ["deleted", "type", "is_paid", "date", "account"] list_display = [ - "deleted", + "date", "description", "type", "account__name", "amount", "account__currency__code", - "date", "reference_date", + "deleted", ] + readonly_fields = ["deleted_at"] actions = ["hard_delete_selected"] From ece44f27265d2beb7bc186ba3563c143fff2937f Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Wed, 22 Jan 2025 01:41:17 -0300 Subject: [PATCH 38/60] feat(import): more UI and endpoints --- app/apps/import_app/forms.py | 9 +- app/apps/import_app/services/v1.py | 18 ++- app/apps/import_app/urls.py | 22 +++- app/apps/import_app/views.py | 60 +++++++++- .../import_app/fragments/profiles/list.html | 38 ++++-- .../import_app/fragments/runs/add.html | 4 +- .../import_app/fragments/runs/list.html | 111 ++++++++++++++++++ 7 files changed, 228 insertions(+), 34 deletions(-) diff --git a/app/apps/import_app/forms.py b/app/apps/import_app/forms.py index 78ee3d7..f300721 100644 --- a/app/apps/import_app/forms.py +++ b/app/apps/import_app/forms.py @@ -55,4 +55,11 @@ class ImportRunFileUploadForm(forms.Form): self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" - self.helper.layout = Layout("file") + self.helper.layout = Layout( + "file", + FormActions( + NoClassSubmit( + "submit", _("Import"), css_class="btn btn-outline-primary w-100" + ), + ), + ) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 0416caf..abda751 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -491,7 +491,6 @@ class ImportService: for row_number, row in enumerate(reader, start=1): self._process_row(row, row_number) - self._increment_totals("processed", value=1) def _validate_file_path(self, file_path: str) -> str: """ @@ -518,15 +517,14 @@ class ImportService: if self.settings.file_type == "csv": self._process_csv(file_path) - if self.import_run.processed_rows == self.import_run.total_rows: - self._update_status("FINISHED") - self._log( - "info", - f"Import completed successfully. " - f"Successful: {self.import_run.successful_rows}, " - f"Failed: {self.import_run.failed_rows}, " - f"Skipped: {self.import_run.skipped_rows}", - ) + self._update_status("FINISHED") + self._log( + "info", + f"Import completed successfully. " + f"Successful: {self.import_run.successful_rows}, " + f"Failed: {self.import_run.failed_rows}, " + f"Skipped: {self.import_run.skipped_rows}", + ) except Exception as e: self._update_status("FAILED") diff --git a/app/apps/import_app/urls.py b/app/apps/import_app/urls.py index c2608a3..beb65ba 100644 --- a/app/apps/import_app/urls.py +++ b/app/apps/import_app/urls.py @@ -13,6 +13,11 @@ urlpatterns = [ views.import_profile_list, name="import_profiles_list", ), + path( + "import/profiles//delete/", + views.import_profile_delete, + name="import_profile_delete", + ), path( "import/profiles/add/", views.import_profile_add, @@ -24,14 +29,19 @@ urlpatterns = [ name="import_profile_edit", ), path( - "import/profiles//runs/", - views.import_run_add, - name="import_profile_runs_index", + "import/profiles//runs/list/", + views.import_runs_list, + name="import_profile_runs_list", ), path( - "import/profiles//runs/list/", - views.import_run_add, - name="import_profile_runs_list", + "import/profiles//runs//log/", + views.import_run_log, + name="import_run_log", + ), + path( + "import/profiles//runs//delete/", + views.import_run_delete, + name="import_run_delete", ), path( "import/profiles//runs/add/", diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index ce65a2b..6b869fd 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -5,6 +5,7 @@ from django.contrib.auth.decorators import login_required from django.core.files.storage import FileSystemStorage from django.http import HttpResponse from django.shortcuts import render, get_object_or_404 +from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods from django.utils.translation import gettext_lazy as _ @@ -107,11 +108,30 @@ def import_profile_edit(request, profile_id): @only_htmx @login_required -@require_http_methods(["GET", "POST"]) -def import_run_list(request, profile_id): +@csrf_exempt +@require_http_methods(["DELETE"]) +def import_profile_delete(request, profile_id): profile = ImportProfile.objects.get(id=profile_id) - runs = ImportRun.objects.filter(profile=profile).order_by("id") + profile.delete() + + messages.success(request, _("Import Profile deleted successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated", + }, + ) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_runs_list(request, profile_id): + profile = ImportProfile.objects.get(id=profile_id) + + runs = ImportRun.objects.filter(profile=profile).order_by("-id") return render( request, @@ -120,6 +140,19 @@ def import_run_list(request, profile_id): ) +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_run_log(request, profile_id, run_id): + run = ImportRun.objects.get(profile__id=profile_id, id=run_id) + + return render( + request, + "import_app/fragments/runs/log.html", + {"run": run}, + ) + + @only_htmx @login_required @require_http_methods(["GET", "POST"]) @@ -140,6 +173,8 @@ def import_run_add(request, profile_id): # Defer the procrastinate task process_import.defer(import_run_id=import_run.id, file_path=file_path) + messages.success(request, _("Import Run queued successfully")) + return HttpResponse( status=204, headers={ @@ -154,3 +189,22 @@ def import_run_add(request, profile_id): "import_app/fragments/runs/add.html", {"form": form, "profile": profile}, ) + + +@only_htmx +@login_required +@csrf_exempt +@require_http_methods(["DELETE"]) +def import_run_delete(request, profile_id, run_id): + run = ImportRun.objects.get(profile__id=profile_id, id=run_id) + + run.delete() + + messages.success(request, _("Run deleted successfully")) + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "updated", + }, + ) diff --git a/app/templates/import_app/fragments/profiles/list.html b/app/templates/import_app/fragments/profiles/list.html index f1f34d2..2872897 100644 --- a/app/templates/import_app/fragments/profiles/list.html +++ b/app/templates/import_app/fragments/profiles/list.html @@ -38,18 +38,32 @@ hx-get="{% url 'import_profile_edit' profile_id=profile.id %}" hx-target="#generic-offcanvas"> -{# #} -{# #} + + + + + + {{ profile.name }} {{ profile.get_version_display }} diff --git a/app/templates/import_app/fragments/runs/add.html b/app/templates/import_app/fragments/runs/add.html index d5a5b89..9997044 100644 --- a/app/templates/import_app/fragments/runs/add.html +++ b/app/templates/import_app/fragments/runs/add.html @@ -2,10 +2,10 @@ {% load i18n %} {% load crispy_forms_tags %} -{% block title %}{% translate 'Import file' %}{% endblock %} +{% block title %}{% translate 'Import file with profile' %} {{ profile.name }}{% endblock %} {% block body %} -
+ {% crispy form %}
{% endblock %} diff --git a/app/templates/import_app/fragments/runs/list.html b/app/templates/import_app/fragments/runs/list.html index 0697d26..f67054c 100644 --- a/app/templates/import_app/fragments/runs/list.html +++ b/app/templates/import_app/fragments/runs/list.html @@ -5,5 +5,116 @@ {% block title %}{% translate 'Runs for ' %}{{ profile.name }}{% endblock %} {% block body %} +
+ {% if runs %} +
+ {% for run in runs %} +
+
+
+ {{ run.get_status_display }} +
+
+
{{ run.id }}({{ run.file_name }})
+
+
+
+
+
+
+ {% trans 'Total Items' %} +
+
+ {{ run.total_rows }} +
+
+
+
+
+
+
+
+ {% trans 'Processed Items' %} +
+
+ {{ run.processed_rows }} +
+
+
+
+ +
+
+
+
+ {% trans 'Skipped Items' %} +
+
+ {{ run.skipped_rows }} +
+
+
+
+ +
+
+
+
+ {% trans 'Failed Items' %} +
+
+ {{ run.failed_rows }} +
+
+
+
+ +
+
+
+
+ {% trans 'Successful Items' %} +
+
+ {{ run.successful_rows }} +
+
+
+
+ +
+
+
+ + + +
+
+
+ {% endfor %} + {% else %} + + {% endif %} +
+
{% endblock %} From cabd03e7e65b9640bcf5f0c1f3d64e05794e234a Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 11:43:35 -0300 Subject: [PATCH 39/60] feat: presets --- app/apps/common/templatetags/json.py | 11 +++++ .../0002_alter_importprofile_name_and_more.py | 23 ++++++++++ app/apps/import_app/models.py | 4 +- app/apps/import_app/services/__init__.py | 2 + app/apps/import_app/services/presets.py | 45 +++++++++++++++++++ app/apps/import_app/urls.py | 5 +++ app/apps/import_app/views.py | 26 ++++++++++- ...alter_usersettings_date_format_and_more.py | 28 ++++++++++++ .../import_app/fragments/profiles/add.html | 10 ++++- .../import_app/fragments/profiles/list.html | 25 ++++++++--- .../fragments/profiles/list_presets.html | 43 ++++++++++++++++++ .../import_app/fragments/runs/log.html | 13 ++++++ 12 files changed, 223 insertions(+), 12 deletions(-) create mode 100644 app/apps/common/templatetags/json.py create mode 100644 app/apps/import_app/migrations/0002_alter_importprofile_name_and_more.py create mode 100644 app/apps/import_app/services/presets.py create mode 100644 app/apps/users/migrations/0014_alter_usersettings_date_format_and_more.py create mode 100644 app/templates/import_app/fragments/profiles/list_presets.html create mode 100644 app/templates/import_app/fragments/runs/log.html diff --git a/app/apps/common/templatetags/json.py b/app/apps/common/templatetags/json.py new file mode 100644 index 0000000..8fb45e2 --- /dev/null +++ b/app/apps/common/templatetags/json.py @@ -0,0 +1,11 @@ +import json + +from django import template + + +register = template.Library() + + +@register.filter("json") +def convert_to_json(value): + return json.dumps(value) diff --git a/app/apps/import_app/migrations/0002_alter_importprofile_name_and_more.py b/app/apps/import_app/migrations/0002_alter_importprofile_name_and_more.py new file mode 100644 index 0000000..efa1ee3 --- /dev/null +++ b/app/apps/import_app/migrations/0002_alter_importprofile_name_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.5 on 2025-01-23 03:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('import_app', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='importprofile', + name='name', + field=models.CharField(max_length=100, unique=True, verbose_name='Name'), + ), + migrations.AlterField( + model_name='importprofile', + name='yaml_config', + field=models.TextField(verbose_name='YAML Configuration'), + ), + ] diff --git a/app/apps/import_app/models.py b/app/apps/import_app/models.py index b489c43..c0224d8 100644 --- a/app/apps/import_app/models.py +++ b/app/apps/import_app/models.py @@ -9,9 +9,9 @@ from apps.import_app.schemas import version_1 class ImportProfile(models.Model): class Versions(models.IntegerChoices): - VERSION_1 = 1, _("Version 1") + VERSION_1 = 1, _("Version") + " 1" - name = models.CharField(max_length=100, verbose_name=_("Name")) + name = models.CharField(max_length=100, verbose_name=_("Name"), unique=True) yaml_config = models.TextField(verbose_name=_("YAML Configuration")) version = models.IntegerField( choices=Versions, diff --git a/app/apps/import_app/services/__init__.py b/app/apps/import_app/services/__init__.py index 6001902..88aa4e8 100644 --- a/app/apps/import_app/services/__init__.py +++ b/app/apps/import_app/services/__init__.py @@ -1 +1,3 @@ from apps.import_app.services.v1 import ImportService as ImportServiceV1 + +from apps.import_app.services.presets import PresetService diff --git a/app/apps/import_app/services/presets.py b/app/apps/import_app/services/presets.py new file mode 100644 index 0000000..15e7ac1 --- /dev/null +++ b/app/apps/import_app/services/presets.py @@ -0,0 +1,45 @@ +import json +from pathlib import Path + +from apps.import_app.models import ImportProfile + + +class PresetService: + PRESET_PATH = "/usr/src/app/import_presets" + + @classmethod + def get_all_presets(cls): + presets = [] + + for folder in Path(cls.PRESET_PATH).iterdir(): + if folder.is_dir(): + manifest_path = folder / "manifest.json" + config_path = folder / "config.yml" + + if manifest_path.exists() and config_path.exists(): + with open(manifest_path) as f: + manifest = json.load(f) + + with open(config_path) as f: + config = json.dumps(f.read()) + + try: + preset = { + "name": manifest.get("name", folder.name), + "description": manifest.get("description", ""), + "message": json.dumps(manifest.get("message", "")), + "authors": manifest.get("author", "").split(","), + "schema_version": (int(manifest.get("schema_version", 1))), + "folder_name": folder.name, + "config": config, + } + + ImportProfile.Versions( + preset["schema_version"] + ) # Check if schema version is valid + except Exception as e: + print(e) + else: + presets.append(preset) + + return presets diff --git a/app/apps/import_app/urls.py b/app/apps/import_app/urls.py index beb65ba..eae9851 100644 --- a/app/apps/import_app/urls.py +++ b/app/apps/import_app/urls.py @@ -3,6 +3,11 @@ import apps.import_app.views as views urlpatterns = [ path("import/", views.import_view, name="import"), + path( + "import/presets/", + views.import_presets_list, + name="import_presets_list", + ), path( "import/profiles/", views.import_profile_index, diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index 6b869fd..720a5e1 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -13,6 +13,7 @@ from apps.common.decorators.htmx import only_htmx from apps.import_app.forms import ImportRunFileUploadForm, ImportProfileForm from apps.import_app.models import ImportRun, ImportProfile from apps.import_app.tasks import process_import +from apps.import_app.services import PresetService def import_view(request): @@ -28,6 +29,18 @@ def import_view(request): return HttpResponse("Hello, world. You're at the polls page.") +@login_required +@require_http_methods(["GET"]) +def import_presets_list(request): + presets = PresetService.get_all_presets() + print(presets) + return render( + request, + "import_app/fragments/profiles/list_presets.html", + {"presets": presets}, + ) + + @login_required @require_http_methods(["GET", "POST"]) def import_profile_index(request): @@ -54,6 +67,8 @@ def import_profile_list(request): @login_required @require_http_methods(["GET", "POST"]) def import_profile_add(request): + message = request.GET.get("message", None) or request.POST.get("message", None) + if request.method == "POST": form = ImportProfileForm(request.POST) @@ -68,12 +83,19 @@ def import_profile_add(request): }, ) else: - form = ImportProfileForm() + print(int(request.GET.get("version", 1))) + form = ImportProfileForm( + initial={ + "name": request.GET.get("name"), + "version": int(request.GET.get("version", 1)), + "yaml_config": request.GET.get("yaml_config"), + } + ) return render( request, "import_app/fragments/profiles/add.html", - {"form": form}, + {"form": form, "message": message}, ) diff --git a/app/apps/users/migrations/0014_alter_usersettings_date_format_and_more.py b/app/apps/users/migrations/0014_alter_usersettings_date_format_and_more.py new file mode 100644 index 0000000..e38b096 --- /dev/null +++ b/app/apps/users/migrations/0014_alter_usersettings_date_format_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.1.5 on 2025-01-23 03:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0013_usersettings_date_format_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='usersettings', + name='date_format', + field=models.CharField(default='SHORT_DATE_FORMAT', max_length=100, verbose_name='Date Format'), + ), + migrations.AlterField( + model_name='usersettings', + name='datetime_format', + field=models.CharField(default='SHORT_DATETIME_FORMAT', max_length=100, verbose_name='Datetime Format'), + ), + migrations.AlterField( + model_name='usersettings', + name='language', + field=models.CharField(choices=[('auto', 'Auto'), ('en', 'English'), ('nl', 'Nederlands'), ('pt-br', 'Português (Brasil)')], default='auto', max_length=10, verbose_name='Language'), + ), + ] diff --git a/app/templates/import_app/fragments/profiles/add.html b/app/templates/import_app/fragments/profiles/add.html index beda873..03eb9a5 100644 --- a/app/templates/import_app/fragments/profiles/add.html +++ b/app/templates/import_app/fragments/profiles/add.html @@ -1,11 +1,19 @@ {% extends 'extends/offcanvas.html' %} +{% load json %} {% load i18n %} {% load crispy_forms_tags %} {% block title %}{% translate 'Add new import profile' %}{% endblock %} {% block body %} -
+{% if message %} + +{% endif %} + {% crispy form %}
{% endblock %} diff --git a/app/templates/import_app/fragments/profiles/list.html b/app/templates/import_app/fragments/profiles/list.html index 2872897..cdc9a83 100644 --- a/app/templates/import_app/fragments/profiles/list.html +++ b/app/templates/import_app/fragments/profiles/list.html @@ -3,13 +3,24 @@
{% spaceless %}
{% translate 'Import Profiles' %} - - + + + +
{% endspaceless %}
diff --git a/app/templates/import_app/fragments/profiles/list_presets.html b/app/templates/import_app/fragments/profiles/list_presets.html new file mode 100644 index 0000000..0b64342 --- /dev/null +++ b/app/templates/import_app/fragments/profiles/list_presets.html @@ -0,0 +1,43 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Import Presets' %}{% endblock %} + +{% block body %} + {% if presets %} + + +{% endblock %} diff --git a/app/templates/import_app/fragments/runs/log.html b/app/templates/import_app/fragments/runs/log.html new file mode 100644 index 0000000..a7445a4 --- /dev/null +++ b/app/templates/import_app/fragments/runs/log.html @@ -0,0 +1,13 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Logs for' %} #{{ run.id }}{% endblock %} + +{% block body %} +
+
+ {{ run.logs|linebreaks }} +
+
+{% endblock %} From bcc96588bf77f53f4221368c10935b8e83ee09a7 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 12:49:50 -0300 Subject: [PATCH 40/60] feat: PWA support --- app/WYGIWYH/settings.py | 41 ++++++++++++++++++++++ app/WYGIWYH/urls.py | 1 + app/static/img/favicon/manifest.json | 41 ---------------------- app/static/img/pwa/splash-640x1136.png | Bin 0 -> 19916 bytes app/static/img/pwa/splash-750x1334.png | Bin 0 -> 23050 bytes app/templates/includes/head/favicons.html | 3 +- app/templates/layouts/base.html | 2 ++ app/templates/layouts/base_auth.html | 4 ++- requirements.txt | 2 +- 9 files changed, 49 insertions(+), 45 deletions(-) delete mode 100644 app/static/img/favicon/manifest.json create mode 100644 app/static/img/pwa/splash-640x1136.png create mode 100644 app/static/img/pwa/splash-750x1334.png diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index b2eeba1..97b2f9c 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -72,6 +72,7 @@ INSTALLED_APPS = [ "apps.rules.apps.RulesConfig", "apps.calendar_view.apps.CalendarViewConfig", "apps.dca.apps.DcaConfig", + "pwa", ] MIDDLEWARE = [ @@ -335,3 +336,43 @@ else: } CACHALOT_UNCACHABLE_TABLES = ("django_migrations", "procrastinate_jobs") + + +# PWA +PWA_APP_NAME = SITE_TITLE +PWA_APP_DESCRIPTION = "A simple and powerful finance tracker" +PWA_APP_THEME_COLOR = "#fbb700" +PWA_APP_BACKGROUND_COLOR = "#222222" +PWA_APP_DISPLAY = "standalone" +PWA_APP_SCOPE = "/" +PWA_APP_ORIENTATION = "any" +PWA_APP_START_URL = "/" +PWA_APP_STATUS_BAR_COLOR = "default" +PWA_APP_ICONS = [ + {"src": "/static/img/favicon/android-icon-192x192.png", "sizes": "192x192"} +] +PWA_APP_ICONS_APPLE = [ + {"src": "/static/img/favicon/apple-icon-180x180.png", "sizes": "180x180"} +] +PWA_APP_SPLASH_SCREEN = [ + { + "src": "/static/img/pwa/splash-640x1136.png", + "media": "(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2)", + } +] +PWA_APP_DIR = "ltr" +PWA_APP_LANG = "en-US" +PWA_APP_SHORTCUTS = [] +PWA_APP_SCREENSHOTS = [ + { + "src": "/static/img/pwa/splash-750x1334.png", + "sizes": "750x1334", + "type": "image/png", + "form_factor": "wide", + }, + { + "src": "/static/img/pwa/splash-750x1334.png", + "sizes": "750x1334", + "type": "image/png", + }, +] diff --git a/app/WYGIWYH/urls.py b/app/WYGIWYH/urls.py index 5a465a5..7aed790 100644 --- a/app/WYGIWYH/urls.py +++ b/app/WYGIWYH/urls.py @@ -27,6 +27,7 @@ urlpatterns = [ path("hijack/", include("hijack.urls")), path("__debug__/", include("debug_toolbar.urls")), path("__reload__/", include("django_browser_reload.urls")), + path("", include("pwa.urls")), # path("api/", include("rest_framework.urls")), path("api/", include("apps.api.urls")), path("api/schema/", SpectacularAPIView.as_view(), name="schema"), diff --git a/app/static/img/favicon/manifest.json b/app/static/img/favicon/manifest.json deleted file mode 100644 index 9d5c19d..0000000 --- a/app/static/img/favicon/manifest.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "WYGIWYH", - "icons": [ - { - "src": "\/static\/img\/favicon\/android-icon-36x36.png", - "sizes": "36x36", - "type": "image\/png", - "density": "0.75" - }, - { - "src": "\/static\/img\/favicon\/android-icon-48x48.png", - "sizes": "48x48", - "type": "image\/png", - "density": "1.0" - }, - { - "src": "\/static\/img\/favicon\/android-icon-72x72.png", - "sizes": "72x72", - "type": "image\/png", - "density": "1.5" - }, - { - "src": "\/static\/img\/favicon\/android-icon-96x96.png", - "sizes": "96x96", - "type": "image\/png", - "density": "2.0" - }, - { - "src": "\/static\/img\/favicon\/android-icon-144x144.png", - "sizes": "144x144", - "type": "image\/png", - "density": "3.0" - }, - { - "src": "\/static\/img\/favicon\/android-icon-192x192.png", - "sizes": "192x192", - "type": "image\/png", - "density": "4.0" - } - ] -} diff --git a/app/static/img/pwa/splash-640x1136.png b/app/static/img/pwa/splash-640x1136.png new file mode 100644 index 0000000000000000000000000000000000000000..8f5db6b432af55ff914fc609cad28557e443c658 GIT binary patch literal 19916 zcmeIac{r5s`!{~i7)*#66qRLAw2&oa8%DP5*|)Oqd-f%a6honsY*|Vud$RAMBo!gM zY-OkH`*^NV@6Y%9eU9gk-|>6?dOpYJ4-WTzF4uX!&ewTf*L{y;BClyE9i=)+1wqhJ z<*V}A5Cq3V5W*cv0qz9E{sh0FBkos?JRyj-h4cf9zd(BmXkic zcC)m(=;z{2Qir5u{M;?9oNT->mNs?{u2`O5H4Qu%2Wu>kfrz?*y1Sf>y~EW24;!5T z4PC1MCo6Gl9vNvWDL)C|z{SSP0^{f6?CL4uhvhl+D*^r{-R9>3N*>m>658^Le^tO0 zmdD=9%Uyz>-`CgoqVJ`PZXS00g5u)h`~pJ!LPC5%gU{37)yu+<&()I)c%Z@@`jEHr zwDNFp_i}J^#gKejSh{(8VR?9{Fn?XkdU<&`SpLhqtLH_F!#mz2lZzl}ehX_)e!+_Z zq_8j;sXzJ>veurY%Rh_g*xWq4_;cm#aJZ7Yg@>n&uD`ntmPgyh)6Lt%%I44m$>^^W zIS(5PFB@xVAps!~J^?{Kp-WQy|C@-k#(x5Q!-o<4KP2|omlXehTju{FE%*OMZW8?tdyGUpAOJaC`kNEDl92WG zvUl^4zG~xPW0zvaP#uAkRqx6BLu`hg#Z5+#Go_(h4KH0#m~b1zcm3+lQ<-= zn}@EOo3pf>g{zMR=B&1jtF?`XjmJ5RwVM@IURKHa3ZIUUh?p|Q+tbDat0#NqlA{iv zmZz4w6#xJ3;{O=-pG?3U026`#UvmRo{A==n6|p=XV6Mb&nBgGkYKOACtgheFgyjsD19&C|4z(!K z_}`k$e@i?P=>GWUv({y^izy4XmksEzG`drVx}$Yo;vgl2e2ek<(jFh2LX1Zwq3oqs zHrB;EJZ&(5alO?3m7dtK#do>y9KPLj%+m=~`5M#3aMAS9Bj=^Q&#pdk3)^H_SS7Nm zg(-jE(O-7lmwrcFYLGd-s{TDFEdfG6Q1J;k3W6GKkvPD$6b>*uAm{KyL6UVh8H6!@ zMBl7(^#Q%i4(T`jTe|~Z4-1>#Bv)v@am7?$v?>W9QgMl>DpMC+4 zCugOtWeN;ve>M#a`mAcw>+jTTk4cF7DFiGI3=MmEVR(&LK~~Sd`CI(ZQ_Z(6 zkqiZ^H;&`3pGY~?_2Tw}D5~2h)0w*ZYh8~`h3h^TaS2Gy(iIicH45RTka_7jW33*g z(JS`DW!%mS`YH>-nM-c^a~wN%N7zC4+B&&X}5eb4FgB z$^wb81hA3`t>4bbKIY>PV3Cj=eKglvjp^CtPlJ1r68cHGzIobS`RDb1>1NM`NwA$E zn~{N|aOv4hJ`Ph`7uDH^vR*MEM68?>W|e2cWD+y1^j-y2oY!m3)n41!)u=O2WrS~e zAu#wiD(?{@z9nX>+`H=o6L_KP!(sI8bH;IAGnJ}tx|LqPD^+t&`Fqif(ZEr~ua(JV zKc}*E4z~q==A%8^K4KL>|NLe3IU}I?MbYAh)1z5shz3yM)VWVb9lrm-PpL0EeP?X& z=sqGXxm*0@nM}yqP*d13jj}D|mBx!m946+c?V5dXTar?>jn3Z5)aLaIGeZYYHNG}c zCD7_udd=L?Nxf0a$4D?hU{H3p>NNIk3aQo3;hGAWSF)!u;jBTUuMCJ!vY8I%^v^Kq zlij=vAtoC4=o_BI@05CFYx*Zi^(y>eEzQ=ShW#X#$R6`I;*@!~parcdHS>#;g zUw3G`D{WIPUT|+*XJD*TRjH$8Y>53uavGUA3IrJbiDfkFo6TaiqfbOv)pPRW*G3-a zleq<~Ej;r|U0F$FCLBXx%x@+wnclcmal?$)$zXXVV^$oOeqJMIX1kdUdywE&mS!UoH~4g8qDTc8bc2(Fn**izSoJ)fG{)!5(6J0bBqyLFAR zaRZ5Kd~jr&Bk1QN(SoPlO&9vjFC2AI4%k_z!EPIepB&uE$C$%mNSZewt(}#(qBn%= z+kssrQ;gr*s*LT-hjTfz=kNt6e51gJ&of`E6$(H=H^kY3hbZ?#e(7g9{$dv*rQfXs znP8yMv&lCYJAqQQ$8jG2P@?_Z)h?YPyP4^bAIkzLbx=3*ys0v$#pj~x{67Tw1q}IF zmhFLu_1@iD7X(2Vjv{2tA1|Amvf-6nA>U<3e(1(0{x&A*Q`Xkgc_#L%XXQJpZ)K;q1yUem$s zXaixV)8mwe?;q-DjO?bMS&qXf@DO5WeDJQzU}>!a`9N(tQ*mzJ!BW?L$Mj69$`zCt zI~*kz0l+!mP-0}IhziE#`>m%6FG@Tpu+qDO>12YVwiqu=d%e7+@aFx`yP%W;SNR(j zd&wF;_qHW70$AEPpwJLOCrg}fNSQ3?bZ|@l1OLI(!rioXEl*0f<1lGO2>wXX#@nfN zPSJFLqUDun$a6NS2L(Yzw#jL5Lr|Wf3ua&QQ!ctayl^*!3RfFg)cefsSI6WWb3Ypp z8tcz>$LK|~{Dz!o-etNj?tMf0n>+$+xQ~G>BqtTa*G8O9>59H{3!s9UefC97WOjne zXHPU1b3qmtObVTQ%R59;(qN5OG%j<`k8V!St~abYiuY5(kS1qSJF`-q=5>XmoNhq3 zstt%!z1RlkXUVo~)LcX~p5WwaSF}`{t=xCf`4W~KM}xdWzxLtR8$aIrl}Zq&D_{E0 z?#m7R8B*VesLk=B6YIUcv+x2Bd`$EZ`-Il^?G!(- zs|jGqnT6meLAT~E{gIt2ZgJMVA{tC}?n##F$w(>IpxDcim~s%h5f~fQZb1dvrYK=7 zIsWs4zqqQ5pJ?kEjd>>$*YaXr)ws+`FLU+;Pu(r7GAFxymn$8>t6z&fwz!nFuwbunhywkoR-t(t+6+LhW!7xZ( zt>4G{O*!v0Ydw^oSJL#1uV&TzIrqCY#<5x?&d}J@HkPfvEwAgyjqDr<*Klid_BwVu zho<`y-jfruFuG^!HAmUkcaAGsH5JzKBw^HNzOc@Jh1@R;nI8$=IWuxoJ&x$&a1X>S zU=KE~-rn>d^M-o4v*#LlQ264L{)&4)IG4-{6Jq0xz9xBYevd~-l`i*UFm+IR}_0U9Lgg5QTO582hqN&FMp8$(= zpVTPPjb6{Y$Fux_36CA#Zn=6fCj6%Iz`($$AQrwMe3Ab}PKANfN9ON8sT-xiply4| z(hKJL?73pWFFDNw@pmxno5@nDk{Uo;yhX)q)N!nrPJuS7)Sa*s2u%3MT#3(OO@T2R zPXrd~?<>(x-xxJ5aHcD&X!*?r3FiCcJNA4jThF$G@MqM(erLBNQJ_qIi;<%F9@J9o zo^@w^=6F#rzk>*t9OAX{(x?dbS1Oy**hoo(pW=Yf-c_^MfRzo$8xnQhYm89wi_#v` z#S#5X{?BfgvE-sn=gj55Z9U|`memnP^At&bM2 zp?bA+<&D@ZHMC!5lT@X@M6LPY}#Jqp-eGZr3&MNy-pC z>I?$Ixiq<306}4wbHp-8I5BE!p~%7Qx!t~R^G@k*LRd0r;>u0gYYp}DrBPMf6lg_= z(~_;eO5ZH8{fH<;crkpRjTBuLL$#WwEDJ~p9qYU?w{q_mfCP7aMJ#Z=rQ9)IeY_AS zPXQxv1X{l_xM%C$$_~lLb)$PHD|+qQ)PnopccSskk6O!nQ6Z4P{ovBq`UYmh76BBN zEHHv>ptDo-@yEZ#;86Gwo!iUqIU#A!=q7-^xrUPAt}7Q2YJ34^h4;Qqtr=@rbKaN$ z=2cAb#^XMFjhl@mRO?i;w~sAobb{&It%I@xpzZK|L-%H;)U~ifh9CM2L>-#7gBb{+ z9MD&{Sl10lHUktude~n*6HISg5C#%P2p-VU*n{EYgPhdK(1c(v=%#7l$GU3klLmDV z{u{U(*2_EQCGaH{Y7yV}<65SKUjIld>UH|bvUH@S3XPxXCOhR==EqGEOnsl9_xk3{ zi3;L^mS5FZ6q*`*nedi_PG9&YCkJHC z)NZ$}TYtMb*hdK(><@8S%e80ZX@(kIUF&j<+N_}X`ebzN$sUqKAHcNRyV_(5;SNYg-kOFk^8kwH3nAcH1U&tYnB2di zccO8*<3Uonn(C~L@nDyBu0lo^)lmgriYyo7jGx(z2)>6YxN9igT7f7n4EK0Z0g)D*fYOoFNHm#3b+#(@XHnybRwY*Rn> z?0$Kp@yA5!>`n*Y8d4P;$Qp(2ck4|rMyu-Ar3KIoUWN2~xGqZ$GvKGNULzN74n|Vy z#LA{18>y-n-;Uhe*rk2JMuvZj^>{PcoT~!qe_DA*cC%H{P8=q1liK#^J2I1DHa!J+ z@s$AX*cLjsl0a&FuTjNw8PTHe2>~?5YLNa1G83PcBVXL$8Sj?rJnVQ=?Xc$Y$i}G3 z3AtU%ktkx16?9EDu3B;OVTxJIxe<*gFR5ULFp}@fg&wvH(4uG8)27M;onA8iyHGx! zedI@j5H$E}BEG2AX4;U>kQ`p@?&~^uEQg=vDwYBgxjFhx(u$}y|5=p>qAqv-iW{zO z!Ztu&Umh=M_1)J8Pjx*n3cqdz)MAaO0rkb7}2n(F$(>1TdU> z&~n#}`Uom+h`Q8qmRPy5SNEzF%6QN19sRz!FOWg+Jj^ihRh-gflm&c(K4}t)tzV2D=7`M_#^#^^~t<^m81%7?Y^cf&h^H)C4TWPurc=vhM4x_351@IEgKU@HmjznR)`o$o@q)889(F93 zZBDNEd;m?%707l$vTuRVb|aHybaVTM(#HFh zcV8$zB|)Ehs*mk2kLzsau#rK@D+PJJ*=(U)DEx`!VUd$&4J+S39dX?nW@2M2x9=Zm zq!}r(@cooB&U*&>xRKRv8hvi$iO|ohkge_bLQ|!8a~e=@yz@CpnkD^)+a;?Qgt_ff zLvQjCD0i~=XrLug&8^IX3%XQd>mTyeGb)-L(rq^=EH1=eh6`{7r3B%imuc&E8?Cun zl-jXPCdfv$;gaB3lA8Xgt?E+`PUw;V;JoXA`9ogloA<}Pu$#*&t#5yT497O^`3!N> z;eJ+eUSx+0ye5AWZrQG6rV46RIucpGTEYP_)C`5lK|O>DQ-2ix2%m|VITitx3Dw!P zjCY;@cf>cy@eQ}yP@!d02=!F zp}}3-t-5tPe04Eeh!vLD813W|T2v!Po{mJ!*^Wd7)5+7L@vP;3U(EAwd<7M!m(ZSx z#AUeFZyY(rse{6^`ZNz>Unyapy+)(~6YHOWbS>m|wGRXC4+1-3&pCHz(>nfj=F>uH z#SEdi)LYvgnlMRq$fwhr$#B=e;4>l>m`(hs3R_DRvv+NnxQ4BWfMYvNUWrhyK_H00~` zPK2%_HZ?F-8LLu&Y@}rfU3c%8tXsY(Q?c<1#F(qNm&FA!phCa-kZ!lz4Xu9EN5J(s zh+XvfNJ68dqurhe0hi%5!HW{5%eo9f*^bP%<5AqC=njC77Y z-%5gkTV*d^6FRPjDA#8_%<40J$Tti`%>y^d)Hw9L3~q2h7jOs-HP*wltj-W~6(IdP zNN0`I!y8Im#9SpvpMYdxU)~JiM2WF18>qvJE#=4h2@Spsp^cJ}4RiGPW2fP`W7=R*dZ7~54ec$H4#_#2@N|a zyvVgpBxA^DhR|#56Gf&7bM+n(xLb&q>)wXjgd3+kMmK#cj~!;n<$bSlh&}aG!)~DF zoZxv`#1-$zh5&lJh)x19FW6l3#0px?x$~MyJqTdTP-rm=I?{MDHvkdUu;D+oZKZ(G zy|a~f8D^-X7mZu)3+6<9DDi{3O^@&&{%rHTnPr(T#$dl zFc-L1MO=|?VOQdW#9uI(EyXaI&c#r7e7pc}*y;Z6=a^Cx9Et>PPQ*simlNdq;f+u42(RejOSF~8j)B_~%9ps(Y3?Zdm z|D2T6-HuJ4c)#499o~<+@5K42Qypr4p7Q*(M#&uI%iO|hWuN%L^UqZ5Wn|C15LID% zrTfa|JL5{%QOCF&Z z#*{LfR%JFXd4@ysW~fjCCpV};ixPO9Ib$Wx3GUyf}RV@W1bUm!Y7s^&ZF>WkD&=?IZ^7s>O2CYqD1l` zABPHgNe?5a9Io~i(ZZgq2t5}%H0uFoj7Xd=&|*XrBshr=s9=PTXi`{OJT2_noX|tW zp+|4r9)}1V)pqD({Qj04EfRP7FvL4t6LeJ==;1JrnR8EMhv@7sotbJKqq2Mk11Cy@ zAoJr6B!65w^&}@8eFgCrwj42x!hfJRbSln8e82@q8F7%(;Z$m2_fUX1B@YApac4{J z3le7?c^DWAD&zwzjNp72*hBhdN=gK#MDZ{=suuR^z#-eALp@I7*d6E&IVVa>9+QjP zYdQ);o;vjRIC3kD3tnvQH(6Uw+wn@|Qewclnepyd9H|l&TBGeVBTaORNxhP-k&VMa zejf5BN>gj8lmy&b$LuQ(EKzEiQs%Kn6V{}mH$~>)`slF7rFHG%`@RRSjWp|uxz>eT z1MNFXO{LvUn@>&a@RMKI`j+0hJ{OqzDj_sk3@h@2Cs=h)Jn?B_?^Ml{Hb2(|gn3i{ z6Y<4*U;U6Ba&%(NCb7y$X1}@aRl?07xxno8j~xUYWk~IZKLn< zTFjOyL0N=^Sb*i}Jmf9+%jlVj&WDTgn75VF9j(M0 z-4c6i{v#s7MJ`;|yYC?zZ@<3s8y(6)toZXHXs@{zPtXd*)aO@TLL0 zRU8?Eb9RD9RY1rQREYGGdoq!MkR%#UZgt@)%Re?zV4(}|(pwP43@}51@~F5l0>XT#QQ|-vJrhczvY}>6dhB`& z`;t7#ssd6#R38G~OT9cDwlyyUFLueGgPlc(UOcSNIu0Oex95@(;5vD&^e6>8d^Y?q zK5L3VwE4r$96wq_4I&P9hB5t9qci}jjq~otuJNz?kn?=sad-6J^=IyGt!njRLl9$} z@Zy5|&|?CQ>99fQKwLuXTW9Bue-R+(Q@qRd(BI+-xB*c~`o>STMv#ojfR)8R0#Yt* zk@~Y{>u-<$Ci#ncE%0}pM5c+3)*te!ZY;b9fy92esXV0b+!te17yMfaomiF0qqz2i zeKo3)DV-C}$|6y6qo`IA5Ix#A!11c)Z&x$w*aer_EHb9^+eT}nz_m;L+xfYcpi64- zqqPS`wcMd`3~KNjQkxrMagBF0%vic6J4TD{8ALV=t80|5?Ogg+H*OVUmR^CzyXd7e zI#ULkOEPOYQ|2_47scLn89k52ld4>`JxBQud(F!LXy+z0A5Ae_pZ8RN%0G8xZCCxJ zsksOczM9a?-S8t~>}`MGAij_)TnB@5$YWdgA8KzCkMfZj@pT-xkF!}_17LMrkZ)THhj+W|e~ z5hcS*3e9HY75*;|?sE`t{Wu-@)#|gaE^8X{behPG|D`=EAmse}PxK zj){OHlON-2NvDh0id$h{(T&d38uRb~;CS3?RN5>f;26|naT6U8s=$Fw7>8!OR8$Tp zEkh&uAH>z8NkSbUJhlY8ej6BX%>KzZ*8zQXZ`=KzOxv%MW}@)o#^oa%hg5x;Est5A zZWbh-h8M?~EkFL?0E%^(P43d6NwiGcwDdu)yD-&LO%N!USud*f=LY3r;&o+jSvIR}FKlz=ejffqLqB&VLP@%c3OIZWPs35<>^ zQpTC2;@de256?2ei{(b(*B)(rmytu91QRDd@Gws15N&MH;=BUnBerXUl{xc=qJM*U zpRPOVv>N~(?!K7j{>nl80cMz!l)>~?++2`LEGHz{v#$d`YYkKrHpB%U zp42Nf{{VL0o3vaqfW<4m0zVwe)ivHwARF{8_rQV!j%C3LK4;%ws_$v9j7F|KJP(hQ zVONiA(3uv^Hofv>&P)QP?tlF$a?O3AM*vNDkW0ew>*KAsWKVUo3-IE5+|FeDm_P9U zv5k_6q{IbqKs9208UKv9k1>EZO5LD`51Lc85avwkEKCfcY%!@Pdo_d8`@%T*QK83a zSqBOYMo%s7HmL4x`+1qIf_d;o0JabXElf;yr1&?VMbC_f#rb2ISD52)r@#*Vy-yh+ zZRPPit1r#fbWiznU3SqOp#8%xMVp`F1i);li6G%+U1RF+yB|jifuSOZsnnLjizvj< z=!$H-m>m4}5A4{@jjd`Dx?g{Qwq!WUHj@uLJ)7mS`h&dvA%<*#T7sjneg;^7v(i&x z`HUlAIyjMR%0$_eWMSlj-RU+q?p8G$Q_P2R$mJieZRPCXBzI zg7uRzG-mx_X@COADQwGR?o$>=nJoo4v6vvngb4qLB}fI3-5ZIV9bFN~6Zx*;wB@GA85~@Uo_EvtuUO zE)g-c)c?w2Ke6vU$5>TG>w5F(!f7InRC!TW0^Rkib@zD%Vb|GUl{6Y@-!~SmCHtEq zYDH;f4T?PdKa7AKDI#6E;~jY67&xqs&@d)pSbKs2CPq;&--Hg8K!xI^Tg=2#u}{a@ zsbON_a;@4P((SE`jOdqSXylCS-rep2ou)uXvJ7Xqz2t}U=FErQ&1EL|Ykgu|$*({Z zjiT;;e{S+m6bU>Xp$ms-@ccR!e*y&F$IE?JB_)w?0X$YoOwIT#=QVH|ET&vGsQe!K zn1=S|T?-V#92$5?zrCeHe17)N(uqoY$0B?;DU=|Jo|^={MoZLTu;MLd%YM*lF}OfR zd02e5(zRcbqs&hf-bCVt%6#-bc9_;TtudQbI~gkygJsajiV-XymnXY~Z2-;sAT0_n zi^juTByo_U#m;4;BRv^5n(=xULZjF75pQq!eouaQ_s#BhFq6ZWS`v$Rk`OlhB*Gk* zG)})=d=GgaKGY;p%u3S|>oPi%zhC)&4VjC=bCfO=2w`tuI!qP=k}WmfD7nf&0b_n^ zTYvo_Ckgh@8Fa{3r-5cpb!LP)a%A5@T-4AtfF|cKBb*nZVaCNa;-WBI;KkI)t+wZi zh&dbr{*H?{@wFs^wYnJD_@JL6hVrl=j4@mHIwWE!Fpwdm0zPI|V#hki)FxyLt18_ajXbr2yU_{()nm!>lyj!G;|p^~x9` zl@fG(W`y<&%jwXXFz^17j>34{5^cCL2}|-=_X;=q-qO{Y%4-Po(B4wdX>;v<+Fo!7 z#=DHZDEM_6qE>*8+iyOkDi39lCuRNCuEnfGbSd}IGu{(AXQit+^S?KpH-B^(?iOK7 z^?NdHFeOtM#0#H>;*<_8cv{j=%=;ghX(1aq7qA~ii;o;~@fL0`>9vntnYEINTI{ov zFvAzeTzF+uaECTFE$QM?1S2VU@zq;LJ|DKmIOjyi(}k;%!Xa;W528R=vpBP4)t5&P z5gezV=*W+*=_f|RVa%a=Qb(QtaFhuZGPS%i4c@n)#~jR8L72|bI1F;4qjury5&n=E zB+yrM0A;zFguUFiOYM|9kYY%mXTI1_eI z6Jh+N9Qjma5{@S1aS|7n11>Z;WnO_4trdp+y1B-vkLq~HXTaq#!WiPh*FerQsn}!L zt?qzv4?np_)^4NG_SAtv4f1CjDCXsA0@;Sti_9Z7W;Wn3;it<|je<2+n}$^%F^( z0?T%9_S180hC)@kt;Fs@_uIf!eX`tNy$-_Yr&WC1zwymtDcrc5ODr0?ucdPd_7Zj& z!dX;^1PeQFLcELA9-0R(ps1vo{i80%vD*p*9Rl_aLtL{TY=Nr`BO4Pxhoqlz<^wJ- zn$ovL_j}eq0d2L_%_XVLsak=;-{gXcD5*O8TraSZ2 zQMCQ#3C)BoJ_c(-V2D*K&r`hrDeh!87zZlUr$mIfLcmF)Lw0!kKm7W18oc=L?=<2A z;i$!iq6oN}ezp*Q`HUL`nIQdNFCR{ZU4QWa0Tjb6BO9bRJ84_eH9UK7 zx=Nozm|uC~#>#IOzExSq0sq`}oMdhU#4eNaa{kh-?<(FwX-}p`^s-xjQ;9svcmw#+($4qIQc~m|GO4o0e$2h>&CNi}Jg z(`xWY-up$%2}+3L;ai@q?Bct2j4=DoiK!zVJGYW%1yL|gfu}=mUjkT3L5j7cOG*!# zog1oYO=4?yLS)D$7{J7-XQ(&@5`ogk+KAn{SWq7U=7|oz4$qAaF?eV389s)ejCuGk z_KA<;z#i9_S2g*hec2$1n^+&^G^x0mNm1zQ*)~O8I5!@9zX_f~hS&C@lxq)qIrlUTf6P{hF^zMYfz#oI zm?EH3HT!${G4F^J*t55&;GLX2Ummxa>A#j*;v5CX+IoOzJx6G0~;Ny|zsS zNP3s@r=nOQs=x!3S?zVwnb(ECUWaEtXW&~28fO10$Fv6Q>4b+)V9_(pWu(}BCe)H@ zOT#@E%Q+#wZzJhXvTc`+_!u=me#+}79|BI>ZOI~&w`*7MMC6Zv%bVDBx8p+2ijeWb z-FR=0TMx246M{B}stv&7%B7_`^-UE_Urkk!zi+(rOui_VLQxiIUMbXNGy9aeTG(C9 zs@$Vz!yu{`jMt-{QS>B(9xwuE|{-Momb#fXD`9O~AUrep3Z)E9stKDmF4B zapb`j&LuRo%gsL|_((bmq=(F4dl_MkCqZK0*O-xbNY~u`rCRU-#V@Uu&_%SEr(?|4 zn+Jd;Ks{snpTasd` z?k82utbgZQ1f-ko&@Dd@SPF=Hl`sq7_7Y+x38(V&;)Bw&)MG;*AaQqtM_P8{Z&}X0 zfADMcalAt820BC~sao*PDe1i~$p@BpDz8eatE+R==tA8!QZK|kyZ&nU64ok!@$`?K z!Fyd|ST3hQC+n?-<#e)CaSdo=Q(e#mAVv<5}{|lk)pkJ*58|_D$*Z5 zO|SLKd@9x+#(wfO{AJJWO&*n@Ca|C3Za*5NwH@l?Q+AqEf|lxHWmlr83+mqDZ8>NqTllf9Cv3 zN@6ph+bZKKBE#dCw{H8;P^C}nWi2HjjUfHY_G))r(G9Ns9P!N@R@e-Fv|c)iU6Jl5 zS8=~R7jchXk5ppgnm5M!XV$KvClnxwv0LT3Y$>fCwXr9&8JWOKx%4O!%QbmAp0rKg z6ZiQ+WlAO?5^6ocJ^xj(K4o=OAaUebLtcoFH(dk&@4|^xS5&sU$7*| zgzWGAJe6gKu?e7&zjF*1vFHEo^guJ$wkg#q&lYz1w?!ZXVu1IJ0_R^-->fcjK&-3x zw)>QA#fVJg&C;@&u2VfMmv4=!%DT~vhtq{DifeO(OqA12qwq1atXFTcby`^|m=i}Y zbPxXyJ*e3%|7rj~HGgSkE^PSsc;+R!ERO>jR{<=AW#b7PH>Zm&T+r5(<5| zi!D?%lG&_=m<)W%E`@^ksf#%wHCnVAIe<3Idvt&PuO-l{=Vxl`a)(-Co0?G7!$58G zqx4kodY_n#e7LA>5GYnwZYb^)4>vAV9jNxf`sC$cmzJ+R>pAxD-Sd|{C6*ToFKSt5 zWnD)Yb7D?XjniXrcZNT>2({5DZ^XjN=Wjn6l4{uA|6b(j!5tbR9isL~Gq_{*7JC{r z*wawJn!iZ7mu=PfAXetcLeWFP;)>W`M_%lS58XcaWIdW5y^QF5FB+VfUli@c`n$%; zLY*lM!Irn|_^!6Qmd;(_qSO&*+A1rL2!Z_B)Xli{HzG;dR_0JuR8&+yjkVP~a*KFr zk1j772>0}*)a2Jv^7L`sv(K*)&YqNcbph@+F|*7NUaXVeW4_+T|FH~Z1|)%VFv9V> zs9a(tgUoNr7(UF)l?ca5!pJ*0rW%r8ZN>_nJ_)%y+*1{onmNsgqfDxi(p={|l5GV+ zytX?2kJ60NT_q?i3Gf*VxLxgQls3q8Gov~G{7krgbT?{UF_&? zaxC=sJWrJohTTzlQhsxS8e*Bw$$KNVJnznki`XTuNa;PD;KxDkQNVI7#n%YtAwiYV zeE03ekvd^B@6QV`7*sP<8;czyJcq7$e?C$jcrK?AYWza0AGjViod^#-ONBYDd@QGS z2l(n1gdnmpwMoop(?=aFhS%j`Eeb=lTkF4bok^nomsOt&jP3_EwO#}N(^gx~r4YL> z+D<}U#{Ni^ci#kDFKsz727d1uG;v^5Ca$L$Z<`ncgJ#06^=Boit|YDDRi8r>kDe5C zbgnk2lA!OTW<@7V?FH6v=$%OhmOxw*Obo~luHkz}a!eQWE5ZM~re-y^&n@0#|dPYKUn+0!|1nTf{x$OucyHvXHd|71UBU^}ys(`uR@ERt9yY(9gw) z`@t1^4c+TwA6Wp1n8MVTtM-Gpuf!W@5jzjg;+audHn7iz*E61=+ytSSS9PuQA}$us zGG6CO8uv@ySz2&cipzwmbngJp%vd_+IyjD8-S_yUseQJYyzWnapT+0?1M0uW%dgfGY zh@#!eq({)D>k0}ABiV9|Rq*zO>pM3wC5~X;zqiSs`5LOsV zW*$iDqSZF|{&=%>w-%HJy_Kk!5_*xd^hzthBL^hhS+t?MBkb#6c+!{wl;||DKfcmR z)!@2s^o+Kc<0L*>>Yzq=f1iQ;v5X;=CS;*3lWvr0!c&?IdZq3-Rn1y6lV{m7S@>9U zz*&58{_GjPfQ6wlz&czkYtgd(mrMQnB#PxP;&db`B!gn4-^LCn`58dT)i z6Z4Fy?k0It%|D0UuFzvg0%lUytLNQP)RkcPc1IfSnbw9cz3=IaR_?BALFmhcr%rS+ zym5M*^!A(u)LWAIwkRZ6P-&fyWes-=)THnQkMA7Q8%Gfb;jm0tVjtU0hp+QV#$`!O zUkLn+xRxwZd|M{oD}nyIaDhqlGK1B$_MaJ)6ZGV$n@(5tf~u2;5!e4V@fmBtEyaQ;>bIb( z#jn1Z=# zTThRFdikwo;sL14grQPPmRX@@U&;Cj;BY?%jvT7?zLFPw%cM>S{^=;9^U^QYjajqc z)B*Bj0roAh zzOD;Hd!0Kskp8MqIkiq_hHbBVizj2*`H-iiZ{ze!&~~S3;4B{hVxd$63)0JuaS>r9 zZ-nfrs>`-MHwE~s=FXnQpScEwHvG zw9ayx5f;gKPhbQoC;!;8B@`MLfKT-mF|swI;T#xb@n#?+)ur) zJi#d+&jsT&fzVWC-Pd6wT0L33O1X_u7Gt}SdbOvkLa20EE{eU=gb|k!+x4hMcpe0i z_^C@k;K7ET0{Dn17&i)xgC{n9oC1{JEY-hO_Me?rYlS&mk@oo@o*OkWTOdH^FgTJ1 z?_lQo)%T!Kt9h0A?9sK}!BiNZN&2%)J1GE7xf~2Z(P<#f-cxs3WNt8zMEQv;XBWDX z7hf5_=1|W$MVS;s)_5@HR%0IJvv#q`ZECWQSxoq{njVKb(MnZpIxpGP|yfd zVNi15T=O}(>jgLz={=8d!)D}u$^2%bT?Q%y=H#px-s&0zBYPBdV`pbef5f!wdB;%; zf95arV1jHc;d`Am$IU6tk>n_W{7Ou|B)7=?ljVZ@4?_#$4bC`z-e4@?qdwdJMTp-V zG-@$2J;&s7h~UfFGoMfTde_W~;Tr5|tMsZbWf*){|JLMKG%E&u$%Np?g?S~P&-PW! zq%fE~PU>=ypzXAhq}yJb+t4i-IUxWj1~>(~mm*xPuG324J;`)i#U_&Zc=<2+=e>(6 zy(u;@#Y*V75)3i%;jz=8u(xww_Vw=f`_097!oHan3?1tnEWvAh)7ioM$pW1o^*#y_ zpu-*O5I5RhA!s=bTKxNWR=%KCr^+lcf)XPR^JTkkq1*ZtWCnnQG3WiBC) z&R4}U6(6I*m|G>CO9i!2Ka+lRwn*vj<9Cskw{CC7ayt$YPrFEt+2Jmkdoto0Wx-^s z4%WSTCUa1DY0hWc@S;!YWLes(=GNYz>A0QPr7;Z9YT`Z~A$(K5hMO@$IBmSU++alD zs!c@Dpm$4-?ZQMX4X%>=*VN?X!NvDglY` z7esCCBNUH^>ped8_4i(J__Tts@C8eWCzGXjh9x`(Gr4l=a-|Nc^SW}TL^ptq5O5zP zS|X4??*H|Q-*{Oi9xWWC*ZnvHC^EHwzuQ-Bo3;R(IR#w9F7fg_Z}=E9lRHcZw+4Ra zpN88g3G+tegfj`3a=*;=d64Ufl`x(E8d*IvKOQUP_*3^H9(DzWFqdx{^E{c%`n7n$ zUXUf3H8R*bRY&`I-Ns4%%ktEwdLtS3ZF%c20pG1e*uY`7dl?c#k)!ZGPw!SgemI=9 z_Cv!4ArX(B(SC0h%r+x+;JvnLNL%R&PNAZS8Si^%<_CHasdb;Gqh%}RObiRj&9OA0 zaC7k{p9)a$)P?b<5*4Z*Rn@~ literal 0 HcmV?d00001 diff --git a/app/static/img/pwa/splash-750x1334.png b/app/static/img/pwa/splash-750x1334.png new file mode 100644 index 0000000000000000000000000000000000000000..93098b38dd6f9d7d0ff320682b84cd7657fe1097 GIT binary patch literal 23050 zcmeFYS5y>Vw=Y_V0t%9&(ukmo13;garhr@yORO*aE9|(h7!Wt z#>>&e$I;!5lPJ^5+TGVjhKJ`2=Rcwqe0;ndt^Y0E&0E0g_=+!aivVz1(8|_ZP*^~S zXci}@^k04{1zT_8@~;vdgy-?%@5;sTSd)j9mp4K;zyl$}qmA%(_w}+t97`Z>`bUVO z7sAR1VJj;lBqGKyB+M@&DlPcGk%&6}V}r84&s|$tyPG1`;-a?J{MMq9H~4Qz3XAjK zMBEVNx0VzZvbGi$vAJoD__v_HTmGw<+txsen24B&xY$kM8^RJd#UzFPz2TVnU-@<2 zeQg|wuE>h~o9#cY9rH>H5~J(l_%|rWi+`f>?;U?b_LuQLrRD#5s%&lkohA=oFPCEo z*xCppToA4ZHy>h>g#SsBt&NnAqmK*XKT@RN;`3iq1h6ONV&!Hp!{g6yi?Flub@Abm z1!g(gyK#CWTshs`T@jr8oWlQ&WB#uRo1YcuRc(ZuEy4@ob&b>3-9|=9LB;kKzmABQ|G@VUH{CD zafP$3LbR=w<(dg{-DlgfT`!$i$PTdkW(Gm6W4o)Mdwt3AB!|MFMQR8l>4JfN7zYRK zjui<(6hfdi(pt`tK%A{8Qb<1Re;EHy3eN+O+&_Od#z{_tU&3DfshFl)InxO;OxE!~ zKmIR_!HMH^&rG}P>M^!1h-D45_v-L+m>I;uNJ{t!Y% zy=_y;t~F;)-g|hJW;;(ifz?02{nZ6Jx-9c+QnOKRNN3!(>JYBEe)D#%RKW`P-6h$& z#w^Bec`1Z59fu2=UXNAV41{7X&5+DJDh`CN+04G#JnR)4kN-6{P&!|y`hG=g{!ZRa z+S2tULlePnOpVuDnOXA*@;^=Q%oBp%2Q)#y=T%h_Ir?`-p-Gb6L|>c0CfLZ1Iy;BE zJFa`$)4z$~;5J%0@g4VDS^n-}r^_i5yj4xc(FJ2442{b=hVxXV_jWnH#Gging$kMC zx;Je3?08HF3lII+1KtBcWs(=z)|h0>oir?{T$F$B`mQTncAKl{>JJ*rRUw_M3Wry& z$5ZWCyH&$#lFDADIdpE=!tHpBJv~B;R)+ivrfK-9{NFbQ>sb>N#LDNHwxKrIq+SBM zL-(g*J9fFrm?WLDmj|zy7!FPblHE(>^;`@Q6A+~7tV?z>H7f3E=V>V*_oG>$ZmLmN z7hCLjt)D458pyPsCzgY1y2l~M*F6Rp8$@e;%XDp4wF*|Mpt&=En*l^(aic2!r@x)3 zY}!;DL!_~CqLPBRaRJK9{;B}~IJVoQ?x$51b9|v(`-%P2NX~-QK=?qPtMae3hW?l3 z?_xQkVH-bw=zp{@e%rYx5oyzW@$16};qa$zeM)dynR+^VJBh^&mGgu-4i$kP^Rp;} z_#C$hEOuw%C;m2nNT|6$;H9w8%QFvQ>syQGJ@G!=1N{_9tGWDuw8mS#B?` z#W1*pW!*)Q!fyw^J-__zJYi||_5;5M?MADN*oCAID31Wi0l~PgPUrTjAzmgKdU+)Z z$AQ1mP9GYIj(^O3k~dxw#%!BRAz+p`&QYgl%7@=z)@`jrSt^`i zs^cGT9AmB;Pw%6(zVhk5i0IS`O_{4q@);|;?7iUSgMG^K3w8shTbTz97h>RHsPmo= z3>PEC9cMmY+4+cwl-jVVkkG4W$tX6M?;ka}P?V8C%K1TahNPqO&GV;SXI+OUhp#Z9 zP=;p?w#bv}wjNWthDzLn&ktHIT?%`dha!zi6IM(Y^EA}VBbVUgc^u#?IgO|Jb5px! z2X{T-Fm4ozk_BqtoYSKp&ewT#PNMeiRFihh$sJ528>VTxfIp~OO@-tOCnFM4DXX zw9Olk<9~za@GB#&s<*dj6vz9m7QCwtC}V0h$VHL^n#GvpH+d&4Ebv>MOm;2P=HGb*jOWPcPo%_gwR9A&Trn+%~&*d5FE zUy)+#nPh6LJsVP%h$G8t@r|CLM%c{~8}74DE3e|^Pn587jP?ni<;8W|v9`L2zsC>v zxVi;@RTG1qlF!tdd3f?!*4%5#+hjhgQ)k!Gx)D09c=H1x#nSqQy7kBRfufGnoiGnn zIo>bW-lbW9ee6w~ZVNtaF-&UlHd+aY$sZTABywW23E zyS1g35+&c7?*OfM2lVt@MpCU`q=|DqdW=<71V%2;2oRUS$4Bjm?+A=%Z+(0hIx_IS zbax%OuW(6^fKu^NZ8O^J3q0eGVbmDd?!I z@S+h?E~lu!0)1bFgfj+isT_&$n$pe{Hyp@x|QVl!l3fWiVwk4tfY|2 z;v9d2@zJgFRo1XFU_twh`r5ni1|?g~{h+AlG6%zTK6xQ82)8Wsz^LEU3uqH-;o&w;1ln9{R_($~uizW$Y7qRFyvaf5tSz-n8=>_qLY;~Z5 zIwp$hu9cX6z})$w2`n=BNqFbvJ9PE74b&Qr82q;Stc+ts80vkEBK=c<49Q3lJx2<4 zH6JZg@f#VvX|${c!9oC2L)k?0je6}yukvLLu|k!PFkQC&L2uA=`{6@u@<7+~{0sO0 zn5M?N3BgFA?@Vs9nH6KxvxPQLm)6X)&y4{XpZSf$bHj<`$xv%g{rjY1=W)i8YnirE zXDq2YVW&&qO1?d?JNPU z3NQY_2pAFPxI{AR8xRkm2vM?zm>Qn1DAOL4u8;n36HvoTi$+&d?oq!?Ab~2M+Mw&4 zws!P*S7$+hX_*+2lXKN+V(IYLb3Rm1*r_GNJLmRc-mHP@6O!Nrs^9(iAm>V=28Lh&M4OYh-c|0DSfFz}tsT}o(g@-eOj95+wVLM>; zNvX5tJmtmXl{W1Fx{!;V!V`;LyXue7u99bD86#7z(}=GgFaCovHzdD&pM!3+eJdD1 zMfC8=DfsJvC7&?_ZrpCOvbFd-BBm~_9PUQ{fzV$ zFhei!Oyb9Oj;r{}rlf<&x{$kdUj8IB5G?}Hd9kder?l@zh>c}1BhnB9?q|fWm+9Z; zExHJWO|p*5y{(JAwpj^oPp&)v{(Pm(@Z;}`5a%vPi385MqIv|-1ORRI_sEG~`x}SA zLWut3mlir1Ki@hp5kS2HsLP;UcSi1kvylWmCSGnfsrB=$n=yKUG^FW-rHFvLl=A{_ z#XylfZB-3?w^G&P)L<9`#R6YMcq7Ptr>V?qNTHX=TZ7DHMjHi9?b^VR@Tvl}#&s`M zIMgad#-x08vyw3e9tRJWztg+yLB|AH%2>QW>MOUMxr!HZ>L6IjNKSwr2tkoI-dt_t zR;xS-xY+_-loGFh=Z!BaB!QR>Fo{2?w*b4fW=@p8MQ7|A2s*fmV;PEhB3t17<)q?O zd=xt5NttoOXB9_sXma4wVS8g@kdaV0HUYKvY`V}ldF+R6T?Xj7@L z=(7*iCE6G`*1GdV4%xPoFo;gr8{!Exz&hz7MMi(J6DGX(MLH*9hb9ArRsR0C!L48@EL;M4tC?6K-C^y( zK(XG6?Nt%Qv#b66r3T@f%bWW3J-e_MrjlAt*rxV%~+zPlo zE|)SRyL%I1_hiV@81QkEaB{i$w4-fuNb$!{%!qorm2Nnk`$u=mxM9^Bo*h=`X)-t3 z*k=F4dpS0?g@H;LzrEGTdQ1!n6xT6N6LPMm@Mo-8JI@ST3R7$#b`OplA`N?%E?Oox zhuJ(Y9HD^{wH3|ywV3rde_Cz2fa*D==p)#al$n#Wf&~rcr3*|v)}XMiHxV^Nb#r*B)`O;?kyfd_!tc=vk!;U5tdHTYD`a1Lp!+$ zGn2P1Lv+u6=XmgNwMhQ*T-c~728a{*j=>M+eG2A%a2LgZ6u`w~?PaRVkU`(idKd1X z^^z+n_F|l!SUv2nL!(u$rGp(2CQ{mo3NTtc(_YVjjVJELyIK-?8Tbz+=e~68p6}S{ zy)$SKV3&qw8IbW-ru!Mvw4pAJYBFTer}tvgr$#r)A;{Ji_hZ2SE7~52y4BX4n^0TZ zHR}AJfEv2FkK0`o4l3an)w?{y=djq5fZ8k^0M0?}H|jh&&1XWc@33Ctn&JDFewUQ+ zBeCf_^t819onOc7yq~ZR6Hu|gRL>whZLkVGtusK@-EQyjF7CF>1ZFiLv=U=epjXg@ z+GZ>W=p7ld#rr6OkIrbb+LKVhWK9cyir~gwaPP_tcmj)?hXL_i%KYi)@MAw-Ps}4otYGdA?ZB99o9I~1#?sx zvbGy1;Zwvx3bAsZWf<(-^pjj`I1?W9;ar6*i=YcU8=%87uFPVHZBGE%V7&?`)Y2Xz zW zf~~^=&4lk+I3oPkKG)2@TQfhiU=Eeypw=Z93-2q;gT7eP)8Rqi^!Zm44&2u613xso zsom>78k|_jZb6PjLo_`K5mJ!DI97|KKvHReBi}1Bf2XZ zG|i?6n`OX0*0@r@K^k_h0cSE>WoKltCd_(jA-kB|-8r#<*cV=DX2~#E@15@-@~XW# z_s6h_^ab{bN?GIbeU$y$XS)ejQE{oDD;-Ip`&^TPza^l36Y=*F;^iA4#N3x=L~H#5 zipKo9Itp%?ft^d=73(qIR8jUTpGozni<>{3AYQfkG*d%ZSrY|dJQ*Hj=VmqxD@_5o zc-%&Hxvvv0e=~a;D?Y}4darL|-nK3ab58wfEHX^w#^6dU&qN(4b}M>3n-ON_{QAsu z)wA)JI=~*bkRM-)9h!cb0)NOBz|GHYy^dnuX%7PR)Ay@M2R565uFn`2vZDh;53GA} zAmFcP>nPlb)Zh&D&KF$|LGYB?WEenK&0F5k19Zr&*`PC9!iiyKdGWA0RASUp1nQbT zxS$c!5)%B363C*FO=CpQJ0!Rf2Cn+--j7Pj+rsAjDW2sGupbQfU#plf3LyX37j!|a zL)2Yd#h{}8;}eg;kHj?xVSpljX`Ff54*HRM?FWe4qO#4UF7CI$6OLM?mhhm6CJ(A^ zf3iA%@;C11xZK-hbj+nAxRmdJK>W&w{QxQbcFI=h8mVy;=_pITCUE(tAXln3&4MY7 zDD;X+m1~Cb%agCSb{PA1ZcxfIIfhpQ5c&&8zZujl?@ohCCM;+Mb>2@Zu!%5hm~5CjG$ibG#T7eSI8WB3*8LdYwvQnSwf-ujvHoeJ8KuQiDDsDKFO zDbF9h%eN=Fqfl!Q>rp{LFpr>DeAi)rf|Vw>IWRovNz`6bGCS(B-1$~RTHM&B8NS<1 z%f)A`OrF@Ed~<5Z+^ntnQAdz@31h8Zpnljhz;b1VPqk_Jb<9PRPJ1%eGeZd;6+#5_ zc?`Prd?gn4WnBqQ*Vg=_BPiHuGn*EDBwaDQz&l6M!il-5f#RN9@7s6yGZ9*$E~Dy& zg}sp)8h8SZi;HvXPaNLA^d~P-t=&><%c_Xlt5rH!iblS2Qi7*)&Fr5eWJfr>7a>sV zK59dOMvLj#(JG=Kk0)9+@s#MHeoaWQ(4^_H9nXEJ4$kOzscW^~5N z<=R`Pq};x!8lTBZg^@XWz|LPxhWld?|=)j>gPuQOlv_gC$prLv_iv_p;{fqGl5Rx|>^%p@%P%O~g= zK>)K1JPxMA#UYHul=BNZ(GC{W1Zo%6TG2wp3r0c;{bp`3u<0}K$z`kjsvQCk_N3lx zB3-A)-M7j<)#=hNHm7{>BN}O`mz!1GSDI}Y-TSzv9$VD+u3+w_nB9JHdG4<7=241&ap5N z>D;WF9|gF2{;t?_vl>2@Uv$7`nPOqzG-^e8rY@iS1C(5zvuaU-dncJGc*ObhB>PA1 z)8YEG{761fMLjg2Z975z_L#GMTM2&extUZY*KrU==y2E7{FYkJ9jl1bsI5_&dP3?< zPYjn35TlKlIP1fCY@2^>*5zJ-^DB3VLfbunm{c>dN|IwShYUE~r)D=QFaBN8;dGyw ziGC(MUS$BFo_T%jlU|mT&UX1}-PdN)~*zu|c(0T<} zSfh?tNr2W{&dmDsc(qB7{HcJ-^Q-ZUnn(+$J6P2kWa$zm} zXl+X+A}K2dFdF#1ZvB{mJ=_YKnm>na<{+qF&V_LZqSY-=TnKKQ4cS3oh(=Qi5j+pj9nz9m~2*kMpSC;CdJxO@vR1Qf<3awe!gn$6!j4 zuWetezISP+cW~ZW!#q%ON0{|P*`PqZ!Cd3`k<4TN(9L)_uG5~T5MZYqxIq zU1${Y+p^$0B*y*dXWwXd*wn}JryhcGe-J@``r+8-sKh@FvmrMhXY3XEanvI#=6$!! zKc+^S;C7HRO`?LKhX?9GJ2+?Dm!t7f6{|}JL6JBXxjzrqZ2FA-9)_a>mlUSw;f$M4 zSTimrSFwHT@V_sk((HpH|C9FXYE6pSg@tnSfX~Lx*k)E@=myxMb^{ZT!+XfO;Vad> z+%scE9KZj#63kZHufu4tM(=x9^`%vyz&?1j;ZaJ{8U3N~w;)=XBqW2YQ zCu&spq=StZT)EIfk8tKv0lMEL3Ff^s*u#S5xPIM>b8JZt74~CuQ!U@eiWj>;{skVD zF@U$iDK|S0;@6R#K^L)E!XQ%PxXhw$P5UVX-A4XYSyI%V@i+R-)OVdhV%V&UAQ+9? z{rCjtHC{CDqcV}jqpSbK(Vil=IlJ7?5l}=E+Hx6F8bEjdbVuuWYLr6`^|dKzE<%*Uwr%Et^G)HVo`%->3#dD zI@*nyaSxhVjk1{QON*gKOn0x8ayj<*bjSQM-1kr4nv%ok2CPa+NqpzCcPV7H5zQA7 z*U2Ef3^WgP``N4bClh1lcljjzo!IXQ+P~o4O9!S1${;_xnO+Dnop^H}IrW6r<)c>G z;y>;+GlR@%)mqGPv`A!2@(}%hY^j*o!_G73Oh@dSV!-{bxZQDMv`5Gb!J0c$2wE5C z@c$4HJOwn^s02CN%}6{mJN2RPr)c6oSr;VYEylsUpJPw$H_P#z%B+RA-WXdqVfm!@ ztw$pX241af_g%U?r4sVh1NfPD9M&Gol6HPrQ-+@$#jt!HuaZz+{miDUMR|wQ zA^fga)QJTPNuFiPjVEe);ZHMbCXCmoAnryH_z)X-&c$iy9?_mlsqU_S4|FGqmH4X-BWQ=YlWJ zy^ir8WEjFH-m1+gnE&0E=0`%IvoA_Rz*7>a^}r2llwYWB?(F@d*IxO}zbMgZM-6}E zu(K>!SV`2LG%fDvsxmsw$i&j=q5+2Qy-?jebL!%uJ?S~Q7ZkDnk-S*5^FX$7Z3e*T zS)885OKdC?F(mIoG@61GniQD`<Qajdu+QH7CiF-9Y}s9 zv|P!MTcJeVQwH3a`O6=QDoqE~?3)Je4u2#WHjijG&jAx(s~du_A)Bd=g(&gRbMumk_LWImWWRD5sq z3aEPIzxRY_x{LsznF3nSjA7R9n!xp<$b^59r~xT6clLT>@hLqFA6jUKZO?>mvxRn3 zQ@JTpx%ME*N4lZLt0!`J=9*u=HNdUPWNbJo9&|r=<^G8}Zp?L^lYm#Q zF{^?FIs1L3BADmzsZ-J723%mKwWKxy#Q5^n^lMzk@HGnUur@Nt4u9?KxIb3EIX)jS zUm5bLOJyoxFMp_Ha=>6RK(J>aI^+QNAIOML9CvO1^?TfVN9-;evRrXOC~JXxFv(WzEKr%lBx)}BXwFa)92#l`;_gwQaTHnhMrGuzU z{eHBYo|k~0c(T8~gbwaN3wFSD={DtV*BB+yI@M|P}Mk;;J?8Bhp5^J zz(p&~&9Pz>E0FqgQ0DL6fJ7t^Euw-pYM8!8hBs&FIa=oUi}j{1IM_A?w!!wmw$EUj zN7teLKl@_Az6&%%Znsg$dqKX;2D`95#6P5HB1v$_t*cmM7K``&FrsoY!kItlFaSFn z`zdfNd#>hFc}!@j<$ni-`!^R4zYs`rXVad&F3bQPp;z8y-TG{v+}Us5$g^`E0RsGqf-|zAcLJ1Vz zttLf1BW#w#Ubk~?PIg&gOZbLkMF*SYUfp{$q$qawJg8SMVhAr(47u5f2Y;A|@}$6& zdXN{9|+x*MUvTN#-$UDWq+64&ewFio|p^+Y&*}FX%Bphv zA!w{0S~!yVn>yV^U~;6xAK~=kAPk=yng0;BJL=dN?c==V9u|$h@5&;@Tc$Q;FSXGB zvy12`*i{_OHsB%nXje7kbcKCGMffpBOGb%?h>DC}!noV~I#H&35~nB5kUi*Sar$rH zEGF)@w|(Py&>giqy};OF?NCvf_B|x!K=rTc7#Pvu-l3M3z3I_Zu4uIZz)_wixhF=m zs7eHgHwyvZY1{0i(NcT&=yLv&{{=Re!WKrQVHX3J?z!|}AvW2Z8kZ&9omKIbd`3s^kIT;Bkir^vpvJ{ZXX-#oZIShIz{0}c=SBU-oR!|zJ(WezZ1 zoPS&$&apXRT80{T45D|FGjo&AylyVB!;ZRGtlj%~!bY@3NSsvfb(bA$A|aCSatr`V zKgV9`H3f#x=}P(a?a3Jg$Hy_wM^eV|HzLZNpnzzG zTEvU&DE4w4D)QMb&P+u7ji%e*3|YYEPF+vf1X4c;p zwdVGZ&`9AlYsZT8w1u)&JP>-l`{4JEZ?PakxdSbSj3Gifw>c<0sET6m1i`~E(SHw7 zZC*D|yGiTg+{A!8qQ!N-wA8853=ev!F1lSHZ;RhugYVJd2BeQPN4iv^cey{b=mF4 zXWN5Yh1;dpf`4{eMpGX~-Hu4FcdvCj)XQx)iP!J66C@z^x#;kx8a-UIewSOR6Opri z$`2ef>Faeo7MM$aL~EP2jwT<2mo?(=&si5W-{Bb$zM^xZt=Tf?KH({|qdxIAQp>&G?hY_lh z0!4Tbbm6pd&sW1v7%%C+6z;b;3S zDfZ-FdM6&AGvt-}NEQe5^}YwIj6&e7w# zpJaFtXOUgWxQl_dMFjk>V)#YO$j`yl;QBj%P69dGbl&*8e^t6E*A{IN7;y~r3ml)H zeL5OV{W9)RhH%Go{mo6$?&w#!ie~mrZpHr5)DthpD?*2A30k=_$$J_M_=|LS64S~eTDHkwFD;;6)M8iQkH?^SDyer!fR98J}93J;R!8Q7V+ zOo+M)@3MOUj`#1egqPcMagL`)Iv8Q2jO6Ukho|iW$;yq~MA`MB8 ztMsQr?YjS_s5qY@_Y5h#L-D%J&ACnVUdCwZBZ?^GtKU(6Fbgvy`=pw&(Nrz)&eL+@ zoD6&XC0lTF^PYCojdoK$fDUI&%`yG-7riOx47a7}?cOhkuGm>REbQuqTr(P;)lLPJ z%6Gvo#Mk3E$p_Hl(y4RueXRkvq+k7Dxfp89=@u{iqTZF`a(?)m7Kf1}6X9r+d5%on(&Rfn>K{J%dsP5jb&lsa*zP4( z5Yw^RFxgzQ3+IzO>F{*Ty_x8y4+qn`UFA{)e8;G1g=?s1l9~BO?m3l+^kTtNQFhZF z#6tD+&79|8S>PC&Pn^s6MPGU5LH?&y!Eb@M=-Ggcrp*i7a|`YD3lWC|r0;qKJ+6m2 zU1nE9{Bj=G47b#QUPV@>9rbncxc;BddnfyT&ACS-U(5ma=Bn7NUH0FL41Uu#+%w!k z2i$(M-U`Q_uJWtEe|+Cn2@i4!53&tCc-p;nLYlxBTcB zo}c}6e}vNX-W~A(zr86L_xVAK?->E&X`9(HgEzy`CtOm6!6O7S7WNSaZlG)9&Vg&> z>Oc6$lseERQAo~%C_mAiv)_87y_HM4J?Iz8<@b_LI6B&+s4M!7{!S zNadf=pG^Pm@>fDnIj|D1?Gj!~oI6%PK|SPF?+|(?9m54AF)x(gsTI6*;roYw$RF8< z+*%G@lC)wx3|3n2%Kk3oo|^?YTII&zS_|$u(c#Z{l9j-K(o0ww40y>JLWlE<+Kv3$ z?&nAOMhV5oh@;*5A zIfmRW=uhv+qns_5J=O}FWGac<#6?yTEH#1Di!_rbL9 zEtK=u`PnC4CERZVsch+vcZq{YzB`a%X7p7=IY;E=Z>St=ijVS>*pO63tw(rF^tI5a zUXsl$0(;p0?zxDW^(f)KUMBOTq`Q@$0@>dYxn=9lJ?~*AROF;oe|`Pp9Bhbb=j#Wt zclY|j{42~c-1C2!2?>M{AC@8fnS`)WhK2IiYmKWVD)I?4`*l|ObFg90sU(YYl|MLk zZzC4z_2-1gXScA`0oR7`j|@Za_+?qrX{m&xBL+gh-922j9<$+8nG|1SuJ3PEa!K^; zO(D~t_EknXJB^^L3^i57>S8kT>&65}y38nhhqb>o!z8Bb7W0N7VJF|%7e z&qP+wU!w8tOd6W(Qd}Y$(_eh1R#y_vx@owUSM1N!t z&1Ot*gmO+X2kFiFG>hMLw^7b6?l#f8rJfXXGbMCWzk>AV%m`m(f=g7s7rgP%6|ee% zk3%na76dvbhBs@C<#9&N9lkM?5(ompy99zq=$}w?$?$CNc_JLJVsG(=qg|rDAN3F? zjK`*i9Ma*_r~OcIlxghBP1F6g4jPdL)Ei2*MM+uP2p8 zA>nD~aHl)aSy6uP%E}ORvnRvRBu;A-$W*#bKVcaRLlhECi*rN!)XDFPrGovkw0N?W zm_Y4m5w#sw%_#qOkNJnP<(sA-I*=o~Uq1Gb=qP;rTdN_#Un0YmNXUqH=x)_)X$pL+ zyTkfD8qgzgbdHWRr5vO&o&(?&!!{*6Pv)WbTz|(+94*~cNUbhe#7704@A*_eez&BR zsDzP8F3(=)W9j-jA@@!K!$4eRj!@G*Tjvg_#=DxUik)6?dzxn-Yv>ypRq5KrSnrm2 zH^Noa&N7IK-f69{cZ@_J@CM*{Su}4s29u!O4D{wF`qfN_yAwsT{K;5Kz~UuFWRao6p{BYXFAZPdAysQ<5Tz}}qu2=LEEB+D{|U;Wgp8f;t(zx? zHN(*ljWO+BKkrwq3BRJln$aj77Gh&MD-q_D&z79O;5aDkS>yX1r zC}$o@FfoHs+*N~>;tIKD&hB^`7*c0r#r5Kpn@SaGH4^*{9y@d^nV0aV{`AOR6MlN7 zZPxN_qYYO_5V#2f??M{tEM+aO9HRUO&*sXA z0q*NBFQmrbi}H)xYkSD%6&yb1mqdq;2ujgIUQ%eXXBo=oz{;Sb{4R}b1(1~Dm^o+8 zPE4(T%HF%D2ZBMjIi*l*AK^bpoeN9a?+#)9w!$HL>t2w*LFH>3eJT0CXMa%azK2j2M`cpq*Nrsk~(S@Ms z-B$qR0Rh5%n`y>Easq;aS1`|Jk}CAxaFCE(oZ%<59kC5dd|XuWa>@8aHEmP<*i|&m zDu}9Zc$E2B)$dlo`AM=mVnHzKHDjgFSHZI){~ zG3Bm_#%qwCn^+k!r66zvQ+yG#&|Tl|^U#$7*(=@h*nRvZIA&;uvQ5AfffD>TD{D`x z{oL17|7lu$ila;X$JdF4ppMBn=Xg|VOBI?l5*#8e23tDO?&8>Jpv3S8%dCDke@!~Vy3Gm!Bk+cq;Ww`Ycy?^J8Q6OVI zr7vkOIe~|hKyTuscD~a+#WbJYz>;oxW;%SzqW_(IUb@ZX0=EwMkQ>nD-6BA0H-yY*{P z+A05Af+`xAA(4b5ngQ!y=C0oIF9ygY9nB;yrw;#G!G8A1jmdN1acs&Z9+?YUHLeOs zS)+T(9ggPSZL;81bEO-zX_c=B&xda0(?o+B^*6Jx9iNg9*)Pt(esrfBiQIS&4iDCk zDZm(<^*y376H}B=suwI9fGWjsw4y-Q8QZh=Xl}K_ZXk$f1lXif9sQveD0vo*=Q~wxC;0vq zI?4m7>8$a($=@;pND$HiCL&|65hdujJo98(z`t3ZOTU(w=KG<@9H2eujqKsT14+Yk>sL(_{oo$GL?Fwia|BWh5xEJ?Ee9tds zor)$w(3yP`AEe|Je|hvrNyexuZSkeBSQ|{o$&sdJ9`-Q9X&N-WS8ck}c9R^BUv)-q zeU%U#NI89&=GIzlZOl! z1u}l&BP>W8nZOlOW|7K7>ehagbmcccP{l&waP8fZP3j)ZJi(e7f>xt;BEPoiS)>}u z71du9b4ga{S<_(Yy;rwX0zMpTy_5JhORbihKO8>%qi-$^v~Ml&ofvf6Xvg?TZEYk7 z68M1P&NKk#l1Qp33G}6Rntvp}(5;bJhix#&18Yu;kOK_?`xB5QZ;ttouR~4(Mz`!J zkhWj%8yFrn?TC+oj}BjzIV(yqH)7iZN>m;$9Dz?4!4UcEzW3-3teap^J#{H#!XnmeN;D0M&3gK_qdujQY}h5AT5!PnI@R2hbEE5r%f91wK(+@87S zrr;L%UYZf+e^f8jj9L1g%N7k#fcwk=NHHO`?OEe zE1MJ>+@Wf@e(An4qMjp^Zyrn%K=kEx+{%x;Q{I$3M4I1WkAc&#O{E0{23xfrQ=KL| ztb?1yX2D0->_F_^mr<_ZM$GZI>!%4tj!)9xiS+tVp}+{jUC-%o0pq%~Bnl|3Mp}YT z<4<~yjTD*bBWyx`R9(?ZMAhYAw3R~n3B+Xh#*HAK!=%e4Zrj8W zA^9R*UAy&%EV1Cr>|V{8XS&mEj*Um3o}Np;wh2K&>F;~GKiaS=fPJoSc1v9DN2jNG zQ$m$#OY0(HJ|`$sEXn#$-+nvyygVZtpIe{6NE{}a-n7k(R7 zL37(vgBG&%Z5n)}r#WNAF1SSk>Fcprah-b$Mv3kbZA?sIAr1+&{<+bqttTDAQHsl0mOZO@6DAyG4Jp zC!K$_@f+dk8dqil1ntro2w53`Ih6s7>*_hlYF!h5rIXO4io2lI3du$+mf8eknp#N0 zt!ng?1e$aneDJg=$ilKuh)9|~qaxGzE&A!VkYofgVpx3kgRtxeJOH7ubfiC6ey1C` z7@=f7oU?ZSK89|`bc_^hia!|e!J`FZI(VDQ`kw2 ztg=g)#zGh%D9Qw5vK~ zx>l96Nvhm<@-X$pmkwC@^Tp}nAqlzm=Yn;Ty&k}cLmwMYj*ID5)mqvh zbt*OiZt8r*oAZlivt-O<_?Y3tmbaJh&f-N?xM@5CQhqq9+G%H#i8wR!ouO)%5%kI` zSe)b$pd#RB))p`$j@bh6Fwc7|Pqg0oE z2Fz+gC`@0E%dA|%9e&rDwNrjzepv2VQk@QWZ^_OyHyHm5vvtPZ8Jh2*-0t~J9Kk&v zjq4UGl30C6;@k862lol+)A#;*pIO-|!}Zc8;w&B*5)DyEeC`*IZcv?#frGgyNF@jn zlE9Iz?;u5H33`JijYt!E`*W+Y*%S|$xPm0rFQt^owHW44SyXDCo@9uDSEdf!EHvWo zb$XHfq3sG371^`H*66KWRsUoJ$a4WRi5riHAK}hPZ}qJ#BUi6NmXuot z`#uw2t;Onu0LKMJdoXp8n<%F)k#i#nV`U{;!b*3;8aQILW-Nt9Lp8dDl+E-?Ds-T%Rv zOO#;Px*BZU{=(!dwf=?~1i>;#83ZBZwq}yuGN@n_btg>d>kEtS?d4Ku-|r-*IV6@e z6@pX_U$-ATN&7?%jt+7v^upWQvvY#iL$8e3Lxv+gYHWcH#;^6=h*N$ZVqi>{MBxwh zY>;C!StdOZv`0mSLwQ>6OU}Qn3p*iwF!_1?Yvr)t+4tqQRoFXzT!HM=GSiBR>`yW| zYXJV%J>o2}*)U(pfk2s#7M>cIiRz2Auygbec-T^kh^Ds`V??&*K3GNH*lw$Iy9p4% z2nkeH0aq=qjkbx!Ay2-I0rSSDS3}WmXrZ@!zdZiHV>C?*g?yeZ`#E&o}P=LJULF>J{_$@(Y>#XTOdN$7O+V+sq#AYH|;AlVaFVOA0E}XpJT&)zEs_!NA1~Uck z&AF3#`k!i#0j-@zu#qlBoP1T*?w|#Oq(NM!LA*nDw8&9W;If{o>)lX~m$H&^9errG) z4b{l%DBSDG9AAx| z57zXQpKWsfu+|c8ub%Z6?E=G5 zo&+nF+Nb8C3UZ*E&YYQ0X9^6_Rl zl{xt5%m*u;e~3!S!fHWK2?E?mE9)M&GS8g|^+T5mAck70 zJoQO4YNTdRV-us3xQ$=;ec?eM`FIpKTcSs5#1SwRu0J&+F-cZW1YDT8-cF1M^vC~p;6e91Q{$6$=2v)dW2U? z>Y=H-##n%rDew>UX8H%J9X}VVj&yNyEst9yX|elf#yRu2Vp2bNa&!$+$ z2^xi-!IlbKWwVX35|eeB>ciuo$Z-W*ieQ4+xDGr#V^CYGI2MP(mxy%Y5wxD{*66&V zSe)}Rd(>XdJvIaHMD`~VVlytGvC7!EKSU6{wTzY|Q}mdle_P{&EmHA1D&TEz%UT*u zGH6~{#G;e;8p0JO;timY7n_ul`Hxp1o-*qI1R3ryvUc+sSqJzytu%;_(-P$4w2tlI zw7>xq1nuMFw0@ZWr!%l|=I`IXKX`fnLpI*Oh=)6N=7J PG90?3kJHP$U>Eim+iGfa literal 0 HcmV?d00001 diff --git a/app/templates/includes/head/favicons.html b/app/templates/includes/head/favicons.html index 52a872b..458ad45 100644 --- a/app/templates/includes/head/favicons.html +++ b/app/templates/includes/head/favicons.html @@ -12,7 +12,6 @@ - - \ No newline at end of file + diff --git a/app/templates/layouts/base.html b/app/templates/layouts/base.html index b55ceba..6545aa9 100644 --- a/app/templates/layouts/base.html +++ b/app/templates/layouts/base.html @@ -1,3 +1,4 @@ +{% load pwa %} {% load formats %} {% load i18n %} {% load title %} @@ -15,6 +16,7 @@ {% include 'includes/head/favicons.html' %} + {% progressive_web_app_meta %} {% include 'includes/styles.html' %} {% block extra_styles %}{% endblock %} diff --git a/app/templates/layouts/base_auth.html b/app/templates/layouts/base_auth.html index c38d0d3..6cd9a16 100644 --- a/app/templates/layouts/base_auth.html +++ b/app/templates/layouts/base_auth.html @@ -1,3 +1,4 @@ +{% load pwa %} {% load title %} {% load webpack_loader %} @@ -11,8 +12,9 @@ {% endblock title %} {% endfilter %} - + {% include 'includes/head/favicons.html' %} + {% progressive_web_app_meta %} {% include 'includes/styles.html' %} {% block extra_styles %}{% endblock %} diff --git a/requirements.txt b/requirements.txt index b4e4f02..808f347 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ django-filter==24.3 django-debug-toolbar==4.3.0 django-cachalot~=2.6.3 django-cotton~=1.2.1 - +django-pwa~=2.0.1 djangorestframework~=3.15.2 drf-spectacular~=0.27.2 From aaee602b713c51a0528fe19b90338b8c1c53caf2 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 12:54:26 -0300 Subject: [PATCH 41/60] refactor: remove django-ace for now --- app/WYGIWYH/settings.py | 1 - requirements.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 960b0ec..9a5a496 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -70,7 +70,6 @@ INSTALLED_APPS = [ "rest_framework", "drf_spectacular", "django_cotton", - "django_ace", "apps.rules.apps.RulesConfig", "apps.calendar_view.apps.CalendarViewConfig", "apps.dca.apps.DcaConfig", diff --git a/requirements.txt b/requirements.txt index 8c24038..af9d39b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ django-filter==24.3 django-debug-toolbar==4.3.0 django-cachalot~=2.6.3 django-cotton~=1.2.1 -django_ace~=1.36.2 djangorestframework~=3.15.2 drf-spectacular~=0.27.2 From a805880e9b52746dc9258b5b71d11410879820b5 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 12:55:01 -0300 Subject: [PATCH 42/60] git: keep import_presets folder --- app/import_presets/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/import_presets/.gitkeep diff --git a/app/import_presets/.gitkeep b/app/import_presets/.gitkeep new file mode 100644 index 0000000..e69de29 From d7de6c17a9e80d8a9c562e27b037ab4ddb64dfb0 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 14:04:40 -0300 Subject: [PATCH 43/60] refactor: remove django-ace for now --- app/apps/import_app/forms.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/apps/import_app/forms.py b/app/apps/import_app/forms.py index f300721..83eb6c4 100644 --- a/app/apps/import_app/forms.py +++ b/app/apps/import_app/forms.py @@ -5,7 +5,6 @@ from crispy_forms.layout import ( ) from django import forms from django.utils.translation import gettext_lazy as _ -from django_ace import AceWidget from apps.import_app.models import ImportProfile from apps.common.widgets.crispy.submit import NoClassSubmit From 962a8efa269e7f4c65227f96cb32a687331d9c41 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 14:04:58 -0300 Subject: [PATCH 44/60] feat(navbar): add import to management menu --- app/templates/includes/navbar.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/templates/includes/navbar.html b/app/templates/includes/navbar.html index 66aafa4..f781d05 100644 --- a/app/templates/includes/navbar.html +++ b/app/templates/includes/navbar.html @@ -120,6 +120,8 @@
  • {% trans 'Automation' %}
  • {% translate 'Rules' %}
    • +
  • {% translate 'Import' %} beta
    • From 4ef4609a96aa3f4551966f8fa9475e5ce89cce65 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 14:24:31 -0300 Subject: [PATCH 45/60] fix(navbar): wrong active link for navbar import item --- app/templates/includes/navbar.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templates/includes/navbar.html b/app/templates/includes/navbar.html index f781d05..a4ea8cc 100644 --- a/app/templates/includes/navbar.html +++ b/app/templates/includes/navbar.html @@ -120,7 +120,7 @@
  • {% trans 'Automation' %}
  • {% translate 'Rules' %}
    • -
  • {% translate 'Import' %} beta
  • From e3d3a7cf91c4b1d0eae4dd90af88176b221984e1 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 14:30:59 -0300 Subject: [PATCH 46/60] feat: add new envs --- .env.example | 6 ++++++ app/WYGIWYH/settings.py | 2 +- app/apps/transactions/models.py | 8 ++++---- app/apps/transactions/tasks.py | 7 ++----- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index d7b1933..376d5dd 100644 --- a/.env.example +++ b/.env.example @@ -18,3 +18,9 @@ SQL_PORT=5432 # Gunicorn WEB_CONCURRENCY=4 + +# App Configs +# Enable this if you want to keep deleted transactions in the database +ENABLE_SOFT_DELETE=false +# If ENABLE_SOFT_DELETE is true, transactions deleted for more than KEEP_DELETED_TRANSACTIONS_FOR days will be truly deleted. Set to 0 to keep all. +KEEP_DELETED_TRANSACTIONS_FOR=365 diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 9a5a496..4067a62 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -337,5 +337,5 @@ else: CACHALOT_UNCACHABLE_TABLES = ("django_migrations", "procrastinate_jobs") -ENABLE_SOFT_DELETION = os.getenv("ENABLE_SOFT_DELETION", "True").lower() == "true" +ENABLE_SOFT_DELETE = os.getenv("ENABLE_SOFT_DELETION", "false").lower() == "true" KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365")) diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index 85ff53a..4b21019 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -18,7 +18,7 @@ logger = logging.getLogger() class SoftDeleteQuerySet(models.QuerySet): def delete(self): - if not settings.ENABLE_SOFT_DELETION: + if not settings.ENABLE_SOFT_DELETE: # If soft deletion is disabled, perform a normal delete return super().delete() @@ -49,7 +49,7 @@ class SoftDeleteQuerySet(models.QuerySet): class SoftDeleteManager(models.Manager): def get_queryset(self): qs = SoftDeleteQuerySet(self.model, using=self._db) - return qs if not settings.ENABLE_SOFT_DELETION else qs.filter(deleted=False) + return qs if not settings.ENABLE_SOFT_DELETE else qs.filter(deleted=False) class AllObjectsManager(models.Manager): @@ -60,7 +60,7 @@ class AllObjectsManager(models.Manager): class DeletedObjectsManager(models.Manager): def get_queryset(self): qs = SoftDeleteQuerySet(self.model, using=self._db) - return qs if not settings.ENABLE_SOFT_DELETION else qs.filter(deleted=True) + return qs if not settings.ENABLE_SOFT_DELETE else qs.filter(deleted=True) class TransactionCategory(models.Model): @@ -227,7 +227,7 @@ class Transaction(models.Model): super().save(*args, **kwargs) def delete(self, *args, **kwargs): - if settings.ENABLE_SOFT_DELETION: + if settings.ENABLE_SOFT_DELETE: self.deleted = True self.deleted_at = timezone.now() self.save() diff --git a/app/apps/transactions/tasks.py b/app/apps/transactions/tasks.py index 5f1c42f..0833f4e 100644 --- a/app/apps/transactions/tasks.py +++ b/app/apps/transactions/tasks.py @@ -29,13 +29,10 @@ def generate_recurring_transactions(timestamp=None): @app.task def cleanup_deleted_transactions(): with cachalot_disabled(): - if ( - settings.ENABLE_SOFT_DELETION - and settings.KEEP_DELETED_TRANSACTIONS_FOR == 0 - ): + if settings.ENABLE_SOFT_DELETE and settings.KEEP_DELETED_TRANSACTIONS_FOR == 0: return "KEEP_DELETED_TRANSACTIONS_FOR is 0, no cleanup performed." - if not settings.ENABLE_SOFT_DELETION: + if not settings.ENABLE_SOFT_DELETE: # Hard delete all soft-deleted transactions deleted_count, _ = Transaction.deleted_objects.all().hard_delete() return ( From 096f24e0a2347359a63adb3ea0f356ef8540f8a6 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 16:32:08 -0300 Subject: [PATCH 47/60] feat(import): cleanup --- app/apps/import_app/schemas.py | 0 app/apps/import_app/schemas/v1.py | 11 ++++++----- app/apps/import_app/services/v1.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) delete mode 100644 app/apps/import_app/schemas.py diff --git a/app/apps/import_app/schemas.py b/app/apps/import_app/schemas.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 22df7c2..f5710f6 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -17,7 +17,6 @@ class CompareDeduplicationRule(BaseModel): class ReplaceTransformationRule(BaseModel): - field: str type: Literal["replace", "regex"] = Field( ..., description="Type of transformation: replace or regex" ) @@ -30,9 +29,8 @@ class ReplaceTransformationRule(BaseModel): class DateFormatTransformationRule(BaseModel): - field: str type: Literal["date_format"] = Field( - ..., description="Type of transformation: replace or regex" + ..., description="Type of transformation: date_format" ) original_format: str = Field(..., description="Original date format") new_format: str = Field(..., description="New date format to use") @@ -50,7 +48,6 @@ class MergeTransformationRule(BaseModel): class SplitTransformationRule(BaseModel): - fields: List[str] type: Literal["split"] separator: str = Field(default=",", description="Separator to use when splitting") index: int | None = Field( @@ -97,6 +94,7 @@ class TransactionAccountMapping(ColumnMapping): target: Literal["account"] = Field(..., description="Transaction field to map to") type: Literal["id", "name"] = "name" coerce_to: Literal["str|int"] = Field("str|int", frozen=True) + required: bool = Field(True, frozen=True) class TransactionTypeMapping(ColumnMapping): @@ -115,6 +113,7 @@ class TransactionDateMapping(ColumnMapping): target: Literal["date"] = Field(..., description="Transaction field to map to") format: List[str] | str coerce_to: Literal["date"] = Field("date", frozen=True) + required: bool = Field(True, frozen=True) class TransactionReferenceDateMapping(ColumnMapping): @@ -128,6 +127,7 @@ class TransactionReferenceDateMapping(ColumnMapping): class TransactionAmountMapping(ColumnMapping): target: Literal["amount"] = Field(..., description="Transaction field to map to") coerce_to: Literal["positive_decimal"] = Field("positive_decimal", frozen=True) + required: bool = Field(True, frozen=True) class TransactionDescriptionMapping(ColumnMapping): @@ -144,6 +144,7 @@ class TransactionNotesMapping(ColumnMapping): class TransactionTagsMapping(ColumnMapping): target: Literal["tags"] = Field(..., description="Transaction field to map to") + type: Literal["id", "name"] = "name" create: bool = Field( default=True, description="Create new tags if they doesn't exist" ) @@ -218,7 +219,7 @@ class EntityNameMapping(ColumnMapping): class EntityActiveMapping(ColumnMapping): - target: Literal["entitiy_active"] = Field(..., description="Entity field to map to") + target: Literal["entity_active"] = Field(..., description="Entity field to map to") coerce_to: Literal["bool"] = Field("bool", frozen=True) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index abda751..d49ef9a 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -200,14 +200,14 @@ class ImportService: # self.import_run.acc.add(category) if "tags" in data: - tag_names = data.pop("tags").split(",") + tag_names = data.pop("tags") for tag_name in tag_names: tag, _ = TransactionTag.objects.get_or_create(name=tag_name.strip()) tags.append(tag) self.import_run.tags.add(tag) if "entities" in data: - entity_names = data.pop("entities").split(",") + entity_names = data.pop("entities") for entity_name in entity_names: entity, _ = TransactionEntity.objects.get_or_create( name=entity_name.strip() From 1c713fac198860569ac3fdd6713dcc7b33d67c57 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 21:07:48 -0300 Subject: [PATCH 48/60] feat(import): add Nuconta preset --- app/import_presets/nuconta/config.yml | 54 ++++++++++++++++++++++++ app/import_presets/nuconta/manifest.json | 7 +++ 2 files changed, 61 insertions(+) create mode 100644 app/import_presets/nuconta/config.yml create mode 100644 app/import_presets/nuconta/manifest.json diff --git a/app/import_presets/nuconta/config.yml b/app/import_presets/nuconta/config.yml new file mode 100644 index 0000000..20fbdc3 --- /dev/null +++ b/app/import_presets/nuconta/config.yml @@ -0,0 +1,54 @@ +settings: + file_type: csv + delimiter: "," + encoding: utf-8 + skip_lines: 0 + importing: transactions + trigger_transaction_rules: true + skip_errors: true + +mapping: + account: + target: account + default: + type: name + + date: + target: date + source: Data + format: "%d/%m/%Y" + + amount: + target: amount + source: Valor + + description: + target: description + source: Descrição + transformations: + - type: split + separator: " - " + index: 0 + + type: + source: "Valor" + target: "type" + detection_method: sign + + notes: + target: notes + source: Notes + + internal_id: + target: internal_id + source: Identificador + + is_paid: + target: is_paid + detection_method: always_paid + +deduplicate: + - type: compare + fields: + - internal_id + match_type: lax diff --git a/app/import_presets/nuconta/manifest.json b/app/import_presets/nuconta/manifest.json new file mode 100644 index 0000000..a9d025f --- /dev/null +++ b/app/import_presets/nuconta/manifest.json @@ -0,0 +1,7 @@ +{ + "author": "eitchtee", + "description": "Importe suas transações da conta corrente do Nubank", + "schema_version": 1, + "name": "Nubank - Conta Corrente", + "message": "Mude '' para o nome da sua Nuconta dentro do WYGIWYH" +} From 1c28dd551359822ef6c390a604dc29ecef89baf8 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 21:08:03 -0300 Subject: [PATCH 49/60] feat(import): show error if YAML is invalid --- app/apps/import_app/models.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/apps/import_app/models.py b/app/apps/import_app/models.py index c0224d8..170b431 100644 --- a/app/apps/import_app/models.py +++ b/app/apps/import_app/models.py @@ -31,7 +31,9 @@ class ImportProfile(models.Model): yaml_data = yaml.safe_load(self.yaml_config) version_1.ImportProfileSchema(**yaml_data) except Exception as e: - raise ValidationError({"yaml_config": _("Invalid YAML Configuration")}) + raise ValidationError( + {"yaml_config": _("Invalid YAML Configuration: ") + str(e)} + ) class ImportRun(models.Model): @@ -79,9 +81,3 @@ class ImportRun(models.Model): failed_rows = models.IntegerField(default=0) started_at = models.DateTimeField(null=True) finished_at = models.DateTimeField(null=True) - - @property - def progress(self): - if self.total_rows == 0: - return 0 - return (self.processed_rows / self.total_rows) * 100 From e4a2b83c8319ff4314e6ff896b8ea50f08dd3872 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 21:08:12 -0300 Subject: [PATCH 50/60] feat: add new envs --- app/WYGIWYH/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 8d97c10..b937e52 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -378,5 +378,5 @@ PWA_APP_SCREENSHOTS = [ }, ] -ENABLE_SOFT_DELETE = os.getenv("ENABLE_SOFT_DELETION", "false").lower() == "true" +ENABLE_SOFT_DELETE = os.getenv("ENABLE_SOFT_DELETE", "false").lower() == "true" KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365")) From d0172b5524a6541e54f1c3136b858a2fb67e89bd Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 21:09:21 -0300 Subject: [PATCH 51/60] feat(import): convert deduplicate fields field into list --- app/apps/import_app/schemas/v1.py | 6 ++---- app/apps/import_app/services/v1.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index f5710f6..07bb5e6 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -4,10 +4,8 @@ from pydantic import BaseModel, Field, model_validator, field_validator class CompareDeduplicationRule(BaseModel): type: Literal["compare"] - fields: Dict = Field( - ..., description="Match header and fields to compare for deduplication" - ) - match_type: Literal["lax", "strict"] + fields: list[str] = Field(..., description="Compare fields for deduplication") + match_type: Literal["lax", "strict"] = "lax" @field_validator("fields", mode="before") def coerce_fields_to_dict(cls, v): diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index d49ef9a..e19886b 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -285,7 +285,7 @@ class ImportService: query = Transaction.all_objects.all().values("id") # Build query conditions for each field in the rule - for field, header in rule.fields.items(): + for field in rule.fields: if field in transaction_data: if rule.match_type == "strict": query = query.filter(**{field: transaction_data[field]}) From 928ad331115cd5057b5d32560d9c2c82ffe606c8 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 21:09:53 -0300 Subject: [PATCH 52/60] feat(import): move required field check to end of process --- app/apps/import_app/services/v1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index e19886b..a992630 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -406,15 +406,15 @@ class ImportService: if value is None: value = mapping.default - if mapping.required and value is None and not mapping.transformations: - raise ValueError(f"Required field {field} is missing") - # Apply transformations if mapping.transformations: value = self._transform_value(value, mapping, row) value = self._coerce_type(value, mapping) + if mapping.required and value is None: + raise ValueError(f"Required field {field} is missing") + if value is not None: # Remove the prefix from the target field target = mapping.target From 38379ab2b188aed7e2510d33f5b8ea2a89b570ff Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 21:12:13 -0300 Subject: [PATCH 53/60] feat(import): try to be more aggressive on cache invalidation --- app/apps/import_app/services/v1.py | 4 +--- app/apps/import_app/tasks.py | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index a992630..b431f2b 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -542,6 +542,4 @@ class ImportService: self.import_run.finished_at = timezone.now() self.import_run.save(update_fields=["finished_at"]) - - if self.import_run.successful_rows >= 1: - cachalot.api.invalidate() + cachalot.api.invalidate() diff --git a/app/apps/import_app/tasks.py b/app/apps/import_app/tasks.py index cf6f3a7..44d63b4 100644 --- a/app/apps/import_app/tasks.py +++ b/app/apps/import_app/tasks.py @@ -1,5 +1,6 @@ import logging +import cachalot.api from procrastinate.contrib.django import app from apps.import_app.models import ImportRun @@ -14,5 +15,7 @@ def process_import(import_run_id: int, file_path: str): import_run = ImportRun.objects.get(id=import_run_id) import_service = ImportServiceV1(import_run) import_service.process_file(file_path) + cachalot.api.invalidate() except ImportRun.DoesNotExist: + cachalot.api.invalidate() raise ValueError(f"ImportRun with id {import_run_id} not found") From 93d04572df40d5f2aebc32372e5f9930ebff1115 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 22:02:45 -0300 Subject: [PATCH 54/60] feat(accounts): make account names unique --- app/apps/accounts/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/apps/accounts/models.py b/app/apps/accounts/models.py index 7e0c824..eed0cd5 100644 --- a/app/apps/accounts/models.py +++ b/app/apps/accounts/models.py @@ -18,7 +18,7 @@ class AccountGroup(models.Model): class Account(models.Model): - name = models.CharField(max_length=255, verbose_name=_("Name")) + name = models.CharField(max_length=255, verbose_name=_("Name"), unique=True) group = models.ForeignKey( AccountGroup, on_delete=models.SET_NULL, From ba2d654f158e17e8d3de074a78cab42515c54b09 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 22:03:02 -0300 Subject: [PATCH 55/60] feat(accounts): make account names unique --- .../0007_make_account_names_unique.py | 38 +++++++++++++++++++ .../migrations/0008_alter_account_name.py | 18 +++++++++ 2 files changed, 56 insertions(+) create mode 100644 app/apps/accounts/migrations/0007_make_account_names_unique.py create mode 100644 app/apps/accounts/migrations/0008_alter_account_name.py diff --git a/app/apps/accounts/migrations/0007_make_account_names_unique.py b/app/apps/accounts/migrations/0007_make_account_names_unique.py new file mode 100644 index 0000000..e570246 --- /dev/null +++ b/app/apps/accounts/migrations/0007_make_account_names_unique.py @@ -0,0 +1,38 @@ +from django.db import migrations, models + + +def make_names_unique(apps, schema_editor): + Account = apps.get_model("accounts", "Account") + + # Get all accounts ordered by id + accounts = Account.objects.all().order_by("id") + + # Track seen names + seen_names = {} + + for account in accounts: + original_name = account.name + counter = seen_names.get(original_name, 0) + + while account.name in seen_names: + counter += 1 + account.name = f"{original_name} ({counter})" + + seen_names[account.name] = counter + account.save() + + +def reverse_migration(apps, schema_editor): + # Can't restore original names, so do nothing + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("accounts", "0006_rename_archived_account_is_archived_and_more"), + ] + + operations = [ + migrations.RunPython(make_names_unique, reverse_migration), + ] diff --git a/app/apps/accounts/migrations/0008_alter_account_name.py b/app/apps/accounts/migrations/0008_alter_account_name.py new file mode 100644 index 0000000..a6a5cfc --- /dev/null +++ b/app/apps/accounts/migrations/0008_alter_account_name.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.5 on 2025-01-24 00:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0007_make_account_names_unique'), + ] + + operations = [ + migrations.AlterField( + model_name='account', + name='name', + field=models.CharField(max_length=255, unique=True, verbose_name='Name'), + ), + ] From 1dc03b0a848985e04bcd2cd63de6fdbafedeb469 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 22:48:23 -0300 Subject: [PATCH 56/60] feat(import:v1:service): respect create and type fields --- app/apps/import_app/services/v1.py | 159 ++++++++++++++++++++++------- 1 file changed, 122 insertions(+), 37 deletions(-) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index b431f2b..dcd5e75 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -23,6 +23,12 @@ from apps.transactions.models import ( TransactionEntity, ) from apps.rules.signals import transaction_created +from apps.import_app.schemas.v1 import ( + TransactionCategoryMapping, + TransactionAccountMapping, + TransactionTagsMapping, + TransactionEntitiesMapping, +) logger = logging.getLogger(__name__) @@ -184,40 +190,127 @@ class ImportService: entities = [] # Handle related objects first if "category" in data: - category_name = data.pop("category") - category, _ = TransactionCategory.objects.get_or_create(name=category_name) - data["category"] = category - self.import_run.categories.add(category) + if "category" in data: + category_name = data.pop("category") + category_mapping = next( + ( + m + for m in self.mapping.values() + if isinstance(m, TransactionCategoryMapping) + and m.target == "category" + ), + None, + ) + print(category_mapping) + + try: + if category_mapping: + if category_mapping.type == "id": + category = TransactionCategory.objects.get(id=category_name) + else: # name + if getattr(category_mapping, "create", False): + category, _ = TransactionCategory.objects.get_or_create( + name=category_name + ) + else: + category = TransactionCategory.objects.filter( + name=category_name + ).first() + + if category: + data["category"] = category + self.import_run.categories.add(category) + except (TransactionCategory.DoesNotExist, ValueError): + # Ignore if category doesn't exist and create is False or not set + data["category"] = None if "account" in data: account_id = data.pop("account") - account = None - if isinstance(account_id, str): - account = Account.objects.get(name=account_id) - elif isinstance(account_id, int): - account = Account.objects.get(id=account_id) - data["account"] = account - # self.import_run.acc.add(category) + account_mapping = next( + ( + m + for m in self.mapping.values() + if isinstance(m, TransactionAccountMapping) + and m.target == "account" + ), + None, + ) + + try: + if account_mapping and account_mapping.type == "id": + account = Account.objects.filter(id=account_id).first() + else: # name + account = Account.objects.filter(name=account_id).first() + + if account: + data["account"] = account + except ValueError: + # Ignore if account doesn't exist + pass if "tags" in data: tag_names = data.pop("tags") + tags_mapping = next( + ( + m + for m in self.mapping.values() + if isinstance(m, TransactionTagsMapping) and m.target == "tags" + ), + None, + ) + for tag_name in tag_names: - tag, _ = TransactionTag.objects.get_or_create(name=tag_name.strip()) - tags.append(tag) - self.import_run.tags.add(tag) + try: + if tags_mapping: + if tags_mapping.type == "id": + tag = TransactionTag.objects.filter(id=tag_name).first() + else: # name + if getattr(tags_mapping, "create", False): + tag, _ = TransactionTag.objects.get_or_create( + name=tag_name.strip() + ) + else: + tag = TransactionTag.objects.filter( + name=tag_name.strip() + ).first() + + if tag: + tags.append(tag) + self.import_run.tags.add(tag) + except ValueError: + # Ignore if tag doesn't exist and create is False or not set + continue if "entities" in data: entity_names = data.pop("entities") - for entity_name in entity_names: - entity, _ = TransactionEntity.objects.get_or_create( - name=entity_name.strip() - ) - entities.append(entity) - self.import_run.entities.add(entity) + entities_mapping = next( + ( + m + for m in self.mapping.values() + if isinstance(m, TransactionEntitiesMapping) + and m.target == "entities" + ), + None, + ) - if "amount" in data: - amount = data.pop("amount") - data["amount"] = abs(Decimal(amount)) + for entity_name in entity_names: + try: + if entities_mapping: + if getattr(entities_mapping, "create", False): + entity, _ = TransactionEntity.objects.get_or_create( + name=entity_name.strip() + ) + else: + entity = TransactionEntity.objects.filter( + name=entity_name.strip() + ).first() + + if entity: + entities.append(entity) + self.import_run.entities.add(entity) + except ValueError: + # Ignore if entity doesn't exist and create is False or not set + continue # Create the transaction new_transaction = Transaction.objects.create(**data) @@ -308,31 +401,23 @@ class ImportService: coerce_to = mapping.coerce_to - if "|" in coerce_to: - types = coerce_to.split("|") - for t in types: - try: - return self._coerce_single_type(value, t, mapping) - except ValueError: - continue - raise ValueError( - f"Could not coerce '{value}' to any of the types: {coerce_to}" - ) - else: - return self._coerce_single_type(value, coerce_to, mapping) + return self._coerce_single_type(value, coerce_to, mapping) + @staticmethod def _coerce_single_type( - self, value: str, coerce_to: str, mapping: version_1.ColumnMapping + value: str, coerce_to: str, mapping: version_1.ColumnMapping ) -> Union[str, int, bool, Decimal, datetime.date, list]: if coerce_to == "str": return str(value) elif coerce_to == "int": + return int(value) + elif coerce_to == "str|int": if hasattr(mapping, "type") and mapping.type == "id": return int(value) elif hasattr(mapping, "type") and mapping.type in ["name", "code"]: return str(value) else: - return int(value) + return str(value) elif coerce_to == "bool": return value.lower() in ["true", "1", "yes", "y", "on"] elif coerce_to == "positive_decimal": From 53175aacb91c2fdf522428c19a8065d929ef5ed7 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 22:49:09 -0300 Subject: [PATCH 57/60] feat(import:templates): change wrong name --- app/templates/import_app/fragments/profiles/list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templates/import_app/fragments/profiles/list.html b/app/templates/import_app/fragments/profiles/list.html index cdc9a83..9c973b2 100644 --- a/app/templates/import_app/fragments/profiles/list.html +++ b/app/templates/import_app/fragments/profiles/list.html @@ -83,7 +83,7 @@ {% else %} - + {% endif %} From f2d32fd7e927d9f5639799613cffb1d34fbff11a Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Thu, 23 Jan 2025 23:52:54 -0300 Subject: [PATCH 58/60] feat(import): final changes for release --- app/apps/import_app/schemas/v1.py | 9 ++------- app/apps/import_app/services/v1.py | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 07bb5e6..01ae643 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -7,12 +7,6 @@ class CompareDeduplicationRule(BaseModel): fields: list[str] = Field(..., description="Compare fields for deduplication") match_type: Literal["lax", "strict"] = "lax" - @field_validator("fields", mode="before") - def coerce_fields_to_dict(cls, v): - if isinstance(v, list): - return {k: v for d in v for k, v in d.items()} - return v - class ReplaceTransformationRule(BaseModel): type: Literal["replace", "regex"] = Field( @@ -103,7 +97,7 @@ class TransactionTypeMapping(ColumnMapping): class TransactionIsPaidMapping(ColumnMapping): target: Literal["is_paid"] = Field(..., description="Transaction field to map to") - detection_method: Literal["sign", "boolean", "always_paid", "always_unpaid"] + detection_method: Literal["boolean", "always_paid", "always_unpaid"] coerce_to: Literal["is_paid"] = Field("is_paid", frozen=True) @@ -151,6 +145,7 @@ class TransactionTagsMapping(ColumnMapping): class TransactionEntitiesMapping(ColumnMapping): target: Literal["entities"] = Field(..., description="Transaction field to map to") + type: Literal["id", "name"] = "name" create: bool = Field( default=True, description="Create new entities if they doesn't exist" ) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index dcd5e75..4e8d8c0 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -296,14 +296,19 @@ class ImportService: for entity_name in entity_names: try: if entities_mapping: - if getattr(entities_mapping, "create", False): - entity, _ = TransactionEntity.objects.get_or_create( - name=entity_name.strip() - ) - else: - entity = TransactionEntity.objects.filter( - name=entity_name.strip() + if entities_mapping.type == "id": + entity = TransactionTag.objects.filter( + id=entity_name ).first() + else: # name + if getattr(entities_mapping, "create", False): + entity, _ = TransactionEntity.objects.get_or_create( + name=entity_name.strip() + ) + else: + entity = TransactionEntity.objects.filter( + name=entity_name.strip() + ).first() if entity: entities.append(entity) @@ -468,9 +473,7 @@ class ImportService: raise ValueError("Invalid transaction type detection method") elif coerce_to == "is_paid": if isinstance(mapping, version_1.TransactionIsPaidMapping): - if mapping.detection_method == "sign": - return not value.startswith("-") - elif mapping.detection_method == "boolean": + if mapping.detection_method == "boolean": return value.lower() in ["true", "1", "yes", "y", "on"] elif mapping.detection_method == "always_paid": return True From d50c84f8e693381ca3f9af5b9e9cadb9cd995a52 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Fri, 24 Jan 2025 00:36:33 -0300 Subject: [PATCH 59/60] refactor: remove debug prints --- app/apps/import_app/services/presets.py | 2 +- app/apps/import_app/services/v1.py | 1 - app/apps/import_app/views.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/apps/import_app/services/presets.py b/app/apps/import_app/services/presets.py index 15e7ac1..824a246 100644 --- a/app/apps/import_app/services/presets.py +++ b/app/apps/import_app/services/presets.py @@ -38,7 +38,7 @@ class PresetService: preset["schema_version"] ) # Check if schema version is valid except Exception as e: - print(e) + pass else: presets.append(preset) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 4e8d8c0..d84935e 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -201,7 +201,6 @@ class ImportService: ), None, ) - print(category_mapping) try: if category_mapping: diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index 720a5e1..1069eca 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -33,7 +33,6 @@ def import_view(request): @require_http_methods(["GET"]) def import_presets_list(request): presets = PresetService.get_all_presets() - print(presets) return render( request, "import_app/fragments/profiles/list_presets.html", @@ -83,7 +82,6 @@ def import_profile_add(request): }, ) else: - print(int(request.GET.get("version", 1))) form = ImportProfileForm( initial={ "name": request.GET.get("name"), From dbea78cd3c2470a9e3120869ea3bbe1af6ceb510 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Fri, 24 Jan 2025 14:22:30 -0300 Subject: [PATCH 60/60] feat(pwa): better offline page and offline request handler --- app/WYGIWYH/settings.py | 1 + app/templates/offline.html | 79 ++++++++++++++++++++++++++++ app/templates/pwa/serviceworker.js | 74 ++++++++++++++++++++++++++ frontend/src/styles/_animations.scss | 35 +++++++++--- frontend/src/styles/style.scss | 24 +++++++++ 5 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 app/templates/offline.html create mode 100644 app/templates/pwa/serviceworker.js diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index b937e52..9b0c7a8 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -377,6 +377,7 @@ PWA_APP_SCREENSHOTS = [ "type": "image/png", }, ] +PWA_SERVICE_WORKER_PATH = BASE_DIR / "templates" / "pwa" / "serviceworker.js" ENABLE_SOFT_DELETE = os.getenv("ENABLE_SOFT_DELETE", "false").lower() == "true" KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365")) diff --git a/app/templates/offline.html b/app/templates/offline.html new file mode 100644 index 0000000..4e738ac --- /dev/null +++ b/app/templates/offline.html @@ -0,0 +1,79 @@ + + + + + + Offline + + + +
    + + + + +

    Either you or your WYGIWYH instance is offline.

    +
    +
    + + + + diff --git a/app/templates/pwa/serviceworker.js b/app/templates/pwa/serviceworker.js new file mode 100644 index 0000000..3dfdfba --- /dev/null +++ b/app/templates/pwa/serviceworker.js @@ -0,0 +1,74 @@ +// Base Service Worker implementation. To use your own Service Worker, set the PWA_SERVICE_WORKER_PATH variable in settings.py + +var staticCacheName = "django-pwa-v" + new Date().getTime(); +var filesToCache = [ + '/offline/', + '/static/css/django-pwa-app.css', + '/static/img/favicon/android-icon-192x192.png', + '/static/img/favicon/apple-icon-180x180.png', + '/static/img/pwa/splash-640x1136.png', + '/static/img/pwa/splash-750x1334.png', +]; + +// Cache on install +self.addEventListener("install", event => { + this.skipWaiting(); + event.waitUntil( + caches.open(staticCacheName) + .then(cache => { + return cache.addAll(filesToCache); + }) + ); +}); + +// Clear cache on activate +self.addEventListener('activate', event => { + event.waitUntil( + caches.keys().then(cacheNames => { + return Promise.all( + cacheNames + .filter(cacheName => (cacheName.startsWith("django-pwa-"))) + .filter(cacheName => (cacheName !== staticCacheName)) + .map(cacheName => caches.delete(cacheName)) + ); + }) + ); +}); + +// Serve from Cache +self.addEventListener("fetch", event => { + event.respondWith( + caches.match(event.request) + .then(response => { + if (response) { + return response; + } + return fetch(event.request).catch(() => { + const isHtmxRequest = event.request.headers.get('HX-Request') === 'true'; + const isHtmxBoosted = event.request.headers.get('HX-Boosted') === 'true'; + + if (!isHtmxRequest || isHtmxBoosted) { + // Serve offline content without changing URL + return caches.match('/offline/').then(offlineResponse => { + if (offlineResponse) { + return offlineResponse.text().then(offlineText => { + return new Response(offlineText, { + status: 200, + headers: {'Content-Type': 'text/html'} + }); + }); + } + // If offline page is not in cache, return a simple offline message + return new Response('

    Offline

    The page is not available offline.

    ', { + status: 200, + headers: {'Content-Type': 'text/html'} + }); + }); + } else { + // For non-boosted HTMX requests, let it fail normally + throw new Error('Network request failed'); + } + }); + }) + ); +}); diff --git a/frontend/src/styles/_animations.scss b/frontend/src/styles/_animations.scss index 765a921..5b9c135 100644 --- a/frontend/src/styles/_animations.scss +++ b/frontend/src/styles/_animations.scss @@ -58,13 +58,21 @@ // HTMX Loading @keyframes spin { - 0% { transform: rotate(0deg); } - 100% { transform: rotate(360deg); } + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } } @keyframes fade-in { - 0% { opacity: 0; } - 100% { opacity: 1; } + 0% { + opacity: 0; + } + 100% { + opacity: 1; + } } .show-loading.htmx-request { @@ -103,7 +111,7 @@ } .swing-out-top-bck { - animation: swing-out-top-bck 0.45s cubic-bezier(0.600, -0.280, 0.735, 0.045) both; + animation: swing-out-top-bck 0.45s cubic-bezier(0.600, -0.280, 0.735, 0.045) both; } /* ---------------------------------------------- @@ -155,7 +163,7 @@ } .scale-in-center { - animation: scale-in-center 0.5s cubic-bezier(0.250, 0.460, 0.450, 0.940) both; + animation: scale-in-center 0.5s cubic-bezier(0.250, 0.460, 0.450, 0.940) both; } /* ---------------------------------------------- @@ -182,5 +190,18 @@ } .scale-out-center { - animation: scale-out-center 0.5s cubic-bezier(0.550, 0.085, 0.680, 0.530) both; + animation: scale-out-center 0.5s cubic-bezier(0.550, 0.085, 0.680, 0.530) both; +} + +@keyframes flash { + 0%, 100% { + opacity: 1; + } + 50% { + opacity: 0.3; + } +} + +.flashing { + animation: flash 1s infinite; } diff --git a/frontend/src/styles/style.scss b/frontend/src/styles/style.scss index 1bb7f8a..7fc980b 100644 --- a/frontend/src/styles/style.scss +++ b/frontend/src/styles/style.scss @@ -53,3 +53,27 @@ select[multiple] { .transaction:has(input[type="checkbox"]:checked) > .transaction-item { background-color: $primary-bg-subtle-dark; } + +.offline { + text-align: center; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + height: 100vh; + background-color: #222; + color: #fbb700; + font-family: Arial, sans-serif; +} + +.wifi-icon { + width: 100px; + height: 100px; +} + +#offline-countdown { + margin-top: 20px; + font-size: 14px; +}