Merge pull request #58

feat: beta import function

.env.example

@@ -18,3 +18,9 @@ SQL_PORT=5432
 
 # Gunicorn
 WEB_CONCURRENCY=4
+
+# App Configs
+# Enable this if you want to keep deleted transactions in the database
+ENABLE_SOFT_DELETE=false
+# If ENABLE_SOFT_DELETE is true, transactions deleted for more than KEEP_DELETED_TRANSACTIONS_FOR days will be truly deleted. Set to 0 to keep all.
+KEEP_DELETED_TRANSACTIONS_FOR=365

app/WYGIWYH/settings.py

@@ -64,6 +64,7 @@ INSTALLED_APPS = [
     "apps.accounts.apps.AccountsConfig",
     "apps.common.apps.CommonConfig",
     "apps.net_worth.apps.NetWorthConfig",
+    "apps.import_app.apps.ImportConfig",
     "apps.api.apps.ApiConfig",
     "cachalot",
     "rest_framework",
@@ -376,3 +377,6 @@ PWA_APP_SCREENSHOTS = [
         "type": "image/png",
     },
 ]
+
+ENABLE_SOFT_DELETE = os.getenv("ENABLE_SOFT_DELETION", "false").lower() == "true"
+KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365"))

app/WYGIWYH/urls.py

@@ -48,4 +48,5 @@ urlpatterns = [
     path("", include("apps.calendar_view.urls")),
     path("", include("apps.dca.urls")),
     path("", include("apps.mini_tools.urls")),
+    path("", include("apps.import_app.urls")),
 ]

app/apps/common/templatetags/json.py

@@ -0,0 +1,11 @@
+import json
+
+from django import template
+
+
+register = template.Library()
+
+
+@register.filter("json")
+def convert_to_json(value):
+    return json.dumps(value)

app/apps/import_app/admin.py

@@ -0,0 +1,6 @@
+from django.contrib import admin
+from apps.import_app import models
+
+# Register your models here.
+admin.site.register(models.ImportRun)
+admin.site.register(models.ImportProfile)

app/apps/import_app/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ImportConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "apps.import_app"

app/apps/import_app/forms.py

@@ -0,0 +1,64 @@
+from crispy_forms.bootstrap import FormActions
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import (
+    Layout,
+)
+from django import forms
+from django.utils.translation import gettext_lazy as _
+
+from apps.import_app.models import ImportProfile
+from apps.common.widgets.crispy.submit import NoClassSubmit
+
+
+class ImportProfileForm(forms.ModelForm):
+    class Meta:
+        model = ImportProfile
+        fields = [
+            "name",
+            "version",
+            "yaml_config",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout("name", "version", "yaml_config")
+
+        if self.instance and self.instance.pk:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Update"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+        else:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Add"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+
+
+class ImportRunFileUploadForm(forms.Form):
+    file = forms.FileField(label=_("Select a file"))
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout(
+            "file",
+            FormActions(
+                NoClassSubmit(
+                    "submit", _("Import"), css_class="btn btn-outline-primary w-100"
+                ),
+            ),
+        )

app/apps/import_app/migrations/0001_initial.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.1.5 on 2025-01-19 00:44
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('currencies', '0006_currency_exchange_currency'),
+        ('transactions', '0028_transaction_internal_note'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImportProfile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+                ('yaml_config', models.TextField(help_text='YAML configuration')),
+                ('version', models.IntegerField(choices=[(1, 'Version 1')], default=1, verbose_name='Version')),
+            ],
+            options={
+                'ordering': ['name'],
+            },
+        ),
+        migrations.CreateModel(
+            name='ImportRun',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('QUEUED', 'Queued'), ('PROCESSING', 'Processing'), ('FAILED', 'Failed'), ('FINISHED', 'Finished')], default='QUEUED', max_length=10, verbose_name='Status')),
+                ('file_name', models.CharField(help_text='File name', max_length=10000)),
+                ('logs', models.TextField(blank=True)),
+                ('processed_rows', models.IntegerField(default=0)),
+                ('total_rows', models.IntegerField(default=0)),
+                ('successful_rows', models.IntegerField(default=0)),
+                ('skipped_rows', models.IntegerField(default=0)),
+                ('failed_rows', models.IntegerField(default=0)),
+                ('started_at', models.DateTimeField(null=True)),
+                ('finished_at', models.DateTimeField(null=True)),
+                ('categories', models.ManyToManyField(related_name='import_runs', to='transactions.transactioncategory')),
+                ('currencies', models.ManyToManyField(related_name='import_runs', to='currencies.currency')),
+                ('entities', models.ManyToManyField(related_name='import_runs', to='transactions.transactionentity')),
+                ('profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='import_app.importprofile')),
+                ('tags', models.ManyToManyField(related_name='import_runs', to='transactions.transactiontag')),
+                ('transactions', models.ManyToManyField(related_name='import_runs', to='transactions.transaction')),
+            ],
+        ),
+    ]

app/apps/import_app/migrations/0002_alter_importprofile_name_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.5 on 2025-01-23 03:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('import_app', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='importprofile',
+            name='name',
+            field=models.CharField(max_length=100, unique=True, verbose_name='Name'),
+        ),
+        migrations.AlterField(
+            model_name='importprofile',
+            name='yaml_config',
+            field=models.TextField(verbose_name='YAML Configuration'),
+        ),
+    ]

app/apps/import_app/models.py

@@ -0,0 +1,87 @@
+import yaml
+
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from apps.import_app.schemas import version_1
+
+
+class ImportProfile(models.Model):
+    class Versions(models.IntegerChoices):
+        VERSION_1 = 1, _("Version") + " 1"
+
+    name = models.CharField(max_length=100, verbose_name=_("Name"), unique=True)
+    yaml_config = models.TextField(verbose_name=_("YAML Configuration"))
+    version = models.IntegerField(
+        choices=Versions,
+        default=Versions.VERSION_1,
+        verbose_name=_("Version"),
+    )
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        ordering = ["name"]
+
+    def clean(self):
+        if self.version and self.version == self.Versions.VERSION_1:
+            try:
+                yaml_data = yaml.safe_load(self.yaml_config)
+                version_1.ImportProfileSchema(**yaml_data)
+            except Exception as e:
+                raise ValidationError({"yaml_config": _("Invalid YAML Configuration")})
+
+
+class ImportRun(models.Model):
+    class Status(models.TextChoices):
+        QUEUED = "QUEUED", _("Queued")
+        PROCESSING = "PROCESSING", _("Processing")
+        FAILED = "FAILED", _("Failed")
+        FINISHED = "FINISHED", _("Finished")
+
+    status = models.CharField(
+        max_length=10,
+        choices=Status,
+        default=Status.QUEUED,
+        verbose_name=_("Status"),
+    )
+    profile = models.ForeignKey(
+        ImportProfile,
+        on_delete=models.CASCADE,
+    )
+    file_name = models.CharField(
+        max_length=10000,
+        help_text=_("File name"),
+    )
+    transactions = models.ManyToManyField(
+        "transactions.Transaction", related_name="import_runs"
+    )
+    tags = models.ManyToManyField(
+        "transactions.TransactionTag", related_name="import_runs"
+    )
+    categories = models.ManyToManyField(
+        "transactions.TransactionCategory", related_name="import_runs"
+    )
+    entities = models.ManyToManyField(
+        "transactions.TransactionEntity", related_name="import_runs"
+    )
+    currencies = models.ManyToManyField(
+        "currencies.Currency", related_name="import_runs"
+    )
+
+    logs = models.TextField(blank=True)
+    processed_rows = models.IntegerField(default=0)
+    total_rows = models.IntegerField(default=0)
+    successful_rows = models.IntegerField(default=0)
+    skipped_rows = models.IntegerField(default=0)
+    failed_rows = models.IntegerField(default=0)
+    started_at = models.DateTimeField(null=True)
+    finished_at = models.DateTimeField(null=True)
+
+    @property
+    def progress(self):
+        if self.total_rows == 0:
+            return 0
+        return (self.processed_rows / self.total_rows) * 100

app/apps/import_app/schemas/__init__.py

@@ -0,0 +1 @@
+import apps.import_app.schemas.v1 as version_1

app/apps/import_app/schemas/v1.py

@@ -0,0 +1,406 @@
+from typing import Dict, List, Optional, Literal
+from pydantic import BaseModel, Field, model_validator, field_validator
+
+
+class CompareDeduplicationRule(BaseModel):
+    type: Literal["compare"]
+    fields: Dict = Field(
+        ..., description="Match header and fields to compare for deduplication"
+    )
+    match_type: Literal["lax", "strict"]
+
+    @field_validator("fields", mode="before")
+    def coerce_fields_to_dict(cls, v):
+        if isinstance(v, list):
+            return {k: v for d in v for k, v in d.items()}
+        return v
+
+
+class ReplaceTransformationRule(BaseModel):
+    field: str
+    type: Literal["replace", "regex"] = Field(
+        ..., description="Type of transformation: replace or regex"
+    )
+    pattern: str = Field(..., description="Pattern to match")
+    replacement: str = Field(..., description="Value to replace with")
+    exclusive: bool = Field(
+        default=False,
+        description="If it should match against the last transformation or the original value",
+    )
+
+
+class DateFormatTransformationRule(BaseModel):
+    field: str
+    type: Literal["date_format"] = Field(
+        ..., description="Type of transformation: replace or regex"
+    )
+    original_format: str = Field(..., description="Original date format")
+    new_format: str = Field(..., description="New date format to use")
+
+
+class HashTransformationRule(BaseModel):
+    fields: List[str]
+    type: Literal["hash"]
+
+
+class MergeTransformationRule(BaseModel):
+    fields: List[str]
+    type: Literal["merge"]
+    separator: str = Field(default=" ", description="Separator to use when merging")
+
+
+class SplitTransformationRule(BaseModel):
+    fields: List[str]
+    type: Literal["split"]
+    separator: str = Field(default=",", description="Separator to use when splitting")
+    index: int | None = Field(
+        default=0, description="Index to return as value. Empty to return all."
+    )
+
+
+class CSVImportSettings(BaseModel):
+    skip_errors: bool = Field(
+        default=False,
+        description="If True, errors during import will be logged and skipped",
+    )
+    file_type: Literal["csv"] = "csv"
+    delimiter: str = Field(default=",", description="CSV delimiter character")
+    encoding: str = Field(default="utf-8", description="File encoding")
+    skip_lines: int = Field(
+        default=0, description="Number of rows to skip at the beginning of the file"
+    )
+    trigger_transaction_rules: bool = True
+    importing: Literal[
+        "transactions", "accounts", "currencies", "categories", "tags", "entities"
+    ]
+
+
+class ColumnMapping(BaseModel):
+    source: Optional[str] = Field(
+        default=None,
+        description="CSV column header. If None, the field will be generated from transformations",
+    )
+    default: Optional[str] = None
+    required: bool = False
+    transformations: Optional[
+        List[
+            ReplaceTransformationRule
+            | DateFormatTransformationRule
+            | HashTransformationRule
+            | MergeTransformationRule
+            | SplitTransformationRule
+        ]
+    ] = Field(default_factory=list)
+
+
+class TransactionAccountMapping(ColumnMapping):
+    target: Literal["account"] = Field(..., description="Transaction field to map to")
+    type: Literal["id", "name"] = "name"
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class TransactionTypeMapping(ColumnMapping):
+    target: Literal["type"] = Field(..., description="Transaction field to map to")
+    detection_method: Literal["sign", "always_income", "always_expense"] = "sign"
+    coerce_to: Literal["transaction_type"] = Field("transaction_type", frozen=True)
+
+
+class TransactionIsPaidMapping(ColumnMapping):
+    target: Literal["is_paid"] = Field(..., description="Transaction field to map to")
+    detection_method: Literal["sign", "boolean", "always_paid", "always_unpaid"]
+    coerce_to: Literal["is_paid"] = Field("is_paid", frozen=True)
+
+
+class TransactionDateMapping(ColumnMapping):
+    target: Literal["date"] = Field(..., description="Transaction field to map to")
+    format: List[str] | str
+    coerce_to: Literal["date"] = Field("date", frozen=True)
+
+
+class TransactionReferenceDateMapping(ColumnMapping):
+    target: Literal["reference_date"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    format: List[str] | str
+    coerce_to: Literal["date"] = Field("date", frozen=True)
+
+
+class TransactionAmountMapping(ColumnMapping):
+    target: Literal["amount"] = Field(..., description="Transaction field to map to")
+    coerce_to: Literal["positive_decimal"] = Field("positive_decimal", frozen=True)
+
+
+class TransactionDescriptionMapping(ColumnMapping):
+    target: Literal["description"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TransactionNotesMapping(ColumnMapping):
+    target: Literal["notes"] = Field(..., description="Transaction field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TransactionTagsMapping(ColumnMapping):
+    target: Literal["tags"] = Field(..., description="Transaction field to map to")
+    create: bool = Field(
+        default=True, description="Create new tags if they doesn't exist"
+    )
+    coerce_to: Literal["list"] = Field("list", frozen=True)
+
+
+class TransactionEntitiesMapping(ColumnMapping):
+    target: Literal["entities"] = Field(..., description="Transaction field to map to")
+    create: bool = Field(
+        default=True, description="Create new entities if they doesn't exist"
+    )
+    coerce_to: Literal["list"] = Field("list", frozen=True)
+
+
+class TransactionCategoryMapping(ColumnMapping):
+    target: Literal["category"] = Field(..., description="Transaction field to map to")
+    create: bool = Field(
+        default=True, description="Create category if it doesn't exist"
+    )
+    type: Literal["id", "name"] = "name"
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class TransactionInternalNoteMapping(ColumnMapping):
+    target: Literal["internal_note"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TransactionInternalIDMapping(ColumnMapping):
+    target: Literal["internal_id"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CategoryNameMapping(ColumnMapping):
+    target: Literal["category_name"] = Field(
+        ..., description="Category field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CategoryMuteMapping(ColumnMapping):
+    target: Literal["category_mute"] = Field(
+        ..., description="Category field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class CategoryActiveMapping(ColumnMapping):
+    target: Literal["category_active"] = Field(
+        ..., description="Category field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class TagNameMapping(ColumnMapping):
+    target: Literal["tag_name"] = Field(..., description="Tag field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TagActiveMapping(ColumnMapping):
+    target: Literal["tag_active"] = Field(..., description="Tag field to map to")
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class EntityNameMapping(ColumnMapping):
+    target: Literal["entity_name"] = Field(..., description="Entity field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class EntityActiveMapping(ColumnMapping):
+    target: Literal["entitiy_active"] = Field(..., description="Entity field to map to")
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class AccountNameMapping(ColumnMapping):
+    target: Literal["account_name"] = Field(..., description="Account field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class AccountGroupMapping(ColumnMapping):
+    target: Literal["account_group"] = Field(..., description="Account field to map to")
+    type: Literal["id", "name"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class AccountCurrencyMapping(ColumnMapping):
+    target: Literal["account_currency"] = Field(
+        ..., description="Account field to map to"
+    )
+    type: Literal["id", "name", "code"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class AccountExchangeCurrencyMapping(ColumnMapping):
+    target: Literal["account_exchange_currency"] = Field(
+        ..., description="Account field to map to"
+    )
+    type: Literal["id", "name", "code"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class AccountIsAssetMapping(ColumnMapping):
+    target: Literal["account_is_asset"] = Field(
+        ..., description="Account field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class AccountIsArchivedMapping(ColumnMapping):
+    target: Literal["account_is_archived"] = Field(
+        ..., description="Account field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class CurrencyCodeMapping(ColumnMapping):
+    target: Literal["currency_code"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencyNameMapping(ColumnMapping):
+    target: Literal["currency_name"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencyDecimalPlacesMapping(ColumnMapping):
+    target: Literal["currency_decimal_places"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["int"] = Field("int", frozen=True)
+
+
+class CurrencyPrefixMapping(ColumnMapping):
+    target: Literal["currency_prefix"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencySuffixMapping(ColumnMapping):
+    target: Literal["currency_suffix"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencyExchangeMapping(ColumnMapping):
+    target: Literal["currency_exchange"] = Field(
+        ..., description="Currency field to map to"
+    )
+    type: Literal["id", "name", "code"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class ImportProfileSchema(BaseModel):
+    settings: CSVImportSettings
+    mapping: Dict[
+        str,
+        TransactionAccountMapping
+        | TransactionTypeMapping
+        | TransactionIsPaidMapping
+        | TransactionDateMapping
+        | TransactionReferenceDateMapping
+        | TransactionAmountMapping
+        | TransactionDescriptionMapping
+        | TransactionNotesMapping
+        | TransactionTagsMapping
+        | TransactionEntitiesMapping
+        | TransactionCategoryMapping
+        | TransactionInternalNoteMapping
+        | TransactionInternalIDMapping
+        | CategoryNameMapping
+        | CategoryMuteMapping
+        | CategoryActiveMapping
+        | TagNameMapping
+        | TagActiveMapping
+        | EntityNameMapping
+        | EntityActiveMapping
+        | AccountNameMapping
+        | AccountGroupMapping
+        | AccountCurrencyMapping
+        | AccountExchangeCurrencyMapping
+        | AccountIsAssetMapping
+        | AccountIsArchivedMapping
+        | CurrencyCodeMapping
+        | CurrencyNameMapping
+        | CurrencyDecimalPlacesMapping
+        | CurrencyPrefixMapping
+        | CurrencySuffixMapping
+        | CurrencyExchangeMapping,
+    ]
+    deduplication: List[CompareDeduplicationRule] = Field(
+        default_factory=list,
+        description="Rules for deduplicating records during import",
+    )
+
+    @model_validator(mode="after")
+    def validate_mappings(self) -> "ImportProfileSchema":
+        import_type = self.settings.importing
+
+        # Define allowed mapping types for each import type
+        allowed_mappings = {
+            "transactions": (
+                TransactionAccountMapping,
+                TransactionTypeMapping,
+                TransactionIsPaidMapping,
+                TransactionDateMapping,
+                TransactionReferenceDateMapping,
+                TransactionAmountMapping,
+                TransactionDescriptionMapping,
+                TransactionNotesMapping,
+                TransactionTagsMapping,
+                TransactionEntitiesMapping,
+                TransactionCategoryMapping,
+                TransactionInternalNoteMapping,
+                TransactionInternalIDMapping,
+            ),
+            "accounts": (
+                AccountNameMapping,
+                AccountGroupMapping,
+                AccountCurrencyMapping,
+                AccountExchangeCurrencyMapping,
+                AccountIsAssetMapping,
+                AccountIsArchivedMapping,
+            ),
+            "currencies": (
+                CurrencyCodeMapping,
+                CurrencyNameMapping,
+                CurrencyDecimalPlacesMapping,
+                CurrencyPrefixMapping,
+                CurrencySuffixMapping,
+                CurrencyExchangeMapping,
+            ),
+            "categories": (
+                CategoryNameMapping,
+                CategoryMuteMapping,
+                CategoryActiveMapping,
+            ),
+            "tags": (TagNameMapping, TagActiveMapping),
+            "entities": (EntityNameMapping, EntityActiveMapping),
+        }
+
+        allowed_types = allowed_mappings[import_type]
+
+        for field_name, mapping in self.mapping.items():
+            if not isinstance(mapping, allowed_types):
+                raise ValueError(
+                    f"Mapping type '{type(mapping).__name__}' is not allowed when importing {import_type}. "
+                    f"Allowed types are: {', '.join(t.__name__ for t in allowed_types)}"
+                )
+
+        return self

app/apps/import_app/services/__init__.py

@@ -0,0 +1,3 @@
+from apps.import_app.services.v1 import ImportService as ImportServiceV1
+
+from apps.import_app.services.presets import PresetService

app/apps/import_app/services/presets.py

@@ -0,0 +1,45 @@
+import json
+from pathlib import Path
+
+from apps.import_app.models import ImportProfile
+
+
+class PresetService:
+    PRESET_PATH = "/usr/src/app/import_presets"
+
+    @classmethod
+    def get_all_presets(cls):
+        presets = []
+
+        for folder in Path(cls.PRESET_PATH).iterdir():
+            if folder.is_dir():
+                manifest_path = folder / "manifest.json"
+                config_path = folder / "config.yml"
+
+                if manifest_path.exists() and config_path.exists():
+                    with open(manifest_path) as f:
+                        manifest = json.load(f)
+
+                    with open(config_path) as f:
+                        config = json.dumps(f.read())
+
+                    try:
+                        preset = {
+                            "name": manifest.get("name", folder.name),
+                            "description": manifest.get("description", ""),
+                            "message": json.dumps(manifest.get("message", "")),
+                            "authors": manifest.get("author", "").split(","),
+                            "schema_version": (int(manifest.get("schema_version", 1))),
+                            "folder_name": folder.name,
+                            "config": config,
+                        }
+
+                        ImportProfile.Versions(
+                            preset["schema_version"]
+                        )  # Check if schema version is valid
+                    except Exception as e:
+                        print(e)
+                    else:
+                        presets.append(preset)
+
+        return presets

app/apps/import_app/services/v1.py

@@ -0,0 +1,547 @@
+import csv
+import hashlib
+import logging
+import os
+import re
+from datetime import datetime
+from decimal import Decimal
+from typing import Dict, Any, Literal, Union
+
+import cachalot.api
+import yaml
+from cachalot.api import cachalot_disabled
+from django.utils import timezone
+
+from apps.accounts.models import Account, AccountGroup
+from apps.currencies.models import Currency
+from apps.import_app.models import ImportRun, ImportProfile
+from apps.import_app.schemas import version_1
+from apps.transactions.models import (
+    Transaction,
+    TransactionCategory,
+    TransactionTag,
+    TransactionEntity,
+)
+from apps.rules.signals import transaction_created
+
+logger = logging.getLogger(__name__)
+
+
+class ImportService:
+    TEMP_DIR = "/usr/src/app/temp"
+
+    def __init__(self, import_run: ImportRun):
+        self.import_run: ImportRun = import_run
+        self.profile: ImportProfile = import_run.profile
+        self.config: version_1.ImportProfileSchema = self._load_config()
+        self.settings: version_1.CSVImportSettings = self.config.settings
+        self.deduplication: list[version_1.CompareDeduplicationRule] = (
+            self.config.deduplication
+        )
+        self.mapping: Dict[str, version_1.ColumnMapping] = self.config.mapping
+
+        # Ensure temp directory exists
+        os.makedirs(self.TEMP_DIR, exist_ok=True)
+
+    def _load_config(self) -> version_1.ImportProfileSchema:
+        yaml_data = yaml.safe_load(self.profile.yaml_config)
+        try:
+            config = version_1.ImportProfileSchema(**yaml_data)
+        except Exception as e:
+            self._log("error", f"Fatal error processing YAML config: {str(e)}")
+            self._update_status("FAILED")
+            raise e
+        else:
+            return config
+
+    def _log(self, level: str, message: str, **kwargs) -> None:
+        """Add a log entry to the import run logs"""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        # Format additional context if present
+        context = ""
+        if kwargs:
+            context = " - " + ", ".join(f"{k}={v}" for k, v in kwargs.items())
+
+        log_line = f"[{timestamp}] {level.upper()}: {message}{context}\n"
+
+        # Append to existing logs
+        self.import_run.logs += log_line
+        self.import_run.save(update_fields=["logs"])
+
+    def _update_totals(
+        self,
+        field: Literal["total", "processed", "successful", "skipped", "failed"],
+        value: int,
+    ) -> None:
+        if field == "total":
+            self.import_run.total_rows = value
+            self.import_run.save(update_fields=["total_rows"])
+        elif field == "processed":
+            self.import_run.processed_rows = value
+            self.import_run.save(update_fields=["processed_rows"])
+        elif field == "successful":
+            self.import_run.successful_rows = value
+            self.import_run.save(update_fields=["successful_rows"])
+        elif field == "skipped":
+            self.import_run.skipped_rows = value
+            self.import_run.save(update_fields=["skipped_rows"])
+        elif field == "failed":
+            self.import_run.failed_rows = value
+            self.import_run.save(update_fields=["failed_rows"])
+
+    def _increment_totals(
+        self,
+        field: Literal["total", "processed", "successful", "skipped", "failed"],
+        value: int,
+    ) -> None:
+        if field == "total":
+            self.import_run.total_rows = self.import_run.total_rows + value
+            self.import_run.save(update_fields=["total_rows"])
+        elif field == "processed":
+            self.import_run.processed_rows = self.import_run.processed_rows + value
+            self.import_run.save(update_fields=["processed_rows"])
+        elif field == "successful":
+            self.import_run.successful_rows = self.import_run.successful_rows + value
+            self.import_run.save(update_fields=["successful_rows"])
+        elif field == "skipped":
+            self.import_run.skipped_rows = self.import_run.skipped_rows + value
+            self.import_run.save(update_fields=["skipped_rows"])
+        elif field == "failed":
+            self.import_run.failed_rows = self.import_run.failed_rows + value
+            self.import_run.save(update_fields=["failed_rows"])
+
+    def _update_status(
+        self, new_status: Literal["PROCESSING", "FAILED", "FINISHED"]
+    ) -> None:
+        if new_status == "PROCESSING":
+            self.import_run.status = ImportRun.Status.PROCESSING
+        elif new_status == "FAILED":
+            self.import_run.status = ImportRun.Status.FAILED
+        elif new_status == "FINISHED":
+            self.import_run.status = ImportRun.Status.FINISHED
+
+        self.import_run.save(update_fields=["status"])
+
+    @staticmethod
+    def _transform_value(
+        value: str, mapping: version_1.ColumnMapping, row: Dict[str, str] = None
+    ) -> Any:
+        transformed = value
+
+        for transform in mapping.transformations:
+            if transform.type == "hash":
+                # Collect all values to be hashed
+                values_to_hash = []
+                for field in transform.fields:
+                    if field in row:
+                        values_to_hash.append(str(row[field]))
+
+                # Create hash from concatenated values
+                if values_to_hash:
+                    concatenated = "|".join(values_to_hash)
+                    transformed = hashlib.sha256(concatenated.encode()).hexdigest()
+
+            elif transform.type == "replace":
+                if transform.exclusive:
+                    transformed = value.replace(
+                        transform.pattern, transform.replacement
+                    )
+                else:
+                    transformed = transformed.replace(
+                        transform.pattern, transform.replacement
+                    )
+            elif transform.type == "regex":
+                if transform.exclusive:
+                    transformed = re.sub(
+                        transform.pattern, transform.replacement, value
+                    )
+                else:
+                    transformed = re.sub(
+                        transform.pattern, transform.replacement, transformed
+                    )
+            elif transform.type == "date_format":
+                transformed = datetime.strptime(
+                    transformed, transform.original_format
+                ).strftime(transform.new_format)
+            elif transform.type == "merge":
+                values_to_merge = []
+                for field in transform.fields:
+                    if field in row:
+                        values_to_merge.append(str(row[field]))
+                transformed = transform.separator.join(values_to_merge)
+            elif transform.type == "split":
+                parts = transformed.split(transform.separator)
+                if transform.index is not None:
+                    transformed = parts[transform.index] if parts else ""
+                else:
+                    transformed = parts
+
+        return transformed
+
+    def _create_transaction(self, data: Dict[str, Any]) -> Transaction:
+        tags = []
+        entities = []
+        # Handle related objects first
+        if "category" in data:
+            category_name = data.pop("category")
+            category, _ = TransactionCategory.objects.get_or_create(name=category_name)
+            data["category"] = category
+            self.import_run.categories.add(category)
+
+        if "account" in data:
+            account_id = data.pop("account")
+            account = None
+            if isinstance(account_id, str):
+                account = Account.objects.get(name=account_id)
+            elif isinstance(account_id, int):
+                account = Account.objects.get(id=account_id)
+            data["account"] = account
+            # self.import_run.acc.add(category)
+
+        if "tags" in data:
+            tag_names = data.pop("tags").split(",")
+            for tag_name in tag_names:
+                tag, _ = TransactionTag.objects.get_or_create(name=tag_name.strip())
+                tags.append(tag)
+                self.import_run.tags.add(tag)
+
+        if "entities" in data:
+            entity_names = data.pop("entities").split(",")
+            for entity_name in entity_names:
+                entity, _ = TransactionEntity.objects.get_or_create(
+                    name=entity_name.strip()
+                )
+                entities.append(entity)
+                self.import_run.entities.add(entity)
+
+        if "amount" in data:
+            amount = data.pop("amount")
+            data["amount"] = abs(Decimal(amount))
+
+        # Create the transaction
+        new_transaction = Transaction.objects.create(**data)
+        self.import_run.transactions.add(new_transaction)
+
+        # Add many-to-many relationships
+        if tags:
+            new_transaction.tags.set(tags)
+        if entities:
+            new_transaction.entities.set(entities)
+
+        if self.settings.trigger_transaction_rules:
+            transaction_created.send(sender=new_transaction)
+
+        return new_transaction
+
+    def _create_account(self, data: Dict[str, Any]) -> Account:
+        if "group" in data:
+            group_name = data.pop("group")
+            group, _ = AccountGroup.objects.get_or_create(name=group_name)
+            data["group"] = group
+
+        # Handle currency references
+        if "currency" in data:
+            currency = Currency.objects.get(code=data["currency"])
+            data["currency"] = currency
+            self.import_run.currencies.add(currency)
+
+        if "exchange_currency" in data:
+            exchange_currency = Currency.objects.get(code=data["exchange_currency"])
+            data["exchange_currency"] = exchange_currency
+            self.import_run.currencies.add(exchange_currency)
+
+        return Account.objects.create(**data)
+
+    def _create_currency(self, data: Dict[str, Any]) -> Currency:
+        # Handle exchange currency reference
+        if "exchange_currency" in data:
+            exchange_currency = Currency.objects.get(code=data["exchange_currency"])
+            data["exchange_currency"] = exchange_currency
+            self.import_run.currencies.add(exchange_currency)
+
+        currency = Currency.objects.create(**data)
+        self.import_run.currencies.add(currency)
+        return currency
+
+    def _create_category(self, data: Dict[str, Any]) -> TransactionCategory:
+        category = TransactionCategory.objects.create(**data)
+        self.import_run.categories.add(category)
+        return category
+
+    def _create_tag(self, data: Dict[str, Any]) -> TransactionTag:
+        tag = TransactionTag.objects.create(**data)
+        self.import_run.tags.add(tag)
+        return tag
+
+    def _create_entity(self, data: Dict[str, Any]) -> TransactionEntity:
+        entity = TransactionEntity.objects.create(**data)
+        self.import_run.entities.add(entity)
+        return entity
+
+    def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool:
+        for rule in self.deduplication:
+            if rule.type == "compare":
+                query = Transaction.all_objects.all().values("id")
+
+                # Build query conditions for each field in the rule
+                for field, header in rule.fields.items():
+                    if field in transaction_data:
+                        if rule.match_type == "strict":
+                            query = query.filter(**{field: transaction_data[field]})
+                        else:  # lax matching
+                            query = query.filter(
+                                **{f"{field}__iexact": transaction_data[field]}
+                            )
+
+                # If we found any matching transaction, it's a duplicate
+                if query.exists():
+                    return True
+
+        return False
+
+    def _coerce_type(
+        self, value: str, mapping: version_1.ColumnMapping
+    ) -> Union[str, int, bool, Decimal, datetime, list]:
+        if not value:
+            return None
+
+        coerce_to = mapping.coerce_to
+
+        if "|" in coerce_to:
+            types = coerce_to.split("|")
+            for t in types:
+                try:
+                    return self._coerce_single_type(value, t, mapping)
+                except ValueError:
+                    continue
+            raise ValueError(
+                f"Could not coerce '{value}' to any of the types: {coerce_to}"
+            )
+        else:
+            return self._coerce_single_type(value, coerce_to, mapping)
+
+    def _coerce_single_type(
+        self, value: str, coerce_to: str, mapping: version_1.ColumnMapping
+    ) -> Union[str, int, bool, Decimal, datetime.date, list]:
+        if coerce_to == "str":
+            return str(value)
+        elif coerce_to == "int":
+            if hasattr(mapping, "type") and mapping.type == "id":
+                return int(value)
+            elif hasattr(mapping, "type") and mapping.type in ["name", "code"]:
+                return str(value)
+            else:
+                return int(value)
+        elif coerce_to == "bool":
+            return value.lower() in ["true", "1", "yes", "y", "on"]
+        elif coerce_to == "positive_decimal":
+            return abs(Decimal(value))
+        elif coerce_to == "date":
+            if isinstance(
+                mapping,
+                (
+                    version_1.TransactionDateMapping,
+                    version_1.TransactionReferenceDateMapping,
+                ),
+            ):
+                formats = (
+                    mapping.format
+                    if isinstance(mapping.format, list)
+                    else [mapping.format]
+                )
+                for fmt in formats:
+                    try:
+                        return datetime.strptime(value, fmt).date()
+                    except ValueError:
+                        continue
+                raise ValueError(
+                    f"Could not parse date '{value}' with any of the provided formats"
+                )
+            else:
+                raise ValueError(
+                    "Date coercion is only supported for TransactionDateMapping and TransactionReferenceDateMapping"
+                )
+        elif coerce_to == "list":
+            return (
+                value
+                if isinstance(value, list)
+                else [item.strip() for item in value.split(",") if item.strip()]
+            )
+        elif coerce_to == "transaction_type":
+            if isinstance(mapping, version_1.TransactionTypeMapping):
+                if mapping.detection_method == "sign":
+                    return (
+                        Transaction.Type.EXPENSE
+                        if value.startswith("-")
+                        else Transaction.Type.INCOME
+                    )
+                elif mapping.detection_method == "always_income":
+                    return Transaction.Type.INCOME
+                elif mapping.detection_method == "always_expense":
+                    return Transaction.Type.EXPENSE
+            raise ValueError("Invalid transaction type detection method")
+        elif coerce_to == "is_paid":
+            if isinstance(mapping, version_1.TransactionIsPaidMapping):
+                if mapping.detection_method == "sign":
+                    return not value.startswith("-")
+                elif mapping.detection_method == "boolean":
+                    return value.lower() in ["true", "1", "yes", "y", "on"]
+                elif mapping.detection_method == "always_paid":
+                    return True
+                elif mapping.detection_method == "always_unpaid":
+                    return False
+            raise ValueError("Invalid is_paid detection method")
+        else:
+            raise ValueError(f"Unsupported coercion type: {coerce_to}")
+
+    def _map_row(self, row: Dict[str, str]) -> Dict[str, Any]:
+        mapped_data = {}
+
+        for field, mapping in self.mapping.items():
+            # If source is None, use None as the initial value
+            value = row.get(mapping.source) if mapping.source else None
+
+            # Use default_value if value is None
+            if value is None:
+                value = mapping.default
+
+            if mapping.required and value is None and not mapping.transformations:
+                raise ValueError(f"Required field {field} is missing")
+
+            # Apply transformations
+            if mapping.transformations:
+                value = self._transform_value(value, mapping, row)
+
+            value = self._coerce_type(value, mapping)
+
+            if value is not None:
+                # Remove the prefix from the target field
+                target = mapping.target
+                if self.settings.importing == "transactions":
+                    mapped_data[target] = value
+                else:
+                    # Remove the model prefix (e.g., "account_" from "account_name")
+                    field_name = target.split("_", 1)[1]
+                    mapped_data[field_name] = value
+
+        return mapped_data
+
+    def _process_row(self, row: Dict[str, str], row_number: int) -> None:
+        try:
+            mapped_data = self._map_row(row)
+
+            if mapped_data:
+                # Handle different import types
+                if self.settings.importing == "transactions":
+                    if self.deduplication and self._check_duplicate_transaction(
+                        mapped_data
+                    ):
+                        self._increment_totals("skipped", 1)
+                        self._log("info", f"Skipped duplicate row {row_number}")
+                        return
+                    self._create_transaction(mapped_data)
+                elif self.settings.importing == "accounts":
+                    self._create_account(mapped_data)
+                elif self.settings.importing == "currencies":
+                    self._create_currency(mapped_data)
+                elif self.settings.importing == "categories":
+                    self._create_category(mapped_data)
+                elif self.settings.importing == "tags":
+                    self._create_tag(mapped_data)
+                elif self.settings.importing == "entities":
+                    self._create_entity(mapped_data)
+
+                self._increment_totals("successful", value=1)
+                self._log("info", f"Successfully processed row {row_number}")
+
+            self._increment_totals("processed", value=1)
+
+        except Exception as e:
+            if not self.settings.skip_errors:
+                self._log("error", f"Fatal error processing row {row_number}: {str(e)}")
+                self._update_status("FAILED")
+                raise
+            else:
+                self._log("warning", f"Error processing row {row_number}: {str(e)}")
+                self._increment_totals("failed", value=1)
+
+            logger.error(f"Fatal error processing row {row_number}", exc_info=e)
+
+    def _process_csv(self, file_path):
+        # First pass: count rows
+        with open(file_path, "r", encoding=self.settings.encoding) as csv_file:
+            # Skip specified number of rows
+            for _ in range(self.settings.skip_lines):
+                next(csv_file)
+
+            reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter)
+            self._update_totals("total", value=sum(1 for _ in reader))
+
+        with open(file_path, "r", encoding=self.settings.encoding) as csv_file:
+            # Skip specified number of rows
+            for _ in range(self.settings.skip_lines):
+                next(csv_file)
+            if self.settings.skip_lines:
+                self._log("info", f"Skipped {self.settings.skip_lines} initial lines")
+
+            reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter)
+
+            self._log("info", f"Starting import with {self.import_run.total_rows} rows")
+
+            for row_number, row in enumerate(reader, start=1):
+                self._process_row(row, row_number)
+
+    def _validate_file_path(self, file_path: str) -> str:
+        """
+        Validates that the file path is within the allowed temporary directory.
+        Returns the absolute path.
+        """
+        abs_path = os.path.abspath(file_path)
+        if not abs_path.startswith(self.TEMP_DIR):
+            raise ValueError(f"Invalid file path. File must be in {self.TEMP_DIR}")
+        return abs_path
+
+    def process_file(self, file_path: str):
+        with cachalot_disabled():
+            # Validate and get absolute path
+            file_path = self._validate_file_path(file_path)
+
+            self._update_status("PROCESSING")
+            self.import_run.started_at = timezone.now()
+            self.import_run.save(update_fields=["started_at"])
+
+            self._log("info", "Starting import process")
+
+            try:
+                if self.settings.file_type == "csv":
+                    self._process_csv(file_path)
+
+                self._update_status("FINISHED")
+                self._log(
+                    "info",
+                    f"Import completed successfully. "
+                    f"Successful: {self.import_run.successful_rows}, "
+                    f"Failed: {self.import_run.failed_rows}, "
+                    f"Skipped: {self.import_run.skipped_rows}",
+                )
+
+            except Exception as e:
+                self._update_status("FAILED")
+                self._log("error", f"Import failed: {str(e)}")
+                raise Exception("Import failed")
+
+            finally:
+                self._log("info", "Cleaning up temporary files")
+                try:
+                    if os.path.exists(file_path):
+                        os.remove(file_path)
+                        self._log("info", f"Deleted temporary file: {file_path}")
+                except OSError as e:
+                    self._log("warning", f"Failed to delete temporary file: {str(e)}")
+
+                self.import_run.finished_at = timezone.now()
+                self.import_run.save(update_fields=["finished_at"])
+
+        if self.import_run.successful_rows >= 1:
+            cachalot.api.invalidate()

app/apps/import_app/tasks.py

@@ -0,0 +1,18 @@
+import logging
+
+from procrastinate.contrib.django import app
+
+from apps.import_app.models import ImportRun
+from apps.import_app.services import ImportServiceV1
+
+logger = logging.getLogger(__name__)
+
+
+@app.task
+def process_import(import_run_id: int, file_path: str):
+    try:
+        import_run = ImportRun.objects.get(id=import_run_id)
+        import_service = ImportServiceV1(import_run)
+        import_service.process_file(file_path)
+    except ImportRun.DoesNotExist:
+        raise ValueError(f"ImportRun with id {import_run_id} not found")

app/apps/import_app/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

app/apps/import_app/urls.py

@@ -0,0 +1,56 @@
+from django.urls import path
+import apps.import_app.views as views
+
+urlpatterns = [
+    path("import/", views.import_view, name="import"),
+    path(
+        "import/presets/",
+        views.import_presets_list,
+        name="import_presets_list",
+    ),
+    path(
+        "import/profiles/",
+        views.import_profile_index,
+        name="import_profiles_index",
+    ),
+    path(
+        "import/profiles/list/",
+        views.import_profile_list,
+        name="import_profiles_list",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/delete/",
+        views.import_profile_delete,
+        name="import_profile_delete",
+    ),
+    path(
+        "import/profiles/add/",
+        views.import_profile_add,
+        name="import_profiles_add",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/edit/",
+        views.import_profile_edit,
+        name="import_profile_edit",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/list/",
+        views.import_runs_list,
+        name="import_profile_runs_list",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/<int:run_id>/log/",
+        views.import_run_log,
+        name="import_run_log",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/<int:run_id>/delete/",
+        views.import_run_delete,
+        name="import_run_delete",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/add/",
+        views.import_run_add,
+        name="import_run_add",
+    ),
+]

app/apps/import_app/views.py

@@ -0,0 +1,232 @@
+import shutil
+
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.core.files.storage import FileSystemStorage
+from django.http import HttpResponse
+from django.shortcuts import render, get_object_or_404
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_http_methods
+from django.utils.translation import gettext_lazy as _
+
+from apps.common.decorators.htmx import only_htmx
+from apps.import_app.forms import ImportRunFileUploadForm, ImportProfileForm
+from apps.import_app.models import ImportRun, ImportProfile
+from apps.import_app.tasks import process_import
+from apps.import_app.services import PresetService
+
+
+def import_view(request):
+    import_profile = ImportProfile.objects.get(id=2)
+    shutil.copyfile(
+        "/usr/src/app/apps/import_app/teste2.csv", "/usr/src/app/temp/teste2.csv"
+    )
+    ir = ImportRun.objects.create(profile=import_profile, file_name="teste.csv")
+    process_import.defer(
+        import_run_id=ir.id,
+        file_path="/usr/src/app/temp/teste2.csv",
+    )
+    return HttpResponse("Hello, world. You're at the polls page.")
+
+
+@login_required
+@require_http_methods(["GET"])
+def import_presets_list(request):
+    presets = PresetService.get_all_presets()
+    print(presets)
+    return render(
+        request,
+        "import_app/fragments/profiles/list_presets.html",
+        {"presets": presets},
+    )
+
+
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_index(request):
+    return render(
+        request,
+        "import_app/pages/profiles_index.html",
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_list(request):
+    profiles = ImportProfile.objects.all()
+
+    return render(
+        request,
+        "import_app/fragments/profiles/list.html",
+        {"profiles": profiles},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_add(request):
+    message = request.GET.get("message", None) or request.POST.get("message", None)
+
+    if request.method == "POST":
+        form = ImportProfileForm(request.POST)
+
+        if form.is_valid():
+            form.save()
+            messages.success(request, _("Import Profile added successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        print(int(request.GET.get("version", 1)))
+        form = ImportProfileForm(
+            initial={
+                "name": request.GET.get("name"),
+                "version": int(request.GET.get("version", 1)),
+                "yaml_config": request.GET.get("yaml_config"),
+            }
+        )
+
+    return render(
+        request,
+        "import_app/fragments/profiles/add.html",
+        {"form": form, "message": message},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_edit(request, profile_id):
+    profile = get_object_or_404(ImportProfile, id=profile_id)
+
+    if request.method == "POST":
+        form = ImportProfileForm(request.POST, instance=profile)
+
+        if form.is_valid():
+            form.save()
+            messages.success(request, _("Import Profile update successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        form = ImportProfileForm(instance=profile)
+
+    return render(
+        request,
+        "import_app/fragments/profiles/edit.html",
+        {"form": form, "profile": profile},
+    )
+
+
+@only_htmx
+@login_required
+@csrf_exempt
+@require_http_methods(["DELETE"])
+def import_profile_delete(request, profile_id):
+    profile = ImportProfile.objects.get(id=profile_id)
+
+    profile.delete()
+
+    messages.success(request, _("Import Profile deleted successfully"))
+
+    return HttpResponse(
+        status=204,
+        headers={
+            "HX-Trigger": "updated",
+        },
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_runs_list(request, profile_id):
+    profile = ImportProfile.objects.get(id=profile_id)
+
+    runs = ImportRun.objects.filter(profile=profile).order_by("-id")
+
+    return render(
+        request,
+        "import_app/fragments/runs/list.html",
+        {"profile": profile, "runs": runs},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_run_log(request, profile_id, run_id):
+    run = ImportRun.objects.get(profile__id=profile_id, id=run_id)
+
+    return render(
+        request,
+        "import_app/fragments/runs/log.html",
+        {"run": run},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_run_add(request, profile_id):
+    profile = ImportProfile.objects.get(id=profile_id)
+
+    if request.method == "POST":
+        form = ImportRunFileUploadForm(request.POST, request.FILES)
+
+        if form.is_valid():
+            uploaded_file = request.FILES["file"]
+            fs = FileSystemStorage(location="/usr/src/app/temp")
+            filename = fs.save(uploaded_file.name, uploaded_file)
+            file_path = fs.path(filename)
+
+            import_run = ImportRun.objects.create(profile=profile, file_name=filename)
+
+            # Defer the procrastinate task
+            process_import.defer(import_run_id=import_run.id, file_path=file_path)
+
+            messages.success(request, _("Import Run queued successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        form = ImportRunFileUploadForm()
+
+    return render(
+        request,
+        "import_app/fragments/runs/add.html",
+        {"form": form, "profile": profile},
+    )
+
+
+@only_htmx
+@login_required
+@csrf_exempt
+@require_http_methods(["DELETE"])
+def import_run_delete(request, profile_id, run_id):
+    run = ImportRun.objects.get(profile__id=profile_id, id=run_id)
+
+    run.delete()
+
+    messages.success(request, _("Run deleted successfully"))
+
+    return HttpResponse(
+        status=204,
+        headers={
+            "HX-Trigger": "updated",
+        },
+    )

app/apps/transactions/admin.py

@@ -12,15 +12,34 @@ from apps.transactions.models import (
 
 @admin.register(Transaction)
 class TransactionModelAdmin(admin.ModelAdmin):
+    def get_queryset(self, request):
+        # Use the all_objects manager to show all transactions, including deleted ones
+        return self.model.all_objects.all()
+
+    list_filter = ["deleted", "type", "is_paid", "date", "account"]
+
     list_display = [
+        "date",
         "description",
         "type",
         "account__name",
         "amount",
         "account__currency__code",
-        "date",
         "reference_date",
+        "deleted",
     ]
+    readonly_fields = ["deleted_at"]
+
+    actions = ["hard_delete_selected"]
+
+    def hard_delete_selected(self, request, queryset):
+        for obj in queryset:
+            obj.hard_delete()
+        self.message_user(
+            request, f"Successfully hard deleted {queryset.count()} transactions."
+        )
+
+    hard_delete_selected.short_description = "Hard delete selected transactions"
 
 
 class TransactionInline(admin.TabularInline):

app/apps/transactions/migrations/0028_transaction_internal_note.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.5 on 2025-01-19 00:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0027_alter_transaction_description'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transaction',
+            name='internal_note',
+            field=models.TextField(blank=True, verbose_name='Internal Note'),
+        ),
+    ]

app/apps/transactions/migrations/0029_alter_transaction_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.1.5 on 2025-01-19 14:59
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0028_transaction_internal_note'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='transaction',
+            options={'default_manager_name': 'objects', 'verbose_name': 'Transaction', 'verbose_name_plural': 'Transactions'},
+        ),
+    ]

app/apps/transactions/migrations/0030_transaction_deleted_transaction_deleted_at.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.5 on 2025-01-19 14:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0029_alter_transaction_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transaction',
+            name='deleted',
+            field=models.BooleanField(default=False, verbose_name='Deleted'),
+        ),
+        migrations.AddField(
+            model_name='transaction',
+            name='deleted_at',
+            field=models.DateTimeField(blank=True, null=True, verbose_name='Deleted At'),
+        ),
+    ]

app/apps/transactions/migrations/0031_alter_transaction_deleted.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.5 on 2025-01-19 15:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0030_transaction_deleted_transaction_deleted_at'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='transaction',
+            name='deleted',
+            field=models.BooleanField(db_index=True, default=False, verbose_name='Deleted'),
+        ),
+    ]

app/apps/transactions/migrations/0032_transaction_created_at_transaction_updated_at.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.1.5 on 2025-01-19 16:48
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('transactions', '0031_alter_transaction_deleted'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transaction',
+            name='created_at',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='transaction',
+            name='updated_at',
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

app/apps/transactions/migrations/0033_transaction_internal_id.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.1.5 on 2025-01-21 01:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("transactions", "0032_transaction_created_at_transaction_updated_at"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="transaction",
+            name="internal_id",
+            field=models.TextField(
+                blank=True, null=True, unique=True, verbose_name="Internal ID"
+            ),
+        ),
+    ]

app/apps/transactions/models.py

@@ -6,6 +6,7 @@ from django.db import models, transaction
 from django.db.models import Q
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from django.conf import settings
 
 from apps.common.fields.month_year import MonthYearModelField
 from apps.common.functions.decimals import truncate_decimal
@@ -15,6 +16,53 @@ from apps.transactions.validators import validate_decimal_places, validate_non_n
 logger = logging.getLogger()
 
 
+class SoftDeleteQuerySet(models.QuerySet):
+    def delete(self):
+        if not settings.ENABLE_SOFT_DELETE:
+            # If soft deletion is disabled, perform a normal delete
+            return super().delete()
+
+        # Separate the queryset into already deleted and not deleted objects
+        already_deleted = self.filter(deleted=True)
+        not_deleted = self.filter(deleted=False)
+
+        # Use a transaction to ensure atomicity
+        with transaction.atomic():
+            # Perform hard delete on already deleted objects
+            hard_deleted_count = already_deleted._raw_delete(already_deleted.db)
+
+            # Perform soft delete on not deleted objects
+            soft_deleted_count = not_deleted.update(
+                deleted=True, deleted_at=timezone.now()
+            )
+
+        # Return a tuple of counts as expected by Django's delete method
+        return (
+            hard_deleted_count + soft_deleted_count,
+            {"Transaction": hard_deleted_count + soft_deleted_count},
+        )
+
+    def hard_delete(self):
+        return super().delete()
+
+
+class SoftDeleteManager(models.Manager):
+    def get_queryset(self):
+        qs = SoftDeleteQuerySet(self.model, using=self._db)
+        return qs if not settings.ENABLE_SOFT_DELETE else qs.filter(deleted=False)
+
+
+class AllObjectsManager(models.Manager):
+    def get_queryset(self):
+        return SoftDeleteQuerySet(self.model, using=self._db)
+
+
+class DeletedObjectsManager(models.Manager):
+    def get_queryset(self):
+        qs = SoftDeleteQuerySet(self.model, using=self._db)
+        return qs if not settings.ENABLE_SOFT_DELETE else qs.filter(deleted=True)
+
+
 class TransactionCategory(models.Model):
     name = models.CharField(max_length=255, verbose_name=_("Name"), unique=True)
     mute = models.BooleanField(default=False, verbose_name=_("Mute"))
@@ -141,11 +189,29 @@ class Transaction(models.Model):
         related_name="transactions",
         verbose_name=_("Recurring Transaction"),
     )
+    internal_note = models.TextField(blank=True, verbose_name=_("Internal Note"))
+    internal_id = models.TextField(
+        blank=True, null=True, unique=True, verbose_name=_("Internal ID")
+    )
+
+    deleted = models.BooleanField(
+        default=False, verbose_name=_("Deleted"), db_index=True
+    )
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+    deleted_at = models.DateTimeField(
+        null=True, blank=True, verbose_name=_("Deleted At")
+    )
+
+    objects = SoftDeleteManager.from_queryset(SoftDeleteQuerySet)()
+    all_objects = AllObjectsManager.from_queryset(SoftDeleteQuerySet)()
+    deleted_objects = DeletedObjectsManager.from_queryset(SoftDeleteQuerySet)()
 
     class Meta:
         verbose_name = _("Transaction")
         verbose_name_plural = _("Transactions")
         db_table = "transactions"
+        default_manager_name = "objects"
 
     def save(self, *args, **kwargs):
         self.amount = truncate_decimal(
@@ -160,6 +226,17 @@ class Transaction(models.Model):
         self.full_clean()
         super().save(*args, **kwargs)
 
+    def delete(self, *args, **kwargs):
+        if settings.ENABLE_SOFT_DELETE:
+            self.deleted = True
+            self.deleted_at = timezone.now()
+            self.save()
+        else:
+            super().delete(*args, **kwargs)
+
+    def hard_delete(self, *args, **kwargs):
+        super().delete(*args, **kwargs)
+
     def exchanged_amount(self):
         if self.account.exchange_currency:
             converted_amount, prefix, suffix, decimal_places = convert(
@@ -178,6 +255,10 @@ class Transaction(models.Model):
 
         return None
 
+    def __str__(self):
+        type_display = self.get_type_display()
+        return f"{self.description} - {type_display} - {self.account} - {self.date}"
+
 
 class InstallmentPlan(models.Model):
     class Recurrence(models.TextChoices):

app/apps/transactions/tasks.py

@@ -1,9 +1,13 @@
 import logging
+from datetime import timedelta
+
+from cachalot.api import cachalot_disabled, invalidate
+from django.utils import timezone
+from django.conf import settings
 
 from procrastinate.contrib.django import app
 
-from apps.transactions.models import RecurringTransaction
+from apps.transactions.models import RecurringTransaction, Transaction
-
 
 logger = logging.getLogger(__name__)
 
@@ -19,3 +23,31 @@ def generate_recurring_transactions(timestamp=None):
             exc_info=True,
         )
         raise e
+
+
+@app.periodic(cron="10 1 * * *")
+@app.task
+def cleanup_deleted_transactions():
+    with cachalot_disabled():
+        if settings.ENABLE_SOFT_DELETE and settings.KEEP_DELETED_TRANSACTIONS_FOR == 0:
+            return "KEEP_DELETED_TRANSACTIONS_FOR is 0, no cleanup performed."
+
+        if not settings.ENABLE_SOFT_DELETE:
+            # Hard delete all soft-deleted transactions
+            deleted_count, _ = Transaction.deleted_objects.all().hard_delete()
+            return (
+                f"Hard deleted {deleted_count} transactions (soft deletion disabled)."
+            )
+
+        # Calculate the cutoff date
+        cutoff_date = timezone.now() - timedelta(
+            days=settings.KEEP_DELETED_TRANSACTIONS_FOR
+        )
+
+    invalidate("transactions.Transaction")
+
+    # Hard delete soft-deleted transactions older than the cutoff date
+    old_transactions = Transaction.deleted_objects.filter(deleted_at__lt=cutoff_date)
+    deleted_count, _ = old_transactions.hard_delete()
+
+    return f"Hard deleted {deleted_count} objects older than {settings.KEEP_DELETED_TRANSACTIONS_FOR} days."

app/apps/users/migrations/0014_alter_usersettings_date_format_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.1.5 on 2025-01-23 03:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0013_usersettings_date_format_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='usersettings',
+            name='date_format',
+            field=models.CharField(default='SHORT_DATE_FORMAT', max_length=100, verbose_name='Date Format'),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='datetime_format',
+            field=models.CharField(default='SHORT_DATETIME_FORMAT', max_length=100, verbose_name='Datetime Format'),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='language',
+            field=models.CharField(choices=[('auto', 'Auto'), ('en', 'English'), ('nl', 'Nederlands'), ('pt-br', 'Português (Brasil)')], default='auto', max_length=10, verbose_name='Language'),
+        ),
+    ]

app/templates/import_app/fragments/profiles/add.html

@@ -0,0 +1,19 @@
+{% extends 'extends/offcanvas.html' %}
+{% load json %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block title %}{% translate 'Add new import profile' %}{% endblock %}
+
+{% block body %}
+{% if message %}
+<div class="alert alert-info" role="alert" id="msg" hx-preserve="true">
+  <h6 class="alert-heading tw-italic tw-font-bold">{% trans 'A message from the author' %}</h6>
+  <hr>
+  <p class="mb-0">{{ message|linebreaksbr }}</p>
+</div>
+{% endif %}
+<form hx-post="{% url 'import_profiles_add' %}" hx-target="#generic-offcanvas" novalidate hx-vals='{"message": {% if message %}{{ message|json }}{% else %}""{% endif %}}'>
+    {% crispy form %}
+</form>
+{% endblock %}

app/templates/import_app/fragments/profiles/edit.html

@@ -0,0 +1,11 @@
+{% extends 'extends/offcanvas.html' %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block title %}{% translate 'Edit import profile' %}{% endblock %}
+
+{% block body %}
+<form hx-post="{% url 'import_profile_edit' profile_id=profile.id %}" hx-target="#generic-offcanvas" novalidate>
+    {% crispy form %}
+</form>
+{% endblock %}

app/templates/import_app/fragments/profiles/list.html

@@ -0,0 +1,90 @@
+{% load i18n %}
+<div class="container px-md-3 py-3 column-gap-5">
+  <div class="tw-text-3xl fw-bold font-monospace tw-w-full mb-3">
+    {% spaceless %}
+    <div>{% translate 'Import Profiles' %}<span>
+      <span class="dropdown" data-bs-toggle="tooltip"
+                data-bs-title="{% translate "Add" %}">
+        <a class="text-decoration-none tw-text-2xl p-1" role="button"
+                data-bs-toggle="dropdown"
+                data-bs-title="{% translate "Add" %}" aria-expanded="false">
+          <i class="fa-solid fa-circle-plus fa-fw"></i>
+        </a>
+        <ul class="dropdown-menu">
+          <li><a class="dropdown-item"
+                 role="button"
+                 hx-get="{% url 'import_profiles_add' %}"
+                 hx-target="#generic-offcanvas">{% trans 'New' %}</a></li>
+          <li><a class="dropdown-item"
+                 role="button"
+                 hx-get="{% url 'import_presets_list' %}"
+                 hx-target="#persistent-generic-offcanvas-left">{% trans 'From preset' %}</a></li>
+        </ul>
+      </span>
+    </span></div>
+    {% endspaceless %}
+  </div>
+
+  <div class="card">
+    <div class="card-body table-responsive">
+      {% if profiles %}
+      <c-config.search></c-config.search>
+      <table class="table table-hover text-nowrap">
+        <thead>
+          <tr>
+            <th scope="col" class="col-auto"></th>
+            <th scope="col" class="col">{% translate 'Name' %}</th>
+            <th scope="col" class="col">{% translate 'Version' %}</th>
+          </tr>
+        </thead>
+        <tbody>
+          {% for profile in profiles %}
+          <tr class="profile">
+            <td class="col-auto">
+              <div class="btn-group" role="group" aria-label="{% translate 'Actions' %}">
+                <a class="btn btn-secondary btn-sm"
+                      role="button"
+                      data-bs-toggle="tooltip"
+                      data-bs-title="{% translate "Edit" %}"
+                      hx-get="{% url 'import_profile_edit' profile_id=profile.id %}"
+                      hx-target="#generic-offcanvas">
+                  <i class="fa-solid fa-pencil fa-fw"></i></a>
+                  <a class="btn btn-secondary btn-sm text-success"
+                        role="button"
+                        data-bs-toggle="tooltip"
+                        data-bs-title="{% translate "Runs" %}"
+                        hx-get="{% url 'import_profile_runs_list' profile_id=profile.id %}"
+                        hx-target="#persistent-generic-offcanvas-left">
+                    <i class="fa-solid fa-person-running fa-fw"></i></a>
+                <a class="btn btn-secondary btn-sm text-primary"
+                        role="button"
+                        data-bs-toggle="tooltip"
+                        data-bs-title="{% translate "Import" %}"
+                        hx-get="{% url 'import_run_add' profile_id=profile.id %}"
+                        hx-target="#generic-offcanvas">
+                    <i class="fa-solid fa-file-import fa-fw"></i></a>
+                <a class="btn btn-secondary btn-sm text-danger"
+                        role="button"
+                        data-bs-toggle="tooltip"
+                        data-bs-title="{% translate "Delete" %}"
+                        hx-delete="{% url 'import_profile_delete' profile_id=profile.id %}"
+                        hx-trigger='confirmed'
+                        data-bypass-on-ctrl="true"
+                        data-title="{% translate "Are you sure?" %}"
+                        data-text="{% translate "You won't be able to revert this!" %}"
+                        data-confirm-text="{% translate "Yes, delete it!" %}"
+                        _="install prompt_swal"><i class="fa-solid fa-trash fa-fw"></i></a>
+              </div>
+            </td>
+            <td class="col">{{ profile.name }}</td>
+            <td class="col">{{ profile.get_version_display }}</td>
+          </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+      {% else %}
+      <c-msg.empty title="{% translate "No accounts" %}" remove-padding></c-msg.empty>
+      {% endif %}
+    </div>
+  </div>
+</div>

app/templates/import_app/fragments/profiles/list_presets.html

@@ -0,0 +1,43 @@
+{% extends 'extends/offcanvas.html' %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block title %}{% translate 'Import Presets' %}{% endblock %}
+
+{% block body %}
+  {% if presets %}
+  <div id="search" class="mb-3">
+    <label class="w-100">
+    <input type="search"
+           class="form-control"
+           placeholder="{% translate 'Search' %}"
+           _="on input or search
+                show < .col /> in <#items/>
+                when its textContent.toLowerCase() contains my value.toLowerCase()"/>
+    </label>
+  </div>
+  <div class="row row-cols-1 g-4" id="items">
+  {% for preset in presets %}
+    <a class="text-decoration-none"
+       role="button"
+       hx-get="{% url 'import_profiles_add' %}"
+       hx-vals='{"yaml_config": {{ preset.config }}, "name": "{{ preset.name }}", "version": "{{ preset.schema_version }}", "message": {{ preset.message }}}'
+       hx-target="#generic-offcanvas">
+
+      <div class="col">
+        <div class="card">
+          <div class="card-body">
+            <h5 class="card-title">{{ preset.name }}</h5>
+            <hr>
+            <p>{{ preset.description }}</p>
+            <p>{% trans 'By' %} {{ preset.authors|join:", " }}</p>
+          </div>
+        </div>
+      </div>
+    </a>
+  {% endfor %}
+  {% else %}
+    <c-msg.empty title="{% translate "No presets yet" %}"></c-msg.empty>
+  {% endif %}
+</div>
+{% endblock %}

app/templates/import_app/fragments/runs/add.html

@@ -0,0 +1,11 @@
+{% extends 'extends/offcanvas.html' %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block title %}{% translate 'Import file with profile' %} {{ profile.name }}{% endblock %}
+
+{% block body %}
+<form hx-post="{% url 'import_run_add' profile_id=profile.id %}" hx-target="#generic-offcanvas" enctype="multipart/form-data" novalidate>
+    {% crispy form %}
+</form>
+{% endblock %}

app/templates/import_app/fragments/runs/list.html

@@ -0,0 +1,120 @@
+{% extends 'extends/offcanvas.html' %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block title %}{% translate 'Runs for ' %}{{ profile.name }}{% endblock %}
+
+{% block body %}
+  <div hx-get="{% url "import_profile_runs_list" profile_id=profile.id %}"
+       hx-trigger="updated from:window"
+       hx-target="closest .offcanvas"
+       class="show-loading"
+       hx-swap="show:none scroll:none">
+    {% if runs %}
+    <div class="row row-cols-1 g-4">
+    {% for run in runs %}
+      <div class="col">
+        <div class="card">
+          <div class="card-header tw-text-sm {% if run.status == run.Status.QUEUED %}tw-text-white{% elif run.status == run.Status.PROCESSING %}text-warning{% elif run.status == run.Status.FINISHED %}text-success{% else %}text-danger{% endif %}">
+            <span><i class="fa-solid {% if run.status == run.Status.QUEUED %}fa-hourglass-half{% elif run.status == run.Status.PROCESSING %}fa-spinner{% elif run.status == run.Status.FINISHED %}fa-check{% else %}fa-xmark{% endif %} fa-fw me-2"></i>{{ run.get_status_display }}</span>
+          </div>
+          <div class="card-body">
+            <h5 class="card-title"><i class="fa-solid fa-hashtag me-1 tw-text-xs tw-text-gray-400"></i>{{ run.id }}<span class="tw-text-xs tw-text-gray-400 ms-1">({{ run.file_name }})</span></h5>
+            <hr>
+            <div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 w-100 g-4">
+              <div class="col">
+                <div class="d-flex flex-row">
+                  <div class="d-flex flex-column">
+                    <div class="text-body-secondary tw-text-xs tw-font-medium">
+                      {% trans 'Total Items' %}
+                    </div>
+                    <div class="tw-text-sm">
+                      {{ run.total_rows }}
+                    </div>
+                  </div>
+                </div>
+              </div>
+
+              <div class="col">
+                <div class="d-flex flex-row">
+                  <div class="d-flex flex-column">
+                    <div class="text-body-secondary tw-text-xs tw-font-medium">
+                      {% trans 'Processed Items' %}
+                    </div>
+                    <div class="tw-text-sm">
+                      {{ run.processed_rows }}
+                    </div>
+                  </div>
+                </div>
+              </div>
+
+              <div class="col">
+                <div class="d-flex flex-row">
+                  <div class="d-flex flex-column">
+                    <div class="text-body-secondary tw-text-xs tw-font-medium">
+                      {% trans 'Skipped Items' %}
+                    </div>
+                    <div class="tw-text-sm">
+                      {{ run.skipped_rows }}
+                    </div>
+                  </div>
+                </div>
+              </div>
+
+              <div class="col">
+                <div class="d-flex flex-row">
+                  <div class="d-flex flex-column">
+                    <div class="text-body-secondary tw-text-xs tw-font-medium">
+                      {% trans 'Failed Items' %}
+                    </div>
+                    <div class="tw-text-sm">
+                      {{ run.failed_rows }}
+                    </div>
+                  </div>
+                </div>
+              </div>
+
+              <div class="col">
+                <div class="d-flex flex-row">
+                  <div class="d-flex flex-column">
+                    <div class="text-body-secondary tw-text-xs tw-font-medium">
+                      {% trans 'Successful Items' %}
+                    </div>
+                    <div class="tw-text-sm">
+                      {{ run.successful_rows }}
+                    </div>
+                  </div>
+                </div>
+              </div>
+
+            </div>
+          </div>
+          <div class="card-footer text-body-secondary">
+            <a class="text-decoration-none text-info"
+               role="button"
+               data-bs-toggle="tooltip"
+               data-bs-title="{% translate "Logs" %}"
+               hx-get="{% url 'import_run_log' profile_id=profile.id run_id=run.id %}"
+               hx-target="#generic-offcanvas"><i class="fa-solid fa-file-lines"></i></a>
+            <a class="text-decoration-none text-danger"
+               role="button"
+               data-bs-toggle="tooltip"
+               data-bs-title="{% translate "Delete" %}"
+               hx-delete="{% url 'import_run_delete' profile_id=profile.id run_id=run.id %}"
+               hx-trigger='confirmed'
+               data-bypass-on-ctrl="true"
+               data-title="{% translate "Are you sure?" %}"
+               data-text="{% translate "You won't be able to revert this! All imported items will be kept." %}"
+               data-confirm-text="{% translate "Yes, delete it!" %}"
+               _="install prompt_swal"><i class="fa-solid fa-trash fa-fw"></i>
+            </a>
+          </div>
+        </div>
+      </div>
+    {% endfor %}
+    {% else %}
+      <c-msg.empty title="{% translate "No runs yet" %}"></c-msg.empty>
+    {% endif %}
+  </div>
+  </div>
+{% endblock %}

app/templates/import_app/fragments/runs/log.html

@@ -0,0 +1,13 @@
+{% extends 'extends/offcanvas.html' %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block title %}{% translate 'Logs for' %} #{{ run.id }}{% endblock %}
+
+{% block body %}
+<div class="card tw-max-h-full tw-overflow-auto">
+  <div class="card-body">
+    {{ run.logs|linebreaks }}
+  </div>
+</div>
+{% endblock %}

app/templates/import_app/pages/profiles_index.html

@@ -0,0 +1,8 @@
+{% extends "layouts/base.html" %}
+{% load i18n %}
+
+{% block title %}{% translate 'Import Profiles' %}{% endblock %}
+
+{% block content %}
+<div hx-get="{% url 'import_profiles_list' %}" hx-trigger="load, updated from:window" class="show-loading"></div>
+{% endblock %}

app/templates/import_app/pages/runs_index.html

@@ -0,0 +1,8 @@
+{% extends "layouts/base.html" %}
+{% load i18n %}
+
+{% block title %}{% translate 'Import Runs' %}{% endblock %}
+
+{% block content %}
+<div hx-get="{% url 'impor' %}" hx-trigger="load, updated from:window" class="show-loading"></div>
+{% endblock %}

app/templates/includes/navbar.html

@@ -120,6 +120,8 @@
             <li><h6 class="dropdown-header">{% trans 'Automation' %}</h6></li>
             <li><a class="dropdown-item {% active_link views='rules_index' %}"
                    href="{% url 'rules_index' %}">{% translate 'Rules' %}</a></li>
+            <li><a class="dropdown-item {% active_link views='import_profiles_index' %}"
+                   href="{% url 'import_profiles_index' %}">{% translate 'Import' %} <span class="badge text-bg-primary">beta</span></a></li>
             <li>
               <hr class="dropdown-divider">
             </li>

docker-compose.dev.yml

@@ -1,6 +1,6 @@
 volumes:
   wygiwyh_dev_postgres_data: {}
-  temp:
+  wygiwyh_temp:
 
 services:
   web: &django
@@ -13,6 +13,7 @@ services:
     volumes:
       - ./app/:/usr/src/app/:z
       - ./frontend/:/usr/src/frontend:z
+      - wygiwyh_temp:/usr/src/app/temp/
     ports:
       - "${OUTBOUND_PORT}:8000"
     env_file:

docker-compose.prod.yml

@@ -9,6 +9,8 @@ services:
       - .env
     depends_on:
       - db
+    volumes:
+      - wygiwyh_temp:/usr/src/app/temp/
     restart: unless-stopped
 
   db:
@@ -29,5 +31,10 @@ services:
       - db
     env_file:
       - .env
+    volumes:
+      - wygiwyh_temp:/usr/src/app/temp/
     command: /start-procrastinate
     restart: unless-stopped
+
+volumes:
+  wygiwyh_temp:

requirements.txt

@@ -24,3 +24,5 @@ requests~=2.32.3
 pytz~=2024.2
 python-dateutil~=2.9.0.post0
 simpleeval~=1.0.0
+pydantic~=2.10.5
+PyYAML~=6.0.2